{ "schema_version": 1, "dataset_git_commit": "70a514f06f9e156be480e2a0b28a1d3380467ace", "dataset_fingerprint": "fc531bdb2e7c9d10e2074519bd3e0d0f66bc8ec970d20f25cdb628054784c6de", "expected_task_count": 292, "run_id": "pareto-20260929", "mock": false, "grading_version": "1", "release": "0.3.0", "evaluation_protocol": "deed0698dca65b5e24f3b06766d2d5344bbf0964e871167cebfc5e4aaba554fa", "finalized_at": "2026-09-29T14:22:21.197670+00:00", "source_updated_at": "2026-09-29T14:18:06.680045+00:00", "source_git_commit": "ac1552d1cd6ddf4092f069b165096204680973a4", "finalizer_git_commit": "ac1552d1cd6ddf4092f069b165096204680973a4", "finalizer_git_dirty": false, "artifact_type": "layoutbench-final-results", "publication_status": "finalized", "benchmark_id": "layoutbench", "version": "0.3.0", "status": "pilot", "human_validation": "not_performed", "numeric_scoring": { "distance": "absolute-px-error", "score_ceiling": 16, "score_formula": "100 * (1 - min(err_px / 16, 1))", "tight_ceiling": 4, "tight_formula": "100 * (1 - min(err_px / 4, 1))", "exact_bands": [ 1, 2, 4, 8 ], "interpretation": "Engineering normalization; not a just-noticeable-difference threshold." }, "dataset_manifest": "dataset/layoutbench-v0.3/manifest.json", "dataset_path": "dataset/layoutbench-v0.3", "comparison": { "scope": "full", "count": 292, "task_ids": [ "layoutbench-blockalign-001", "layoutbench-blockalign-002", "layoutbench-blockalign-003", "layoutbench-blockalign-004", "layoutbench-blockalign-005", "layoutbench-blockalign-006", "layoutbench-blockalign-007", "layoutbench-blockalign-008", "layoutbench-blockalign-009", "layoutbench-blockalign-010", "layoutbench-blockalign-011", "layoutbench-blockalign-012", "layoutbench-crossalign-001", "layoutbench-crossalign-002", "layoutbench-crossalign-003", "layoutbench-crossalign-004", "layoutbench-crossalign-005", "layoutbench-crossalign-006", "layoutbench-crossalign-007", "layoutbench-crossalign-008", "layoutbench-crossalign-009", "layoutbench-crossalign-010", "layoutbench-crossalign-011", "layoutbench-crossalign-012", "layoutbench-crossalign-013", "layoutbench-crossalign-014", "layoutbench-crossalign-015", "layoutbench-crossalign-016", "layoutbench-direction-001", "layoutbench-direction-002", "layoutbench-direction-003", "layoutbench-direction-004", "layoutbench-direction-005", "layoutbench-direction-006", "layoutbench-direction-007", "layoutbench-direction-008", "layoutbench-distribution-001", "layoutbench-distribution-002", "layoutbench-distribution-003", "layoutbench-distribution-004", "layoutbench-distribution-005", "layoutbench-distribution-006", "layoutbench-distribution-007", "layoutbench-distribution-008", "layoutbench-distribution-009", "layoutbench-distribution-010", "layoutbench-distribution-011", "layoutbench-distribution-012", "layoutbench-distribution-013", "layoutbench-distribution-014", "layoutbench-distribution-015", "layoutbench-distribution-016", "layoutbench-distribution-017", "layoutbench-distribution-018", "layoutbench-distribution-019", "layoutbench-distribution-020", "layoutbench-distribution-021", "layoutbench-distribution-022", "layoutbench-distribution-023", "layoutbench-distribution-024", "layoutbench-gapcompare-001", "layoutbench-gapcompare-002", "layoutbench-gapcompare-003", "layoutbench-gapcompare-004", "layoutbench-gapcompare-005", "layoutbench-gapcompare-006", "layoutbench-gapcompare-007", "layoutbench-gapcompare-008", "layoutbench-gapcompare-009", "layoutbench-gapcompare-010", "layoutbench-gapcompare-011", "layoutbench-gapcompare-012", "layoutbench-gridcols-001", "layoutbench-gridcols-002", "layoutbench-gridcols-003", "layoutbench-gridcols-004", "layoutbench-gridcols-005", "layoutbench-gridcols-006", "layoutbench-gridcols-007", "layoutbench-gridcols-008", "layoutbench-gridcols-009", "layoutbench-gridcols-010", "layoutbench-gridcols-011", "layoutbench-gridcols-012", "layoutbench-gridgaps-001", "layoutbench-gridgaps-002", "layoutbench-gridgaps-003", "layoutbench-gridgaps-004", "layoutbench-gridgaps-005", "layoutbench-gridgaps-006", "layoutbench-gridgaps-007", "layoutbench-gridgaps-008", "layoutbench-gridgaps-009", "layoutbench-gridgaps-010", "layoutbench-gridgaps-011", "layoutbench-gridgaps-012", "layoutbench-gridrows-001", "layoutbench-gridrows-002", "layoutbench-gridrows-003", "layoutbench-gridrows-004", "layoutbench-gridrows-005", "layoutbench-gridrows-006", "layoutbench-gridrows-007", "layoutbench-gridrows-008", "layoutbench-gridrows-009", "layoutbench-gridrows-010", "layoutbench-gridrows-011", "layoutbench-gridrows-012", "layoutbench-gridspan-001", "layoutbench-gridspan-002", "layoutbench-gridspan-003", "layoutbench-gridspan-004", "layoutbench-gridspan-005", "layoutbench-gridspan-006", "layoutbench-gridspan-007", "layoutbench-gridspan-008", "layoutbench-gridspan-009", "layoutbench-gridspan-010", "layoutbench-gridspan-011", "layoutbench-gridspan-012", "layoutbench-gridtracks-001", "layoutbench-gridtracks-002", "layoutbench-gridtracks-003", "layoutbench-gridtracks-004", "layoutbench-gridtracks-005", "layoutbench-gridtracks-006", "layoutbench-gridtracks-007", "layoutbench-gridtracks-008", "layoutbench-gridtracks-009", "layoutbench-gridtracks-010", "layoutbench-gridtracks-011", "layoutbench-gridtracks-012", "layoutbench-groupgap-001", "layoutbench-groupgap-002", "layoutbench-groupgap-003", "layoutbench-groupgap-004", "layoutbench-groupgap-005", "layoutbench-groupgap-006", "layoutbench-groupgap-007", "layoutbench-groupgap-008", "layoutbench-groupgap-009", "layoutbench-groupgap-010", "layoutbench-groupgap-011", "layoutbench-groupgap-012", "layoutbench-nestedflow-001", "layoutbench-nestedflow-002", "layoutbench-nestedflow-003", "layoutbench-nestedflow-004", "layoutbench-nestedflow-005", "layoutbench-nestedflow-006", "layoutbench-nestedflow-007", "layoutbench-nestedflow-008", "layoutbench-nestedflow-009", "layoutbench-nestedflow-010", "layoutbench-nestedflow-011", "layoutbench-nestedflow-012", "layoutbench-nestedflow-013", "layoutbench-nestedflow-014", "layoutbench-nestedflow-015", "layoutbench-nestedflow-016", "layoutbench-nestedflow-017", "layoutbench-nestedflow-018", "layoutbench-nestedflow-019", "layoutbench-nestedflow-020", "layoutbench-nestedflow-021", "layoutbench-nestedflow-022", "layoutbench-nestedflow-023", "layoutbench-nestedflow-024", "layoutbench-nestedwrap-001", "layoutbench-nestedwrap-002", "layoutbench-nestedwrap-003", "layoutbench-nestedwrap-004", "layoutbench-nestedwrap-005", "layoutbench-nestedwrap-006", "layoutbench-nestedwrap-007", "layoutbench-nestedwrap-008", "layoutbench-nestedwrap-009", "layoutbench-nestedwrap-010", "layoutbench-nestedwrap-011", "layoutbench-nestedwrap-012", "layoutbench-nestedwrap-013", "layoutbench-nestedwrap-014", "layoutbench-nestedwrap-015", "layoutbench-nestedwrap-016", "layoutbench-nestedwrap-017", "layoutbench-nestedwrap-018", "layoutbench-nestedwrap-019", "layoutbench-nestedwrap-020", "layoutbench-nestedwrap-021", "layoutbench-nestedwrap-022", "layoutbench-nestedwrap-023", "layoutbench-nestedwrap-024", "layoutbench-padcompare-001", "layoutbench-padcompare-002", "layoutbench-padcompare-003", "layoutbench-padcompare-004", "layoutbench-padcompare-005", "layoutbench-padcompare-006", "layoutbench-padcompare-007", "layoutbench-padcompare-008", "layoutbench-padcompare-009", "layoutbench-padcompare-010", "layoutbench-padcompare-011", "layoutbench-padcompare-012", "layoutbench-textalign-001", "layoutbench-textalign-002", "layoutbench-textalign-003", "layoutbench-textalign-004", "layoutbench-textalign-005", "layoutbench-textalign-006", "layoutbench-textalign-007", "layoutbench-textalign-008", "layoutbench-textalign-009", "layoutbench-textalign-010", "layoutbench-textalign-011", "layoutbench-textalign-012", "layoutbench-textalign-013", "layoutbench-textalign-014", "layoutbench-textalign-015", "layoutbench-textalign-016", "layoutbench-textcolumns-001", "layoutbench-textcolumns-002", "layoutbench-textcolumns-003", "layoutbench-textcolumns-004", "layoutbench-textcolumns-005", "layoutbench-textcolumns-006", "layoutbench-textcolumns-007", "layoutbench-textcolumns-008", "layoutbench-textcolumns-009", "layoutbench-textcolumns-010", "layoutbench-textcolumns-011", "layoutbench-textcolumns-012", "layoutbench-topflow-001", "layoutbench-topflow-002", "layoutbench-topflow-003", "layoutbench-topflow-004", "layoutbench-topflow-005", "layoutbench-topflow-006", "layoutbench-topflow-007", "layoutbench-topflow-008", "layoutbench-topflow-009", "layoutbench-topflow-010", "layoutbench-topflow-011", "layoutbench-topflow-012", "layoutbench-topflow-013", "layoutbench-topflow-014", "layoutbench-topflow-015", "layoutbench-topflow-016", "layoutbench-topflow-017", "layoutbench-topflow-018", "layoutbench-topflow-019", "layoutbench-topflow-020", "layoutbench-topflow-021", "layoutbench-topflow-022", "layoutbench-topflow-023", "layoutbench-topflow-024", "layoutbench-widthcompare-001", "layoutbench-widthcompare-002", "layoutbench-widthcompare-003", "layoutbench-widthcompare-004", "layoutbench-widthcompare-005", "layoutbench-widthcompare-006", "layoutbench-widthcompare-007", "layoutbench-widthcompare-008", "layoutbench-widthcompare-009", "layoutbench-widthcompare-010", "layoutbench-widthcompare-011", "layoutbench-widthcompare-012", "layoutbench-wrapalign-001", "layoutbench-wrapalign-002", "layoutbench-wrapalign-003", "layoutbench-wrapalign-004", "layoutbench-wrapalign-005", "layoutbench-wrapalign-006", "layoutbench-wrapalign-007", "layoutbench-wrapalign-008", "layoutbench-wrapalign-009", "layoutbench-wrapalign-010", "layoutbench-wrapalign-011", "layoutbench-wrapalign-012", "layoutbench-wrapcount-001", "layoutbench-wrapcount-002", "layoutbench-wrapcount-003", "layoutbench-wrapcount-004", "layoutbench-wrapcount-005", "layoutbench-wrapcount-006", "layoutbench-wrapcount-007", "layoutbench-wrapcount-008", "layoutbench-wrapcount-009", "layoutbench-wrapcount-010", "layoutbench-wrapcount-011", "layoutbench-wrapcount-012" ], "cohort_fingerprint": "56e2fb761613c5caa49412d9ea2c156fe82287109203dd73d974c23290fb64da", "model_ids": [ "pareto" ] }, "cost_basis": "Recorded metered token usage multiplied by recorded model pricing per scored input; excludes unmetered reserves and separate infrastructure attempts.", "statistical_scope": "Descriptive finite pilot. Related images share groups; no independent-sample confidence intervals or combined cross-family score.", "campaign": { "final_response_count": 292, "attempt_count": 292, "infrastructure_attempt_count": 0, "infrastructure_error_kinds": {}, "unresolved_infrastructure_task_count": 0, "spent_cost_usd": 2.4456175, "budget_usd": null, "cost_incomplete": false, "estimated_cost_attempt_count": 0 }, "models": [ { "model_id": "pareto", "model_config": { "api_key_env": "PARETO_API_KEY", "base_url": "http://127.0.0.1:8088/v1", "display_name": "Pareto", "enabled": true, "id": "pareto", "input_per_m": 2.5, "max_output_tokens": 4096, "model": "pareto", "notes": "Pareto 26.9 from Unbiased AI via local adapter proxy to https://api.unbiased.ai/v1/messages", "output_per_m": 7.5, "pricing_notes": "Standard online API rates $2.50/M input, $7.50/M output; cached input $0.25/M.", "pricing_source_url": "https://unbiased.ai/pricing", "provider": "anthropic", "source_url": "https://unbiased.ai/" }, "model_config_fingerprint": "45eb4bc3747be6a3944e91a903f2a4ccb647cd70fd8a4078e8907c26da8d2c8e", "status": "finalized", "coverage_status": "complete", "included_in_comparison": true, "availability_status": "complete", "run_status": "complete", "run_reason": null, "attempted_task_count": 292, "unresolved_infrastructure_task_count": 0, "observed_task_count": 292, "expected_task_count": 292, "comparison_task_count": 292, "families": { "blockalign": { "kind": "choice", "count": 12, "valid_count": 12, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 12, "accuracy": 1.0, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 4 }, "B": { "B": 4 }, "C": { "C": 4 } }, "mean_api_response_cost_usd": 0.00768375, "cost_known_response_count": 12, "mean_latency_sec": 2.2203576781903394, "latency_known_response_count": 12, "group_count": 12 }, "crossalign": { "kind": "choice", "count": 16, "valid_count": 16, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 16, "accuracy": 1.0, "chance_accuracy": 0.25, "confusion": { "A": { "A": 4 }, "B": { "B": 4 }, "C": { "C": 4 }, "D": { "D": 4 } }, "mean_api_response_cost_usd": 0.008174375, "cost_known_response_count": 16, "mean_latency_sec": 2.2853235815127846, "latency_known_response_count": 16, "group_count": 16 }, "direction": { "kind": "choice", "count": 8, "valid_count": 8, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 8, "accuracy": 1.0, "chance_accuracy": 0.5, "confusion": { "A": { "A": 4 }, "B": { "B": 4 } }, "mean_api_response_cost_usd": 0.00671625, "cost_known_response_count": 8, "mean_latency_sec": 1.72160610294668, "latency_known_response_count": 8, "group_count": 8 }, "distribution": { "kind": "choice", "count": 24, "valid_count": 24, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 23, "accuracy": 0.9583333333333334, "chance_accuracy": 0.16666666666666666, "confusion": { "A": { "A": 4 }, "B": { "B": 3, "A": 1 }, "C": { "C": 4 }, "D": { "D": 4 }, "E": { "E": 4 }, "F": { "F": 4 } }, "mean_api_response_cost_usd": 0.012024374999999999, "cost_known_response_count": 24, "mean_latency_sec": 4.461305249365978, "latency_known_response_count": 24, "group_count": 24 }, "gapcompare": { "kind": "choice", "count": 12, "valid_count": 9, "invalid_count": 3, "validity_rate": 0.75, "correct_count": 9, "accuracy": 0.75, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 3, "invalid": 1 }, "B": { "invalid": 1, "B": 3 }, "C": { "C": 3, "invalid": 1 } }, "mean_api_response_cost_usd": 0.008561250000000001, "cost_known_response_count": 12, "mean_latency_sec": 2.5504060374223627, "latency_known_response_count": 12, "group_count": 12 }, "gridcols": { "kind": "choice", "count": 12, "valid_count": 12, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 12, "accuracy": 1.0, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 4 }, "B": { "B": 4 }, "C": { "C": 4 } }, "mean_api_response_cost_usd": 0.00684, "cost_known_response_count": 12, "mean_latency_sec": 1.9028895854911145, "latency_known_response_count": 12, "group_count": 12 }, "gridgaps": { "kind": "choice", "count": 12, "valid_count": 10, "invalid_count": 2, "validity_rate": 0.8333333333333334, "correct_count": 8, "accuracy": 0.6666666666666666, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 2, "invalid": 1, "B": 1 }, "B": { "B": 3, "C": 1 }, "C": { "C": 3, "invalid": 1 } }, "mean_api_response_cost_usd": 0.012613749999999998, "cost_known_response_count": 12, "mean_latency_sec": 4.570957416920767, "latency_known_response_count": 12, "group_count": 12 }, "gridrows": { "kind": "choice", "count": 12, "valid_count": 12, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 12, "accuracy": 1.0, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 4 }, "B": { "B": 4 }, "C": { "C": 4 } }, "mean_api_response_cost_usd": 0.0068762499999999996, "cost_known_response_count": 12, "mean_latency_sec": 1.7937589385158692, "latency_known_response_count": 12, "group_count": 12 }, "gridspan": { "kind": "choice", "count": 12, "valid_count": 12, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 12, "accuracy": 1.0, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 4 }, "B": { "B": 4 }, "C": { "C": 4 } }, "mean_api_response_cost_usd": 0.007703124999999998, "cost_known_response_count": 12, "mean_latency_sec": 2.2703312745046182, "latency_known_response_count": 12, "group_count": 12 }, "gridtracks": { "kind": "choice", "count": 12, "valid_count": 12, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 12, "accuracy": 1.0, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 4 }, "B": { "B": 4 }, "C": { "C": 4 } }, "mean_api_response_cost_usd": 0.007631875, "cost_known_response_count": 12, "mean_latency_sec": 2.0468162483302876, "latency_known_response_count": 12, "group_count": 12 }, "groupgap": { "kind": "choice", "count": 12, "valid_count": 11, "invalid_count": 1, "validity_rate": 0.9166666666666666, "correct_count": 11, "accuracy": 0.9166666666666666, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 3, "invalid": 1 }, "B": { "B": 4 }, "C": { "C": 4 } }, "mean_api_response_cost_usd": 0.00904, "cost_known_response_count": 12, "mean_latency_sec": 3.080139108855898, "latency_known_response_count": 12, "group_count": 12 }, "nestedflow": { "kind": "choice", "count": 24, "valid_count": 24, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 24, "accuracy": 1.0, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 8 }, "B": { "B": 8 }, "C": { "C": 8 } }, "mean_api_response_cost_usd": 0.00699125, "cost_known_response_count": 24, "mean_latency_sec": 1.8491567237166844, "latency_known_response_count": 24, "group_count": 24 }, "nestedwrap": { "kind": "choice", "count": 24, "valid_count": 21, "invalid_count": 3, "validity_rate": 0.875, "correct_count": 21, "accuracy": 0.875, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 7, "invalid": 1 }, "B": { "B": 8 }, "C": { "C": 6, "invalid": 2 } }, "mean_api_response_cost_usd": 0.008269062499999999, "cost_known_response_count": 24, "mean_latency_sec": 2.7151038353331387, "latency_known_response_count": 24, "group_count": 24 }, "padcompare": { "kind": "choice", "count": 12, "valid_count": 11, "invalid_count": 1, "validity_rate": 0.9166666666666666, "correct_count": 9, "accuracy": 0.75, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 3, "invalid": 1 }, "B": { "B": 3, "A": 1 }, "C": { "C": 3, "A": 1 } }, "mean_api_response_cost_usd": 0.011597499999999998, "cost_known_response_count": 12, "mean_latency_sec": 4.397030739637557, "latency_known_response_count": 12, "group_count": 12 }, "textalign": { "kind": "choice", "count": 16, "valid_count": 13, "invalid_count": 3, "validity_rate": 0.8125, "correct_count": 10, "accuracy": 0.625, "chance_accuracy": 0.25, "confusion": { "A": { "A": 3, "D": 1 }, "B": { "B": 4 }, "C": { "C": 1, "D": 1, "A": 1, "invalid": 1 }, "D": { "invalid": 2, "D": 2 } }, "mean_api_response_cost_usd": 0.009660312500000002, "cost_known_response_count": 16, "mean_latency_sec": 3.696879255498061, "latency_known_response_count": 16, "group_count": 16 }, "textcolumns": { "kind": "choice", "count": 12, "valid_count": 12, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 12, "accuracy": 1.0, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 4 }, "B": { "B": 4 }, "C": { "C": 4 } }, "mean_api_response_cost_usd": 0.006920000000000002, "cost_known_response_count": 12, "mean_latency_sec": 1.9665560659680825, "latency_known_response_count": 12, "group_count": 12 }, "topflow": { "kind": "choice", "count": 24, "valid_count": 24, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 24, "accuracy": 1.0, "chance_accuracy": 0.5, "confusion": { "A": { "A": 12 }, "B": { "B": 12 } }, "mean_api_response_cost_usd": 0.0067325, "cost_known_response_count": 24, "mean_latency_sec": 1.785266268978982, "latency_known_response_count": 24, "group_count": 24 }, "widthcompare": { "kind": "choice", "count": 12, "valid_count": 12, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 12, "accuracy": 1.0, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 4 }, "B": { "B": 4 }, "C": { "C": 4 } }, "mean_api_response_cost_usd": 0.007297499999999999, "cost_known_response_count": 12, "mean_latency_sec": 1.9539558584025751, "latency_known_response_count": 12, "group_count": 12 }, "wrapalign": { "kind": "choice", "count": 12, "valid_count": 12, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 12, "accuracy": 1.0, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 4 }, "B": { "B": 4 }, "C": { "C": 4 } }, "mean_api_response_cost_usd": 0.007846875000000001, "cost_known_response_count": 12, "mean_latency_sec": 2.3413978639097572, "latency_known_response_count": 12, "group_count": 12 }, "wrapcount": { "kind": "choice", "count": 12, "valid_count": 12, "invalid_count": 0, "validity_rate": 1.0, "correct_count": 12, "accuracy": 1.0, "chance_accuracy": 0.3333333333333333, "confusion": { "A": { "A": 4 }, "B": { "B": 4 }, "C": { "C": 4 } }, "mean_api_response_cost_usd": 0.006898125000000001, "cost_known_response_count": 12, "mean_latency_sec": 1.609895121480804, "latency_known_response_count": 12, "group_count": 12 } }, "mean_api_response_cost_usd": 0.008375402397260274, "cost_known_response_count": 292, "mean_latency_sec": 2.60754098398898, "latency_known_response_count": 292 } ], "attempts": [ { "sequence": 1, "attempt_id": "f9135343-95ed-4ae6-ad0e-0c603ed027ca", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-gridrows-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0067025, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "B" }, "group_id": "gridrows-3-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-3-dark-0.png", "input_tokens": 2549, "key_errors": null, "latency_sec": 2.217542486032471, "model_name": "pareto", "output_tokens": 44, "prediction": { "choice": "B" }, "prompt_sha256": "0fad5d0a9261a17e68aa8de2ef5fb0207dc8af585c3baaeabfc584af8b5329f5", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-29T14:16:51.108694+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0067025, "created_at": "2026-09-29T14:16:51.108Z" }, { "sequence": 2, "attempt_id": "0091aece-9ee4-4ad2-bc2a-983a6cb78bea", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-gridcols-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0069275, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "C" }, "group_id": "gridcols-3-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-dark-0.png", "input_tokens": 2549, "key_errors": null, "latency_sec": 2.21444767084904, "model_name": "pareto", "output_tokens": 74, "prediction": { "choice": "C" }, "prompt_sha256": "4e97151663d1e3e37046a23842abcce2e96ecf3d2204566c717e9014b170f248", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-29T14:16:51.114237+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0069275, "created_at": "2026-09-29T14:16:51.114Z" }, { "sequence": 3, "attempt_id": "d61be90b-6daa-4ca6-ab0a-5426336bfc0a", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-wrapalign-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00798, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "B" }, "group_id": "wrapalign-left-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-left-light-0.png", "input_tokens": 2568, "key_errors": null, "latency_sec": 2.491852676961571, "model_name": "pareto", "output_tokens": 208, "prediction": { "choice": "B" }, "prompt_sha256": "eaad4b2f9e91b0501c9e0bbd973bee743f4eec05e7ad65bdddca7302eaa806eb", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-29T14:16:51.388566+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00798, "created_at": "2026-09-29T14:16:51.388Z" }, { "sequence": 4, "attempt_id": "103cda5a-d632-4275-b810-f692bfbe019e", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-nestedwrap-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0076175, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-21", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-21.png", "input_tokens": 2579, "key_errors": null, "latency_sec": 2.594237416051328, "model_name": "pareto", "output_tokens": 156, "prediction": { "choice": "A" }, "prompt_sha256": "a242868807b1158889b152c7c9664c235ebd10352a9a4a541f0a0af30a7fb4a5", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-29T14:16:51.493978+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0076175, "created_at": "2026-09-29T14:16:51.494Z" }, { "sequence": 5, "attempt_id": "8fc67a64-23df-4913-bcfd-76f1f828697c", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-blockalign-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.007505, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "C" }, "group_id": "blockalign-left-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-left-dark-1.png", "input_tokens": 2576, "key_errors": null, "latency_sec": 2.5952626201324165, "model_name": "pareto", "output_tokens": 142, "prediction": { "choice": "C" }, "prompt_sha256": "aec8632f26486517e0ed07640068efdb11c079f166b430d6a2eb3e7ecb5c47b1", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-29T14:16:51.500914+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.007505, "created_at": "2026-09-29T14:16:51.502Z" }, { "sequence": 6, "attempt_id": "dbeb29e7-ead6-4522-8a71-ef1f2eb99798", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-gridspan-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0083425, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "C" }, "group_id": "gridspan-1-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-1-dark-0.png", "input_tokens": 2572, "key_errors": null, "latency_sec": 2.6277199380565435, "model_name": "pareto", "output_tokens": 255, "prediction": { "choice": "C" }, "prompt_sha256": "4c2619c1f3339203e066968a6911df07e41f49d553c6c3cb2be0b1e81e76a29b", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-29T14:16:51.513236+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0083425, "created_at": "2026-09-29T14:16:51.513Z" }, { "sequence": 7, "attempt_id": "3f494385-0497-4142-b3da-40084b79885e", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-crossalign-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.008375, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "C" }, "group_id": "crossalign-start-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-start-light-1.png", "input_tokens": 2642, "key_errors": null, "latency_sec": 2.816447041928768, "model_name": "pareto", "output_tokens": 236, "prediction": { "choice": "C" }, "prompt_sha256": "a0cc0893516679f3ab248b3cfb7b1e000bacbb17beb9b9bd9dcc6099c756222f", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-29T14:16:51.714613+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.008375, "created_at": "2026-09-29T14:16:51.714Z" }, { "sequence": 8, "attempt_id": "9b32235b-3a60-4bd7-a260-36c3be1d9daf", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-groupgap-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00867, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "A" }, "group_id": "groupgap-equal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-equal-dark-0.png", "input_tokens": 2583, "key_errors": null, "latency_sec": 3.0499927271157503, "model_name": "pareto", "output_tokens": 295, "prediction": { "choice": "A" }, "prompt_sha256": "a4ae0e9e5d9435b02d3759ecf1850c36024f893c4de107ea8256f2c5052eaa53", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-29T14:16:51.957095+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00867, "created_at": "2026-09-29T14:16:51.957Z" }, { "sequence": 9, "attempt_id": "f081bf08-8aee-4a08-9938-48c7ddc49118", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-padcompare-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00934, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "B" }, "group_id": "padcompare-horizontal-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-horizontal-dark-1.png", "input_tokens": 2593, "key_errors": null, "latency_sec": 3.247768934816122, "model_name": "pareto", "output_tokens": 381, "prediction": { "choice": "B" }, "prompt_sha256": "4f484ecb32307572888516788b76d03dcfb86406579ae3871dbbdd5d50c24d37", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-29T14:16:52.135401+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00934, "created_at": "2026-09-29T14:16:52.135Z" }, { "sequence": 10, "attempt_id": "92fe92d4-029b-48bf-b737-23aef6d04a0b", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-nestedflow-014", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0068425, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-14", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-14.png", "input_tokens": 2569, "key_errors": null, "latency_sec": 1.6075586378574371, "model_name": "pareto", "output_tokens": 56, "prediction": { "choice": "C" }, "prompt_sha256": "ac0c30958153584a4b94fe042ea1cbf718f020f9f66dd474a09c514effd2f9f9", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-29T14:16:53.003340+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-014", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0068425, "created_at": "2026-09-29T14:16:53.003Z" }, { "sequence": 11, "attempt_id": "f561b614-eea0-4707-9926-50e2d2e18d33", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-gridgaps-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00785, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "A" }, "group_id": "gridgaps-vertical-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-vertical-light-0.png", "input_tokens": 2567, "key_errors": null, "latency_sec": 1.919954146957025, "model_name": "pareto", "output_tokens": 191, "prediction": { "choice": "A" }, "prompt_sha256": "da1f2cca453f507bde83a2ac6d19b503fd7ab1c709bd4a77ee1046cf28ba5c59", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-29T14:16:53.043183+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00785, "created_at": "2026-09-29T14:16:53.043Z" }, { "sequence": 12, "attempt_id": "f40db26f-baa2-4b98-b8ea-bb64b0bb2b4c", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-wrapcount-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0068375, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "B" }, "group_id": "wrapcount-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-3-dark-1.png", "input_tokens": 2546, "key_errors": null, "latency_sec": 1.7878159550018609, "model_name": "pareto", "output_tokens": 63, "prediction": { "choice": "B" }, "prompt_sha256": "15d8bd94924cd0eb8efdec17da21c6da4f97b10dd9a51e2d28e533f308413109", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-29T14:16:53.297271+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0068375, "created_at": "2026-09-29T14:16:53.297Z" }, { "sequence": 13, "attempt_id": "b3ba2e0d-bc1d-4cd2-a1e3-c8a9144ed3f0", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-groupgap-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.007845, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "A" }, "group_id": "groupgap-between-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-between-dark-1.png", "input_tokens": 2583, "key_errors": null, "latency_sec": 1.8564705101307482, "model_name": "pareto", "output_tokens": 185, "prediction": { "choice": "A" }, "prompt_sha256": "6cba71084ece6f53e834e34e24c720eaabde46f305bb7ec0454830a5812e840e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-29T14:16:53.357670+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.007845, "created_at": "2026-09-29T14:16:53.357Z" }, { "sequence": 14, "attempt_id": "f30e15b4-b4da-470b-b85e-ea7b57a2189f", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-topflow-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0067525, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-11", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-11.png", "input_tokens": 2566, "key_errors": null, "latency_sec": 2.274386632023379, "model_name": "pareto", "output_tokens": 45, "prediction": { "choice": "B" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-29T14:16:54.004974+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0067525, "created_at": "2026-09-29T14:16:54.005Z" }, { "sequence": 15, "attempt_id": "7ce0410b-4aa0-4437-8367-f506d28a13af", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-textcolumns-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00664, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "C" }, "group_id": "textcolumns-2-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-2-dark-1.png", "input_tokens": 2545, "key_errors": null, "latency_sec": 2.5256814828608185, "model_name": "pareto", "output_tokens": 37, "prediction": { "choice": "C" }, "prompt_sha256": "9709598fa2c03afd3c8191b7a476b8aa1593f8488926311cbd6c9d5952354d67", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-29T14:16:54.045429+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00664, "created_at": "2026-09-29T14:16:54.045Z" }, { "sequence": 16, "attempt_id": "138fef8c-20b5-4cc2-bca6-4035c21b1e79", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-textalign-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0095525, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "B" }, "group_id": "textalign-right-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-right-light-0.png", "input_tokens": 2597, "key_errors": null, "latency_sec": 3.2158248180057853, "model_name": "pareto", "output_tokens": 408, "prediction": { "choice": "B" }, "prompt_sha256": "4d2473b3f62dd5f7bdca37b6285ca5748e1b3eadc7e0f04cb6fd68e2978a908b", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-29T14:16:54.336966+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0095525, "created_at": "2026-09-29T14:16:54.337Z" }, { "sequence": 17, "attempt_id": "e3ddc425-6e29-4720-889f-50e09a7f5a31", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-widthcompare-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0070675, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "C" }, "group_id": "widthcompare-last-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-last-dark-0.png", "input_tokens": 2560, "key_errors": null, "latency_sec": 1.5063107998576015, "model_name": "pareto", "output_tokens": 89, "prediction": { "choice": "C" }, "prompt_sha256": "875f019d57875304ded0af8f075989118a0e4fcf9ca43c9eba1cda4f64a503a0", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-29T14:16:54.517053+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0070675, "created_at": "2026-09-29T14:16:54.517Z" }, { "sequence": 18, "attempt_id": "2ef64892-9c50-44ce-9789-286cd319497a", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-gridspan-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0087775, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "B" }, "group_id": "gridspan-1-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-1-light-0.png", "input_tokens": 2572, "key_errors": null, "latency_sec": 2.6471281989943236, "model_name": "pareto", "output_tokens": 313, "prediction": { "choice": "B" }, "prompt_sha256": "036502f601ca514f2b291d9a63cb6d8de5371510c2e08f970e9cb252d2c52ce2", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-29T14:16:54.611947+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0087775, "created_at": "2026-09-29T14:16:54.612Z" }, { "sequence": 19, "attempt_id": "1e5a95f7-d78f-44f7-baae-91374d4eadc6", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-gridgaps-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0088325, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "B" }, "group_id": "gridgaps-vertical-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-vertical-dark-0.png", "input_tokens": 2567, "key_errors": null, "latency_sec": 2.699047068832442, "model_name": "pareto", "output_tokens": 322, "prediction": { "choice": "B" }, "prompt_sha256": "9bd6b2738f2f8c2f8f4f1ac19c1985fb2970e8e7972ea985cb663167fcc029e4", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-29T14:16:54.841430+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0088325, "created_at": "2026-09-29T14:16:54.841Z" }, { "sequence": 20, "attempt_id": "b1c4c090-5e65-4368-aacc-dbf7f80acd3d", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-wrapcount-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00692, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "A" }, "group_id": "wrapcount-2-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-2-dark-1.png", "input_tokens": 2546, "key_errors": null, "latency_sec": 1.8786774198524654, "model_name": "pareto", "output_tokens": 74, "prediction": { "choice": "A" }, "prompt_sha256": "f426cc4a860aebf2f83ce2ed8bb35c244b740a3f0e0196bacf7e87e4c01825c0", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-29T14:16:54.929259+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00692, "created_at": "2026-09-29T14:16:54.929Z" }, { "sequence": 21, "attempt_id": "981b910c-9031-4539-ada1-cd0c236ae4b5", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-topflow-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0067225, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-06", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-06.png", "input_tokens": 2566, "key_errors": null, "latency_sec": 1.781356168910861, "model_name": "pareto", "output_tokens": 41, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-29T14:16:55.146899+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0067225, "created_at": "2026-09-29T14:16:55.147Z" }, { "sequence": 22, "attempt_id": "7d30a12f-8521-4107-88ab-27660b3947c0", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-textcolumns-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006835, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "A" }, "group_id": "textcolumns-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-3-light-1.png", "input_tokens": 2545, "key_errors": null, "latency_sec": 1.9135226719081402, "model_name": "pareto", "output_tokens": 63, "prediction": { "choice": "A" }, "prompt_sha256": "7ab8a26bbcac7d24004e4a74f0ea411de080ccc6cfed0989403acf58e56e0339", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-29T14:16:55.218339+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.006835, "created_at": "2026-09-29T14:16:55.218Z" }, { "sequence": 23, "attempt_id": "84bea602-d614-4905-a602-ef258ef45b0c", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-wrapcount-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006845, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "B" }, "group_id": "wrapcount-2-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-2-dark-0.png", "input_tokens": 2546, "key_errors": null, "latency_sec": 1.5166279340628535, "model_name": "pareto", "output_tokens": 64, "prediction": { "choice": "B" }, "prompt_sha256": "d5a11da43aa9608fe90b20a74d3d14e9dbf3731c33be73ca93aad96c927ccc7d", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-29T14:16:55.570021+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.006845, "created_at": "2026-09-29T14:16:55.570Z" }, { "sequence": 24, "attempt_id": "c4096bf5-d28e-445f-9aca-32b9b5aff575", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-widthcompare-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0076225, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "C" }, "group_id": "widthcompare-equal-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-equal-dark-1.png", "input_tokens": 2560, "key_errors": null, "latency_sec": 1.8592016680631787, "model_name": "pareto", "output_tokens": 163, "prediction": { "choice": "C" }, "prompt_sha256": "807b859493212c8c914db2b1d9395b00171e3c91df87b66b9663b6cc7b65c485", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-29T14:16:55.872183+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0076225, "created_at": "2026-09-29T14:16:55.872Z" }, { "sequence": 25, "attempt_id": "94c550df-5ebb-4d93-8e42-38a7f1a907ae", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-gridtracks-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0076375, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "B" }, "group_id": "gridtracks-right-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-right-light-1.png", "input_tokens": 2569, "key_errors": null, "latency_sec": 1.7599668740294874, "model_name": "pareto", "output_tokens": 162, "prediction": { "choice": "B" }, "prompt_sha256": "81f56b69372d1aac7eb5a641448a893a03f044c68d3ed320fc7646d8a1c13ed9", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-29T14:16:56.105198+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0076375, "created_at": "2026-09-29T14:16:56.105Z" }, { "sequence": 26, "attempt_id": "b6fec9af-ee55-4447-87c2-80da8f7e0dda", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-wrapcount-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0067925, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "A" }, "group_id": "wrapcount-1-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-1-light-1.png", "input_tokens": 2546, "key_errors": null, "latency_sec": 1.5533016379922628, "model_name": "pareto", "output_tokens": 57, "prediction": { "choice": "A" }, "prompt_sha256": "d5a11da43aa9608fe90b20a74d3d14e9dbf3731c33be73ca93aad96c927ccc7d", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-29T14:16:56.403416+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0067925, "created_at": "2026-09-29T14:16:56.403Z" }, { "sequence": 27, "attempt_id": "c8c50b5a-bcd4-4cfc-a56e-630c398fb1e4", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-gridspan-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0076375, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "C" }, "group_id": "gridspan-2-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-2-light-0.png", "input_tokens": 2572, "key_errors": null, "latency_sec": 2.249529456021264, "model_name": "pareto", "output_tokens": 161, "prediction": { "choice": "C" }, "prompt_sha256": "036502f601ca514f2b291d9a63cb6d8de5371510c2e08f970e9cb252d2c52ce2", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-29T14:16:56.774699+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0076375, "created_at": "2026-09-29T14:16:56.774Z" }, { "sequence": 28, "attempt_id": "973a0c17-79ba-47b7-97ce-27f8245eec32", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-nestedflow-023", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0068575, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-23", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-23.png", "input_tokens": 2569, "key_errors": null, "latency_sec": 1.6700430971104652, "model_name": "pareto", "output_tokens": 58, "prediction": { "choice": "A" }, "prompt_sha256": "6cb509a4fb3ecab9a334fb061a4aa543245f6985aa65f331cb06646a50f35dfe", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-29T14:16:56.825602+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-023", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0068575, "created_at": "2026-09-29T14:16:56.825Z" }, { "sequence": 29, "attempt_id": "22b8bc09-4733-43f7-8a0a-74597bca0f2b", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-nestedwrap-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00773, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-26", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-26.png", "input_tokens": 2579, "key_errors": null, "latency_sec": 2.26339330105111, "model_name": "pareto", "output_tokens": 171, "prediction": { "choice": "A" }, "prompt_sha256": "dbe3324244c1acc8302aa7a2f89b338ba07244a1490efb8ddaf45fc8d47a978d", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-29T14:16:56.889064+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00773, "created_at": "2026-09-29T14:16:56.889Z" }, { "sequence": 30, "attempt_id": "ee004b1f-8ad7-4cc0-9ba5-d5e248e80fe6", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-gridrows-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0067925, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "C" }, "group_id": "gridcols-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-light-1.png", "input_tokens": 2549, "key_errors": null, "latency_sec": 1.855449269991368, "model_name": "pareto", "output_tokens": 56, "prediction": { "choice": "C" }, "prompt_sha256": "da4194e14f8d3c86eb007c251e2f977baeaf16f3ed67610f233d5aaa1e16745d", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-29T14:16:57.434750+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0067925, "created_at": "2026-09-29T14:16:57.434Z" }, { "sequence": 31, "attempt_id": "b60d8957-d87c-49d4-80b8-84bce54b22c3", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-direction-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00675, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "A" }, "group_id": "direction-column-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-column-dark-0.png", "input_tokens": 2547, "key_errors": null, "latency_sec": 2.2802412561140954, "model_name": "pareto", "output_tokens": 51, "prediction": { "choice": "A" }, "prompt_sha256": "3768f3feb47a0750f78e6a2c372e707b1b87f7af93fd1fa6b13ef50f1092f226", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-29T14:16:57.507226+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00675, "created_at": "2026-09-29T14:16:57.507Z" }, { "sequence": 32, "attempt_id": "3906b3d4-86db-4a5f-b0a1-7e9a53ca755b", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-wrapalign-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00756, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "C" }, "group_id": "wrapalign-center-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-center-dark-1.png", "input_tokens": 2568, "key_errors": null, "latency_sec": 2.9206901909783483, "model_name": "pareto", "output_tokens": 152, "prediction": { "choice": "C" }, "prompt_sha256": "4f5e9513c600f9a4c3e41e06f8b5947480ecacc47063f687d8ace9fa210713f1", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-29T14:16:57.858589+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00756, "created_at": "2026-09-29T14:16:57.858Z" }, { "sequence": 33, "attempt_id": "aab2530d-3d96-4c7e-ba70-5f060c349dd1", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-nestedflow-020", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00748, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-20", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-20.png", "input_tokens": 2569, "key_errors": null, "latency_sec": 2.030281794955954, "model_name": "pareto", "output_tokens": 141, "prediction": { "choice": "A" }, "prompt_sha256": "377bcd13e265380fd865ccbfd9819e64dfd2c4c3b2980e09c6e4d88924162f6c", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-29T14:16:57.911567+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-020", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00748, "created_at": "2026-09-29T14:16:57.911Z" }, { "sequence": 34, "attempt_id": "5b7049af-5df2-4d35-a24d-3d179643c1ad", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-wrapalign-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0074625, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "B" }, "group_id": "wrapalign-center-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-center-dark-0.png", "input_tokens": 2568, "key_errors": null, "latency_sec": 1.6495219201315194, "model_name": "pareto", "output_tokens": 139, "prediction": { "choice": "B" }, "prompt_sha256": "6f9f40796910da778fa5f30bd2cb43da95875ca8d6fb47db39f42ffb70cbab76", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-29T14:16:58.062637+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0074625, "created_at": "2026-09-29T14:16:58.062Z" }, { "sequence": 35, "attempt_id": "d29e1164-5c50-4b93-af1a-72b126939db9", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-distribution-022", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0198725, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "C" }, "group_id": "distribution-evenly-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-evenly-light-1.png", "input_tokens": 2648, "key_errors": null, "latency_sec": 9.195097668096423, "model_name": "pareto", "output_tokens": 1767, "prediction": { "choice": "C" }, "prompt_sha256": "40b9c4de265a2cf5726c7450b659e9a657270b652500eb7da5fcf557c0fe9c72", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-29T14:16:58.097585+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-022", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0198725, "created_at": "2026-09-29T14:16:58.097Z" }, { "sequence": 36, "attempt_id": "ab2a2860-eb49-43bd-9166-96b57cae07c1", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-gridspan-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.007165, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "B" }, "group_id": "gridspan-3-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-3-light-0.png", "input_tokens": 2572, "key_errors": null, "latency_sec": 1.556035669054836, "model_name": "pareto", "output_tokens": 98, "prediction": { "choice": "B" }, "prompt_sha256": "9320fc919a5f33d10a5a97b8269a21a6fc8fa9787daa8025e876c4a4452a6d4a", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-29T14:16:58.340160+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.007165, "created_at": "2026-09-29T14:16:58.340Z" }, { "sequence": 37, "attempt_id": "b6bdb128-c7c7-4384-85b2-13a66f236e4c", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-distribution-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00887, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "A" }, "group_id": "distribution-end-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-end-dark-0.png", "input_tokens": 2648, "key_errors": null, "latency_sec": 2.447814923012629, "model_name": "pareto", "output_tokens": 300, "prediction": { "choice": "A" }, "prompt_sha256": "230bd5c48fe9323eb18d8e68c17b30c36e29e314d1401da5a141c036bd28ebb6", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-29T14:16:58.561770+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00887, "created_at": "2026-09-29T14:16:58.561Z" }, { "sequence": 38, "attempt_id": "1af3b436-ca1d-4c5b-bcb6-fd5ca3b97de5", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-crossalign-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00866, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "A" }, "group_id": "crossalign-start-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-start-dark-0.png", "input_tokens": 2642, "key_errors": null, "latency_sec": 2.290109053021297, "model_name": "pareto", "output_tokens": 274, "prediction": { "choice": "A" }, "prompt_sha256": "e6e3e7be0a2b7e91666344c6f0c3a0f931a9ea63d47c2dbd9e1c4cb334420e83", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-29T14:16:59.125175+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00866, "created_at": "2026-09-29T14:16:59.125Z" }, { "sequence": 39, "attempt_id": "2cc54899-f663-414d-9244-0d47964f03db", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-gapcompare-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.007655, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "A" }, "group_id": "gapcompare-equal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-equal-light-0.png", "input_tokens": 2567, "key_errors": null, "latency_sec": 1.7814098601229489, "model_name": "pareto", "output_tokens": 165, "prediction": { "choice": "A" }, "prompt_sha256": "2aacf3dbef42dbefa25b359df3a8af3c1541ab2712c25e9b3c3f5cf8371b79e4", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-29T14:16:59.225974+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.007655, "created_at": "2026-09-29T14:16:59.226Z" }, { "sequence": 40, "attempt_id": "ed3322e2-97d9-48f9-a41c-b674a9eb6113", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-nestedflow-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0068575, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-10", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-10.png", "input_tokens": 2569, "key_errors": null, "latency_sec": 1.753918498987332, "model_name": "pareto", "output_tokens": 58, "prediction": { "choice": "B" }, "prompt_sha256": "a555c9a37a814a2a2637ad2f6a1177955732dce4f82ecebc074c8930d36c5293", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-29T14:16:59.271172+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0068575, "created_at": "2026-09-29T14:16:59.271Z" }, { "sequence": 41, "attempt_id": "11579e47-39d8-4299-9b06-bdc5cd39883a", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-topflow-024", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0067, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-24", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-24.png", "input_tokens": 2566, "key_errors": null, "latency_sec": 1.385108107002452, "model_name": "pareto", "output_tokens": 38, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-29T14:16:59.457682+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-024", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0067, "created_at": "2026-09-29T14:16:59.457Z" }, { "sequence": 42, "attempt_id": "8583f35a-68ca-4406-ad7d-330fee31b7dc", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-topflow-021", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0066775, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-21", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-21.png", "input_tokens": 2566, "key_errors": null, "latency_sec": 1.57088692788966, "model_name": "pareto", "output_tokens": 35, "prediction": { "choice": "A" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-29T14:16:59.499291+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-021", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0066775, "created_at": "2026-09-29T14:16:59.499Z" }, { "sequence": 43, "attempt_id": "d0afe608-9c15-4dec-8c95-96906bd47b40", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-widthcompare-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0072175, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "A" }, "group_id": "widthcompare-last-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-last-dark-1.png", "input_tokens": 2560, "key_errors": null, "latency_sec": 1.801532695069909, "model_name": "pareto", "output_tokens": 109, "prediction": { "choice": "A" }, "prompt_sha256": "cd2c2f1f86d4ef1f19fa7f880569a29637d27eed1e41cdded418dc476e8498ed", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-29T14:16:59.670323+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0072175, "created_at": "2026-09-29T14:16:59.670Z" }, { "sequence": 44, "attempt_id": "c7d7a8ec-5dc9-43c9-ace3-8341997802a7", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-gridgaps-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009035, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "C" }, "group_id": "gridgaps-horizontal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-horizontal-light-0.png", "input_tokens": 2567, "key_errors": null, "latency_sec": 2.869866468012333, "model_name": "pareto", "output_tokens": 349, "prediction": { "choice": "C" }, "prompt_sha256": "9bd6b2738f2f8c2f8f4f1ac19c1985fb2970e8e7972ea985cb663167fcc029e4", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-29T14:16:59.769046+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.009035, "created_at": "2026-09-29T14:16:59.769Z" }, { "sequence": 45, "attempt_id": "dc6a4a8a-7185-42ba-a799-d063d5e30a56", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-distribution-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.008825, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "E" }, "group_id": "distribution-center-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-center-light-1.png", "input_tokens": 2648, "key_errors": null, "latency_sec": 2.2895193859003484, "model_name": "pareto", "output_tokens": 294, "prediction": { "choice": "E" }, "prompt_sha256": "2f316a4393b531fa7b5e9b8090c0a3c148e22a78e6e8f9382c57e24f29d56be7", "provider": "anthropic", "raw_prediction": "{\"choice\": \"E\"}", "recorded_at": "2026-09-29T14:17:00.639986+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.008825, "created_at": "2026-09-29T14:17:00.640Z" }, { "sequence": 46, "attempt_id": "99c28c6c-6a27-4c77-ab7a-0b41e0caebe8", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-padcompare-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.007975, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "A" }, "group_id": "padcompare-vertical-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-vertical-light-0.png", "input_tokens": 2593, "key_errors": null, "latency_sec": 2.5880502050276846, "model_name": "pareto", "output_tokens": 199, "prediction": { "choice": "A" }, "prompt_sha256": "0abb90b71247ce9a0876c99ad3ef52ea5a9ae09bfffb2875eec73b10166640c1", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-29T14:17:00.696099+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.007975, "created_at": "2026-09-29T14:17:00.696Z" }, { "sequence": 47, "attempt_id": "450ce05d-4050-4f59-b188-c3bd3246b4a9", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-gridtracks-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0073975, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "C" }, "group_id": "gridtracks-right-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-right-dark-1.png", "input_tokens": 2569, "key_errors": null, "latency_sec": 2.0346484950277954, "model_name": "pareto", "output_tokens": 130, "prediction": { "choice": "C" }, "prompt_sha256": "330f46df745e35a409a07fe87ab9184b25be7cbe0c4a58bdce6f2fce80123f3a", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-29T14:17:01.170553+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0073975, "created_at": "2026-09-29T14:17:01.170Z" }, { "sequence": 48, "attempt_id": "ad18155c-32ac-4c42-aa9c-ee26c719f37b", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-crossalign-013", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0083225, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "C" }, "group_id": "crossalign-stretch-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-stretch-light-0.png", "input_tokens": 2642, "key_errors": null, "latency_sec": 2.1384620149619877, "model_name": "pareto", "output_tokens": 229, "prediction": { "choice": "C" }, "prompt_sha256": "42e3d5b05e35df09d8f58ddf18bbea16275da96264b53e5005136200c350e010", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-29T14:17:01.374899+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-013", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0083225, "created_at": "2026-09-29T14:17:01.375Z" }, { "sequence": 49, "attempt_id": "81f5cfe5-a502-409e-badd-0604db72d5d7", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-wrapalign-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.008175, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "C" }, "group_id": "wrapalign-right-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-right-light-1.png", "input_tokens": 2568, "key_errors": null, "latency_sec": 2.1771751458290964, "model_name": "pareto", "output_tokens": 234, "prediction": { "choice": "C" }, "prompt_sha256": "be68205720b4630afe6748f490c25dd02332bab19b459ce5a69565f78da84e8a", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-29T14:17:01.458925+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.008175, "created_at": "2026-09-29T14:17:01.459Z" }, { "sequence": 50, "attempt_id": "13b27c9a-55b3-4bd4-b3f0-c2eab1b8ab37", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-gridspan-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0070525, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "B" }, "group_id": "gridspan-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-3-light-1.png", "input_tokens": 2572, "key_errors": null, "latency_sec": 2.148014223901555, "model_name": "pareto", "output_tokens": 83, "prediction": { "choice": "B" }, "prompt_sha256": "4c2619c1f3339203e066968a6911df07e41f49d553c6c3cb2be0b1e81e76a29b", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-29T14:17:01.657795+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0070525, "created_at": "2026-09-29T14:17:01.657Z" }, { "sequence": 51, "attempt_id": "16d12972-bc82-4877-a9bf-cf10e0691eb0", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-gridtracks-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00766, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "B" }, "group_id": "gridcols-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-dark-1.png", "input_tokens": 2569, "key_errors": null, "latency_sec": 2.091701610945165, "model_name": "pareto", "output_tokens": 165, "prediction": { "choice": "B" }, "prompt_sha256": "e1686611bde92953032219f0663fed17a23573139af7c26937579757a13cc24d", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-29T14:17:01.773239+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00766, "created_at": "2026-09-29T14:17:01.773Z" }, { "sequence": 52, "attempt_id": "12607f01-8393-40ba-a695-bc0392dc2795", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-gridgaps-009", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.0101825, "err": null, "error": "Expecting value: line 1 column 1 (char 0)", "error_kind": "invalid_response", "family": "gridgaps", "ground_truth": { "choice": "A" }, "group_id": "gridgaps-equal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-equal-light-0.png", "input_tokens": 2567, "key_errors": null, "latency_sec": 3.483148311963305, "model_name": "pareto", "output_tokens": 502, "prediction": {}, "prompt_sha256": "9bd6b2738f2f8c2f8f4f1ac19c1985fb2970e8e7972ea985cb663167fcc029e4", "provider": "anthropic", "raw_prediction": "A", "recorded_at": "2026-09-29T14:17:02.056334+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-gridgaps-009", "tight_score": null, "unmetered_attempts": 0, "valid": false, "within_bands": null }, "cost_usd": 0.0101825, "created_at": "2026-09-29T14:17:02.056Z" }, { "sequence": 53, "attempt_id": "0af64825-ea14-4b33-b8a8-ffe933f82f59", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-topflow-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0067225, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-05", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-05.png", "input_tokens": 2566, "key_errors": null, "latency_sec": 1.6422356590628624, "model_name": "pareto", "output_tokens": 41, "prediction": { "choice": "B" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-29T14:17:02.293549+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0067225, "created_at": "2026-09-29T14:17:02.293Z" }, { "sequence": 54, "attempt_id": "5e3328a0-b42c-4b34-a42b-61d518d912f4", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-distribution-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0085175, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "E" }, "group_id": "distribution-center-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-center-light-0.png", "input_tokens": 2648, "key_errors": null, "latency_sec": 2.5328790300991386, "model_name": "pareto", "output_tokens": 253, "prediction": { "choice": "E" }, "prompt_sha256": "d47c19cab91f33f4d25cba7cf52697e79dfc50d2011d63234517770b7bbe3d42", "provider": "anthropic", "raw_prediction": "{\"choice\": \"E\"}", "recorded_at": "2026-09-29T14:17:02.313807+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0085175, "created_at": "2026-09-29T14:17:02.313Z" }, { "sequence": 55, "attempt_id": "7ce0b279-dcf6-4a29-8e68-74ad12d6789b", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-nestedwrap-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0087425, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-24", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-24.png", "input_tokens": 2579, "key_errors": null, "latency_sec": 2.3873048461973667, "model_name": "pareto", "output_tokens": 306, "prediction": { "choice": "C" }, "prompt_sha256": "1019a8d1887a379785b7f0f2096c38ba800c5c05f4005e0a2fc11959e378500f", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-29T14:17:03.094763+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0087425, "created_at": "2026-09-29T14:17:03.094Z" }, { "sequence": 56, "attempt_id": "43926545-fa95-4489-aa6e-762f62b90ee3", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-wrapalign-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00708, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "B" }, "group_id": "wrapalign-center-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-center-light-1.png", "input_tokens": 2568, "key_errors": null, "latency_sec": 1.6822827658616006, "model_name": "pareto", "output_tokens": 88, "prediction": { "choice": "B" }, "prompt_sha256": "71d9f4106dcceb21abcbdbcb7978862ff824dca3aef4bc9d0cc134a9007d7be3", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-29T14:17:03.153638+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00708, "created_at": "2026-09-29T14:17:03.153Z" }, { "sequence": 57, "attempt_id": "ff5ec6a6-71b9-45e9-8e79-53886f0d741d", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-topflow-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0067525, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-04", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-04.png", "input_tokens": 2566, "key_errors": null, "latency_sec": 1.7966016689315438, "model_name": "pareto", "output_tokens": 45, "prediction": { "choice": "B" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-29T14:17:04.123399+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0067525, "created_at": "2026-09-29T14:17:04.123Z" }, { "sequence": 58, "attempt_id": "0dd9b4ea-8a80-485b-9c43-c7ac331b3eea", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-gridgaps-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.008795, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "C" }, "group_id": "gridgaps-horizontal-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-horizontal-dark-1.png", "input_tokens": 2567, "key_errors": null, "latency_sec": 3.1083950740285218, "model_name": "pareto", "output_tokens": 317, "prediction": { "choice": "C" }, "prompt_sha256": "6fc908b851f777215935a481f34941f8c8db739f40ec24d39d8b889fd6975989", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-29T14:17:04.300457+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.008795, "created_at": "2026-09-29T14:17:04.300Z" }, { "sequence": 59, "attempt_id": "2cdca63c-71b0-486a-9288-cb388e864857", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-direction-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00672, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "B" }, "group_id": "direction-column-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-column-dark-1.png", "input_tokens": 2547, "key_errors": null, "latency_sec": 1.5193028808571398, "model_name": "pareto", "output_tokens": 47, "prediction": { "choice": "B" }, "prompt_sha256": "84ba6fd0e7012086e1f258af7368901e4550a2b9a6e285f06f25a8ab71464ec2", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-29T14:17:04.686176+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00672, "created_at": "2026-09-29T14:17:04.686Z" }, { "sequence": 60, "attempt_id": "5cdc70b9-b72e-4f73-8768-4bbec8024e3f", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-gapcompare-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.008375, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "A" }, "group_id": "gapcompare-first-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-first-dark-1.png", "input_tokens": 2567, "key_errors": null, "latency_sec": 2.9411765050608665, "model_name": "pareto", "output_tokens": 261, "prediction": { "choice": "A" }, "prompt_sha256": "ae03239d9758e90707007a297d168c1b204598f40d5440779fcfa12bd3cdb399", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-29T14:17:04.726710+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.008375, "created_at": "2026-09-29T14:17:04.726Z" }, { "sequence": 61, "attempt_id": "8f62052d-af50-46b4-a396-a307d509514c", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-textalign-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00866, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "A" }, "group_id": "textalign-left-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-left-dark-0.png", "input_tokens": 2597, "key_errors": null, "latency_sec": 3.220576724037528, "model_name": "pareto", "output_tokens": 289, "prediction": { "choice": "A" }, "prompt_sha256": "a46b47bd0291db684271663183741f72c6e90a5628697cdf5c907a9da6067b3d", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-29T14:17:04.900392+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00866, "created_at": "2026-09-29T14:17:04.900Z" }, { "sequence": 62, "attempt_id": "e2bdd2d0-d5cf-4dd7-ab33-815d000f22af", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-padcompare-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00763, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "B" }, "group_id": "padcompare-horizontal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-horizontal-dark-0.png", "input_tokens": 2593, "key_errors": null, "latency_sec": 1.9478382568340749, "model_name": "pareto", "output_tokens": 153, "prediction": { "choice": "B" }, "prompt_sha256": "4f484ecb32307572888516788b76d03dcfb86406579ae3871dbbdd5d50c24d37", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-29T14:17:05.055396+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00763, "created_at": "2026-09-29T14:17:05.055Z" }, { "sequence": 63, "attempt_id": "8e471dbc-e25e-4745-8b73-96e9f9efd8d9", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-groupgap-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.010425, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "C" }, "group_id": "groupgap-equal-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-equal-dark-1.png", "input_tokens": 2583, "key_errors": null, "latency_sec": 4.4971969809848815, "model_name": "pareto", "output_tokens": 529, "prediction": { "choice": "C" }, "prompt_sha256": "b5587c95ce8398dca0e206a8a222a0859cb2781184dde718bbe27409262b14c1", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-29T14:17:05.883589+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.010425, "created_at": "2026-09-29T14:17:05.883Z" }, { "sequence": 64, "attempt_id": "32f51013-8ff0-43f0-bd92-3d91d9c731f1", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-distribution-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00869, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "B" }, "group_id": "distribution-center-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-center-dark-0.png", "input_tokens": 2648, "key_errors": null, "latency_sec": 2.2318063939455897, "model_name": "pareto", "output_tokens": 276, "prediction": { "choice": "B" }, "prompt_sha256": "d3aa9dd8445fc739c14269d23f3f90c4093713cd7ec7f0fe052ca16e2f8ae3b4", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-29T14:17:06.546433+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00869, "created_at": "2026-09-29T14:17:06.546Z" }, { "sequence": 65, "attempt_id": "4f97502c-71da-4cbd-94e6-90302c42d96d", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-blockalign-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0073625, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "C" }, "group_id": "blockalign-right-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-right-dark-0.png", "input_tokens": 2576, "key_errors": null, "latency_sec": 1.8565552250947803, "model_name": "pareto", "output_tokens": 123, "prediction": { "choice": "C" }, "prompt_sha256": "19f470e5b91763a91fec11553496167c6525902526ca96fe05ea2d483d2acc16", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-29T14:17:06.562580+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0073625, "created_at": "2026-09-29T14:17:06.562Z" }, { "sequence": 66, "attempt_id": "87f311da-2424-416b-bc86-eed8844d7ee1", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-distribution-023", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.008705, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "A" }, "group_id": "distribution-evenly-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-evenly-dark-0.png", "input_tokens": 2648, "key_errors": null, "latency_sec": 2.261234342120588, "model_name": "pareto", "output_tokens": 278, "prediction": { "choice": "A" }, "prompt_sha256": "d8a96a07c7be49c0aea175aadd7302fe6c7f6601952712884e24cd44b6fec66a", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-29T14:17:07.001644+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-023", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.008705, "created_at": "2026-09-29T14:17:07.001Z" }, { "sequence": 67, "attempt_id": "f978068a-2e6b-4665-8782-07213be54b7e", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-gridcols-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0067775, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "C" }, "group_id": "gridcols-4-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-4-light-0.png", "input_tokens": 2549, "key_errors": null, "latency_sec": 2.3828110119793564, "model_name": "pareto", "output_tokens": 54, "prediction": { "choice": "C" }, "prompt_sha256": "baf29085a3966715a493e43b25eb411faa9ffe2033ec17653423007441d24033", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-29T14:17:07.297834+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0067775, "created_at": "2026-09-29T14:17:07.297Z" }, { "sequence": 68, "attempt_id": "7dd74269-8d4a-400c-b398-04db10015ca0", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-nestedflow-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0068875, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-08", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-08.png", "input_tokens": 2569, "key_errors": null, "latency_sec": 1.5578654820565134, "model_name": "pareto", "output_tokens": 62, "prediction": { "choice": "A" }, "prompt_sha256": "a555c9a37a814a2a2637ad2f6a1177955732dce4f82ecebc074c8930d36c5293", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-29T14:17:07.455986+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0068875, "created_at": "2026-09-29T14:17:07.456Z" }, { "sequence": 69, "attempt_id": "35067642-5d71-4763-92a8-8772ec6babee", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-textalign-016", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0095525, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "C" }, "group_id": "textalign-justify-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-justify-dark-1.png", "input_tokens": 2597, "key_errors": null, "latency_sec": 5.388723507057875, "model_name": "pareto", "output_tokens": 408, "prediction": { "choice": "C" }, "prompt_sha256": "2e47c2a304900033c1e7233c922b86d826e6949f29106d3fa5c967cc727df497", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-29T14:17:07.470976+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-016", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0095525, "created_at": "2026-09-29T14:17:07.472Z" }, { "sequence": 70, "attempt_id": "c1f50925-b4e1-4e12-9221-73c64c6ccbe6", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-crossalign-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0079325, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "A" }, "group_id": "crossalign-center-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-center-dark-1.png", "input_tokens": 2642, "key_errors": null, "latency_sec": 2.7289232739713043, "model_name": "pareto", "output_tokens": 177, "prediction": { "choice": "A" }, "prompt_sha256": "a0cc0893516679f3ab248b3cfb7b1e000bacbb17beb9b9bd9dcc6099c756222f", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-29T14:17:07.798335+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0079325, "created_at": "2026-09-29T14:17:07.798Z" }, { "sequence": 71, "attempt_id": "5ca334c3-91ea-4db0-92d1-74335ab056c1", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-nestedwrap-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0084275, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-12", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-12.png", "input_tokens": 2579, "key_errors": null, "latency_sec": 3.772160931956023, "model_name": "pareto", "output_tokens": 264, "prediction": { "choice": "A" }, "prompt_sha256": "3ba51907b89444fcea345fda34c9692faefd56e2b5c6671b282c0da644fdccb7", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-29T14:17:07.909530+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0084275, "created_at": "2026-09-29T14:17:07.909Z" }, { "sequence": 72, "attempt_id": "f5869d1f-aa70-484f-aea9-f8554c6da078", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-wrapcount-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0068525, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "B" }, "group_id": "wrapcount-3-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-3-light-0.png", "input_tokens": 2546, "key_errors": null, "latency_sec": 1.508646213915199, "model_name": "pareto", "output_tokens": 65, "prediction": { "choice": "B" }, "prompt_sha256": "a879d0677f7b7d6bd13a35743cb1a9461192ed1362a649bede5b7da8c73785da", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-29T14:17:08.071506+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0068525, "created_at": "2026-09-29T14:17:08.071Z" }, { "sequence": 73, "attempt_id": "22605bca-84db-4591-899c-d3cefe3c8cfb", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-gridrows-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0068, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "C" }, "group_id": "gridrows-2-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-2-dark-1.png", "input_tokens": 2549, "key_errors": null, "latency_sec": 1.6707531670108438, "model_name": "pareto", "output_tokens": 57, "prediction": { "choice": "C" }, "prompt_sha256": "bc0885c5d8e75e638e58b055b8a0c362e439458552c2f596d459edf6097e3ea3", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-29T14:17:08.247968+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0068, "created_at": "2026-09-29T14:17:08.248Z" }, { "sequence": 74, "attempt_id": "bcab263d-0d5a-4b0c-88c6-f3b32bbe839f", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-textalign-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0117575, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "A" }, "group_id": "textalign-right-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-right-dark-0.png", "input_tokens": 2597, "key_errors": null, "latency_sec": 6.189292537048459, "model_name": "pareto", "output_tokens": 702, "prediction": { "choice": "A" }, "prompt_sha256": "774f6052d034e009780326c2721c2a22fa373bdb75a2015c34e12e26984fdc20", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-29T14:17:08.495622+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0117575, "created_at": "2026-09-29T14:17:08.495Z" }, { "sequence": 75, "attempt_id": "ea03fc8a-2c1b-43df-8b5c-b5f454ba6375", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-gridrows-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0068225, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "A" }, "group_id": "gridcols-2-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-2-light-0.png", "input_tokens": 2549, "key_errors": null, "latency_sec": 1.5042487250175327, "model_name": "pareto", "output_tokens": 60, "prediction": { "choice": "A" }, "prompt_sha256": "0fad5d0a9261a17e68aa8de2ef5fb0207dc8af585c3baaeabfc584af8b5329f5", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-29T14:17:08.975540+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0068225, "created_at": "2026-09-29T14:17:08.975Z" }, { "sequence": 76, "attempt_id": "94022e64-81b8-4f15-82e2-b17f8a1fbdf7", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-gapcompare-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0074675, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "C" }, "group_id": "gapcompare-last-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-last-light-1.png", "input_tokens": 2567, "key_errors": null, "latency_sec": 1.9044157040771097, "model_name": "pareto", "output_tokens": 140, "prediction": { "choice": "C" }, "prompt_sha256": "bb41f3da30db1063257895b1a55fa7d2a40b0d638c3bd0111db12e799eb0d481", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-29T14:17:09.224657+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0074675, "created_at": "2026-09-29T14:17:09.224Z" }, { "sequence": 77, "attempt_id": "61f764d0-346d-45d3-9e05-896ac8a8532e", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-gridrows-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006725, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "B" }, "group_id": "gridrows-2-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-2-light-1.png", "input_tokens": 2549, "key_errors": null, "latency_sec": 1.4601874460931867, "model_name": "pareto", "output_tokens": 47, "prediction": { "choice": "B" }, "prompt_sha256": "fdee2ada4386cbfdbf34a82a0b96ff2b416af121ed3440edf26b052476c9811a", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-29T14:17:09.275521+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.006725, "created_at": "2026-09-29T14:17:09.275Z" }, { "sequence": 78, "attempt_id": "84105f2a-8762-4358-9098-114e54543c91", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-distribution-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009125, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "D" }, "group_id": "distribution-start-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-start-dark-1.png", "input_tokens": 2648, "key_errors": null, "latency_sec": 2.736703363014385, "model_name": "pareto", "output_tokens": 334, "prediction": { "choice": "D" }, "prompt_sha256": "3ad96dc5182ffe205957a476703a3fa61818fc870ed5be5256b09869b0bc7323", "provider": "anthropic", "raw_prediction": "{\"choice\": \"D\"}", "recorded_at": "2026-09-29T14:17:09.753176+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.009125, "created_at": "2026-09-29T14:17:09.753Z" }, { "sequence": 79, "attempt_id": "84212912-8a3d-4941-a57e-ffa9e44644e8", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-textalign-014", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.00764, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "C" }, "group_id": "textalign-justify-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-justify-light-1.png", "input_tokens": 2597, "key_errors": null, "latency_sec": 2.4986159331165254, "model_name": "pareto", "output_tokens": 153, "prediction": { "choice": "D" }, "prompt_sha256": "2e47c2a304900033c1e7233c922b86d826e6949f29106d3fa5c967cc727df497", "provider": "anthropic", "raw_prediction": "{\"choice\": \"D\"}", "recorded_at": "2026-09-29T14:17:09.985065+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-textalign-014", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00764, "created_at": "2026-09-29T14:17:09.985Z" }, { "sequence": 80, "attempt_id": "982862bc-2e0d-4fab-ae17-a8e2de0d40e8", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-padcompare-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0079825, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "A" }, "group_id": "padcompare-horizontal-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-horizontal-light-1.png", "input_tokens": 2593, "key_errors": null, "latency_sec": 2.0002577379345894, "model_name": "pareto", "output_tokens": 200, "prediction": { "choice": "A" }, "prompt_sha256": "08c2d8beb6c4a367fb4f9166e02fb0ce76d3bd79ca0c35a18250c8312e79a7ef", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-29T14:17:10.088326+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0079825, "created_at": "2026-09-29T14:17:10.088Z" }, { "sequence": 81, "attempt_id": "90fd2c31-dd18-4c3f-a97b-6236512f8cc8", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-blockalign-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0077375, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "A" }, "group_id": "blockalign-left-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-left-light-1.png", "input_tokens": 2576, "key_errors": null, "latency_sec": 2.3804736270103604, "model_name": "pareto", "output_tokens": 173, "prediction": { "choice": "A" }, "prompt_sha256": "e32486067ac25398fcca8ac9d957270d096fb6771f17cdb332d35edf376b0f86", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-29T14:17:10.305321+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0077375, "created_at": "2026-09-29T14:17:10.305Z" }, { "sequence": 82, "attempt_id": "d15abad6-5339-4d9a-bf5b-623211f13212", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-nestedflow-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0069925, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-03", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-03.png", "input_tokens": 2569, "key_errors": null, "latency_sec": 2.3197911200113595, "model_name": "pareto", "output_tokens": 76, "prediction": { "choice": "B" }, "prompt_sha256": "377bcd13e265380fd865ccbfd9819e64dfd2c4c3b2980e09c6e4d88924162f6c", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-29T14:17:10.584393+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0069925, "created_at": "2026-09-29T14:17:10.584Z" }, { "sequence": 83, "attempt_id": "b4912120-defd-4abc-b7eb-3827f5a84efb", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-gridtracks-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0075475, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "A" }, "group_id": "gridtracks-left-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-left-dark-1.png", "input_tokens": 2569, "key_errors": null, "latency_sec": 1.6432427940890193, "model_name": "pareto", "output_tokens": 150, "prediction": { "choice": "A" }, "prompt_sha256": "330f46df745e35a409a07fe87ab9184b25be7cbe0c4a58bdce6f2fce80123f3a", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-29T14:17:10.635563+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0075475, "created_at": "2026-09-29T14:17:10.635Z" }, { "sequence": 84, "attempt_id": "12a878e3-69fb-4d96-b2d2-8dd40979f1bc", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-gridrows-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00713, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "B" }, "group_id": "gridrows-4-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-4-light-1.png", "input_tokens": 2549, "key_errors": null, "latency_sec": 2.204713589977473, "model_name": "pareto", "output_tokens": 101, "prediction": { "choice": "B" }, "prompt_sha256": "8dbf44e12608a972028adfbb37ba6ec0f4b2e39e4ef1034ec56a4d2fe458ec94", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-29T14:17:10.717025+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00713, "created_at": "2026-09-29T14:17:10.717Z" }, { "sequence": 85, "attempt_id": "9b693261-801b-49af-ac9e-0aa9e5aefd15", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-direction-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0066825, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "B" }, "group_id": "direction-column-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-column-light-0.png", "input_tokens": 2547, "key_errors": null, "latency_sec": 2.0044628409668803, "model_name": "pareto", "output_tokens": 42, "prediction": { "choice": "B" }, "prompt_sha256": "84ba6fd0e7012086e1f258af7368901e4550a2b9a6e285f06f25a8ab71464ec2", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-29T14:17:11.245418+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0066825, "created_at": "2026-09-29T14:17:11.245Z" }, { "sequence": 86, "attempt_id": "90efea53-7895-4899-8bc9-c53af9754b98", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-gapcompare-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00746, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "C" }, "group_id": "gapcompare-last-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-last-dark-1.png", "input_tokens": 2567, "key_errors": null, "latency_sec": 2.2421872389968485, "model_name": "pareto", "output_tokens": 139, "prediction": { "choice": "C" }, "prompt_sha256": "7c67aaf8a81903fb008ead49ab0c337d815dd8e5b1f984eb1c2d9e3cddf9fd65", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-29T14:17:11.534418+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00746, "created_at": "2026-09-29T14:17:11.534Z" }, { "sequence": 87, "attempt_id": "286a93fc-4b4f-4be9-b5e4-300bb6a42d61", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-wrapcount-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006845, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "C" }, "group_id": "wrapcount-2-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-2-light-1.png", "input_tokens": 2546, "key_errors": null, "latency_sec": 1.5227400579024106, "model_name": "pareto", "output_tokens": 64, "prediction": { "choice": "C" }, "prompt_sha256": "e0ef52abca5ee7a42abae5e3fabccb2a95599a04ff3d41502af290e06d4711bd", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-29T14:17:11.844755+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.006845, "created_at": "2026-09-29T14:17:11.844Z" }, { "sequence": 88, "attempt_id": "8405e230-6488-434a-839b-551576feb0c8", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-padcompare-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0252475, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "C" }, "group_id": "padcompare-equal-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-equal-dark-1.png", "input_tokens": 2593, "key_errors": null, "latency_sec": 12.974288318073377, "model_name": "pareto", "output_tokens": 2502, "prediction": { "choice": "C" }, "prompt_sha256": "168f7fb4256da669d5b660759ed5f66d9bc1fe6fa5e0cc2f680f4674e926e39a", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-29T14:17:12.443829+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0252475, "created_at": "2026-09-29T14:17:12.443Z" }, { "sequence": 89, "attempt_id": "710ff21b-060d-4b4c-9ba9-498634f80ad6", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-wrapalign-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0086325, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "C" }, "group_id": "wrapalign-right-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-right-dark-0.png", "input_tokens": 2568, "key_errors": null, "latency_sec": 2.4915586579591036, "model_name": "pareto", "output_tokens": 295, "prediction": { "choice": "C" }, "prompt_sha256": "71d9f4106dcceb21abcbdbcb7978862ff824dca3aef4bc9d0cc134a9007d7be3", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-29T14:17:12.494397+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0086325, "created_at": "2026-09-29T14:17:12.494Z" }, { "sequence": 90, "attempt_id": "b8cc74b0-3c9e-4589-8997-fcfbcc7664ed", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-nestedflow-019", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.007105, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-19", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-19.png", "input_tokens": 2569, "key_errors": null, "latency_sec": 1.9600911207962781, "model_name": "pareto", "output_tokens": 91, "prediction": { "choice": "C" }, "prompt_sha256": "6cb509a4fb3ecab9a334fb061a4aa543245f6985aa65f331cb06646a50f35dfe", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-29T14:17:12.561896+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-019", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.007105, "created_at": "2026-09-29T14:17:12.562Z" }, { "sequence": 91, "attempt_id": "f02819b0-4e0c-412f-8721-732a2b93aa88", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-crossalign-015", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00797, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "D" }, "group_id": "crossalign-stretch-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-stretch-dark-0.png", "input_tokens": 2642, "key_errors": null, "latency_sec": 2.0068532151635736, "model_name": "pareto", "output_tokens": 182, "prediction": { "choice": "D" }, "prompt_sha256": "a0cc0893516679f3ab248b3cfb7b1e000bacbb17beb9b9bd9dcc6099c756222f", "provider": "anthropic", "raw_prediction": "{\"choice\": \"D\"}", "recorded_at": "2026-09-29T14:17:12.658501+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-015", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00797, "created_at": "2026-09-29T14:17:12.658Z" }, { "sequence": 92, "attempt_id": "a635abc3-f6c4-4874-8ede-dc4002eda393", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-textcolumns-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0067225, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "A" }, "group_id": "textcolumns-3-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-3-light-0.png", "input_tokens": 2545, "key_errors": null, "latency_sec": 1.9599044870119542, "model_name": "pareto", "output_tokens": 48, "prediction": { "choice": "A" }, "prompt_sha256": "7ab8a26bbcac7d24004e4a74f0ea411de080ccc6cfed0989403acf58e56e0339", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-29T14:17:12.693331+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0067225, "created_at": "2026-09-29T14:17:12.693Z" }, { "sequence": 93, "attempt_id": "68d80e0e-c3a2-4419-8a1d-820102df3f25", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-nestedflow-024", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0068725, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-24", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-24.png", "input_tokens": 2569, "key_errors": null, "latency_sec": 1.3849481861107051, "model_name": "pareto", "output_tokens": 60, "prediction": { "choice": "B" }, "prompt_sha256": "45c0efef1d7925275aea025bb1909f609ffc45577974778b58b34f1829f807e2", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-29T14:17:12.936970+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-024", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0068725, "created_at": "2026-09-29T14:17:12.937Z" }, { "sequence": 94, "attempt_id": "e2874513-555f-4c30-aac3-a1cdeeb389b8", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-nestedwrap-021", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0089, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-37", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-37.png", "input_tokens": 2579, "key_errors": null, "latency_sec": 2.863118896028027, "model_name": "pareto", "output_tokens": 327, "prediction": { "choice": "C" }, "prompt_sha256": "a242868807b1158889b152c7c9664c235ebd10352a9a4a541f0a0af30a7fb4a5", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-29T14:17:12.967663+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-021", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0089, "created_at": "2026-09-29T14:17:12.967Z" }, { "sequence": 95, "attempt_id": "93b8bf03-ac91-4d48-b6d0-7c5ac9de4730", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-textalign-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0105575, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "B" }, "group_id": "textalign-center-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-center-light-1.png", "input_tokens": 2597, "key_errors": null, "latency_sec": 3.240374278044328, "model_name": "pareto", "output_tokens": 542, "prediction": { "choice": "B" }, "prompt_sha256": "2c9a6eb7abbff326cf2a2f2c942bc4e9b7e820c30052fd8719ee75f2d63826cd", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-29T14:17:13.016712+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0105575, "created_at": "2026-09-29T14:17:13.016Z" }, { "sequence": 96, "attempt_id": "1d0dfea6-95ec-451b-a2a7-d0177f141992", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-gridrows-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006725, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "C" }, "group_id": "gridcols-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-dark-1.png", "input_tokens": 2549, "key_errors": null, "latency_sec": 1.8197080141399056, "model_name": "pareto", "output_tokens": 47, "prediction": { "choice": "C" }, "prompt_sha256": "da4194e14f8d3c86eb007c251e2f977baeaf16f3ed67610f233d5aaa1e16745d", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-29T14:17:13.083223+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.006725, "created_at": "2026-09-29T14:17:13.083Z" }, { "sequence": 97, "attempt_id": "c41b02e7-5bec-4ba5-b9aa-a4899622a48c", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-gridcols-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0068225, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "A" }, "group_id": "gridcols-2-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-2-light-0.png", "input_tokens": 2549, "key_errors": null, "latency_sec": 1.6934874539729208, "model_name": "pareto", "output_tokens": 60, "prediction": { "choice": "A" }, "prompt_sha256": "baf29085a3966715a493e43b25eb411faa9ffe2033ec17653423007441d24033", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-29T14:17:13.559852+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0068225, "created_at": "2026-09-29T14:17:13.560Z" }, { "sequence": 98, "attempt_id": "98e831a0-c31c-4361-a60a-be48044e9543", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-gridspan-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00694, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "B" }, "group_id": "gridspan-3-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-3-dark-0.png", "input_tokens": 2572, "key_errors": null, "latency_sec": 1.5883238189853728, "model_name": "pareto", "output_tokens": 68, "prediction": { "choice": "B" }, "prompt_sha256": "4c2619c1f3339203e066968a6911df07e41f49d553c6c3cb2be0b1e81e76a29b", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-29T14:17:14.265646+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00694, "created_at": "2026-09-29T14:17:14.265Z" }, { "sequence": 99, "attempt_id": "e1a8541e-7e45-4669-a0d9-a38a65a37ebb", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-nestedwrap-018", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0075575, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-34", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-34.png", "input_tokens": 2579, "key_errors": null, "latency_sec": 2.038081169128418, "model_name": "pareto", "output_tokens": 148, "prediction": { "choice": "A" }, "prompt_sha256": "1019a8d1887a379785b7f0f2096c38ba800c5c05f4005e0a2fc11959e378500f", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-29T14:17:14.749762+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-018", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0075575, "created_at": "2026-09-29T14:17:14.749Z" }, { "sequence": 100, "attempt_id": "2c2fec34-3218-4562-8d62-9a43ed8485fd", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-gridcols-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006875, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "A" }, "group_id": "gridcols-3-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-light-0.png", "input_tokens": 2549, "key_errors": null, "latency_sec": 1.871909359935671, "model_name": "pareto", "output_tokens": 67, "prediction": { "choice": "A" }, "prompt_sha256": "f1530d19c11e2af386da59f0ec25ff3124022397de4c3b03a6bc80ea51e1422e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-29T14:17:14.973763+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.006875, "created_at": "2026-09-29T14:17:14.973Z" }, { "sequence": 101, "attempt_id": "f790a5e0-27c0-44bf-ab08-5a4c8378c865", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-wrapcount-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0067925, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "A" }, "group_id": "wrapcount-1-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-1-dark-0.png", "input_tokens": 2546, "key_errors": null, "latency_sec": 1.5618619991000742, "model_name": "pareto", "output_tokens": 57, "prediction": { "choice": "A" }, "prompt_sha256": "d5a11da43aa9608fe90b20a74d3d14e9dbf3731c33be73ca93aad96c927ccc7d", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-29T14:17:15.140654+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0067925, "created_at": "2026-09-29T14:17:15.140Z" }, { "sequence": 102, "attempt_id": "18ae31d3-2e24-43c0-b01d-726de6e8e087", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-gridgaps-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0081125, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "C" }, "group_id": "gridgaps-vertical-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-vertical-light-1.png", "input_tokens": 2567, "key_errors": null, "latency_sec": 2.270721456967294, "model_name": "pareto", "output_tokens": 226, "prediction": { "choice": "C" }, "prompt_sha256": "e313bf537665702a78ba5d1571e6c2f2fbe0c9caf46ed390c4d20e71f215f3d1", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-29T14:17:15.256786+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0081125, "created_at": "2026-09-29T14:17:15.256Z" }, { "sequence": 103, "attempt_id": "fc98fd8f-e8c4-43ef-8ff7-3f9991d26ee0", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-gridtracks-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0077275, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "C" }, "group_id": "gridgaps-equal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-equal-light-0.png", "input_tokens": 2569, "key_errors": null, "latency_sec": 2.4002444290090352, "model_name": "pareto", "output_tokens": 174, "prediction": { "choice": "C" }, "prompt_sha256": "10cd7f5950372ee9cf1d2df732ca333086531870973115e171dc7abfa750b35c", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-29T14:17:15.435223+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0077275, "created_at": "2026-09-29T14:17:15.435Z" }, { "sequence": 104, "attempt_id": "28f02b1d-bf8d-45a5-a283-63caff4f1c39", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-nestedwrap-020", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.008405, "err": null, "error": "Expecting value: line 1 column 1 (char 0)", "error_kind": "invalid_response", "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-36", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-36.png", "input_tokens": 2579, "key_errors": null, "latency_sec": 3.0643772380426526, "model_name": "pareto", "output_tokens": 261, "prediction": {}, "prompt_sha256": "1019a8d1887a379785b7f0f2096c38ba800c5c05f4005e0a2fc11959e378500f", "provider": "anthropic", "raw_prediction": "A", "recorded_at": "2026-09-29T14:17:15.526968+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-nestedwrap-020", "tight_score": null, "unmetered_attempts": 0, "valid": false, "within_bands": null }, "cost_usd": 0.008405, "created_at": "2026-09-29T14:17:15.527Z" }, { "sequence": 105, "attempt_id": "74b9b61b-1c31-484d-9ae9-399dbbffced9", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-wrapalign-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0078675, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "A" }, "group_id": "wrapalign-left-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-left-dark-1.png", "input_tokens": 2568, "key_errors": null, "latency_sec": 2.5824706580024213, "model_name": "pareto", "output_tokens": 193, "prediction": { "choice": "A" }, "prompt_sha256": "4f5e9513c600f9a4c3e41e06f8b5947480ecacc47063f687d8ace9fa210713f1", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-29T14:17:15.545943+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0078675, "created_at": "2026-09-29T14:17:15.547Z" }, { "sequence": 106, "attempt_id": "4ac00e87-6d10-492b-b736-1c1ac43a48b8", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-distribution-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00896, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "E" }, "group_id": "distribution-end-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-end-light-1.png", "input_tokens": 2648, "key_errors": null, "latency_sec": 3.107987100025639, "model_name": "pareto", "output_tokens": 312, "prediction": { "choice": "E" }, "prompt_sha256": "aebd9c0440974ace931268cc69fd04b3961d3ac9b3895fb8243ac6e96f4cc134", "provider": "anthropic", "raw_prediction": "{\"choice\": \"E\"}", "recorded_at": "2026-09-29T14:17:15.688043+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00896, "created_at": "2026-09-29T14:17:15.688Z" }, { "sequence": 107, "attempt_id": "df954a50-52e2-4ca9-962f-3c508af5e408", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-wrapcount-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006755, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "A" }, "group_id": "wrapcount-1-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-1-light-0.png", "input_tokens": 2546, "key_errors": null, "latency_sec": 1.5577019138727337, "model_name": "pareto", "output_tokens": 52, "prediction": { "choice": "A" }, "prompt_sha256": "15d8bd94924cd0eb8efdec17da21c6da4f97b10dd9a51e2d28e533f308413109", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-29T14:17:16.550247+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.006755, "created_at": "2026-09-29T14:17:16.550Z" }, { "sequence": 108, "attempt_id": "afe54140-50c3-4122-938a-6e969e04ec1f", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-gridcols-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0068975, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "B" }, "group_id": "gridcols-4-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-4-dark-0.png", "input_tokens": 2549, "key_errors": null, "latency_sec": 1.6350384878460318, "model_name": "pareto", "output_tokens": 70, "prediction": { "choice": "B" }, "prompt_sha256": "f456b84b6ca321b64c95c2bb2cfe766f21207eb4901be3ddea7233116bb62356", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-29T14:17:17.200355+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0068975, "created_at": "2026-09-29T14:17:17.200Z" }, { "sequence": 109, "attempt_id": "7c68341c-3fae-41cf-8c02-a491ddad81d4", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-nestedwrap-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.007535, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-27", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-27.png", "input_tokens": 2579, "key_errors": null, "latency_sec": 2.5154739229474217, "model_name": "pareto", "output_tokens": 145, "prediction": { "choice": "A" }, "prompt_sha256": "dbe3324244c1acc8302aa7a2f89b338ba07244a1490efb8ddaf45fc8d47a978d", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-29T14:17:17.791048+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.007535, "created_at": "2026-09-29T14:17:17.791Z" }, { "sequence": 110, "attempt_id": "4e40f8a6-5a35-4cf8-b539-12001d3e6214", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-nestedwrap-015", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.008, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-31", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-31.png", "input_tokens": 2579, "key_errors": null, "latency_sec": 2.2591891877818853, "model_name": "pareto", "output_tokens": 207, "prediction": { "choice": "B" }, "prompt_sha256": "3ba51907b89444fcea345fda34c9692faefd56e2b5c6671b282c0da644fdccb7", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-29T14:17:17.966301+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-015", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.008, "created_at": "2026-09-29T14:17:17.966Z" }, { "sequence": 111, "attempt_id": "eda39cc8-3e81-4139-93c4-bc0ae918da8e", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-distribution-024", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009545, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "C" }, "group_id": "distribution-evenly-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-evenly-dark-1.png", "input_tokens": 2648, "key_errors": null, "latency_sec": 3.7027019588276744, "model_name": "pareto", "output_tokens": 390, "prediction": { "choice": "C" }, "prompt_sha256": "01948a6a96d60ae4f02c8775cea12be647823408f239745e622e7c22d625ffc8", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-29T14:17:17.987500+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-024", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.009545, "created_at": "2026-09-29T14:17:17.987Z" }, { "sequence": 112, "attempt_id": "780c4750-a607-4d5f-b081-3a5f805f422b", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-nestedwrap-023", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0080225, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-39", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-39.png", "input_tokens": 2579, "key_errors": null, "latency_sec": 2.4959922039415687, "model_name": "pareto", "output_tokens": 210, "prediction": { "choice": "B" }, "prompt_sha256": "1019a8d1887a379785b7f0f2096c38ba800c5c05f4005e0a2fc11959e378500f", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-29T14:17:18.042177+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-023", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0080225, "created_at": "2026-09-29T14:17:18.042Z" }, { "sequence": 113, "attempt_id": "66bd6fba-c8e7-4264-9d13-a0b914c6bc32", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-direction-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0066975, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "B" }, "group_id": "direction-row-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-row-light-0.png", "input_tokens": 2547, "key_errors": null, "latency_sec": 1.484726507915184, "model_name": "pareto", "output_tokens": 44, "prediction": { "choice": "B" }, "prompt_sha256": "3768f3feb47a0750f78e6a2c372e707b1b87f7af93fd1fa6b13ef50f1092f226", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-29T14:17:18.061030+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0066975, "created_at": "2026-09-29T14:17:18.061Z" }, { "sequence": 114, "attempt_id": "5fdef264-2a13-4bfb-b50b-63d684bad01d", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-gapcompare-005", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.010805, "err": null, "error": "Expecting value: line 1 column 1 (char 0)", "error_kind": "invalid_response", "family": "gapcompare", "ground_truth": { "choice": "A" }, "group_id": "gapcompare-last-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-last-light-0.png", "input_tokens": 2567, "key_errors": null, "latency_sec": 3.720825847936794, "model_name": "pareto", "output_tokens": 585, "prediction": {}, "prompt_sha256": "ab1cd666bf62495c439cacec79574bb2f8ac552c98f8627324d3801b52d6d81b", "provider": "anthropic", "raw_prediction": "A", "recorded_at": "2026-09-29T14:17:18.489528+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-gapcompare-005", "tight_score": null, "unmetered_attempts": 0, "valid": false, "within_bands": null }, "cost_usd": 0.010805, "created_at": "2026-09-29T14:17:18.489Z" }, { "sequence": 115, "attempt_id": "11698333-4896-4799-a8ec-be2228feabc8", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-groupgap-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01122, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "A" }, "group_id": "groupgap-within-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-within-light-1.png", "input_tokens": 2583, "key_errors": null, "latency_sec": 4.048435197910294, "model_name": "pareto", "output_tokens": 635, "prediction": { "choice": "A" }, "prompt_sha256": "b5587c95ce8398dca0e206a8a222a0859cb2781184dde718bbe27409262b14c1", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-29T14:17:19.207450+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.01122, "created_at": "2026-09-29T14:17:19.207Z" }, { "sequence": 116, "attempt_id": "f523a694-1718-4d96-9131-598a26201e1c", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-gridcols-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0067025, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "A" }, "group_id": "gridcols-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-dark-1.png", "input_tokens": 2549, "key_errors": null, "latency_sec": 1.4715389341581613, "model_name": "pareto", "output_tokens": 44, "prediction": { "choice": "A" }, "prompt_sha256": "f1530d19c11e2af386da59f0ec25ff3124022397de4c3b03a6bc80ea51e1422e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-29T14:17:19.532943+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0067025, "created_at": "2026-09-29T14:17:19.533Z" }, { "sequence": 117, "attempt_id": "952bf921-2e0e-4372-b3ef-4d54d5ec2a05", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-gridrows-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0070025, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "A" }, "group_id": "gridrows-4-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-4-dark-0.png", "input_tokens": 2549, "key_errors": null, "latency_sec": 1.7226293310523033, "model_name": "pareto", "output_tokens": 84, "prediction": { "choice": "A" }, "prompt_sha256": "2c2e1ea513c52f4b927e59a676294b441e8aa0cf98f9851e570c4ad2bd9396af", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-29T14:17:19.708670+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0070025, "created_at": "2026-09-29T14:17:19.708Z" }, { "sequence": 118, "attempt_id": "3c41fbf9-52a2-497d-950a-3f3c20e59f57", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-padcompare-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.007585, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "B" }, "group_id": "padcompare-vertical-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-vertical-dark-1.png", "input_tokens": 2593, "key_errors": null, "latency_sec": 2.5182823319919407, "model_name": "pareto", "output_tokens": 147, "prediction": { "choice": "B" }, "prompt_sha256": "168f7fb4256da669d5b660759ed5f66d9bc1fe6fa5e0cc2f680f4674e926e39a", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-29T14:17:19.738212+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.007585, "created_at": "2026-09-29T14:17:19.738Z" }, { "sequence": 119, "attempt_id": "991609b1-4f4f-4585-81b1-4490919448ee", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-nestedflow-022", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006835, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-22", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-22.png", "input_tokens": 2569, "key_errors": null, "latency_sec": 1.5995723251253366, "model_name": "pareto", "output_tokens": 55, "prediction": { "choice": "A" }, "prompt_sha256": "ac0c30958153584a4b94fe042ea1cbf718f020f9f66dd474a09c514effd2f9f9", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-29T14:17:20.108681+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-022", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.006835, "created_at": "2026-09-29T14:17:20.108Z" }, { "sequence": 120, "attempt_id": "b7e305ea-3246-4d1d-89ba-50a86f18aaa3", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-nestedwrap-024", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.00836, "err": null, "error": "Expecting value: line 1 column 1 (char 0)", "error_kind": "invalid_response", "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-40", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-40.png", "input_tokens": 2579, "key_errors": null, "latency_sec": 2.361248614033684, "model_name": "pareto", "output_tokens": 255, "prediction": {}, "prompt_sha256": "dbe3324244c1acc8302aa7a2f89b338ba07244a1490efb8ddaf45fc8d47a978d", "provider": "anthropic", "raw_prediction": "C", "recorded_at": "2026-09-29T14:17:20.172425+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-nestedwrap-024", "tight_score": null, "unmetered_attempts": 0, "valid": false, "within_bands": null }, "cost_usd": 0.00836, "created_at": "2026-09-29T14:17:20.172Z" }, { "sequence": 121, "attempt_id": "bfc714d3-4d20-4407-bda7-cc4b56193916", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-crossalign-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0079625, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "B" }, "group_id": "crossalign-start-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-start-light-0.png", "input_tokens": 2642, "key_errors": null, "latency_sec": 2.185381606919691, "model_name": "pareto", "output_tokens": 181, "prediction": { "choice": "B" }, "prompt_sha256": "0f1e528120c9711dd50b5a3ee0dd5163897f25273cfd6b0ec66485192ac32e79", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-29T14:17:20.191942+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0079625, "created_at": "2026-09-29T14:17:20.192Z" }, { "sequence": 122, "attempt_id": "53fbfa53-97e3-404a-ae0d-c9f76107215a", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-blockalign-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0073175, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "B" }, "group_id": "blockalign-left-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-left-dark-0.png", "input_tokens": 2576, "key_errors": null, "latency_sec": 2.1267722349148244, "model_name": "pareto", "output_tokens": 117, "prediction": { "choice": "B" }, "prompt_sha256": "7d6254d951604c657e71aeb7cae9178b26bc4d3dd7020cdf265225567557e5dd", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-29T14:17:21.354172+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0073175, "created_at": "2026-09-29T14:17:21.354Z" }, { "sequence": 123, "attempt_id": "bed63d97-f9ce-475a-b0dc-ac0a5aa2191d", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-gridspan-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00709, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "C" }, "group_id": "gridspan-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-3-dark-1.png", "input_tokens": 2572, "key_errors": null, "latency_sec": 1.6518914070911705, "model_name": "pareto", "output_tokens": 88, "prediction": { "choice": "C" }, "prompt_sha256": "8f345bb2bced18424e3b0095ebe76c3e49a3f618e53f0a1cf0fe32cf1f45be8c", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-29T14:17:21.379879+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00709, "created_at": "2026-09-29T14:17:21.379Z" }, { "sequence": 124, "attempt_id": "addf450f-34df-4071-ad3e-959cfecfc7ef", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-gridgaps-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00773, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "B" }, "group_id": "gridgaps-horizontal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-horizontal-dark-0.png", "input_tokens": 2567, "key_errors": null, "latency_sec": 2.202348052058369, "model_name": "pareto", "output_tokens": 175, "prediction": { "choice": "B" }, "prompt_sha256": "bb599c345aa6aaf2e8a00a558db18c18ea46033edf36cd1b879c2785b58dce8b", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-29T14:17:21.755008+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00773, "created_at": "2026-09-29T14:17:21.755Z" }, { "sequence": 125, "attempt_id": "909499a8-faf6-4267-8eb4-0d9aa4ed4cb4", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-distribution-019", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0103325, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "D" }, "group_id": "distribution-around-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-around-dark-0.png", "input_tokens": 2648, "key_errors": null, "latency_sec": 3.7252997681498528, "model_name": "pareto", "output_tokens": 495, "prediction": { "choice": "D" }, "prompt_sha256": "4dc80907c36a071b7b8d24fb1b232e7423f18ee5163be9b22e0a35c4d658cc94", "provider": "anthropic", "raw_prediction": "{\"choice\": \"D\"}", "recorded_at": "2026-09-29T14:17:21.805888+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-019", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0103325, "created_at": "2026-09-29T14:17:21.806Z" }, { "sequence": 126, "attempt_id": "762788f4-46c0-4804-82cc-29636b49f4fc", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-textcolumns-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006595, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "B" }, "group_id": "textcolumns-2-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-2-light-1.png", "input_tokens": 2545, "key_errors": null, "latency_sec": 1.5949952921364456, "model_name": "pareto", "output_tokens": 31, "prediction": { "choice": "B" }, "prompt_sha256": "7ab8a26bbcac7d24004e4a74f0ea411de080ccc6cfed0989403acf58e56e0339", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-29T14:17:21.825276+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.006595, "created_at": "2026-09-29T14:17:21.826Z" }, { "sequence": 127, "attempt_id": "0abefac1-a6b3-40f2-a4f3-acfa120c500d", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-nestedflow-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0076, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-06", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-06.png", "input_tokens": 2569, "key_errors": null, "latency_sec": 2.286201292881742, "model_name": "pareto", "output_tokens": 157, "prediction": { "choice": "B" }, "prompt_sha256": "4cd5f1bb86e36238445efff79cfd13b01ed6916cd094da4a341b7112a2f75bfe", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-29T14:17:22.043491+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0076, "created_at": "2026-09-29T14:17:22.043Z" }, { "sequence": 128, "attempt_id": "ea0254f6-5636-4017-b504-f99170169cef", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-gridtracks-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00844, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "C" }, "group_id": "gridgaps-equal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-equal-dark-0.png", "input_tokens": 2569, "key_errors": null, "latency_sec": 2.1394046710338444, "model_name": "pareto", "output_tokens": 269, "prediction": { "choice": "C" }, "prompt_sha256": "970b359b82046186b75b0e492f6832a2ce3e8a83fea0f87acd3cecf2c795cb79", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-29T14:17:22.331655+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00844, "created_at": "2026-09-29T14:17:22.331Z" }, { "sequence": 129, "attempt_id": "3cb83e59-59e5-4bf5-86be-5656bcb4de73", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-distribution-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0157175, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "A" }, "group_id": "distribution-end-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-end-light-0.png", "input_tokens": 2648, "key_errors": null, "latency_sec": 6.891848739003763, "model_name": "pareto", "output_tokens": 1213, "prediction": { "choice": "A" }, "prompt_sha256": "17648faa8c38d72326eb56153ca43b16f1a49276e5acb8d5110bb4bb0e9299d1", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-29T14:17:22.352010+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0157175, "created_at": "2026-09-29T14:17:22.352Z" }, { "sequence": 130, "attempt_id": "fec7ceb1-0412-47ab-a77a-53f366fb636b", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-wrapcount-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0068975, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "C" }, "group_id": "wrapcount-2-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-2-light-0.png", "input_tokens": 2546, "key_errors": null, "latency_sec": 1.5242184379603714, "model_name": "pareto", "output_tokens": 71, "prediction": { "choice": "C" }, "prompt_sha256": "e0ef52abca5ee7a42abae5e3fabccb2a95599a04ff3d41502af290e06d4711bd", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-29T14:17:23.369649+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0068975, "created_at": "2026-09-29T14:17:23.369Z" }, { "sequence": 131, "attempt_id": "d8115716-10ae-4f2d-84d4-55b8fee5f3a2", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-blockalign-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0077375, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "C" }, "group_id": "blockalign-center-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-center-light-1.png", "input_tokens": 2576, "key_errors": null, "latency_sec": 2.035596271045506, "model_name": "pareto", "output_tokens": 173, "prediction": { "choice": "C" }, "prompt_sha256": "7d6254d951604c657e71aeb7cae9178b26bc4d3dd7020cdf265225567557e5dd", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-29T14:17:23.811015+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0077375, "created_at": "2026-09-29T14:17:23.811Z" }, { "sequence": 132, "attempt_id": "529eecb0-1359-4ec3-8dc1-4926f58102f2", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-padcompare-009", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.0084775, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "B" }, "group_id": "padcompare-equal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-equal-light-0.png", "input_tokens": 2593, "key_errors": null, "latency_sec": 2.5328552729915828, "model_name": "pareto", "output_tokens": 266, "prediction": { "choice": "A" }, "prompt_sha256": "0abb90b71247ce9a0876c99ad3ef52ea5a9ae09bfffb2875eec73b10166640c1", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-29T14:17:23.907648+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-padcompare-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0084775, "created_at": "2026-09-29T14:17:23.907Z" }, { "sequence": 133, "attempt_id": "b45261b6-2ada-4a98-ab10-cd0b22057578", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-direction-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00669, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "B" }, "group_id": "direction-row-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-row-dark-0.png", "input_tokens": 2547, "key_errors": null, "latency_sec": 1.642727104946971, "model_name": "pareto", "output_tokens": 43, "prediction": { "choice": "B" }, "prompt_sha256": "3768f3feb47a0750f78e6a2c372e707b1b87f7af93fd1fa6b13ef50f1092f226", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-29T14:17:25.033824+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00669, "created_at": "2026-09-29T14:17:25.033Z" }, { "sequence": 134, "attempt_id": "b5b78109-da54-405b-bac8-c464616dabc5", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-distribution-014", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.011585, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "A" }, "group_id": "distribution-between-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-between-light-1.png", "input_tokens": 2648, "key_errors": null, "latency_sec": 3.7238588260952383, "model_name": "pareto", "output_tokens": 662, "prediction": { "choice": "A" }, "prompt_sha256": "a69521ff42316a2cf6dd875d17ed524c23ebb9d37a359ef46a9898ae2d02aae0", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-29T14:17:25.122323+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-014", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.011585, "created_at": "2026-09-29T14:17:25.122Z" }, { "sequence": 135, "attempt_id": "772b0797-9eb5-463a-9f71-5071a2970af0", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-nestedwrap-017", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.008, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-33", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-33.png", "input_tokens": 2579, "key_errors": null, "latency_sec": 2.8197626220062375, "model_name": "pareto", "output_tokens": 207, "prediction": { "choice": "A" }, "prompt_sha256": "1019a8d1887a379785b7f0f2096c38ba800c5c05f4005e0a2fc11959e378500f", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-29T14:17:25.192914+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-017", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.008, "created_at": "2026-09-29T14:17:25.193Z" }, { "sequence": 136, "attempt_id": "57abb89b-b739-4592-8304-ebcc27a26fb5", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-nestedflow-016", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.007225, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-16", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-16.png", "input_tokens": 2569, "key_errors": null, "latency_sec": 1.5939314209390432, "model_name": "pareto", "output_tokens": 107, "prediction": { "choice": "B" }, "prompt_sha256": "377bcd13e265380fd865ccbfd9819e64dfd2c4c3b2980e09c6e4d88924162f6c", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-29T14:17:25.426247+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-016", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.007225, "created_at": "2026-09-29T14:17:25.426Z" }, { "sequence": 137, "attempt_id": "87e74379-bd0e-40fe-87b2-3352cbc768d8", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-nestedwrap-022", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.010955, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-38", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-38.png", "input_tokens": 2579, "key_errors": null, "latency_sec": 3.6633410227950662, "model_name": "pareto", "output_tokens": 601, "prediction": { "choice": "B" }, "prompt_sha256": "3ba51907b89444fcea345fda34c9692faefd56e2b5c6671b282c0da644fdccb7", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-29T14:17:25.488869+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-022", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.010955, "created_at": "2026-09-29T14:17:25.488Z" }, { "sequence": 138, "attempt_id": "028bec69-346f-42ef-8ea2-f5519ced8217", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-topflow-014", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00688, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-14", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-14.png", "input_tokens": 2566, "key_errors": null, "latency_sec": 1.6908708759583533, "model_name": "pareto", "output_tokens": 62, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-29T14:17:25.619836+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-014", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00688, "created_at": "2026-09-29T14:17:25.619Z" }, { "sequence": 139, "attempt_id": "c93f5c1c-ef27-453d-9d0a-322790846d00", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-textalign-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0107375, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "B" }, "group_id": "textalign-center-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-center-dark-0.png", "input_tokens": 2597, "key_errors": null, "latency_sec": 4.221947029000148, "model_name": "pareto", "output_tokens": 566, "prediction": { "choice": "B" }, "prompt_sha256": "9a155e0b3c7bbbbff68050bc1a1e72dea59500ab480af46a3cd3b87e25de70bf", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-29T14:17:26.574256+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0107375, "created_at": "2026-09-29T14:17:26.574Z" }, { "sequence": 140, "attempt_id": "e4174119-5ef8-4ec5-addf-5fd515a10fbd", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-wrapcount-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0068525, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "C" }, "group_id": "wrapcount-3-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-3-dark-0.png", "input_tokens": 2546, "key_errors": null, "latency_sec": 1.5159034670796245, "model_name": "pareto", "output_tokens": 65, "prediction": { "choice": "C" }, "prompt_sha256": "d5a11da43aa9608fe90b20a74d3d14e9dbf3731c33be73ca93aad96c927ccc7d", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-29T14:17:27.025873+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0068525, "created_at": "2026-09-29T14:17:27.026Z" }, { "sequence": 141, "attempt_id": "d10346bd-ce88-4114-a9de-58e5ea0b7e1a", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-crossalign-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0077825, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "D" }, "group_id": "crossalign-center-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-center-dark-0.png", "input_tokens": 2642, "key_errors": null, "latency_sec": 2.470191791187972, "model_name": "pareto", "output_tokens": 157, "prediction": { "choice": "D" }, "prompt_sha256": "e6e3e7be0a2b7e91666344c6f0c3a0f931a9ea63d47c2dbd9e1c4cb334420e83", "provider": "anthropic", "raw_prediction": "{\"choice\": \"D\"}", "recorded_at": "2026-09-29T14:17:27.613320+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0077825, "created_at": "2026-09-29T14:17:27.613Z" }, { "sequence": 142, "attempt_id": "56ee236f-0886-4d1f-ab23-5ba3d45cac24", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-groupgap-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009345, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "C" }, "group_id": "groupgap-within-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-within-light-0.png", "input_tokens": 2583, "key_errors": null, "latency_sec": 2.4534390489570796, "model_name": "pareto", "output_tokens": 385, "prediction": { "choice": "C" }, "prompt_sha256": "6cba71084ece6f53e834e34e24c720eaabde46f305bb7ec0454830a5812e840e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-29T14:17:27.666512+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.009345, "created_at": "2026-09-29T14:17:27.666Z" }, { "sequence": 143, "attempt_id": "bec45785-16d0-4818-8b15-6058855cfc7c", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-blockalign-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0078275, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "A" }, "group_id": "blockalign-center-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-center-dark-0.png", "input_tokens": 2576, "key_errors": null, "latency_sec": 2.6830995879136026, "model_name": "pareto", "output_tokens": 185, "prediction": { "choice": "A" }, "prompt_sha256": "19f470e5b91763a91fec11553496167c6525902526ca96fe05ea2d483d2acc16", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-29T14:17:27.738589+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0078275, "created_at": "2026-09-29T14:17:27.738Z" }, { "sequence": 144, "attempt_id": "ff10ef06-3ce9-4312-a6f6-4320411298ef", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-wrapalign-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0084675, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "A" }, "group_id": "wrapalign-left-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-left-dark-0.png", "input_tokens": 2568, "key_errors": null, "latency_sec": 2.4887428351212293, "model_name": "pareto", "output_tokens": 273, "prediction": { "choice": "A" }, "prompt_sha256": "71d9f4106dcceb21abcbdbcb7978862ff824dca3aef4bc9d0cc134a9007d7be3", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-29T14:17:28.128891+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0084675, "created_at": "2026-09-29T14:17:28.129Z" }, { "sequence": 145, "attempt_id": "286810cd-7b60-4f81-9a07-000b0567e5bb", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-blockalign-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009065, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "B" }, "group_id": "blockalign-right-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-right-dark-1.png", "input_tokens": 2576, "key_errors": null, "latency_sec": 2.703711590031162, "model_name": "pareto", "output_tokens": 350, "prediction": { "choice": "B" }, "prompt_sha256": "aec8632f26486517e0ed07640068efdb11c079f166b430d6a2eb3e7ecb5c47b1", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-29T14:17:28.151227+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.009065, "created_at": "2026-09-29T14:17:28.151Z" }, { "sequence": 146, "attempt_id": "c7046431-bced-461f-ada0-ea7a5c637954", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-crossalign-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0085775, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "D" }, "group_id": "crossalign-start-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-start-dark-1.png", "input_tokens": 2642, "key_errors": null, "latency_sec": 2.161463828990236, "model_name": "pareto", "output_tokens": 263, "prediction": { "choice": "D" }, "prompt_sha256": "a4e6df1b241ba522334256d095328511f40be1d02d97470c916d2ec188ae062f", "provider": "anthropic", "raw_prediction": "{\"choice\": \"D\"}", "recorded_at": "2026-09-29T14:17:29.209588+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0085775, "created_at": "2026-09-29T14:17:29.209Z" }, { "sequence": 147, "attempt_id": "0fd1f9c3-c862-4ad2-987f-e06e46b88f94", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-gridspan-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0075625, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "A" }, "group_id": "gridspan-1-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-1-light-1.png", "input_tokens": 2572, "key_errors": null, "latency_sec": 1.9681051161605865, "model_name": "pareto", "output_tokens": 151, "prediction": { "choice": "A" }, "prompt_sha256": "9320fc919a5f33d10a5a97b8269a21a6fc8fa9787daa8025e876c4a4452a6d4a", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-29T14:17:29.605559+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0075625, "created_at": "2026-09-29T14:17:29.605Z" }, { "sequence": 148, "attempt_id": "79b30b60-82bd-478b-944f-9b6cf3209e84", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-nestedflow-017", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006895, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-17", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-17.png", "input_tokens": 2569, "key_errors": null, "latency_sec": 2.178813843987882, "model_name": "pareto", "output_tokens": 63, "prediction": { "choice": "A" }, "prompt_sha256": "377bcd13e265380fd865ccbfd9819e64dfd2c4c3b2980e09c6e4d88924162f6c", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-29T14:17:29.864761+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-017", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.006895, "created_at": "2026-09-29T14:17:29.864Z" }, { "sequence": 149, "attempt_id": "52f9ca17-923c-4afe-9c55-11c9b3d97688", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-topflow-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0067, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-09", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-09.png", "input_tokens": 2566, "key_errors": null, "latency_sec": 1.7413938730023801, "model_name": "pareto", "output_tokens": 38, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-29T14:17:29.912096+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0067, "created_at": "2026-09-29T14:17:29.912Z" }, { "sequence": 150, "attempt_id": "4927f6b7-a624-44fe-9438-af50490c8a20", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-crossalign-014", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00956, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "B" }, "group_id": "crossalign-stretch-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-stretch-light-1.png", "input_tokens": 2642, "key_errors": null, "latency_sec": 3.3831742801703513, "model_name": "pareto", "output_tokens": 394, "prediction": { "choice": "B" }, "prompt_sha256": "ea02b690a39fed2c99e991ca10571f8f5e75cfea4e2db5349dafc50758191f40", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-29T14:17:29.979337+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-014", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00956, "created_at": "2026-09-29T14:17:29.979Z" }, { "sequence": 151, "attempt_id": "4f1ffc2f-3302-4b41-8b8a-4ce566e468bd", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-gridgaps-010", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.0437525, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "A" }, "group_id": "gridcols-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-light-1.png", "input_tokens": 5192, "key_errors": null, "latency_sec": 17.980661551002413, "model_name": "pareto", "output_tokens": 4103, "prediction": { "choice": "B" }, "prompt_sha256": "bb599c345aa6aaf2e8a00a558db18c18ea46033edf36cd1b879c2785b58dce8b", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-29T14:17:30.496232+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-gridgaps-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0437525, "created_at": "2026-09-29T14:17:30.496Z" }, { "sequence": 152, "attempt_id": "516abbde-6e0f-4642-82e4-01d845ea1b23", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-gridgaps-012", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.0097325, "err": null, "error": "Expecting value: line 1 column 1 (char 0)", "error_kind": "invalid_response", "family": "gridgaps", "ground_truth": { "choice": "C" }, "group_id": "gridcols-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-dark-1.png", "input_tokens": 2567, "key_errors": null, "latency_sec": 2.967900612158701, "model_name": "pareto", "output_tokens": 442, "prediction": {}, "prompt_sha256": "b680534db587454f307f9837d6da890e5af0576bf084907b76dd89e0815cc97c", "provider": "anthropic", "raw_prediction": "C", "recorded_at": "2026-09-29T14:17:31.117192+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-gridgaps-012", "tight_score": null, "unmetered_attempts": 0, "valid": false, "within_bands": null }, "cost_usd": 0.0097325, "created_at": "2026-09-29T14:17:31.117Z" }, { "sequence": 153, "attempt_id": "da80b19f-b36d-4cac-b079-6aa3cdb3e1a5", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-distribution-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0116375, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "D" }, "group_id": "distribution-center-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-center-dark-1.png", "input_tokens": 2648, "key_errors": null, "latency_sec": 3.7103604848962277, "model_name": "pareto", "output_tokens": 669, "prediction": { "choice": "D" }, "prompt_sha256": "27147e1f86f207706e058c66a175f2db6da63ea3c8dfd58dc353d60bc9922569", "provider": "anthropic", "raw_prediction": "{\"choice\": \"D\"}", "recorded_at": "2026-09-29T14:17:31.467954+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0116375, "created_at": "2026-09-29T14:17:31.468Z" }, { "sequence": 154, "attempt_id": "a6679712-2b5d-492c-8d07-abd41a2c55b8", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-topflow-018", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0067675, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-18", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-18.png", "input_tokens": 2566, "key_errors": null, "latency_sec": 1.6873109070584178, "model_name": "pareto", "output_tokens": 47, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-29T14:17:31.619401+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-018", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0067675, "created_at": "2026-09-29T14:17:31.619Z" }, { "sequence": 155, "attempt_id": "678cf65a-4664-4de4-a59e-c9b625f801ca", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-gridgaps-011", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.020945, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "B" }, "group_id": "gridgaps-equal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-equal-dark-0.png", "input_tokens": 2567, "key_errors": null, "latency_sec": 9.572733517037705, "model_name": "pareto", "output_tokens": 1937, "prediction": { "choice": "C" }, "prompt_sha256": "e313bf537665702a78ba5d1571e6c2f2fbe0c9caf46ed390c4d20e71f215f3d1", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-29T14:17:31.641538+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-gridgaps-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.020945, "created_at": "2026-09-29T14:17:31.642Z" }, { "sequence": 156, "attempt_id": "b7d77d05-ccd8-4f84-a6ed-6a0bdb383274", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-blockalign-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.007535, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "B" }, "group_id": "blockalign-center-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-center-light-0.png", "input_tokens": 2576, "key_errors": null, "latency_sec": 2.036298847058788, "model_name": "pareto", "output_tokens": 146, "prediction": { "choice": "B" }, "prompt_sha256": "376fb84b749ec9e6ed85a09a2f7a073f4bb3962b1c113f20e7e3bcf9e146f18b", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-29T14:17:31.922134+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.007535, "created_at": "2026-09-29T14:17:31.922Z" }, { "sequence": 157, "attempt_id": "2386aaf2-4c54-46c4-afab-db898b3bbac3", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-gapcompare-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0088025, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "A" }, "group_id": "gapcompare-equal-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-equal-light-1.png", "input_tokens": 2567, "key_errors": null, "latency_sec": 2.3181441437918693, "model_name": "pareto", "output_tokens": 318, "prediction": { "choice": "A" }, "prompt_sha256": "2aacf3dbef42dbefa25b359df3a8af3c1541ab2712c25e9b3c3f5cf8371b79e4", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-29T14:17:31.944578+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0088025, "created_at": "2026-09-29T14:17:31.944Z" }, { "sequence": 158, "attempt_id": "062bf106-e7b6-4420-baef-e82c97a2599e", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-nestedwrap-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0076475, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-11", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-11.png", "input_tokens": 2579, "key_errors": null, "latency_sec": 3.329857981065288, "model_name": "pareto", "output_tokens": 160, "prediction": { "choice": "C" }, "prompt_sha256": "7005b025e68bce70267cd8198f60bc975d0dd5a8eea3134d829280ee9656a35b", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-29T14:17:32.560470+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0076475, "created_at": "2026-09-29T14:17:32.560Z" }, { "sequence": 159, "attempt_id": "5798ecfa-ef33-42d2-bcff-0c6277b3e1c5", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-textcolumns-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0074125, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "A" }, "group_id": "textcolumns-1-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-1-light-0.png", "input_tokens": 2545, "key_errors": null, "latency_sec": 2.0632782699540257, "model_name": "pareto", "output_tokens": 140, "prediction": { "choice": "A" }, "prompt_sha256": "7fb32e0fd4fc7490498a3b322716d502bc83af4be984e371808188e1a654a309", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-29T14:17:32.582321+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0074125, "created_at": "2026-09-29T14:17:32.583Z" }, { "sequence": 160, "attempt_id": "0a13fdf5-55b0-4014-9ab4-68c20866e672", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-direction-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0066825, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "A" }, "group_id": "direction-row-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-row-light-1.png", "input_tokens": 2547, "key_errors": null, "latency_sec": 1.4987934639211744, "model_name": "pareto", "output_tokens": 42, "prediction": { "choice": "A" }, "prompt_sha256": "84ba6fd0e7012086e1f258af7368901e4550a2b9a6e285f06f25a8ab71464ec2", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-29T14:17:33.161957+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0066825, "created_at": "2026-09-29T14:17:33.162Z" }, { "sequence": 161, "attempt_id": "6b1767c4-159a-4f7e-89a9-754452d0d176", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-widthcompare-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0077725, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "B" }, "group_id": "widthcompare-equal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-equal-light-0.png", "input_tokens": 2560, "key_errors": null, "latency_sec": 2.2953475201502442, "model_name": "pareto", "output_tokens": 183, "prediction": { "choice": "B" }, "prompt_sha256": "cd2c2f1f86d4ef1f19fa7f880569a29637d27eed1e41cdded418dc476e8498ed", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-29T14:17:33.434037+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0077725, "created_at": "2026-09-29T14:17:33.434Z" }, { "sequence": 162, "attempt_id": "712481a3-0b4d-4446-bed8-c152d317b0a1", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-nestedwrap-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.007475, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-25", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-25.png", "input_tokens": 2579, "key_errors": null, "latency_sec": 2.06300320988521, "model_name": "pareto", "output_tokens": 137, "prediction": { "choice": "C" }, "prompt_sha256": "3ba51907b89444fcea345fda34c9692faefd56e2b5c6671b282c0da644fdccb7", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-29T14:17:33.552925+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.007475, "created_at": "2026-09-29T14:17:33.553Z" }, { "sequence": 163, "attempt_id": "64553059-3b97-4ad5-a88a-13f291a8e621", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-gridtracks-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0073975, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "B" }, "group_id": "gridtracks-left-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-left-light-0.png", "input_tokens": 2569, "key_errors": null, "latency_sec": 2.0035907689016312, "model_name": "pareto", "output_tokens": 130, "prediction": { "choice": "B" }, "prompt_sha256": "aa2cac59ecf81aa683f25bd6bee52028be6daa4a3ce7c514aca8929ad1d53167", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-29T14:17:33.645398+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0073975, "created_at": "2026-09-29T14:17:33.645Z" }, { "sequence": 164, "attempt_id": "04611ff5-31a6-4c58-91f5-fdc3adad534b", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-topflow-016", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006685, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-16", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-16.png", "input_tokens": 2566, "key_errors": null, "latency_sec": 1.7123219221830368, "model_name": "pareto", "output_tokens": 36, "prediction": { "choice": "A" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-29T14:17:33.679024+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-016", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.006685, "created_at": "2026-09-29T14:17:33.679Z" }, { "sequence": 165, "attempt_id": "990561da-462d-4e64-bc0c-41a6b2db1297", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-padcompare-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0084925, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "A" }, "group_id": "padcompare-vertical-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-vertical-dark-0.png", "input_tokens": 2593, "key_errors": null, "latency_sec": 2.4030937959905714, "model_name": "pareto", "output_tokens": 268, "prediction": { "choice": "A" }, "prompt_sha256": "4f484ecb32307572888516788b76d03dcfb86406579ae3871dbbdd5d50c24d37", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-29T14:17:34.347976+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0084925, "created_at": "2026-09-29T14:17:34.348Z" }, { "sequence": 166, "attempt_id": "c17f4598-19a6-4126-bf00-10ac374a4272", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-crossalign-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0080525, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "C" }, "group_id": "crossalign-end-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-end-light-1.png", "input_tokens": 2642, "key_errors": null, "latency_sec": 1.9344107639044523, "model_name": "pareto", "output_tokens": 193, "prediction": { "choice": "C" }, "prompt_sha256": "259aeefd7e7471d977cd6004803e1298c147569226cadca5947c59db9f3e2731", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-29T14:17:34.516986+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0080525, "created_at": "2026-09-29T14:17:34.517Z" }, { "sequence": 167, "attempt_id": "f692b85e-2568-4d22-a500-ff0b9c029c60", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-nestedflow-021", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00691, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-21", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-21.png", "input_tokens": 2569, "key_errors": null, "latency_sec": 1.4790599481202662, "model_name": "pareto", "output_tokens": 65, "prediction": { "choice": "B" }, "prompt_sha256": "a555c9a37a814a2a2637ad2f6a1177955732dce4f82ecebc074c8930d36c5293", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-29T14:17:34.663170+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-021", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00691, "created_at": "2026-09-29T14:17:34.663Z" }, { "sequence": 168, "attempt_id": "7c00c52c-9026-4e20-a8ed-e402d7629a78", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-topflow-022", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0066925, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-22", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-22.png", "input_tokens": 2566, "key_errors": null, "latency_sec": 1.5933829401619732, "model_name": "pareto", "output_tokens": 37, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-29T14:17:35.167911+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-022", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0066925, "created_at": "2026-09-29T14:17:35.168Z" }, { "sequence": 169, "attempt_id": "7659aef3-e69c-4c9b-9d3b-c75910cb49ef", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-wrapalign-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0074775, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "C" }, "group_id": "wrapalign-left-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-left-light-1.png", "input_tokens": 2568, "key_errors": null, "latency_sec": 1.8943017560523003, "model_name": "pareto", "output_tokens": 141, "prediction": { "choice": "C" }, "prompt_sha256": "3df695a5de14c72a5a5dd88e4cced5a400dc2307a605c97919c60104bfb7a338", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-29T14:17:35.350693+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0074775, "created_at": "2026-09-29T14:17:35.350Z" }, { "sequence": 170, "attempt_id": "2a353abc-fc2a-4a31-ab6b-901f0dbc0865", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-gridgaps-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0082175, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "A" }, "group_id": "gridgaps-vertical-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-vertical-dark-1.png", "input_tokens": 2567, "key_errors": null, "latency_sec": 3.3943443831522018, "model_name": "pareto", "output_tokens": 240, "prediction": { "choice": "A" }, "prompt_sha256": "da1f2cca453f507bde83a2ac6d19b503fd7ab1c709bd4a77ee1046cf28ba5c59", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-29T14:17:36.011399+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0082175, "created_at": "2026-09-29T14:17:36.011Z" }, { "sequence": 171, "attempt_id": "380106bc-c026-4c6c-bd61-7b823b661b37", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-widthcompare-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00772, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "C" }, "group_id": "widthcompare-equal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-equal-dark-0.png", "input_tokens": 2560, "key_errors": null, "latency_sec": 2.4650111000519246, "model_name": "pareto", "output_tokens": 176, "prediction": { "choice": "C" }, "prompt_sha256": "807b859493212c8c914db2b1d9395b00171e3c91df87b66b9663b6cc7b65c485", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-29T14:17:36.166156+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00772, "created_at": "2026-09-29T14:17:36.166Z" }, { "sequence": 172, "attempt_id": "9749a9e7-d350-4d99-b6da-2f6ab6770592", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-nestedflow-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.007075, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-05", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-05.png", "input_tokens": 2569, "key_errors": null, "latency_sec": 1.824355703080073, "model_name": "pareto", "output_tokens": 87, "prediction": { "choice": "C" }, "prompt_sha256": "45c0efef1d7925275aea025bb1909f609ffc45577974778b58b34f1829f807e2", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-29T14:17:36.195249+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.007075, "created_at": "2026-09-29T14:17:36.195Z" }, { "sequence": 173, "attempt_id": "d3a681aa-f399-437d-bdf4-5113cb3df6e8", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-gridspan-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00796, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "A" }, "group_id": "gridspan-2-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-2-dark-1.png", "input_tokens": 2572, "key_errors": null, "latency_sec": 2.604833025019616, "model_name": "pareto", "output_tokens": 204, "prediction": { "choice": "A" }, "prompt_sha256": "c03d42c3e0b3f012910b0d501b5da5eff14e458eec8ebf179f14644511f48500", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-29T14:17:36.271979+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00796, "created_at": "2026-09-29T14:17:36.272Z" }, { "sequence": 174, "attempt_id": "623aeb4c-20cf-47f1-ab6f-03581b6419fc", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-gridrows-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0074375, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "A" }, "group_id": "gridrows-4-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-4-dark-1.png", "input_tokens": 2549, "key_errors": null, "latency_sec": 1.9172324549872428, "model_name": "pareto", "output_tokens": 142, "prediction": { "choice": "A" }, "prompt_sha256": "bc0885c5d8e75e638e58b055b8a0c362e439458552c2f596d459edf6097e3ea3", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-29T14:17:36.457204+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0074375, "created_at": "2026-09-29T14:17:36.457Z" }, { "sequence": 175, "attempt_id": "7460979b-02b1-40c7-b708-7ea60ff7d60f", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-textalign-010", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.0078725, "err": null, "error": "Expecting value: line 1 column 1 (char 0)", "error_kind": "invalid_response", "family": "textalign", "ground_truth": { "choice": "D" }, "group_id": "textalign-right-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-right-light-1.png", "input_tokens": 2597, "key_errors": null, "latency_sec": 2.2757514391560107, "model_name": "pareto", "output_tokens": 184, "prediction": {}, "prompt_sha256": "05e7cf07d1f1be0bf5dff1e08481cc4ad5c2acfdf246c6fe69f426c659c19fea", "provider": "anthropic", "raw_prediction": "A", "recorded_at": "2026-09-29T14:17:36.961355+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-textalign-010", "tight_score": null, "unmetered_attempts": 0, "valid": false, "within_bands": null }, "cost_usd": 0.0078725, "created_at": "2026-09-29T14:17:36.961Z" }, { "sequence": 176, "attempt_id": "18c142ce-c893-4969-9f9d-5952dd89bfb0", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-padcompare-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0079525, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "C" }, "group_id": "padcompare-horizontal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-horizontal-light-0.png", "input_tokens": 2593, "key_errors": null, "latency_sec": 2.0416087941266596, "model_name": "pareto", "output_tokens": 196, "prediction": { "choice": "C" }, "prompt_sha256": "0abb90b71247ce9a0876c99ad3ef52ea5a9ae09bfffb2875eec73b10166640c1", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-29T14:17:37.233012+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0079525, "created_at": "2026-09-29T14:17:37.233Z" }, { "sequence": 177, "attempt_id": "6d2e6698-3add-425e-b591-49ca6d04dcfc", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-widthcompare-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.007045, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "B" }, "group_id": "widthcompare-last-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-last-light-0.png", "input_tokens": 2560, "key_errors": null, "latency_sec": 2.125079007819295, "model_name": "pareto", "output_tokens": 86, "prediction": { "choice": "B" }, "prompt_sha256": "756f6b3e431f583d190c4d469c2409153beb0d173c07c749e93a1af92d2e6f8e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-29T14:17:38.313992+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.007045, "created_at": "2026-09-29T14:17:38.314Z" }, { "sequence": 178, "attempt_id": "3770d572-ae46-45f9-8855-366754a53100", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-groupgap-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0084525, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "B" }, "group_id": "groupgap-between-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-between-dark-0.png", "input_tokens": 2583, "key_errors": null, "latency_sec": 2.073339018970728, "model_name": "pareto", "output_tokens": 266, "prediction": { "choice": "B" }, "prompt_sha256": "b5587c95ce8398dca0e206a8a222a0859cb2781184dde718bbe27409262b14c1", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-29T14:17:38.367356+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0084525, "created_at": "2026-09-29T14:17:38.367Z" }, { "sequence": 179, "attempt_id": "496d3916-16f5-45be-a216-f5f6f347649b", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-gridrows-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00671, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "C" }, "group_id": "gridrows-3-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-3-light-0.png", "input_tokens": 2549, "key_errors": null, "latency_sec": 1.6556845589075238, "model_name": "pareto", "output_tokens": 45, "prediction": { "choice": "C" }, "prompt_sha256": "da4194e14f8d3c86eb007c251e2f977baeaf16f3ed67610f233d5aaa1e16745d", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-29T14:17:38.640541+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00671, "created_at": "2026-09-29T14:17:38.640Z" }, { "sequence": 180, "attempt_id": "878848c1-41e1-43e4-99dd-1cf1cca9a969", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-textcolumns-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0074275, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "B" }, "group_id": "textcolumns-1-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-1-dark-1.png", "input_tokens": 2545, "key_errors": null, "latency_sec": 2.1682536948937923, "model_name": "pareto", "output_tokens": 142, "prediction": { "choice": "B" }, "prompt_sha256": "9709598fa2c03afd3c8191b7a476b8aa1593f8488926311cbd6c9d5952354d67", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-29T14:17:38.665170+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0074275, "created_at": "2026-09-29T14:17:38.665Z" }, { "sequence": 181, "attempt_id": "83c5f2e2-6822-40d6-b115-6634414cd7c4", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-gapcompare-011", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.00842, "err": null, "error": "Expecting value: line 1 column 1 (char 0)", "error_kind": "invalid_response", "family": "gapcompare", "ground_truth": { "choice": "B" }, "group_id": "gapcompare-equal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-equal-dark-0.png", "input_tokens": 2567, "key_errors": null, "latency_sec": 2.651479404885322, "model_name": "pareto", "output_tokens": 267, "prediction": {}, "prompt_sha256": "ab1cd666bf62495c439cacec79574bb2f8ac552c98f8627324d3801b52d6d81b", "provider": "anthropic", "raw_prediction": "B", "recorded_at": "2026-09-29T14:17:38.869294+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-gapcompare-011", "tight_score": null, "unmetered_attempts": 0, "valid": false, "within_bands": null }, "cost_usd": 0.00842, "created_at": "2026-09-29T14:17:38.869Z" }, { "sequence": 182, "attempt_id": "4f176ef1-7314-485e-8645-b3a29053377f", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-gapcompare-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.010295, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "C" }, "group_id": "gapcompare-first-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-first-light-0.png", "input_tokens": 2567, "key_errors": null, "latency_sec": 3.0154006741940975, "model_name": "pareto", "output_tokens": 517, "prediction": { "choice": "C" }, "prompt_sha256": "2aacf3dbef42dbefa25b359df3a8af3c1541ab2712c25e9b3c3f5cf8371b79e4", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-29T14:17:39.050265+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.010295, "created_at": "2026-09-29T14:17:39.050Z" }, { "sequence": 183, "attempt_id": "484a998d-8d0c-40fe-a48e-525640a48f3c", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-distribution-020", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01049, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "F" }, "group_id": "distribution-around-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-around-dark-1.png", "input_tokens": 2648, "key_errors": null, "latency_sec": 3.958048290805891, "model_name": "pareto", "output_tokens": 516, "prediction": { "choice": "F" }, "prompt_sha256": "4dcc7229487e3d355fc6f147248270e76223b590fa32a706ea21638c7421eced", "provider": "anthropic", "raw_prediction": "{\"choice\": \"F\"}", "recorded_at": "2026-09-29T14:17:39.331721+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-020", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.01049, "created_at": "2026-09-29T14:17:39.331Z" }, { "sequence": 184, "attempt_id": "3cc9676c-c444-46d2-9393-d1a7956d1f87", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-nestedwrap-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0079625, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-23", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-23.png", "input_tokens": 2579, "key_errors": null, "latency_sec": 2.144976287148893, "model_name": "pareto", "output_tokens": 202, "prediction": { "choice": "A" }, "prompt_sha256": "a242868807b1158889b152c7c9664c235ebd10352a9a4a541f0a0af30a7fb4a5", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-29T14:17:39.401600+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0079625, "created_at": "2026-09-29T14:17:39.401Z" }, { "sequence": 185, "attempt_id": "96b20e57-4363-498e-bd9e-2a0796edc1f5", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-padcompare-010", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.020395, "err": null, "error": "Expecting value: line 1 column 1 (char 0)", "error_kind": "invalid_response", "family": "padcompare", "ground_truth": { "choice": "A" }, "group_id": "padcompare-equal-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-equal-light-1.png", "input_tokens": 2593, "key_errors": null, "latency_sec": 9.886003009974957, "model_name": "pareto", "output_tokens": 1855, "prediction": {}, "prompt_sha256": "07819e1c7e8686a5ffdde745114a9c6e8df1c7ac485dc319c5c72c24e1b67dae", "provider": "anthropic", "raw_prediction": "A", "recorded_at": "2026-09-29T14:17:39.885880+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-padcompare-010", "tight_score": null, "unmetered_attempts": 0, "valid": false, "within_bands": null }, "cost_usd": 0.020395, "created_at": "2026-09-29T14:17:39.886Z" }, { "sequence": 186, "attempt_id": "8d015355-529d-431b-91d6-121b6dd2e883", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-topflow-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0067225, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-01", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-01.png", "input_tokens": 2566, "key_errors": null, "latency_sec": 1.8700051179621369, "model_name": "pareto", "output_tokens": 41, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-29T14:17:40.535488+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0067225, "created_at": "2026-09-29T14:17:40.535Z" }, { "sequence": 187, "attempt_id": "f331b81c-3511-4312-a9ec-8426d6b124df", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-direction-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006765, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "A" }, "group_id": "direction-column-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-column-light-1.png", "input_tokens": 2547, "key_errors": null, "latency_sec": 1.561554656829685, "model_name": "pareto", "output_tokens": 53, "prediction": { "choice": "A" }, "prompt_sha256": "3768f3feb47a0750f78e6a2c372e707b1b87f7af93fd1fa6b13ef50f1092f226", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-29T14:17:40.635880+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.006765, "created_at": "2026-09-29T14:17:40.635Z" }, { "sequence": 188, "attempt_id": "6b215d47-67f3-40be-8e27-32205ac62bef", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-blockalign-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0075725, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "B" }, "group_id": "blockalign-right-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-right-light-1.png", "input_tokens": 2576, "key_errors": null, "latency_sec": 2.023338290862739, "model_name": "pareto", "output_tokens": 151, "prediction": { "choice": "B" }, "prompt_sha256": "238f37fc59f145a67466f3f7e834594604126429a37dc7fd71fda81a0610afe4", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-29T14:17:40.713059+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0075725, "created_at": "2026-09-29T14:17:40.713Z" }, { "sequence": 189, "attempt_id": "2ca8853a-6513-4bea-ba85-5f2563a1ef3e", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-crossalign-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.007445, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "C" }, "group_id": "crossalign-end-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-end-dark-0.png", "input_tokens": 2642, "key_errors": null, "latency_sec": 1.750789768062532, "model_name": "pareto", "output_tokens": 112, "prediction": { "choice": "C" }, "prompt_sha256": "7179fb2e3aaf1ed7b815d29c079e764245857c5a2a81c76dca5d8cd0b7fcd9e8", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-29T14:17:41.175703+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.007445, "created_at": "2026-09-29T14:17:41.175Z" }, { "sequence": 190, "attempt_id": "f4843eb3-be9d-4901-a6cb-f5200cdbc33d", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-textalign-015", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.008465, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "C" }, "group_id": "textalign-justify-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-justify-dark-0.png", "input_tokens": 2597, "key_errors": null, "latency_sec": 3.0027507601771504, "model_name": "pareto", "output_tokens": 263, "prediction": { "choice": "A" }, "prompt_sha256": "9a155e0b3c7bbbbff68050bc1a1e72dea59500ab480af46a3cd3b87e25de70bf", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-29T14:17:41.341207+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-textalign-015", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.008465, "created_at": "2026-09-29T14:17:41.341Z" }, { "sequence": 191, "attempt_id": "8aa72454-5f8d-4570-bae9-af8fe6048c52", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-nestedwrap-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0086075, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-22", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-22.png", "input_tokens": 2579, "key_errors": null, "latency_sec": 2.156134741846472, "model_name": "pareto", "output_tokens": 288, "prediction": { "choice": "B" }, "prompt_sha256": "a10b800c7bde04b869b770c30fd7ed4b80b25795d3668fcd2fd6f199bd9b600e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-29T14:17:41.512962+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0086075, "created_at": "2026-09-29T14:17:41.513Z" }, { "sequence": 192, "attempt_id": "9e846ac9-249e-4293-b404-b54dda80ac3e", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-crossalign-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00797, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "D" }, "group_id": "crossalign-center-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-center-light-1.png", "input_tokens": 2642, "key_errors": null, "latency_sec": 2.0570322540588677, "model_name": "pareto", "output_tokens": 182, "prediction": { "choice": "D" }, "prompt_sha256": "e6e3e7be0a2b7e91666344c6f0c3a0f931a9ea63d47c2dbd9e1c4cb334420e83", "provider": "anthropic", "raw_prediction": "{\"choice\": \"D\"}", "recorded_at": "2026-09-29T14:17:41.967820+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00797, "created_at": "2026-09-29T14:17:41.967Z" }, { "sequence": 193, "attempt_id": "eff9fb9d-f79e-4070-b3ee-153b59ef31ff", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-distribution-018", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.0441575, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "B" }, "group_id": "distribution-around-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-around-light-1.png", "input_tokens": 5354, "key_errors": null, "latency_sec": 21.95545280002989, "model_name": "pareto", "output_tokens": 4103, "prediction": { "choice": "A" }, "prompt_sha256": "964044cc4630a4303ac98062eed5f27cbfd8d60ca2ff55cefa53ce651a6151e1", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-29T14:17:42.084476+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-distribution-018", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0441575, "created_at": "2026-09-29T14:17:42.084Z" }, { "sequence": 194, "attempt_id": "79fdfd61-1e65-4cb2-b29d-29595b6f50ef", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-nestedwrap-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00905, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-28", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-28.png", "input_tokens": 2579, "key_errors": null, "latency_sec": 3.70287779904902, "model_name": "pareto", "output_tokens": 347, "prediction": { "choice": "C" }, "prompt_sha256": "3ba51907b89444fcea345fda34c9692faefd56e2b5c6671b282c0da644fdccb7", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-29T14:17:42.112826+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00905, "created_at": "2026-09-29T14:17:42.113Z" }, { "sequence": 195, "attempt_id": "677ebef5-ff59-4262-9764-e2df9ae43f10", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-gapcompare-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0074, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "B" }, "group_id": "gapcompare-last-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-last-dark-0.png", "input_tokens": 2567, "key_errors": null, "latency_sec": 1.7383661018684506, "model_name": "pareto", "output_tokens": 131, "prediction": { "choice": "B" }, "prompt_sha256": "2aacf3dbef42dbefa25b359df3a8af3c1541ab2712c25e9b3c3f5cf8371b79e4", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-29T14:17:42.398327+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0074, "created_at": "2026-09-29T14:17:42.398Z" }, { "sequence": 196, "attempt_id": "b68764eb-111a-4400-8009-5b64563dde90", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-gridtracks-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0082075, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "B" }, "group_id": "gridcols-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-light-1.png", "input_tokens": 2569, "key_errors": null, "latency_sec": 2.146391484187916, "model_name": "pareto", "output_tokens": 238, "prediction": { "choice": "B" }, "prompt_sha256": "330f46df745e35a409a07fe87ab9184b25be7cbe0c4a58bdce6f2fce80123f3a", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-29T14:17:42.707091+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0082075, "created_at": "2026-09-29T14:17:42.707Z" }, { "sequence": 197, "attempt_id": "cd4111af-d1e7-4475-9b7a-a2f9d6723f0b", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-distribution-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0103925, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "B" }, "group_id": "distribution-start-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-start-dark-0.png", "input_tokens": 2648, "key_errors": null, "latency_sec": 3.835082483012229, "model_name": "pareto", "output_tokens": 503, "prediction": { "choice": "B" }, "prompt_sha256": "1eda56ca21b06fb039bc76a8c3d84de28b1ce8490ca33890a7e3b2bcf7d45437", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-29T14:17:42.733604+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0103925, "created_at": "2026-09-29T14:17:42.735Z" }, { "sequence": 198, "attempt_id": "399441a3-af7f-4f96-a9a4-54ce18b79c3f", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-topflow-023", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006685, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-23", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-23.png", "input_tokens": 2566, "key_errors": null, "latency_sec": 1.653541567036882, "model_name": "pareto", "output_tokens": 36, "prediction": { "choice": "A" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-29T14:17:42.854462+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-023", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.006685, "created_at": "2026-09-29T14:17:42.854Z" }, { "sequence": 199, "attempt_id": "6142d319-d299-4782-9a98-3f07a6eda4a4", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-nestedwrap-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0071825, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-09", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-09.png", "input_tokens": 2579, "key_errors": null, "latency_sec": 2.339739317074418, "model_name": "pareto", "output_tokens": 98, "prediction": { "choice": "B" }, "prompt_sha256": "a10b800c7bde04b869b770c30fd7ed4b80b25795d3668fcd2fd6f199bd9b600e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-29T14:17:43.076568+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0071825, "created_at": "2026-09-29T14:17:43.076Z" }, { "sequence": 200, "attempt_id": "94c033f9-59f0-4182-98cf-c85bb0db20a8", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-blockalign-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0074525, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "A" }, "group_id": "blockalign-right-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-right-light-0.png", "input_tokens": 2576, "key_errors": null, "latency_sec": 1.9350436450913548, "model_name": "pareto", "output_tokens": 135, "prediction": { "choice": "A" }, "prompt_sha256": "7d6254d951604c657e71aeb7cae9178b26bc4d3dd7020cdf265225567557e5dd", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-29T14:17:43.301224+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0074525, "created_at": "2026-09-29T14:17:43.301Z" }, { "sequence": 201, "attempt_id": "f2082d51-5bc1-400f-ad18-dc7218675055", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-topflow-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0067525, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-12", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-12.png", "input_tokens": 2566, "key_errors": null, "latency_sec": 1.6304461141116917, "model_name": "pareto", "output_tokens": 45, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-29T14:17:43.623784+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0067525, "created_at": "2026-09-29T14:17:43.623Z" }, { "sequence": 202, "attempt_id": "fd46f4fd-362f-4ad2-a112-958346267fe7", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-widthcompare-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00697, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "C" }, "group_id": "widthcompare-first-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-first-light-1.png", "input_tokens": 2560, "key_errors": null, "latency_sec": 1.610805595992133, "model_name": "pareto", "output_tokens": 76, "prediction": { "choice": "C" }, "prompt_sha256": "cd2c2f1f86d4ef1f19fa7f880569a29637d27eed1e41cdded418dc476e8498ed", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-29T14:17:43.723981+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00697, "created_at": "2026-09-29T14:17:43.724Z" }, { "sequence": 203, "attempt_id": "4a32b2d6-a65d-4505-ab7e-36cb6c370217", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-topflow-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0067525, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-08", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-08.png", "input_tokens": 2566, "key_errors": null, "latency_sec": 1.4852710850536823, "model_name": "pareto", "output_tokens": 45, "prediction": { "choice": "B" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-29T14:17:43.909242+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0067525, "created_at": "2026-09-29T14:17:43.909Z" }, { "sequence": 204, "attempt_id": "a45df715-efe5-4ce4-baad-45d6555b5dea", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-textcolumns-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006595, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "C" }, "group_id": "textcolumns-3-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-3-dark-0.png", "input_tokens": 2545, "key_errors": null, "latency_sec": 1.8905359450727701, "model_name": "pareto", "output_tokens": 31, "prediction": { "choice": "C" }, "prompt_sha256": "7fb32e0fd4fc7490498a3b322716d502bc83af4be984e371808188e1a654a309", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-29T14:17:44.029791+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.006595, "created_at": "2026-09-29T14:17:44.029Z" }, { "sequence": 205, "attempt_id": "a01cee2f-7e3e-42f2-9dd2-513e2813f42d", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-textcolumns-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006745, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "C" }, "group_id": "textcolumns-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-3-dark-1.png", "input_tokens": 2545, "key_errors": null, "latency_sec": 1.872590949991718, "model_name": "pareto", "output_tokens": 51, "prediction": { "choice": "C" }, "prompt_sha256": "78899f74135f81158adab4e7446b61a7bda49f0ad810f8b42dad28a3640486f6", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-29T14:17:44.632534+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.006745, "created_at": "2026-09-29T14:17:44.632Z" }, { "sequence": 206, "attempt_id": "2f084d23-49d7-4ba0-a4bb-49d54164d0e1", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-gridrows-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0067625, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "A" }, "group_id": "gridcols-2-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-2-dark-0.png", "input_tokens": 2549, "key_errors": null, "latency_sec": 1.975827471818775, "model_name": "pareto", "output_tokens": 52, "prediction": { "choice": "A" }, "prompt_sha256": "0fad5d0a9261a17e68aa8de2ef5fb0207dc8af585c3baaeabfc584af8b5329f5", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-29T14:17:44.856330+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0067625, "created_at": "2026-09-29T14:17:44.856Z" }, { "sequence": 207, "attempt_id": "d3c31695-5558-40a2-a711-e004ab759687", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-nestedflow-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00679, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-04", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-04.png", "input_tokens": 2569, "key_errors": null, "latency_sec": 2.1796916800085455, "model_name": "pareto", "output_tokens": 49, "prediction": { "choice": "C" }, "prompt_sha256": "a555c9a37a814a2a2637ad2f6a1177955732dce4f82ecebc074c8930d36c5293", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-29T14:17:44.913563+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00679, "created_at": "2026-09-29T14:17:44.913Z" }, { "sequence": 208, "attempt_id": "b44c1bd2-70c0-423d-98f3-5f84d352b22e", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-nestedflow-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00682, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-09", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-09.png", "input_tokens": 2569, "key_errors": null, "latency_sec": 1.618158380035311, "model_name": "pareto", "output_tokens": 53, "prediction": { "choice": "B" }, "prompt_sha256": "a555c9a37a814a2a2637ad2f6a1177955732dce4f82ecebc074c8930d36c5293", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-29T14:17:44.952340+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00682, "created_at": "2026-09-29T14:17:44.953Z" }, { "sequence": 209, "attempt_id": "8da1342f-e90e-402f-ae1b-efd1d3bd8703", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-topflow-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0067, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-02", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-02.png", "input_tokens": 2566, "key_errors": null, "latency_sec": 1.7601971072144806, "model_name": "pareto", "output_tokens": 38, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-29T14:17:45.410614+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0067, "created_at": "2026-09-29T14:17:45.410Z" }, { "sequence": 210, "attempt_id": "aba5bd06-e7be-4298-9217-ffec2c922cd2", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-gridspan-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0075325, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "A" }, "group_id": "gridspan-2-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-2-light-1.png", "input_tokens": 2572, "key_errors": null, "latency_sec": 1.9048811018001288, "model_name": "pareto", "output_tokens": 147, "prediction": { "choice": "A" }, "prompt_sha256": "4c2619c1f3339203e066968a6911df07e41f49d553c6c3cb2be0b1e81e76a29b", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-29T14:17:45.655570+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0075325, "created_at": "2026-09-29T14:17:45.655Z" }, { "sequence": 211, "attempt_id": "2860f392-1e3f-4b63-95b1-99afd2bcad1f", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-gridcols-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0070175, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "B" }, "group_id": "gridcols-2-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-2-light-1.png", "input_tokens": 2549, "key_errors": null, "latency_sec": 2.162986687151715, "model_name": "pareto", "output_tokens": 86, "prediction": { "choice": "B" }, "prompt_sha256": "0853a31c9fbb6301b3f371c91baacf09c13f4a5be3b3e7a0613b6b02d27a8b19", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-29T14:17:46.098794+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0070175, "created_at": "2026-09-29T14:17:46.098Z" }, { "sequence": 212, "attempt_id": "abad799b-c5dc-477a-a556-f1fc53b01d74", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-nestedflow-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00703, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-07", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-07.png", "input_tokens": 2569, "key_errors": null, "latency_sec": 2.1243503771256655, "model_name": "pareto", "output_tokens": 81, "prediction": { "choice": "A" }, "prompt_sha256": "377bcd13e265380fd865ccbfd9819e64dfd2c4c3b2980e09c6e4d88924162f6c", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-29T14:17:46.179830+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00703, "created_at": "2026-09-29T14:17:46.179Z" }, { "sequence": 213, "attempt_id": "499bcd73-4a4d-446b-9d25-b697a6539c0e", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-widthcompare-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0070225, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "B" }, "group_id": "widthcompare-first-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-first-light-0.png", "input_tokens": 2560, "key_errors": null, "latency_sec": 1.5687876418232918, "model_name": "pareto", "output_tokens": 83, "prediction": { "choice": "B" }, "prompt_sha256": "875f019d57875304ded0af8f075989118a0e4fcf9ca43c9eba1cda4f64a503a0", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-29T14:17:46.521437+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0070225, "created_at": "2026-09-29T14:17:46.521Z" }, { "sequence": 214, "attempt_id": "257f63f4-93e4-483f-ab3d-6808bd9b1f9d", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-nestedflow-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0069625, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-11", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-11.png", "input_tokens": 2569, "key_errors": null, "latency_sec": 1.6575980000197887, "model_name": "pareto", "output_tokens": 72, "prediction": { "choice": "C" }, "prompt_sha256": "377bcd13e265380fd865ccbfd9819e64dfd2c4c3b2980e09c6e4d88924162f6c", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-29T14:17:46.549635+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0069625, "created_at": "2026-09-29T14:17:46.550Z" }, { "sequence": 215, "attempt_id": "769b7d28-1ec5-467e-8042-fb40ffe3d765", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-textcolumns-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00733, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "B" }, "group_id": "textcolumns-1-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-1-light-1.png", "input_tokens": 2545, "key_errors": null, "latency_sec": 1.9127576209139079, "model_name": "pareto", "output_tokens": 129, "prediction": { "choice": "B" }, "prompt_sha256": "9709598fa2c03afd3c8191b7a476b8aa1593f8488926311cbd6c9d5952354d67", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-29T14:17:46.577663+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00733, "created_at": "2026-09-29T14:17:46.579Z" }, { "sequence": 216, "attempt_id": "883f5f17-7352-44fd-88ef-9468bc356400", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-distribution-017", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0089525, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "F" }, "group_id": "distribution-around-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-around-light-0.png", "input_tokens": 2648, "key_errors": null, "latency_sec": 2.3350224760361016, "model_name": "pareto", "output_tokens": 311, "prediction": { "choice": "F" }, "prompt_sha256": "bbb61c606caa02f5696f7942e1fe764803a2a2fa41fb5bea98e450a25158c219", "provider": "anthropic", "raw_prediction": "{\"choice\": \"F\"}", "recorded_at": "2026-09-29T14:17:47.314358+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-017", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0089525, "created_at": "2026-09-29T14:17:47.314Z" }, { "sequence": 217, "attempt_id": "1c89a56d-afc5-4ffc-9d9c-ceb91b9531fa", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-topflow-019", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00667, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-19", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-19.png", "input_tokens": 2566, "key_errors": null, "latency_sec": 1.4752840739674866, "model_name": "pareto", "output_tokens": 34, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-29T14:17:47.602042+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-019", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00667, "created_at": "2026-09-29T14:17:47.602Z" }, { "sequence": 218, "attempt_id": "5b55249f-1273-462a-9099-874604a6ef0b", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-groupgap-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009285, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "C" }, "group_id": "groupgap-within-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-within-dark-0.png", "input_tokens": 2583, "key_errors": null, "latency_sec": 2.630145997973159, "model_name": "pareto", "output_tokens": 377, "prediction": { "choice": "C" }, "prompt_sha256": "6cba71084ece6f53e834e34e24c720eaabde46f305bb7ec0454830a5812e840e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-29T14:17:48.068170+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.009285, "created_at": "2026-09-29T14:17:48.068Z" }, { "sequence": 219, "attempt_id": "a747d1ff-b9d9-4944-98ed-8b82bfaf9a1c", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-gridcols-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00683, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "B" }, "group_id": "gridcols-2-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-2-dark-0.png", "input_tokens": 2549, "key_errors": null, "latency_sec": 1.904915687860921, "model_name": "pareto", "output_tokens": 61, "prediction": { "choice": "B" }, "prompt_sha256": "4e97151663d1e3e37046a23842abcce2e96ecf3d2204566c717e9014b170f248", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-29T14:17:48.510289+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00683, "created_at": "2026-09-29T14:17:48.510Z" }, { "sequence": 220, "attempt_id": "6045a3a0-8144-452f-990c-30cad0d3f40e", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-topflow-015", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006685, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-15", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-15.png", "input_tokens": 2566, "key_errors": null, "latency_sec": 1.9638949721120298, "model_name": "pareto", "output_tokens": 36, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-29T14:17:48.541826+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-015", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.006685, "created_at": "2026-09-29T14:17:48.542Z" }, { "sequence": 221, "attempt_id": "d3625964-4eb7-44f7-8101-ba7b5d03c420", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-widthcompare-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.007165, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "B" }, "group_id": "widthcompare-first-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-first-dark-1.png", "input_tokens": 2560, "key_errors": null, "latency_sec": 2.580907064024359, "model_name": "pareto", "output_tokens": 102, "prediction": { "choice": "B" }, "prompt_sha256": "875f019d57875304ded0af8f075989118a0e4fcf9ca43c9eba1cda4f64a503a0", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-29T14:17:49.130866+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.007165, "created_at": "2026-09-29T14:17:49.131Z" }, { "sequence": 222, "attempt_id": "261abb7c-019f-423e-8ff0-b6dcb7656f20", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-gridcols-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006815, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "B" }, "group_id": "gridcols-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-light-1.png", "input_tokens": 2549, "key_errors": null, "latency_sec": 1.5730073500890285, "model_name": "pareto", "output_tokens": 59, "prediction": { "choice": "B" }, "prompt_sha256": "baf29085a3966715a493e43b25eb411faa9ffe2033ec17653423007441d24033", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-29T14:17:49.203680+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.006815, "created_at": "2026-09-29T14:17:49.203Z" }, { "sequence": 223, "attempt_id": "68f05e2e-8ce0-4944-989a-4fc318cf5216", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-groupgap-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0090525, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "B" }, "group_id": "groupgap-equal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-equal-light-0.png", "input_tokens": 2583, "key_errors": null, "latency_sec": 3.2952831611037254, "model_name": "pareto", "output_tokens": 346, "prediction": { "choice": "B" }, "prompt_sha256": "b4672b836da84079fbff370d5a3a7a8410525141d624bd71e50f7c11264338f7", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-29T14:17:49.502057+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0090525, "created_at": "2026-09-29T14:17:49.502Z" }, { "sequence": 224, "attempt_id": "d9c34034-9aa2-47b4-9bf4-551f429ecd42", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-textalign-013", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0152, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "A" }, "group_id": "textalign-justify-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-justify-light-0.png", "input_tokens": 2597, "key_errors": null, "latency_sec": 6.594838043907657, "model_name": "pareto", "output_tokens": 1161, "prediction": { "choice": "A" }, "prompt_sha256": "02fcd4a4c952217ee96fdbbca5e92af61005f02abe9611e017f571e23cb193e2", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-29T14:17:49.697642+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-013", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0152, "created_at": "2026-09-29T14:17:49.697Z" }, { "sequence": 225, "attempt_id": "429291b5-7a4f-42f6-8697-aaef5f386bb0", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-padcompare-011", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.0194725, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "C" }, "group_id": "padcompare-equal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-equal-dark-0.png", "input_tokens": 2593, "key_errors": null, "latency_sec": 8.268409000942484, "model_name": "pareto", "output_tokens": 1732, "prediction": { "choice": "A" }, "prompt_sha256": "4f484ecb32307572888516788b76d03dcfb86406579ae3871dbbdd5d50c24d37", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-29T14:17:49.806787+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-padcompare-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0194725, "created_at": "2026-09-29T14:17:49.806Z" }, { "sequence": 226, "attempt_id": "8068d9cd-9199-455c-b288-89a0299be69c", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-groupgap-010", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.0075225, "err": null, "error": "Expecting value: line 1 column 1 (char 0)", "error_kind": "invalid_response", "family": "groupgap", "ground_truth": { "choice": "A" }, "group_id": "groupgap-equal-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-equal-light-1.png", "input_tokens": 2583, "key_errors": null, "latency_sec": 2.9354980350472033, "model_name": "pareto", "output_tokens": 142, "prediction": {}, "prompt_sha256": "a4ae0e9e5d9435b02d3759ecf1850c36024f893c4de107ea8256f2c5052eaa53", "provider": "anthropic", "raw_prediction": "A", "recorded_at": "2026-09-29T14:17:50.278819+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-groupgap-010", "tight_score": null, "unmetered_attempts": 0, "valid": false, "within_bands": null }, "cost_usd": 0.0075225, "created_at": "2026-09-29T14:17:50.278Z" }, { "sequence": 227, "attempt_id": "a62131fd-b8a4-4386-a062-74288586e87d", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-gridtracks-011", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0072625, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "A" }, "group_id": "gridtracks-right-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-right-dark-0.png", "input_tokens": 2569, "key_errors": null, "latency_sec": 1.764803681988269, "model_name": "pareto", "output_tokens": 112, "prediction": { "choice": "A" }, "prompt_sha256": "e1686611bde92953032219f0663fed17a23573139af7c26937579757a13cc24d", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-29T14:17:50.334361+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0072625, "created_at": "2026-09-29T14:17:50.334Z" }, { "sequence": 228, "attempt_id": "8e1689ac-0352-422a-97f4-0e91b0ae424f", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-gridtracks-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0075625, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "A" }, "group_id": "gridtracks-left-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-left-light-1.png", "input_tokens": 2569, "key_errors": null, "latency_sec": 1.8880702739115804, "model_name": "pareto", "output_tokens": 152, "prediction": { "choice": "A" }, "prompt_sha256": "10cd7f5950372ee9cf1d2df732ca333086531870973115e171dc7abfa750b35c", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-29T14:17:50.427990+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0075625, "created_at": "2026-09-29T14:17:50.428Z" }, { "sequence": 229, "attempt_id": "ab9d1662-e0b9-4452-99ee-db360a476348", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-textalign-005", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.011495, "err": null, "error": "Expecting value: line 1 column 1 (char 0)", "error_kind": "invalid_response", "family": "textalign", "ground_truth": { "choice": "C" }, "group_id": "textalign-center-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-center-light-0.png", "input_tokens": 2597, "key_errors": null, "latency_sec": 4.869142596144229, "model_name": "pareto", "output_tokens": 667, "prediction": {}, "prompt_sha256": "a46b47bd0291db684271663183741f72c6e90a5628697cdf5c907a9da6067b3d", "provider": "anthropic", "raw_prediction": "C", "recorded_at": "2026-09-29T14:17:50.552365+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-textalign-005", "tight_score": null, "unmetered_attempts": 0, "valid": false, "within_bands": null }, "cost_usd": 0.011495, "created_at": "2026-09-29T14:17:50.552Z" }, { "sequence": 230, "attempt_id": "c1af443d-4b57-4331-ac2b-d6a79137187f", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-crossalign-016", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00833, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "B" }, "group_id": "crossalign-stretch-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-stretch-dark-1.png", "input_tokens": 2642, "key_errors": null, "latency_sec": 2.5430529578588903, "model_name": "pareto", "output_tokens": 230, "prediction": { "choice": "B" }, "prompt_sha256": "fed2c2756d1bfbfa4a2899a0ada5790db38828dab3d9688a7a0f2ac04c6b6676", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-29T14:17:50.640380+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-016", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00833, "created_at": "2026-09-29T14:17:50.640Z" }, { "sequence": 231, "attempt_id": "a112af1d-8549-4b5d-94c5-73436473e6ec", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-textcolumns-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00661, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "C" }, "group_id": "textcolumns-2-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-2-light-0.png", "input_tokens": 2545, "key_errors": null, "latency_sec": 1.7989499571267515, "model_name": "pareto", "output_tokens": 33, "prediction": { "choice": "C" }, "prompt_sha256": "4888a14115f1b07fd0ac541d1950a02ffd9073681cd57eaacd9a4eb745a47694", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-29T14:17:50.958563+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00661, "created_at": "2026-09-29T14:17:50.958Z" }, { "sequence": 232, "attempt_id": "fe0adaab-c37d-4522-a96b-465c43f17a90", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-wrapcount-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006845, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "C" }, "group_id": "wrapcount-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-3-light-1.png", "input_tokens": 2546, "key_errors": null, "latency_sec": 1.6703446209430695, "model_name": "pareto", "output_tokens": 64, "prediction": { "choice": "C" }, "prompt_sha256": "d5a11da43aa9608fe90b20a74d3d14e9dbf3731c33be73ca93aad96c927ccc7d", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-29T14:17:51.200882+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.006845, "created_at": "2026-09-29T14:17:51.201Z" }, { "sequence": 233, "attempt_id": "092b52f8-498e-41e5-9b64-65fb83c0fddc", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-gridcols-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0069275, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "A" }, "group_id": "gridcols-4-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-4-light-1.png", "input_tokens": 2549, "key_errors": null, "latency_sec": 2.1292437959928066, "model_name": "pareto", "output_tokens": 74, "prediction": { "choice": "A" }, "prompt_sha256": "69a0dbb7b95e4d65f59a6c7934d34ff8a7253b73dbb6ab0dfaf6ce730257753e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-29T14:17:51.362460+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0069275, "created_at": "2026-09-29T14:17:51.362Z" }, { "sequence": 234, "attempt_id": "9d048525-37be-489a-8e63-820eb0e65d4a", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-crossalign-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00824, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "B" }, "group_id": "crossalign-end-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-end-dark-1.png", "input_tokens": 2642, "key_errors": null, "latency_sec": 2.023086281027645, "model_name": "pareto", "output_tokens": 218, "prediction": { "choice": "B" }, "prompt_sha256": "a741f50a81efd135db5359d6779e3d6988fa29ccc1e2f727cb701189f06bf4d8", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-29T14:17:51.749133+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00824, "created_at": "2026-09-29T14:17:51.749Z" }, { "sequence": 235, "attempt_id": "5b15d4f2-43ce-4128-b967-c2c904694ccd", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-topflow-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006775, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-07", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-07.png", "input_tokens": 2566, "key_errors": null, "latency_sec": 2.187021662015468, "model_name": "pareto", "output_tokens": 48, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-29T14:17:52.551247+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.006775, "created_at": "2026-09-29T14:17:52.551Z" }, { "sequence": 236, "attempt_id": "51332c52-6119-4947-addf-699b9b29005a", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-gridspan-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0078325, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "A" }, "group_id": "gridspan-2-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-2-dark-0.png", "input_tokens": 2572, "key_errors": null, "latency_sec": 2.9438704641070217, "model_name": "pareto", "output_tokens": 187, "prediction": { "choice": "A" }, "prompt_sha256": "4c2619c1f3339203e066968a6911df07e41f49d553c6c3cb2be0b1e81e76a29b", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-29T14:17:52.779867+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0078325, "created_at": "2026-09-29T14:17:52.779Z" }, { "sequence": 237, "attempt_id": "0cdc1fb0-5e6a-4b97-93a9-779945b77604", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-topflow-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0067675, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-03", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-03.png", "input_tokens": 2566, "key_errors": null, "latency_sec": 1.827246906934306, "model_name": "pareto", "output_tokens": 47, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-29T14:17:53.219205+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0067675, "created_at": "2026-09-29T14:17:53.219Z" }, { "sequence": 238, "attempt_id": "25a81dbb-8062-4cb4-a7d8-d34fc101eaa9", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-nestedflow-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00697, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-02", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-02.png", "input_tokens": 2569, "key_errors": null, "latency_sec": 2.0807702669408172, "model_name": "pareto", "output_tokens": 73, "prediction": { "choice": "C" }, "prompt_sha256": "ac0c30958153584a4b94fe042ea1cbf718f020f9f66dd474a09c514effd2f9f9", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-29T14:17:53.312300+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00697, "created_at": "2026-09-29T14:17:53.312Z" }, { "sequence": 239, "attempt_id": "aeb7dcd9-5b10-4e3e-8c73-ddbee9c8fb5b", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-distribution-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0101, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "D" }, "group_id": "distribution-start-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-start-light-0.png", "input_tokens": 2648, "key_errors": null, "latency_sec": 3.0047242019791156, "model_name": "pareto", "output_tokens": 464, "prediction": { "choice": "D" }, "prompt_sha256": "c5e8a062bab89a0ada9699f67d71f11574c44b07f47fe6b471893162d79e822e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"D\"}", "recorded_at": "2026-09-29T14:17:53.461238+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0101, "created_at": "2026-09-29T14:17:53.461Z" }, { "sequence": 240, "attempt_id": "7106edb9-1f8d-4311-bbac-ca063cd8b6f1", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-distribution-021", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.008405, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "B" }, "group_id": "distribution-evenly-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-evenly-light-0.png", "input_tokens": 2648, "key_errors": null, "latency_sec": 2.694969051051885, "model_name": "pareto", "output_tokens": 238, "prediction": { "choice": "B" }, "prompt_sha256": "bb6a0458622253c31a96dd71203dfe1750682c555db8bd324eda28766af2ea8d", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-29T14:17:53.684180+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-021", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.008405, "created_at": "2026-09-29T14:17:53.684Z" }, { "sequence": 241, "attempt_id": "e0bff592-fce1-42ac-a435-e952d26b821c", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-textalign-002", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.009605, "err": null, "error": "Expecting value: line 1 column 1 (char 0)", "error_kind": "invalid_response", "family": "textalign", "ground_truth": { "choice": "D" }, "group_id": "textalign-left-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-left-light-1.png", "input_tokens": 2597, "key_errors": null, "latency_sec": 3.5359742981381714, "model_name": "pareto", "output_tokens": 415, "prediction": {}, "prompt_sha256": "2e47c2a304900033c1e7233c922b86d826e6949f29106d3fa5c967cc727df497", "provider": "anthropic", "raw_prediction": "C", "recorded_at": "2026-09-29T14:17:54.117572+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-textalign-002", "tight_score": null, "unmetered_attempts": 0, "valid": false, "within_bands": null }, "cost_usd": 0.009605, "created_at": "2026-09-29T14:17:54.117Z" }, { "sequence": 242, "attempt_id": "2c6f6299-199a-4c8a-b461-8dc8f2f7495e", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-crossalign-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.008375, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "A" }, "group_id": "crossalign-center-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-center-light-0.png", "input_tokens": 2642, "key_errors": null, "latency_sec": 2.347847797907889, "model_name": "pareto", "output_tokens": 236, "prediction": { "choice": "A" }, "prompt_sha256": "ea02b690a39fed2c99e991ca10571f8f5e75cfea4e2db5349dafc50758191f40", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-29T14:17:54.930428+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.008375, "created_at": "2026-09-29T14:17:54.930Z" }, { "sequence": 243, "attempt_id": "c5d1765a-98e4-4cb7-9868-bc35e3326f0f", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-gapcompare-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0110375, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "B" }, "group_id": "gapcompare-first-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-first-light-1.png", "input_tokens": 2567, "key_errors": null, "latency_sec": 4.270517039112747, "model_name": "pareto", "output_tokens": 616, "prediction": { "choice": "B" }, "prompt_sha256": "bb41f3da30db1063257895b1a55fa7d2a40b0d638c3bd0111db12e799eb0d481", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-29T14:17:54.961984+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0110375, "created_at": "2026-09-29T14:17:54.963Z" }, { "sequence": 244, "attempt_id": "09c6092e-58ad-4c93-b036-ac068f7b849f", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-wrapalign-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00768, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "B" }, "group_id": "wrapalign-right-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-right-light-0.png", "input_tokens": 2568, "key_errors": null, "latency_sec": 3.2962478541303426, "model_name": "pareto", "output_tokens": 168, "prediction": { "choice": "B" }, "prompt_sha256": "3df695a5de14c72a5a5dd88e4cced5a400dc2307a605c97919c60104bfb7a338", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-29T14:17:55.076405+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00768, "created_at": "2026-09-29T14:17:55.076Z" }, { "sequence": 245, "attempt_id": "04184763-3cd8-4c5e-96e3-dc969bbc08a1", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-gridcols-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00668, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "C" }, "group_id": "gridcols-4-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-4-dark-1.png", "input_tokens": 2549, "key_errors": null, "latency_sec": 1.7810945990495384, "model_name": "pareto", "output_tokens": 41, "prediction": { "choice": "C" }, "prompt_sha256": "baf29085a3966715a493e43b25eb411faa9ffe2033ec17653423007441d24033", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-29T14:17:55.122983+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00668, "created_at": "2026-09-29T14:17:55.123Z" }, { "sequence": 246, "attempt_id": "fbbc16f7-a657-4c8c-81a9-014ee735bb13", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-topflow-020", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0067, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-20", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-20.png", "input_tokens": 2566, "key_errors": null, "latency_sec": 2.322535689920187, "model_name": "pareto", "output_tokens": 38, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-29T14:17:55.153438+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-020", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0067, "created_at": "2026-09-29T14:17:55.154Z" }, { "sequence": 247, "attempt_id": "5888a7bb-34f1-46cf-8406-a7ca5e0965a1", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-nestedwrap-013", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.008105, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-29", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-29.png", "input_tokens": 2579, "key_errors": null, "latency_sec": 2.274803881999105, "model_name": "pareto", "output_tokens": 221, "prediction": { "choice": "B" }, "prompt_sha256": "1019a8d1887a379785b7f0f2096c38ba800c5c05f4005e0a2fc11959e378500f", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-29T14:17:55.525272+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-013", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.008105, "created_at": "2026-09-29T14:17:55.525Z" }, { "sequence": 248, "attempt_id": "8f2cd19b-20e0-4c50-b202-72dfa30adc0d", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-gapcompare-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0074675, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "B" }, "group_id": "gapcompare-first-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-first-dark-0.png", "input_tokens": 2567, "key_errors": null, "latency_sec": 1.8522727340459824, "model_name": "pareto", "output_tokens": 140, "prediction": { "choice": "B" }, "prompt_sha256": "bb41f3da30db1063257895b1a55fa7d2a40b0d638c3bd0111db12e799eb0d481", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-29T14:17:55.575807+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0074675, "created_at": "2026-09-29T14:17:55.575Z" }, { "sequence": 249, "attempt_id": "63799ce5-ffe3-4973-87f0-132050517498", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-groupgap-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01086, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "C" }, "group_id": "groupgap-within-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-within-dark-1.png", "input_tokens": 2583, "key_errors": null, "latency_sec": 5.301955471979454, "model_name": "pareto", "output_tokens": 587, "prediction": { "choice": "C" }, "prompt_sha256": "a4ae0e9e5d9435b02d3759ecf1850c36024f893c4de107ea8256f2c5052eaa53", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-29T14:17:55.610780+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.01086, "created_at": "2026-09-29T14:17:55.610Z" }, { "sequence": 250, "attempt_id": "681ee6a6-4449-4eea-8d94-35a80b2ef6e2", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-nestedwrap-019", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0080975, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-35", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-35.png", "input_tokens": 2579, "key_errors": null, "latency_sec": 2.859820842044428, "model_name": "pareto", "output_tokens": 220, "prediction": { "choice": "C" }, "prompt_sha256": "a10b800c7bde04b869b770c30fd7ed4b80b25795d3668fcd2fd6f199bd9b600e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-29T14:17:56.351767+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-019", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0080975, "created_at": "2026-09-29T14:17:56.351Z" }, { "sequence": 251, "attempt_id": "8a79a1f5-d62c-4e1c-8485-1ccf009f635e", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-gridgaps-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00818, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "B" }, "group_id": "gridgaps-horizontal-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-horizontal-light-1.png", "input_tokens": 2567, "key_errors": null, "latency_sec": 2.3823683608788997, "model_name": "pareto", "output_tokens": 235, "prediction": { "choice": "B" }, "prompt_sha256": "bb599c345aa6aaf2e8a00a558db18c18ea46033edf36cd1b879c2785b58dce8b", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-29T14:17:56.533877+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00818, "created_at": "2026-09-29T14:17:56.533Z" }, { "sequence": 252, "attempt_id": "59a35217-1be2-4ea8-924b-92035069e2d1", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-nestedflow-013", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0069625, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-13", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-13.png", "input_tokens": 2569, "key_errors": null, "latency_sec": 2.003039717907086, "model_name": "pareto", "output_tokens": 72, "prediction": { "choice": "B" }, "prompt_sha256": "6cb509a4fb3ecab9a334fb061a4aa543245f6985aa65f331cb06646a50f35dfe", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-29T14:17:57.156799+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-013", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0069625, "created_at": "2026-09-29T14:17:57.156Z" }, { "sequence": 253, "attempt_id": "921c6c21-3105-48e2-8e7a-277dd66400ee", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-blockalign-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0076175, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "A" }, "group_id": "blockalign-left-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-left-light-0.png", "input_tokens": 2576, "key_errors": null, "latency_sec": 2.140302923042327, "model_name": "pareto", "output_tokens": 157, "prediction": { "choice": "A" }, "prompt_sha256": "e32486067ac25398fcca8ac9d957270d096fb6771f17cdb332d35edf376b0f86", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-29T14:17:57.327829+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0076175, "created_at": "2026-09-29T14:17:57.327Z" }, { "sequence": 254, "attempt_id": "a0208384-ef0c-4c49-b069-2ce0a704b38d", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-widthcompare-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0075925, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "A" }, "group_id": "widthcompare-equal-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-equal-light-1.png", "input_tokens": 2560, "key_errors": null, "latency_sec": 2.4131321778986603, "model_name": "pareto", "output_tokens": 159, "prediction": { "choice": "A" }, "prompt_sha256": "875f019d57875304ded0af8f075989118a0e4fcf9ca43c9eba1cda4f64a503a0", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-29T14:17:57.375358+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0075925, "created_at": "2026-09-29T14:17:57.375Z" }, { "sequence": 255, "attempt_id": "e034441b-65e0-41a9-8947-ecdcff87cef5", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-gridtracks-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.007465, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "A" }, "group_id": "gridtracks-left-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-left-dark-0.png", "input_tokens": 2569, "key_errors": null, "latency_sec": 2.3992168370168656, "model_name": "pareto", "output_tokens": 139, "prediction": { "choice": "A" }, "prompt_sha256": "10cd7f5950372ee9cf1d2df732ca333086531870973115e171dc7abfa750b35c", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-29T14:17:57.506258+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.007465, "created_at": "2026-09-29T14:17:57.506Z" }, { "sequence": 256, "attempt_id": "aeaa8f69-e7b6-4f13-9d03-b73dc8a3cd1d", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-textalign-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0070325, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "D" }, "group_id": "textalign-left-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-left-light-0.png", "input_tokens": 2597, "key_errors": null, "latency_sec": 2.0401099419686943, "model_name": "pareto", "output_tokens": 72, "prediction": { "choice": "D" }, "prompt_sha256": "2e47c2a304900033c1e7233c922b86d826e6949f29106d3fa5c967cc727df497", "provider": "anthropic", "raw_prediction": "{\"choice\": \"D\"}", "recorded_at": "2026-09-29T14:17:57.596572+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0070325, "created_at": "2026-09-29T14:17:57.596Z" }, { "sequence": 257, "attempt_id": "efd9f6fb-608d-4ce5-8718-fb41ea27aea2", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-blockalign-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.007475, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "C" }, "group_id": "blockalign-center-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-center-dark-1.png", "input_tokens": 2576, "key_errors": null, "latency_sec": 2.127837276086211, "model_name": "pareto", "output_tokens": 138, "prediction": { "choice": "C" }, "prompt_sha256": "7d6254d951604c657e71aeb7cae9178b26bc4d3dd7020cdf265225567557e5dd", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-29T14:17:57.769309+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.007475, "created_at": "2026-09-29T14:17:57.769Z" }, { "sequence": 258, "attempt_id": "45e6f984-a16c-4959-ac89-4bcd891b5a7b", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-direction-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0067425, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "A" }, "group_id": "direction-row-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-row-dark-1.png", "input_tokens": 2547, "key_errors": null, "latency_sec": 1.7810401120223105, "model_name": "pareto", "output_tokens": 50, "prediction": { "choice": "A" }, "prompt_sha256": "84ba6fd0e7012086e1f258af7368901e4550a2b9a6e285f06f25a8ab71464ec2", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-29T14:17:58.164703+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0067425, "created_at": "2026-09-29T14:17:58.164Z" }, { "sequence": 259, "attempt_id": "2c2b9718-fd6f-418e-b690-72f89fb4f722", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-textalign-008", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01004, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "B" }, "group_id": "textalign-center-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-center-dark-1.png", "input_tokens": 2597, "key_errors": null, "latency_sec": 3.5573331071063876, "model_name": "pareto", "output_tokens": 473, "prediction": { "choice": "B" }, "prompt_sha256": "2e47c2a304900033c1e7233c922b86d826e6949f29106d3fa5c967cc727df497", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-29T14:17:58.550127+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.01004, "created_at": "2026-09-29T14:17:58.550Z" }, { "sequence": 260, "attempt_id": "59360efa-07bd-4266-9e93-0c614fc84a42", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-gridrows-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006905, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "B" }, "group_id": "gridrows-4-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-4-light-0.png", "input_tokens": 2549, "key_errors": null, "latency_sec": 1.5211307471618056, "model_name": "pareto", "output_tokens": 71, "prediction": { "choice": "B" }, "prompt_sha256": "da4194e14f8d3c86eb007c251e2f977baeaf16f3ed67610f233d5aaa1e16745d", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-29T14:17:58.927614+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.006905, "created_at": "2026-09-29T14:17:58.927Z" }, { "sequence": 261, "attempt_id": "2c6cceb9-7076-4882-adbe-1b8e6ee07ab0", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-textcolumns-007", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006595, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "A" }, "group_id": "textcolumns-2-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-2-dark-0.png", "input_tokens": 2545, "key_errors": null, "latency_sec": 1.794996763812378, "model_name": "pareto", "output_tokens": 31, "prediction": { "choice": "A" }, "prompt_sha256": "78899f74135f81158adab4e7446b61a7bda49f0ad810f8b42dad28a3640486f6", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-29T14:17:58.984417+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.006595, "created_at": "2026-09-29T14:17:58.984Z" }, { "sequence": 262, "attempt_id": "831740c8-dd46-46ff-8a72-8ed337c1feb8", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-wrapcount-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0075425, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "B" }, "group_id": "wrapcount-1-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-1-dark-1.png", "input_tokens": 2546, "key_errors": null, "latency_sec": 1.7209018000867218, "model_name": "pareto", "output_tokens": 157, "prediction": { "choice": "B" }, "prompt_sha256": "f426cc4a860aebf2f83ce2ed8bb35c244b740a3f0e0196bacf7e87e4c01825c0", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-29T14:17:59.080979+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0075425, "created_at": "2026-09-29T14:17:59.081Z" }, { "sequence": 263, "attempt_id": "4228d0b0-02f8-4f07-b2f8-286dc95202b3", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-groupgap-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0078375, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "B" }, "group_id": "groupgap-between-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-between-light-1.png", "input_tokens": 2583, "key_errors": null, "latency_sec": 2.5406151991337538, "model_name": "pareto", "output_tokens": 184, "prediction": { "choice": "B" }, "prompt_sha256": "b5587c95ce8398dca0e206a8a222a0859cb2781184dde718bbe27409262b14c1", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-29T14:17:59.115491+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0078375, "created_at": "2026-09-29T14:17:59.116Z" }, { "sequence": 264, "attempt_id": "7124d981-a63b-4571-b8f2-a2d60c7d37bc", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-nestedflow-018", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0071725, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-18", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-18.png", "input_tokens": 2569, "key_errors": null, "latency_sec": 1.9802759170997888, "model_name": "pareto", "output_tokens": 100, "prediction": { "choice": "C" }, "prompt_sha256": "45c0efef1d7925275aea025bb1909f609ffc45577974778b58b34f1829f807e2", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-29T14:17:59.521076+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-018", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0071725, "created_at": "2026-09-29T14:17:59.521Z" }, { "sequence": 265, "attempt_id": "69858c0c-85d3-45cd-bcaa-30961879bc43", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-textcolumns-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0075325, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "B" }, "group_id": "textcolumns-1-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-1-dark-0.png", "input_tokens": 2545, "key_errors": null, "latency_sec": 2.103205655934289, "model_name": "pareto", "output_tokens": 156, "prediction": { "choice": "B" }, "prompt_sha256": "78899f74135f81158adab4e7446b61a7bda49f0ad810f8b42dad28a3640486f6", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-29T14:17:59.749607+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0075325, "created_at": "2026-09-29T14:17:59.749Z" }, { "sequence": 266, "attempt_id": "a33f576c-049b-4d14-b04a-c84cbfde1500", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-topflow-010", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0067525, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-10", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-10.png", "input_tokens": 2566, "key_errors": null, "latency_sec": 2.241855821106583, "model_name": "pareto", "output_tokens": 45, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-29T14:18:00.825674+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0067525, "created_at": "2026-09-29T14:18:00.825Z" }, { "sequence": 267, "attempt_id": "17bd889c-1ac4-4809-9f4c-d4d0675909ec", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-nestedflow-015", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006805, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-15", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-15.png", "input_tokens": 2569, "key_errors": null, "latency_sec": 1.731446553952992, "model_name": "pareto", "output_tokens": 51, "prediction": { "choice": "C" }, "prompt_sha256": "ac0c30958153584a4b94fe042ea1cbf718f020f9f66dd474a09c514effd2f9f9", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-29T14:18:00.882055+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-015", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.006805, "created_at": "2026-09-29T14:18:00.882Z" }, { "sequence": 268, "attempt_id": "ac6480a6-c28a-4042-a6c0-172a799aa18f", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-distribution-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01343, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "F" }, "group_id": "distribution-end-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-end-dark-1.png", "input_tokens": 2648, "key_errors": null, "latency_sec": 5.653783267131075, "model_name": "pareto", "output_tokens": 908, "prediction": { "choice": "F" }, "prompt_sha256": "ba31519a816a02b7e10e25462cd6a6242eafaf332f7ae72c4e8dc04ba3caa324", "provider": "anthropic", "raw_prediction": "{\"choice\": \"F\"}", "recorded_at": "2026-09-29T14:18:01.260777+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.01343, "created_at": "2026-09-29T14:18:01.260Z" }, { "sequence": 269, "attempt_id": "6e7518f1-f155-417c-ae26-b33e8aa033bc", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-nestedwrap-014", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.00959, "err": null, "error": "Expecting value: line 1 column 1 (char 0)", "error_kind": "invalid_response", "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-30", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-30.png", "input_tokens": 2579, "key_errors": null, "latency_sec": 3.653300137957558, "model_name": "pareto", "output_tokens": 419, "prediction": {}, "prompt_sha256": "a242868807b1158889b152c7c9664c235ebd10352a9a4a541f0a0af30a7fb4a5", "provider": "anthropic", "raw_prediction": "C", "recorded_at": "2026-09-29T14:18:01.454066+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-nestedwrap-014", "tight_score": null, "unmetered_attempts": 0, "valid": false, "within_bands": null }, "cost_usd": 0.00959, "created_at": "2026-09-29T14:18:01.454Z" }, { "sequence": 270, "attempt_id": "f5735611-1287-4ed0-9651-8b5c937cda25", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-gridcols-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0068075, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "C" }, "group_id": "gridcols-2-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-2-dark-1.png", "input_tokens": 2549, "key_errors": null, "latency_sec": 2.014193987008184, "model_name": "pareto", "output_tokens": 58, "prediction": { "choice": "C" }, "prompt_sha256": "69a0dbb7b95e4d65f59a6c7934d34ff8a7253b73dbb6ab0dfaf6ce730257753e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-29T14:18:01.568809+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0068075, "created_at": "2026-09-29T14:18:01.568Z" }, { "sequence": 271, "attempt_id": "2aa55ccb-0fde-4c47-975b-4ab4544aba51", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-nestedwrap-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.008315, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-10", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-10.png", "input_tokens": 2579, "key_errors": null, "latency_sec": 2.8175799809396267, "model_name": "pareto", "output_tokens": 249, "prediction": { "choice": "B" }, "prompt_sha256": "a10b800c7bde04b869b770c30fd7ed4b80b25795d3668fcd2fd6f199bd9b600e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-29T14:18:01.777512+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.008315, "created_at": "2026-09-29T14:18:01.777Z" }, { "sequence": 272, "attempt_id": "08950d49-41fa-47b2-8912-7e2892d1a6c6", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-wrapalign-005", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.007605, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "A" }, "group_id": "wrapalign-center-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-center-light-0.png", "input_tokens": 2568, "key_errors": null, "latency_sec": 2.156712754862383, "model_name": "pareto", "output_tokens": 158, "prediction": { "choice": "A" }, "prompt_sha256": "3df695a5de14c72a5a5dd88e4cced5a400dc2307a605c97919c60104bfb7a338", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-29T14:18:01.938903+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.007605, "created_at": "2026-09-29T14:18:01.939Z" }, { "sequence": 273, "attempt_id": "37574653-dbae-4e8d-9a86-6a35dbd57de2", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-distribution-002", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0095975, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "F" }, "group_id": "distribution-start-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-start-light-1.png", "input_tokens": 2648, "key_errors": null, "latency_sec": 3.5013921468053013, "model_name": "pareto", "output_tokens": 397, "prediction": { "choice": "F" }, "prompt_sha256": "a0ccf33b0dbbc426b80b82e74fdfbafb7cd7dff4573b6bb132185c8b476a57b2", "provider": "anthropic", "raw_prediction": "{\"choice\":\"F\"}", "recorded_at": "2026-09-29T14:18:02.517458+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0095975, "created_at": "2026-09-29T14:18:02.517Z" }, { "sequence": 274, "attempt_id": "190d7c65-6044-4ca4-9c6a-f406799324cf", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-distribution-015", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0097175, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "C" }, "group_id": "distribution-between-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-between-dark-0.png", "input_tokens": 2648, "key_errors": null, "latency_sec": 3.426031371811405, "model_name": "pareto", "output_tokens": 413, "prediction": { "choice": "C" }, "prompt_sha256": "2844fd3c7e35599c196ac5243ab693c9f28551ca0f691e83a1b632f51c83444b", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-29T14:18:02.561030+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-015", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0097175, "created_at": "2026-09-29T14:18:02.562Z" }, { "sequence": 275, "attempt_id": "68697252-6180-46cd-a044-da5820df1cce", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-distribution-016", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.013145, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "C" }, "group_id": "distribution-between-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-between-dark-1.png", "input_tokens": 2648, "key_errors": null, "latency_sec": 4.8105664409231395, "model_name": "pareto", "output_tokens": 870, "prediction": { "choice": "C" }, "prompt_sha256": "18d8a135f2c101591d404cbfe01f629ac78319d2093678f977ecd0929055dab8", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-29T14:18:03.008670+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-016", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.013145, "created_at": "2026-09-29T14:18:03.008Z" }, { "sequence": 276, "attempt_id": "68381e9f-1a3f-4801-883a-9b64dc94441b", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-gridtracks-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0072775, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "C" }, "group_id": "gridtracks-right-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-right-light-0.png", "input_tokens": 2569, "key_errors": null, "latency_sec": 2.2905130598228425, "model_name": "pareto", "output_tokens": 114, "prediction": { "choice": "C" }, "prompt_sha256": "aa2cac59ecf81aa683f25bd6bee52028be6daa4a3ce7c514aca8929ad1d53167", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-29T14:18:03.150860+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0072775, "created_at": "2026-09-29T14:18:03.150Z" }, { "sequence": 277, "attempt_id": "2a756d4b-8a56-45cb-b7ff-072a2735d8fa", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-widthcompare-003", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0071875, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "A" }, "group_id": "widthcompare-first-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-first-dark-0.png", "input_tokens": 2560, "key_errors": null, "latency_sec": 1.625844496069476, "model_name": "pareto", "output_tokens": 105, "prediction": { "choice": "A" }, "prompt_sha256": "756f6b3e431f583d190c4d469c2409153beb0d173c07c749e93a1af92d2e6f8e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-29T14:18:03.437865+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0071875, "created_at": "2026-09-29T14:18:03.438Z" }, { "sequence": 278, "attempt_id": "c226d164-526f-4ff8-ab21-f6113a157539", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-nestedflow-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0068725, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-01", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-01.png", "input_tokens": 2569, "key_errors": null, "latency_sec": 1.9859295259229839, "model_name": "pareto", "output_tokens": 60, "prediction": { "choice": "A" }, "prompt_sha256": "4cd5f1bb86e36238445efff79cfd13b01ed6916cd094da4a341b7112a2f75bfe", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-29T14:18:03.473428+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0068725, "created_at": "2026-09-29T14:18:03.474Z" }, { "sequence": 279, "attempt_id": "cddf4388-7b19-4f8f-9cfc-96cd7f7d33e6", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-wrapalign-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.008175, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "A" }, "group_id": "wrapalign-right-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-right-dark-1.png", "input_tokens": 2568, "key_errors": null, "latency_sec": 2.265217151027173, "model_name": "pareto", "output_tokens": 234, "prediction": { "choice": "A" }, "prompt_sha256": "6f9f40796910da778fa5f30bd2cb43da95875ca8d6fb47db39f42ffb70cbab76", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-29T14:18:03.559274+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.008175, "created_at": "2026-09-29T14:18:03.559Z" }, { "sequence": 280, "attempt_id": "2f8d0219-b1cb-4366-a45a-acf782520335", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-gapcompare-012", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.00755, "err": null, "error": "Expecting value: line 1 column 1 (char 0)", "error_kind": "invalid_response", "family": "gapcompare", "ground_truth": { "choice": "C" }, "group_id": "gapcompare-equal-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-equal-dark-1.png", "input_tokens": 2567, "key_errors": null, "latency_sec": 2.1686771949753165, "model_name": "pareto", "output_tokens": 151, "prediction": {}, "prompt_sha256": "e88ed59689802f6ceda5c2ff1f4a55b2556783dabe7507fad3d3173dad609864", "provider": "anthropic", "raw_prediction": "C", "recorded_at": "2026-09-29T14:18:03.771393+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-gapcompare-012", "tight_score": null, "unmetered_attempts": 0, "valid": false, "within_bands": null }, "cost_usd": 0.00755, "created_at": "2026-09-29T14:18:03.771Z" }, { "sequence": 281, "attempt_id": "8e73a5a9-7d8a-4413-88e3-0e1adba99f1a", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-topflow-013", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0066775, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-13", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-13.png", "input_tokens": 2566, "key_errors": null, "latency_sec": 1.59511176077649, "model_name": "pareto", "output_tokens": 35, "prediction": { "choice": "A" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-29T14:18:04.204763+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-013", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0066775, "created_at": "2026-09-29T14:18:04.204Z" }, { "sequence": 282, "attempt_id": "aae690a5-1945-4089-9c39-377a458cb273", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-distribution-013", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009815, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "E" }, "group_id": "distribution-between-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-between-light-0.png", "input_tokens": 2648, "key_errors": null, "latency_sec": 3.339141472009942, "model_name": "pareto", "output_tokens": 426, "prediction": { "choice": "E" }, "prompt_sha256": "23da23350386069bba58b0668e6b5df5af67fac05943b8826730fffd8ce8e9ef", "provider": "anthropic", "raw_prediction": "{\"choice\": \"E\"}", "recorded_at": "2026-09-29T14:18:04.253783+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-013", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.009815, "created_at": "2026-09-29T14:18:04.253Z" }, { "sequence": 283, "attempt_id": "dbfa7cd4-1167-47d8-a3a3-ec4fde5aaf66", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-nestedflow-012", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00697, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-12", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-12.png", "input_tokens": 2569, "key_errors": null, "latency_sec": 1.772068478167057, "model_name": "pareto", "output_tokens": 73, "prediction": { "choice": "A" }, "prompt_sha256": "ac0c30958153584a4b94fe042ea1cbf718f020f9f66dd474a09c514effd2f9f9", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-29T14:18:04.816442+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00697, "created_at": "2026-09-29T14:18:04.816Z" }, { "sequence": 284, "attempt_id": "d7b78109-6646-402f-b0cb-ba5427791a75", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-widthcompare-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0071875, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "A" }, "group_id": "widthcompare-last-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-last-light-1.png", "input_tokens": 2560, "key_errors": null, "latency_sec": 1.5955105340108275, "model_name": "pareto", "output_tokens": 105, "prediction": { "choice": "A" }, "prompt_sha256": "807b859493212c8c914db2b1d9395b00171e3c91df87b66b9663b6cc7b65c485", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-29T14:18:05.107540+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0071875, "created_at": "2026-09-29T14:18:05.107Z" }, { "sequence": 285, "attempt_id": "20dd29cd-7235-42ae-a016-823febe06e50", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-nestedwrap-016", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0081725, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-32", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-32.png", "input_tokens": 2579, "key_errors": null, "latency_sec": 2.7227164970245212, "model_name": "pareto", "output_tokens": 230, "prediction": { "choice": "B" }, "prompt_sha256": "3ba51907b89444fcea345fda34c9692faefd56e2b5c6671b282c0da644fdccb7", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-29T14:18:05.284049+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-016", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0081725, "created_at": "2026-09-29T14:18:05.284Z" }, { "sequence": 286, "attempt_id": "2f173f88-19d6-463f-8bbd-2d4a24db2208", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-gridspan-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.008545, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "C" }, "group_id": "gridspan-1-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-1-dark-1.png", "input_tokens": 2572, "key_errors": null, "latency_sec": 3.3536428748629987, "model_name": "pareto", "output_tokens": 282, "prediction": { "choice": "C" }, "prompt_sha256": "4c2619c1f3339203e066968a6911df07e41f49d553c6c3cb2be0b1e81e76a29b", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-29T14:18:05.325735+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.008545, "created_at": "2026-09-29T14:18:05.325Z" }, { "sequence": 287, "attempt_id": "0db3fb03-3233-4776-b227-131ed4a932f3", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-topflow-017", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0068875, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-17", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-17.png", "input_tokens": 2566, "key_errors": null, "latency_sec": 1.9581228950992227, "model_name": "pareto", "output_tokens": 63, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-29T14:18:05.431804+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-017", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0068875, "created_at": "2026-09-29T14:18:05.431Z" }, { "sequence": 288, "attempt_id": "f7d1a905-2e78-449d-9fe5-7c8aac63795f", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-crossalign-009", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.007235, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "A" }, "group_id": "crossalign-end-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-end-light-0.png", "input_tokens": 2642, "key_errors": null, "latency_sec": 1.7279513750690967, "model_name": "pareto", "output_tokens": 84, "prediction": { "choice": "A" }, "prompt_sha256": "42e3d5b05e35df09d8f58ddf18bbea16275da96264b53e5005136200c350e010", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-29T14:18:05.533884+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.007235, "created_at": "2026-09-29T14:18:05.534Z" }, { "sequence": 289, "attempt_id": "ba14ab27-7690-4eef-a022-6f542d982264", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-groupgap-001", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.007965, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "B" }, "group_id": "groupgap-between-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-between-light-0.png", "input_tokens": 2583, "key_errors": null, "latency_sec": 2.279297956964001, "model_name": "pareto", "output_tokens": 201, "prediction": { "choice": "B" }, "prompt_sha256": "a4ae0e9e5d9435b02d3759ecf1850c36024f893c4de107ea8256f2c5052eaa53", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-29T14:18:05.871791+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.007965, "created_at": "2026-09-29T14:18:05.871Z" }, { "sequence": 290, "attempt_id": "694037d6-441f-45a0-b3c3-6c1ccc56206b", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-textalign-004", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0083375, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "D" }, "group_id": "textalign-left-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-left-dark-1.png", "input_tokens": 2597, "key_errors": null, "latency_sec": 2.9426800389774144, "model_name": "pareto", "output_tokens": 246, "prediction": { "choice": "D" }, "prompt_sha256": "02fcd4a4c952217ee96fdbbca5e92af61005f02abe9611e017f571e23cb193e2", "provider": "anthropic", "raw_prediction": "{\"choice\": \"D\"}", "recorded_at": "2026-09-29T14:18:06.127817+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.0083375, "created_at": "2026-09-29T14:18:06.127Z" }, { "sequence": 291, "attempt_id": "6af07903-0bfc-46b4-8d91-aaa76c6ec742", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-textalign-012", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.00806, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "A" }, "group_id": "textalign-right-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-right-dark-1.png", "input_tokens": 2597, "key_errors": null, "latency_sec": 2.3561330360826105, "model_name": "pareto", "output_tokens": 209, "prediction": { "choice": "D" }, "prompt_sha256": "ee3ac587972345495a05631df96b48614d67cf083b60577be8181fd19c9043fd", "provider": "anthropic", "raw_prediction": "{\"choice\": \"D\"}", "recorded_at": "2026-09-29T14:18:06.604381+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-textalign-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00806, "created_at": "2026-09-29T14:18:06.604Z" }, { "sequence": 292, "attempt_id": "8a7a7b88-8c20-4290-a009-e5580cf17637", "run_id": "pareto-20260929", "model_id": "pareto", "task_id": "layoutbench-padcompare-006", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00862, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "C" }, "group_id": "padcompare-vertical-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-vertical-light-1.png", "input_tokens": 2593, "key_errors": null, "latency_sec": 2.3559132169466466, "model_name": "pareto", "output_tokens": 285, "prediction": { "choice": "C" }, "prompt_sha256": "07819e1c7e8686a5ffdde745114a9c6e8df1c7ac485dc319c5c72c24e1b67dae", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-29T14:18:06.644579+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null }, "cost_usd": 0.00862, "created_at": "2026-09-29T14:18:06.644Z" } ], "specimens": [ { "task_id": "layoutbench-direction-001", "family": "direction", "group_id": "direction-row-light-0", "image_filename": "direction-row-light-0.png", "image_sha256": "4727c71ba70b443edf13136f7c7219a32ba1e65babccee09327f3918fee6ebf6", "ground_truth": { "choice": "B" }, "prompt": "How are the cards inside the outlined frame arranged?\nA: Stacked in one vertical column\nB: Side by side in one horizontal row\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 0, "answer": "row", "options": [ "column", "row" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-direction-002", "family": "direction", "group_id": "direction-row-light-1", "image_filename": "direction-row-light-1.png", "image_sha256": "fd32b6baf1afbd4af974ea300b59601ffa3f2b26ef50baef8dc56cf1351752c1", "ground_truth": { "choice": "A" }, "prompt": "How are the cards inside the outlined frame arranged?\nA: Side by side in one horizontal row\nB: Stacked in one vertical column\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 1, "answer": "row", "options": [ "row", "column" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-direction-003", "family": "direction", "group_id": "direction-row-dark-0", "image_filename": "direction-row-dark-0.png", "image_sha256": "e1054056f7333f3d07d4db078449b382741a50c1967ea87d3873042aeedd21b1", "ground_truth": { "choice": "B" }, "prompt": "How are the cards inside the outlined frame arranged?\nA: Stacked in one vertical column\nB: Side by side in one horizontal row\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 0, "answer": "row", "options": [ "column", "row" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-direction-004", "family": "direction", "group_id": "direction-row-dark-1", "image_filename": "direction-row-dark-1.png", "image_sha256": "5d5d514272958fc67e30dfb635e8303dfec36af16d578450c675d558b918289a", "ground_truth": { "choice": "A" }, "prompt": "How are the cards inside the outlined frame arranged?\nA: Side by side in one horizontal row\nB: Stacked in one vertical column\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 1, "answer": "row", "options": [ "row", "column" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-direction-005", "family": "direction", "group_id": "direction-column-light-0", "image_filename": "direction-column-light-0.png", "image_sha256": "d21144267d483280335db6abe4118b89b6b54f3abdc03c4fc9642a782294cfe8", "ground_truth": { "choice": "B" }, "prompt": "How are the cards inside the outlined frame arranged?\nA: Side by side in one horizontal row\nB: Stacked in one vertical column\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 0, "answer": "column", "options": [ "row", "column" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-direction-006", "family": "direction", "group_id": "direction-column-light-1", "image_filename": "direction-column-light-1.png", "image_sha256": "39fb86e81eb182c6ed20beacb763e3fd5b3fb3b4a2791e22d8d44c41725de7a3", "ground_truth": { "choice": "A" }, "prompt": "How are the cards inside the outlined frame arranged?\nA: Stacked in one vertical column\nB: Side by side in one horizontal row\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 1, "answer": "column", "options": [ "column", "row" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-direction-007", "family": "direction", "group_id": "direction-column-dark-0", "image_filename": "direction-column-dark-0.png", "image_sha256": "f6d975c91e1cfa5205f4bd29576f5a82451dd51773c94d735d4a878702c812e2", "ground_truth": { "choice": "A" }, "prompt": "How are the cards inside the outlined frame arranged?\nA: Stacked in one vertical column\nB: Side by side in one horizontal row\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 0, "answer": "column", "options": [ "column", "row" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-direction-008", "family": "direction", "group_id": "direction-column-dark-1", "image_filename": "direction-column-dark-1.png", "image_sha256": "1666a5a779971149d3acca75b3f20edfab2f6ca286e8f7d76d53825ceb8001a7", "ground_truth": { "choice": "B" }, "prompt": "How are the cards inside the outlined frame arranged?\nA: Side by side in one horizontal row\nB: Stacked in one vertical column\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 1, "answer": "column", "options": [ "row", "column" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-distribution-001", "family": "distribution", "group_id": "distribution-start-light-0", "image_filename": "distribution-start-light-0.png", "image_sha256": "6934f78a7a7500fa45f29d87d1be1c389ca15393173d7b916a6b76e77434b623", "ground_truth": { "choice": "D" }, "prompt": "Consider the sequence of cards along its long direction inside the outlined frame. Start means left for a row and top for a column. Which description fits the empty spaces measured from card edges to the INSIDE of the frame border?\nA: Packed at the end, touching that inner edge\nB: All interior gaps and both outer spaces are equal\nC: First and last cards touch opposite inner edges; equal gaps between cards\nD: Packed at the start, touching that inner edge\nE: Equal outer spaces, each half the size of an interior gap\nF: Packed together in the middle, with much larger equal outer spaces\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 0, "answer": "start", "options": [ "end", "evenly", "between", "start", "around", "center" ], "factors": { "direction": "row" }, "lines": {} } }, { "task_id": "layoutbench-distribution-002", "family": "distribution", "group_id": "distribution-start-light-1", "image_filename": "distribution-start-light-1.png", "image_sha256": "560524dd93b7f70b5c26f8209854f407b5a654a4ba9891ae330f59b45175786e", "ground_truth": { "choice": "F" }, "prompt": "Consider the sequence of cards along its long direction inside the outlined frame. Start means left for a row and top for a column. Which description fits the empty spaces measured from card edges to the INSIDE of the frame border?\nA: Equal outer spaces, each half the size of an interior gap\nB: All interior gaps and both outer spaces are equal\nC: Packed at the end, touching that inner edge\nD: First and last cards touch opposite inner edges; equal gaps between cards\nE: Packed together in the middle, with much larger equal outer spaces\nF: Packed at the start, touching that inner edge\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 1, "answer": "start", "options": [ "around", "evenly", "end", "between", "center", "start" ], "factors": { "direction": "column" }, "lines": {} } }, { "task_id": "layoutbench-distribution-003", "family": "distribution", "group_id": "distribution-start-dark-0", "image_filename": "distribution-start-dark-0.png", "image_sha256": "790c3146f9dc58d06c1996bad5505080ae3a558de99ae89f730a8ea9b630e44f", "ground_truth": { "choice": "B" }, "prompt": "Consider the sequence of cards along its long direction inside the outlined frame. Start means left for a row and top for a column. Which description fits the empty spaces measured from card edges to the INSIDE of the frame border?\nA: Packed at the end, touching that inner edge\nB: Packed at the start, touching that inner edge\nC: First and last cards touch opposite inner edges; equal gaps between cards\nD: Equal outer spaces, each half the size of an interior gap\nE: All interior gaps and both outer spaces are equal\nF: Packed together in the middle, with much larger equal outer spaces\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 0, "answer": "start", "options": [ "end", "start", "between", "around", "evenly", "center" ], "factors": { "direction": "row" }, "lines": {} } }, { "task_id": "layoutbench-distribution-004", "family": "distribution", "group_id": "distribution-start-dark-1", "image_filename": "distribution-start-dark-1.png", "image_sha256": "d864575a8d583a9d95d519e3bd43dee0aea9682de4157f2e0ee81039d193d5ae", "ground_truth": { "choice": "D" }, "prompt": "Consider the sequence of cards along its long direction inside the outlined frame. Start means left for a row and top for a column. Which description fits the empty spaces measured from card edges to the INSIDE of the frame border?\nA: All interior gaps and both outer spaces are equal\nB: Packed together in the middle, with much larger equal outer spaces\nC: Packed at the end, touching that inner edge\nD: Packed at the start, touching that inner edge\nE: First and last cards touch opposite inner edges; equal gaps between cards\nF: Equal outer spaces, each half the size of an interior gap\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 1, "answer": "start", "options": [ "evenly", "center", "end", "start", "between", "around" ], "factors": { "direction": "column" }, "lines": {} } }, { "task_id": "layoutbench-distribution-005", "family": "distribution", "group_id": "distribution-center-light-0", "image_filename": "distribution-center-light-0.png", "image_sha256": "13c3926f98cda2973153eb0ba3bedd05993b22cd5dd6cea1cd3b3c131f6682ad", "ground_truth": { "choice": "E" }, "prompt": "Consider the sequence of cards along its long direction inside the outlined frame. Start means left for a row and top for a column. Which description fits the empty spaces measured from card edges to the INSIDE of the frame border?\nA: Equal outer spaces, each half the size of an interior gap\nB: Packed at the start, touching that inner edge\nC: First and last cards touch opposite inner edges; equal gaps between cards\nD: All interior gaps and both outer spaces are equal\nE: Packed together in the middle, with much larger equal outer spaces\nF: Packed at the end, touching that inner edge\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 0, "answer": "center", "options": [ "around", "start", "between", "evenly", "center", "end" ], "factors": { "direction": "row" }, "lines": {} } }, { "task_id": "layoutbench-distribution-006", "family": "distribution", "group_id": "distribution-center-light-1", "image_filename": "distribution-center-light-1.png", "image_sha256": "8bd42fb49d0f74f0c78e00cf29761b4a502708d81ed47075c6c3c679ed917065", "ground_truth": { "choice": "E" }, "prompt": "Consider the sequence of cards along its long direction inside the outlined frame. Start means left for a row and top for a column. Which description fits the empty spaces measured from card edges to the INSIDE of the frame border?\nA: All interior gaps and both outer spaces are equal\nB: Packed at the end, touching that inner edge\nC: First and last cards touch opposite inner edges; equal gaps between cards\nD: Packed at the start, touching that inner edge\nE: Packed together in the middle, with much larger equal outer spaces\nF: Equal outer spaces, each half the size of an interior gap\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 1, "answer": "center", "options": [ "evenly", "end", "between", "start", "center", "around" ], "factors": { "direction": "column" }, "lines": {} } }, { "task_id": "layoutbench-distribution-007", "family": "distribution", "group_id": "distribution-center-dark-0", "image_filename": "distribution-center-dark-0.png", "image_sha256": "17a87c1e7fa01a4df18a1aff869fafcb6802f07f1e5bcd2a346ed8e438bff341", "ground_truth": { "choice": "B" }, "prompt": "Consider the sequence of cards along its long direction inside the outlined frame. Start means left for a row and top for a column. Which description fits the empty spaces measured from card edges to the INSIDE of the frame border?\nA: Equal outer spaces, each half the size of an interior gap\nB: Packed together in the middle, with much larger equal outer spaces\nC: First and last cards touch opposite inner edges; equal gaps between cards\nD: All interior gaps and both outer spaces are equal\nE: Packed at the start, touching that inner edge\nF: Packed at the end, touching that inner edge\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 0, "answer": "center", "options": [ "around", "center", "between", "evenly", "start", "end" ], "factors": { "direction": "row" }, "lines": {} } }, { "task_id": "layoutbench-distribution-008", "family": "distribution", "group_id": "distribution-center-dark-1", "image_filename": "distribution-center-dark-1.png", "image_sha256": "52dfc295a559681a6d9dc1869036b888253a906071c83f6bdc5d06a79b0303bb", "ground_truth": { "choice": "D" }, "prompt": "Consider the sequence of cards along its long direction inside the outlined frame. Start means left for a row and top for a column. Which description fits the empty spaces measured from card edges to the INSIDE of the frame border?\nA: Packed at the end, touching that inner edge\nB: Equal outer spaces, each half the size of an interior gap\nC: First and last cards touch opposite inner edges; equal gaps between cards\nD: Packed together in the middle, with much larger equal outer spaces\nE: All interior gaps and both outer spaces are equal\nF: Packed at the start, touching that inner edge\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 1, "answer": "center", "options": [ "end", "around", "between", "center", "evenly", "start" ], "factors": { "direction": "column" }, "lines": {} } }, { "task_id": "layoutbench-distribution-009", "family": "distribution", "group_id": "distribution-end-light-0", "image_filename": "distribution-end-light-0.png", "image_sha256": "0f1f44e01f09213d9b8bece93b73369403c328ce4e1e6940178615f2b86b4377", "ground_truth": { "choice": "A" }, "prompt": "Consider the sequence of cards along its long direction inside the outlined frame. Start means left for a row and top for a column. Which description fits the empty spaces measured from card edges to the INSIDE of the frame border?\nA: Packed at the end, touching that inner edge\nB: Packed at the start, touching that inner edge\nC: Equal outer spaces, each half the size of an interior gap\nD: All interior gaps and both outer spaces are equal\nE: First and last cards touch opposite inner edges; equal gaps between cards\nF: Packed together in the middle, with much larger equal outer spaces\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 0, "answer": "end", "options": [ "end", "start", "around", "evenly", "between", "center" ], "factors": { "direction": "row" }, "lines": {} } }, { "task_id": "layoutbench-distribution-010", "family": "distribution", "group_id": "distribution-end-light-1", "image_filename": "distribution-end-light-1.png", "image_sha256": "53bb5e19abdd405e840d6ced432488840f4df2a5f6b756a3b85f373891e1e9eb", "ground_truth": { "choice": "E" }, "prompt": "Consider the sequence of cards along its long direction inside the outlined frame. Start means left for a row and top for a column. Which description fits the empty spaces measured from card edges to the INSIDE of the frame border?\nA: Packed at the start, touching that inner edge\nB: First and last cards touch opposite inner edges; equal gaps between cards\nC: All interior gaps and both outer spaces are equal\nD: Packed together in the middle, with much larger equal outer spaces\nE: Packed at the end, touching that inner edge\nF: Equal outer spaces, each half the size of an interior gap\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 1, "answer": "end", "options": [ "start", "between", "evenly", "center", "end", "around" ], "factors": { "direction": "column" }, "lines": {} } }, { "task_id": "layoutbench-distribution-011", "family": "distribution", "group_id": "distribution-end-dark-0", "image_filename": "distribution-end-dark-0.png", "image_sha256": "34d41ee3ce9e087fe3964190afacf247479019abaf31ebc0951e6ae733b789da", "ground_truth": { "choice": "A" }, "prompt": "Consider the sequence of cards along its long direction inside the outlined frame. Start means left for a row and top for a column. Which description fits the empty spaces measured from card edges to the INSIDE of the frame border?\nA: Packed at the end, touching that inner edge\nB: Equal outer spaces, each half the size of an interior gap\nC: All interior gaps and both outer spaces are equal\nD: First and last cards touch opposite inner edges; equal gaps between cards\nE: Packed at the start, touching that inner edge\nF: Packed together in the middle, with much larger equal outer spaces\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 0, "answer": "end", "options": [ "end", "around", "evenly", "between", "start", "center" ], "factors": { "direction": "row" }, "lines": {} } }, { "task_id": "layoutbench-distribution-012", "family": "distribution", "group_id": "distribution-end-dark-1", "image_filename": "distribution-end-dark-1.png", "image_sha256": "aebd5f15e877aa03cdce4a3ec9c330f1fd9d0bb855b03c46294777c28a61a4b0", "ground_truth": { "choice": "F" }, "prompt": "Consider the sequence of cards along its long direction inside the outlined frame. Start means left for a row and top for a column. Which description fits the empty spaces measured from card edges to the INSIDE of the frame border?\nA: Packed at the start, touching that inner edge\nB: Packed together in the middle, with much larger equal outer spaces\nC: All interior gaps and both outer spaces are equal\nD: First and last cards touch opposite inner edges; equal gaps between cards\nE: Equal outer spaces, each half the size of an interior gap\nF: Packed at the end, touching that inner edge\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 1, "answer": "end", "options": [ "start", "center", "evenly", "between", "around", "end" ], "factors": { "direction": "column" }, "lines": {} } }, { "task_id": "layoutbench-distribution-013", "family": "distribution", "group_id": "distribution-between-light-0", "image_filename": "distribution-between-light-0.png", "image_sha256": "a23b333539717be6f0e6ddd82f7b50e5140e9848c24411f1aac5986ba15cb22d", "ground_truth": { "choice": "E" }, "prompt": "Consider the sequence of cards along its long direction inside the outlined frame. Start means left for a row and top for a column. Which description fits the empty spaces measured from card edges to the INSIDE of the frame border?\nA: Packed at the end, touching that inner edge\nB: Packed together in the middle, with much larger equal outer spaces\nC: Equal outer spaces, each half the size of an interior gap\nD: Packed at the start, touching that inner edge\nE: First and last cards touch opposite inner edges; equal gaps between cards\nF: All interior gaps and both outer spaces are equal\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 0, "answer": "between", "options": [ "end", "center", "around", "start", "between", "evenly" ], "factors": { "direction": "row" }, "lines": {} } }, { "task_id": "layoutbench-distribution-014", "family": "distribution", "group_id": "distribution-between-light-1", "image_filename": "distribution-between-light-1.png", "image_sha256": "6c346d11652388193a2cf6f477eac372490c7634ebe9cd08e1b1cab5bc7e067b", "ground_truth": { "choice": "A" }, "prompt": "Consider the sequence of cards along its long direction inside the outlined frame. Start means left for a row and top for a column. Which description fits the empty spaces measured from card edges to the INSIDE of the frame border?\nA: First and last cards touch opposite inner edges; equal gaps between cards\nB: Packed at the start, touching that inner edge\nC: Packed together in the middle, with much larger equal outer spaces\nD: All interior gaps and both outer spaces are equal\nE: Equal outer spaces, each half the size of an interior gap\nF: Packed at the end, touching that inner edge\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 1, "answer": "between", "options": [ "between", "start", "center", "evenly", "around", "end" ], "factors": { "direction": "column" }, "lines": {} } }, { "task_id": "layoutbench-distribution-015", "family": "distribution", "group_id": "distribution-between-dark-0", "image_filename": "distribution-between-dark-0.png", "image_sha256": "bbb7973d3d6383e82636e2e44b0d85832cce3c07ec05fb8782370843df3a95d1", "ground_truth": { "choice": "C" }, "prompt": "Consider the sequence of cards along its long direction inside the outlined frame. Start means left for a row and top for a column. Which description fits the empty spaces measured from card edges to the INSIDE of the frame border?\nA: Packed at the end, touching that inner edge\nB: All interior gaps and both outer spaces are equal\nC: First and last cards touch opposite inner edges; equal gaps between cards\nD: Equal outer spaces, each half the size of an interior gap\nE: Packed at the start, touching that inner edge\nF: Packed together in the middle, with much larger equal outer spaces\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 0, "answer": "between", "options": [ "end", "evenly", "between", "around", "start", "center" ], "factors": { "direction": "row" }, "lines": {} } }, { "task_id": "layoutbench-distribution-016", "family": "distribution", "group_id": "distribution-between-dark-1", "image_filename": "distribution-between-dark-1.png", "image_sha256": "2d1a97f606ce03901126522702df733c5d44eba62fd3ad5dce41960ca5b95beb", "ground_truth": { "choice": "C" }, "prompt": "Consider the sequence of cards along its long direction inside the outlined frame. Start means left for a row and top for a column. Which description fits the empty spaces measured from card edges to the INSIDE of the frame border?\nA: Equal outer spaces, each half the size of an interior gap\nB: Packed at the end, touching that inner edge\nC: First and last cards touch opposite inner edges; equal gaps between cards\nD: All interior gaps and both outer spaces are equal\nE: Packed together in the middle, with much larger equal outer spaces\nF: Packed at the start, touching that inner edge\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 1, "answer": "between", "options": [ "around", "end", "between", "evenly", "center", "start" ], "factors": { "direction": "column" }, "lines": {} } }, { "task_id": "layoutbench-distribution-017", "family": "distribution", "group_id": "distribution-around-light-0", "image_filename": "distribution-around-light-0.png", "image_sha256": "c8df91b1509d235baf54176aecad2860fd8e1f7d3b689e2f51b5a8fa1f0b5358", "ground_truth": { "choice": "F" }, "prompt": "Consider the sequence of cards along its long direction inside the outlined frame. Start means left for a row and top for a column. Which description fits the empty spaces measured from card edges to the INSIDE of the frame border?\nA: Packed at the start, touching that inner edge\nB: Packed at the end, touching that inner edge\nC: All interior gaps and both outer spaces are equal\nD: First and last cards touch opposite inner edges; equal gaps between cards\nE: Packed together in the middle, with much larger equal outer spaces\nF: Equal outer spaces, each half the size of an interior gap\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 0, "answer": "around", "options": [ "start", "end", "evenly", "between", "center", "around" ], "factors": { "direction": "row" }, "lines": {} } }, { "task_id": "layoutbench-distribution-018", "family": "distribution", "group_id": "distribution-around-light-1", "image_filename": "distribution-around-light-1.png", "image_sha256": "929150f26d59093cadf1318a5b1f7adc98d5eb94059522d1260fd864509097e1", "ground_truth": { "choice": "B" }, "prompt": "Consider the sequence of cards along its long direction inside the outlined frame. Start means left for a row and top for a column. Which description fits the empty spaces measured from card edges to the INSIDE of the frame border?\nA: First and last cards touch opposite inner edges; equal gaps between cards\nB: Equal outer spaces, each half the size of an interior gap\nC: All interior gaps and both outer spaces are equal\nD: Packed together in the middle, with much larger equal outer spaces\nE: Packed at the start, touching that inner edge\nF: Packed at the end, touching that inner edge\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 1, "answer": "around", "options": [ "between", "around", "evenly", "center", "start", "end" ], "factors": { "direction": "column" }, "lines": {} } }, { "task_id": "layoutbench-distribution-019", "family": "distribution", "group_id": "distribution-around-dark-0", "image_filename": "distribution-around-dark-0.png", "image_sha256": "93a4b61825ca5ee5a9c388742f7bc77ec952dc7aac5da336b7cb812d629bea58", "ground_truth": { "choice": "D" }, "prompt": "Consider the sequence of cards along its long direction inside the outlined frame. Start means left for a row and top for a column. Which description fits the empty spaces measured from card edges to the INSIDE of the frame border?\nA: First and last cards touch opposite inner edges; equal gaps between cards\nB: Packed at the start, touching that inner edge\nC: Packed at the end, touching that inner edge\nD: Equal outer spaces, each half the size of an interior gap\nE: Packed together in the middle, with much larger equal outer spaces\nF: All interior gaps and both outer spaces are equal\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 0, "answer": "around", "options": [ "between", "start", "end", "around", "center", "evenly" ], "factors": { "direction": "row" }, "lines": {} } }, { "task_id": "layoutbench-distribution-020", "family": "distribution", "group_id": "distribution-around-dark-1", "image_filename": "distribution-around-dark-1.png", "image_sha256": "c5bee743a5cdf0f55ad8f8e2ad4d0da59843024cf9f0520da9f216dd3e698a27", "ground_truth": { "choice": "F" }, "prompt": "Consider the sequence of cards along its long direction inside the outlined frame. Start means left for a row and top for a column. Which description fits the empty spaces measured from card edges to the INSIDE of the frame border?\nA: All interior gaps and both outer spaces are equal\nB: Packed together in the middle, with much larger equal outer spaces\nC: Packed at the start, touching that inner edge\nD: Packed at the end, touching that inner edge\nE: First and last cards touch opposite inner edges; equal gaps between cards\nF: Equal outer spaces, each half the size of an interior gap\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 1, "answer": "around", "options": [ "evenly", "center", "start", "end", "between", "around" ], "factors": { "direction": "column" }, "lines": {} } }, { "task_id": "layoutbench-distribution-021", "family": "distribution", "group_id": "distribution-evenly-light-0", "image_filename": "distribution-evenly-light-0.png", "image_sha256": "4b0fa8ccce73a4842ec16bf47515acc5e5ac08922350410eb12ff459a45e6510", "ground_truth": { "choice": "B" }, "prompt": "Consider the sequence of cards along its long direction inside the outlined frame. Start means left for a row and top for a column. Which description fits the empty spaces measured from card edges to the INSIDE of the frame border?\nA: First and last cards touch opposite inner edges; equal gaps between cards\nB: All interior gaps and both outer spaces are equal\nC: Packed at the end, touching that inner edge\nD: Equal outer spaces, each half the size of an interior gap\nE: Packed together in the middle, with much larger equal outer spaces\nF: Packed at the start, touching that inner edge\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 0, "answer": "evenly", "options": [ "between", "evenly", "end", "around", "center", "start" ], "factors": { "direction": "row" }, "lines": {} } }, { "task_id": "layoutbench-distribution-022", "family": "distribution", "group_id": "distribution-evenly-light-1", "image_filename": "distribution-evenly-light-1.png", "image_sha256": "43750ea9b51206622f9572020ec769262ab52a0a187b81d5f06e5f27b420418a", "ground_truth": { "choice": "C" }, "prompt": "Consider the sequence of cards along its long direction inside the outlined frame. Start means left for a row and top for a column. Which description fits the empty spaces measured from card edges to the INSIDE of the frame border?\nA: First and last cards touch opposite inner edges; equal gaps between cards\nB: Packed at the start, touching that inner edge\nC: All interior gaps and both outer spaces are equal\nD: Equal outer spaces, each half the size of an interior gap\nE: Packed together in the middle, with much larger equal outer spaces\nF: Packed at the end, touching that inner edge\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 1, "answer": "evenly", "options": [ "between", "start", "evenly", "around", "center", "end" ], "factors": { "direction": "column" }, "lines": {} } }, { "task_id": "layoutbench-distribution-023", "family": "distribution", "group_id": "distribution-evenly-dark-0", "image_filename": "distribution-evenly-dark-0.png", "image_sha256": "33cc4f41d7465f2ccc190c633d0d7d72468749ededb015c4f8094951e6de1c9f", "ground_truth": { "choice": "A" }, "prompt": "Consider the sequence of cards along its long direction inside the outlined frame. Start means left for a row and top for a column. Which description fits the empty spaces measured from card edges to the INSIDE of the frame border?\nA: All interior gaps and both outer spaces are equal\nB: Equal outer spaces, each half the size of an interior gap\nC: Packed together in the middle, with much larger equal outer spaces\nD: First and last cards touch opposite inner edges; equal gaps between cards\nE: Packed at the start, touching that inner edge\nF: Packed at the end, touching that inner edge\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 0, "answer": "evenly", "options": [ "evenly", "around", "center", "between", "start", "end" ], "factors": { "direction": "row" }, "lines": {} } }, { "task_id": "layoutbench-distribution-024", "family": "distribution", "group_id": "distribution-evenly-dark-1", "image_filename": "distribution-evenly-dark-1.png", "image_sha256": "a4233e3b1be0fd9f809700538a6fec7a18fa9a1028c25109dd2a5067d5f71c97", "ground_truth": { "choice": "C" }, "prompt": "Consider the sequence of cards along its long direction inside the outlined frame. Start means left for a row and top for a column. Which description fits the empty spaces measured from card edges to the INSIDE of the frame border?\nA: Packed together in the middle, with much larger equal outer spaces\nB: First and last cards touch opposite inner edges; equal gaps between cards\nC: All interior gaps and both outer spaces are equal\nD: Packed at the start, touching that inner edge\nE: Equal outer spaces, each half the size of an interior gap\nF: Packed at the end, touching that inner edge\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 1, "answer": "evenly", "options": [ "center", "between", "evenly", "start", "around", "end" ], "factors": { "direction": "column" }, "lines": {} } }, { "task_id": "layoutbench-crossalign-001", "family": "crossalign", "group_id": "crossalign-start-light-0", "image_filename": "crossalign-start-light-0.png", "image_sha256": "6807ee4d1058fe188ee5fa9d536c5fcb6d1cbe8fd637b8d0beab1a91852be6d1", "ground_truth": { "choice": "B" }, "prompt": "Across the direction perpendicular to the sequence of cards, how do their visible boxes align inside the outlined frame? For a horizontal sequence compare vertical positions; for a vertical sequence compare horizontal positions.\nA: Every box is centered on that cross axis, with empty space on both sides of the box\nB: Every box touches the top (horizontal sequence) or left (vertical sequence) inner edge, leaving space before the opposite edge\nC: Every box fills the frame along that cross axis\nD: Every box touches the bottom (horizontal sequence) or right (vertical sequence) inner edge, leaving space before the opposite edge\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 0, "answer": "start", "options": [ "center", "start", "stretch", "end" ], "factors": { "direction": "row" }, "lines": {} } }, { "task_id": "layoutbench-crossalign-002", "family": "crossalign", "group_id": "crossalign-start-light-1", "image_filename": "crossalign-start-light-1.png", "image_sha256": "0cae12efbadf95299208c915af596834d304f9733727a30f7fbee377b19d5049", "ground_truth": { "choice": "C" }, "prompt": "Across the direction perpendicular to the sequence of cards, how do their visible boxes align inside the outlined frame? For a horizontal sequence compare vertical positions; for a vertical sequence compare horizontal positions.\nA: Every box is centered on that cross axis, with empty space on both sides of the box\nB: Every box touches the bottom (horizontal sequence) or right (vertical sequence) inner edge, leaving space before the opposite edge\nC: Every box touches the top (horizontal sequence) or left (vertical sequence) inner edge, leaving space before the opposite edge\nD: Every box fills the frame along that cross axis\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 1, "answer": "start", "options": [ "center", "end", "start", "stretch" ], "factors": { "direction": "column" }, "lines": {} } }, { "task_id": "layoutbench-crossalign-003", "family": "crossalign", "group_id": "crossalign-start-dark-0", "image_filename": "crossalign-start-dark-0.png", "image_sha256": "aa0f2e839dd073a842ee9ca332cc700c32e3500e36189064d19376517955a337", "ground_truth": { "choice": "A" }, "prompt": "Across the direction perpendicular to the sequence of cards, how do their visible boxes align inside the outlined frame? For a horizontal sequence compare vertical positions; for a vertical sequence compare horizontal positions.\nA: Every box touches the top (horizontal sequence) or left (vertical sequence) inner edge, leaving space before the opposite edge\nB: Every box fills the frame along that cross axis\nC: Every box touches the bottom (horizontal sequence) or right (vertical sequence) inner edge, leaving space before the opposite edge\nD: Every box is centered on that cross axis, with empty space on both sides of the box\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 0, "answer": "start", "options": [ "start", "stretch", "end", "center" ], "factors": { "direction": "row" }, "lines": {} } }, { "task_id": "layoutbench-crossalign-004", "family": "crossalign", "group_id": "crossalign-start-dark-1", "image_filename": "crossalign-start-dark-1.png", "image_sha256": "7f4312cb95176955c2fa3e79c7d57089b4cc2a2be6fd78c7257dfb3150b77d72", "ground_truth": { "choice": "D" }, "prompt": "Across the direction perpendicular to the sequence of cards, how do their visible boxes align inside the outlined frame? For a horizontal sequence compare vertical positions; for a vertical sequence compare horizontal positions.\nA: Every box fills the frame along that cross axis\nB: Every box is centered on that cross axis, with empty space on both sides of the box\nC: Every box touches the bottom (horizontal sequence) or right (vertical sequence) inner edge, leaving space before the opposite edge\nD: Every box touches the top (horizontal sequence) or left (vertical sequence) inner edge, leaving space before the opposite edge\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 1, "answer": "start", "options": [ "stretch", "center", "end", "start" ], "factors": { "direction": "column" }, "lines": {} } }, { "task_id": "layoutbench-crossalign-005", "family": "crossalign", "group_id": "crossalign-center-light-0", "image_filename": "crossalign-center-light-0.png", "image_sha256": "f13bac549430082588ecdbbff1f7a18b203eff675ffda99f97236b94bcfd583d", "ground_truth": { "choice": "A" }, "prompt": "Across the direction perpendicular to the sequence of cards, how do their visible boxes align inside the outlined frame? For a horizontal sequence compare vertical positions; for a vertical sequence compare horizontal positions.\nA: Every box is centered on that cross axis, with empty space on both sides of the box\nB: Every box fills the frame along that cross axis\nC: Every box touches the bottom (horizontal sequence) or right (vertical sequence) inner edge, leaving space before the opposite edge\nD: Every box touches the top (horizontal sequence) or left (vertical sequence) inner edge, leaving space before the opposite edge\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 0, "answer": "center", "options": [ "center", "stretch", "end", "start" ], "factors": { "direction": "row" }, "lines": {} } }, { "task_id": "layoutbench-crossalign-006", "family": "crossalign", "group_id": "crossalign-center-light-1", "image_filename": "crossalign-center-light-1.png", "image_sha256": "5ad39e8f0bb40f9dc4206073ab07b3eb472d8a62234eb00c2d6242b6f8297fe1", "ground_truth": { "choice": "D" }, "prompt": "Across the direction perpendicular to the sequence of cards, how do their visible boxes align inside the outlined frame? For a horizontal sequence compare vertical positions; for a vertical sequence compare horizontal positions.\nA: Every box touches the top (horizontal sequence) or left (vertical sequence) inner edge, leaving space before the opposite edge\nB: Every box fills the frame along that cross axis\nC: Every box touches the bottom (horizontal sequence) or right (vertical sequence) inner edge, leaving space before the opposite edge\nD: Every box is centered on that cross axis, with empty space on both sides of the box\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 1, "answer": "center", "options": [ "start", "stretch", "end", "center" ], "factors": { "direction": "column" }, "lines": {} } }, { "task_id": "layoutbench-crossalign-007", "family": "crossalign", "group_id": "crossalign-center-dark-0", "image_filename": "crossalign-center-dark-0.png", "image_sha256": "720dd90c5a75cc47e8d7b4d0707fe3e587c4c3777ebced9d28c59ee8f1f6f976", "ground_truth": { "choice": "D" }, "prompt": "Across the direction perpendicular to the sequence of cards, how do their visible boxes align inside the outlined frame? For a horizontal sequence compare vertical positions; for a vertical sequence compare horizontal positions.\nA: Every box touches the top (horizontal sequence) or left (vertical sequence) inner edge, leaving space before the opposite edge\nB: Every box fills the frame along that cross axis\nC: Every box touches the bottom (horizontal sequence) or right (vertical sequence) inner edge, leaving space before the opposite edge\nD: Every box is centered on that cross axis, with empty space on both sides of the box\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 0, "answer": "center", "options": [ "start", "stretch", "end", "center" ], "factors": { "direction": "row" }, "lines": {} } }, { "task_id": "layoutbench-crossalign-008", "family": "crossalign", "group_id": "crossalign-center-dark-1", "image_filename": "crossalign-center-dark-1.png", "image_sha256": "96ba038b9c1e15b95e02d28c9c0a0a669468dbaf0b0e7d9a0c1219e14a988a03", "ground_truth": { "choice": "A" }, "prompt": "Across the direction perpendicular to the sequence of cards, how do their visible boxes align inside the outlined frame? For a horizontal sequence compare vertical positions; for a vertical sequence compare horizontal positions.\nA: Every box is centered on that cross axis, with empty space on both sides of the box\nB: Every box touches the bottom (horizontal sequence) or right (vertical sequence) inner edge, leaving space before the opposite edge\nC: Every box touches the top (horizontal sequence) or left (vertical sequence) inner edge, leaving space before the opposite edge\nD: Every box fills the frame along that cross axis\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 1, "answer": "center", "options": [ "center", "end", "start", "stretch" ], "factors": { "direction": "column" }, "lines": {} } }, { "task_id": "layoutbench-crossalign-009", "family": "crossalign", "group_id": "crossalign-end-light-0", "image_filename": "crossalign-end-light-0.png", "image_sha256": "71494b9eb2615d6b9b01b12bc414029d27e898f4b876eacadd29a95689c57800", "ground_truth": { "choice": "A" }, "prompt": "Across the direction perpendicular to the sequence of cards, how do their visible boxes align inside the outlined frame? For a horizontal sequence compare vertical positions; for a vertical sequence compare horizontal positions.\nA: Every box touches the bottom (horizontal sequence) or right (vertical sequence) inner edge, leaving space before the opposite edge\nB: Every box is centered on that cross axis, with empty space on both sides of the box\nC: Every box fills the frame along that cross axis\nD: Every box touches the top (horizontal sequence) or left (vertical sequence) inner edge, leaving space before the opposite edge\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 0, "answer": "end", "options": [ "end", "center", "stretch", "start" ], "factors": { "direction": "row" }, "lines": {} } }, { "task_id": "layoutbench-crossalign-010", "family": "crossalign", "group_id": "crossalign-end-light-1", "image_filename": "crossalign-end-light-1.png", "image_sha256": "a3effd6b3a5d358a1b500bd4dbd255c12c9b4ee4e38f59f4f8e7fc21474409bb", "ground_truth": { "choice": "C" }, "prompt": "Across the direction perpendicular to the sequence of cards, how do their visible boxes align inside the outlined frame? For a horizontal sequence compare vertical positions; for a vertical sequence compare horizontal positions.\nA: Every box touches the top (horizontal sequence) or left (vertical sequence) inner edge, leaving space before the opposite edge\nB: Every box is centered on that cross axis, with empty space on both sides of the box\nC: Every box touches the bottom (horizontal sequence) or right (vertical sequence) inner edge, leaving space before the opposite edge\nD: Every box fills the frame along that cross axis\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 1, "answer": "end", "options": [ "start", "center", "end", "stretch" ], "factors": { "direction": "column" }, "lines": {} } }, { "task_id": "layoutbench-crossalign-011", "family": "crossalign", "group_id": "crossalign-end-dark-0", "image_filename": "crossalign-end-dark-0.png", "image_sha256": "b98a71784ef92ee7def38fc19bdb426fece3ab21027d5470f622150e822a2f62", "ground_truth": { "choice": "C" }, "prompt": "Across the direction perpendicular to the sequence of cards, how do their visible boxes align inside the outlined frame? For a horizontal sequence compare vertical positions; for a vertical sequence compare horizontal positions.\nA: Every box fills the frame along that cross axis\nB: Every box touches the top (horizontal sequence) or left (vertical sequence) inner edge, leaving space before the opposite edge\nC: Every box touches the bottom (horizontal sequence) or right (vertical sequence) inner edge, leaving space before the opposite edge\nD: Every box is centered on that cross axis, with empty space on both sides of the box\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 0, "answer": "end", "options": [ "stretch", "start", "end", "center" ], "factors": { "direction": "row" }, "lines": {} } }, { "task_id": "layoutbench-crossalign-012", "family": "crossalign", "group_id": "crossalign-end-dark-1", "image_filename": "crossalign-end-dark-1.png", "image_sha256": "0f154b21beb82164b6c260a6854bf151614a4d774e77176c9e52dc93db112b2e", "ground_truth": { "choice": "B" }, "prompt": "Across the direction perpendicular to the sequence of cards, how do their visible boxes align inside the outlined frame? For a horizontal sequence compare vertical positions; for a vertical sequence compare horizontal positions.\nA: Every box touches the top (horizontal sequence) or left (vertical sequence) inner edge, leaving space before the opposite edge\nB: Every box touches the bottom (horizontal sequence) or right (vertical sequence) inner edge, leaving space before the opposite edge\nC: Every box fills the frame along that cross axis\nD: Every box is centered on that cross axis, with empty space on both sides of the box\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 1, "answer": "end", "options": [ "start", "end", "stretch", "center" ], "factors": { "direction": "column" }, "lines": {} } }, { "task_id": "layoutbench-crossalign-013", "family": "crossalign", "group_id": "crossalign-stretch-light-0", "image_filename": "crossalign-stretch-light-0.png", "image_sha256": "53a6b5623c72652a94e534ed326d0df08ba8974c4ffe15355782b84a94649797", "ground_truth": { "choice": "C" }, "prompt": "Across the direction perpendicular to the sequence of cards, how do their visible boxes align inside the outlined frame? For a horizontal sequence compare vertical positions; for a vertical sequence compare horizontal positions.\nA: Every box touches the bottom (horizontal sequence) or right (vertical sequence) inner edge, leaving space before the opposite edge\nB: Every box is centered on that cross axis, with empty space on both sides of the box\nC: Every box fills the frame along that cross axis\nD: Every box touches the top (horizontal sequence) or left (vertical sequence) inner edge, leaving space before the opposite edge\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 0, "answer": "stretch", "options": [ "end", "center", "stretch", "start" ], "factors": { "direction": "row" }, "lines": {} } }, { "task_id": "layoutbench-crossalign-014", "family": "crossalign", "group_id": "crossalign-stretch-light-1", "image_filename": "crossalign-stretch-light-1.png", "image_sha256": "eb41f2095a8ca7c1512aa2814ea9df3855e816dbc2b43d13007c1606a4970ac0", "ground_truth": { "choice": "B" }, "prompt": "Across the direction perpendicular to the sequence of cards, how do their visible boxes align inside the outlined frame? For a horizontal sequence compare vertical positions; for a vertical sequence compare horizontal positions.\nA: Every box is centered on that cross axis, with empty space on both sides of the box\nB: Every box fills the frame along that cross axis\nC: Every box touches the bottom (horizontal sequence) or right (vertical sequence) inner edge, leaving space before the opposite edge\nD: Every box touches the top (horizontal sequence) or left (vertical sequence) inner edge, leaving space before the opposite edge\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 1, "answer": "stretch", "options": [ "center", "stretch", "end", "start" ], "factors": { "direction": "column" }, "lines": {} } }, { "task_id": "layoutbench-crossalign-015", "family": "crossalign", "group_id": "crossalign-stretch-dark-0", "image_filename": "crossalign-stretch-dark-0.png", "image_sha256": "4c3ef31f845a579e2c208a6d64e9cf419d03b44cd29f4552cdf9b43bfe8c8a34", "ground_truth": { "choice": "D" }, "prompt": "Across the direction perpendicular to the sequence of cards, how do their visible boxes align inside the outlined frame? For a horizontal sequence compare vertical positions; for a vertical sequence compare horizontal positions.\nA: Every box is centered on that cross axis, with empty space on both sides of the box\nB: Every box touches the bottom (horizontal sequence) or right (vertical sequence) inner edge, leaving space before the opposite edge\nC: Every box touches the top (horizontal sequence) or left (vertical sequence) inner edge, leaving space before the opposite edge\nD: Every box fills the frame along that cross axis\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 0, "answer": "stretch", "options": [ "center", "end", "start", "stretch" ], "factors": { "direction": "row" }, "lines": {} } }, { "task_id": "layoutbench-crossalign-016", "family": "crossalign", "group_id": "crossalign-stretch-dark-1", "image_filename": "crossalign-stretch-dark-1.png", "image_sha256": "f41b2bbfe51e1d891395bf465a844bd5437136fb1ebf12f686255f22018471f7", "ground_truth": { "choice": "B" }, "prompt": "Across the direction perpendicular to the sequence of cards, how do their visible boxes align inside the outlined frame? For a horizontal sequence compare vertical positions; for a vertical sequence compare horizontal positions.\nA: Every box touches the bottom (horizontal sequence) or right (vertical sequence) inner edge, leaving space before the opposite edge\nB: Every box fills the frame along that cross axis\nC: Every box touches the top (horizontal sequence) or left (vertical sequence) inner edge, leaving space before the opposite edge\nD: Every box is centered on that cross axis, with empty space on both sides of the box\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 1, "answer": "stretch", "options": [ "end", "stretch", "start", "center" ], "factors": { "direction": "column" }, "lines": {} } }, { "task_id": "layoutbench-textalign-001", "family": "textalign", "group_id": "textalign-left-light-0", "image_filename": "textalign-left-light-0.png", "image_sha256": "8905dae024a605d690165f00c9acb675ae2c30148a35af4664128c8df89a3a88", "ground_truth": { "choice": "D" }, "prompt": "How are the lines of body text aligned within the outlined text region? For both-edge justification, disregard the short final line.\nA: Right edges line up; left edges are ragged\nB: Line centers line up; both edges are ragged\nC: Both left and right edges line up on the nonfinal lines\nD: Left edges line up; right edges are ragged\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 0, "answer": "left", "options": [ "right", "center", "justify", "left" ], "factors": {}, "lines": { "text-0": [ { "x": 161, "y": 109, "width": 456, "height": 21 }, { "x": 161, "y": 136.890625, "width": 455, "height": 21 }, { "x": 161, "y": 164.78125, "width": 447, "height": 21 }, { "x": 161, "y": 192.671875, "width": 472, "height": 21 }, { "x": 161, "y": 220.5625, "width": 447, "height": 21 }, { "x": 161, "y": 248.453125, "width": 444, "height": 21 }, { "x": 161, "y": 276.34375, "width": 198, "height": 21 } ] } } }, { "task_id": "layoutbench-textalign-002", "family": "textalign", "group_id": "textalign-left-light-1", "image_filename": "textalign-left-light-1.png", "image_sha256": "ee1471024f264f11528a7c363c7042a96114b65dafaf828365f21c0ee8b18f61", "ground_truth": { "choice": "D" }, "prompt": "How are the lines of body text aligned within the outlined text region? For both-edge justification, disregard the short final line.\nA: Right edges line up; left edges are ragged\nB: Line centers line up; both edges are ragged\nC: Both left and right edges line up on the nonfinal lines\nD: Left edges line up; right edges are ragged\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 1, "answer": "left", "options": [ "right", "center", "justify", "left" ], "factors": {}, "lines": { "text-0": [ { "x": 161, "y": 109, "width": 478, "height": 21 }, { "x": 161, "y": 136.890625, "width": 477, "height": 21 }, { "x": 161, "y": 164.78125, "width": 447, "height": 21 }, { "x": 161, "y": 192.671875, "width": 468, "height": 21 }, { "x": 161, "y": 220.5625, "width": 473, "height": 21 }, { "x": 161, "y": 248.453125, "width": 440, "height": 21 } ] } } }, { "task_id": "layoutbench-textalign-003", "family": "textalign", "group_id": "textalign-left-dark-0", "image_filename": "textalign-left-dark-0.png", "image_sha256": "1348d5f6f99b1b3052c1122cdf87f87f4b89988bb44fe5441d0b56a16e3ebd47", "ground_truth": { "choice": "A" }, "prompt": "How are the lines of body text aligned within the outlined text region? For both-edge justification, disregard the short final line.\nA: Left edges line up; right edges are ragged\nB: Right edges line up; left edges are ragged\nC: Line centers line up; both edges are ragged\nD: Both left and right edges line up on the nonfinal lines\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 0, "answer": "left", "options": [ "left", "right", "center", "justify" ], "factors": {}, "lines": { "text-0": [ { "x": 161, "y": 109, "width": 456, "height": 21 }, { "x": 161, "y": 136.890625, "width": 455, "height": 21 }, { "x": 161, "y": 164.78125, "width": 447, "height": 21 }, { "x": 161, "y": 192.671875, "width": 472, "height": 21 }, { "x": 161, "y": 220.5625, "width": 447, "height": 21 }, { "x": 161, "y": 248.453125, "width": 444, "height": 21 }, { "x": 161, "y": 276.34375, "width": 198, "height": 21 } ] } } }, { "task_id": "layoutbench-textalign-004", "family": "textalign", "group_id": "textalign-left-dark-1", "image_filename": "textalign-left-dark-1.png", "image_sha256": "690dfd731d6e5ff618ff3c0a1e4cb2d0acd4c750841bd95380926277bc1a70ab", "ground_truth": { "choice": "D" }, "prompt": "How are the lines of body text aligned within the outlined text region? For both-edge justification, disregard the short final line.\nA: Both left and right edges line up on the nonfinal lines\nB: Line centers line up; both edges are ragged\nC: Right edges line up; left edges are ragged\nD: Left edges line up; right edges are ragged\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 1, "answer": "left", "options": [ "justify", "center", "right", "left" ], "factors": {}, "lines": { "text-0": [ { "x": 161, "y": 109, "width": 478, "height": 21 }, { "x": 161, "y": 136.890625, "width": 477, "height": 21 }, { "x": 161, "y": 164.78125, "width": 447, "height": 21 }, { "x": 161, "y": 192.671875, "width": 468, "height": 21 }, { "x": 161, "y": 220.5625, "width": 473, "height": 21 }, { "x": 161, "y": 248.453125, "width": 440, "height": 21 } ] } } }, { "task_id": "layoutbench-textalign-005", "family": "textalign", "group_id": "textalign-center-light-0", "image_filename": "textalign-center-light-0.png", "image_sha256": "b594092feff844f3f0fcdf6b1e53ada3d71dbd3de071f8e9c2bf69f62a9c44fe", "ground_truth": { "choice": "C" }, "prompt": "How are the lines of body text aligned within the outlined text region? For both-edge justification, disregard the short final line.\nA: Left edges line up; right edges are ragged\nB: Right edges line up; left edges are ragged\nC: Line centers line up; both edges are ragged\nD: Both left and right edges line up on the nonfinal lines\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 0, "answer": "center", "options": [ "left", "right", "center", "justify" ], "factors": {}, "lines": { "text-0": [ { "x": 172, "y": 109, "width": 456, "height": 21 }, { "x": 172.5, "y": 136.890625, "width": 455, "height": 21 }, { "x": 176.5, "y": 164.78125, "width": 447, "height": 21 }, { "x": 164, "y": 192.671875, "width": 472, "height": 21 }, { "x": 176.5, "y": 220.5625, "width": 447, "height": 21 }, { "x": 178, "y": 248.453125, "width": 444, "height": 21 }, { "x": 301, "y": 276.34375, "width": 198, "height": 21 } ] } } }, { "task_id": "layoutbench-textalign-006", "family": "textalign", "group_id": "textalign-center-light-1", "image_filename": "textalign-center-light-1.png", "image_sha256": "b02b85c997c0d88abe7f20411efc5cb120b5eb8425bac446f1571f488162549b", "ground_truth": { "choice": "B" }, "prompt": "How are the lines of body text aligned within the outlined text region? For both-edge justification, disregard the short final line.\nA: Both left and right edges line up on the nonfinal lines\nB: Line centers line up; both edges are ragged\nC: Left edges line up; right edges are ragged\nD: Right edges line up; left edges are ragged\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 1, "answer": "center", "options": [ "justify", "center", "left", "right" ], "factors": {}, "lines": { "text-0": [ { "x": 161, "y": 109, "width": 478, "height": 21 }, { "x": 161.5, "y": 136.890625, "width": 477, "height": 21 }, { "x": 176.5, "y": 164.78125, "width": 447, "height": 21 }, { "x": 166, "y": 192.671875, "width": 468, "height": 21 }, { "x": 163.5, "y": 220.5625, "width": 473, "height": 21 }, { "x": 180, "y": 248.453125, "width": 440, "height": 21 } ] } } }, { "task_id": "layoutbench-textalign-007", "family": "textalign", "group_id": "textalign-center-dark-0", "image_filename": "textalign-center-dark-0.png", "image_sha256": "872023a65bac1e205fa0a7212f44da2c74dc327d6d1c3bef2433a7372394dbe1", "ground_truth": { "choice": "B" }, "prompt": "How are the lines of body text aligned within the outlined text region? For both-edge justification, disregard the short final line.\nA: Left edges line up; right edges are ragged\nB: Line centers line up; both edges are ragged\nC: Both left and right edges line up on the nonfinal lines\nD: Right edges line up; left edges are ragged\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 0, "answer": "center", "options": [ "left", "center", "justify", "right" ], "factors": {}, "lines": { "text-0": [ { "x": 172, "y": 109, "width": 456, "height": 21 }, { "x": 172.5, "y": 136.890625, "width": 455, "height": 21 }, { "x": 176.5, "y": 164.78125, "width": 447, "height": 21 }, { "x": 164, "y": 192.671875, "width": 472, "height": 21 }, { "x": 176.5, "y": 220.5625, "width": 447, "height": 21 }, { "x": 178, "y": 248.453125, "width": 444, "height": 21 }, { "x": 301, "y": 276.34375, "width": 198, "height": 21 } ] } } }, { "task_id": "layoutbench-textalign-008", "family": "textalign", "group_id": "textalign-center-dark-1", "image_filename": "textalign-center-dark-1.png", "image_sha256": "45b0cd0ec48f539edae5b70512f48d1c6e420ae9b7645352ba41b60a3e66c3b1", "ground_truth": { "choice": "B" }, "prompt": "How are the lines of body text aligned within the outlined text region? For both-edge justification, disregard the short final line.\nA: Right edges line up; left edges are ragged\nB: Line centers line up; both edges are ragged\nC: Both left and right edges line up on the nonfinal lines\nD: Left edges line up; right edges are ragged\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 1, "answer": "center", "options": [ "right", "center", "justify", "left" ], "factors": {}, "lines": { "text-0": [ { "x": 161, "y": 109, "width": 478, "height": 21 }, { "x": 161.5, "y": 136.890625, "width": 477, "height": 21 }, { "x": 176.5, "y": 164.78125, "width": 447, "height": 21 }, { "x": 166, "y": 192.671875, "width": 468, "height": 21 }, { "x": 163.5, "y": 220.5625, "width": 473, "height": 21 }, { "x": 180, "y": 248.453125, "width": 440, "height": 21 } ] } } }, { "task_id": "layoutbench-textalign-009", "family": "textalign", "group_id": "textalign-right-light-0", "image_filename": "textalign-right-light-0.png", "image_sha256": "c5d24cf77febfe0fb2e3f5fc92a1bb797941653bb7935afa134bdb8e7c45ce04", "ground_truth": { "choice": "B" }, "prompt": "How are the lines of body text aligned within the outlined text region? For both-edge justification, disregard the short final line.\nA: Line centers line up; both edges are ragged\nB: Right edges line up; left edges are ragged\nC: Left edges line up; right edges are ragged\nD: Both left and right edges line up on the nonfinal lines\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 0, "answer": "right", "options": [ "center", "right", "left", "justify" ], "factors": {}, "lines": { "text-0": [ { "x": 183, "y": 109, "width": 456, "height": 21 }, { "x": 184, "y": 136.890625, "width": 455, "height": 21 }, { "x": 192, "y": 164.78125, "width": 447, "height": 21 }, { "x": 167, "y": 192.671875, "width": 472, "height": 21 }, { "x": 192, "y": 220.5625, "width": 447, "height": 21 }, { "x": 195, "y": 248.453125, "width": 444, "height": 21 }, { "x": 441, "y": 276.34375, "width": 198, "height": 21 } ] } } }, { "task_id": "layoutbench-textalign-010", "family": "textalign", "group_id": "textalign-right-light-1", "image_filename": "textalign-right-light-1.png", "image_sha256": "878c98bd98d4dde02acdab98e70c7b0400853970164a91c31ec91b364fbace89", "ground_truth": { "choice": "D" }, "prompt": "How are the lines of body text aligned within the outlined text region? For both-edge justification, disregard the short final line.\nA: Line centers line up; both edges are ragged\nB: Both left and right edges line up on the nonfinal lines\nC: Left edges line up; right edges are ragged\nD: Right edges line up; left edges are ragged\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 1, "answer": "right", "options": [ "center", "justify", "left", "right" ], "factors": {}, "lines": { "text-0": [ { "x": 161, "y": 109, "width": 478, "height": 21 }, { "x": 162, "y": 136.890625, "width": 477, "height": 21 }, { "x": 192, "y": 164.78125, "width": 447, "height": 21 }, { "x": 171, "y": 192.671875, "width": 468, "height": 21 }, { "x": 166, "y": 220.5625, "width": 473, "height": 21 }, { "x": 199, "y": 248.453125, "width": 440, "height": 21 } ] } } }, { "task_id": "layoutbench-textalign-011", "family": "textalign", "group_id": "textalign-right-dark-0", "image_filename": "textalign-right-dark-0.png", "image_sha256": "1ef0c33a06dccfc603e96da71ed17ce5fe3c73cf4515e4c3a04ca269c0745a53", "ground_truth": { "choice": "A" }, "prompt": "How are the lines of body text aligned within the outlined text region? For both-edge justification, disregard the short final line.\nA: Right edges line up; left edges are ragged\nB: Both left and right edges line up on the nonfinal lines\nC: Left edges line up; right edges are ragged\nD: Line centers line up; both edges are ragged\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 0, "answer": "right", "options": [ "right", "justify", "left", "center" ], "factors": {}, "lines": { "text-0": [ { "x": 183, "y": 109, "width": 456, "height": 21 }, { "x": 184, "y": 136.890625, "width": 455, "height": 21 }, { "x": 192, "y": 164.78125, "width": 447, "height": 21 }, { "x": 167, "y": 192.671875, "width": 472, "height": 21 }, { "x": 192, "y": 220.5625, "width": 447, "height": 21 }, { "x": 195, "y": 248.453125, "width": 444, "height": 21 }, { "x": 441, "y": 276.34375, "width": 198, "height": 21 } ] } } }, { "task_id": "layoutbench-textalign-012", "family": "textalign", "group_id": "textalign-right-dark-1", "image_filename": "textalign-right-dark-1.png", "image_sha256": "75f7c9502d145943c1d28d485e0f5c7242aa7c65081d473a3c69eabd2b4572b0", "ground_truth": { "choice": "A" }, "prompt": "How are the lines of body text aligned within the outlined text region? For both-edge justification, disregard the short final line.\nA: Right edges line up; left edges are ragged\nB: Left edges line up; right edges are ragged\nC: Both left and right edges line up on the nonfinal lines\nD: Line centers line up; both edges are ragged\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 1, "answer": "right", "options": [ "right", "left", "justify", "center" ], "factors": {}, "lines": { "text-0": [ { "x": 161, "y": 109, "width": 478, "height": 21 }, { "x": 162, "y": 136.890625, "width": 477, "height": 21 }, { "x": 192, "y": 164.78125, "width": 447, "height": 21 }, { "x": 171, "y": 192.671875, "width": 468, "height": 21 }, { "x": 166, "y": 220.5625, "width": 473, "height": 21 }, { "x": 199, "y": 248.453125, "width": 440, "height": 21 } ] } } }, { "task_id": "layoutbench-textalign-013", "family": "textalign", "group_id": "textalign-justify-light-0", "image_filename": "textalign-justify-light-0.png", "image_sha256": "15f73d663b263c3199bc2af63f82b7006773fb0d85d4555b56c25f121110b02e", "ground_truth": { "choice": "A" }, "prompt": "How are the lines of body text aligned within the outlined text region? For both-edge justification, disregard the short final line.\nA: Both left and right edges line up on the nonfinal lines\nB: Line centers line up; both edges are ragged\nC: Right edges line up; left edges are ragged\nD: Left edges line up; right edges are ragged\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 0, "answer": "justify", "options": [ "justify", "center", "right", "left" ], "factors": {}, "lines": { "text-0": [ { "x": 161, "y": 109, "width": 478, "height": 21 }, { "x": 161, "y": 136.890625, "width": 478, "height": 21 }, { "x": 161, "y": 164.78125, "width": 478, "height": 21 }, { "x": 161, "y": 192.671875, "width": 478, "height": 21 }, { "x": 161, "y": 220.5625, "width": 478, "height": 21 }, { "x": 161, "y": 248.453125, "width": 478, "height": 21 }, { "x": 161, "y": 276.34375, "width": 198, "height": 21 } ] } } }, { "task_id": "layoutbench-textalign-014", "family": "textalign", "group_id": "textalign-justify-light-1", "image_filename": "textalign-justify-light-1.png", "image_sha256": "0d6d4e7e771c5395d582ea32b75e5f05ed47b24f35cb10c6df00e9aea3e89c10", "ground_truth": { "choice": "C" }, "prompt": "How are the lines of body text aligned within the outlined text region? For both-edge justification, disregard the short final line.\nA: Right edges line up; left edges are ragged\nB: Line centers line up; both edges are ragged\nC: Both left and right edges line up on the nonfinal lines\nD: Left edges line up; right edges are ragged\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 1, "answer": "justify", "options": [ "right", "center", "justify", "left" ], "factors": {}, "lines": { "text-0": [ { "x": 161, "y": 109, "width": 478, "height": 21 }, { "x": 161, "y": 136.890625, "width": 478, "height": 21 }, { "x": 161, "y": 164.78125, "width": 478, "height": 21 }, { "x": 161, "y": 192.671875, "width": 478, "height": 21 }, { "x": 161, "y": 220.5625, "width": 478, "height": 21 }, { "x": 161, "y": 248.453125, "width": 440, "height": 21 } ] } } }, { "task_id": "layoutbench-textalign-015", "family": "textalign", "group_id": "textalign-justify-dark-0", "image_filename": "textalign-justify-dark-0.png", "image_sha256": "772229d3293d13622df3a014b260d2337cce0625b141f5aabf5fcdf60c81b542", "ground_truth": { "choice": "C" }, "prompt": "How are the lines of body text aligned within the outlined text region? For both-edge justification, disregard the short final line.\nA: Left edges line up; right edges are ragged\nB: Line centers line up; both edges are ragged\nC: Both left and right edges line up on the nonfinal lines\nD: Right edges line up; left edges are ragged\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 0, "answer": "justify", "options": [ "left", "center", "justify", "right" ], "factors": {}, "lines": { "text-0": [ { "x": 161, "y": 109, "width": 478, "height": 21 }, { "x": 161, "y": 136.890625, "width": 478, "height": 21 }, { "x": 161, "y": 164.78125, "width": 478, "height": 21 }, { "x": 161, "y": 192.671875, "width": 478, "height": 21 }, { "x": 161, "y": 220.5625, "width": 478, "height": 21 }, { "x": 161, "y": 248.453125, "width": 478, "height": 21 }, { "x": 161, "y": 276.34375, "width": 198, "height": 21 } ] } } }, { "task_id": "layoutbench-textalign-016", "family": "textalign", "group_id": "textalign-justify-dark-1", "image_filename": "textalign-justify-dark-1.png", "image_sha256": "0afbc53f0bbe07c398b162d83ab208a158e0bdcfdbcebc35ad1c1277226694a5", "ground_truth": { "choice": "C" }, "prompt": "How are the lines of body text aligned within the outlined text region? For both-edge justification, disregard the short final line.\nA: Right edges line up; left edges are ragged\nB: Line centers line up; both edges are ragged\nC: Both left and right edges line up on the nonfinal lines\nD: Left edges line up; right edges are ragged\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 1, "answer": "justify", "options": [ "right", "center", "justify", "left" ], "factors": {}, "lines": { "text-0": [ { "x": 161, "y": 109, "width": 478, "height": 21 }, { "x": 161, "y": 136.890625, "width": 478, "height": 21 }, { "x": 161, "y": 164.78125, "width": 478, "height": 21 }, { "x": 161, "y": 192.671875, "width": 478, "height": 21 }, { "x": 161, "y": 220.5625, "width": 478, "height": 21 }, { "x": 161, "y": 248.453125, "width": 440, "height": 21 } ] } } }, { "task_id": "layoutbench-textcolumns-001", "family": "textcolumns", "group_id": "textcolumns-1-light-0", "image_filename": "textcolumns-1-light-0.png", "image_sha256": "647a54a44dbdf58ed0c52f65c4005d87df8a7a897cfae12ef5a20936a48159d6", "ground_truth": { "choice": "A" }, "prompt": "How many side-by-side columns of body text are visible inside the frame?\nA: One\nB: Two\nC: Three\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 0, "answer": "1", "options": [ "1", "2", "3" ], "factors": {}, "lines": { "text-0": [ { "x": 96, "y": 109, "width": 582, "height": 21 }, { "x": 96, "y": 136.890625, "width": 587, "height": 21 }, { "x": 96, "y": 164.78125, "width": 603, "height": 21 }, { "x": 96, "y": 192.671875, "width": 564, "height": 21 }, { "x": 96, "y": 220.5625, "width": 595, "height": 21 } ] } } }, { "task_id": "layoutbench-textcolumns-002", "family": "textcolumns", "group_id": "textcolumns-1-light-1", "image_filename": "textcolumns-1-light-1.png", "image_sha256": "40803215467b1223ccfda17fb9ab588c5b4c9adb3540352f6e95fced4c21b653", "ground_truth": { "choice": "B" }, "prompt": "How many side-by-side columns of body text are visible inside the frame?\nA: Three\nB: One\nC: Two\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 1, "answer": "1", "options": [ "3", "1", "2" ], "factors": {}, "lines": { "text-0": [ { "x": 96, "y": 109, "width": 608, "height": 21 }, { "x": 96, "y": 136.890625, "width": 608, "height": 21 }, { "x": 96, "y": 164.78125, "width": 608, "height": 21 }, { "x": 96, "y": 192.671875, "width": 608, "height": 21 }, { "x": 96, "y": 220.5625, "width": 589, "height": 21 } ] } } }, { "task_id": "layoutbench-textcolumns-003", "family": "textcolumns", "group_id": "textcolumns-1-dark-0", "image_filename": "textcolumns-1-dark-0.png", "image_sha256": "27b5472a838867447f80e48badcf1d9d2e3f0ae65da1397310c70dbfddf56e98", "ground_truth": { "choice": "B" }, "prompt": "How many side-by-side columns of body text are visible inside the frame?\nA: Two\nB: One\nC: Three\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 0, "answer": "1", "options": [ "2", "1", "3" ], "factors": {}, "lines": { "text-0": [ { "x": 96, "y": 109, "width": 582, "height": 21 }, { "x": 96, "y": 136.890625, "width": 587, "height": 21 }, { "x": 96, "y": 164.78125, "width": 603, "height": 21 }, { "x": 96, "y": 192.671875, "width": 564, "height": 21 }, { "x": 96, "y": 220.5625, "width": 595, "height": 21 } ] } } }, { "task_id": "layoutbench-textcolumns-004", "family": "textcolumns", "group_id": "textcolumns-1-dark-1", "image_filename": "textcolumns-1-dark-1.png", "image_sha256": "7df9505249c2768203f9ff31775983aa126e6baa54a7efa86615b24c1f5a68f7", "ground_truth": { "choice": "B" }, "prompt": "How many side-by-side columns of body text are visible inside the frame?\nA: Three\nB: One\nC: Two\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 1, "answer": "1", "options": [ "3", "1", "2" ], "factors": {}, "lines": { "text-0": [ { "x": 96, "y": 109, "width": 608, "height": 21 }, { "x": 96, "y": 136.890625, "width": 608, "height": 21 }, { "x": 96, "y": 164.78125, "width": 608, "height": 21 }, { "x": 96, "y": 192.671875, "width": 608, "height": 21 }, { "x": 96, "y": 220.5625, "width": 589, "height": 21 } ] } } }, { "task_id": "layoutbench-textcolumns-005", "family": "textcolumns", "group_id": "textcolumns-2-light-0", "image_filename": "textcolumns-2-light-0.png", "image_sha256": "1b118e44838601277d8a40b7fdf57a57d49f6284a106025703dd7a6cd27a7870", "ground_truth": { "choice": "C" }, "prompt": "How many side-by-side columns of body text are visible inside the frame?\nA: One\nB: Three\nC: Two\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 0, "answer": "2", "options": [ "1", "3", "2" ], "factors": {}, "lines": { "text-0": [ { "x": 96, "y": 109, "width": 264, "height": 21 }, { "x": 96, "y": 136.890625, "width": 259, "height": 21 }, { "x": 96, "y": 164.78125, "width": 248, "height": 21 }, { "x": 96, "y": 192.671875, "width": 272, "height": 21 }, { "x": 96, "y": 220.5625, "width": 257, "height": 21 }, { "x": 96, "y": 248.453125, "width": 264, "height": 21 }, { "x": 96, "y": 276.34375, "width": 248, "height": 21 }, { "x": 96, "y": 304.234375, "width": 269, "height": 21 }, { "x": 96, "y": 332.125, "width": 225, "height": 21 }, { "x": 96, "y": 360.015625, "width": 281, "height": 21 }, { "x": 96, "y": 387.90625, "width": 219, "height": 21 }, { "x": 96, "y": 415.796875, "width": 83, "height": 21 } ], "text-1": [ { "x": 415, "y": 109, "width": 264, "height": 21 }, { "x": 415, "y": 136.890625, "width": 259, "height": 21 }, { "x": 415, "y": 164.78125, "width": 248, "height": 21 }, { "x": 415, "y": 192.671875, "width": 272, "height": 21 }, { "x": 415, "y": 220.5625, "width": 257, "height": 21 }, { "x": 415, "y": 248.453125, "width": 264, "height": 21 }, { "x": 415, "y": 276.34375, "width": 248, "height": 21 }, { "x": 415, "y": 304.234375, "width": 269, "height": 21 }, { "x": 415, "y": 332.125, "width": 225, "height": 21 }, { "x": 415, "y": 360.015625, "width": 281, "height": 21 }, { "x": 415, "y": 387.90625, "width": 219, "height": 21 }, { "x": 415, "y": 415.796875, "width": 83, "height": 21 } ] } } }, { "task_id": "layoutbench-textcolumns-006", "family": "textcolumns", "group_id": "textcolumns-2-light-1", "image_filename": "textcolumns-2-light-1.png", "image_sha256": "525124e99c513c25cd9e95a5a1258ab03a28b2d979e87620882c0dfb69f08dd4", "ground_truth": { "choice": "B" }, "prompt": "How many side-by-side columns of body text are visible inside the frame?\nA: Three\nB: Two\nC: One\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 1, "answer": "2", "options": [ "3", "2", "1" ], "factors": {}, "lines": { "text-0": [ { "x": 96, "y": 109, "width": 289, "height": 21 }, { "x": 96, "y": 136.890625, "width": 289, "height": 21 }, { "x": 96, "y": 164.78125, "width": 289, "height": 21 }, { "x": 96, "y": 192.671875, "width": 289, "height": 21 }, { "x": 96, "y": 220.5625, "width": 289, "height": 21 }, { "x": 96, "y": 248.453125, "width": 289, "height": 21 }, { "x": 96, "y": 276.34375, "width": 289, "height": 21 }, { "x": 96, "y": 304.234375, "width": 289, "height": 21 }, { "x": 96, "y": 332.125, "width": 289, "height": 21 }, { "x": 96, "y": 360.015625, "width": 289, "height": 21 }, { "x": 96, "y": 387.90625, "width": 41, "height": 21 } ], "text-1": [ { "x": 415, "y": 109, "width": 289, "height": 21 }, { "x": 415, "y": 136.890625, "width": 289, "height": 21 }, { "x": 415, "y": 164.78125, "width": 289, "height": 21 }, { "x": 415, "y": 192.671875, "width": 289, "height": 21 }, { "x": 415, "y": 220.5625, "width": 289, "height": 21 }, { "x": 415, "y": 248.453125, "width": 289, "height": 21 }, { "x": 415, "y": 276.34375, "width": 289, "height": 21 }, { "x": 415, "y": 304.234375, "width": 289, "height": 21 }, { "x": 415, "y": 332.125, "width": 289, "height": 21 }, { "x": 415, "y": 360.015625, "width": 289, "height": 21 }, { "x": 415, "y": 387.90625, "width": 41, "height": 21 } ] } } }, { "task_id": "layoutbench-textcolumns-007", "family": "textcolumns", "group_id": "textcolumns-2-dark-0", "image_filename": "textcolumns-2-dark-0.png", "image_sha256": "12c99f09af63a4826d287da83f0af98df75b5c8a3284edaf7b095659a18fc382", "ground_truth": { "choice": "A" }, "prompt": "How many side-by-side columns of body text are visible inside the frame?\nA: Two\nB: One\nC: Three\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 0, "answer": "2", "options": [ "2", "1", "3" ], "factors": {}, "lines": { "text-0": [ { "x": 96, "y": 109, "width": 264, "height": 21 }, { "x": 96, "y": 136.890625, "width": 259, "height": 21 }, { "x": 96, "y": 164.78125, "width": 248, "height": 21 }, { "x": 96, "y": 192.671875, "width": 272, "height": 21 }, { "x": 96, "y": 220.5625, "width": 257, "height": 21 }, { "x": 96, "y": 248.453125, "width": 264, "height": 21 }, { "x": 96, "y": 276.34375, "width": 248, "height": 21 }, { "x": 96, "y": 304.234375, "width": 269, "height": 21 }, { "x": 96, "y": 332.125, "width": 225, "height": 21 }, { "x": 96, "y": 360.015625, "width": 281, "height": 21 }, { "x": 96, "y": 387.90625, "width": 219, "height": 21 }, { "x": 96, "y": 415.796875, "width": 83, "height": 21 } ], "text-1": [ { "x": 415, "y": 109, "width": 264, "height": 21 }, { "x": 415, "y": 136.890625, "width": 259, "height": 21 }, { "x": 415, "y": 164.78125, "width": 248, "height": 21 }, { "x": 415, "y": 192.671875, "width": 272, "height": 21 }, { "x": 415, "y": 220.5625, "width": 257, "height": 21 }, { "x": 415, "y": 248.453125, "width": 264, "height": 21 }, { "x": 415, "y": 276.34375, "width": 248, "height": 21 }, { "x": 415, "y": 304.234375, "width": 269, "height": 21 }, { "x": 415, "y": 332.125, "width": 225, "height": 21 }, { "x": 415, "y": 360.015625, "width": 281, "height": 21 }, { "x": 415, "y": 387.90625, "width": 219, "height": 21 }, { "x": 415, "y": 415.796875, "width": 83, "height": 21 } ] } } }, { "task_id": "layoutbench-textcolumns-008", "family": "textcolumns", "group_id": "textcolumns-2-dark-1", "image_filename": "textcolumns-2-dark-1.png", "image_sha256": "74ab4de2876fb8d5522ecda1941b9f62d79a0a767b87d50c31b95fb17c71eac0", "ground_truth": { "choice": "C" }, "prompt": "How many side-by-side columns of body text are visible inside the frame?\nA: Three\nB: One\nC: Two\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 1, "answer": "2", "options": [ "3", "1", "2" ], "factors": {}, "lines": { "text-0": [ { "x": 96, "y": 109, "width": 289, "height": 21 }, { "x": 96, "y": 136.890625, "width": 289, "height": 21 }, { "x": 96, "y": 164.78125, "width": 289, "height": 21 }, { "x": 96, "y": 192.671875, "width": 289, "height": 21 }, { "x": 96, "y": 220.5625, "width": 289, "height": 21 }, { "x": 96, "y": 248.453125, "width": 289, "height": 21 }, { "x": 96, "y": 276.34375, "width": 289, "height": 21 }, { "x": 96, "y": 304.234375, "width": 289, "height": 21 }, { "x": 96, "y": 332.125, "width": 289, "height": 21 }, { "x": 96, "y": 360.015625, "width": 289, "height": 21 }, { "x": 96, "y": 387.90625, "width": 41, "height": 21 } ], "text-1": [ { "x": 415, "y": 109, "width": 289, "height": 21 }, { "x": 415, "y": 136.890625, "width": 289, "height": 21 }, { "x": 415, "y": 164.78125, "width": 289, "height": 21 }, { "x": 415, "y": 192.671875, "width": 289, "height": 21 }, { "x": 415, "y": 220.5625, "width": 289, "height": 21 }, { "x": 415, "y": 248.453125, "width": 289, "height": 21 }, { "x": 415, "y": 276.34375, "width": 289, "height": 21 }, { "x": 415, "y": 304.234375, "width": 289, "height": 21 }, { "x": 415, "y": 332.125, "width": 289, "height": 21 }, { "x": 415, "y": 360.015625, "width": 289, "height": 21 }, { "x": 415, "y": 387.90625, "width": 41, "height": 21 } ] } } }, { "task_id": "layoutbench-textcolumns-009", "family": "textcolumns", "group_id": "textcolumns-3-light-0", "image_filename": "textcolumns-3-light-0.png", "image_sha256": "f2895ce13820d9dd5ed7d55e031fbce6dce0cf407cef425b44a97888998078ae", "ground_truth": { "choice": "A" }, "prompt": "How many side-by-side columns of body text are visible inside the frame?\nA: Three\nB: Two\nC: One\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 0, "answer": "3", "options": [ "3", "2", "1" ], "factors": {}, "lines": { "text-0": [ { "x": 96, "y": 108, "width": 161, "height": 18 }, { "x": 96, "y": 131.25, "width": 161, "height": 18 }, { "x": 96, "y": 154.5, "width": 168, "height": 18 }, { "x": 96, "y": 177.75, "width": 167, "height": 18 }, { "x": 96, "y": 201, "width": 182, "height": 18 }, { "x": 96, "y": 224.25, "width": 173, "height": 18 }, { "x": 96, "y": 247.5, "width": 147, "height": 18 }, { "x": 96, "y": 270.75, "width": 158, "height": 18 }, { "x": 96, "y": 294, "width": 159, "height": 18 }, { "x": 96, "y": 317.25, "width": 127, "height": 18 }, { "x": 96, "y": 340.5, "width": 150, "height": 18 }, { "x": 96, "y": 363.75, "width": 146, "height": 18 }, { "x": 96, "y": 387, "width": 158, "height": 18 }, { "x": 96, "y": 410.25, "width": 167, "height": 18 }, { "x": 96, "y": 433.5, "width": 142, "height": 18 }, { "x": 96, "y": 456.75, "width": 71, "height": 18 } ], "text-1": [ { "x": 308.65625, "y": 108, "width": 161, "height": 18 }, { "x": 308.65625, "y": 131.25, "width": 161, "height": 18 }, { "x": 308.65625, "y": 154.5, "width": 168, "height": 18 }, { "x": 308.65625, "y": 177.75, "width": 167, "height": 18 }, { "x": 308.65625, "y": 201, "width": 182, "height": 18 }, { "x": 308.65625, "y": 224.25, "width": 173, "height": 18 }, { "x": 308.65625, "y": 247.5, "width": 147, "height": 18 }, { "x": 308.65625, "y": 270.75, "width": 158, "height": 18 }, { "x": 308.65625, "y": 294, "width": 159, "height": 18 }, { "x": 308.65625, "y": 317.25, "width": 127, "height": 18 }, { "x": 308.65625, "y": 340.5, "width": 150, "height": 18 }, { "x": 308.65625, "y": 363.75, "width": 146, "height": 18 }, { "x": 308.65625, "y": 387, "width": 158, "height": 18 }, { "x": 308.65625, "y": 410.25, "width": 167, "height": 18 }, { "x": 308.65625, "y": 433.5, "width": 142, "height": 18 }, { "x": 308.65625, "y": 456.75, "width": 71, "height": 18 } ], "text-2": [ { "x": 521.328125, "y": 108, "width": 161, "height": 18 }, { "x": 521.328125, "y": 131.25, "width": 161, "height": 18 }, { "x": 521.328125, "y": 154.5, "width": 168, "height": 18 }, { "x": 521.328125, "y": 177.75, "width": 167, "height": 18 }, { "x": 521.328125, "y": 201, "width": 182, "height": 18 }, { "x": 521.328125, "y": 224.25, "width": 173, "height": 18 }, { "x": 521.328125, "y": 247.5, "width": 147, "height": 18 }, { "x": 521.328125, "y": 270.75, "width": 158, "height": 18 }, { "x": 521.328125, "y": 294, "width": 159, "height": 18 }, { "x": 521.328125, "y": 317.25, "width": 127, "height": 18 }, { "x": 521.328125, "y": 340.5, "width": 150, "height": 18 }, { "x": 521.328125, "y": 363.75, "width": 146, "height": 18 }, { "x": 521.328125, "y": 387, "width": 158, "height": 18 }, { "x": 521.328125, "y": 410.25, "width": 167, "height": 18 }, { "x": 521.328125, "y": 433.5, "width": 142, "height": 18 }, { "x": 521.328125, "y": 456.75, "width": 71, "height": 18 } ] } } }, { "task_id": "layoutbench-textcolumns-010", "family": "textcolumns", "group_id": "textcolumns-3-light-1", "image_filename": "textcolumns-3-light-1.png", "image_sha256": "be4ccaa0a2d8241899bed3bd19071abb8f6906c288c7b4a99f38abebf33fdef4", "ground_truth": { "choice": "A" }, "prompt": "How many side-by-side columns of body text are visible inside the frame?\nA: Three\nB: Two\nC: One\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 1, "answer": "3", "options": [ "3", "2", "1" ], "factors": {}, "lines": { "text-0": [ { "x": 96, "y": 108, "width": 182.65625, "height": 18 }, { "x": 96, "y": 131.25, "width": 182.65625, "height": 18 }, { "x": 96, "y": 154.5, "width": 182.65625, "height": 18 }, { "x": 96, "y": 177.75, "width": 182.65625, "height": 18 }, { "x": 96, "y": 201, "width": 182.65625, "height": 18 }, { "x": 96, "y": 224.25, "width": 182.65625, "height": 18 }, { "x": 96, "y": 247.5, "width": 182.65625, "height": 18 }, { "x": 96, "y": 270.75, "width": 182.65625, "height": 18 }, { "x": 96, "y": 294, "width": 182.65625, "height": 18 }, { "x": 96, "y": 317.25, "width": 182.65625, "height": 18 }, { "x": 96, "y": 340.5, "width": 182.65625, "height": 18 }, { "x": 96, "y": 363.75, "width": 182.65625, "height": 18 }, { "x": 96, "y": 387, "width": 182.65625, "height": 18 }, { "x": 96, "y": 410.25, "width": 182.65625, "height": 18 }, { "x": 96, "y": 433.5, "width": 182.65625, "height": 18 }, { "x": 96, "y": 456.75, "width": 93, "height": 18 } ], "text-1": [ { "x": 308.65625, "y": 108, "width": 182.671875, "height": 18 }, { "x": 308.65625, "y": 131.25, "width": 182.671875, "height": 18 }, { "x": 308.65625, "y": 154.5, "width": 182.671875, "height": 18 }, { "x": 308.65625, "y": 177.75, "width": 182.671875, "height": 18 }, { "x": 308.65625, "y": 201, "width": 182.671875, "height": 18 }, { "x": 308.65625, "y": 224.25, "width": 182.671875, "height": 18 }, { "x": 308.65625, "y": 247.5, "width": 182.671875, "height": 18 }, { "x": 308.65625, "y": 270.75, "width": 182.671875, "height": 18 }, { "x": 308.65625, "y": 294, "width": 182.671875, "height": 18 }, { "x": 308.65625, "y": 317.25, "width": 182.671875, "height": 18 }, { "x": 308.65625, "y": 340.5, "width": 182.671875, "height": 18 }, { "x": 308.65625, "y": 363.75, "width": 182.671875, "height": 18 }, { "x": 308.65625, "y": 387, "width": 182.671875, "height": 18 }, { "x": 308.65625, "y": 410.25, "width": 182.671875, "height": 18 }, { "x": 308.65625, "y": 433.5, "width": 182.671875, "height": 18 }, { "x": 308.65625, "y": 456.75, "width": 93, "height": 18 } ], "text-2": [ { "x": 521.328125, "y": 108, "width": 182.671875, "height": 18 }, { "x": 521.328125, "y": 131.25, "width": 182.671875, "height": 18 }, { "x": 521.328125, "y": 154.5, "width": 182.671875, "height": 18 }, { "x": 521.328125, "y": 177.75, "width": 182.671875, "height": 18 }, { "x": 521.328125, "y": 201, "width": 182.671875, "height": 18 }, { "x": 521.328125, "y": 224.25, "width": 182.671875, "height": 18 }, { "x": 521.328125, "y": 247.5, "width": 182.671875, "height": 18 }, { "x": 521.328125, "y": 270.75, "width": 182.671875, "height": 18 }, { "x": 521.328125, "y": 294, "width": 182.671875, "height": 18 }, { "x": 521.328125, "y": 317.25, "width": 182.671875, "height": 18 }, { "x": 521.328125, "y": 340.5, "width": 182.671875, "height": 18 }, { "x": 521.328125, "y": 363.75, "width": 182.671875, "height": 18 }, { "x": 521.328125, "y": 387, "width": 182.671875, "height": 18 }, { "x": 521.328125, "y": 410.25, "width": 182.671875, "height": 18 }, { "x": 521.328125, "y": 433.5, "width": 182.671875, "height": 18 }, { "x": 521.328125, "y": 456.75, "width": 93, "height": 18 } ] } } }, { "task_id": "layoutbench-textcolumns-011", "family": "textcolumns", "group_id": "textcolumns-3-dark-0", "image_filename": "textcolumns-3-dark-0.png", "image_sha256": "c6c81995a5ca8f20eae37ab32c85ec41daffada38a97d893c356e682a2a13502", "ground_truth": { "choice": "C" }, "prompt": "How many side-by-side columns of body text are visible inside the frame?\nA: One\nB: Two\nC: Three\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 0, "answer": "3", "options": [ "1", "2", "3" ], "factors": {}, "lines": { "text-0": [ { "x": 96, "y": 108, "width": 161, "height": 18 }, { "x": 96, "y": 131.25, "width": 161, "height": 18 }, { "x": 96, "y": 154.5, "width": 168, "height": 18 }, { "x": 96, "y": 177.75, "width": 167, "height": 18 }, { "x": 96, "y": 201, "width": 182, "height": 18 }, { "x": 96, "y": 224.25, "width": 173, "height": 18 }, { "x": 96, "y": 247.5, "width": 147, "height": 18 }, { "x": 96, "y": 270.75, "width": 158, "height": 18 }, { "x": 96, "y": 294, "width": 159, "height": 18 }, { "x": 96, "y": 317.25, "width": 127, "height": 18 }, { "x": 96, "y": 340.5, "width": 150, "height": 18 }, { "x": 96, "y": 363.75, "width": 146, "height": 18 }, { "x": 96, "y": 387, "width": 158, "height": 18 }, { "x": 96, "y": 410.25, "width": 167, "height": 18 }, { "x": 96, "y": 433.5, "width": 142, "height": 18 }, { "x": 96, "y": 456.75, "width": 71, "height": 18 } ], "text-1": [ { "x": 308.65625, "y": 108, "width": 161, "height": 18 }, { "x": 308.65625, "y": 131.25, "width": 161, "height": 18 }, { "x": 308.65625, "y": 154.5, "width": 168, "height": 18 }, { "x": 308.65625, "y": 177.75, "width": 167, "height": 18 }, { "x": 308.65625, "y": 201, "width": 182, "height": 18 }, { "x": 308.65625, "y": 224.25, "width": 173, "height": 18 }, { "x": 308.65625, "y": 247.5, "width": 147, "height": 18 }, { "x": 308.65625, "y": 270.75, "width": 158, "height": 18 }, { "x": 308.65625, "y": 294, "width": 159, "height": 18 }, { "x": 308.65625, "y": 317.25, "width": 127, "height": 18 }, { "x": 308.65625, "y": 340.5, "width": 150, "height": 18 }, { "x": 308.65625, "y": 363.75, "width": 146, "height": 18 }, { "x": 308.65625, "y": 387, "width": 158, "height": 18 }, { "x": 308.65625, "y": 410.25, "width": 167, "height": 18 }, { "x": 308.65625, "y": 433.5, "width": 142, "height": 18 }, { "x": 308.65625, "y": 456.75, "width": 71, "height": 18 } ], "text-2": [ { "x": 521.328125, "y": 108, "width": 161, "height": 18 }, { "x": 521.328125, "y": 131.25, "width": 161, "height": 18 }, { "x": 521.328125, "y": 154.5, "width": 168, "height": 18 }, { "x": 521.328125, "y": 177.75, "width": 167, "height": 18 }, { "x": 521.328125, "y": 201, "width": 182, "height": 18 }, { "x": 521.328125, "y": 224.25, "width": 173, "height": 18 }, { "x": 521.328125, "y": 247.5, "width": 147, "height": 18 }, { "x": 521.328125, "y": 270.75, "width": 158, "height": 18 }, { "x": 521.328125, "y": 294, "width": 159, "height": 18 }, { "x": 521.328125, "y": 317.25, "width": 127, "height": 18 }, { "x": 521.328125, "y": 340.5, "width": 150, "height": 18 }, { "x": 521.328125, "y": 363.75, "width": 146, "height": 18 }, { "x": 521.328125, "y": 387, "width": 158, "height": 18 }, { "x": 521.328125, "y": 410.25, "width": 167, "height": 18 }, { "x": 521.328125, "y": 433.5, "width": 142, "height": 18 }, { "x": 521.328125, "y": 456.75, "width": 71, "height": 18 } ] } } }, { "task_id": "layoutbench-textcolumns-012", "family": "textcolumns", "group_id": "textcolumns-3-dark-1", "image_filename": "textcolumns-3-dark-1.png", "image_sha256": "7a2112804a99ff45987bc12cc3b11a9f8ae59924997cf456e5f6904c8e636228", "ground_truth": { "choice": "C" }, "prompt": "How many side-by-side columns of body text are visible inside the frame?\nA: Two\nB: One\nC: Three\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 1, "answer": "3", "options": [ "2", "1", "3" ], "factors": {}, "lines": { "text-0": [ { "x": 96, "y": 108, "width": 182.65625, "height": 18 }, { "x": 96, "y": 131.25, "width": 182.65625, "height": 18 }, { "x": 96, "y": 154.5, "width": 182.65625, "height": 18 }, { "x": 96, "y": 177.75, "width": 182.65625, "height": 18 }, { "x": 96, "y": 201, "width": 182.65625, "height": 18 }, { "x": 96, "y": 224.25, "width": 182.65625, "height": 18 }, { "x": 96, "y": 247.5, "width": 182.65625, "height": 18 }, { "x": 96, "y": 270.75, "width": 182.65625, "height": 18 }, { "x": 96, "y": 294, "width": 182.65625, "height": 18 }, { "x": 96, "y": 317.25, "width": 182.65625, "height": 18 }, { "x": 96, "y": 340.5, "width": 182.65625, "height": 18 }, { "x": 96, "y": 363.75, "width": 182.65625, "height": 18 }, { "x": 96, "y": 387, "width": 182.65625, "height": 18 }, { "x": 96, "y": 410.25, "width": 182.65625, "height": 18 }, { "x": 96, "y": 433.5, "width": 182.65625, "height": 18 }, { "x": 96, "y": 456.75, "width": 93, "height": 18 } ], "text-1": [ { "x": 308.65625, "y": 108, "width": 182.671875, "height": 18 }, { "x": 308.65625, "y": 131.25, "width": 182.671875, "height": 18 }, { "x": 308.65625, "y": 154.5, "width": 182.671875, "height": 18 }, { "x": 308.65625, "y": 177.75, "width": 182.671875, "height": 18 }, { "x": 308.65625, "y": 201, "width": 182.671875, "height": 18 }, { "x": 308.65625, "y": 224.25, "width": 182.671875, "height": 18 }, { "x": 308.65625, "y": 247.5, "width": 182.671875, "height": 18 }, { "x": 308.65625, "y": 270.75, "width": 182.671875, "height": 18 }, { "x": 308.65625, "y": 294, "width": 182.671875, "height": 18 }, { "x": 308.65625, "y": 317.25, "width": 182.671875, "height": 18 }, { "x": 308.65625, "y": 340.5, "width": 182.671875, "height": 18 }, { "x": 308.65625, "y": 363.75, "width": 182.671875, "height": 18 }, { "x": 308.65625, "y": 387, "width": 182.671875, "height": 18 }, { "x": 308.65625, "y": 410.25, "width": 182.671875, "height": 18 }, { "x": 308.65625, "y": 433.5, "width": 182.671875, "height": 18 }, { "x": 308.65625, "y": 456.75, "width": 93, "height": 18 } ], "text-2": [ { "x": 521.328125, "y": 108, "width": 182.671875, "height": 18 }, { "x": 521.328125, "y": 131.25, "width": 182.671875, "height": 18 }, { "x": 521.328125, "y": 154.5, "width": 182.671875, "height": 18 }, { "x": 521.328125, "y": 177.75, "width": 182.671875, "height": 18 }, { "x": 521.328125, "y": 201, "width": 182.671875, "height": 18 }, { "x": 521.328125, "y": 224.25, "width": 182.671875, "height": 18 }, { "x": 521.328125, "y": 247.5, "width": 182.671875, "height": 18 }, { "x": 521.328125, "y": 270.75, "width": 182.671875, "height": 18 }, { "x": 521.328125, "y": 294, "width": 182.671875, "height": 18 }, { "x": 521.328125, "y": 317.25, "width": 182.671875, "height": 18 }, { "x": 521.328125, "y": 340.5, "width": 182.671875, "height": 18 }, { "x": 521.328125, "y": 363.75, "width": 182.671875, "height": 18 }, { "x": 521.328125, "y": 387, "width": 182.671875, "height": 18 }, { "x": 521.328125, "y": 410.25, "width": 182.671875, "height": 18 }, { "x": 521.328125, "y": 433.5, "width": 182.671875, "height": 18 }, { "x": 521.328125, "y": 456.75, "width": 93, "height": 18 } ] } } }, { "task_id": "layoutbench-gridcols-001", "family": "gridcols", "group_id": "gridcols-2-light-0", "image_filename": "gridcols-2-light-0.png", "image_sha256": "dd1adba5db8f486b008cacd9f7a2028a6507ad49ac0dca1b4949847f0766c8c5", "ground_truth": { "choice": "A" }, "prompt": "How many columns of cards are in the rectangular grid? Count the cards across one complete row.\nA: Two\nB: Three\nC: Four\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 0, "answer": "2", "options": [ "2", "3", "4" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-gridcols-002", "family": "gridcols", "group_id": "gridcols-2-light-1", "image_filename": "gridcols-2-light-1.png", "image_sha256": "18acda7f7c618a05f19af3a8dbb4fb07588a9ae063feef024e7382c07aea9b8e", "ground_truth": { "choice": "B" }, "prompt": "How many columns of cards are in the rectangular grid? Count the cards across one complete row.\nA: Three\nB: Two\nC: Four\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 1, "answer": "2", "options": [ "3", "2", "4" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-gridcols-003", "family": "gridcols", "group_id": "gridcols-2-dark-0", "image_filename": "gridcols-2-dark-0.png", "image_sha256": "6fa5f99244d01e00f27463ac84f40c12de35ba8eaf860bae7d733ab364e2719d", "ground_truth": { "choice": "B" }, "prompt": "How many columns of cards are in the rectangular grid? Count the cards across one complete row.\nA: Four\nB: Two\nC: Three\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 0, "answer": "2", "options": [ "4", "2", "3" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-gridcols-004", "family": "gridcols", "group_id": "gridcols-2-dark-1", "image_filename": "gridcols-2-dark-1.png", "image_sha256": "2d88c8d5030683a96dee466b06a1b7ea7c529c97f2cd5fcdb52d6c1e5e9510ac", "ground_truth": { "choice": "C" }, "prompt": "How many columns of cards are in the rectangular grid? Count the cards across one complete row.\nA: Four\nB: Three\nC: Two\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 1, "answer": "2", "options": [ "4", "3", "2" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-gridcols-005", "family": "gridcols", "group_id": "gridcols-3-light-0", "image_filename": "gridcols-3-light-0.png", "image_sha256": "812b272d8b53cebe7c9205b8ba1e2e3954c08a0c9c5046e8b6ae392c120da238", "ground_truth": { "choice": "A" }, "prompt": "How many columns of cards are in the rectangular grid? Count the cards across one complete row.\nA: Three\nB: Four\nC: Two\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 0, "answer": "3", "options": [ "3", "4", "2" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-gridcols-006", "family": "gridcols", "group_id": "gridcols-3-light-1", "image_filename": "gridcols-3-light-1.png", "image_sha256": "69534ac5ed527b2dde3d2db25199c2447e246348ba45210cc6b13b58820a515f", "ground_truth": { "choice": "B" }, "prompt": "How many columns of cards are in the rectangular grid? Count the cards across one complete row.\nA: Two\nB: Three\nC: Four\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 1, "answer": "3", "options": [ "2", "3", "4" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-gridcols-007", "family": "gridcols", "group_id": "gridcols-3-dark-0", "image_filename": "gridcols-3-dark-0.png", "image_sha256": "0351f12f5f4f2d3aaf8680babc2d3df35120e94659f2c12ef163521e381c0bb1", "ground_truth": { "choice": "C" }, "prompt": "How many columns of cards are in the rectangular grid? Count the cards across one complete row.\nA: Four\nB: Two\nC: Three\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 0, "answer": "3", "options": [ "4", "2", "3" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-gridcols-008", "family": "gridcols", "group_id": "gridcols-3-dark-1", "image_filename": "gridcols-3-dark-1.png", "image_sha256": "f36d992202281f254b489f04029ec746b38366eab24c49e9acb5351a92d12070", "ground_truth": { "choice": "A" }, "prompt": "How many columns of cards are in the rectangular grid? Count the cards across one complete row.\nA: Three\nB: Four\nC: Two\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 1, "answer": "3", "options": [ "3", "4", "2" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-gridcols-009", "family": "gridcols", "group_id": "gridcols-4-light-0", "image_filename": "gridcols-4-light-0.png", "image_sha256": "e211130610b8d41a59b8cd8fd303fa7db8a2f567813c3147048f9e2da39ef8e9", "ground_truth": { "choice": "C" }, "prompt": "How many columns of cards are in the rectangular grid? Count the cards across one complete row.\nA: Two\nB: Three\nC: Four\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 0, "answer": "4", "options": [ "2", "3", "4" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-gridcols-010", "family": "gridcols", "group_id": "gridcols-4-light-1", "image_filename": "gridcols-4-light-1.png", "image_sha256": "bc6a8cdd59ff4756891a2ce2ca122f9d163a0309f35ce4cfa2be501275912c79", "ground_truth": { "choice": "A" }, "prompt": "How many columns of cards are in the rectangular grid? Count the cards across one complete row.\nA: Four\nB: Three\nC: Two\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 1, "answer": "4", "options": [ "4", "3", "2" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-gridcols-011", "family": "gridcols", "group_id": "gridcols-4-dark-0", "image_filename": "gridcols-4-dark-0.png", "image_sha256": "e39694825eef9f0fb1cfc923574d8533e29bb331a8dcc3242912551a3541e0b4", "ground_truth": { "choice": "B" }, "prompt": "How many columns of cards are in the rectangular grid? Count the cards across one complete row.\nA: Two\nB: Four\nC: Three\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 0, "answer": "4", "options": [ "2", "4", "3" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-gridcols-012", "family": "gridcols", "group_id": "gridcols-4-dark-1", "image_filename": "gridcols-4-dark-1.png", "image_sha256": "9e1de6feb34b56bcf4a18793fc16d571ba0c5e1235f5e1a632dc4d02b2467509", "ground_truth": { "choice": "C" }, "prompt": "How many columns of cards are in the rectangular grid? Count the cards across one complete row.\nA: Two\nB: Three\nC: Four\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 1, "answer": "4", "options": [ "2", "3", "4" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-gridrows-001", "family": "gridrows", "group_id": "gridcols-2-light-0", "image_filename": "gridcols-2-light-0.png", "image_sha256": "dd1adba5db8f486b008cacd9f7a2028a6507ad49ac0dca1b4949847f0766c8c5", "ground_truth": { "choice": "A" }, "prompt": "How many rows of cards are in the rectangular grid? Count the cards down one complete column.\nA: Two\nB: Three\nC: Four\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 0, "answer": "2", "options": [ "2", "3", "4" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-gridrows-002", "family": "gridrows", "group_id": "gridrows-2-light-1", "image_filename": "gridrows-2-light-1.png", "image_sha256": "b009117053a4957a26a13c5912b060e79a8905602543a636e5d5bc24f0bb5fdc", "ground_truth": { "choice": "B" }, "prompt": "How many rows of cards are in the rectangular grid? Count the cards down one complete column.\nA: Three\nB: Two\nC: Four\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 1, "answer": "2", "options": [ "3", "2", "4" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-gridrows-003", "family": "gridrows", "group_id": "gridcols-2-dark-0", "image_filename": "gridcols-2-dark-0.png", "image_sha256": "6fa5f99244d01e00f27463ac84f40c12de35ba8eaf860bae7d733ab364e2719d", "ground_truth": { "choice": "A" }, "prompt": "How many rows of cards are in the rectangular grid? Count the cards down one complete column.\nA: Two\nB: Three\nC: Four\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 0, "answer": "2", "options": [ "2", "3", "4" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-gridrows-004", "family": "gridrows", "group_id": "gridrows-2-dark-1", "image_filename": "gridrows-2-dark-1.png", "image_sha256": "5e9c134086843fcd5447a5084b3ac4b874bc3adf55c621600fecc742d7848b69", "ground_truth": { "choice": "C" }, "prompt": "How many rows of cards are in the rectangular grid? Count the cards down one complete column.\nA: Four\nB: Three\nC: Two\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 1, "answer": "2", "options": [ "4", "3", "2" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-gridrows-005", "family": "gridrows", "group_id": "gridrows-3-light-0", "image_filename": "gridrows-3-light-0.png", "image_sha256": "fc323619c2a79c900e3e6636a884ea6e63b4d11008b77983221ccddf8630431c", "ground_truth": { "choice": "C" }, "prompt": "How many rows of cards are in the rectangular grid? Count the cards down one complete column.\nA: Two\nB: Four\nC: Three\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 0, "answer": "3", "options": [ "2", "4", "3" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-gridrows-006", "family": "gridrows", "group_id": "gridcols-3-light-1", "image_filename": "gridcols-3-light-1.png", "image_sha256": "69534ac5ed527b2dde3d2db25199c2447e246348ba45210cc6b13b58820a515f", "ground_truth": { "choice": "C" }, "prompt": "How many rows of cards are in the rectangular grid? Count the cards down one complete column.\nA: Two\nB: Four\nC: Three\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 1, "answer": "3", "options": [ "2", "4", "3" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-gridrows-007", "family": "gridrows", "group_id": "gridrows-3-dark-0", "image_filename": "gridrows-3-dark-0.png", "image_sha256": "08e8f1cfe3c14b1b92802273a6cf783b7ea92e9ef4e18d20ae389945677f6ce1", "ground_truth": { "choice": "B" }, "prompt": "How many rows of cards are in the rectangular grid? Count the cards down one complete column.\nA: Two\nB: Three\nC: Four\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 0, "answer": "3", "options": [ "2", "3", "4" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-gridrows-008", "family": "gridrows", "group_id": "gridcols-3-dark-1", "image_filename": "gridcols-3-dark-1.png", "image_sha256": "f36d992202281f254b489f04029ec746b38366eab24c49e9acb5351a92d12070", "ground_truth": { "choice": "C" }, "prompt": "How many rows of cards are in the rectangular grid? Count the cards down one complete column.\nA: Two\nB: Four\nC: Three\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 1, "answer": "3", "options": [ "2", "4", "3" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-gridrows-009", "family": "gridrows", "group_id": "gridrows-4-light-0", "image_filename": "gridrows-4-light-0.png", "image_sha256": "0263bdb2d490f291a2ce043686bb6b3dfb903c51bdbbd4d0a530f2221d5ea841", "ground_truth": { "choice": "B" }, "prompt": "How many rows of cards are in the rectangular grid? Count the cards down one complete column.\nA: Two\nB: Four\nC: Three\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 0, "answer": "4", "options": [ "2", "4", "3" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-gridrows-010", "family": "gridrows", "group_id": "gridrows-4-light-1", "image_filename": "gridrows-4-light-1.png", "image_sha256": "1288f934549ad03f39e810de459e88579b0e56973d8d16943ed545e696ce38e5", "ground_truth": { "choice": "B" }, "prompt": "How many rows of cards are in the rectangular grid? Count the cards down one complete column.\nA: Three\nB: Four\nC: Two\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 1, "answer": "4", "options": [ "3", "4", "2" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-gridrows-011", "family": "gridrows", "group_id": "gridrows-4-dark-0", "image_filename": "gridrows-4-dark-0.png", "image_sha256": "a128b8a14494801896465b87c2bb8e40d905e235ae1f582d44a9d47eca5ee818", "ground_truth": { "choice": "A" }, "prompt": "How many rows of cards are in the rectangular grid? Count the cards down one complete column.\nA: Four\nB: Two\nC: Three\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 0, "answer": "4", "options": [ "4", "2", "3" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-gridrows-012", "family": "gridrows", "group_id": "gridrows-4-dark-1", "image_filename": "gridrows-4-dark-1.png", "image_sha256": "3957f045bd97a3caa2d62cc2142d59dc4dc84baac1891cbb3ac37b106d945a4a", "ground_truth": { "choice": "A" }, "prompt": "How many rows of cards are in the rectangular grid? Count the cards down one complete column.\nA: Four\nB: Three\nC: Two\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 1, "answer": "4", "options": [ "4", "3", "2" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-gridgaps-001", "family": "gridgaps", "group_id": "gridgaps-horizontal-light-0", "image_filename": "gridgaps-horizontal-light-0.png", "image_sha256": "d1df4b92e915ddce23934a0ee957fdeeca596611f34b69d00d2c977c04a6e51a", "ground_truth": { "choice": "C" }, "prompt": "Compare the horizontal empty gutters between columns with the vertical empty gutters between rows. Measure between the visible card boxes.\nA: The two gutter sizes are equal\nB: Vertical gutters are larger\nC: Horizontal gutters are larger\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 0, "answer": "horizontal", "options": [ "equal", "vertical", "horizontal" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-gridgaps-002", "family": "gridgaps", "group_id": "gridgaps-horizontal-light-1", "image_filename": "gridgaps-horizontal-light-1.png", "image_sha256": "a2a50b94b7d51187e4f2fdb7eb64a670ab44c5ca9ab5e5ee13c7d4d356f93b02", "ground_truth": { "choice": "B" }, "prompt": "Compare the horizontal empty gutters between columns with the vertical empty gutters between rows. Measure between the visible card boxes.\nA: The two gutter sizes are equal\nB: Horizontal gutters are larger\nC: Vertical gutters are larger\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 1, "answer": "horizontal", "options": [ "equal", "horizontal", "vertical" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-gridgaps-003", "family": "gridgaps", "group_id": "gridgaps-horizontal-dark-0", "image_filename": "gridgaps-horizontal-dark-0.png", "image_sha256": "a91d4a48ec49d3ca21202f9c33652a1599677a01df08b79eea7727304de7f146", "ground_truth": { "choice": "B" }, "prompt": "Compare the horizontal empty gutters between columns with the vertical empty gutters between rows. Measure between the visible card boxes.\nA: The two gutter sizes are equal\nB: Horizontal gutters are larger\nC: Vertical gutters are larger\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 0, "answer": "horizontal", "options": [ "equal", "horizontal", "vertical" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-gridgaps-004", "family": "gridgaps", "group_id": "gridgaps-horizontal-dark-1", "image_filename": "gridgaps-horizontal-dark-1.png", "image_sha256": "90160df0c6846be09f70273be7958da4107c42e26621506e429530e0f98d2f0c", "ground_truth": { "choice": "C" }, "prompt": "Compare the horizontal empty gutters between columns with the vertical empty gutters between rows. Measure between the visible card boxes.\nA: Vertical gutters are larger\nB: The two gutter sizes are equal\nC: Horizontal gutters are larger\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 1, "answer": "horizontal", "options": [ "vertical", "equal", "horizontal" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-gridgaps-005", "family": "gridgaps", "group_id": "gridgaps-vertical-light-0", "image_filename": "gridgaps-vertical-light-0.png", "image_sha256": "feb0fd98985c729d1671414e8e0efcd3572776d57c692076cbe3175aadd95f68", "ground_truth": { "choice": "A" }, "prompt": "Compare the horizontal empty gutters between columns with the vertical empty gutters between rows. Measure between the visible card boxes.\nA: Vertical gutters are larger\nB: Horizontal gutters are larger\nC: The two gutter sizes are equal\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 0, "answer": "vertical", "options": [ "vertical", "horizontal", "equal" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-gridgaps-006", "family": "gridgaps", "group_id": "gridgaps-vertical-light-1", "image_filename": "gridgaps-vertical-light-1.png", "image_sha256": "ec764924ec14bc8efee7302302fd6c57178068714d3c76dc20d67a7a01493948", "ground_truth": { "choice": "C" }, "prompt": "Compare the horizontal empty gutters between columns with the vertical empty gutters between rows. Measure between the visible card boxes.\nA: Horizontal gutters are larger\nB: The two gutter sizes are equal\nC: Vertical gutters are larger\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 1, "answer": "vertical", "options": [ "horizontal", "equal", "vertical" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-gridgaps-007", "family": "gridgaps", "group_id": "gridgaps-vertical-dark-0", "image_filename": "gridgaps-vertical-dark-0.png", "image_sha256": "8573792e663152c325cb5e7b97290691a02f38c1e4a1c02406f12f1bd4b26a9b", "ground_truth": { "choice": "B" }, "prompt": "Compare the horizontal empty gutters between columns with the vertical empty gutters between rows. Measure between the visible card boxes.\nA: The two gutter sizes are equal\nB: Vertical gutters are larger\nC: Horizontal gutters are larger\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 0, "answer": "vertical", "options": [ "equal", "vertical", "horizontal" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-gridgaps-008", "family": "gridgaps", "group_id": "gridgaps-vertical-dark-1", "image_filename": "gridgaps-vertical-dark-1.png", "image_sha256": "751cafd93f1cbd9a21e83671b34e152b80fad69dcf1b4ccbc888340cf467761f", "ground_truth": { "choice": "A" }, "prompt": "Compare the horizontal empty gutters between columns with the vertical empty gutters between rows. Measure between the visible card boxes.\nA: Vertical gutters are larger\nB: Horizontal gutters are larger\nC: The two gutter sizes are equal\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 1, "answer": "vertical", "options": [ "vertical", "horizontal", "equal" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-gridgaps-009", "family": "gridgaps", "group_id": "gridgaps-equal-light-0", "image_filename": "gridgaps-equal-light-0.png", "image_sha256": "1da20fc6045872218e5145c61d94b6a600a27b80e86335a110050b9fed8704dd", "ground_truth": { "choice": "A" }, "prompt": "Compare the horizontal empty gutters between columns with the vertical empty gutters between rows. Measure between the visible card boxes.\nA: The two gutter sizes are equal\nB: Vertical gutters are larger\nC: Horizontal gutters are larger\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 0, "answer": "equal", "options": [ "equal", "vertical", "horizontal" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-gridgaps-010", "family": "gridgaps", "group_id": "gridcols-3-light-1", "image_filename": "gridcols-3-light-1.png", "image_sha256": "69534ac5ed527b2dde3d2db25199c2447e246348ba45210cc6b13b58820a515f", "ground_truth": { "choice": "A" }, "prompt": "Compare the horizontal empty gutters between columns with the vertical empty gutters between rows. Measure between the visible card boxes.\nA: The two gutter sizes are equal\nB: Horizontal gutters are larger\nC: Vertical gutters are larger\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 1, "answer": "equal", "options": [ "equal", "horizontal", "vertical" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-gridgaps-011", "family": "gridgaps", "group_id": "gridgaps-equal-dark-0", "image_filename": "gridgaps-equal-dark-0.png", "image_sha256": "7665a3776248fd9fce596c339b45ce73d5c2580c85fa6e2991252f0e49c78bda", "ground_truth": { "choice": "B" }, "prompt": "Compare the horizontal empty gutters between columns with the vertical empty gutters between rows. Measure between the visible card boxes.\nA: Horizontal gutters are larger\nB: The two gutter sizes are equal\nC: Vertical gutters are larger\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 0, "answer": "equal", "options": [ "horizontal", "equal", "vertical" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-gridgaps-012", "family": "gridgaps", "group_id": "gridcols-3-dark-1", "image_filename": "gridcols-3-dark-1.png", "image_sha256": "f36d992202281f254b489f04029ec746b38366eab24c49e9acb5351a92d12070", "ground_truth": { "choice": "C" }, "prompt": "Compare the horizontal empty gutters between columns with the vertical empty gutters between rows. Measure between the visible card boxes.\nA: Horizontal gutters are larger\nB: Vertical gutters are larger\nC: The two gutter sizes are equal\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 1, "answer": "equal", "options": [ "horizontal", "vertical", "equal" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-gridtracks-001", "family": "gridtracks", "group_id": "gridgaps-equal-light-0", "image_filename": "gridgaps-equal-light-0.png", "image_sha256": "1da20fc6045872218e5145c61d94b6a600a27b80e86335a110050b9fed8704dd", "ground_truth": { "choice": "C" }, "prompt": "Compare the widths of the three card columns in this grid.\nA: The left column is wider; the other two have equal widths\nB: The right column is wider; the other two have equal widths\nC: All three columns have equal widths\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 0, "answer": "equal", "options": [ "left", "right", "equal" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-gridtracks-002", "family": "gridtracks", "group_id": "gridcols-3-light-1", "image_filename": "gridcols-3-light-1.png", "image_sha256": "69534ac5ed527b2dde3d2db25199c2447e246348ba45210cc6b13b58820a515f", "ground_truth": { "choice": "B" }, "prompt": "Compare the widths of the three card columns in this grid.\nA: The left column is wider; the other two have equal widths\nB: All three columns have equal widths\nC: The right column is wider; the other two have equal widths\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 1, "answer": "equal", "options": [ "left", "equal", "right" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-gridtracks-003", "family": "gridtracks", "group_id": "gridgaps-equal-dark-0", "image_filename": "gridgaps-equal-dark-0.png", "image_sha256": "7665a3776248fd9fce596c339b45ce73d5c2580c85fa6e2991252f0e49c78bda", "ground_truth": { "choice": "C" }, "prompt": "Compare the widths of the three card columns in this grid.\nA: The right column is wider; the other two have equal widths\nB: The left column is wider; the other two have equal widths\nC: All three columns have equal widths\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 0, "answer": "equal", "options": [ "right", "left", "equal" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-gridtracks-004", "family": "gridtracks", "group_id": "gridcols-3-dark-1", "image_filename": "gridcols-3-dark-1.png", "image_sha256": "f36d992202281f254b489f04029ec746b38366eab24c49e9acb5351a92d12070", "ground_truth": { "choice": "B" }, "prompt": "Compare the widths of the three card columns in this grid.\nA: The right column is wider; the other two have equal widths\nB: All three columns have equal widths\nC: The left column is wider; the other two have equal widths\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 1, "answer": "equal", "options": [ "right", "equal", "left" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-gridtracks-005", "family": "gridtracks", "group_id": "gridtracks-left-light-0", "image_filename": "gridtracks-left-light-0.png", "image_sha256": "f0e212b23db4ed3fd338d44001acedf73a067b4acd417f0be84571c5b1c1194b", "ground_truth": { "choice": "B" }, "prompt": "Compare the widths of the three card columns in this grid.\nA: All three columns have equal widths\nB: The left column is wider; the other two have equal widths\nC: The right column is wider; the other two have equal widths\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 0, "answer": "left", "options": [ "equal", "left", "right" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-gridtracks-006", "family": "gridtracks", "group_id": "gridtracks-left-light-1", "image_filename": "gridtracks-left-light-1.png", "image_sha256": "dc9b6fcb5eadf9ee9062cbe849f88fece0711aaedfc79e7d23091c2b14b3453a", "ground_truth": { "choice": "A" }, "prompt": "Compare the widths of the three card columns in this grid.\nA: The left column is wider; the other two have equal widths\nB: The right column is wider; the other two have equal widths\nC: All three columns have equal widths\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 1, "answer": "left", "options": [ "left", "right", "equal" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-gridtracks-007", "family": "gridtracks", "group_id": "gridtracks-left-dark-0", "image_filename": "gridtracks-left-dark-0.png", "image_sha256": "4a60bd7e28b6abf63d6da7792b1389ed101663e36c934e18d7e7442b1068b675", "ground_truth": { "choice": "A" }, "prompt": "Compare the widths of the three card columns in this grid.\nA: The left column is wider; the other two have equal widths\nB: The right column is wider; the other two have equal widths\nC: All three columns have equal widths\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 0, "answer": "left", "options": [ "left", "right", "equal" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-gridtracks-008", "family": "gridtracks", "group_id": "gridtracks-left-dark-1", "image_filename": "gridtracks-left-dark-1.png", "image_sha256": "cbe9f6b29fe105ea4b597e4cf9b5b060144d6b288c4a1419275bf2dae1c88060", "ground_truth": { "choice": "A" }, "prompt": "Compare the widths of the three card columns in this grid.\nA: The left column is wider; the other two have equal widths\nB: All three columns have equal widths\nC: The right column is wider; the other two have equal widths\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 1, "answer": "left", "options": [ "left", "equal", "right" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-gridtracks-009", "family": "gridtracks", "group_id": "gridtracks-right-light-0", "image_filename": "gridtracks-right-light-0.png", "image_sha256": "83daf8ff9fcb22586aaf6a8e6f08685f9482c2b784497be7152330d19948f140", "ground_truth": { "choice": "C" }, "prompt": "Compare the widths of the three card columns in this grid.\nA: All three columns have equal widths\nB: The left column is wider; the other two have equal widths\nC: The right column is wider; the other two have equal widths\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 0, "answer": "right", "options": [ "equal", "left", "right" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-gridtracks-010", "family": "gridtracks", "group_id": "gridtracks-right-light-1", "image_filename": "gridtracks-right-light-1.png", "image_sha256": "a51b46ab70129e8bb6e796e2ea7f918179eecc486a81197479d87c92e80714ab", "ground_truth": { "choice": "B" }, "prompt": "Compare the widths of the three card columns in this grid.\nA: All three columns have equal widths\nB: The right column is wider; the other two have equal widths\nC: The left column is wider; the other two have equal widths\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 1, "answer": "right", "options": [ "equal", "right", "left" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-gridtracks-011", "family": "gridtracks", "group_id": "gridtracks-right-dark-0", "image_filename": "gridtracks-right-dark-0.png", "image_sha256": "2b905573edc99531abce8079598897def2803d677b689e827289ce6ddcffa185", "ground_truth": { "choice": "A" }, "prompt": "Compare the widths of the three card columns in this grid.\nA: The right column is wider; the other two have equal widths\nB: All three columns have equal widths\nC: The left column is wider; the other two have equal widths\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 0, "answer": "right", "options": [ "right", "equal", "left" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-gridtracks-012", "family": "gridtracks", "group_id": "gridtracks-right-dark-1", "image_filename": "gridtracks-right-dark-1.png", "image_sha256": "978062e15a160e803e73e5afec4bc42eb18130f177742f3dcf740d8d973863dc", "ground_truth": { "choice": "C" }, "prompt": "Compare the widths of the three card columns in this grid.\nA: The left column is wider; the other two have equal widths\nB: All three columns have equal widths\nC: The right column is wider; the other two have equal widths\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 1, "answer": "right", "options": [ "left", "equal", "right" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-gridspan-001", "family": "gridspan", "group_id": "gridspan-1-light-0", "image_filename": "gridspan-1-light-0.png", "image_sha256": "5113879de5447472367eb5324a0417433871bd3529aa4fd1b0fca3e1fb588f04", "ground_truth": { "choice": "B" }, "prompt": "Use the three equal-width cards in the lower row as column guides. How many of those column positions does the single card in the upper row cover, including any intervening gutters?\nA: Three column positions\nB: One column position\nC: Two column positions\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 0, "answer": "1", "options": [ "3", "1", "2" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-gridspan-002", "family": "gridspan", "group_id": "gridspan-1-light-1", "image_filename": "gridspan-1-light-1.png", "image_sha256": "09dad52c37707c839600f15ea6c03ae0e92ed6343e095a863164b615e50dd864", "ground_truth": { "choice": "A" }, "prompt": "Use the three equal-width cards in the lower row as column guides. How many of those column positions does the single card in the upper row cover, including any intervening gutters?\nA: One column position\nB: Three column positions\nC: Two column positions\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 1, "answer": "1", "options": [ "1", "3", "2" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-gridspan-003", "family": "gridspan", "group_id": "gridspan-1-dark-0", "image_filename": "gridspan-1-dark-0.png", "image_sha256": "78154cae227f20d2389899d9f67326fee3b1b4da7118129cc02f5048a9f73f91", "ground_truth": { "choice": "C" }, "prompt": "Use the three equal-width cards in the lower row as column guides. How many of those column positions does the single card in the upper row cover, including any intervening gutters?\nA: Two column positions\nB: Three column positions\nC: One column position\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 0, "answer": "1", "options": [ "2", "3", "1" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-gridspan-004", "family": "gridspan", "group_id": "gridspan-1-dark-1", "image_filename": "gridspan-1-dark-1.png", "image_sha256": "a4a1298de87a7d5fa55542610f90f34f5acb449bfdcf54e32103f6ad43bcfcfc", "ground_truth": { "choice": "C" }, "prompt": "Use the three equal-width cards in the lower row as column guides. How many of those column positions does the single card in the upper row cover, including any intervening gutters?\nA: Two column positions\nB: Three column positions\nC: One column position\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 1, "answer": "1", "options": [ "2", "3", "1" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-gridspan-005", "family": "gridspan", "group_id": "gridspan-2-light-0", "image_filename": "gridspan-2-light-0.png", "image_sha256": "278545a076abcbca9cc809b4abeba5cd6ad47b3d10207960ee7b5a8dcb0eaadb", "ground_truth": { "choice": "C" }, "prompt": "Use the three equal-width cards in the lower row as column guides. How many of those column positions does the single card in the upper row cover, including any intervening gutters?\nA: Three column positions\nB: One column position\nC: Two column positions\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 0, "answer": "2", "options": [ "3", "1", "2" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-gridspan-006", "family": "gridspan", "group_id": "gridspan-2-light-1", "image_filename": "gridspan-2-light-1.png", "image_sha256": "b4896fa3bb1c23f884710e9cf0009678819726c2336bbaf5a9ef90e07b516f79", "ground_truth": { "choice": "A" }, "prompt": "Use the three equal-width cards in the lower row as column guides. How many of those column positions does the single card in the upper row cover, including any intervening gutters?\nA: Two column positions\nB: Three column positions\nC: One column position\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 1, "answer": "2", "options": [ "2", "3", "1" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-gridspan-007", "family": "gridspan", "group_id": "gridspan-2-dark-0", "image_filename": "gridspan-2-dark-0.png", "image_sha256": "c84bc711ebd52999d21ed6d805b2b96728de31f1fa3800d43d933c5732653776", "ground_truth": { "choice": "A" }, "prompt": "Use the three equal-width cards in the lower row as column guides. How many of those column positions does the single card in the upper row cover, including any intervening gutters?\nA: Two column positions\nB: Three column positions\nC: One column position\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 0, "answer": "2", "options": [ "2", "3", "1" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-gridspan-008", "family": "gridspan", "group_id": "gridspan-2-dark-1", "image_filename": "gridspan-2-dark-1.png", "image_sha256": "b717f60b0c9fb8b369bb5c9ee941300b7e69a0da0f76dc59ced325d02b0f1d12", "ground_truth": { "choice": "A" }, "prompt": "Use the three equal-width cards in the lower row as column guides. How many of those column positions does the single card in the upper row cover, including any intervening gutters?\nA: Two column positions\nB: One column position\nC: Three column positions\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 1, "answer": "2", "options": [ "2", "1", "3" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-gridspan-009", "family": "gridspan", "group_id": "gridspan-3-light-0", "image_filename": "gridspan-3-light-0.png", "image_sha256": "27cbb3bea8a70078841833f43663b15abc63ce04cb2fdd13f0554fcca8403181", "ground_truth": { "choice": "B" }, "prompt": "Use the three equal-width cards in the lower row as column guides. How many of those column positions does the single card in the upper row cover, including any intervening gutters?\nA: One column position\nB: Three column positions\nC: Two column positions\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 0, "answer": "3", "options": [ "1", "3", "2" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-gridspan-010", "family": "gridspan", "group_id": "gridspan-3-light-1", "image_filename": "gridspan-3-light-1.png", "image_sha256": "a08da053dedba4b439a7657a7b565da080619f7c2d798e06bf83b0332662d684", "ground_truth": { "choice": "B" }, "prompt": "Use the three equal-width cards in the lower row as column guides. How many of those column positions does the single card in the upper row cover, including any intervening gutters?\nA: Two column positions\nB: Three column positions\nC: One column position\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 1, "answer": "3", "options": [ "2", "3", "1" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-gridspan-011", "family": "gridspan", "group_id": "gridspan-3-dark-0", "image_filename": "gridspan-3-dark-0.png", "image_sha256": "753e859b97c8e2317e7adaec19cc8e3217a106f93542354c9142624c065d4b25", "ground_truth": { "choice": "B" }, "prompt": "Use the three equal-width cards in the lower row as column guides. How many of those column positions does the single card in the upper row cover, including any intervening gutters?\nA: Two column positions\nB: Three column positions\nC: One column position\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 0, "answer": "3", "options": [ "2", "3", "1" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-gridspan-012", "family": "gridspan", "group_id": "gridspan-3-dark-1", "image_filename": "gridspan-3-dark-1.png", "image_sha256": "6e476c5eb1b07e94a4090068ec8282b846207d210e01096215498232e50fc0a3", "ground_truth": { "choice": "C" }, "prompt": "Use the three equal-width cards in the lower row as column guides. How many of those column positions does the single card in the upper row cover, including any intervening gutters?\nA: One column position\nB: Two column positions\nC: Three column positions\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 1, "answer": "3", "options": [ "1", "2", "3" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-wrapcount-001", "family": "wrapcount", "group_id": "wrapcount-1-light-0", "image_filename": "wrapcount-1-light-0.png", "image_sha256": "8068924055f3c867ac47f96855b526f9f0d9e55ab8fbaac9b7ed7e585fb689b5", "ground_truth": { "choice": "A" }, "prompt": "How many horizontal rows of small cards are visible inside the frame?\nA: One row\nB: Three rows\nC: Two rows\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 0, "answer": "1", "options": [ "1", "3", "2" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-wrapcount-002", "family": "wrapcount", "group_id": "wrapcount-1-light-1", "image_filename": "wrapcount-1-light-1.png", "image_sha256": "b138b6a69d71fb2b1d7a71ca58ae615ee35f2e6fecfd457ad7baf5e3107a2e1f", "ground_truth": { "choice": "A" }, "prompt": "How many horizontal rows of small cards are visible inside the frame?\nA: One row\nB: Two rows\nC: Three rows\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 1, "answer": "1", "options": [ "1", "2", "3" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-wrapcount-003", "family": "wrapcount", "group_id": "wrapcount-1-dark-0", "image_filename": "wrapcount-1-dark-0.png", "image_sha256": "4e90c93c89ba8154640bc88379ec46db9a2587de7204e07599befad4162dbaba", "ground_truth": { "choice": "A" }, "prompt": "How many horizontal rows of small cards are visible inside the frame?\nA: One row\nB: Two rows\nC: Three rows\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 0, "answer": "1", "options": [ "1", "2", "3" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-wrapcount-004", "family": "wrapcount", "group_id": "wrapcount-1-dark-1", "image_filename": "wrapcount-1-dark-1.png", "image_sha256": "c44a030d6f639864188327ac3752600da70dbf170eabe627c3a4680bde2021a6", "ground_truth": { "choice": "B" }, "prompt": "How many horizontal rows of small cards are visible inside the frame?\nA: Two rows\nB: One row\nC: Three rows\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 1, "answer": "1", "options": [ "2", "1", "3" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-wrapcount-005", "family": "wrapcount", "group_id": "wrapcount-2-light-0", "image_filename": "wrapcount-2-light-0.png", "image_sha256": "5f6dd957b89f6660e886cf03eb4bb0b81e6622b22c6bd9a77d7f46901f842a60", "ground_truth": { "choice": "C" }, "prompt": "How many horizontal rows of small cards are visible inside the frame?\nA: Three rows\nB: One row\nC: Two rows\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 0, "answer": "2", "options": [ "3", "1", "2" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-wrapcount-006", "family": "wrapcount", "group_id": "wrapcount-2-light-1", "image_filename": "wrapcount-2-light-1.png", "image_sha256": "14588c5f12de52cd51d23e17743279f5a2468f2327046fa33ce54c3a2a42e858", "ground_truth": { "choice": "C" }, "prompt": "How many horizontal rows of small cards are visible inside the frame?\nA: Three rows\nB: One row\nC: Two rows\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 1, "answer": "2", "options": [ "3", "1", "2" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-wrapcount-007", "family": "wrapcount", "group_id": "wrapcount-2-dark-0", "image_filename": "wrapcount-2-dark-0.png", "image_sha256": "0cbda80278412db7692f3674b521e0172e8248f5c3be3ff3bc457cd8e5ecb64e", "ground_truth": { "choice": "B" }, "prompt": "How many horizontal rows of small cards are visible inside the frame?\nA: One row\nB: Two rows\nC: Three rows\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 0, "answer": "2", "options": [ "1", "2", "3" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-wrapcount-008", "family": "wrapcount", "group_id": "wrapcount-2-dark-1", "image_filename": "wrapcount-2-dark-1.png", "image_sha256": "c4909ee6b6a7455c7db8e81a4e5bef93cab7854991d7afad54cdaaaeb3dc686c", "ground_truth": { "choice": "A" }, "prompt": "How many horizontal rows of small cards are visible inside the frame?\nA: Two rows\nB: One row\nC: Three rows\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 1, "answer": "2", "options": [ "2", "1", "3" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-wrapcount-009", "family": "wrapcount", "group_id": "wrapcount-3-light-0", "image_filename": "wrapcount-3-light-0.png", "image_sha256": "b668d6650ec1a07ab46e39d21546acd28abf3067a94981341b313b25e004e851", "ground_truth": { "choice": "B" }, "prompt": "How many horizontal rows of small cards are visible inside the frame?\nA: Two rows\nB: Three rows\nC: One row\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 0, "answer": "3", "options": [ "2", "3", "1" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-wrapcount-010", "family": "wrapcount", "group_id": "wrapcount-3-light-1", "image_filename": "wrapcount-3-light-1.png", "image_sha256": "f8015e74d11ec40cdd964eb03da44aecf26a02985e94e3f4d59e2ddd8db3df39", "ground_truth": { "choice": "C" }, "prompt": "How many horizontal rows of small cards are visible inside the frame?\nA: One row\nB: Two rows\nC: Three rows\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 1, "answer": "3", "options": [ "1", "2", "3" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-wrapcount-011", "family": "wrapcount", "group_id": "wrapcount-3-dark-0", "image_filename": "wrapcount-3-dark-0.png", "image_sha256": "10adae27ebac8e11278eecbfaccd35c3de4cfd63ff5a461dbc98e5e6174d24b9", "ground_truth": { "choice": "C" }, "prompt": "How many horizontal rows of small cards are visible inside the frame?\nA: One row\nB: Two rows\nC: Three rows\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 0, "answer": "3", "options": [ "1", "2", "3" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-wrapcount-012", "family": "wrapcount", "group_id": "wrapcount-3-dark-1", "image_filename": "wrapcount-3-dark-1.png", "image_sha256": "1b47a0b631596604ccfe93ebf5f3b23d730064cbc4ff5a9058e18f7d1466dabd", "ground_truth": { "choice": "B" }, "prompt": "How many horizontal rows of small cards are visible inside the frame?\nA: One row\nB: Three rows\nC: Two rows\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 1, "answer": "3", "options": [ "1", "3", "2" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-wrapalign-001", "family": "wrapalign", "group_id": "wrapalign-left-light-0", "image_filename": "wrapalign-left-light-0.png", "image_sha256": "2581d72463a54c2b3a78c2a211ec00f349c59a01182a1d656c4ef80b17801ef3", "ground_truth": { "choice": "B" }, "prompt": "Look only at the shorter final row of cards. How is that row placed horizontally inside the outlined frame?\nA: Against the right inner edge\nB: Against the left inner edge\nC: Centered, with equal empty space on either side\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 0, "answer": "left", "options": [ "right", "left", "center" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-wrapalign-002", "family": "wrapalign", "group_id": "wrapalign-left-light-1", "image_filename": "wrapalign-left-light-1.png", "image_sha256": "1fefa1c83320903e377f47458054c254955bff38821e503f6dbd782067a07837", "ground_truth": { "choice": "C" }, "prompt": "Look only at the shorter final row of cards. How is that row placed horizontally inside the outlined frame?\nA: Centered, with equal empty space on either side\nB: Against the right inner edge\nC: Against the left inner edge\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 1, "answer": "left", "options": [ "center", "right", "left" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-wrapalign-003", "family": "wrapalign", "group_id": "wrapalign-left-dark-0", "image_filename": "wrapalign-left-dark-0.png", "image_sha256": "287e44f6667d59cb8c85cf4a4da75119a5fe70556912bd058edc0ec36d3d10db", "ground_truth": { "choice": "A" }, "prompt": "Look only at the shorter final row of cards. How is that row placed horizontally inside the outlined frame?\nA: Against the left inner edge\nB: Centered, with equal empty space on either side\nC: Against the right inner edge\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 0, "answer": "left", "options": [ "left", "center", "right" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-wrapalign-004", "family": "wrapalign", "group_id": "wrapalign-left-dark-1", "image_filename": "wrapalign-left-dark-1.png", "image_sha256": "895ca53e74780ff9f2db4a7014d254c744e96c604f04852c67e10b5ae2358f04", "ground_truth": { "choice": "A" }, "prompt": "Look only at the shorter final row of cards. How is that row placed horizontally inside the outlined frame?\nA: Against the left inner edge\nB: Against the right inner edge\nC: Centered, with equal empty space on either side\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 1, "answer": "left", "options": [ "left", "right", "center" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-wrapalign-005", "family": "wrapalign", "group_id": "wrapalign-center-light-0", "image_filename": "wrapalign-center-light-0.png", "image_sha256": "3cfbcc89320fba4607808bfa17a424c6da3e1ebf4d3193a409b0f67fb61f03a6", "ground_truth": { "choice": "A" }, "prompt": "Look only at the shorter final row of cards. How is that row placed horizontally inside the outlined frame?\nA: Centered, with equal empty space on either side\nB: Against the right inner edge\nC: Against the left inner edge\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 0, "answer": "center", "options": [ "center", "right", "left" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-wrapalign-006", "family": "wrapalign", "group_id": "wrapalign-center-light-1", "image_filename": "wrapalign-center-light-1.png", "image_sha256": "14f301c90bae0a2f04c1c81aa88cd6e69afe342201096274b1b627daa98e6d0f", "ground_truth": { "choice": "B" }, "prompt": "Look only at the shorter final row of cards. How is that row placed horizontally inside the outlined frame?\nA: Against the left inner edge\nB: Centered, with equal empty space on either side\nC: Against the right inner edge\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 1, "answer": "center", "options": [ "left", "center", "right" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-wrapalign-007", "family": "wrapalign", "group_id": "wrapalign-center-dark-0", "image_filename": "wrapalign-center-dark-0.png", "image_sha256": "e215aec67f064c31061b6b9a1b87e2ff18d75e7f92cf40f0cbfe474ca18bfd9a", "ground_truth": { "choice": "B" }, "prompt": "Look only at the shorter final row of cards. How is that row placed horizontally inside the outlined frame?\nA: Against the right inner edge\nB: Centered, with equal empty space on either side\nC: Against the left inner edge\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 0, "answer": "center", "options": [ "right", "center", "left" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-wrapalign-008", "family": "wrapalign", "group_id": "wrapalign-center-dark-1", "image_filename": "wrapalign-center-dark-1.png", "image_sha256": "41825df3c3836b765b03940b2125e8bff705eb9efc29c091d680ce1b54f45916", "ground_truth": { "choice": "C" }, "prompt": "Look only at the shorter final row of cards. How is that row placed horizontally inside the outlined frame?\nA: Against the left inner edge\nB: Against the right inner edge\nC: Centered, with equal empty space on either side\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 1, "answer": "center", "options": [ "left", "right", "center" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-wrapalign-009", "family": "wrapalign", "group_id": "wrapalign-right-light-0", "image_filename": "wrapalign-right-light-0.png", "image_sha256": "31c0f17c4693d44976136e660a71fdbbe49e08112dfb5f0ae4a9e792e8e86076", "ground_truth": { "choice": "B" }, "prompt": "Look only at the shorter final row of cards. How is that row placed horizontally inside the outlined frame?\nA: Centered, with equal empty space on either side\nB: Against the right inner edge\nC: Against the left inner edge\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 0, "answer": "right", "options": [ "center", "right", "left" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-wrapalign-010", "family": "wrapalign", "group_id": "wrapalign-right-light-1", "image_filename": "wrapalign-right-light-1.png", "image_sha256": "6256166889394e1725ed5772027cfb2ff6f3bd865044984f95a2dc949f49b616", "ground_truth": { "choice": "C" }, "prompt": "Look only at the shorter final row of cards. How is that row placed horizontally inside the outlined frame?\nA: Centered, with equal empty space on either side\nB: Against the left inner edge\nC: Against the right inner edge\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 1, "answer": "right", "options": [ "center", "left", "right" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-wrapalign-011", "family": "wrapalign", "group_id": "wrapalign-right-dark-0", "image_filename": "wrapalign-right-dark-0.png", "image_sha256": "52b53c48b26628aaceb090540a6a4cf590a6c34886ba826815bf8cf18ba21208", "ground_truth": { "choice": "C" }, "prompt": "Look only at the shorter final row of cards. How is that row placed horizontally inside the outlined frame?\nA: Against the left inner edge\nB: Centered, with equal empty space on either side\nC: Against the right inner edge\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 0, "answer": "right", "options": [ "left", "center", "right" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-wrapalign-012", "family": "wrapalign", "group_id": "wrapalign-right-dark-1", "image_filename": "wrapalign-right-dark-1.png", "image_sha256": "a80216528ef5adb2ac7e9a6b14b045d146618630f8aa4ef53c57255a09780f33", "ground_truth": { "choice": "A" }, "prompt": "Look only at the shorter final row of cards. How is that row placed horizontally inside the outlined frame?\nA: Against the right inner edge\nB: Centered, with equal empty space on either side\nC: Against the left inner edge\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 1, "answer": "right", "options": [ "right", "center", "left" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-gapcompare-001", "family": "gapcompare", "group_id": "gapcompare-first-light-0", "image_filename": "gapcompare-first-light-0.png", "image_sha256": "cc419375281a36f3a49b6d9072f8b636e4a61d20f6df2e324c32900271406e3a", "ground_truth": { "choice": "C" }, "prompt": "Compare the two empty gaps between the three card boxes, reading left to right for a row or top to bottom for a column.\nA: The gaps are equal\nB: The second gap is larger\nC: The first gap is larger\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 0, "answer": "first", "options": [ "equal", "last", "first" ], "factors": { "direction": "row" }, "lines": {} } }, { "task_id": "layoutbench-gapcompare-002", "family": "gapcompare", "group_id": "gapcompare-first-light-1", "image_filename": "gapcompare-first-light-1.png", "image_sha256": "2e49a4e095e3abcfc4c60afb148f03bf6ae4ecd2aee9bd87fa1048bbdeb9f335", "ground_truth": { "choice": "B" }, "prompt": "Compare the two empty gaps between the three card boxes, reading left to right for a row or top to bottom for a column.\nA: The gaps are equal\nB: The first gap is larger\nC: The second gap is larger\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 1, "answer": "first", "options": [ "equal", "first", "last" ], "factors": { "direction": "column" }, "lines": {} } }, { "task_id": "layoutbench-gapcompare-003", "family": "gapcompare", "group_id": "gapcompare-first-dark-0", "image_filename": "gapcompare-first-dark-0.png", "image_sha256": "ece00f4c71da4c7da23e19231e904477aefcf1a791afb53334a71b5887ee7d65", "ground_truth": { "choice": "B" }, "prompt": "Compare the two empty gaps between the three card boxes, reading left to right for a row or top to bottom for a column.\nA: The gaps are equal\nB: The first gap is larger\nC: The second gap is larger\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 0, "answer": "first", "options": [ "equal", "first", "last" ], "factors": { "direction": "row" }, "lines": {} } }, { "task_id": "layoutbench-gapcompare-004", "family": "gapcompare", "group_id": "gapcompare-first-dark-1", "image_filename": "gapcompare-first-dark-1.png", "image_sha256": "e137bcfefabfc208bd6aabcfcb44a1301b33070dffcd646a323365fe21a842e8", "ground_truth": { "choice": "A" }, "prompt": "Compare the two empty gaps between the three card boxes, reading left to right for a row or top to bottom for a column.\nA: The first gap is larger\nB: The second gap is larger\nC: The gaps are equal\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 1, "answer": "first", "options": [ "first", "last", "equal" ], "factors": { "direction": "column" }, "lines": {} } }, { "task_id": "layoutbench-gapcompare-005", "family": "gapcompare", "group_id": "gapcompare-last-light-0", "image_filename": "gapcompare-last-light-0.png", "image_sha256": "54e5f23e4961afdcbe4a89d5d98726e3afd3e67b8f49ddd8f8ae7ff6de8639f4", "ground_truth": { "choice": "A" }, "prompt": "Compare the two empty gaps between the three card boxes, reading left to right for a row or top to bottom for a column.\nA: The second gap is larger\nB: The gaps are equal\nC: The first gap is larger\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 0, "answer": "last", "options": [ "last", "equal", "first" ], "factors": { "direction": "row" }, "lines": {} } }, { "task_id": "layoutbench-gapcompare-006", "family": "gapcompare", "group_id": "gapcompare-last-light-1", "image_filename": "gapcompare-last-light-1.png", "image_sha256": "c4c3576b8259241e6f91d9d74fe8c249548ff44fc3a2a22c489e6617f52f2d07", "ground_truth": { "choice": "C" }, "prompt": "Compare the two empty gaps between the three card boxes, reading left to right for a row or top to bottom for a column.\nA: The gaps are equal\nB: The first gap is larger\nC: The second gap is larger\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 1, "answer": "last", "options": [ "equal", "first", "last" ], "factors": { "direction": "column" }, "lines": {} } }, { "task_id": "layoutbench-gapcompare-007", "family": "gapcompare", "group_id": "gapcompare-last-dark-0", "image_filename": "gapcompare-last-dark-0.png", "image_sha256": "74e309b66c3c476507138caea045dd8032fa8ec676990f4dd59fc0cf7f6ba09a", "ground_truth": { "choice": "B" }, "prompt": "Compare the two empty gaps between the three card boxes, reading left to right for a row or top to bottom for a column.\nA: The gaps are equal\nB: The second gap is larger\nC: The first gap is larger\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 0, "answer": "last", "options": [ "equal", "last", "first" ], "factors": { "direction": "row" }, "lines": {} } }, { "task_id": "layoutbench-gapcompare-008", "family": "gapcompare", "group_id": "gapcompare-last-dark-1", "image_filename": "gapcompare-last-dark-1.png", "image_sha256": "d93a1a8cf7126019bac18613708f8cb50bf094ee9e3b761088a28d8b5e1dd7f4", "ground_truth": { "choice": "C" }, "prompt": "Compare the two empty gaps between the three card boxes, reading left to right for a row or top to bottom for a column.\nA: The first gap is larger\nB: The gaps are equal\nC: The second gap is larger\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 1, "answer": "last", "options": [ "first", "equal", "last" ], "factors": { "direction": "column" }, "lines": {} } }, { "task_id": "layoutbench-gapcompare-009", "family": "gapcompare", "group_id": "gapcompare-equal-light-0", "image_filename": "gapcompare-equal-light-0.png", "image_sha256": "3a5f7e2741ed276c056812d0d9942666c7ef98bcf6fc6b9da467751543047ccf", "ground_truth": { "choice": "A" }, "prompt": "Compare the two empty gaps between the three card boxes, reading left to right for a row or top to bottom for a column.\nA: The gaps are equal\nB: The second gap is larger\nC: The first gap is larger\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 0, "answer": "equal", "options": [ "equal", "last", "first" ], "factors": { "direction": "row" }, "lines": {} } }, { "task_id": "layoutbench-gapcompare-010", "family": "gapcompare", "group_id": "gapcompare-equal-light-1", "image_filename": "gapcompare-equal-light-1.png", "image_sha256": "8256b1f42e57aa77e7f20f68bf3de037068ed496e65d80a22235aa3df55dc585", "ground_truth": { "choice": "A" }, "prompt": "Compare the two empty gaps between the three card boxes, reading left to right for a row or top to bottom for a column.\nA: The gaps are equal\nB: The second gap is larger\nC: The first gap is larger\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 1, "answer": "equal", "options": [ "equal", "last", "first" ], "factors": { "direction": "column" }, "lines": {} } }, { "task_id": "layoutbench-gapcompare-011", "family": "gapcompare", "group_id": "gapcompare-equal-dark-0", "image_filename": "gapcompare-equal-dark-0.png", "image_sha256": "a48214a509738ad6006af8ab96f7de94e73ddc76969af1026c7526367ebbeda9", "ground_truth": { "choice": "B" }, "prompt": "Compare the two empty gaps between the three card boxes, reading left to right for a row or top to bottom for a column.\nA: The second gap is larger\nB: The gaps are equal\nC: The first gap is larger\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 0, "answer": "equal", "options": [ "last", "equal", "first" ], "factors": { "direction": "row" }, "lines": {} } }, { "task_id": "layoutbench-gapcompare-012", "family": "gapcompare", "group_id": "gapcompare-equal-dark-1", "image_filename": "gapcompare-equal-dark-1.png", "image_sha256": "e534c1c57f40c439acac755cc8919d1e7a7af12d6dc1591091adab2a2c650a27", "ground_truth": { "choice": "C" }, "prompt": "Compare the two empty gaps between the three card boxes, reading left to right for a row or top to bottom for a column.\nA: The second gap is larger\nB: The first gap is larger\nC: The gaps are equal\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 1, "answer": "equal", "options": [ "last", "first", "equal" ], "factors": { "direction": "column" }, "lines": {} } }, { "task_id": "layoutbench-padcompare-001", "family": "padcompare", "group_id": "padcompare-horizontal-light-0", "image_filename": "padcompare-horizontal-light-0.png", "image_sha256": "5b296a8ad48fae5b768bb9600813d4fc21f7e93126a111479312b3bb5a04d626", "ground_truth": { "choice": "C" }, "prompt": "The tinted rectangle is centered inside the outlined frame. Compare the horizontal inset (frame to rectangle on either side) with the vertical inset (frame to rectangle above or below). Measure to the rectangle's visible outer edge, not to its text.\nA: The vertical inset is larger\nB: The horizontal and vertical insets are equal\nC: The horizontal inset is larger\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 0, "answer": "horizontal", "options": [ "vertical", "equal", "horizontal" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-padcompare-002", "family": "padcompare", "group_id": "padcompare-horizontal-light-1", "image_filename": "padcompare-horizontal-light-1.png", "image_sha256": "95ce6788a34ca142b9bb9bb0d99322e25cb680df13310774674cff9a8e3a0132", "ground_truth": { "choice": "A" }, "prompt": "The tinted rectangle is centered inside the outlined frame. Compare the horizontal inset (frame to rectangle on either side) with the vertical inset (frame to rectangle above or below). Measure to the rectangle's visible outer edge, not to its text.\nA: The horizontal inset is larger\nB: The horizontal and vertical insets are equal\nC: The vertical inset is larger\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 1, "answer": "horizontal", "options": [ "horizontal", "equal", "vertical" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-padcompare-003", "family": "padcompare", "group_id": "padcompare-horizontal-dark-0", "image_filename": "padcompare-horizontal-dark-0.png", "image_sha256": "b277e9f0173d3fc4170b0a539af2a0500980de929c094a164aa512c3ee71ffa6", "ground_truth": { "choice": "B" }, "prompt": "The tinted rectangle is centered inside the outlined frame. Compare the horizontal inset (frame to rectangle on either side) with the vertical inset (frame to rectangle above or below). Measure to the rectangle's visible outer edge, not to its text.\nA: The vertical inset is larger\nB: The horizontal inset is larger\nC: The horizontal and vertical insets are equal\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 0, "answer": "horizontal", "options": [ "vertical", "horizontal", "equal" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-padcompare-004", "family": "padcompare", "group_id": "padcompare-horizontal-dark-1", "image_filename": "padcompare-horizontal-dark-1.png", "image_sha256": "f4207b17388390034c7a395a612c36f56bc7937d7f6a7e092c10b9a3cc935266", "ground_truth": { "choice": "B" }, "prompt": "The tinted rectangle is centered inside the outlined frame. Compare the horizontal inset (frame to rectangle on either side) with the vertical inset (frame to rectangle above or below). Measure to the rectangle's visible outer edge, not to its text.\nA: The vertical inset is larger\nB: The horizontal inset is larger\nC: The horizontal and vertical insets are equal\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 1, "answer": "horizontal", "options": [ "vertical", "horizontal", "equal" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-padcompare-005", "family": "padcompare", "group_id": "padcompare-vertical-light-0", "image_filename": "padcompare-vertical-light-0.png", "image_sha256": "456a4f4f79d28afcce09489c8ec09dfd8ea636271ca39b6031d7e9dd8a15beaf", "ground_truth": { "choice": "A" }, "prompt": "The tinted rectangle is centered inside the outlined frame. Compare the horizontal inset (frame to rectangle on either side) with the vertical inset (frame to rectangle above or below). Measure to the rectangle's visible outer edge, not to its text.\nA: The vertical inset is larger\nB: The horizontal and vertical insets are equal\nC: The horizontal inset is larger\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 0, "answer": "vertical", "options": [ "vertical", "equal", "horizontal" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-padcompare-006", "family": "padcompare", "group_id": "padcompare-vertical-light-1", "image_filename": "padcompare-vertical-light-1.png", "image_sha256": "acd7ccb49452169ee11251745f7b9861449993832d13e3bf9b758234d3ecb3ff", "ground_truth": { "choice": "C" }, "prompt": "The tinted rectangle is centered inside the outlined frame. Compare the horizontal inset (frame to rectangle on either side) with the vertical inset (frame to rectangle above or below). Measure to the rectangle's visible outer edge, not to its text.\nA: The horizontal and vertical insets are equal\nB: The horizontal inset is larger\nC: The vertical inset is larger\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 1, "answer": "vertical", "options": [ "equal", "horizontal", "vertical" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-padcompare-007", "family": "padcompare", "group_id": "padcompare-vertical-dark-0", "image_filename": "padcompare-vertical-dark-0.png", "image_sha256": "a5b99dab5bcad52f3e412facd7e2f2c95482da7381589baaa4ca8fd39e510916", "ground_truth": { "choice": "A" }, "prompt": "The tinted rectangle is centered inside the outlined frame. Compare the horizontal inset (frame to rectangle on either side) with the vertical inset (frame to rectangle above or below). Measure to the rectangle's visible outer edge, not to its text.\nA: The vertical inset is larger\nB: The horizontal inset is larger\nC: The horizontal and vertical insets are equal\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 0, "answer": "vertical", "options": [ "vertical", "horizontal", "equal" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-padcompare-008", "family": "padcompare", "group_id": "padcompare-vertical-dark-1", "image_filename": "padcompare-vertical-dark-1.png", "image_sha256": "a6a85191cf30f3994733c9356d031a968e78e82b8989e6f77e1c43c3cdda825d", "ground_truth": { "choice": "B" }, "prompt": "The tinted rectangle is centered inside the outlined frame. Compare the horizontal inset (frame to rectangle on either side) with the vertical inset (frame to rectangle above or below). Measure to the rectangle's visible outer edge, not to its text.\nA: The horizontal inset is larger\nB: The vertical inset is larger\nC: The horizontal and vertical insets are equal\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 1, "answer": "vertical", "options": [ "horizontal", "vertical", "equal" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-padcompare-009", "family": "padcompare", "group_id": "padcompare-equal-light-0", "image_filename": "padcompare-equal-light-0.png", "image_sha256": "1da94e96dd505f78b50f88bdef08b1ee369de72aa7834e1f62f462496ff2f737", "ground_truth": { "choice": "B" }, "prompt": "The tinted rectangle is centered inside the outlined frame. Compare the horizontal inset (frame to rectangle on either side) with the vertical inset (frame to rectangle above or below). Measure to the rectangle's visible outer edge, not to its text.\nA: The vertical inset is larger\nB: The horizontal and vertical insets are equal\nC: The horizontal inset is larger\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 0, "answer": "equal", "options": [ "vertical", "equal", "horizontal" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-padcompare-010", "family": "padcompare", "group_id": "padcompare-equal-light-1", "image_filename": "padcompare-equal-light-1.png", "image_sha256": "762069b171a8cb02d979c2dbccf6780e96ed4281ff459650617d24547a6d8440", "ground_truth": { "choice": "A" }, "prompt": "The tinted rectangle is centered inside the outlined frame. Compare the horizontal inset (frame to rectangle on either side) with the vertical inset (frame to rectangle above or below). Measure to the rectangle's visible outer edge, not to its text.\nA: The horizontal and vertical insets are equal\nB: The horizontal inset is larger\nC: The vertical inset is larger\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 1, "answer": "equal", "options": [ "equal", "horizontal", "vertical" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-padcompare-011", "family": "padcompare", "group_id": "padcompare-equal-dark-0", "image_filename": "padcompare-equal-dark-0.png", "image_sha256": "6e2ae037b0751829b32e931f8f8ac2283f1189a3e618a794f7c3f6ca40204e13", "ground_truth": { "choice": "C" }, "prompt": "The tinted rectangle is centered inside the outlined frame. Compare the horizontal inset (frame to rectangle on either side) with the vertical inset (frame to rectangle above or below). Measure to the rectangle's visible outer edge, not to its text.\nA: The vertical inset is larger\nB: The horizontal inset is larger\nC: The horizontal and vertical insets are equal\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 0, "answer": "equal", "options": [ "vertical", "horizontal", "equal" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-padcompare-012", "family": "padcompare", "group_id": "padcompare-equal-dark-1", "image_filename": "padcompare-equal-dark-1.png", "image_sha256": "a1d6b43cf6f3cc3d7b51892312f98e040635893623072d8855ef47012de30c97", "ground_truth": { "choice": "C" }, "prompt": "The tinted rectangle is centered inside the outlined frame. Compare the horizontal inset (frame to rectangle on either side) with the vertical inset (frame to rectangle above or below). Measure to the rectangle's visible outer edge, not to its text.\nA: The horizontal inset is larger\nB: The vertical inset is larger\nC: The horizontal and vertical insets are equal\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 1, "answer": "equal", "options": [ "horizontal", "vertical", "equal" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-blockalign-001", "family": "blockalign", "group_id": "blockalign-left-light-0", "image_filename": "blockalign-left-light-0.png", "image_sha256": "93999cc78a3c0e18d5947e27d1aab602edb9d703fd5dde8b6198e6ebf5331c6f", "ground_truth": { "choice": "A" }, "prompt": "Where is the tinted text block placed horizontally within the outer frame? Judge the block's outer edges, independently of how its text lines align.\nA: Against the left inner edge\nB: Centered, with equal empty space on either side\nC: Against the right inner edge\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 0, "answer": "left", "options": [ "left", "center", "right" ], "factors": {}, "lines": { "block": [ { "x": 93, "y": 178.015625, "width": 287, "height": 19 }, { "x": 93, "y": 202.8125, "width": 283, "height": 19 }, { "x": 93, "y": 227.609375, "width": 243, "height": 19 }, { "x": 93, "y": 252.40625, "width": 265, "height": 19 }, { "x": 93, "y": 277.203125, "width": 275, "height": 19 }, { "x": 93, "y": 302, "width": 275, "height": 19 }, { "x": 93, "y": 326.796875, "width": 284, "height": 19 }, { "x": 93, "y": 351.59375, "width": 276, "height": 19 }, { "x": 93, "y": 376.390625, "width": 284, "height": 19 }, { "x": 93, "y": 401.1875, "width": 119, "height": 19 } ] } } }, { "task_id": "layoutbench-blockalign-002", "family": "blockalign", "group_id": "blockalign-left-light-1", "image_filename": "blockalign-left-light-1.png", "image_sha256": "9cbdadaabfaf7d384d94aea382fafd7741085a67e8f8fb3db8999ff1453c07bd", "ground_truth": { "choice": "A" }, "prompt": "Where is the tinted text block placed horizontally within the outer frame? Judge the block's outer edges, independently of how its text lines align.\nA: Against the left inner edge\nB: Centered, with equal empty space on either side\nC: Against the right inner edge\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 1, "answer": "left", "options": [ "left", "center", "right" ], "factors": {}, "lines": { "block": [ { "x": 100, "y": 190.40625, "width": 291, "height": 19 }, { "x": 130, "y": 215.203125, "width": 261, "height": 19 }, { "x": 93, "y": 240, "width": 298, "height": 19 }, { "x": 127, "y": 264.796875, "width": 264, "height": 19 }, { "x": 141, "y": 289.59375, "width": 250, "height": 19 }, { "x": 98, "y": 314.390625, "width": 293, "height": 19 }, { "x": 102, "y": 339.1875, "width": 289, "height": 19 }, { "x": 135, "y": 363.984375, "width": 256, "height": 19 }, { "x": 131, "y": 388.78125, "width": 260, "height": 19 } ] } } }, { "task_id": "layoutbench-blockalign-003", "family": "blockalign", "group_id": "blockalign-left-dark-0", "image_filename": "blockalign-left-dark-0.png", "image_sha256": "03da03c4b96165a5e33d3aabd08529c7d210d32d241380a56542c2fa5f5ed98e", "ground_truth": { "choice": "B" }, "prompt": "Where is the tinted text block placed horizontally within the outer frame? Judge the block's outer edges, independently of how its text lines align.\nA: Against the right inner edge\nB: Against the left inner edge\nC: Centered, with equal empty space on either side\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 0, "answer": "left", "options": [ "right", "left", "center" ], "factors": {}, "lines": { "block": [ { "x": 93, "y": 178.015625, "width": 287, "height": 19 }, { "x": 93, "y": 202.8125, "width": 283, "height": 19 }, { "x": 93, "y": 227.609375, "width": 243, "height": 19 }, { "x": 93, "y": 252.40625, "width": 265, "height": 19 }, { "x": 93, "y": 277.203125, "width": 275, "height": 19 }, { "x": 93, "y": 302, "width": 275, "height": 19 }, { "x": 93, "y": 326.796875, "width": 284, "height": 19 }, { "x": 93, "y": 351.59375, "width": 276, "height": 19 }, { "x": 93, "y": 376.390625, "width": 284, "height": 19 }, { "x": 93, "y": 401.1875, "width": 119, "height": 19 } ] } } }, { "task_id": "layoutbench-blockalign-004", "family": "blockalign", "group_id": "blockalign-left-dark-1", "image_filename": "blockalign-left-dark-1.png", "image_sha256": "a8da2329c460938a38739b8672030c9a0b81bb9e89cc71d36ed63c9ba51bb447", "ground_truth": { "choice": "C" }, "prompt": "Where is the tinted text block placed horizontally within the outer frame? Judge the block's outer edges, independently of how its text lines align.\nA: Centered, with equal empty space on either side\nB: Against the right inner edge\nC: Against the left inner edge\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 1, "answer": "left", "options": [ "center", "right", "left" ], "factors": {}, "lines": { "block": [ { "x": 100, "y": 190.40625, "width": 291, "height": 19 }, { "x": 130, "y": 215.203125, "width": 261, "height": 19 }, { "x": 93, "y": 240, "width": 298, "height": 19 }, { "x": 127, "y": 264.796875, "width": 264, "height": 19 }, { "x": 141, "y": 289.59375, "width": 250, "height": 19 }, { "x": 98, "y": 314.390625, "width": 293, "height": 19 }, { "x": 102, "y": 339.1875, "width": 289, "height": 19 }, { "x": 135, "y": 363.984375, "width": 256, "height": 19 }, { "x": 131, "y": 388.78125, "width": 260, "height": 19 } ] } } }, { "task_id": "layoutbench-blockalign-005", "family": "blockalign", "group_id": "blockalign-center-light-0", "image_filename": "blockalign-center-light-0.png", "image_sha256": "c16726c63796edc5b14ff5d8c93c69fd7f9160ee5cf2aac6d56abd2151ca6911", "ground_truth": { "choice": "B" }, "prompt": "Where is the tinted text block placed horizontally within the outer frame? Judge the block's outer edges, independently of how its text lines align.\nA: Against the right inner edge\nB: Centered, with equal empty space on either side\nC: Against the left inner edge\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 0, "answer": "center", "options": [ "right", "center", "left" ], "factors": {}, "lines": { "block": [ { "x": 251, "y": 178.015625, "width": 287, "height": 19 }, { "x": 251, "y": 202.8125, "width": 283, "height": 19 }, { "x": 251, "y": 227.609375, "width": 243, "height": 19 }, { "x": 251, "y": 252.40625, "width": 265, "height": 19 }, { "x": 251, "y": 277.203125, "width": 275, "height": 19 }, { "x": 251, "y": 302, "width": 275, "height": 19 }, { "x": 251, "y": 326.796875, "width": 284, "height": 19 }, { "x": 251, "y": 351.59375, "width": 276, "height": 19 }, { "x": 251, "y": 376.390625, "width": 284, "height": 19 }, { "x": 251, "y": 401.1875, "width": 119, "height": 19 } ] } } }, { "task_id": "layoutbench-blockalign-006", "family": "blockalign", "group_id": "blockalign-center-light-1", "image_filename": "blockalign-center-light-1.png", "image_sha256": "c1d2ea027ca7c5544c3228e0e7ef3c1b763372f0e5abc3d6f8b8e03feac01fd2", "ground_truth": { "choice": "C" }, "prompt": "Where is the tinted text block placed horizontally within the outer frame? Judge the block's outer edges, independently of how its text lines align.\nA: Against the right inner edge\nB: Against the left inner edge\nC: Centered, with equal empty space on either side\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 1, "answer": "center", "options": [ "right", "left", "center" ], "factors": {}, "lines": { "block": [ { "x": 258, "y": 190.40625, "width": 291, "height": 19 }, { "x": 288, "y": 215.203125, "width": 261, "height": 19 }, { "x": 251, "y": 240, "width": 298, "height": 19 }, { "x": 285, "y": 264.796875, "width": 264, "height": 19 }, { "x": 299, "y": 289.59375, "width": 250, "height": 19 }, { "x": 256, "y": 314.390625, "width": 293, "height": 19 }, { "x": 260, "y": 339.1875, "width": 289, "height": 19 }, { "x": 293, "y": 363.984375, "width": 256, "height": 19 }, { "x": 289, "y": 388.78125, "width": 260, "height": 19 } ] } } }, { "task_id": "layoutbench-blockalign-007", "family": "blockalign", "group_id": "blockalign-center-dark-0", "image_filename": "blockalign-center-dark-0.png", "image_sha256": "009f707473ba4f5e85536576eea3c4f35b761c70440f013cf9038979f8469288", "ground_truth": { "choice": "A" }, "prompt": "Where is the tinted text block placed horizontally within the outer frame? Judge the block's outer edges, independently of how its text lines align.\nA: Centered, with equal empty space on either side\nB: Against the left inner edge\nC: Against the right inner edge\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 0, "answer": "center", "options": [ "center", "left", "right" ], "factors": {}, "lines": { "block": [ { "x": 251, "y": 178.015625, "width": 287, "height": 19 }, { "x": 251, "y": 202.8125, "width": 283, "height": 19 }, { "x": 251, "y": 227.609375, "width": 243, "height": 19 }, { "x": 251, "y": 252.40625, "width": 265, "height": 19 }, { "x": 251, "y": 277.203125, "width": 275, "height": 19 }, { "x": 251, "y": 302, "width": 275, "height": 19 }, { "x": 251, "y": 326.796875, "width": 284, "height": 19 }, { "x": 251, "y": 351.59375, "width": 276, "height": 19 }, { "x": 251, "y": 376.390625, "width": 284, "height": 19 }, { "x": 251, "y": 401.1875, "width": 119, "height": 19 } ] } } }, { "task_id": "layoutbench-blockalign-008", "family": "blockalign", "group_id": "blockalign-center-dark-1", "image_filename": "blockalign-center-dark-1.png", "image_sha256": "ba60d8ec38eaff61c59a1dc8310d86bde2a0541f1b9068137d1e622d1bbd3996", "ground_truth": { "choice": "C" }, "prompt": "Where is the tinted text block placed horizontally within the outer frame? Judge the block's outer edges, independently of how its text lines align.\nA: Against the right inner edge\nB: Against the left inner edge\nC: Centered, with equal empty space on either side\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 1, "answer": "center", "options": [ "right", "left", "center" ], "factors": {}, "lines": { "block": [ { "x": 258, "y": 190.40625, "width": 291, "height": 19 }, { "x": 288, "y": 215.203125, "width": 261, "height": 19 }, { "x": 251, "y": 240, "width": 298, "height": 19 }, { "x": 285, "y": 264.796875, "width": 264, "height": 19 }, { "x": 299, "y": 289.59375, "width": 250, "height": 19 }, { "x": 256, "y": 314.390625, "width": 293, "height": 19 }, { "x": 260, "y": 339.1875, "width": 289, "height": 19 }, { "x": 293, "y": 363.984375, "width": 256, "height": 19 }, { "x": 289, "y": 388.78125, "width": 260, "height": 19 } ] } } }, { "task_id": "layoutbench-blockalign-009", "family": "blockalign", "group_id": "blockalign-right-light-0", "image_filename": "blockalign-right-light-0.png", "image_sha256": "d4c23856fcf5b4b04e2d3b939e7d4117d75a41b3f1c01b96444692528c135684", "ground_truth": { "choice": "A" }, "prompt": "Where is the tinted text block placed horizontally within the outer frame? Judge the block's outer edges, independently of how its text lines align.\nA: Against the right inner edge\nB: Against the left inner edge\nC: Centered, with equal empty space on either side\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 0, "answer": "right", "options": [ "right", "left", "center" ], "factors": {}, "lines": { "block": [ { "x": 409, "y": 178.015625, "width": 287, "height": 19 }, { "x": 409, "y": 202.8125, "width": 283, "height": 19 }, { "x": 409, "y": 227.609375, "width": 243, "height": 19 }, { "x": 409, "y": 252.40625, "width": 265, "height": 19 }, { "x": 409, "y": 277.203125, "width": 275, "height": 19 }, { "x": 409, "y": 302, "width": 275, "height": 19 }, { "x": 409, "y": 326.796875, "width": 284, "height": 19 }, { "x": 409, "y": 351.59375, "width": 276, "height": 19 }, { "x": 409, "y": 376.390625, "width": 284, "height": 19 }, { "x": 409, "y": 401.1875, "width": 119, "height": 19 } ] } } }, { "task_id": "layoutbench-blockalign-010", "family": "blockalign", "group_id": "blockalign-right-light-1", "image_filename": "blockalign-right-light-1.png", "image_sha256": "58d73942b4c7479d04a47fbcda3c873327a521a91cfb09cdb589dc9394711ed0", "ground_truth": { "choice": "B" }, "prompt": "Where is the tinted text block placed horizontally within the outer frame? Judge the block's outer edges, independently of how its text lines align.\nA: Against the left inner edge\nB: Against the right inner edge\nC: Centered, with equal empty space on either side\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 1, "answer": "right", "options": [ "left", "right", "center" ], "factors": {}, "lines": { "block": [ { "x": 416, "y": 190.40625, "width": 291, "height": 19 }, { "x": 446, "y": 215.203125, "width": 261, "height": 19 }, { "x": 409, "y": 240, "width": 298, "height": 19 }, { "x": 443, "y": 264.796875, "width": 264, "height": 19 }, { "x": 457, "y": 289.59375, "width": 250, "height": 19 }, { "x": 414, "y": 314.390625, "width": 293, "height": 19 }, { "x": 418, "y": 339.1875, "width": 289, "height": 19 }, { "x": 451, "y": 363.984375, "width": 256, "height": 19 }, { "x": 447, "y": 388.78125, "width": 260, "height": 19 } ] } } }, { "task_id": "layoutbench-blockalign-011", "family": "blockalign", "group_id": "blockalign-right-dark-0", "image_filename": "blockalign-right-dark-0.png", "image_sha256": "f34c2f431c1a1ae5e21bdc12bda54a4ff809d38515b4627acdc2caabfae9bfbf", "ground_truth": { "choice": "C" }, "prompt": "Where is the tinted text block placed horizontally within the outer frame? Judge the block's outer edges, independently of how its text lines align.\nA: Centered, with equal empty space on either side\nB: Against the left inner edge\nC: Against the right inner edge\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 0, "answer": "right", "options": [ "center", "left", "right" ], "factors": {}, "lines": { "block": [ { "x": 409, "y": 178.015625, "width": 287, "height": 19 }, { "x": 409, "y": 202.8125, "width": 283, "height": 19 }, { "x": 409, "y": 227.609375, "width": 243, "height": 19 }, { "x": 409, "y": 252.40625, "width": 265, "height": 19 }, { "x": 409, "y": 277.203125, "width": 275, "height": 19 }, { "x": 409, "y": 302, "width": 275, "height": 19 }, { "x": 409, "y": 326.796875, "width": 284, "height": 19 }, { "x": 409, "y": 351.59375, "width": 276, "height": 19 }, { "x": 409, "y": 376.390625, "width": 284, "height": 19 }, { "x": 409, "y": 401.1875, "width": 119, "height": 19 } ] } } }, { "task_id": "layoutbench-blockalign-012", "family": "blockalign", "group_id": "blockalign-right-dark-1", "image_filename": "blockalign-right-dark-1.png", "image_sha256": "2afc47f7c1b07f0c593f365a0f9b7a03cf87bdac25a2574b857e1313c1a8ab8f", "ground_truth": { "choice": "B" }, "prompt": "Where is the tinted text block placed horizontally within the outer frame? Judge the block's outer edges, independently of how its text lines align.\nA: Centered, with equal empty space on either side\nB: Against the right inner edge\nC: Against the left inner edge\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 1, "answer": "right", "options": [ "center", "right", "left" ], "factors": {}, "lines": { "block": [ { "x": 416, "y": 190.40625, "width": 291, "height": 19 }, { "x": 446, "y": 215.203125, "width": 261, "height": 19 }, { "x": 409, "y": 240, "width": 298, "height": 19 }, { "x": 443, "y": 264.796875, "width": 264, "height": 19 }, { "x": 457, "y": 289.59375, "width": 250, "height": 19 }, { "x": 414, "y": 314.390625, "width": 293, "height": 19 }, { "x": 418, "y": 339.1875, "width": 289, "height": 19 }, { "x": 451, "y": 363.984375, "width": 256, "height": 19 }, { "x": 447, "y": 388.78125, "width": 260, "height": 19 } ] } } }, { "task_id": "layoutbench-widthcompare-001", "family": "widthcompare", "group_id": "widthcompare-first-light-0", "image_filename": "widthcompare-first-light-0.png", "image_sha256": "46860f6ee36c88d21df117584a6a7d1bee3dbbdddde639fd12a2c34a6689826c", "ground_truth": { "choice": "B" }, "prompt": "Compare the widths of the two tinted panels, from their visible left edge to right edge.\nA: Their widths are equal\nB: The left panel is wider\nC: The right panel is wider\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 0, "answer": "first", "options": [ "equal", "first", "last" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-widthcompare-002", "family": "widthcompare", "group_id": "widthcompare-first-light-1", "image_filename": "widthcompare-first-light-1.png", "image_sha256": "ba75b11311674a94c11f44abb0d3ff347d0f39a52d78d2f39b3a82d6e5a79a42", "ground_truth": { "choice": "C" }, "prompt": "Compare the widths of the two tinted panels, from their visible left edge to right edge.\nA: The right panel is wider\nB: Their widths are equal\nC: The left panel is wider\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 1, "answer": "first", "options": [ "last", "equal", "first" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-widthcompare-003", "family": "widthcompare", "group_id": "widthcompare-first-dark-0", "image_filename": "widthcompare-first-dark-0.png", "image_sha256": "aa7026825d6facf9b94cefd1b70fa35bdca85928c7185f0a30027647b969fdc1", "ground_truth": { "choice": "A" }, "prompt": "Compare the widths of the two tinted panels, from their visible left edge to right edge.\nA: The left panel is wider\nB: The right panel is wider\nC: Their widths are equal\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 0, "answer": "first", "options": [ "first", "last", "equal" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-widthcompare-004", "family": "widthcompare", "group_id": "widthcompare-first-dark-1", "image_filename": "widthcompare-first-dark-1.png", "image_sha256": "2744b96c2c1184f933a8b49483327c6690744736058300c3d8404d8cf5435183", "ground_truth": { "choice": "B" }, "prompt": "Compare the widths of the two tinted panels, from their visible left edge to right edge.\nA: Their widths are equal\nB: The left panel is wider\nC: The right panel is wider\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 1, "answer": "first", "options": [ "equal", "first", "last" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-widthcompare-005", "family": "widthcompare", "group_id": "widthcompare-last-light-0", "image_filename": "widthcompare-last-light-0.png", "image_sha256": "fd61274910c02eb937ec1bb8f30b5cc754d5da33c564585598acceee3074c616", "ground_truth": { "choice": "B" }, "prompt": "Compare the widths of the two tinted panels, from their visible left edge to right edge.\nA: The left panel is wider\nB: The right panel is wider\nC: Their widths are equal\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 0, "answer": "last", "options": [ "first", "last", "equal" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-widthcompare-006", "family": "widthcompare", "group_id": "widthcompare-last-light-1", "image_filename": "widthcompare-last-light-1.png", "image_sha256": "ff0d815424a947f8f6975d2aad9c9fd96d3709c09822ac70db434a8f8d1d6e64", "ground_truth": { "choice": "A" }, "prompt": "Compare the widths of the two tinted panels, from their visible left edge to right edge.\nA: The right panel is wider\nB: The left panel is wider\nC: Their widths are equal\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 1, "answer": "last", "options": [ "last", "first", "equal" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-widthcompare-007", "family": "widthcompare", "group_id": "widthcompare-last-dark-0", "image_filename": "widthcompare-last-dark-0.png", "image_sha256": "898d6547a4e05b4b219a2c121ad66a3546a5fcbba53d03419b7c97fa84bca25d", "ground_truth": { "choice": "C" }, "prompt": "Compare the widths of the two tinted panels, from their visible left edge to right edge.\nA: Their widths are equal\nB: The left panel is wider\nC: The right panel is wider\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 0, "answer": "last", "options": [ "equal", "first", "last" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-widthcompare-008", "family": "widthcompare", "group_id": "widthcompare-last-dark-1", "image_filename": "widthcompare-last-dark-1.png", "image_sha256": "1cf129e23880d6e00dcab320ef154e9e9762c9d86e4c56842e04fd1d19419a8a", "ground_truth": { "choice": "A" }, "prompt": "Compare the widths of the two tinted panels, from their visible left edge to right edge.\nA: The right panel is wider\nB: Their widths are equal\nC: The left panel is wider\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 1, "answer": "last", "options": [ "last", "equal", "first" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-widthcompare-009", "family": "widthcompare", "group_id": "widthcompare-equal-light-0", "image_filename": "widthcompare-equal-light-0.png", "image_sha256": "2d59ff6eb852db55988086f0037c1a1815945900c24d37930b5271755553c87a", "ground_truth": { "choice": "B" }, "prompt": "Compare the widths of the two tinted panels, from their visible left edge to right edge.\nA: The right panel is wider\nB: Their widths are equal\nC: The left panel is wider\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 0, "answer": "equal", "options": [ "last", "equal", "first" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-widthcompare-010", "family": "widthcompare", "group_id": "widthcompare-equal-light-1", "image_filename": "widthcompare-equal-light-1.png", "image_sha256": "cc3219943ba440fefa67f513a731f5112c14d3b2a8ea874d26f3835e5e4ed08b", "ground_truth": { "choice": "A" }, "prompt": "Compare the widths of the two tinted panels, from their visible left edge to right edge.\nA: Their widths are equal\nB: The left panel is wider\nC: The right panel is wider\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 1, "answer": "equal", "options": [ "equal", "first", "last" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-widthcompare-011", "family": "widthcompare", "group_id": "widthcompare-equal-dark-0", "image_filename": "widthcompare-equal-dark-0.png", "image_sha256": "87e943ef52547d0aa4da2c6c86e801853d7a2c5d19215f84741e040b00eaef20", "ground_truth": { "choice": "C" }, "prompt": "Compare the widths of the two tinted panels, from their visible left edge to right edge.\nA: The right panel is wider\nB: The left panel is wider\nC: Their widths are equal\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 0, "answer": "equal", "options": [ "last", "first", "equal" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-widthcompare-012", "family": "widthcompare", "group_id": "widthcompare-equal-dark-1", "image_filename": "widthcompare-equal-dark-1.png", "image_sha256": "56f921860765116da9a904be6c93172ae73204166e028a79897a6b609210e801", "ground_truth": { "choice": "C" }, "prompt": "Compare the widths of the two tinted panels, from their visible left edge to right edge.\nA: The right panel is wider\nB: The left panel is wider\nC: Their widths are equal\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 1, "answer": "equal", "options": [ "last", "first", "equal" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-groupgap-001", "family": "groupgap", "group_id": "groupgap-between-light-0", "image_filename": "groupgap-between-light-0.png", "image_sha256": "ceb82f5b741bb1c82eeecadad888a8a37634bdfcd13b518fd090579d29d3f722", "ground_truth": { "choice": "B" }, "prompt": "The cards form two groups marked Harbor and Meadow. Compare the gap between the groups' nearest card boxes with the gaps between cards within each group. Ignore the small group captions.\nA: The between-group and within-group gaps are equal\nB: The between-group gap is larger\nC: The within-group gaps are larger\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 0, "answer": "between", "options": [ "equal", "between", "within" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-groupgap-002", "family": "groupgap", "group_id": "groupgap-between-light-1", "image_filename": "groupgap-between-light-1.png", "image_sha256": "d89e016a56363e65ec619535629b869fc91ad54552753715f0ee63b7e8f98baa", "ground_truth": { "choice": "B" }, "prompt": "The cards form two groups marked Harbor and Meadow. Compare the gap between the groups' nearest card boxes with the gaps between cards within each group. Ignore the small group captions.\nA: The within-group gaps are larger\nB: The between-group gap is larger\nC: The between-group and within-group gaps are equal\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 1, "answer": "between", "options": [ "within", "between", "equal" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-groupgap-003", "family": "groupgap", "group_id": "groupgap-between-dark-0", "image_filename": "groupgap-between-dark-0.png", "image_sha256": "70bb1bfdfa598530782f35f113acef2c91778b67a15b798d32679c8f73533ec4", "ground_truth": { "choice": "B" }, "prompt": "The cards form two groups marked Harbor and Meadow. Compare the gap between the groups' nearest card boxes with the gaps between cards within each group. Ignore the small group captions.\nA: The within-group gaps are larger\nB: The between-group gap is larger\nC: The between-group and within-group gaps are equal\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 0, "answer": "between", "options": [ "within", "between", "equal" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-groupgap-004", "family": "groupgap", "group_id": "groupgap-between-dark-1", "image_filename": "groupgap-between-dark-1.png", "image_sha256": "c6a918cb13023fed221b6ebe3f6b8d2bc0884e5a9cc46b9222eeee4a3560b66b", "ground_truth": { "choice": "A" }, "prompt": "The cards form two groups marked Harbor and Meadow. Compare the gap between the groups' nearest card boxes with the gaps between cards within each group. Ignore the small group captions.\nA: The between-group gap is larger\nB: The between-group and within-group gaps are equal\nC: The within-group gaps are larger\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 1, "answer": "between", "options": [ "between", "equal", "within" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-groupgap-005", "family": "groupgap", "group_id": "groupgap-within-light-0", "image_filename": "groupgap-within-light-0.png", "image_sha256": "d6b8b3fb38cdca6a7808c902b5c8d76e8bcb1b139ffdcf5d4bc6b0b18fb6723d", "ground_truth": { "choice": "C" }, "prompt": "The cards form two groups marked Harbor and Meadow. Compare the gap between the groups' nearest card boxes with the gaps between cards within each group. Ignore the small group captions.\nA: The between-group gap is larger\nB: The between-group and within-group gaps are equal\nC: The within-group gaps are larger\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 0, "answer": "within", "options": [ "between", "equal", "within" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-groupgap-006", "family": "groupgap", "group_id": "groupgap-within-light-1", "image_filename": "groupgap-within-light-1.png", "image_sha256": "012c7b0c080a5dbf52ef141cbadb2baf491c0a1ca10ad5f08c06074b104afc7a", "ground_truth": { "choice": "A" }, "prompt": "The cards form two groups marked Harbor and Meadow. Compare the gap between the groups' nearest card boxes with the gaps between cards within each group. Ignore the small group captions.\nA: The within-group gaps are larger\nB: The between-group gap is larger\nC: The between-group and within-group gaps are equal\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 1, "answer": "within", "options": [ "within", "between", "equal" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-groupgap-007", "family": "groupgap", "group_id": "groupgap-within-dark-0", "image_filename": "groupgap-within-dark-0.png", "image_sha256": "aa29f9546ddc83ca8a3072b2c656282e867da95f84e38d7c5605c639ee2c8592", "ground_truth": { "choice": "C" }, "prompt": "The cards form two groups marked Harbor and Meadow. Compare the gap between the groups' nearest card boxes with the gaps between cards within each group. Ignore the small group captions.\nA: The between-group gap is larger\nB: The between-group and within-group gaps are equal\nC: The within-group gaps are larger\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 0, "answer": "within", "options": [ "between", "equal", "within" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-groupgap-008", "family": "groupgap", "group_id": "groupgap-within-dark-1", "image_filename": "groupgap-within-dark-1.png", "image_sha256": "c032eff2522bf2eb2c35de95e00a4acef8d668150b4afbc593c49ee6f27c6f59", "ground_truth": { "choice": "C" }, "prompt": "The cards form two groups marked Harbor and Meadow. Compare the gap between the groups' nearest card boxes with the gaps between cards within each group. Ignore the small group captions.\nA: The between-group and within-group gaps are equal\nB: The between-group gap is larger\nC: The within-group gaps are larger\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 1, "answer": "within", "options": [ "equal", "between", "within" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-groupgap-009", "family": "groupgap", "group_id": "groupgap-equal-light-0", "image_filename": "groupgap-equal-light-0.png", "image_sha256": "324fb72a105e935b01471b580b3e9acc2cfa26b3f44664ef57e61f3e8ecd0a2b", "ground_truth": { "choice": "B" }, "prompt": "The cards form two groups marked Harbor and Meadow. Compare the gap between the groups' nearest card boxes with the gaps between cards within each group. Ignore the small group captions.\nA: The within-group gaps are larger\nB: The between-group and within-group gaps are equal\nC: The between-group gap is larger\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 0, "answer": "equal", "options": [ "within", "equal", "between" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-groupgap-010", "family": "groupgap", "group_id": "groupgap-equal-light-1", "image_filename": "groupgap-equal-light-1.png", "image_sha256": "a966dcb73dc3edbb12a3a4ab4b5704bbf7a00a2e9f98f6da52213be89f9ede84", "ground_truth": { "choice": "A" }, "prompt": "The cards form two groups marked Harbor and Meadow. Compare the gap between the groups' nearest card boxes with the gaps between cards within each group. Ignore the small group captions.\nA: The between-group and within-group gaps are equal\nB: The between-group gap is larger\nC: The within-group gaps are larger\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 1, "answer": "equal", "options": [ "equal", "between", "within" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-groupgap-011", "family": "groupgap", "group_id": "groupgap-equal-dark-0", "image_filename": "groupgap-equal-dark-0.png", "image_sha256": "10450b9a549a8d0c0c1e980fdf8564ee520b408eb1d1da67f729fec64bea7f41", "ground_truth": { "choice": "A" }, "prompt": "The cards form two groups marked Harbor and Meadow. Compare the gap between the groups' nearest card boxes with the gaps between cards within each group. Ignore the small group captions.\nA: The between-group and within-group gaps are equal\nB: The between-group gap is larger\nC: The within-group gaps are larger\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 0, "answer": "equal", "options": [ "equal", "between", "within" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-groupgap-012", "family": "groupgap", "group_id": "groupgap-equal-dark-1", "image_filename": "groupgap-equal-dark-1.png", "image_sha256": "09e520206c0615b65a9ac6a5325091542c1eae2b92f556e98542028f188524a2", "ground_truth": { "choice": "C" }, "prompt": "The cards form two groups marked Harbor and Meadow. Compare the gap between the groups' nearest card boxes with the gaps between cards within each group. Ignore the small group captions.\nA: The within-group gaps are larger\nB: The between-group gap is larger\nC: The between-group and within-group gaps are equal\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 1, "answer": "equal", "options": [ "within", "between", "equal" ], "factors": {}, "lines": {} } }, { "task_id": "layoutbench-topflow-001", "family": "topflow", "group_id": "h-01", "image_filename": "h-01.png", "image_sha256": "741b99f72d76efde3e95c36ef17639ab23de30d920877562d55f355743f9a36e", "ground_truth": { "choice": "A" }, "prompt": "Look at the two large outlined sections headed Field notes and Harbor log. At the top level, how are these two sections arranged? Ignore the arrangement of cards inside either section.\nA: Side by side horizontally\nB: Stacked vertically\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 0, "answer": "row", "options": [ "row", "column" ], "factors": { "parent": "row", "child": "row" }, "lines": {} } }, { "task_id": "layoutbench-nestedflow-001", "family": "nestedflow", "group_id": "h-01", "image_filename": "h-01.png", "image_sha256": "741b99f72d76efde3e95c36ef17639ab23de30d920877562d55f355743f9a36e", "ground_truth": { "choice": "A" }, "prompt": "Look inside the large section headed Field notes. How are its small cards arranged? Ignore how Field notes is placed relative to Harbor log.\nA: One horizontal row\nB: One vertical column\nC: Multiple horizontal rows, with a shorter final row\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 0, "answer": "row", "options": [ "row", "column", "wrapped" ], "factors": { "parent": "row", "child": "row" }, "lines": {} } }, { "task_id": "layoutbench-topflow-002", "family": "topflow", "group_id": "h-02", "image_filename": "h-02.png", "image_sha256": "ce9bdffee41108c8564bbae34842327d1c77448ad21c047a46f0f1eb83ad5afe", "ground_truth": { "choice": "A" }, "prompt": "Look at the two large outlined sections headed Field notes and Harbor log. At the top level, how are these two sections arranged? Ignore the arrangement of cards inside either section.\nA: Side by side horizontally\nB: Stacked vertically\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 1, "answer": "row", "options": [ "row", "column" ], "factors": { "parent": "row", "child": "row" }, "lines": {} } }, { "task_id": "layoutbench-nestedflow-002", "family": "nestedflow", "group_id": "h-02", "image_filename": "h-02.png", "image_sha256": "ce9bdffee41108c8564bbae34842327d1c77448ad21c047a46f0f1eb83ad5afe", "ground_truth": { "choice": "C" }, "prompt": "Look inside the large section headed Field notes. How are its small cards arranged? Ignore how Field notes is placed relative to Harbor log.\nA: Multiple horizontal rows, with a shorter final row\nB: One vertical column\nC: One horizontal row\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 1, "answer": "row", "options": [ "wrapped", "column", "row" ], "factors": { "parent": "row", "child": "row" }, "lines": {} } }, { "task_id": "layoutbench-topflow-003", "family": "topflow", "group_id": "h-03", "image_filename": "h-03.png", "image_sha256": "b5012b0f72f8c622150f264bba9b29e3987c9950f49fca02dd53e6b86335c785", "ground_truth": { "choice": "A" }, "prompt": "Look at the two large outlined sections headed Field notes and Harbor log. At the top level, how are these two sections arranged? Ignore the arrangement of cards inside either section.\nA: Side by side horizontally\nB: Stacked vertically\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 0, "answer": "row", "options": [ "row", "column" ], "factors": { "parent": "row", "child": "row" }, "lines": {} } }, { "task_id": "layoutbench-nestedflow-003", "family": "nestedflow", "group_id": "h-03", "image_filename": "h-03.png", "image_sha256": "b5012b0f72f8c622150f264bba9b29e3987c9950f49fca02dd53e6b86335c785", "ground_truth": { "choice": "B" }, "prompt": "Look inside the large section headed Field notes. How are its small cards arranged? Ignore how Field notes is placed relative to Harbor log.\nA: One vertical column\nB: One horizontal row\nC: Multiple horizontal rows, with a shorter final row\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 0, "answer": "row", "options": [ "column", "row", "wrapped" ], "factors": { "parent": "row", "child": "row" }, "lines": {} } }, { "task_id": "layoutbench-topflow-004", "family": "topflow", "group_id": "h-04", "image_filename": "h-04.png", "image_sha256": "ee6e41ee6423d82929e6b2792f099c30e41e9b4f093a59891f3dff64d49519a1", "ground_truth": { "choice": "B" }, "prompt": "Look at the two large outlined sections headed Field notes and Harbor log. At the top level, how are these two sections arranged? Ignore the arrangement of cards inside either section.\nA: Stacked vertically\nB: Side by side horizontally\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 1, "answer": "row", "options": [ "column", "row" ], "factors": { "parent": "row", "child": "row" }, "lines": {} } }, { "task_id": "layoutbench-nestedflow-004", "family": "nestedflow", "group_id": "h-04", "image_filename": "h-04.png", "image_sha256": "ee6e41ee6423d82929e6b2792f099c30e41e9b4f093a59891f3dff64d49519a1", "ground_truth": { "choice": "C" }, "prompt": "Look inside the large section headed Field notes. How are its small cards arranged? Ignore how Field notes is placed relative to Harbor log.\nA: One vertical column\nB: Multiple horizontal rows, with a shorter final row\nC: One horizontal row\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 1, "answer": "row", "options": [ "column", "wrapped", "row" ], "factors": { "parent": "row", "child": "row" }, "lines": {} } }, { "task_id": "layoutbench-topflow-005", "family": "topflow", "group_id": "h-05", "image_filename": "h-05.png", "image_sha256": "b8553aa6a3374f08b6ac80b8dd5164e6d34ead6b3cd6507ee7890d9231eae4a0", "ground_truth": { "choice": "B" }, "prompt": "Look at the two large outlined sections headed Field notes and Harbor log. At the top level, how are these two sections arranged? Ignore the arrangement of cards inside either section.\nA: Stacked vertically\nB: Side by side horizontally\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 0, "answer": "row", "options": [ "column", "row" ], "factors": { "parent": "row", "child": "column" }, "lines": {} } }, { "task_id": "layoutbench-nestedflow-005", "family": "nestedflow", "group_id": "h-05", "image_filename": "h-05.png", "image_sha256": "b8553aa6a3374f08b6ac80b8dd5164e6d34ead6b3cd6507ee7890d9231eae4a0", "ground_truth": { "choice": "C" }, "prompt": "Look inside the large section headed Field notes. How are its small cards arranged? Ignore how Field notes is placed relative to Harbor log.\nA: One horizontal row\nB: Multiple horizontal rows, with a shorter final row\nC: One vertical column\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 0, "answer": "column", "options": [ "row", "wrapped", "column" ], "factors": { "parent": "row", "child": "column" }, "lines": {} } }, { "task_id": "layoutbench-topflow-006", "family": "topflow", "group_id": "h-06", "image_filename": "h-06.png", "image_sha256": "dead20b06b2b84a2e55ee13e0ae10470d565bc40d3183bf46215687a91046935", "ground_truth": { "choice": "A" }, "prompt": "Look at the two large outlined sections headed Field notes and Harbor log. At the top level, how are these two sections arranged? Ignore the arrangement of cards inside either section.\nA: Side by side horizontally\nB: Stacked vertically\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 1, "answer": "row", "options": [ "row", "column" ], "factors": { "parent": "row", "child": "column" }, "lines": {} } }, { "task_id": "layoutbench-nestedflow-006", "family": "nestedflow", "group_id": "h-06", "image_filename": "h-06.png", "image_sha256": "dead20b06b2b84a2e55ee13e0ae10470d565bc40d3183bf46215687a91046935", "ground_truth": { "choice": "B" }, "prompt": "Look inside the large section headed Field notes. How are its small cards arranged? Ignore how Field notes is placed relative to Harbor log.\nA: One horizontal row\nB: One vertical column\nC: Multiple horizontal rows, with a shorter final row\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 1, "answer": "column", "options": [ "row", "column", "wrapped" ], "factors": { "parent": "row", "child": "column" }, "lines": {} } }, { "task_id": "layoutbench-topflow-007", "family": "topflow", "group_id": "h-07", "image_filename": "h-07.png", "image_sha256": "deb9c818c6b4cd55dd8f62920c258e5e4e50c69aa5ca326621242f7464d0e3d3", "ground_truth": { "choice": "A" }, "prompt": "Look at the two large outlined sections headed Field notes and Harbor log. At the top level, how are these two sections arranged? Ignore the arrangement of cards inside either section.\nA: Side by side horizontally\nB: Stacked vertically\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 0, "answer": "row", "options": [ "row", "column" ], "factors": { "parent": "row", "child": "column" }, "lines": {} } }, { "task_id": "layoutbench-nestedflow-007", "family": "nestedflow", "group_id": "h-07", "image_filename": "h-07.png", "image_sha256": "deb9c818c6b4cd55dd8f62920c258e5e4e50c69aa5ca326621242f7464d0e3d3", "ground_truth": { "choice": "A" }, "prompt": "Look inside the large section headed Field notes. How are its small cards arranged? Ignore how Field notes is placed relative to Harbor log.\nA: One vertical column\nB: One horizontal row\nC: Multiple horizontal rows, with a shorter final row\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 0, "answer": "column", "options": [ "column", "row", "wrapped" ], "factors": { "parent": "row", "child": "column" }, "lines": {} } }, { "task_id": "layoutbench-topflow-008", "family": "topflow", "group_id": "h-08", "image_filename": "h-08.png", "image_sha256": "ce7968612ee74659b54fe1e40ac2eb572c8db02c4ad660a30af89a5a762aec4d", "ground_truth": { "choice": "B" }, "prompt": "Look at the two large outlined sections headed Field notes and Harbor log. At the top level, how are these two sections arranged? Ignore the arrangement of cards inside either section.\nA: Stacked vertically\nB: Side by side horizontally\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 1, "answer": "row", "options": [ "column", "row" ], "factors": { "parent": "row", "child": "column" }, "lines": {} } }, { "task_id": "layoutbench-nestedflow-008", "family": "nestedflow", "group_id": "h-08", "image_filename": "h-08.png", "image_sha256": "ce7968612ee74659b54fe1e40ac2eb572c8db02c4ad660a30af89a5a762aec4d", "ground_truth": { "choice": "A" }, "prompt": "Look inside the large section headed Field notes. How are its small cards arranged? Ignore how Field notes is placed relative to Harbor log.\nA: One vertical column\nB: Multiple horizontal rows, with a shorter final row\nC: One horizontal row\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 1, "answer": "column", "options": [ "column", "wrapped", "row" ], "factors": { "parent": "row", "child": "column" }, "lines": {} } }, { "task_id": "layoutbench-topflow-009", "family": "topflow", "group_id": "h-09", "image_filename": "h-09.png", "image_sha256": "882f6b21e43601e586930da4bae5ac5ceb3173a0854b23000f9184b87a6a71df", "ground_truth": { "choice": "A" }, "prompt": "Look at the two large outlined sections headed Field notes and Harbor log. At the top level, how are these two sections arranged? Ignore the arrangement of cards inside either section.\nA: Side by side horizontally\nB: Stacked vertically\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 0, "answer": "row", "options": [ "row", "column" ], "factors": { "parent": "row", "child": "wrapped", "alignment": "left" }, "lines": {} } }, { "task_id": "layoutbench-nestedflow-009", "family": "nestedflow", "group_id": "h-09", "image_filename": "h-09.png", "image_sha256": "882f6b21e43601e586930da4bae5ac5ceb3173a0854b23000f9184b87a6a71df", "ground_truth": { "choice": "B" }, "prompt": "Look inside the large section headed Field notes. How are its small cards arranged? Ignore how Field notes is placed relative to Harbor log.\nA: One vertical column\nB: Multiple horizontal rows, with a shorter final row\nC: One horizontal row\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 0, "answer": "wrapped", "options": [ "column", "wrapped", "row" ], "factors": { "parent": "row", "child": "wrapped", "alignment": "left" }, "lines": {} } }, { "task_id": "layoutbench-nestedwrap-001", "family": "nestedwrap", "group_id": "h-09", "image_filename": "h-09.png", "image_sha256": "882f6b21e43601e586930da4bae5ac5ceb3173a0854b23000f9184b87a6a71df", "ground_truth": { "choice": "B" }, "prompt": "Inside Field notes, look only at the shorter final row of cards. How is it placed horizontally inside its own outlined inner frame? Ignore the surrounding page's arrangement.\nA: Against that frame's right inner edge\nB: Against that frame's left inner edge\nC: Centered in that frame\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 0, "answer": "left", "options": [ "right", "left", "center" ], "factors": { "parent": "row", "child": "wrapped", "alignment": "left" }, "lines": {} } }, { "task_id": "layoutbench-topflow-010", "family": "topflow", "group_id": "h-10", "image_filename": "h-10.png", "image_sha256": "38331671e2ad6b023b8c85ebfed1a00bfcf4e065af92fc235e80bce6eae12ec6", "ground_truth": { "choice": "A" }, "prompt": "Look at the two large outlined sections headed Field notes and Harbor log. At the top level, how are these two sections arranged? Ignore the arrangement of cards inside either section.\nA: Side by side horizontally\nB: Stacked vertically\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 1, "answer": "row", "options": [ "row", "column" ], "factors": { "parent": "row", "child": "wrapped", "alignment": "left" }, "lines": {} } }, { "task_id": "layoutbench-nestedflow-010", "family": "nestedflow", "group_id": "h-10", "image_filename": "h-10.png", "image_sha256": "38331671e2ad6b023b8c85ebfed1a00bfcf4e065af92fc235e80bce6eae12ec6", "ground_truth": { "choice": "B" }, "prompt": "Look inside the large section headed Field notes. How are its small cards arranged? Ignore how Field notes is placed relative to Harbor log.\nA: One vertical column\nB: Multiple horizontal rows, with a shorter final row\nC: One horizontal row\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 1, "answer": "wrapped", "options": [ "column", "wrapped", "row" ], "factors": { "parent": "row", "child": "wrapped", "alignment": "left" }, "lines": {} } }, { "task_id": "layoutbench-nestedwrap-002", "family": "nestedwrap", "group_id": "h-10", "image_filename": "h-10.png", "image_sha256": "38331671e2ad6b023b8c85ebfed1a00bfcf4e065af92fc235e80bce6eae12ec6", "ground_truth": { "choice": "B" }, "prompt": "Inside Field notes, look only at the shorter final row of cards. How is it placed horizontally inside its own outlined inner frame? Ignore the surrounding page's arrangement.\nA: Against that frame's right inner edge\nB: Against that frame's left inner edge\nC: Centered in that frame\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 1, "answer": "left", "options": [ "right", "left", "center" ], "factors": { "parent": "row", "child": "wrapped", "alignment": "left" }, "lines": {} } }, { "task_id": "layoutbench-topflow-011", "family": "topflow", "group_id": "h-11", "image_filename": "h-11.png", "image_sha256": "11fab0be75f11a84e4ce883389ccfd4ad1de5a331d43fe5f2125483fb75c0414", "ground_truth": { "choice": "B" }, "prompt": "Look at the two large outlined sections headed Field notes and Harbor log. At the top level, how are these two sections arranged? Ignore the arrangement of cards inside either section.\nA: Stacked vertically\nB: Side by side horizontally\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 0, "answer": "row", "options": [ "column", "row" ], "factors": { "parent": "row", "child": "wrapped", "alignment": "left" }, "lines": {} } }, { "task_id": "layoutbench-nestedflow-011", "family": "nestedflow", "group_id": "h-11", "image_filename": "h-11.png", "image_sha256": "11fab0be75f11a84e4ce883389ccfd4ad1de5a331d43fe5f2125483fb75c0414", "ground_truth": { "choice": "C" }, "prompt": "Look inside the large section headed Field notes. How are its small cards arranged? Ignore how Field notes is placed relative to Harbor log.\nA: One vertical column\nB: One horizontal row\nC: Multiple horizontal rows, with a shorter final row\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 0, "answer": "wrapped", "options": [ "column", "row", "wrapped" ], "factors": { "parent": "row", "child": "wrapped", "alignment": "left" }, "lines": {} } }, { "task_id": "layoutbench-nestedwrap-003", "family": "nestedwrap", "group_id": "h-11", "image_filename": "h-11.png", "image_sha256": "11fab0be75f11a84e4ce883389ccfd4ad1de5a331d43fe5f2125483fb75c0414", "ground_truth": { "choice": "C" }, "prompt": "Inside Field notes, look only at the shorter final row of cards. How is it placed horizontally inside its own outlined inner frame? Ignore the surrounding page's arrangement.\nA: Against that frame's right inner edge\nB: Centered in that frame\nC: Against that frame's left inner edge\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 0, "answer": "left", "options": [ "right", "center", "left" ], "factors": { "parent": "row", "child": "wrapped", "alignment": "left" }, "lines": {} } }, { "task_id": "layoutbench-topflow-012", "family": "topflow", "group_id": "h-12", "image_filename": "h-12.png", "image_sha256": "ce42e30c0bffa72ca47a923d41027233a6c7d1f57dd0e861e7e55d5c45b37e7c", "ground_truth": { "choice": "A" }, "prompt": "Look at the two large outlined sections headed Field notes and Harbor log. At the top level, how are these two sections arranged? Ignore the arrangement of cards inside either section.\nA: Side by side horizontally\nB: Stacked vertically\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 1, "answer": "row", "options": [ "row", "column" ], "factors": { "parent": "row", "child": "wrapped", "alignment": "left" }, "lines": {} } }, { "task_id": "layoutbench-nestedflow-012", "family": "nestedflow", "group_id": "h-12", "image_filename": "h-12.png", "image_sha256": "ce42e30c0bffa72ca47a923d41027233a6c7d1f57dd0e861e7e55d5c45b37e7c", "ground_truth": { "choice": "A" }, "prompt": "Look inside the large section headed Field notes. How are its small cards arranged? Ignore how Field notes is placed relative to Harbor log.\nA: Multiple horizontal rows, with a shorter final row\nB: One vertical column\nC: One horizontal row\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 1, "answer": "wrapped", "options": [ "wrapped", "column", "row" ], "factors": { "parent": "row", "child": "wrapped", "alignment": "left" }, "lines": {} } }, { "task_id": "layoutbench-nestedwrap-004", "family": "nestedwrap", "group_id": "h-12", "image_filename": "h-12.png", "image_sha256": "ce42e30c0bffa72ca47a923d41027233a6c7d1f57dd0e861e7e55d5c45b37e7c", "ground_truth": { "choice": "A" }, "prompt": "Inside Field notes, look only at the shorter final row of cards. How is it placed horizontally inside its own outlined inner frame? Ignore the surrounding page's arrangement.\nA: Against that frame's left inner edge\nB: Against that frame's right inner edge\nC: Centered in that frame\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 1, "answer": "left", "options": [ "left", "right", "center" ], "factors": { "parent": "row", "child": "wrapped", "alignment": "left" }, "lines": {} } }, { "task_id": "layoutbench-topflow-013", "family": "topflow", "group_id": "h-13", "image_filename": "h-13.png", "image_sha256": "73cc1b0894680664693d0cd4727ba8fbf7e28c29dd8826cd4bf67e92b220a67e", "ground_truth": { "choice": "A" }, "prompt": "Look at the two large outlined sections headed Field notes and Harbor log. At the top level, how are these two sections arranged? Ignore the arrangement of cards inside either section.\nA: Stacked vertically\nB: Side by side horizontally\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 0, "answer": "column", "options": [ "column", "row" ], "factors": { "parent": "column", "child": "row" }, "lines": {} } }, { "task_id": "layoutbench-nestedflow-013", "family": "nestedflow", "group_id": "h-13", "image_filename": "h-13.png", "image_sha256": "73cc1b0894680664693d0cd4727ba8fbf7e28c29dd8826cd4bf67e92b220a67e", "ground_truth": { "choice": "B" }, "prompt": "Look inside the large section headed Field notes. How are its small cards arranged? Ignore how Field notes is placed relative to Harbor log.\nA: Multiple horizontal rows, with a shorter final row\nB: One horizontal row\nC: One vertical column\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 0, "answer": "row", "options": [ "wrapped", "row", "column" ], "factors": { "parent": "column", "child": "row" }, "lines": {} } }, { "task_id": "layoutbench-topflow-014", "family": "topflow", "group_id": "h-14", "image_filename": "h-14.png", "image_sha256": "b9c75a3dc7e8f5176ebc4a845e41fe06c3e91bdc3fdb02ff91cb6cf8bd63dc8f", "ground_truth": { "choice": "B" }, "prompt": "Look at the two large outlined sections headed Field notes and Harbor log. At the top level, how are these two sections arranged? Ignore the arrangement of cards inside either section.\nA: Side by side horizontally\nB: Stacked vertically\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 1, "answer": "column", "options": [ "row", "column" ], "factors": { "parent": "column", "child": "row" }, "lines": {} } }, { "task_id": "layoutbench-nestedflow-014", "family": "nestedflow", "group_id": "h-14", "image_filename": "h-14.png", "image_sha256": "b9c75a3dc7e8f5176ebc4a845e41fe06c3e91bdc3fdb02ff91cb6cf8bd63dc8f", "ground_truth": { "choice": "C" }, "prompt": "Look inside the large section headed Field notes. How are its small cards arranged? Ignore how Field notes is placed relative to Harbor log.\nA: Multiple horizontal rows, with a shorter final row\nB: One vertical column\nC: One horizontal row\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 1, "answer": "row", "options": [ "wrapped", "column", "row" ], "factors": { "parent": "column", "child": "row" }, "lines": {} } }, { "task_id": "layoutbench-topflow-015", "family": "topflow", "group_id": "h-15", "image_filename": "h-15.png", "image_sha256": "2a0d01a2f64b8d14df5ce81eaf53efb7422372ae9757687eb7f14ea7d7bc6916", "ground_truth": { "choice": "B" }, "prompt": "Look at the two large outlined sections headed Field notes and Harbor log. At the top level, how are these two sections arranged? Ignore the arrangement of cards inside either section.\nA: Side by side horizontally\nB: Stacked vertically\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 0, "answer": "column", "options": [ "row", "column" ], "factors": { "parent": "column", "child": "row" }, "lines": {} } }, { "task_id": "layoutbench-nestedflow-015", "family": "nestedflow", "group_id": "h-15", "image_filename": "h-15.png", "image_sha256": "2a0d01a2f64b8d14df5ce81eaf53efb7422372ae9757687eb7f14ea7d7bc6916", "ground_truth": { "choice": "C" }, "prompt": "Look inside the large section headed Field notes. How are its small cards arranged? Ignore how Field notes is placed relative to Harbor log.\nA: Multiple horizontal rows, with a shorter final row\nB: One vertical column\nC: One horizontal row\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 0, "answer": "row", "options": [ "wrapped", "column", "row" ], "factors": { "parent": "column", "child": "row" }, "lines": {} } }, { "task_id": "layoutbench-topflow-016", "family": "topflow", "group_id": "h-16", "image_filename": "h-16.png", "image_sha256": "81202a6843a545ef0629321d9309200371f961552c69b24027c4136e5c77e49a", "ground_truth": { "choice": "A" }, "prompt": "Look at the two large outlined sections headed Field notes and Harbor log. At the top level, how are these two sections arranged? Ignore the arrangement of cards inside either section.\nA: Stacked vertically\nB: Side by side horizontally\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 1, "answer": "column", "options": [ "column", "row" ], "factors": { "parent": "column", "child": "row" }, "lines": {} } }, { "task_id": "layoutbench-nestedflow-016", "family": "nestedflow", "group_id": "h-16", "image_filename": "h-16.png", "image_sha256": "81202a6843a545ef0629321d9309200371f961552c69b24027c4136e5c77e49a", "ground_truth": { "choice": "B" }, "prompt": "Look inside the large section headed Field notes. How are its small cards arranged? Ignore how Field notes is placed relative to Harbor log.\nA: One vertical column\nB: One horizontal row\nC: Multiple horizontal rows, with a shorter final row\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 1, "answer": "row", "options": [ "column", "row", "wrapped" ], "factors": { "parent": "column", "child": "row" }, "lines": {} } }, { "task_id": "layoutbench-topflow-017", "family": "topflow", "group_id": "h-17", "image_filename": "h-17.png", "image_sha256": "a6dde0c147d50a2fb7a2347d51e9e3351dde82f2e501be2b338b4ceeaad6883f", "ground_truth": { "choice": "B" }, "prompt": "Look at the two large outlined sections headed Field notes and Harbor log. At the top level, how are these two sections arranged? Ignore the arrangement of cards inside either section.\nA: Side by side horizontally\nB: Stacked vertically\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 0, "answer": "column", "options": [ "row", "column" ], "factors": { "parent": "column", "child": "column" }, "lines": {} } }, { "task_id": "layoutbench-nestedflow-017", "family": "nestedflow", "group_id": "h-17", "image_filename": "h-17.png", "image_sha256": "a6dde0c147d50a2fb7a2347d51e9e3351dde82f2e501be2b338b4ceeaad6883f", "ground_truth": { "choice": "A" }, "prompt": "Look inside the large section headed Field notes. How are its small cards arranged? Ignore how Field notes is placed relative to Harbor log.\nA: One vertical column\nB: One horizontal row\nC: Multiple horizontal rows, with a shorter final row\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 0, "answer": "column", "options": [ "column", "row", "wrapped" ], "factors": { "parent": "column", "child": "column" }, "lines": {} } }, { "task_id": "layoutbench-topflow-018", "family": "topflow", "group_id": "h-18", "image_filename": "h-18.png", "image_sha256": "d0bbda389e76accbcb4c2d62e9931cca31806af19f961963f3a5f618ced07cb8", "ground_truth": { "choice": "B" }, "prompt": "Look at the two large outlined sections headed Field notes and Harbor log. At the top level, how are these two sections arranged? Ignore the arrangement of cards inside either section.\nA: Side by side horizontally\nB: Stacked vertically\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 1, "answer": "column", "options": [ "row", "column" ], "factors": { "parent": "column", "child": "column" }, "lines": {} } }, { "task_id": "layoutbench-nestedflow-018", "family": "nestedflow", "group_id": "h-18", "image_filename": "h-18.png", "image_sha256": "d0bbda389e76accbcb4c2d62e9931cca31806af19f961963f3a5f618ced07cb8", "ground_truth": { "choice": "C" }, "prompt": "Look inside the large section headed Field notes. How are its small cards arranged? Ignore how Field notes is placed relative to Harbor log.\nA: One horizontal row\nB: Multiple horizontal rows, with a shorter final row\nC: One vertical column\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 1, "answer": "column", "options": [ "row", "wrapped", "column" ], "factors": { "parent": "column", "child": "column" }, "lines": {} } }, { "task_id": "layoutbench-topflow-019", "family": "topflow", "group_id": "h-19", "image_filename": "h-19.png", "image_sha256": "747f3f9149d4ec8088138a66363ee8c3ca1a73699c6e0ec0bc06930070b028be", "ground_truth": { "choice": "B" }, "prompt": "Look at the two large outlined sections headed Field notes and Harbor log. At the top level, how are these two sections arranged? Ignore the arrangement of cards inside either section.\nA: Side by side horizontally\nB: Stacked vertically\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 0, "answer": "column", "options": [ "row", "column" ], "factors": { "parent": "column", "child": "column" }, "lines": {} } }, { "task_id": "layoutbench-nestedflow-019", "family": "nestedflow", "group_id": "h-19", "image_filename": "h-19.png", "image_sha256": "747f3f9149d4ec8088138a66363ee8c3ca1a73699c6e0ec0bc06930070b028be", "ground_truth": { "choice": "C" }, "prompt": "Look inside the large section headed Field notes. How are its small cards arranged? Ignore how Field notes is placed relative to Harbor log.\nA: Multiple horizontal rows, with a shorter final row\nB: One horizontal row\nC: One vertical column\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 0, "answer": "column", "options": [ "wrapped", "row", "column" ], "factors": { "parent": "column", "child": "column" }, "lines": {} } }, { "task_id": "layoutbench-topflow-020", "family": "topflow", "group_id": "h-20", "image_filename": "h-20.png", "image_sha256": "c206898a1cce5a359c56a498bbf996ed59750d7550277738befcbc065e071678", "ground_truth": { "choice": "B" }, "prompt": "Look at the two large outlined sections headed Field notes and Harbor log. At the top level, how are these two sections arranged? Ignore the arrangement of cards inside either section.\nA: Side by side horizontally\nB: Stacked vertically\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 1, "answer": "column", "options": [ "row", "column" ], "factors": { "parent": "column", "child": "column" }, "lines": {} } }, { "task_id": "layoutbench-nestedflow-020", "family": "nestedflow", "group_id": "h-20", "image_filename": "h-20.png", "image_sha256": "c206898a1cce5a359c56a498bbf996ed59750d7550277738befcbc065e071678", "ground_truth": { "choice": "A" }, "prompt": "Look inside the large section headed Field notes. How are its small cards arranged? Ignore how Field notes is placed relative to Harbor log.\nA: One vertical column\nB: One horizontal row\nC: Multiple horizontal rows, with a shorter final row\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 1, "answer": "column", "options": [ "column", "row", "wrapped" ], "factors": { "parent": "column", "child": "column" }, "lines": {} } }, { "task_id": "layoutbench-topflow-021", "family": "topflow", "group_id": "h-21", "image_filename": "h-21.png", "image_sha256": "1d4800c3ed8fc4fed51234e11ff9e69e776daf5d7e63e0dd8973bdcffd70fb26", "ground_truth": { "choice": "A" }, "prompt": "Look at the two large outlined sections headed Field notes and Harbor log. At the top level, how are these two sections arranged? Ignore the arrangement of cards inside either section.\nA: Stacked vertically\nB: Side by side horizontally\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 0, "answer": "column", "options": [ "column", "row" ], "factors": { "parent": "column", "child": "wrapped", "alignment": "left" }, "lines": {} } }, { "task_id": "layoutbench-nestedflow-021", "family": "nestedflow", "group_id": "h-21", "image_filename": "h-21.png", "image_sha256": "1d4800c3ed8fc4fed51234e11ff9e69e776daf5d7e63e0dd8973bdcffd70fb26", "ground_truth": { "choice": "B" }, "prompt": "Look inside the large section headed Field notes. How are its small cards arranged? Ignore how Field notes is placed relative to Harbor log.\nA: One vertical column\nB: Multiple horizontal rows, with a shorter final row\nC: One horizontal row\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 0, "answer": "wrapped", "options": [ "column", "wrapped", "row" ], "factors": { "parent": "column", "child": "wrapped", "alignment": "left" }, "lines": {} } }, { "task_id": "layoutbench-nestedwrap-005", "family": "nestedwrap", "group_id": "h-21", "image_filename": "h-21.png", "image_sha256": "1d4800c3ed8fc4fed51234e11ff9e69e776daf5d7e63e0dd8973bdcffd70fb26", "ground_truth": { "choice": "A" }, "prompt": "Inside Field notes, look only at the shorter final row of cards. How is it placed horizontally inside its own outlined inner frame? Ignore the surrounding page's arrangement.\nA: Against that frame's left inner edge\nB: Centered in that frame\nC: Against that frame's right inner edge\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 0, "answer": "left", "options": [ "left", "center", "right" ], "factors": { "parent": "column", "child": "wrapped", "alignment": "left" }, "lines": {} } }, { "task_id": "layoutbench-topflow-022", "family": "topflow", "group_id": "h-22", "image_filename": "h-22.png", "image_sha256": "a7b0cd9eb4f906fa4967205ed5db6e0c7e71282dd1312c6b47690b8aa501cfa9", "ground_truth": { "choice": "B" }, "prompt": "Look at the two large outlined sections headed Field notes and Harbor log. At the top level, how are these two sections arranged? Ignore the arrangement of cards inside either section.\nA: Side by side horizontally\nB: Stacked vertically\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 1, "answer": "column", "options": [ "row", "column" ], "factors": { "parent": "column", "child": "wrapped", "alignment": "left" }, "lines": {} } }, { "task_id": "layoutbench-nestedflow-022", "family": "nestedflow", "group_id": "h-22", "image_filename": "h-22.png", "image_sha256": "a7b0cd9eb4f906fa4967205ed5db6e0c7e71282dd1312c6b47690b8aa501cfa9", "ground_truth": { "choice": "A" }, "prompt": "Look inside the large section headed Field notes. How are its small cards arranged? Ignore how Field notes is placed relative to Harbor log.\nA: Multiple horizontal rows, with a shorter final row\nB: One vertical column\nC: One horizontal row\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 1, "answer": "wrapped", "options": [ "wrapped", "column", "row" ], "factors": { "parent": "column", "child": "wrapped", "alignment": "left" }, "lines": {} } }, { "task_id": "layoutbench-nestedwrap-006", "family": "nestedwrap", "group_id": "h-22", "image_filename": "h-22.png", "image_sha256": "a7b0cd9eb4f906fa4967205ed5db6e0c7e71282dd1312c6b47690b8aa501cfa9", "ground_truth": { "choice": "B" }, "prompt": "Inside Field notes, look only at the shorter final row of cards. How is it placed horizontally inside its own outlined inner frame? Ignore the surrounding page's arrangement.\nA: Against that frame's right inner edge\nB: Against that frame's left inner edge\nC: Centered in that frame\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 1, "answer": "left", "options": [ "right", "left", "center" ], "factors": { "parent": "column", "child": "wrapped", "alignment": "left" }, "lines": {} } }, { "task_id": "layoutbench-topflow-023", "family": "topflow", "group_id": "h-23", "image_filename": "h-23.png", "image_sha256": "6047742c65fc42dcbf3ff66b9a3f2d8c60e28fdb3910e0ff99529f204c2de7ff", "ground_truth": { "choice": "A" }, "prompt": "Look at the two large outlined sections headed Field notes and Harbor log. At the top level, how are these two sections arranged? Ignore the arrangement of cards inside either section.\nA: Stacked vertically\nB: Side by side horizontally\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 0, "answer": "column", "options": [ "column", "row" ], "factors": { "parent": "column", "child": "wrapped", "alignment": "left" }, "lines": {} } }, { "task_id": "layoutbench-nestedflow-023", "family": "nestedflow", "group_id": "h-23", "image_filename": "h-23.png", "image_sha256": "6047742c65fc42dcbf3ff66b9a3f2d8c60e28fdb3910e0ff99529f204c2de7ff", "ground_truth": { "choice": "A" }, "prompt": "Look inside the large section headed Field notes. How are its small cards arranged? Ignore how Field notes is placed relative to Harbor log.\nA: Multiple horizontal rows, with a shorter final row\nB: One horizontal row\nC: One vertical column\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 0, "answer": "wrapped", "options": [ "wrapped", "row", "column" ], "factors": { "parent": "column", "child": "wrapped", "alignment": "left" }, "lines": {} } }, { "task_id": "layoutbench-nestedwrap-007", "family": "nestedwrap", "group_id": "h-23", "image_filename": "h-23.png", "image_sha256": "6047742c65fc42dcbf3ff66b9a3f2d8c60e28fdb3910e0ff99529f204c2de7ff", "ground_truth": { "choice": "A" }, "prompt": "Inside Field notes, look only at the shorter final row of cards. How is it placed horizontally inside its own outlined inner frame? Ignore the surrounding page's arrangement.\nA: Against that frame's left inner edge\nB: Centered in that frame\nC: Against that frame's right inner edge\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 0, "answer": "left", "options": [ "left", "center", "right" ], "factors": { "parent": "column", "child": "wrapped", "alignment": "left" }, "lines": {} } }, { "task_id": "layoutbench-topflow-024", "family": "topflow", "group_id": "h-24", "image_filename": "h-24.png", "image_sha256": "78b83f2cb2d0ed8ba47fc3705d2d914b8daabf05c5775d59531abcff06fce754", "ground_truth": { "choice": "B" }, "prompt": "Look at the two large outlined sections headed Field notes and Harbor log. At the top level, how are these two sections arranged? Ignore the arrangement of cards inside either section.\nA: Side by side horizontally\nB: Stacked vertically\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 1, "answer": "column", "options": [ "row", "column" ], "factors": { "parent": "column", "child": "wrapped", "alignment": "left" }, "lines": {} } }, { "task_id": "layoutbench-nestedflow-024", "family": "nestedflow", "group_id": "h-24", "image_filename": "h-24.png", "image_sha256": "78b83f2cb2d0ed8ba47fc3705d2d914b8daabf05c5775d59531abcff06fce754", "ground_truth": { "choice": "B" }, "prompt": "Look inside the large section headed Field notes. How are its small cards arranged? Ignore how Field notes is placed relative to Harbor log.\nA: One horizontal row\nB: Multiple horizontal rows, with a shorter final row\nC: One vertical column\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 1, "answer": "wrapped", "options": [ "row", "wrapped", "column" ], "factors": { "parent": "column", "child": "wrapped", "alignment": "left" }, "lines": {} } }, { "task_id": "layoutbench-nestedwrap-008", "family": "nestedwrap", "group_id": "h-24", "image_filename": "h-24.png", "image_sha256": "78b83f2cb2d0ed8ba47fc3705d2d914b8daabf05c5775d59531abcff06fce754", "ground_truth": { "choice": "C" }, "prompt": "Inside Field notes, look only at the shorter final row of cards. How is it placed horizontally inside its own outlined inner frame? Ignore the surrounding page's arrangement.\nA: Centered in that frame\nB: Against that frame's right inner edge\nC: Against that frame's left inner edge\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 1, "answer": "left", "options": [ "center", "right", "left" ], "factors": { "parent": "column", "child": "wrapped", "alignment": "left" }, "lines": {} } }, { "task_id": "layoutbench-nestedwrap-009", "family": "nestedwrap", "group_id": "h-25", "image_filename": "h-25.png", "image_sha256": "27855b2e6fced92816b6fb010b410ab15744b572d561dd18cb2a672e49939fbe", "ground_truth": { "choice": "C" }, "prompt": "Inside Field notes, look only at the shorter final row of cards. How is it placed horizontally inside its own outlined inner frame? Ignore the surrounding page's arrangement.\nA: Against that frame's left inner edge\nB: Against that frame's right inner edge\nC: Centered in that frame\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 0, "answer": "center", "options": [ "left", "right", "center" ], "factors": { "parent": "row", "child": "wrapped", "alignment": "center" }, "lines": {} } }, { "task_id": "layoutbench-nestedwrap-010", "family": "nestedwrap", "group_id": "h-26", "image_filename": "h-26.png", "image_sha256": "0286d84e3f7b26d0056ade80f04cea37c03f43c2806c0972e3f58fd7d010635b", "ground_truth": { "choice": "A" }, "prompt": "Inside Field notes, look only at the shorter final row of cards. How is it placed horizontally inside its own outlined inner frame? Ignore the surrounding page's arrangement.\nA: Centered in that frame\nB: Against that frame's left inner edge\nC: Against that frame's right inner edge\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 1, "answer": "center", "options": [ "center", "left", "right" ], "factors": { "parent": "row", "child": "wrapped", "alignment": "center" }, "lines": {} } }, { "task_id": "layoutbench-nestedwrap-011", "family": "nestedwrap", "group_id": "h-27", "image_filename": "h-27.png", "image_sha256": "d6307216da29e0a4f2bf1f2c84210ded0dba685ea928332720cf5ab92cf79e31", "ground_truth": { "choice": "A" }, "prompt": "Inside Field notes, look only at the shorter final row of cards. How is it placed horizontally inside its own outlined inner frame? Ignore the surrounding page's arrangement.\nA: Centered in that frame\nB: Against that frame's left inner edge\nC: Against that frame's right inner edge\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 0, "answer": "center", "options": [ "center", "left", "right" ], "factors": { "parent": "row", "child": "wrapped", "alignment": "center" }, "lines": {} } }, { "task_id": "layoutbench-nestedwrap-012", "family": "nestedwrap", "group_id": "h-28", "image_filename": "h-28.png", "image_sha256": "544e484f4d9c02418e4406dd6c79ad4ab511897f3d55f9848531250a717c6755", "ground_truth": { "choice": "C" }, "prompt": "Inside Field notes, look only at the shorter final row of cards. How is it placed horizontally inside its own outlined inner frame? Ignore the surrounding page's arrangement.\nA: Against that frame's left inner edge\nB: Against that frame's right inner edge\nC: Centered in that frame\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 1, "answer": "center", "options": [ "left", "right", "center" ], "factors": { "parent": "row", "child": "wrapped", "alignment": "center" }, "lines": {} } }, { "task_id": "layoutbench-nestedwrap-013", "family": "nestedwrap", "group_id": "h-29", "image_filename": "h-29.png", "image_sha256": "bf9a6f99ad42c19b670a7394e559760fd81c6052e8717afdd345a0b1a1e79202", "ground_truth": { "choice": "B" }, "prompt": "Inside Field notes, look only at the shorter final row of cards. How is it placed horizontally inside its own outlined inner frame? Ignore the surrounding page's arrangement.\nA: Centered in that frame\nB: Against that frame's right inner edge\nC: Against that frame's left inner edge\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 0, "answer": "right", "options": [ "center", "right", "left" ], "factors": { "parent": "row", "child": "wrapped", "alignment": "right" }, "lines": {} } }, { "task_id": "layoutbench-nestedwrap-014", "family": "nestedwrap", "group_id": "h-30", "image_filename": "h-30.png", "image_sha256": "5efdb664d144ae4df5c7dc476be71682041956693b29cb6c9d303f35bbaf20f9", "ground_truth": { "choice": "C" }, "prompt": "Inside Field notes, look only at the shorter final row of cards. How is it placed horizontally inside its own outlined inner frame? Ignore the surrounding page's arrangement.\nA: Against that frame's left inner edge\nB: Centered in that frame\nC: Against that frame's right inner edge\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 1, "answer": "right", "options": [ "left", "center", "right" ], "factors": { "parent": "row", "child": "wrapped", "alignment": "right" }, "lines": {} } }, { "task_id": "layoutbench-nestedwrap-015", "family": "nestedwrap", "group_id": "h-31", "image_filename": "h-31.png", "image_sha256": "7d477ca9e59a35587d4499445de2aec0a1ee19fb7a3bee4cb7c398271db3bb5e", "ground_truth": { "choice": "B" }, "prompt": "Inside Field notes, look only at the shorter final row of cards. How is it placed horizontally inside its own outlined inner frame? Ignore the surrounding page's arrangement.\nA: Against that frame's left inner edge\nB: Against that frame's right inner edge\nC: Centered in that frame\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 0, "answer": "right", "options": [ "left", "right", "center" ], "factors": { "parent": "row", "child": "wrapped", "alignment": "right" }, "lines": {} } }, { "task_id": "layoutbench-nestedwrap-016", "family": "nestedwrap", "group_id": "h-32", "image_filename": "h-32.png", "image_sha256": "eae65c052f264b8402769a5300480339f38fab9bcddc4b88f221d157a2680e1a", "ground_truth": { "choice": "B" }, "prompt": "Inside Field notes, look only at the shorter final row of cards. How is it placed horizontally inside its own outlined inner frame? Ignore the surrounding page's arrangement.\nA: Against that frame's left inner edge\nB: Against that frame's right inner edge\nC: Centered in that frame\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 1, "answer": "right", "options": [ "left", "right", "center" ], "factors": { "parent": "row", "child": "wrapped", "alignment": "right" }, "lines": {} } }, { "task_id": "layoutbench-nestedwrap-017", "family": "nestedwrap", "group_id": "h-33", "image_filename": "h-33.png", "image_sha256": "a0fdde798f6344c8ff03337e534ae7467ea10432fabc306bcf4b3040fe978b28", "ground_truth": { "choice": "A" }, "prompt": "Inside Field notes, look only at the shorter final row of cards. How is it placed horizontally inside its own outlined inner frame? Ignore the surrounding page's arrangement.\nA: Centered in that frame\nB: Against that frame's right inner edge\nC: Against that frame's left inner edge\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 0, "answer": "center", "options": [ "center", "right", "left" ], "factors": { "parent": "column", "child": "wrapped", "alignment": "center" }, "lines": {} } }, { "task_id": "layoutbench-nestedwrap-018", "family": "nestedwrap", "group_id": "h-34", "image_filename": "h-34.png", "image_sha256": "a7b0142776ab45bd9f68c91aac9951adb73f83d76af73357c62636ba25023da7", "ground_truth": { "choice": "A" }, "prompt": "Inside Field notes, look only at the shorter final row of cards. How is it placed horizontally inside its own outlined inner frame? Ignore the surrounding page's arrangement.\nA: Centered in that frame\nB: Against that frame's right inner edge\nC: Against that frame's left inner edge\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 1, "answer": "center", "options": [ "center", "right", "left" ], "factors": { "parent": "column", "child": "wrapped", "alignment": "center" }, "lines": {} } }, { "task_id": "layoutbench-nestedwrap-019", "family": "nestedwrap", "group_id": "h-35", "image_filename": "h-35.png", "image_sha256": "97dd5a262593af54f5ef693d1ec37feb67bf4900ab404d1828b3e4e94467fb39", "ground_truth": { "choice": "C" }, "prompt": "Inside Field notes, look only at the shorter final row of cards. How is it placed horizontally inside its own outlined inner frame? Ignore the surrounding page's arrangement.\nA: Against that frame's right inner edge\nB: Against that frame's left inner edge\nC: Centered in that frame\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 0, "answer": "center", "options": [ "right", "left", "center" ], "factors": { "parent": "column", "child": "wrapped", "alignment": "center" }, "lines": {} } }, { "task_id": "layoutbench-nestedwrap-020", "family": "nestedwrap", "group_id": "h-36", "image_filename": "h-36.png", "image_sha256": "175d95b20cafef3137e69ede0ef0c8a19d5274444584930d3157cfe571c20a35", "ground_truth": { "choice": "A" }, "prompt": "Inside Field notes, look only at the shorter final row of cards. How is it placed horizontally inside its own outlined inner frame? Ignore the surrounding page's arrangement.\nA: Centered in that frame\nB: Against that frame's right inner edge\nC: Against that frame's left inner edge\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 1, "answer": "center", "options": [ "center", "right", "left" ], "factors": { "parent": "column", "child": "wrapped", "alignment": "center" }, "lines": {} } }, { "task_id": "layoutbench-nestedwrap-021", "family": "nestedwrap", "group_id": "h-37", "image_filename": "h-37.png", "image_sha256": "7feb231c6385acf9349d6a1d4ad713c28886b149c9ae395e4bc3c63d6c8bda0b", "ground_truth": { "choice": "C" }, "prompt": "Inside Field notes, look only at the shorter final row of cards. How is it placed horizontally inside its own outlined inner frame? Ignore the surrounding page's arrangement.\nA: Against that frame's left inner edge\nB: Centered in that frame\nC: Against that frame's right inner edge\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 0, "answer": "right", "options": [ "left", "center", "right" ], "factors": { "parent": "column", "child": "wrapped", "alignment": "right" }, "lines": {} } }, { "task_id": "layoutbench-nestedwrap-022", "family": "nestedwrap", "group_id": "h-38", "image_filename": "h-38.png", "image_sha256": "373d52b50a1bb6d35560a59d1f4b2c687bc040177d8d76171ad1ba0ef1f392af", "ground_truth": { "choice": "B" }, "prompt": "Inside Field notes, look only at the shorter final row of cards. How is it placed horizontally inside its own outlined inner frame? Ignore the surrounding page's arrangement.\nA: Against that frame's left inner edge\nB: Against that frame's right inner edge\nC: Centered in that frame\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "light", "variant": 1, "answer": "right", "options": [ "left", "right", "center" ], "factors": { "parent": "column", "child": "wrapped", "alignment": "right" }, "lines": {} } }, { "task_id": "layoutbench-nestedwrap-023", "family": "nestedwrap", "group_id": "h-39", "image_filename": "h-39.png", "image_sha256": "8ba7648a28cac3e204f80cfd1748fad8e60cdf0697e98b688ad73f96d71df193", "ground_truth": { "choice": "B" }, "prompt": "Inside Field notes, look only at the shorter final row of cards. How is it placed horizontally inside its own outlined inner frame? Ignore the surrounding page's arrangement.\nA: Centered in that frame\nB: Against that frame's right inner edge\nC: Against that frame's left inner edge\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 0, "answer": "right", "options": [ "center", "right", "left" ], "factors": { "parent": "column", "child": "wrapped", "alignment": "right" }, "lines": {} } }, { "task_id": "layoutbench-nestedwrap-024", "family": "nestedwrap", "group_id": "h-40", "image_filename": "h-40.png", "image_sha256": "a4efeff5aef04660008577c6a8bef60f07d4d801cca4eb52a973313e4c936b56", "ground_truth": { "choice": "C" }, "prompt": "Inside Field notes, look only at the shorter final row of cards. How is it placed horizontally inside its own outlined inner frame? Ignore the surrounding page's arrangement.\nA: Centered in that frame\nB: Against that frame's left inner edge\nC: Against that frame's right inner edge\nReturn only a JSON object with one key, \"choice\", whose value is the selected option letter.", "design": { "kind": "qualitative", "theme": "dark", "variant": 1, "answer": "right", "options": [ "center", "left", "right" ], "factors": { "parent": "column", "child": "wrapped", "alignment": "right" }, "lines": {} } } ], "results": [ { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "921c6c21-3105-48e2-8e7a-277dd66400ee", "result_sha256": "f7430fba77c85a9502d320f2b79e7e3197e0514f8d5993baae99beeeb6898fcf", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0076175, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "A" }, "group_id": "blockalign-left-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-left-light-0.png", "input_tokens": 2576, "key_errors": null, "latency_sec": 2.140302923042327, "model_name": "pareto", "output_tokens": 157, "prediction": { "choice": "A" }, "prompt_sha256": "e32486067ac25398fcca8ac9d957270d096fb6771f17cdb332d35edf376b0f86", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-29T14:17:57.327829+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "90fd2c31-dd18-4c3f-a97b-6236512f8cc8", "result_sha256": "15c69ab9c5075357d1ecf2737d05e29d84b775b9530a15bad486146cad7ad78d", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0077375, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "A" }, "group_id": "blockalign-left-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-left-light-1.png", "input_tokens": 2576, "key_errors": null, "latency_sec": 2.3804736270103604, "model_name": "pareto", "output_tokens": 173, "prediction": { "choice": "A" }, "prompt_sha256": "e32486067ac25398fcca8ac9d957270d096fb6771f17cdb332d35edf376b0f86", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-29T14:17:10.305321+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "53fbfa53-97e3-404a-ae0d-c9f76107215a", "result_sha256": "64336428701960148ff57c06f731268edd07130269083583ad316a483eaf1988", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0073175, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "B" }, "group_id": "blockalign-left-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-left-dark-0.png", "input_tokens": 2576, "key_errors": null, "latency_sec": 2.1267722349148244, "model_name": "pareto", "output_tokens": 117, "prediction": { "choice": "B" }, "prompt_sha256": "7d6254d951604c657e71aeb7cae9178b26bc4d3dd7020cdf265225567557e5dd", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-29T14:17:21.354172+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "8fc67a64-23df-4913-bcfd-76f1f828697c", "result_sha256": "25ff787db08be099812f92fc3889bfdcbd53ab41fdaf1337876dbc41a4701d50", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.007505, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "C" }, "group_id": "blockalign-left-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-left-dark-1.png", "input_tokens": 2576, "key_errors": null, "latency_sec": 2.5952626201324165, "model_name": "pareto", "output_tokens": 142, "prediction": { "choice": "C" }, "prompt_sha256": "aec8632f26486517e0ed07640068efdb11c079f166b430d6a2eb3e7ecb5c47b1", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-29T14:16:51.500914+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "b7d77d05-ccd8-4f84-a6ed-6a0bdb383274", "result_sha256": "2d4e36e9c65b931e25014140880d6a0bcc8914e2e9e39a561bfb2c79f7c98b45", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.007535, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "B" }, "group_id": "blockalign-center-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-center-light-0.png", "input_tokens": 2576, "key_errors": null, "latency_sec": 2.036298847058788, "model_name": "pareto", "output_tokens": 146, "prediction": { "choice": "B" }, "prompt_sha256": "376fb84b749ec9e6ed85a09a2f7a073f4bb3962b1c113f20e7e3bcf9e146f18b", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-29T14:17:31.922134+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "d8115716-10ae-4f2d-84d4-55b8fee5f3a2", "result_sha256": "e8d7b997efdc147c5f8009beff2876b383122cfb53f78eed8f685255e8ec54ef", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0077375, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "C" }, "group_id": "blockalign-center-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-center-light-1.png", "input_tokens": 2576, "key_errors": null, "latency_sec": 2.035596271045506, "model_name": "pareto", "output_tokens": 173, "prediction": { "choice": "C" }, "prompt_sha256": "7d6254d951604c657e71aeb7cae9178b26bc4d3dd7020cdf265225567557e5dd", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-29T14:17:23.811015+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "bec45785-16d0-4818-8b15-6058855cfc7c", "result_sha256": "fb896625dd467c26ee0fcbf90c76fc1cffe33d611ad5c6394f85cc9e0fb1c59c", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0078275, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "A" }, "group_id": "blockalign-center-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-center-dark-0.png", "input_tokens": 2576, "key_errors": null, "latency_sec": 2.6830995879136026, "model_name": "pareto", "output_tokens": 185, "prediction": { "choice": "A" }, "prompt_sha256": "19f470e5b91763a91fec11553496167c6525902526ca96fe05ea2d483d2acc16", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-29T14:17:27.738589+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "efd9f6fb-608d-4ce5-8718-fb41ea27aea2", "result_sha256": "24400809bef4f615b5982dba01820a4224ff61dfb52444f7b833b513c50bffae", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.007475, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "C" }, "group_id": "blockalign-center-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-center-dark-1.png", "input_tokens": 2576, "key_errors": null, "latency_sec": 2.127837276086211, "model_name": "pareto", "output_tokens": 138, "prediction": { "choice": "C" }, "prompt_sha256": "7d6254d951604c657e71aeb7cae9178b26bc4d3dd7020cdf265225567557e5dd", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-29T14:17:57.769309+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "94c033f9-59f0-4182-98cf-c85bb0db20a8", "result_sha256": "9beb09561579e2efee58c7fb63f4e60782b0e82502520af615d54b65a0327c5d", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0074525, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "A" }, "group_id": "blockalign-right-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-right-light-0.png", "input_tokens": 2576, "key_errors": null, "latency_sec": 1.9350436450913548, "model_name": "pareto", "output_tokens": 135, "prediction": { "choice": "A" }, "prompt_sha256": "7d6254d951604c657e71aeb7cae9178b26bc4d3dd7020cdf265225567557e5dd", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-29T14:17:43.301224+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "6b215d47-67f3-40be-8e27-32205ac62bef", "result_sha256": "bf8ed329390f168d0ca9c0aa5544098c1fffb25d40184bf8edf6343a01b53029", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0075725, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "B" }, "group_id": "blockalign-right-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-right-light-1.png", "input_tokens": 2576, "key_errors": null, "latency_sec": 2.023338290862739, "model_name": "pareto", "output_tokens": 151, "prediction": { "choice": "B" }, "prompt_sha256": "238f37fc59f145a67466f3f7e834594604126429a37dc7fd71fda81a0610afe4", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-29T14:17:40.713059+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "4f97502c-71da-4cbd-94e6-90302c42d96d", "result_sha256": "56f5ffc6bc77bd58e9fa32a06fed48af6ffe3d316e537f1bad9837e329e04b36", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0073625, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "C" }, "group_id": "blockalign-right-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-right-dark-0.png", "input_tokens": 2576, "key_errors": null, "latency_sec": 1.8565552250947803, "model_name": "pareto", "output_tokens": 123, "prediction": { "choice": "C" }, "prompt_sha256": "19f470e5b91763a91fec11553496167c6525902526ca96fe05ea2d483d2acc16", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-29T14:17:06.562580+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "286810cd-7b60-4f81-9a07-000b0567e5bb", "result_sha256": "2395214aae461cfaa9edbf4ccc8cf9debfeab19b9d2dbd0cb1bd4829d80ed599", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009065, "err": null, "error": null, "error_kind": null, "family": "blockalign", "ground_truth": { "choice": "B" }, "group_id": "blockalign-right-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/blockalign-right-dark-1.png", "input_tokens": 2576, "key_errors": null, "latency_sec": 2.703711590031162, "model_name": "pareto", "output_tokens": 350, "prediction": { "choice": "B" }, "prompt_sha256": "aec8632f26486517e0ed07640068efdb11c079f166b430d6a2eb3e7ecb5c47b1", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-29T14:17:28.151227+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-blockalign-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "bfc714d3-4d20-4407-bda7-cc4b56193916", "result_sha256": "87bd2aa08479ead8b32be7513f66df87933cfdbb3a36a1fb8cf9a2542f3115cb", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0079625, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "B" }, "group_id": "crossalign-start-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-start-light-0.png", "input_tokens": 2642, "key_errors": null, "latency_sec": 2.185381606919691, "model_name": "pareto", "output_tokens": 181, "prediction": { "choice": "B" }, "prompt_sha256": "0f1e528120c9711dd50b5a3ee0dd5163897f25273cfd6b0ec66485192ac32e79", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-29T14:17:20.191942+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "3f494385-0497-4142-b3da-40084b79885e", "result_sha256": "86f3937eb59f97099f67c2fadc4ae8a7d3ecea1c33e7c7bb7327bb1d3016ae7a", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.008375, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "C" }, "group_id": "crossalign-start-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-start-light-1.png", "input_tokens": 2642, "key_errors": null, "latency_sec": 2.816447041928768, "model_name": "pareto", "output_tokens": 236, "prediction": { "choice": "C" }, "prompt_sha256": "a0cc0893516679f3ab248b3cfb7b1e000bacbb17beb9b9bd9dcc6099c756222f", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-29T14:16:51.714613+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "1af3b436-ca1d-4c5b-bcb6-fd5ca3b97de5", "result_sha256": "a7398638ff23dd8343fbe65364c574e5aa69fe3e1223524f962b036d0c76cba8", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00866, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "A" }, "group_id": "crossalign-start-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-start-dark-0.png", "input_tokens": 2642, "key_errors": null, "latency_sec": 2.290109053021297, "model_name": "pareto", "output_tokens": 274, "prediction": { "choice": "A" }, "prompt_sha256": "e6e3e7be0a2b7e91666344c6f0c3a0f931a9ea63d47c2dbd9e1c4cb334420e83", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-29T14:16:59.125175+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "c7046431-bced-461f-ada0-ea7a5c637954", "result_sha256": "52167c6c9a09b9239bb986e90437fd41e758452e8fc299dfcf6d03d13bb1abc2", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0085775, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "D" }, "group_id": "crossalign-start-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-start-dark-1.png", "input_tokens": 2642, "key_errors": null, "latency_sec": 2.161463828990236, "model_name": "pareto", "output_tokens": 263, "prediction": { "choice": "D" }, "prompt_sha256": "a4e6df1b241ba522334256d095328511f40be1d02d97470c916d2ec188ae062f", "provider": "anthropic", "raw_prediction": "{\"choice\": \"D\"}", "recorded_at": "2026-09-29T14:17:29.209588+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "2c6f6299-199a-4c8a-b461-8dc8f2f7495e", "result_sha256": "c0ee5965605dd80f4206c6b4ecb148bf0aa63237ae7f19686dcecd6c1e0b2ee0", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.008375, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "A" }, "group_id": "crossalign-center-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-center-light-0.png", "input_tokens": 2642, "key_errors": null, "latency_sec": 2.347847797907889, "model_name": "pareto", "output_tokens": 236, "prediction": { "choice": "A" }, "prompt_sha256": "ea02b690a39fed2c99e991ca10571f8f5e75cfea4e2db5349dafc50758191f40", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-29T14:17:54.930428+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "9e846ac9-249e-4293-b404-b54dda80ac3e", "result_sha256": "5912c1d092f2fa99cf1c4aa2705624b4b70580cef9060901b7a0a8f07d58bfee", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00797, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "D" }, "group_id": "crossalign-center-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-center-light-1.png", "input_tokens": 2642, "key_errors": null, "latency_sec": 2.0570322540588677, "model_name": "pareto", "output_tokens": 182, "prediction": { "choice": "D" }, "prompt_sha256": "e6e3e7be0a2b7e91666344c6f0c3a0f931a9ea63d47c2dbd9e1c4cb334420e83", "provider": "anthropic", "raw_prediction": "{\"choice\": \"D\"}", "recorded_at": "2026-09-29T14:17:41.967820+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "d10346bd-ce88-4114-a9de-58e5ea0b7e1a", "result_sha256": "cda9f03fdc1f14f72e6c0dbca31abcd9e1ae6895c37c94065093e0b315a59e83", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0077825, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "D" }, "group_id": "crossalign-center-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-center-dark-0.png", "input_tokens": 2642, "key_errors": null, "latency_sec": 2.470191791187972, "model_name": "pareto", "output_tokens": 157, "prediction": { "choice": "D" }, "prompt_sha256": "e6e3e7be0a2b7e91666344c6f0c3a0f931a9ea63d47c2dbd9e1c4cb334420e83", "provider": "anthropic", "raw_prediction": "{\"choice\": \"D\"}", "recorded_at": "2026-09-29T14:17:27.613320+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "c1f50925-b4e1-4e12-9221-73c64c6ccbe6", "result_sha256": "4c84f57cc44e1d4738e79e5f6177fc5e32690516d2b0c4b05e18501c65b1ae3f", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0079325, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "A" }, "group_id": "crossalign-center-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-center-dark-1.png", "input_tokens": 2642, "key_errors": null, "latency_sec": 2.7289232739713043, "model_name": "pareto", "output_tokens": 177, "prediction": { "choice": "A" }, "prompt_sha256": "a0cc0893516679f3ab248b3cfb7b1e000bacbb17beb9b9bd9dcc6099c756222f", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-29T14:17:07.798335+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "f7d1a905-2e78-449d-9fe5-7c8aac63795f", "result_sha256": "a51ab884c303fa803d6a1ea069f43cb689d891da7667b2657241cfc06057bf14", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.007235, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "A" }, "group_id": "crossalign-end-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-end-light-0.png", "input_tokens": 2642, "key_errors": null, "latency_sec": 1.7279513750690967, "model_name": "pareto", "output_tokens": 84, "prediction": { "choice": "A" }, "prompt_sha256": "42e3d5b05e35df09d8f58ddf18bbea16275da96264b53e5005136200c350e010", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-29T14:18:05.533884+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "c17f4598-19a6-4126-bf00-10ac374a4272", "result_sha256": "23e55cf069ec873dc509e3975b0742efff29883a97f20d4373a94e71a740d2a7", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0080525, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "C" }, "group_id": "crossalign-end-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-end-light-1.png", "input_tokens": 2642, "key_errors": null, "latency_sec": 1.9344107639044523, "model_name": "pareto", "output_tokens": 193, "prediction": { "choice": "C" }, "prompt_sha256": "259aeefd7e7471d977cd6004803e1298c147569226cadca5947c59db9f3e2731", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-29T14:17:34.516986+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "2ca8853a-6513-4bea-ba85-5f2563a1ef3e", "result_sha256": "00572d98c70c3c963fe95450d1696e5ec0c8ba7f613cd60f77369328c17cca38", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.007445, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "C" }, "group_id": "crossalign-end-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-end-dark-0.png", "input_tokens": 2642, "key_errors": null, "latency_sec": 1.750789768062532, "model_name": "pareto", "output_tokens": 112, "prediction": { "choice": "C" }, "prompt_sha256": "7179fb2e3aaf1ed7b815d29c079e764245857c5a2a81c76dca5d8cd0b7fcd9e8", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-29T14:17:41.175703+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "9d048525-37be-489a-8e63-820eb0e65d4a", "result_sha256": "1a334fd8b406b19983e6922202fc1017499e6c5fba77350158c7ebbe646310f0", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00824, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "B" }, "group_id": "crossalign-end-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-end-dark-1.png", "input_tokens": 2642, "key_errors": null, "latency_sec": 2.023086281027645, "model_name": "pareto", "output_tokens": 218, "prediction": { "choice": "B" }, "prompt_sha256": "a741f50a81efd135db5359d6779e3d6988fa29ccc1e2f727cb701189f06bf4d8", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-29T14:17:51.749133+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "ad18155c-32ac-4c42-aa9c-ee26c719f37b", "result_sha256": "5d4bd1db101d00e2555b41834e67075a77ed71b6c24d5811c78de0a6bbf28750", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0083225, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "C" }, "group_id": "crossalign-stretch-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-stretch-light-0.png", "input_tokens": 2642, "key_errors": null, "latency_sec": 2.1384620149619877, "model_name": "pareto", "output_tokens": 229, "prediction": { "choice": "C" }, "prompt_sha256": "42e3d5b05e35df09d8f58ddf18bbea16275da96264b53e5005136200c350e010", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-29T14:17:01.374899+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-013", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "4927f6b7-a624-44fe-9438-af50490c8a20", "result_sha256": "a7af3cd771c2ccff6b66d5b0454891f517e67da1865c9fbc010b55802bd06051", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00956, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "B" }, "group_id": "crossalign-stretch-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-stretch-light-1.png", "input_tokens": 2642, "key_errors": null, "latency_sec": 3.3831742801703513, "model_name": "pareto", "output_tokens": 394, "prediction": { "choice": "B" }, "prompt_sha256": "ea02b690a39fed2c99e991ca10571f8f5e75cfea4e2db5349dafc50758191f40", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-29T14:17:29.979337+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-014", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "f02819b0-4e0c-412f-8721-732a2b93aa88", "result_sha256": "8b31352b458ead131172fa9fcbf2ad6ca97c0ab057b57b88fee7b1385ca0d7c7", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00797, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "D" }, "group_id": "crossalign-stretch-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-stretch-dark-0.png", "input_tokens": 2642, "key_errors": null, "latency_sec": 2.0068532151635736, "model_name": "pareto", "output_tokens": 182, "prediction": { "choice": "D" }, "prompt_sha256": "a0cc0893516679f3ab248b3cfb7b1e000bacbb17beb9b9bd9dcc6099c756222f", "provider": "anthropic", "raw_prediction": "{\"choice\": \"D\"}", "recorded_at": "2026-09-29T14:17:12.658501+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-015", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "c1af443d-4b57-4331-ac2b-d6a79137187f", "result_sha256": "67008d9199e78b0905bb4fc61679f267005fd792069be468ed8113ae93daa29f", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00833, "err": null, "error": null, "error_kind": null, "family": "crossalign", "ground_truth": { "choice": "B" }, "group_id": "crossalign-stretch-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/crossalign-stretch-dark-1.png", "input_tokens": 2642, "key_errors": null, "latency_sec": 2.5430529578588903, "model_name": "pareto", "output_tokens": 230, "prediction": { "choice": "B" }, "prompt_sha256": "fed2c2756d1bfbfa4a2899a0ada5790db38828dab3d9688a7a0f2ac04c6b6676", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-29T14:17:50.640380+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-crossalign-016", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "66bd6fba-c8e7-4264-9d13-a0b914c6bc32", "result_sha256": "d90e39cbbcab465ef60a0f0a4153f3576b51f14836cf76d2d71064d503a90fc1", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0066975, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "B" }, "group_id": "direction-row-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-row-light-0.png", "input_tokens": 2547, "key_errors": null, "latency_sec": 1.484726507915184, "model_name": "pareto", "output_tokens": 44, "prediction": { "choice": "B" }, "prompt_sha256": "3768f3feb47a0750f78e6a2c372e707b1b87f7af93fd1fa6b13ef50f1092f226", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-29T14:17:18.061030+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "0a13fdf5-55b0-4014-9ab4-68c20866e672", "result_sha256": "a4ac74a90469bf1d71f9cc355a2bfff83d644c5681cf0bfcd8bf24b7e0ce7f02", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0066825, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "A" }, "group_id": "direction-row-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-row-light-1.png", "input_tokens": 2547, "key_errors": null, "latency_sec": 1.4987934639211744, "model_name": "pareto", "output_tokens": 42, "prediction": { "choice": "A" }, "prompt_sha256": "84ba6fd0e7012086e1f258af7368901e4550a2b9a6e285f06f25a8ab71464ec2", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-29T14:17:33.161957+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "b45261b6-2ada-4a98-ab10-cd0b22057578", "result_sha256": "8af711e1a5f11810e639e30bfa7b8a3031d4c026e7f5a1adfb89c0859d4b6920", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00669, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "B" }, "group_id": "direction-row-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-row-dark-0.png", "input_tokens": 2547, "key_errors": null, "latency_sec": 1.642727104946971, "model_name": "pareto", "output_tokens": 43, "prediction": { "choice": "B" }, "prompt_sha256": "3768f3feb47a0750f78e6a2c372e707b1b87f7af93fd1fa6b13ef50f1092f226", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-29T14:17:25.033824+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "45e6f984-a16c-4959-ac89-4bcd891b5a7b", "result_sha256": "8c91ac2b5206d81a5c3d080037cd9da4ca5e5b45017488692dfc903e89d5b5b4", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0067425, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "A" }, "group_id": "direction-row-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-row-dark-1.png", "input_tokens": 2547, "key_errors": null, "latency_sec": 1.7810401120223105, "model_name": "pareto", "output_tokens": 50, "prediction": { "choice": "A" }, "prompt_sha256": "84ba6fd0e7012086e1f258af7368901e4550a2b9a6e285f06f25a8ab71464ec2", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-29T14:17:58.164703+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "9b693261-801b-49af-ac9e-0aa9e5aefd15", "result_sha256": "e888e003b520cdc7925c7b1741c6b0c4ee0a2c7babdf83e8a1b34f2aa78006bb", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0066825, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "B" }, "group_id": "direction-column-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-column-light-0.png", "input_tokens": 2547, "key_errors": null, "latency_sec": 2.0044628409668803, "model_name": "pareto", "output_tokens": 42, "prediction": { "choice": "B" }, "prompt_sha256": "84ba6fd0e7012086e1f258af7368901e4550a2b9a6e285f06f25a8ab71464ec2", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-29T14:17:11.245418+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "f331b81c-3511-4312-a9ec-8426d6b124df", "result_sha256": "5928c8d6b1d2119175e38103bdd0dffe8bd898cbe5f7b95faf71d32b83bffd26", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006765, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "A" }, "group_id": "direction-column-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-column-light-1.png", "input_tokens": 2547, "key_errors": null, "latency_sec": 1.561554656829685, "model_name": "pareto", "output_tokens": 53, "prediction": { "choice": "A" }, "prompt_sha256": "3768f3feb47a0750f78e6a2c372e707b1b87f7af93fd1fa6b13ef50f1092f226", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-29T14:17:40.635880+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "b60d8957-d87c-49d4-80b8-84bce54b22c3", "result_sha256": "2aa390da1b2961d5e24eb9d9161fa47ab89ccaf59f2deacc2d1eb1bd11601b2c", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00675, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "A" }, "group_id": "direction-column-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-column-dark-0.png", "input_tokens": 2547, "key_errors": null, "latency_sec": 2.2802412561140954, "model_name": "pareto", "output_tokens": 51, "prediction": { "choice": "A" }, "prompt_sha256": "3768f3feb47a0750f78e6a2c372e707b1b87f7af93fd1fa6b13ef50f1092f226", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-29T14:16:57.507226+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "2cdca63c-71b0-486a-9288-cb388e864857", "result_sha256": "dc7fb9cc6448582852060a2fcdea984d0207e8e0978e4c1e0d9fffd869b09a66", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00672, "err": null, "error": null, "error_kind": null, "family": "direction", "ground_truth": { "choice": "B" }, "group_id": "direction-column-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/direction-column-dark-1.png", "input_tokens": 2547, "key_errors": null, "latency_sec": 1.5193028808571398, "model_name": "pareto", "output_tokens": 47, "prediction": { "choice": "B" }, "prompt_sha256": "84ba6fd0e7012086e1f258af7368901e4550a2b9a6e285f06f25a8ab71464ec2", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-29T14:17:04.686176+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-direction-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "aeb7dcd9-5b10-4e3e-8c73-ddbee9c8fb5b", "result_sha256": "8e0e33445c84e4819fba8fec338971a71a26f6c587d11f87fe8b2948763cde7c", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0101, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "D" }, "group_id": "distribution-start-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-start-light-0.png", "input_tokens": 2648, "key_errors": null, "latency_sec": 3.0047242019791156, "model_name": "pareto", "output_tokens": 464, "prediction": { "choice": "D" }, "prompt_sha256": "c5e8a062bab89a0ada9699f67d71f11574c44b07f47fe6b471893162d79e822e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"D\"}", "recorded_at": "2026-09-29T14:17:53.461238+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "37574653-dbae-4e8d-9a86-6a35dbd57de2", "result_sha256": "0a461cbdb832b414b3dd3b5d71bac6df5aa1d7c9970a807ffc1ca172a972f49a", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0095975, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "F" }, "group_id": "distribution-start-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-start-light-1.png", "input_tokens": 2648, "key_errors": null, "latency_sec": 3.5013921468053013, "model_name": "pareto", "output_tokens": 397, "prediction": { "choice": "F" }, "prompt_sha256": "a0ccf33b0dbbc426b80b82e74fdfbafb7cd7dff4573b6bb132185c8b476a57b2", "provider": "anthropic", "raw_prediction": "{\"choice\":\"F\"}", "recorded_at": "2026-09-29T14:18:02.517458+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "cd4111af-d1e7-4475-9b7a-a2f9d6723f0b", "result_sha256": "4bd7eb9b1f2f365bc75cf1ee59916ba2cb12e5bb3611db93ffc5edfa80fd1f84", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0103925, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "B" }, "group_id": "distribution-start-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-start-dark-0.png", "input_tokens": 2648, "key_errors": null, "latency_sec": 3.835082483012229, "model_name": "pareto", "output_tokens": 503, "prediction": { "choice": "B" }, "prompt_sha256": "1eda56ca21b06fb039bc76a8c3d84de28b1ce8490ca33890a7e3b2bcf7d45437", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-29T14:17:42.733604+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "84105f2a-8762-4358-9098-114e54543c91", "result_sha256": "00680825bc13b466139773f8548786c127ed2d9ed713ec7be2183ae5528a25bf", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009125, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "D" }, "group_id": "distribution-start-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-start-dark-1.png", "input_tokens": 2648, "key_errors": null, "latency_sec": 2.736703363014385, "model_name": "pareto", "output_tokens": 334, "prediction": { "choice": "D" }, "prompt_sha256": "3ad96dc5182ffe205957a476703a3fa61818fc870ed5be5256b09869b0bc7323", "provider": "anthropic", "raw_prediction": "{\"choice\": \"D\"}", "recorded_at": "2026-09-29T14:17:09.753176+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "5e3328a0-b42c-4b34-a42b-61d518d912f4", "result_sha256": "9544efdadd574a2bb154b6096f8d3c01fecac6382cf4ecb04cfed76281052867", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0085175, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "E" }, "group_id": "distribution-center-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-center-light-0.png", "input_tokens": 2648, "key_errors": null, "latency_sec": 2.5328790300991386, "model_name": "pareto", "output_tokens": 253, "prediction": { "choice": "E" }, "prompt_sha256": "d47c19cab91f33f4d25cba7cf52697e79dfc50d2011d63234517770b7bbe3d42", "provider": "anthropic", "raw_prediction": "{\"choice\": \"E\"}", "recorded_at": "2026-09-29T14:17:02.313807+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "dc6a4a8a-7185-42ba-a799-d063d5e30a56", "result_sha256": "2c78dd41d206207444238e3211eee9c7391db0c28ca7d4f8ae6505c8a2569067", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.008825, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "E" }, "group_id": "distribution-center-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-center-light-1.png", "input_tokens": 2648, "key_errors": null, "latency_sec": 2.2895193859003484, "model_name": "pareto", "output_tokens": 294, "prediction": { "choice": "E" }, "prompt_sha256": "2f316a4393b531fa7b5e9b8090c0a3c148e22a78e6e8f9382c57e24f29d56be7", "provider": "anthropic", "raw_prediction": "{\"choice\": \"E\"}", "recorded_at": "2026-09-29T14:17:00.639986+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "32f51013-8ff0-43f0-bd92-3d91d9c731f1", "result_sha256": "3413a8472609c20f8dd930cfdbb106875b375ac97ddc3eefabba258f0f3dc6e1", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00869, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "B" }, "group_id": "distribution-center-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-center-dark-0.png", "input_tokens": 2648, "key_errors": null, "latency_sec": 2.2318063939455897, "model_name": "pareto", "output_tokens": 276, "prediction": { "choice": "B" }, "prompt_sha256": "d3aa9dd8445fc739c14269d23f3f90c4093713cd7ec7f0fe052ca16e2f8ae3b4", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-29T14:17:06.546433+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "da80b19f-b36d-4cac-b079-6aa3cdb3e1a5", "result_sha256": "914b71deab737a45bda32a4b87be17d0bc83fa3711fb95b33ea3815299f0b28c", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0116375, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "D" }, "group_id": "distribution-center-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-center-dark-1.png", "input_tokens": 2648, "key_errors": null, "latency_sec": 3.7103604848962277, "model_name": "pareto", "output_tokens": 669, "prediction": { "choice": "D" }, "prompt_sha256": "27147e1f86f207706e058c66a175f2db6da63ea3c8dfd58dc353d60bc9922569", "provider": "anthropic", "raw_prediction": "{\"choice\": \"D\"}", "recorded_at": "2026-09-29T14:17:31.467954+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "3cb83e59-59e5-4bf5-86be-5656bcb4de73", "result_sha256": "d7bbff0fbb602280b45a69576a56e4ba415d5e7f5b9f5fd4e2c809a351476c16", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0157175, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "A" }, "group_id": "distribution-end-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-end-light-0.png", "input_tokens": 2648, "key_errors": null, "latency_sec": 6.891848739003763, "model_name": "pareto", "output_tokens": 1213, "prediction": { "choice": "A" }, "prompt_sha256": "17648faa8c38d72326eb56153ca43b16f1a49276e5acb8d5110bb4bb0e9299d1", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-29T14:17:22.352010+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "4ac00e87-6d10-492b-b736-1c1ac43a48b8", "result_sha256": "0bd32856e69d1f074bf757c29ff429c5ee9d379b9f3644498da188f4372bb833", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00896, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "E" }, "group_id": "distribution-end-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-end-light-1.png", "input_tokens": 2648, "key_errors": null, "latency_sec": 3.107987100025639, "model_name": "pareto", "output_tokens": 312, "prediction": { "choice": "E" }, "prompt_sha256": "aebd9c0440974ace931268cc69fd04b3961d3ac9b3895fb8243ac6e96f4cc134", "provider": "anthropic", "raw_prediction": "{\"choice\": \"E\"}", "recorded_at": "2026-09-29T14:17:15.688043+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "b6bdb128-c7c7-4384-85b2-13a66f236e4c", "result_sha256": "a2f9c15ed09b7406e577ac8c9bfc26dbde2f0c4fb326626097b91b114e0dbf37", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00887, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "A" }, "group_id": "distribution-end-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-end-dark-0.png", "input_tokens": 2648, "key_errors": null, "latency_sec": 2.447814923012629, "model_name": "pareto", "output_tokens": 300, "prediction": { "choice": "A" }, "prompt_sha256": "230bd5c48fe9323eb18d8e68c17b30c36e29e314d1401da5a141c036bd28ebb6", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-29T14:16:58.561770+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "ac6480a6-c28a-4042-a6c0-172a799aa18f", "result_sha256": "f420999945c627ba173ddaa230e6e9134cc739d756cbae92527f99470d845b3d", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01343, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "F" }, "group_id": "distribution-end-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-end-dark-1.png", "input_tokens": 2648, "key_errors": null, "latency_sec": 5.653783267131075, "model_name": "pareto", "output_tokens": 908, "prediction": { "choice": "F" }, "prompt_sha256": "ba31519a816a02b7e10e25462cd6a6242eafaf332f7ae72c4e8dc04ba3caa324", "provider": "anthropic", "raw_prediction": "{\"choice\": \"F\"}", "recorded_at": "2026-09-29T14:18:01.260777+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "aae690a5-1945-4089-9c39-377a458cb273", "result_sha256": "81568a0d9d4f4873543f4deb54fb8697d0875f4ce5f4b6661301c8e92dce7a85", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009815, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "E" }, "group_id": "distribution-between-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-between-light-0.png", "input_tokens": 2648, "key_errors": null, "latency_sec": 3.339141472009942, "model_name": "pareto", "output_tokens": 426, "prediction": { "choice": "E" }, "prompt_sha256": "23da23350386069bba58b0668e6b5df5af67fac05943b8826730fffd8ce8e9ef", "provider": "anthropic", "raw_prediction": "{\"choice\": \"E\"}", "recorded_at": "2026-09-29T14:18:04.253783+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-013", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "b5b78109-da54-405b-bac8-c464616dabc5", "result_sha256": "c4ea1124eaa5069a079da8801f2206b79136e08b494b5ea4918b95cb86b834c8", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.011585, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "A" }, "group_id": "distribution-between-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-between-light-1.png", "input_tokens": 2648, "key_errors": null, "latency_sec": 3.7238588260952383, "model_name": "pareto", "output_tokens": 662, "prediction": { "choice": "A" }, "prompt_sha256": "a69521ff42316a2cf6dd875d17ed524c23ebb9d37a359ef46a9898ae2d02aae0", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-29T14:17:25.122323+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-014", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "190d7c65-6044-4ca4-9c6a-f406799324cf", "result_sha256": "e7369c0bd17013472e99e9c526a74217794fc5b02454b5db696d30c0e113205b", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0097175, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "C" }, "group_id": "distribution-between-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-between-dark-0.png", "input_tokens": 2648, "key_errors": null, "latency_sec": 3.426031371811405, "model_name": "pareto", "output_tokens": 413, "prediction": { "choice": "C" }, "prompt_sha256": "2844fd3c7e35599c196ac5243ab693c9f28551ca0f691e83a1b632f51c83444b", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-29T14:18:02.561030+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-015", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "68697252-6180-46cd-a044-da5820df1cce", "result_sha256": "91f71bd08900e5db5d0da6ce2dac0b5fe7ceaf09e7484458bd5d88c8cfacfed4", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.013145, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "C" }, "group_id": "distribution-between-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-between-dark-1.png", "input_tokens": 2648, "key_errors": null, "latency_sec": 4.8105664409231395, "model_name": "pareto", "output_tokens": 870, "prediction": { "choice": "C" }, "prompt_sha256": "18d8a135f2c101591d404cbfe01f629ac78319d2093678f977ecd0929055dab8", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-29T14:18:03.008670+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-016", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "883f5f17-7352-44fd-88ef-9468bc356400", "result_sha256": "15e737bb9adf65a102fc21c3df9e26aee370071b9af7d9159d834a7668e3b211", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0089525, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "F" }, "group_id": "distribution-around-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-around-light-0.png", "input_tokens": 2648, "key_errors": null, "latency_sec": 2.3350224760361016, "model_name": "pareto", "output_tokens": 311, "prediction": { "choice": "F" }, "prompt_sha256": "bbb61c606caa02f5696f7942e1fe764803a2a2fa41fb5bea98e450a25158c219", "provider": "anthropic", "raw_prediction": "{\"choice\": \"F\"}", "recorded_at": "2026-09-29T14:17:47.314358+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-017", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "eff9fb9d-f79e-4070-b3ee-153b59ef31ff", "result_sha256": "a57e9785e3b30890db41a2a1c1af33689168e9948f1f842970caecdecfb5f0a3", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.0441575, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "B" }, "group_id": "distribution-around-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-around-light-1.png", "input_tokens": 5354, "key_errors": null, "latency_sec": 21.95545280002989, "model_name": "pareto", "output_tokens": 4103, "prediction": { "choice": "A" }, "prompt_sha256": "964044cc4630a4303ac98062eed5f27cbfd8d60ca2ff55cefa53ce651a6151e1", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-29T14:17:42.084476+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-distribution-018", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "909499a8-faf6-4267-8eb4-0d9aa4ed4cb4", "result_sha256": "1ffda28eaedcf88a125d09d40c808b0dbc343d95aeb1c737e571a3cdf6d6203c", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0103325, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "D" }, "group_id": "distribution-around-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-around-dark-0.png", "input_tokens": 2648, "key_errors": null, "latency_sec": 3.7252997681498528, "model_name": "pareto", "output_tokens": 495, "prediction": { "choice": "D" }, "prompt_sha256": "4dc80907c36a071b7b8d24fb1b232e7423f18ee5163be9b22e0a35c4d658cc94", "provider": "anthropic", "raw_prediction": "{\"choice\": \"D\"}", "recorded_at": "2026-09-29T14:17:21.805888+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-019", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "484a998d-8d0c-40fe-a48e-525640a48f3c", "result_sha256": "d186d1e9635cfa9930790fa2103e53c5044a76939b7064e0648486c784d06e84", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01049, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "F" }, "group_id": "distribution-around-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-around-dark-1.png", "input_tokens": 2648, "key_errors": null, "latency_sec": 3.958048290805891, "model_name": "pareto", "output_tokens": 516, "prediction": { "choice": "F" }, "prompt_sha256": "4dcc7229487e3d355fc6f147248270e76223b590fa32a706ea21638c7421eced", "provider": "anthropic", "raw_prediction": "{\"choice\": \"F\"}", "recorded_at": "2026-09-29T14:17:39.331721+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-020", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "7106edb9-1f8d-4311-bbac-ca063cd8b6f1", "result_sha256": "68d7e3cb7363fd4d61805bcf52bd114ca67136f6d21a98f21fd6ff86cf46b244", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.008405, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "B" }, "group_id": "distribution-evenly-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-evenly-light-0.png", "input_tokens": 2648, "key_errors": null, "latency_sec": 2.694969051051885, "model_name": "pareto", "output_tokens": 238, "prediction": { "choice": "B" }, "prompt_sha256": "bb6a0458622253c31a96dd71203dfe1750682c555db8bd324eda28766af2ea8d", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-29T14:17:53.684180+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-021", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "d29e1164-5c50-4b93-af1a-72b126939db9", "result_sha256": "9933c70f10014671ed3023af43616df2c5c37cf8240661fb005bc59a48a233bf", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0198725, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "C" }, "group_id": "distribution-evenly-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-evenly-light-1.png", "input_tokens": 2648, "key_errors": null, "latency_sec": 9.195097668096423, "model_name": "pareto", "output_tokens": 1767, "prediction": { "choice": "C" }, "prompt_sha256": "40b9c4de265a2cf5726c7450b659e9a657270b652500eb7da5fcf557c0fe9c72", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-29T14:16:58.097585+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-022", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "87f311da-2424-416b-bc86-eed8844d7ee1", "result_sha256": "84b8c4f3e8bf43507d1c823b939d5d348d61aa52e1c941eac0dc5fc1e3a1872f", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.008705, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "A" }, "group_id": "distribution-evenly-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-evenly-dark-0.png", "input_tokens": 2648, "key_errors": null, "latency_sec": 2.261234342120588, "model_name": "pareto", "output_tokens": 278, "prediction": { "choice": "A" }, "prompt_sha256": "d8a96a07c7be49c0aea175aadd7302fe6c7f6601952712884e24cd44b6fec66a", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-29T14:17:07.001644+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-023", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "eda39cc8-3e81-4139-93c4-bc0ae918da8e", "result_sha256": "8f5d7914a66caacde5358943904636fda276d7e8a60ae17a14669d6524acbe81", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009545, "err": null, "error": null, "error_kind": null, "family": "distribution", "ground_truth": { "choice": "C" }, "group_id": "distribution-evenly-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/distribution-evenly-dark-1.png", "input_tokens": 2648, "key_errors": null, "latency_sec": 3.7027019588276744, "model_name": "pareto", "output_tokens": 390, "prediction": { "choice": "C" }, "prompt_sha256": "01948a6a96d60ae4f02c8775cea12be647823408f239745e622e7c22d625ffc8", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-29T14:17:17.987500+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-distribution-024", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "4f176ef1-7314-485e-8645-b3a29053377f", "result_sha256": "3584c70e80117d9e84c03d97cbeba5669e2942092e54e15ef318968bd27e818d", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.010295, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "C" }, "group_id": "gapcompare-first-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-first-light-0.png", "input_tokens": 2567, "key_errors": null, "latency_sec": 3.0154006741940975, "model_name": "pareto", "output_tokens": 517, "prediction": { "choice": "C" }, "prompt_sha256": "2aacf3dbef42dbefa25b359df3a8af3c1541ab2712c25e9b3c3f5cf8371b79e4", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-29T14:17:39.050265+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "c5d1765a-98e4-4cb7-9868-bc35e3326f0f", "result_sha256": "447886e992fa2c35f0290365e97725a5d6b08a4d6a2b74aa9a24c28f541d185f", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0110375, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "B" }, "group_id": "gapcompare-first-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-first-light-1.png", "input_tokens": 2567, "key_errors": null, "latency_sec": 4.270517039112747, "model_name": "pareto", "output_tokens": 616, "prediction": { "choice": "B" }, "prompt_sha256": "bb41f3da30db1063257895b1a55fa7d2a40b0d638c3bd0111db12e799eb0d481", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-29T14:17:54.961984+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "8f2cd19b-20e0-4c50-b202-72dfa30adc0d", "result_sha256": "b7e0033033e827ff154f35d48178c585a6f38005064a3eef33610784773ea048", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0074675, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "B" }, "group_id": "gapcompare-first-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-first-dark-0.png", "input_tokens": 2567, "key_errors": null, "latency_sec": 1.8522727340459824, "model_name": "pareto", "output_tokens": 140, "prediction": { "choice": "B" }, "prompt_sha256": "bb41f3da30db1063257895b1a55fa7d2a40b0d638c3bd0111db12e799eb0d481", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-29T14:17:55.575807+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "5cdc70b9-b72e-4f73-8768-4bbec8024e3f", "result_sha256": "607fa839d6141a732b5d6baa7aaaba6ddb8c2a5e45dc40934d12d1b4749d06df", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.008375, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "A" }, "group_id": "gapcompare-first-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-first-dark-1.png", "input_tokens": 2567, "key_errors": null, "latency_sec": 2.9411765050608665, "model_name": "pareto", "output_tokens": 261, "prediction": { "choice": "A" }, "prompt_sha256": "ae03239d9758e90707007a297d168c1b204598f40d5440779fcfa12bd3cdb399", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-29T14:17:04.726710+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "5fdef264-2a13-4bfb-b50b-63d684bad01d", "result_sha256": "0fa15530dd69894204bec789dba70608e9284e2f22be9e63c419971fbd1cf623", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.010805, "err": null, "error": "Expecting value: line 1 column 1 (char 0)", "error_kind": "invalid_response", "family": "gapcompare", "ground_truth": { "choice": "A" }, "group_id": "gapcompare-last-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-last-light-0.png", "input_tokens": 2567, "key_errors": null, "latency_sec": 3.720825847936794, "model_name": "pareto", "output_tokens": 585, "prediction": {}, "prompt_sha256": "ab1cd666bf62495c439cacec79574bb2f8ac552c98f8627324d3801b52d6d81b", "provider": "anthropic", "raw_prediction": "A", "recorded_at": "2026-09-29T14:17:18.489528+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-gapcompare-005", "tight_score": null, "unmetered_attempts": 0, "valid": false, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "94022e64-81b8-4f15-82e2-b17f8a1fbdf7", "result_sha256": "69d7afc1fbe590fcac2290786d489e71f7692de8e1cf8635bbc1f3fd5acc69a1", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0074675, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "C" }, "group_id": "gapcompare-last-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-last-light-1.png", "input_tokens": 2567, "key_errors": null, "latency_sec": 1.9044157040771097, "model_name": "pareto", "output_tokens": 140, "prediction": { "choice": "C" }, "prompt_sha256": "bb41f3da30db1063257895b1a55fa7d2a40b0d638c3bd0111db12e799eb0d481", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-29T14:17:09.224657+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "677ebef5-ff59-4262-9764-e2df9ae43f10", "result_sha256": "d0942c4134a10c5e52c9a982f8e8a322eaf36ca5a0d5766355e86c1f5c92bfad", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0074, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "B" }, "group_id": "gapcompare-last-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-last-dark-0.png", "input_tokens": 2567, "key_errors": null, "latency_sec": 1.7383661018684506, "model_name": "pareto", "output_tokens": 131, "prediction": { "choice": "B" }, "prompt_sha256": "2aacf3dbef42dbefa25b359df3a8af3c1541ab2712c25e9b3c3f5cf8371b79e4", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-29T14:17:42.398327+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "90efea53-7895-4899-8bc9-c53af9754b98", "result_sha256": "7a1e6f05dd8bb7bbdb612807ca4033ab4c8bfc8eec731f0887a576416456a583", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00746, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "C" }, "group_id": "gapcompare-last-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-last-dark-1.png", "input_tokens": 2567, "key_errors": null, "latency_sec": 2.2421872389968485, "model_name": "pareto", "output_tokens": 139, "prediction": { "choice": "C" }, "prompt_sha256": "7c67aaf8a81903fb008ead49ab0c337d815dd8e5b1f984eb1c2d9e3cddf9fd65", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-29T14:17:11.534418+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "2cc54899-f663-414d-9244-0d47964f03db", "result_sha256": "724713117bbd630e27b228dc487e7bfa6cd0907ab19fa2e6c3e6c91842b91c61", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.007655, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "A" }, "group_id": "gapcompare-equal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-equal-light-0.png", "input_tokens": 2567, "key_errors": null, "latency_sec": 1.7814098601229489, "model_name": "pareto", "output_tokens": 165, "prediction": { "choice": "A" }, "prompt_sha256": "2aacf3dbef42dbefa25b359df3a8af3c1541ab2712c25e9b3c3f5cf8371b79e4", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-29T14:16:59.225974+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "2386aaf2-4c54-46c4-afab-db898b3bbac3", "result_sha256": "47cc7d7db70ecded6ec929839dd9f5df39341e102f27fa23b539662c1effd498", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0088025, "err": null, "error": null, "error_kind": null, "family": "gapcompare", "ground_truth": { "choice": "A" }, "group_id": "gapcompare-equal-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-equal-light-1.png", "input_tokens": 2567, "key_errors": null, "latency_sec": 2.3181441437918693, "model_name": "pareto", "output_tokens": 318, "prediction": { "choice": "A" }, "prompt_sha256": "2aacf3dbef42dbefa25b359df3a8af3c1541ab2712c25e9b3c3f5cf8371b79e4", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-29T14:17:31.944578+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gapcompare-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "83c5f2e2-6822-40d6-b115-6634414cd7c4", "result_sha256": "1149bb3bf04fd6f076154e815227fec3805291b0d4d94809e20983844295ccdd", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.00842, "err": null, "error": "Expecting value: line 1 column 1 (char 0)", "error_kind": "invalid_response", "family": "gapcompare", "ground_truth": { "choice": "B" }, "group_id": "gapcompare-equal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-equal-dark-0.png", "input_tokens": 2567, "key_errors": null, "latency_sec": 2.651479404885322, "model_name": "pareto", "output_tokens": 267, "prediction": {}, "prompt_sha256": "ab1cd666bf62495c439cacec79574bb2f8ac552c98f8627324d3801b52d6d81b", "provider": "anthropic", "raw_prediction": "B", "recorded_at": "2026-09-29T14:17:38.869294+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-gapcompare-011", "tight_score": null, "unmetered_attempts": 0, "valid": false, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "2f8d0219-b1cb-4366-a45a-acf782520335", "result_sha256": "ac7788ba8cc7c3dfb9e78dc02eb0e82e0bc8ed7d0e4711554e5dbdcd0c1463ed", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.00755, "err": null, "error": "Expecting value: line 1 column 1 (char 0)", "error_kind": "invalid_response", "family": "gapcompare", "ground_truth": { "choice": "C" }, "group_id": "gapcompare-equal-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gapcompare-equal-dark-1.png", "input_tokens": 2567, "key_errors": null, "latency_sec": 2.1686771949753165, "model_name": "pareto", "output_tokens": 151, "prediction": {}, "prompt_sha256": "e88ed59689802f6ceda5c2ff1f4a55b2556783dabe7507fad3d3173dad609864", "provider": "anthropic", "raw_prediction": "C", "recorded_at": "2026-09-29T14:18:03.771393+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-gapcompare-012", "tight_score": null, "unmetered_attempts": 0, "valid": false, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "c41b02e7-5bec-4ba5-b9aa-a4899622a48c", "result_sha256": "259e4052f53e83dadefaa9c64b9e387756a3a0a7f9f0bf97322bb49af7504e3b", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0068225, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "A" }, "group_id": "gridcols-2-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-2-light-0.png", "input_tokens": 2549, "key_errors": null, "latency_sec": 1.6934874539729208, "model_name": "pareto", "output_tokens": 60, "prediction": { "choice": "A" }, "prompt_sha256": "baf29085a3966715a493e43b25eb411faa9ffe2033ec17653423007441d24033", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-29T14:17:13.559852+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "2860f392-1e3f-4b63-95b1-99afd2bcad1f", "result_sha256": "2146c93dbe22771269731bb1ca7964e2e4ab98455e48567120ee5b7c93be1c44", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0070175, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "B" }, "group_id": "gridcols-2-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-2-light-1.png", "input_tokens": 2549, "key_errors": null, "latency_sec": 2.162986687151715, "model_name": "pareto", "output_tokens": 86, "prediction": { "choice": "B" }, "prompt_sha256": "0853a31c9fbb6301b3f371c91baacf09c13f4a5be3b3e7a0613b6b02d27a8b19", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-29T14:17:46.098794+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "a747d1ff-b9d9-4944-98ed-8b82bfaf9a1c", "result_sha256": "8c570cb856f40a78d089e391b1ac7f595ebe1f6bdeb8f85d815ea405982f44c2", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00683, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "B" }, "group_id": "gridcols-2-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-2-dark-0.png", "input_tokens": 2549, "key_errors": null, "latency_sec": 1.904915687860921, "model_name": "pareto", "output_tokens": 61, "prediction": { "choice": "B" }, "prompt_sha256": "4e97151663d1e3e37046a23842abcce2e96ecf3d2204566c717e9014b170f248", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-29T14:17:48.510289+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "f5735611-1287-4ed0-9651-8b5c937cda25", "result_sha256": "00c16cf8d7ed120b84303d914638c1e7bab97d726a6fde9e48f69c4946aa81d3", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0068075, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "C" }, "group_id": "gridcols-2-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-2-dark-1.png", "input_tokens": 2549, "key_errors": null, "latency_sec": 2.014193987008184, "model_name": "pareto", "output_tokens": 58, "prediction": { "choice": "C" }, "prompt_sha256": "69a0dbb7b95e4d65f59a6c7934d34ff8a7253b73dbb6ab0dfaf6ce730257753e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-29T14:18:01.568809+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "2c2fec34-3218-4562-8d62-9a43ed8485fd", "result_sha256": "c0fca94a76160664613ea0b4d65770e3efaf1498c7fa3a7ca9c8278a0d9f7625", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006875, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "A" }, "group_id": "gridcols-3-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-light-0.png", "input_tokens": 2549, "key_errors": null, "latency_sec": 1.871909359935671, "model_name": "pareto", "output_tokens": 67, "prediction": { "choice": "A" }, "prompt_sha256": "f1530d19c11e2af386da59f0ec25ff3124022397de4c3b03a6bc80ea51e1422e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-29T14:17:14.973763+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "261abb7c-019f-423e-8ff0-b6dcb7656f20", "result_sha256": "f5d62e3fbdc4fb987118fd7d3ed213b0ff6d2b076b90ddee5d720e65f49ae26c", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006815, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "B" }, "group_id": "gridcols-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-light-1.png", "input_tokens": 2549, "key_errors": null, "latency_sec": 1.5730073500890285, "model_name": "pareto", "output_tokens": 59, "prediction": { "choice": "B" }, "prompt_sha256": "baf29085a3966715a493e43b25eb411faa9ffe2033ec17653423007441d24033", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-29T14:17:49.203680+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "0091aece-9ee4-4ad2-bc2a-983a6cb78bea", "result_sha256": "a16514fe25b711f77fbf9246c78e0b1067fa9d7ca4707673196827dbbc6b1fd9", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0069275, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "C" }, "group_id": "gridcols-3-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-dark-0.png", "input_tokens": 2549, "key_errors": null, "latency_sec": 2.21444767084904, "model_name": "pareto", "output_tokens": 74, "prediction": { "choice": "C" }, "prompt_sha256": "4e97151663d1e3e37046a23842abcce2e96ecf3d2204566c717e9014b170f248", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-29T14:16:51.114237+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "f523a694-1718-4d96-9131-598a26201e1c", "result_sha256": "065e22202f84057690b91b30009bb543be09b6f4765fc005bd87fc5fee7cf02f", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0067025, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "A" }, "group_id": "gridcols-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-dark-1.png", "input_tokens": 2549, "key_errors": null, "latency_sec": 1.4715389341581613, "model_name": "pareto", "output_tokens": 44, "prediction": { "choice": "A" }, "prompt_sha256": "f1530d19c11e2af386da59f0ec25ff3124022397de4c3b03a6bc80ea51e1422e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-29T14:17:19.532943+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "f978068a-2e6b-4665-8782-07213be54b7e", "result_sha256": "b0c543786f17a9c4cebaace522c251a67b2d0eb64f3b7a281679350e11e0af39", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0067775, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "C" }, "group_id": "gridcols-4-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-4-light-0.png", "input_tokens": 2549, "key_errors": null, "latency_sec": 2.3828110119793564, "model_name": "pareto", "output_tokens": 54, "prediction": { "choice": "C" }, "prompt_sha256": "baf29085a3966715a493e43b25eb411faa9ffe2033ec17653423007441d24033", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-29T14:17:07.297834+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "092b52f8-498e-41e5-9b64-65fb83c0fddc", "result_sha256": "6be7751c346bf7df94a194b7cb35c4c36df11d0618dbe994fbe87be1efdeae1e", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0069275, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "A" }, "group_id": "gridcols-4-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-4-light-1.png", "input_tokens": 2549, "key_errors": null, "latency_sec": 2.1292437959928066, "model_name": "pareto", "output_tokens": 74, "prediction": { "choice": "A" }, "prompt_sha256": "69a0dbb7b95e4d65f59a6c7934d34ff8a7253b73dbb6ab0dfaf6ce730257753e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-29T14:17:51.362460+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "afe54140-50c3-4122-938a-6e969e04ec1f", "result_sha256": "4115a6247dd5eb60ab6831b4c4a1536c5cf7c2356555364254330a284020f340", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0068975, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "B" }, "group_id": "gridcols-4-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-4-dark-0.png", "input_tokens": 2549, "key_errors": null, "latency_sec": 1.6350384878460318, "model_name": "pareto", "output_tokens": 70, "prediction": { "choice": "B" }, "prompt_sha256": "f456b84b6ca321b64c95c2bb2cfe766f21207eb4901be3ddea7233116bb62356", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-29T14:17:17.200355+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "04184763-3cd8-4c5e-96e3-dc969bbc08a1", "result_sha256": "082607ae12838498c82556ad6ccb1f62454c0c45083f7e6675b9c2208be21388", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00668, "err": null, "error": null, "error_kind": null, "family": "gridcols", "ground_truth": { "choice": "C" }, "group_id": "gridcols-4-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-4-dark-1.png", "input_tokens": 2549, "key_errors": null, "latency_sec": 1.7810945990495384, "model_name": "pareto", "output_tokens": 41, "prediction": { "choice": "C" }, "prompt_sha256": "baf29085a3966715a493e43b25eb411faa9ffe2033ec17653423007441d24033", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-29T14:17:55.122983+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridcols-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "c7d7a8ec-5dc9-43c9-ace3-8341997802a7", "result_sha256": "076e318d079ade4a32e07583519df35ac2cb7d672e4b322a141cc2dfb22f8f2f", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009035, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "C" }, "group_id": "gridgaps-horizontal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-horizontal-light-0.png", "input_tokens": 2567, "key_errors": null, "latency_sec": 2.869866468012333, "model_name": "pareto", "output_tokens": 349, "prediction": { "choice": "C" }, "prompt_sha256": "9bd6b2738f2f8c2f8f4f1ac19c1985fb2970e8e7972ea985cb663167fcc029e4", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-29T14:16:59.769046+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "8a79a1f5-d62c-4e1c-8485-1ccf009f635e", "result_sha256": "0b4b0299f94110f327ff8b79ca121360eb0fe1921282597e2d590af4cc0d845b", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00818, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "B" }, "group_id": "gridgaps-horizontal-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-horizontal-light-1.png", "input_tokens": 2567, "key_errors": null, "latency_sec": 2.3823683608788997, "model_name": "pareto", "output_tokens": 235, "prediction": { "choice": "B" }, "prompt_sha256": "bb599c345aa6aaf2e8a00a558db18c18ea46033edf36cd1b879c2785b58dce8b", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-29T14:17:56.533877+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "addf450f-34df-4071-ad3e-959cfecfc7ef", "result_sha256": "25a7f86ada8a9ddcdf1c26570824e27da0e245f22189b5486b6e175def30b8c1", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00773, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "B" }, "group_id": "gridgaps-horizontal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-horizontal-dark-0.png", "input_tokens": 2567, "key_errors": null, "latency_sec": 2.202348052058369, "model_name": "pareto", "output_tokens": 175, "prediction": { "choice": "B" }, "prompt_sha256": "bb599c345aa6aaf2e8a00a558db18c18ea46033edf36cd1b879c2785b58dce8b", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-29T14:17:21.755008+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "0dd9b4ea-8a80-485b-9c43-c7ac331b3eea", "result_sha256": "b26fd14fdfb87720949224fa726c2f13c87c1562fde4bf4a9c109e1b2261488e", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.008795, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "C" }, "group_id": "gridgaps-horizontal-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-horizontal-dark-1.png", "input_tokens": 2567, "key_errors": null, "latency_sec": 3.1083950740285218, "model_name": "pareto", "output_tokens": 317, "prediction": { "choice": "C" }, "prompt_sha256": "6fc908b851f777215935a481f34941f8c8db739f40ec24d39d8b889fd6975989", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-29T14:17:04.300457+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "f561b614-eea0-4707-9926-50e2d2e18d33", "result_sha256": "21271a9381dff80f69af9b64e299a2eb88b7e93b6bd3443da87026fb4d2cbdac", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00785, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "A" }, "group_id": "gridgaps-vertical-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-vertical-light-0.png", "input_tokens": 2567, "key_errors": null, "latency_sec": 1.919954146957025, "model_name": "pareto", "output_tokens": 191, "prediction": { "choice": "A" }, "prompt_sha256": "da1f2cca453f507bde83a2ac6d19b503fd7ab1c709bd4a77ee1046cf28ba5c59", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-29T14:16:53.043183+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "18ae31d3-2e24-43c0-b01d-726de6e8e087", "result_sha256": "e1467ed29db8dbebb290f093a79c33e0a45bb7ae6184dc0e703b1862bc4317cf", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0081125, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "C" }, "group_id": "gridgaps-vertical-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-vertical-light-1.png", "input_tokens": 2567, "key_errors": null, "latency_sec": 2.270721456967294, "model_name": "pareto", "output_tokens": 226, "prediction": { "choice": "C" }, "prompt_sha256": "e313bf537665702a78ba5d1571e6c2f2fbe0c9caf46ed390c4d20e71f215f3d1", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-29T14:17:15.256786+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "1e5a95f7-d78f-44f7-baae-91374d4eadc6", "result_sha256": "d58f152f39492d879c30b4719d29b7473abb243c3b391d8785f26be54a1fe184", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0088325, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "B" }, "group_id": "gridgaps-vertical-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-vertical-dark-0.png", "input_tokens": 2567, "key_errors": null, "latency_sec": 2.699047068832442, "model_name": "pareto", "output_tokens": 322, "prediction": { "choice": "B" }, "prompt_sha256": "9bd6b2738f2f8c2f8f4f1ac19c1985fb2970e8e7972ea985cb663167fcc029e4", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-29T14:16:54.841430+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "2a353abc-fc2a-4a31-ab6b-901f0dbc0865", "result_sha256": "83d7a5037687dda301d7bbc57470247439fae370b454e6cf9eca038fc33b9a21", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0082175, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "A" }, "group_id": "gridgaps-vertical-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-vertical-dark-1.png", "input_tokens": 2567, "key_errors": null, "latency_sec": 3.3943443831522018, "model_name": "pareto", "output_tokens": 240, "prediction": { "choice": "A" }, "prompt_sha256": "da1f2cca453f507bde83a2ac6d19b503fd7ab1c709bd4a77ee1046cf28ba5c59", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-29T14:17:36.011399+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridgaps-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "12607f01-8393-40ba-a695-bc0392dc2795", "result_sha256": "8c362c03482f02d52f7391e7b002ca4d243f37fd57e285faa709c0ff23d5cec6", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.0101825, "err": null, "error": "Expecting value: line 1 column 1 (char 0)", "error_kind": "invalid_response", "family": "gridgaps", "ground_truth": { "choice": "A" }, "group_id": "gridgaps-equal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-equal-light-0.png", "input_tokens": 2567, "key_errors": null, "latency_sec": 3.483148311963305, "model_name": "pareto", "output_tokens": 502, "prediction": {}, "prompt_sha256": "9bd6b2738f2f8c2f8f4f1ac19c1985fb2970e8e7972ea985cb663167fcc029e4", "provider": "anthropic", "raw_prediction": "A", "recorded_at": "2026-09-29T14:17:02.056334+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-gridgaps-009", "tight_score": null, "unmetered_attempts": 0, "valid": false, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "4f1ffc2f-3302-4b41-8b8a-4ce566e468bd", "result_sha256": "89dafc2ca912cce680c024bc32cb6a11d27d6b38db8f512734649c223ff0aa54", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.0437525, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "A" }, "group_id": "gridcols-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-light-1.png", "input_tokens": 5192, "key_errors": null, "latency_sec": 17.980661551002413, "model_name": "pareto", "output_tokens": 4103, "prediction": { "choice": "B" }, "prompt_sha256": "bb599c345aa6aaf2e8a00a558db18c18ea46033edf36cd1b879c2785b58dce8b", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-29T14:17:30.496232+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-gridgaps-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "678cf65a-4664-4de4-a59e-c9b625f801ca", "result_sha256": "ff5df830df5d0f1feaa097106c36fc8bbfcb00134f4985b0fde81ee332e13082", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.020945, "err": null, "error": null, "error_kind": null, "family": "gridgaps", "ground_truth": { "choice": "B" }, "group_id": "gridgaps-equal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-equal-dark-0.png", "input_tokens": 2567, "key_errors": null, "latency_sec": 9.572733517037705, "model_name": "pareto", "output_tokens": 1937, "prediction": { "choice": "C" }, "prompt_sha256": "e313bf537665702a78ba5d1571e6c2f2fbe0c9caf46ed390c4d20e71f215f3d1", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-29T14:17:31.641538+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-gridgaps-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "516abbde-6e0f-4642-82e4-01d845ea1b23", "result_sha256": "06eba78368135fdfb0d7fac74018ab5173e192f95e731fd0df133a6fcafa4e29", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.0097325, "err": null, "error": "Expecting value: line 1 column 1 (char 0)", "error_kind": "invalid_response", "family": "gridgaps", "ground_truth": { "choice": "C" }, "group_id": "gridcols-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-dark-1.png", "input_tokens": 2567, "key_errors": null, "latency_sec": 2.967900612158701, "model_name": "pareto", "output_tokens": 442, "prediction": {}, "prompt_sha256": "b680534db587454f307f9837d6da890e5af0576bf084907b76dd89e0815cc97c", "provider": "anthropic", "raw_prediction": "C", "recorded_at": "2026-09-29T14:17:31.117192+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-gridgaps-012", "tight_score": null, "unmetered_attempts": 0, "valid": false, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "ea03fc8a-2c1b-43df-8b5c-b5f454ba6375", "result_sha256": "772c0d00590a00233e37d5b88e88b5cc1770946e57162823539f294a4702127a", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0068225, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "A" }, "group_id": "gridcols-2-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-2-light-0.png", "input_tokens": 2549, "key_errors": null, "latency_sec": 1.5042487250175327, "model_name": "pareto", "output_tokens": 60, "prediction": { "choice": "A" }, "prompt_sha256": "0fad5d0a9261a17e68aa8de2ef5fb0207dc8af585c3baaeabfc584af8b5329f5", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-29T14:17:08.975540+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "61f764d0-346d-45d3-9e05-896ac8a8532e", "result_sha256": "4f8d6066910b14cf843be80161769271563bea014f1e1ade04dca3775b3459ab", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006725, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "B" }, "group_id": "gridrows-2-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-2-light-1.png", "input_tokens": 2549, "key_errors": null, "latency_sec": 1.4601874460931867, "model_name": "pareto", "output_tokens": 47, "prediction": { "choice": "B" }, "prompt_sha256": "fdee2ada4386cbfdbf34a82a0b96ff2b416af121ed3440edf26b052476c9811a", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-29T14:17:09.275521+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "2f084d23-49d7-4ba0-a4bb-49d54164d0e1", "result_sha256": "ea8d0858ce41bf8a8cf320e5f00803f00f71345f9f94c72e1b506856e3366330", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0067625, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "A" }, "group_id": "gridcols-2-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-2-dark-0.png", "input_tokens": 2549, "key_errors": null, "latency_sec": 1.975827471818775, "model_name": "pareto", "output_tokens": 52, "prediction": { "choice": "A" }, "prompt_sha256": "0fad5d0a9261a17e68aa8de2ef5fb0207dc8af585c3baaeabfc584af8b5329f5", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-29T14:17:44.856330+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "22605bca-84db-4591-899c-d3cefe3c8cfb", "result_sha256": "9214914a234ae2bef391ebbe9f817b09fb19cb4c565eb2bf033f2effc6f851fb", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0068, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "C" }, "group_id": "gridrows-2-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-2-dark-1.png", "input_tokens": 2549, "key_errors": null, "latency_sec": 1.6707531670108438, "model_name": "pareto", "output_tokens": 57, "prediction": { "choice": "C" }, "prompt_sha256": "bc0885c5d8e75e638e58b055b8a0c362e439458552c2f596d459edf6097e3ea3", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-29T14:17:08.247968+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "496d3916-16f5-45be-a216-f5f6f347649b", "result_sha256": "abbc215d3785ee057c78a0d8ccea6fd2e2cd3e728d8413dfaeb94e2da52fac57", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00671, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "C" }, "group_id": "gridrows-3-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-3-light-0.png", "input_tokens": 2549, "key_errors": null, "latency_sec": 1.6556845589075238, "model_name": "pareto", "output_tokens": 45, "prediction": { "choice": "C" }, "prompt_sha256": "da4194e14f8d3c86eb007c251e2f977baeaf16f3ed67610f233d5aaa1e16745d", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-29T14:17:38.640541+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "ee004b1f-8ad7-4cc0-9ba5-d5e248e80fe6", "result_sha256": "dd53587cc18b110c0f0838e58e04a85fa9f3eae020f86543dfe0f3b1475e7dd9", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0067925, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "C" }, "group_id": "gridcols-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-light-1.png", "input_tokens": 2549, "key_errors": null, "latency_sec": 1.855449269991368, "model_name": "pareto", "output_tokens": 56, "prediction": { "choice": "C" }, "prompt_sha256": "da4194e14f8d3c86eb007c251e2f977baeaf16f3ed67610f233d5aaa1e16745d", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-29T14:16:57.434750+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "f9135343-95ed-4ae6-ad0e-0c603ed027ca", "result_sha256": "7580ffa51cc7d65946e9fcb0e4e54394a915c46ae05d00182bc7d63538309d68", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0067025, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "B" }, "group_id": "gridrows-3-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-3-dark-0.png", "input_tokens": 2549, "key_errors": null, "latency_sec": 2.217542486032471, "model_name": "pareto", "output_tokens": 44, "prediction": { "choice": "B" }, "prompt_sha256": "0fad5d0a9261a17e68aa8de2ef5fb0207dc8af585c3baaeabfc584af8b5329f5", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-29T14:16:51.108694+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "1d0dfea6-95ec-451b-a2a7-d0177f141992", "result_sha256": "da1f59b0ac21198b919c2a22c0db3a02f5ee20356ba5ea72d987e6be87289667", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006725, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "C" }, "group_id": "gridcols-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-dark-1.png", "input_tokens": 2549, "key_errors": null, "latency_sec": 1.8197080141399056, "model_name": "pareto", "output_tokens": 47, "prediction": { "choice": "C" }, "prompt_sha256": "da4194e14f8d3c86eb007c251e2f977baeaf16f3ed67610f233d5aaa1e16745d", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-29T14:17:13.083223+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "59360efa-07bd-4266-9e93-0c614fc84a42", "result_sha256": "4412e67592b505010c2e82c8837b6115fa9fe60b760ee5857e7367d2a603d119", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006905, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "B" }, "group_id": "gridrows-4-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-4-light-0.png", "input_tokens": 2549, "key_errors": null, "latency_sec": 1.5211307471618056, "model_name": "pareto", "output_tokens": 71, "prediction": { "choice": "B" }, "prompt_sha256": "da4194e14f8d3c86eb007c251e2f977baeaf16f3ed67610f233d5aaa1e16745d", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-29T14:17:58.927614+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "12a878e3-69fb-4d96-b2d2-8dd40979f1bc", "result_sha256": "e4ce71aed880e88b65eebd1154bddcc923bf58aad56ce5a92d4c0899a41051b4", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00713, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "B" }, "group_id": "gridrows-4-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-4-light-1.png", "input_tokens": 2549, "key_errors": null, "latency_sec": 2.204713589977473, "model_name": "pareto", "output_tokens": 101, "prediction": { "choice": "B" }, "prompt_sha256": "8dbf44e12608a972028adfbb37ba6ec0f4b2e39e4ef1034ec56a4d2fe458ec94", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-29T14:17:10.717025+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "952bf921-2e0e-4372-b3ef-4d54d5ec2a05", "result_sha256": "653b8bbf787f8eb6ac4a23372cf1c99e839a3457f99dc597b3ae6c4329da2a92", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0070025, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "A" }, "group_id": "gridrows-4-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-4-dark-0.png", "input_tokens": 2549, "key_errors": null, "latency_sec": 1.7226293310523033, "model_name": "pareto", "output_tokens": 84, "prediction": { "choice": "A" }, "prompt_sha256": "2c2e1ea513c52f4b927e59a676294b441e8aa0cf98f9851e570c4ad2bd9396af", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-29T14:17:19.708670+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "623aeb4c-20cf-47f1-ab6f-03581b6419fc", "result_sha256": "6307159ae4537754c434beb9e737ae2667798bc1a73c32193e554d5d89e1d8e8", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0074375, "err": null, "error": null, "error_kind": null, "family": "gridrows", "ground_truth": { "choice": "A" }, "group_id": "gridrows-4-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridrows-4-dark-1.png", "input_tokens": 2549, "key_errors": null, "latency_sec": 1.9172324549872428, "model_name": "pareto", "output_tokens": 142, "prediction": { "choice": "A" }, "prompt_sha256": "bc0885c5d8e75e638e58b055b8a0c362e439458552c2f596d459edf6097e3ea3", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-29T14:17:36.457204+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridrows-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "2ef64892-9c50-44ce-9789-286cd319497a", "result_sha256": "b96b19eaa994c665095c12bc7071c66d3868f276a172847789718240272d9070", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0087775, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "B" }, "group_id": "gridspan-1-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-1-light-0.png", "input_tokens": 2572, "key_errors": null, "latency_sec": 2.6471281989943236, "model_name": "pareto", "output_tokens": 313, "prediction": { "choice": "B" }, "prompt_sha256": "036502f601ca514f2b291d9a63cb6d8de5371510c2e08f970e9cb252d2c52ce2", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-29T14:16:54.611947+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "0fd1f9c3-c862-4ad2-987f-e06e46b88f94", "result_sha256": "66924b21746a37126ad14d61573c1e015c119472af6ce03992c64ebe7d041973", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0075625, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "A" }, "group_id": "gridspan-1-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-1-light-1.png", "input_tokens": 2572, "key_errors": null, "latency_sec": 1.9681051161605865, "model_name": "pareto", "output_tokens": 151, "prediction": { "choice": "A" }, "prompt_sha256": "9320fc919a5f33d10a5a97b8269a21a6fc8fa9787daa8025e876c4a4452a6d4a", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-29T14:17:29.605559+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "dbeb29e7-ead6-4522-8a71-ef1f2eb99798", "result_sha256": "d75d412d5982d1c0a7b3aaaddabe5eb73216299c73cd8680c038eb9be1ccb448", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0083425, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "C" }, "group_id": "gridspan-1-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-1-dark-0.png", "input_tokens": 2572, "key_errors": null, "latency_sec": 2.6277199380565435, "model_name": "pareto", "output_tokens": 255, "prediction": { "choice": "C" }, "prompt_sha256": "4c2619c1f3339203e066968a6911df07e41f49d553c6c3cb2be0b1e81e76a29b", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-29T14:16:51.513236+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "2f173f88-19d6-463f-8bbd-2d4a24db2208", "result_sha256": "7f84faccc47b61a18887cdc4b067fbd02f46c3bb8219aea4164f80873140a186", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.008545, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "C" }, "group_id": "gridspan-1-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-1-dark-1.png", "input_tokens": 2572, "key_errors": null, "latency_sec": 3.3536428748629987, "model_name": "pareto", "output_tokens": 282, "prediction": { "choice": "C" }, "prompt_sha256": "4c2619c1f3339203e066968a6911df07e41f49d553c6c3cb2be0b1e81e76a29b", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-29T14:18:05.325735+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "c8c50b5a-bcd4-4cfc-a56e-630c398fb1e4", "result_sha256": "be7c08ea4772ed9909846987da271e4214373e4882a86dc268e65605e144b5fd", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0076375, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "C" }, "group_id": "gridspan-2-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-2-light-0.png", "input_tokens": 2572, "key_errors": null, "latency_sec": 2.249529456021264, "model_name": "pareto", "output_tokens": 161, "prediction": { "choice": "C" }, "prompt_sha256": "036502f601ca514f2b291d9a63cb6d8de5371510c2e08f970e9cb252d2c52ce2", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-29T14:16:56.774699+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "aba5bd06-e7be-4298-9217-ffec2c922cd2", "result_sha256": "2e18198155b8edadd8aab011ed50eac7f078e9f9c349e308871b07c6a11e1b03", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0075325, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "A" }, "group_id": "gridspan-2-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-2-light-1.png", "input_tokens": 2572, "key_errors": null, "latency_sec": 1.9048811018001288, "model_name": "pareto", "output_tokens": 147, "prediction": { "choice": "A" }, "prompt_sha256": "4c2619c1f3339203e066968a6911df07e41f49d553c6c3cb2be0b1e81e76a29b", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-29T14:17:45.655570+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "51332c52-6119-4947-addf-699b9b29005a", "result_sha256": "fcbfcafb05e050f96e5f056e0e1bbc1e39c5488b2c878fb2c0c4d9f75ec9678b", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0078325, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "A" }, "group_id": "gridspan-2-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-2-dark-0.png", "input_tokens": 2572, "key_errors": null, "latency_sec": 2.9438704641070217, "model_name": "pareto", "output_tokens": 187, "prediction": { "choice": "A" }, "prompt_sha256": "4c2619c1f3339203e066968a6911df07e41f49d553c6c3cb2be0b1e81e76a29b", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-29T14:17:52.779867+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "d3a681aa-f399-437d-bdf4-5113cb3df6e8", "result_sha256": "21c7c764e10355e8ccf16dad44fb09d3dea72fbdb5ec94537d2c360522dac422", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00796, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "A" }, "group_id": "gridspan-2-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-2-dark-1.png", "input_tokens": 2572, "key_errors": null, "latency_sec": 2.604833025019616, "model_name": "pareto", "output_tokens": 204, "prediction": { "choice": "A" }, "prompt_sha256": "c03d42c3e0b3f012910b0d501b5da5eff14e458eec8ebf179f14644511f48500", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-29T14:17:36.271979+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "ab2a2860-eb49-43bd-9166-96b57cae07c1", "result_sha256": "7e785a145b2319ad89909e3b19c120dd8567e016f12d7ccaff2ea4a9a44b458b", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.007165, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "B" }, "group_id": "gridspan-3-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-3-light-0.png", "input_tokens": 2572, "key_errors": null, "latency_sec": 1.556035669054836, "model_name": "pareto", "output_tokens": 98, "prediction": { "choice": "B" }, "prompt_sha256": "9320fc919a5f33d10a5a97b8269a21a6fc8fa9787daa8025e876c4a4452a6d4a", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-29T14:16:58.340160+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "13b27c9a-55b3-4bd4-b3f0-c2eab1b8ab37", "result_sha256": "de669e383bfd9fae9f39ecdc824e80188f210979736b6c76f7b5cf44e0eb1f67", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0070525, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "B" }, "group_id": "gridspan-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-3-light-1.png", "input_tokens": 2572, "key_errors": null, "latency_sec": 2.148014223901555, "model_name": "pareto", "output_tokens": 83, "prediction": { "choice": "B" }, "prompt_sha256": "4c2619c1f3339203e066968a6911df07e41f49d553c6c3cb2be0b1e81e76a29b", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-29T14:17:01.657795+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "98e831a0-c31c-4361-a60a-be48044e9543", "result_sha256": "2572920848d59e35256497e7ccb2b009e6f0002153aa961ccee581e2950a9b3b", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00694, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "B" }, "group_id": "gridspan-3-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-3-dark-0.png", "input_tokens": 2572, "key_errors": null, "latency_sec": 1.5883238189853728, "model_name": "pareto", "output_tokens": 68, "prediction": { "choice": "B" }, "prompt_sha256": "4c2619c1f3339203e066968a6911df07e41f49d553c6c3cb2be0b1e81e76a29b", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-29T14:17:14.265646+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "bed63d97-f9ce-475a-b0dc-ac0a5aa2191d", "result_sha256": "3efa2f1ad6c18a1648f6fbdd4bf54960b9f4286ca1566829d1afcdfad43ba788", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00709, "err": null, "error": null, "error_kind": null, "family": "gridspan", "ground_truth": { "choice": "C" }, "group_id": "gridspan-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridspan-3-dark-1.png", "input_tokens": 2572, "key_errors": null, "latency_sec": 1.6518914070911705, "model_name": "pareto", "output_tokens": 88, "prediction": { "choice": "C" }, "prompt_sha256": "8f345bb2bced18424e3b0095ebe76c3e49a3f618e53f0a1cf0fe32cf1f45be8c", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-29T14:17:21.379879+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridspan-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "fc98fd8f-e8c4-43ef-8ff7-3f9991d26ee0", "result_sha256": "6a29aba25e3f5804863f09cb9274e539b45e97bdba6848f37746c40f3987450e", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0077275, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "C" }, "group_id": "gridgaps-equal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-equal-light-0.png", "input_tokens": 2569, "key_errors": null, "latency_sec": 2.4002444290090352, "model_name": "pareto", "output_tokens": 174, "prediction": { "choice": "C" }, "prompt_sha256": "10cd7f5950372ee9cf1d2df732ca333086531870973115e171dc7abfa750b35c", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-29T14:17:15.435223+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "b68764eb-111a-4400-8009-5b64563dde90", "result_sha256": "306969e870abdfe0ade99d90c35119b14aec6669210117afe594ff7b26146482", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0082075, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "B" }, "group_id": "gridcols-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-light-1.png", "input_tokens": 2569, "key_errors": null, "latency_sec": 2.146391484187916, "model_name": "pareto", "output_tokens": 238, "prediction": { "choice": "B" }, "prompt_sha256": "330f46df745e35a409a07fe87ab9184b25be7cbe0c4a58bdce6f2fce80123f3a", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-29T14:17:42.707091+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "ea0254f6-5636-4017-b504-f99170169cef", "result_sha256": "6476c887690c63071120b5875fef16e1804d44774314451c84e1c7fcbc180e2e", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00844, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "C" }, "group_id": "gridgaps-equal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridgaps-equal-dark-0.png", "input_tokens": 2569, "key_errors": null, "latency_sec": 2.1394046710338444, "model_name": "pareto", "output_tokens": 269, "prediction": { "choice": "C" }, "prompt_sha256": "970b359b82046186b75b0e492f6832a2ce3e8a83fea0f87acd3cecf2c795cb79", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-29T14:17:22.331655+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "16d12972-bc82-4877-a9bf-cf10e0691eb0", "result_sha256": "e44d7d652e9bbdb01cac44c1fafa2bc82fb202cab0b7b8246c14219d03c748ea", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00766, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "B" }, "group_id": "gridcols-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridcols-3-dark-1.png", "input_tokens": 2569, "key_errors": null, "latency_sec": 2.091701610945165, "model_name": "pareto", "output_tokens": 165, "prediction": { "choice": "B" }, "prompt_sha256": "e1686611bde92953032219f0663fed17a23573139af7c26937579757a13cc24d", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-29T14:17:01.773239+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "64553059-3b97-4ad5-a88a-13f291a8e621", "result_sha256": "c8c04dafff8c6974084e4f25512b68e4adb2c7f6044f3f4df23f8657b3c26a4c", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0073975, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "B" }, "group_id": "gridtracks-left-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-left-light-0.png", "input_tokens": 2569, "key_errors": null, "latency_sec": 2.0035907689016312, "model_name": "pareto", "output_tokens": 130, "prediction": { "choice": "B" }, "prompt_sha256": "aa2cac59ecf81aa683f25bd6bee52028be6daa4a3ce7c514aca8929ad1d53167", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-29T14:17:33.645398+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "8e1689ac-0352-422a-97f4-0e91b0ae424f", "result_sha256": "75a6f38bcc7bc0298dba7926ca341b7af131ba7731a2a6fae6a30b49dd45da0b", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0075625, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "A" }, "group_id": "gridtracks-left-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-left-light-1.png", "input_tokens": 2569, "key_errors": null, "latency_sec": 1.8880702739115804, "model_name": "pareto", "output_tokens": 152, "prediction": { "choice": "A" }, "prompt_sha256": "10cd7f5950372ee9cf1d2df732ca333086531870973115e171dc7abfa750b35c", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-29T14:17:50.427990+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "e034441b-65e0-41a9-8947-ecdcff87cef5", "result_sha256": "4a6a5ffaa4566e01a07b2e623ffb90773e9dc99ca2102f07a17fce9d40d33d9a", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.007465, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "A" }, "group_id": "gridtracks-left-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-left-dark-0.png", "input_tokens": 2569, "key_errors": null, "latency_sec": 2.3992168370168656, "model_name": "pareto", "output_tokens": 139, "prediction": { "choice": "A" }, "prompt_sha256": "10cd7f5950372ee9cf1d2df732ca333086531870973115e171dc7abfa750b35c", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-29T14:17:57.506258+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "b4912120-defd-4abc-b7eb-3827f5a84efb", "result_sha256": "4ac4b0d4f6bdf0298884469fe64bc28dbbd526e260a33ebc4ca21faaf08c66d2", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0075475, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "A" }, "group_id": "gridtracks-left-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-left-dark-1.png", "input_tokens": 2569, "key_errors": null, "latency_sec": 1.6432427940890193, "model_name": "pareto", "output_tokens": 150, "prediction": { "choice": "A" }, "prompt_sha256": "330f46df745e35a409a07fe87ab9184b25be7cbe0c4a58bdce6f2fce80123f3a", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-29T14:17:10.635563+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "68381e9f-1a3f-4801-883a-9b64dc94441b", "result_sha256": "c186675e1397a64a40fe293317bc2762cf44024849eb2f2526a7d6377401a608", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0072775, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "C" }, "group_id": "gridtracks-right-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-right-light-0.png", "input_tokens": 2569, "key_errors": null, "latency_sec": 2.2905130598228425, "model_name": "pareto", "output_tokens": 114, "prediction": { "choice": "C" }, "prompt_sha256": "aa2cac59ecf81aa683f25bd6bee52028be6daa4a3ce7c514aca8929ad1d53167", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-29T14:18:03.150860+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "94c550df-5ebb-4d93-8e42-38a7f1a907ae", "result_sha256": "1d5d9d00cc430f3e0e2beff035ada7d6d67da5e5ff1d988a75a53d93ad7d9fd6", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0076375, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "B" }, "group_id": "gridtracks-right-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-right-light-1.png", "input_tokens": 2569, "key_errors": null, "latency_sec": 1.7599668740294874, "model_name": "pareto", "output_tokens": 162, "prediction": { "choice": "B" }, "prompt_sha256": "81f56b69372d1aac7eb5a641448a893a03f044c68d3ed320fc7646d8a1c13ed9", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-29T14:16:56.105198+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "a62131fd-b8a4-4386-a062-74288586e87d", "result_sha256": "5d13c8a16884ed58aba25bba5067a7f3d62d92680de28f823692fcd6334a4a16", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0072625, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "A" }, "group_id": "gridtracks-right-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-right-dark-0.png", "input_tokens": 2569, "key_errors": null, "latency_sec": 1.764803681988269, "model_name": "pareto", "output_tokens": 112, "prediction": { "choice": "A" }, "prompt_sha256": "e1686611bde92953032219f0663fed17a23573139af7c26937579757a13cc24d", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-29T14:17:50.334361+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "450ce05d-4050-4f59-b188-c3bd3246b4a9", "result_sha256": "e34227d7f5daf09496fe8771ff1f1c8b1bd273afcba75961db1afac30d93ce0e", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0073975, "err": null, "error": null, "error_kind": null, "family": "gridtracks", "ground_truth": { "choice": "C" }, "group_id": "gridtracks-right-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/gridtracks-right-dark-1.png", "input_tokens": 2569, "key_errors": null, "latency_sec": 2.0346484950277954, "model_name": "pareto", "output_tokens": 130, "prediction": { "choice": "C" }, "prompt_sha256": "330f46df745e35a409a07fe87ab9184b25be7cbe0c4a58bdce6f2fce80123f3a", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-29T14:17:01.170553+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-gridtracks-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "ba14ab27-7690-4eef-a022-6f542d982264", "result_sha256": "2ab0bc4168afa54f5606641c08bdd615d7695134a674b4d893eb18ba47a7e6a3", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.007965, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "B" }, "group_id": "groupgap-between-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-between-light-0.png", "input_tokens": 2583, "key_errors": null, "latency_sec": 2.279297956964001, "model_name": "pareto", "output_tokens": 201, "prediction": { "choice": "B" }, "prompt_sha256": "a4ae0e9e5d9435b02d3759ecf1850c36024f893c4de107ea8256f2c5052eaa53", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-29T14:18:05.871791+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "4228d0b0-02f8-4f07-b2f8-286dc95202b3", "result_sha256": "a1e00eb38c350006b2b703a1d20080f37f7836adb2de93a4e07b08ce484a504c", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0078375, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "B" }, "group_id": "groupgap-between-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-between-light-1.png", "input_tokens": 2583, "key_errors": null, "latency_sec": 2.5406151991337538, "model_name": "pareto", "output_tokens": 184, "prediction": { "choice": "B" }, "prompt_sha256": "b5587c95ce8398dca0e206a8a222a0859cb2781184dde718bbe27409262b14c1", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-29T14:17:59.115491+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "3770d572-ae46-45f9-8855-366754a53100", "result_sha256": "ec71de3ae5ec8e0839df620f9371bf580e209dfb93dacadfdb9001a85c04b722", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0084525, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "B" }, "group_id": "groupgap-between-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-between-dark-0.png", "input_tokens": 2583, "key_errors": null, "latency_sec": 2.073339018970728, "model_name": "pareto", "output_tokens": 266, "prediction": { "choice": "B" }, "prompt_sha256": "b5587c95ce8398dca0e206a8a222a0859cb2781184dde718bbe27409262b14c1", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-29T14:17:38.367356+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "b3ba2e0d-bc1d-4cd2-a1e3-c8a9144ed3f0", "result_sha256": "1c295ae13f5854e1c23694bf98a980ec213ff13cb6af5a54a32cc3dca493df88", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.007845, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "A" }, "group_id": "groupgap-between-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-between-dark-1.png", "input_tokens": 2583, "key_errors": null, "latency_sec": 1.8564705101307482, "model_name": "pareto", "output_tokens": 185, "prediction": { "choice": "A" }, "prompt_sha256": "6cba71084ece6f53e834e34e24c720eaabde46f305bb7ec0454830a5812e840e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-29T14:16:53.357670+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "56ee236f-0886-4d1f-ab23-5ba3d45cac24", "result_sha256": "781dd63a749b87bfebf47d64b2154b7d1b74a347f43063928934911c5dba2bf8", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009345, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "C" }, "group_id": "groupgap-within-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-within-light-0.png", "input_tokens": 2583, "key_errors": null, "latency_sec": 2.4534390489570796, "model_name": "pareto", "output_tokens": 385, "prediction": { "choice": "C" }, "prompt_sha256": "6cba71084ece6f53e834e34e24c720eaabde46f305bb7ec0454830a5812e840e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-29T14:17:27.666512+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "11698333-4896-4799-a8ec-be2228feabc8", "result_sha256": "f68403792730772190ca183237afe67670559071560c3e58adfb2e0f2d340052", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01122, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "A" }, "group_id": "groupgap-within-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-within-light-1.png", "input_tokens": 2583, "key_errors": null, "latency_sec": 4.048435197910294, "model_name": "pareto", "output_tokens": 635, "prediction": { "choice": "A" }, "prompt_sha256": "b5587c95ce8398dca0e206a8a222a0859cb2781184dde718bbe27409262b14c1", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-29T14:17:19.207450+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "5b55249f-1273-462a-9099-874604a6ef0b", "result_sha256": "4a0c25062c3c7e8c4aae414dcc74f9b09136a57df631adef64c3a07a0e7fd7c2", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.009285, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "C" }, "group_id": "groupgap-within-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-within-dark-0.png", "input_tokens": 2583, "key_errors": null, "latency_sec": 2.630145997973159, "model_name": "pareto", "output_tokens": 377, "prediction": { "choice": "C" }, "prompt_sha256": "6cba71084ece6f53e834e34e24c720eaabde46f305bb7ec0454830a5812e840e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-29T14:17:48.068170+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "63799ce5-ffe3-4973-87f0-132050517498", "result_sha256": "7e2df72b0898f924f20f1a6d72e677441ee52e03292c429f88d1fc51f8b8db9e", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01086, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "C" }, "group_id": "groupgap-within-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-within-dark-1.png", "input_tokens": 2583, "key_errors": null, "latency_sec": 5.301955471979454, "model_name": "pareto", "output_tokens": 587, "prediction": { "choice": "C" }, "prompt_sha256": "a4ae0e9e5d9435b02d3759ecf1850c36024f893c4de107ea8256f2c5052eaa53", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-29T14:17:55.610780+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "68f05e2e-8ce0-4944-989a-4fc318cf5216", "result_sha256": "d7959e9f59703145574f3b214ad17878429d64a99e7df7ddee7a232afb30963b", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0090525, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "B" }, "group_id": "groupgap-equal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-equal-light-0.png", "input_tokens": 2583, "key_errors": null, "latency_sec": 3.2952831611037254, "model_name": "pareto", "output_tokens": 346, "prediction": { "choice": "B" }, "prompt_sha256": "b4672b836da84079fbff370d5a3a7a8410525141d624bd71e50f7c11264338f7", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-29T14:17:49.502057+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "8068d9cd-9199-455c-b288-89a0299be69c", "result_sha256": "1eaa06ea74a22cf1676ef43d1b7eae29f062a669eeaa5f1dbaceb40c78538a1e", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.0075225, "err": null, "error": "Expecting value: line 1 column 1 (char 0)", "error_kind": "invalid_response", "family": "groupgap", "ground_truth": { "choice": "A" }, "group_id": "groupgap-equal-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-equal-light-1.png", "input_tokens": 2583, "key_errors": null, "latency_sec": 2.9354980350472033, "model_name": "pareto", "output_tokens": 142, "prediction": {}, "prompt_sha256": "a4ae0e9e5d9435b02d3759ecf1850c36024f893c4de107ea8256f2c5052eaa53", "provider": "anthropic", "raw_prediction": "A", "recorded_at": "2026-09-29T14:17:50.278819+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-groupgap-010", "tight_score": null, "unmetered_attempts": 0, "valid": false, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "9b32235b-3a60-4bd7-a260-36c3be1d9daf", "result_sha256": "a400667aeec20e7ab796f155caf727cdea19fbc059c465722e9955f895b7a408", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00867, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "A" }, "group_id": "groupgap-equal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-equal-dark-0.png", "input_tokens": 2583, "key_errors": null, "latency_sec": 3.0499927271157503, "model_name": "pareto", "output_tokens": 295, "prediction": { "choice": "A" }, "prompt_sha256": "a4ae0e9e5d9435b02d3759ecf1850c36024f893c4de107ea8256f2c5052eaa53", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-29T14:16:51.957095+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "8e471dbc-e25e-4745-8b73-96e9f9efd8d9", "result_sha256": "dc0c6306384dd2cfb733dd50cce7d1770a95f0aa7fa4669be9f4bae46cd42ccb", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.010425, "err": null, "error": null, "error_kind": null, "family": "groupgap", "ground_truth": { "choice": "C" }, "group_id": "groupgap-equal-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/groupgap-equal-dark-1.png", "input_tokens": 2583, "key_errors": null, "latency_sec": 4.4971969809848815, "model_name": "pareto", "output_tokens": 529, "prediction": { "choice": "C" }, "prompt_sha256": "b5587c95ce8398dca0e206a8a222a0859cb2781184dde718bbe27409262b14c1", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-29T14:17:05.883589+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-groupgap-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "c226d164-526f-4ff8-ab21-f6113a157539", "result_sha256": "d71f4c8ef0512864b987fcf88c81132231fad9b10c38a18de2c8cf1ee6724572", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0068725, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-01", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-01.png", "input_tokens": 2569, "key_errors": null, "latency_sec": 1.9859295259229839, "model_name": "pareto", "output_tokens": 60, "prediction": { "choice": "A" }, "prompt_sha256": "4cd5f1bb86e36238445efff79cfd13b01ed6916cd094da4a341b7112a2f75bfe", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-29T14:18:03.473428+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "25a81dbb-8062-4cb4-a7d8-d34fc101eaa9", "result_sha256": "1dcd05bd59b4c8661188a0a7b936bdbeadf741dbba4b869d41672d0931e910aa", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00697, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-02", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-02.png", "input_tokens": 2569, "key_errors": null, "latency_sec": 2.0807702669408172, "model_name": "pareto", "output_tokens": 73, "prediction": { "choice": "C" }, "prompt_sha256": "ac0c30958153584a4b94fe042ea1cbf718f020f9f66dd474a09c514effd2f9f9", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-29T14:17:53.312300+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "d15abad6-5339-4d9a-bf5b-623211f13212", "result_sha256": "298dee6853ca88bd9ea48c5fba513836f2e031d8cacee6d303fe954609e27d2a", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0069925, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-03", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-03.png", "input_tokens": 2569, "key_errors": null, "latency_sec": 2.3197911200113595, "model_name": "pareto", "output_tokens": 76, "prediction": { "choice": "B" }, "prompt_sha256": "377bcd13e265380fd865ccbfd9819e64dfd2c4c3b2980e09c6e4d88924162f6c", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-29T14:17:10.584393+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "d3c31695-5558-40a2-a711-e004ab759687", "result_sha256": "c2394b437a96f5cad2318504a9dfe7527ec633353e559c3d906a3762a993db24", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00679, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-04", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-04.png", "input_tokens": 2569, "key_errors": null, "latency_sec": 2.1796916800085455, "model_name": "pareto", "output_tokens": 49, "prediction": { "choice": "C" }, "prompt_sha256": "a555c9a37a814a2a2637ad2f6a1177955732dce4f82ecebc074c8930d36c5293", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-29T14:17:44.913563+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "9749a9e7-d350-4d99-b6da-2f6ab6770592", "result_sha256": "3d5be76c0adab6fce8ab17b21722f62721702f6d481ae73e7b249aa43b9f7f36", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.007075, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-05", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-05.png", "input_tokens": 2569, "key_errors": null, "latency_sec": 1.824355703080073, "model_name": "pareto", "output_tokens": 87, "prediction": { "choice": "C" }, "prompt_sha256": "45c0efef1d7925275aea025bb1909f609ffc45577974778b58b34f1829f807e2", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-29T14:17:36.195249+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "0abefac1-a6b3-40f2-a4f3-acfa120c500d", "result_sha256": "d00c8b7c8063f1d74d5f05449af214f1902266d64032a30c4d27c125602862b6", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0076, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-06", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-06.png", "input_tokens": 2569, "key_errors": null, "latency_sec": 2.286201292881742, "model_name": "pareto", "output_tokens": 157, "prediction": { "choice": "B" }, "prompt_sha256": "4cd5f1bb86e36238445efff79cfd13b01ed6916cd094da4a341b7112a2f75bfe", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-29T14:17:22.043491+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "abad799b-c5dc-477a-a556-f1fc53b01d74", "result_sha256": "8b39676556d19b5748504cb9a8cb06aa69f6b7da18fc892fa57fd41630eef5fe", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00703, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-07", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-07.png", "input_tokens": 2569, "key_errors": null, "latency_sec": 2.1243503771256655, "model_name": "pareto", "output_tokens": 81, "prediction": { "choice": "A" }, "prompt_sha256": "377bcd13e265380fd865ccbfd9819e64dfd2c4c3b2980e09c6e4d88924162f6c", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-29T14:17:46.179830+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "7dd74269-8d4a-400c-b398-04db10015ca0", "result_sha256": "d4907269c921d45a5ce8b8072c3e04bb69ada68f853820f3597290aab18e643d", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0068875, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-08", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-08.png", "input_tokens": 2569, "key_errors": null, "latency_sec": 1.5578654820565134, "model_name": "pareto", "output_tokens": 62, "prediction": { "choice": "A" }, "prompt_sha256": "a555c9a37a814a2a2637ad2f6a1177955732dce4f82ecebc074c8930d36c5293", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-29T14:17:07.455986+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "b44c1bd2-70c0-423d-98f3-5f84d352b22e", "result_sha256": "a40bb728f68a7dc0c84ce505e5f0a7db4f91a85f14319ef558df64d3f9be0569", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00682, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-09", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-09.png", "input_tokens": 2569, "key_errors": null, "latency_sec": 1.618158380035311, "model_name": "pareto", "output_tokens": 53, "prediction": { "choice": "B" }, "prompt_sha256": "a555c9a37a814a2a2637ad2f6a1177955732dce4f82ecebc074c8930d36c5293", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-29T14:17:44.952340+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "ed3322e2-97d9-48f9-a41c-b674a9eb6113", "result_sha256": "513086e0d52b67646fd357be9bade1b747705a9e4525646d58c172380080187b", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0068575, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-10", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-10.png", "input_tokens": 2569, "key_errors": null, "latency_sec": 1.753918498987332, "model_name": "pareto", "output_tokens": 58, "prediction": { "choice": "B" }, "prompt_sha256": "a555c9a37a814a2a2637ad2f6a1177955732dce4f82ecebc074c8930d36c5293", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-29T14:16:59.271172+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "257f63f4-93e4-483f-ab3d-6808bd9b1f9d", "result_sha256": "3792c7eb9a5514f156fd32f7c360b447319872f229f47da4cecdb1809cf23853", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0069625, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-11", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-11.png", "input_tokens": 2569, "key_errors": null, "latency_sec": 1.6575980000197887, "model_name": "pareto", "output_tokens": 72, "prediction": { "choice": "C" }, "prompt_sha256": "377bcd13e265380fd865ccbfd9819e64dfd2c4c3b2980e09c6e4d88924162f6c", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-29T14:17:46.549635+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "dbfa7cd4-1167-47d8-a3a3-ec4fde5aaf66", "result_sha256": "8fb9cf8b968022baad776703081a7cf824b858851d5a77c84628e1b19dae609a", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00697, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-12", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-12.png", "input_tokens": 2569, "key_errors": null, "latency_sec": 1.772068478167057, "model_name": "pareto", "output_tokens": 73, "prediction": { "choice": "A" }, "prompt_sha256": "ac0c30958153584a4b94fe042ea1cbf718f020f9f66dd474a09c514effd2f9f9", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-29T14:18:04.816442+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "59a35217-1be2-4ea8-924b-92035069e2d1", "result_sha256": "c2ae289d86917739e0137b5ff6bde3ba030936ee8a0ad63f8f8769a970cc903e", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0069625, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-13", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-13.png", "input_tokens": 2569, "key_errors": null, "latency_sec": 2.003039717907086, "model_name": "pareto", "output_tokens": 72, "prediction": { "choice": "B" }, "prompt_sha256": "6cb509a4fb3ecab9a334fb061a4aa543245f6985aa65f331cb06646a50f35dfe", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-29T14:17:57.156799+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-013", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "92fe92d4-029b-48bf-b737-23aef6d04a0b", "result_sha256": "f158c95a54b6370da0728ce17530dd743c864c645f20845de12558157d8955ed", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0068425, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-14", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-14.png", "input_tokens": 2569, "key_errors": null, "latency_sec": 1.6075586378574371, "model_name": "pareto", "output_tokens": 56, "prediction": { "choice": "C" }, "prompt_sha256": "ac0c30958153584a4b94fe042ea1cbf718f020f9f66dd474a09c514effd2f9f9", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-29T14:16:53.003340+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-014", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "17bd889c-1ac4-4809-9f4c-d4d0675909ec", "result_sha256": "6e3bff75ea4c15e9e67e48f4fedac1a040de1de41839033bc8578134e948ba21", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006805, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-15", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-15.png", "input_tokens": 2569, "key_errors": null, "latency_sec": 1.731446553952992, "model_name": "pareto", "output_tokens": 51, "prediction": { "choice": "C" }, "prompt_sha256": "ac0c30958153584a4b94fe042ea1cbf718f020f9f66dd474a09c514effd2f9f9", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-29T14:18:00.882055+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-015", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "57abb89b-b739-4592-8304-ebcc27a26fb5", "result_sha256": "686232ef066326d2edfc403b0446e844fdfcdbea3bf76f20c8ac3c6a82226b19", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.007225, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-16", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-16.png", "input_tokens": 2569, "key_errors": null, "latency_sec": 1.5939314209390432, "model_name": "pareto", "output_tokens": 107, "prediction": { "choice": "B" }, "prompt_sha256": "377bcd13e265380fd865ccbfd9819e64dfd2c4c3b2980e09c6e4d88924162f6c", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-29T14:17:25.426247+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-016", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "79b30b60-82bd-478b-944f-9b6cf3209e84", "result_sha256": "95b8b7787ec812228dc00298cf033968a3bf57424e9234b4193f544b3fb9a7f9", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006895, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-17", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-17.png", "input_tokens": 2569, "key_errors": null, "latency_sec": 2.178813843987882, "model_name": "pareto", "output_tokens": 63, "prediction": { "choice": "A" }, "prompt_sha256": "377bcd13e265380fd865ccbfd9819e64dfd2c4c3b2980e09c6e4d88924162f6c", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-29T14:17:29.864761+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-017", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "7124d981-a63b-4571-b8f2-a2d60c7d37bc", "result_sha256": "9054d4a081266783e065d63de409994d2c0bfe53cbcc33d747a7772bbcf15cd0", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0071725, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-18", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-18.png", "input_tokens": 2569, "key_errors": null, "latency_sec": 1.9802759170997888, "model_name": "pareto", "output_tokens": 100, "prediction": { "choice": "C" }, "prompt_sha256": "45c0efef1d7925275aea025bb1909f609ffc45577974778b58b34f1829f807e2", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-29T14:17:59.521076+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-018", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "b8cc74b0-3c9e-4589-8997-fcfbcc7664ed", "result_sha256": "6e76e439993e6500ae6c96647eb9650885c759d423802580712b58a3154da6a2", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.007105, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "C" }, "group_id": "h-19", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-19.png", "input_tokens": 2569, "key_errors": null, "latency_sec": 1.9600911207962781, "model_name": "pareto", "output_tokens": 91, "prediction": { "choice": "C" }, "prompt_sha256": "6cb509a4fb3ecab9a334fb061a4aa543245f6985aa65f331cb06646a50f35dfe", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-29T14:17:12.561896+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-019", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "aab2530d-3d96-4c7e-ba70-5f060c349dd1", "result_sha256": "64806a1061c5585fb118608413355877663bb03a1a83850fc9040fff6a8e0015", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00748, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-20", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-20.png", "input_tokens": 2569, "key_errors": null, "latency_sec": 2.030281794955954, "model_name": "pareto", "output_tokens": 141, "prediction": { "choice": "A" }, "prompt_sha256": "377bcd13e265380fd865ccbfd9819e64dfd2c4c3b2980e09c6e4d88924162f6c", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-29T14:16:57.911567+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-020", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "f692b85e-2568-4d22-a500-ff0b9c029c60", "result_sha256": "b8929238d73047180aee82e5b8abd5466dd36e58b674a6e9e6652badca5c4604", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00691, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-21", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-21.png", "input_tokens": 2569, "key_errors": null, "latency_sec": 1.4790599481202662, "model_name": "pareto", "output_tokens": 65, "prediction": { "choice": "B" }, "prompt_sha256": "a555c9a37a814a2a2637ad2f6a1177955732dce4f82ecebc074c8930d36c5293", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-29T14:17:34.663170+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-021", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "991609b1-4f4f-4585-81b1-4490919448ee", "result_sha256": "8b1aac8cbf900615415905a10463ae7cf2a72d106d0a1cab7308ce284329eb80", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006835, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-22", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-22.png", "input_tokens": 2569, "key_errors": null, "latency_sec": 1.5995723251253366, "model_name": "pareto", "output_tokens": 55, "prediction": { "choice": "A" }, "prompt_sha256": "ac0c30958153584a4b94fe042ea1cbf718f020f9f66dd474a09c514effd2f9f9", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-29T14:17:20.108681+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-022", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "973a0c17-79ba-47b7-97ce-27f8245eec32", "result_sha256": "35c7887861ba3ac8d810eca559b0dc95e2b24775d388408d97be594452e9c9df", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0068575, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "A" }, "group_id": "h-23", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-23.png", "input_tokens": 2569, "key_errors": null, "latency_sec": 1.6700430971104652, "model_name": "pareto", "output_tokens": 58, "prediction": { "choice": "A" }, "prompt_sha256": "6cb509a4fb3ecab9a334fb061a4aa543245f6985aa65f331cb06646a50f35dfe", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-29T14:16:56.825602+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-023", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "68d80e0e-c3a2-4419-8a1d-820102df3f25", "result_sha256": "e9cc9270ade89aaacd248f92384e40ef1dbb305a61b6723d7b69ba9e91e6fa8d", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0068725, "err": null, "error": null, "error_kind": null, "family": "nestedflow", "ground_truth": { "choice": "B" }, "group_id": "h-24", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-24.png", "input_tokens": 2569, "key_errors": null, "latency_sec": 1.3849481861107051, "model_name": "pareto", "output_tokens": 60, "prediction": { "choice": "B" }, "prompt_sha256": "45c0efef1d7925275aea025bb1909f609ffc45577974778b58b34f1829f807e2", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-29T14:17:12.936970+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedflow-024", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "6142d319-d299-4782-9a98-3f07a6eda4a4", "result_sha256": "cc342e2d7c614bcf94b59274c340980ce5246200123a1384109eb4c546281b71", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0071825, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-09", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-09.png", "input_tokens": 2579, "key_errors": null, "latency_sec": 2.339739317074418, "model_name": "pareto", "output_tokens": 98, "prediction": { "choice": "B" }, "prompt_sha256": "a10b800c7bde04b869b770c30fd7ed4b80b25795d3668fcd2fd6f199bd9b600e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-29T14:17:43.076568+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "2aa55ccb-0fde-4c47-975b-4ab4544aba51", "result_sha256": "9eb03ddf99c10f38eb286dae5b338b89d564812d8842de3f6418f0e9a445d607", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.008315, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-10", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-10.png", "input_tokens": 2579, "key_errors": null, "latency_sec": 2.8175799809396267, "model_name": "pareto", "output_tokens": 249, "prediction": { "choice": "B" }, "prompt_sha256": "a10b800c7bde04b869b770c30fd7ed4b80b25795d3668fcd2fd6f199bd9b600e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-29T14:18:01.777512+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "062bf106-e7b6-4420-baef-e82c97a2599e", "result_sha256": "38a4cedb5ee2c73ac08d87c1196b716073042ec3a58d7a54e34a37633a308756", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0076475, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-11", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-11.png", "input_tokens": 2579, "key_errors": null, "latency_sec": 3.329857981065288, "model_name": "pareto", "output_tokens": 160, "prediction": { "choice": "C" }, "prompt_sha256": "7005b025e68bce70267cd8198f60bc975d0dd5a8eea3134d829280ee9656a35b", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-29T14:17:32.560470+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "5ca334c3-91ea-4db0-92d1-74335ab056c1", "result_sha256": "492c5e0e74a68647b5926e1f817b69b860b014c7a0cb9c276690241349233a43", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0084275, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-12", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-12.png", "input_tokens": 2579, "key_errors": null, "latency_sec": 3.772160931956023, "model_name": "pareto", "output_tokens": 264, "prediction": { "choice": "A" }, "prompt_sha256": "3ba51907b89444fcea345fda34c9692faefd56e2b5c6671b282c0da644fdccb7", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-29T14:17:07.909530+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "103cda5a-d632-4275-b810-f692bfbe019e", "result_sha256": "ecee87ddbdc57f826aab65d659cd393847da37a2ffdb0ad3298b18da1e799a02", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0076175, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-21", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-21.png", "input_tokens": 2579, "key_errors": null, "latency_sec": 2.594237416051328, "model_name": "pareto", "output_tokens": 156, "prediction": { "choice": "A" }, "prompt_sha256": "a242868807b1158889b152c7c9664c235ebd10352a9a4a541f0a0af30a7fb4a5", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-29T14:16:51.493978+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "8aa72454-5f8d-4570-bae9-af8fe6048c52", "result_sha256": "97ce2984682f15c2383ad540c69bb40208a87179f589efa409d67c1edb0aceac", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0086075, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-22", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-22.png", "input_tokens": 2579, "key_errors": null, "latency_sec": 2.156134741846472, "model_name": "pareto", "output_tokens": 288, "prediction": { "choice": "B" }, "prompt_sha256": "a10b800c7bde04b869b770c30fd7ed4b80b25795d3668fcd2fd6f199bd9b600e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-29T14:17:41.512962+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "3cc9676c-c444-46d2-9393-d1a7956d1f87", "result_sha256": "825fc1a272f366154a405bd13a329c6a837ac3b538ec80ab3d6bcc7cd8a3cb7e", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0079625, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-23", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-23.png", "input_tokens": 2579, "key_errors": null, "latency_sec": 2.144976287148893, "model_name": "pareto", "output_tokens": 202, "prediction": { "choice": "A" }, "prompt_sha256": "a242868807b1158889b152c7c9664c235ebd10352a9a4a541f0a0af30a7fb4a5", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-29T14:17:39.401600+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "7ce0b279-dcf6-4a29-8e68-74ad12d6789b", "result_sha256": "2429c2948da617a74ec8275a30dd580f853531bc60b53f15810cbf75dda9f65b", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0087425, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-24", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-24.png", "input_tokens": 2579, "key_errors": null, "latency_sec": 2.3873048461973667, "model_name": "pareto", "output_tokens": 306, "prediction": { "choice": "C" }, "prompt_sha256": "1019a8d1887a379785b7f0f2096c38ba800c5c05f4005e0a2fc11959e378500f", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-29T14:17:03.094763+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "712481a3-0b4d-4446-bed8-c152d317b0a1", "result_sha256": "6972a1d086238d33aa37c88ef1958fef6f56dddfe014422f14229da8d7ac816d", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.007475, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-25", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-25.png", "input_tokens": 2579, "key_errors": null, "latency_sec": 2.06300320988521, "model_name": "pareto", "output_tokens": 137, "prediction": { "choice": "C" }, "prompt_sha256": "3ba51907b89444fcea345fda34c9692faefd56e2b5c6671b282c0da644fdccb7", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-29T14:17:33.552925+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "22b8bc09-4733-43f7-8a0a-74597bca0f2b", "result_sha256": "9aa0431516c405bc6ca3b2250c7fd735f5c341a4a72c85c9bb0d5fcb54773a2c", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00773, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-26", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-26.png", "input_tokens": 2579, "key_errors": null, "latency_sec": 2.26339330105111, "model_name": "pareto", "output_tokens": 171, "prediction": { "choice": "A" }, "prompt_sha256": "dbe3324244c1acc8302aa7a2f89b338ba07244a1490efb8ddaf45fc8d47a978d", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-29T14:16:56.889064+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "7c68341c-3fae-41cf-8c02-a491ddad81d4", "result_sha256": "3e19375ff4eeb04ad5db10d2cd534663892a8815cd996dbfefef80a0e1189ec7", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.007535, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-27", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-27.png", "input_tokens": 2579, "key_errors": null, "latency_sec": 2.5154739229474217, "model_name": "pareto", "output_tokens": 145, "prediction": { "choice": "A" }, "prompt_sha256": "dbe3324244c1acc8302aa7a2f89b338ba07244a1490efb8ddaf45fc8d47a978d", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-29T14:17:17.791048+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "79fdfd61-1e65-4cb2-b29d-29595b6f50ef", "result_sha256": "bf04b0b1f64303293f64133acb343b5e30dc263d16e9142b58a6b028904a7ed3", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00905, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-28", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-28.png", "input_tokens": 2579, "key_errors": null, "latency_sec": 3.70287779904902, "model_name": "pareto", "output_tokens": 347, "prediction": { "choice": "C" }, "prompt_sha256": "3ba51907b89444fcea345fda34c9692faefd56e2b5c6671b282c0da644fdccb7", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-29T14:17:42.112826+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "5888a7bb-34f1-46cf-8406-a7ca5e0965a1", "result_sha256": "2513f77fab4ba79a7cc63c9931fd0dbd5cf43ce4f9f1137e63c024edbefda8d3", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.008105, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-29", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-29.png", "input_tokens": 2579, "key_errors": null, "latency_sec": 2.274803881999105, "model_name": "pareto", "output_tokens": 221, "prediction": { "choice": "B" }, "prompt_sha256": "1019a8d1887a379785b7f0f2096c38ba800c5c05f4005e0a2fc11959e378500f", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-29T14:17:55.525272+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-013", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "6e7518f1-f155-417c-ae26-b33e8aa033bc", "result_sha256": "07ba86611c5050437551494b845fef7cfc3a54ff269d074309267a0bf296da7d", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.00959, "err": null, "error": "Expecting value: line 1 column 1 (char 0)", "error_kind": "invalid_response", "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-30", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-30.png", "input_tokens": 2579, "key_errors": null, "latency_sec": 3.653300137957558, "model_name": "pareto", "output_tokens": 419, "prediction": {}, "prompt_sha256": "a242868807b1158889b152c7c9664c235ebd10352a9a4a541f0a0af30a7fb4a5", "provider": "anthropic", "raw_prediction": "C", "recorded_at": "2026-09-29T14:18:01.454066+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-nestedwrap-014", "tight_score": null, "unmetered_attempts": 0, "valid": false, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "4e40f8a6-5a35-4cf8-b539-12001d3e6214", "result_sha256": "9bf2147e635cf9fa5d5bb04aecd79c0b148283b5564cdc0b80b4e744c60689cf", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.008, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-31", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-31.png", "input_tokens": 2579, "key_errors": null, "latency_sec": 2.2591891877818853, "model_name": "pareto", "output_tokens": 207, "prediction": { "choice": "B" }, "prompt_sha256": "3ba51907b89444fcea345fda34c9692faefd56e2b5c6671b282c0da644fdccb7", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-29T14:17:17.966301+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-015", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "20dd29cd-7235-42ae-a016-823febe06e50", "result_sha256": "f4bab1fd8966c2f862278ba29bb0affb1d9b8e67d58aac6f9d0d035d6a18df3b", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0081725, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-32", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-32.png", "input_tokens": 2579, "key_errors": null, "latency_sec": 2.7227164970245212, "model_name": "pareto", "output_tokens": 230, "prediction": { "choice": "B" }, "prompt_sha256": "3ba51907b89444fcea345fda34c9692faefd56e2b5c6671b282c0da644fdccb7", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-29T14:18:05.284049+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-016", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "772b0797-9eb5-463a-9f71-5071a2970af0", "result_sha256": "2c1d88fb8a244be99d2bb56f952f1b4f38565a3acda2d0a0bc3723645fa3a0bf", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.008, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-33", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-33.png", "input_tokens": 2579, "key_errors": null, "latency_sec": 2.8197626220062375, "model_name": "pareto", "output_tokens": 207, "prediction": { "choice": "A" }, "prompt_sha256": "1019a8d1887a379785b7f0f2096c38ba800c5c05f4005e0a2fc11959e378500f", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-29T14:17:25.192914+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-017", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "e1a8541e-7e45-4669-a0d9-a38a65a37ebb", "result_sha256": "00b0544a84091b4e6b8925fa197f066668a5ca204392b38e016f080cc761b5b2", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0075575, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-34", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-34.png", "input_tokens": 2579, "key_errors": null, "latency_sec": 2.038081169128418, "model_name": "pareto", "output_tokens": 148, "prediction": { "choice": "A" }, "prompt_sha256": "1019a8d1887a379785b7f0f2096c38ba800c5c05f4005e0a2fc11959e378500f", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-29T14:17:14.749762+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-018", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "681ee6a6-4449-4eea-8d94-35a80b2ef6e2", "result_sha256": "89104ea7b47a3cc1d265fc89ded096181d4ef503e5d89263288d0c211f580e05", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0080975, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-35", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-35.png", "input_tokens": 2579, "key_errors": null, "latency_sec": 2.859820842044428, "model_name": "pareto", "output_tokens": 220, "prediction": { "choice": "C" }, "prompt_sha256": "a10b800c7bde04b869b770c30fd7ed4b80b25795d3668fcd2fd6f199bd9b600e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-29T14:17:56.351767+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-019", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "28f02b1d-bf8d-45a5-a283-63caff4f1c39", "result_sha256": "973618e69a2a2af4d919ca9d5cb3f44fefbee0e4907dcf1ce1fb54eed20b05aa", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.008405, "err": null, "error": "Expecting value: line 1 column 1 (char 0)", "error_kind": "invalid_response", "family": "nestedwrap", "ground_truth": { "choice": "A" }, "group_id": "h-36", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-36.png", "input_tokens": 2579, "key_errors": null, "latency_sec": 3.0643772380426526, "model_name": "pareto", "output_tokens": 261, "prediction": {}, "prompt_sha256": "1019a8d1887a379785b7f0f2096c38ba800c5c05f4005e0a2fc11959e378500f", "provider": "anthropic", "raw_prediction": "A", "recorded_at": "2026-09-29T14:17:15.526968+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-nestedwrap-020", "tight_score": null, "unmetered_attempts": 0, "valid": false, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "e2874513-555f-4c30-aac3-a1cdeeb389b8", "result_sha256": "2f7a35a3fc7203f06aa45dc37108c04fc12b3d4ef68424e736e3b39fb1ace2a2", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0089, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-37", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-37.png", "input_tokens": 2579, "key_errors": null, "latency_sec": 2.863118896028027, "model_name": "pareto", "output_tokens": 327, "prediction": { "choice": "C" }, "prompt_sha256": "a242868807b1158889b152c7c9664c235ebd10352a9a4a541f0a0af30a7fb4a5", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-29T14:17:12.967663+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-021", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "87e74379-bd0e-40fe-87b2-3352cbc768d8", "result_sha256": "fd71839049bd7bc43017659b09e897ea9793a723a8d73f36b06ab888ead158f0", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.010955, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-38", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-38.png", "input_tokens": 2579, "key_errors": null, "latency_sec": 3.6633410227950662, "model_name": "pareto", "output_tokens": 601, "prediction": { "choice": "B" }, "prompt_sha256": "3ba51907b89444fcea345fda34c9692faefd56e2b5c6671b282c0da644fdccb7", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-29T14:17:25.488869+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-022", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "780c4750-a607-4d5f-b081-3a5f805f422b", "result_sha256": "8e85d0231a39ece007ed94fadf57757d3c5ac88f28073fe69d885d98b1e34750", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0080225, "err": null, "error": null, "error_kind": null, "family": "nestedwrap", "ground_truth": { "choice": "B" }, "group_id": "h-39", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-39.png", "input_tokens": 2579, "key_errors": null, "latency_sec": 2.4959922039415687, "model_name": "pareto", "output_tokens": 210, "prediction": { "choice": "B" }, "prompt_sha256": "1019a8d1887a379785b7f0f2096c38ba800c5c05f4005e0a2fc11959e378500f", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-29T14:17:18.042177+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-nestedwrap-023", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "b7e305ea-3246-4d1d-89ba-50a86f18aaa3", "result_sha256": "8ce3d5097bb939e6af5e7963a4dea6b9f812f469ae4d7fbb9d794531ca2fced3", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.00836, "err": null, "error": "Expecting value: line 1 column 1 (char 0)", "error_kind": "invalid_response", "family": "nestedwrap", "ground_truth": { "choice": "C" }, "group_id": "h-40", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-40.png", "input_tokens": 2579, "key_errors": null, "latency_sec": 2.361248614033684, "model_name": "pareto", "output_tokens": 255, "prediction": {}, "prompt_sha256": "dbe3324244c1acc8302aa7a2f89b338ba07244a1490efb8ddaf45fc8d47a978d", "provider": "anthropic", "raw_prediction": "C", "recorded_at": "2026-09-29T14:17:20.172425+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-nestedwrap-024", "tight_score": null, "unmetered_attempts": 0, "valid": false, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "18c142ce-c893-4969-9f9d-5952dd89bfb0", "result_sha256": "dc0328657ee4bd1a59be5527e9541ac73d37493dec58b9a45a9316bd3d35ca20", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0079525, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "C" }, "group_id": "padcompare-horizontal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-horizontal-light-0.png", "input_tokens": 2593, "key_errors": null, "latency_sec": 2.0416087941266596, "model_name": "pareto", "output_tokens": 196, "prediction": { "choice": "C" }, "prompt_sha256": "0abb90b71247ce9a0876c99ad3ef52ea5a9ae09bfffb2875eec73b10166640c1", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-29T14:17:37.233012+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "982862bc-2e0d-4fab-ae17-a8e2de0d40e8", "result_sha256": "2cc6932dc8e513941bf6bd903c450a43caf3b8f7981f4c8106c79b5a6e97afb3", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0079825, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "A" }, "group_id": "padcompare-horizontal-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-horizontal-light-1.png", "input_tokens": 2593, "key_errors": null, "latency_sec": 2.0002577379345894, "model_name": "pareto", "output_tokens": 200, "prediction": { "choice": "A" }, "prompt_sha256": "08c2d8beb6c4a367fb4f9166e02fb0ce76d3bd79ca0c35a18250c8312e79a7ef", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-29T14:17:10.088326+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "e2bdd2d0-d5cf-4dd7-ab33-815d000f22af", "result_sha256": "776c332a82ee067224451baf1301f9e0eb2b78a92da6b984fce459df46b68c65", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00763, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "B" }, "group_id": "padcompare-horizontal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-horizontal-dark-0.png", "input_tokens": 2593, "key_errors": null, "latency_sec": 1.9478382568340749, "model_name": "pareto", "output_tokens": 153, "prediction": { "choice": "B" }, "prompt_sha256": "4f484ecb32307572888516788b76d03dcfb86406579ae3871dbbdd5d50c24d37", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-29T14:17:05.055396+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "f081bf08-8aee-4a08-9938-48c7ddc49118", "result_sha256": "80d5dd575b810ab10ff0e8ad37d86f0f7feaedc4f37bf35d3c8aa684c8981979", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00934, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "B" }, "group_id": "padcompare-horizontal-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-horizontal-dark-1.png", "input_tokens": 2593, "key_errors": null, "latency_sec": 3.247768934816122, "model_name": "pareto", "output_tokens": 381, "prediction": { "choice": "B" }, "prompt_sha256": "4f484ecb32307572888516788b76d03dcfb86406579ae3871dbbdd5d50c24d37", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-29T14:16:52.135401+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "99c28c6c-6a27-4c77-ab7a-0b41e0caebe8", "result_sha256": "ab5a5239bbb9f71a0ae3c2eedd3ace717e32e61b89f5502d3a9bf7d94abb74a0", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.007975, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "A" }, "group_id": "padcompare-vertical-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-vertical-light-0.png", "input_tokens": 2593, "key_errors": null, "latency_sec": 2.5880502050276846, "model_name": "pareto", "output_tokens": 199, "prediction": { "choice": "A" }, "prompt_sha256": "0abb90b71247ce9a0876c99ad3ef52ea5a9ae09bfffb2875eec73b10166640c1", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-29T14:17:00.696099+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "8a7a7b88-8c20-4290-a009-e5580cf17637", "result_sha256": "d4b52f55489cb5c47043c2d2c1a41df414125528e7cab155ac37234a922402d1", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00862, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "C" }, "group_id": "padcompare-vertical-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-vertical-light-1.png", "input_tokens": 2593, "key_errors": null, "latency_sec": 2.3559132169466466, "model_name": "pareto", "output_tokens": 285, "prediction": { "choice": "C" }, "prompt_sha256": "07819e1c7e8686a5ffdde745114a9c6e8df1c7ac485dc319c5c72c24e1b67dae", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-29T14:18:06.644579+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "990561da-462d-4e64-bc0c-41a6b2db1297", "result_sha256": "0faf8f31afef335df805ca115989e0051f3db800429d1e073c1e8800878d224b", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0084925, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "A" }, "group_id": "padcompare-vertical-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-vertical-dark-0.png", "input_tokens": 2593, "key_errors": null, "latency_sec": 2.4030937959905714, "model_name": "pareto", "output_tokens": 268, "prediction": { "choice": "A" }, "prompt_sha256": "4f484ecb32307572888516788b76d03dcfb86406579ae3871dbbdd5d50c24d37", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-29T14:17:34.347976+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "3c41fbf9-52a2-497d-950a-3f3c20e59f57", "result_sha256": "0137f11e73dafd5931b4d0dfab4782fc8db7ad1268b1fcbbc5e7375209e1fd8c", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.007585, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "B" }, "group_id": "padcompare-vertical-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-vertical-dark-1.png", "input_tokens": 2593, "key_errors": null, "latency_sec": 2.5182823319919407, "model_name": "pareto", "output_tokens": 147, "prediction": { "choice": "B" }, "prompt_sha256": "168f7fb4256da669d5b660759ed5f66d9bc1fe6fa5e0cc2f680f4674e926e39a", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-29T14:17:19.738212+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "529eecb0-1359-4ec3-8dc1-4926f58102f2", "result_sha256": "a87e32446e2667f1a363ecec78728cc1952f1bda8b0c8983b498f5355ae926c7", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.0084775, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "B" }, "group_id": "padcompare-equal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-equal-light-0.png", "input_tokens": 2593, "key_errors": null, "latency_sec": 2.5328552729915828, "model_name": "pareto", "output_tokens": 266, "prediction": { "choice": "A" }, "prompt_sha256": "0abb90b71247ce9a0876c99ad3ef52ea5a9ae09bfffb2875eec73b10166640c1", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-29T14:17:23.907648+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-padcompare-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "96b20e57-4363-498e-bd9e-2a0796edc1f5", "result_sha256": "579a8a3d5eb4ae723d44536ad1041954beabfabc1b96a15ed26ad289413b19b3", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.020395, "err": null, "error": "Expecting value: line 1 column 1 (char 0)", "error_kind": "invalid_response", "family": "padcompare", "ground_truth": { "choice": "A" }, "group_id": "padcompare-equal-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-equal-light-1.png", "input_tokens": 2593, "key_errors": null, "latency_sec": 9.886003009974957, "model_name": "pareto", "output_tokens": 1855, "prediction": {}, "prompt_sha256": "07819e1c7e8686a5ffdde745114a9c6e8df1c7ac485dc319c5c72c24e1b67dae", "provider": "anthropic", "raw_prediction": "A", "recorded_at": "2026-09-29T14:17:39.885880+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-padcompare-010", "tight_score": null, "unmetered_attempts": 0, "valid": false, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "429291b5-7a4f-42f6-8697-aaef5f386bb0", "result_sha256": "990a16b9d4c997f10762770730c811701ffa5ccd156004f25af489f64d4aec89", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.0194725, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "C" }, "group_id": "padcompare-equal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-equal-dark-0.png", "input_tokens": 2593, "key_errors": null, "latency_sec": 8.268409000942484, "model_name": "pareto", "output_tokens": 1732, "prediction": { "choice": "A" }, "prompt_sha256": "4f484ecb32307572888516788b76d03dcfb86406579ae3871dbbdd5d50c24d37", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-29T14:17:49.806787+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-padcompare-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "8405e230-6488-434a-839b-551576feb0c8", "result_sha256": "07ad6fd664d85e04e8571172947397bec7cde972cc4e062cd34ca7b30c544e82", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0252475, "err": null, "error": null, "error_kind": null, "family": "padcompare", "ground_truth": { "choice": "C" }, "group_id": "padcompare-equal-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/padcompare-equal-dark-1.png", "input_tokens": 2593, "key_errors": null, "latency_sec": 12.974288318073377, "model_name": "pareto", "output_tokens": 2502, "prediction": { "choice": "C" }, "prompt_sha256": "168f7fb4256da669d5b660759ed5f66d9bc1fe6fa5e0cc2f680f4674e926e39a", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-29T14:17:12.443829+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-padcompare-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "aeaa8f69-e7b6-4f13-9d03-b73dc8a3cd1d", "result_sha256": "0454a64c91dec6289f40b4259729520234bbb4e5c56ff39d1df0830f2ae2f56b", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0070325, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "D" }, "group_id": "textalign-left-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-left-light-0.png", "input_tokens": 2597, "key_errors": null, "latency_sec": 2.0401099419686943, "model_name": "pareto", "output_tokens": 72, "prediction": { "choice": "D" }, "prompt_sha256": "2e47c2a304900033c1e7233c922b86d826e6949f29106d3fa5c967cc727df497", "provider": "anthropic", "raw_prediction": "{\"choice\": \"D\"}", "recorded_at": "2026-09-29T14:17:57.596572+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "e0bff592-fce1-42ac-a435-e952d26b821c", "result_sha256": "3ea00232567c541afcf51e5f114a1126554ba90a8a269c657ea60f3bb377c8d5", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.009605, "err": null, "error": "Expecting value: line 1 column 1 (char 0)", "error_kind": "invalid_response", "family": "textalign", "ground_truth": { "choice": "D" }, "group_id": "textalign-left-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-left-light-1.png", "input_tokens": 2597, "key_errors": null, "latency_sec": 3.5359742981381714, "model_name": "pareto", "output_tokens": 415, "prediction": {}, "prompt_sha256": "2e47c2a304900033c1e7233c922b86d826e6949f29106d3fa5c967cc727df497", "provider": "anthropic", "raw_prediction": "C", "recorded_at": "2026-09-29T14:17:54.117572+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-textalign-002", "tight_score": null, "unmetered_attempts": 0, "valid": false, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "8f62052d-af50-46b4-a396-a307d509514c", "result_sha256": "ca99b92daf2a1fcbb432b1eef2b480f0ef8c08bd53090b9b5cd48694ef7a690c", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00866, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "A" }, "group_id": "textalign-left-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-left-dark-0.png", "input_tokens": 2597, "key_errors": null, "latency_sec": 3.220576724037528, "model_name": "pareto", "output_tokens": 289, "prediction": { "choice": "A" }, "prompt_sha256": "a46b47bd0291db684271663183741f72c6e90a5628697cdf5c907a9da6067b3d", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-29T14:17:04.900392+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "694037d6-441f-45a0-b3c3-6c1ccc56206b", "result_sha256": "b6044b10a2aabc572025a03aea4a068f548ef2347783e6600c47ff14f1ea01d4", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0083375, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "D" }, "group_id": "textalign-left-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-left-dark-1.png", "input_tokens": 2597, "key_errors": null, "latency_sec": 2.9426800389774144, "model_name": "pareto", "output_tokens": 246, "prediction": { "choice": "D" }, "prompt_sha256": "02fcd4a4c952217ee96fdbbca5e92af61005f02abe9611e017f571e23cb193e2", "provider": "anthropic", "raw_prediction": "{\"choice\": \"D\"}", "recorded_at": "2026-09-29T14:18:06.127817+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "ab9d1662-e0b9-4452-99ee-db360a476348", "result_sha256": "ea3d71dee099e5c6df4bfb92634c98a822d128f1410635ab64a6908b550175c8", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.011495, "err": null, "error": "Expecting value: line 1 column 1 (char 0)", "error_kind": "invalid_response", "family": "textalign", "ground_truth": { "choice": "C" }, "group_id": "textalign-center-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-center-light-0.png", "input_tokens": 2597, "key_errors": null, "latency_sec": 4.869142596144229, "model_name": "pareto", "output_tokens": 667, "prediction": {}, "prompt_sha256": "a46b47bd0291db684271663183741f72c6e90a5628697cdf5c907a9da6067b3d", "provider": "anthropic", "raw_prediction": "C", "recorded_at": "2026-09-29T14:17:50.552365+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-textalign-005", "tight_score": null, "unmetered_attempts": 0, "valid": false, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "93b8bf03-ac91-4d48-b6d0-7c5ac9de4730", "result_sha256": "98e40b7a940d01b263dafcf4f050f8336d662f3231bb3a18573a7c69b2bd0934", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0105575, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "B" }, "group_id": "textalign-center-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-center-light-1.png", "input_tokens": 2597, "key_errors": null, "latency_sec": 3.240374278044328, "model_name": "pareto", "output_tokens": 542, "prediction": { "choice": "B" }, "prompt_sha256": "2c9a6eb7abbff326cf2a2f2c942bc4e9b7e820c30052fd8719ee75f2d63826cd", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-29T14:17:13.016712+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "c93f5c1c-ef27-453d-9d0a-322790846d00", "result_sha256": "9018c0d28402f5798612d53d5f8cfdf77341b766b8941f77f04aefce72758463", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0107375, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "B" }, "group_id": "textalign-center-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-center-dark-0.png", "input_tokens": 2597, "key_errors": null, "latency_sec": 4.221947029000148, "model_name": "pareto", "output_tokens": 566, "prediction": { "choice": "B" }, "prompt_sha256": "9a155e0b3c7bbbbff68050bc1a1e72dea59500ab480af46a3cd3b87e25de70bf", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-29T14:17:26.574256+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "2c2b9718-fd6f-418e-b690-72f89fb4f722", "result_sha256": "860a5e815fdf17c162e6401a842978d31caec55a7034d8f9ea34fa7e2c014e89", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.01004, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "B" }, "group_id": "textalign-center-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-center-dark-1.png", "input_tokens": 2597, "key_errors": null, "latency_sec": 3.5573331071063876, "model_name": "pareto", "output_tokens": 473, "prediction": { "choice": "B" }, "prompt_sha256": "2e47c2a304900033c1e7233c922b86d826e6949f29106d3fa5c967cc727df497", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-29T14:17:58.550127+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "138fef8c-20b5-4cc2-bca6-4035c21b1e79", "result_sha256": "2f5fa82389557064a05ea9d4072d79ed88e9dd284512568e4594f20be051d24b", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0095525, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "B" }, "group_id": "textalign-right-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-right-light-0.png", "input_tokens": 2597, "key_errors": null, "latency_sec": 3.2158248180057853, "model_name": "pareto", "output_tokens": 408, "prediction": { "choice": "B" }, "prompt_sha256": "4d2473b3f62dd5f7bdca37b6285ca5748e1b3eadc7e0f04cb6fd68e2978a908b", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-29T14:16:54.336966+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "7460979b-02b1-40c7-b708-7ea60ff7d60f", "result_sha256": "4b99b841228cc45c848fbce8e1b3701442fd656e6bd1a35acfa161c7c743b7e8", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.0078725, "err": null, "error": "Expecting value: line 1 column 1 (char 0)", "error_kind": "invalid_response", "family": "textalign", "ground_truth": { "choice": "D" }, "group_id": "textalign-right-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-right-light-1.png", "input_tokens": 2597, "key_errors": null, "latency_sec": 2.2757514391560107, "model_name": "pareto", "output_tokens": 184, "prediction": {}, "prompt_sha256": "05e7cf07d1f1be0bf5dff1e08481cc4ad5c2acfdf246c6fe69f426c659c19fea", "provider": "anthropic", "raw_prediction": "A", "recorded_at": "2026-09-29T14:17:36.961355+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-textalign-010", "tight_score": null, "unmetered_attempts": 0, "valid": false, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "bcab263d-0d5a-4b0c-88c6-f3b32bbe839f", "result_sha256": "c57f00815ee5f5adb958522e85c7d8477f1e6192975ff3c91b17ac99285cf4cf", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0117575, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "A" }, "group_id": "textalign-right-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-right-dark-0.png", "input_tokens": 2597, "key_errors": null, "latency_sec": 6.189292537048459, "model_name": "pareto", "output_tokens": 702, "prediction": { "choice": "A" }, "prompt_sha256": "774f6052d034e009780326c2721c2a22fa373bdb75a2015c34e12e26984fdc20", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-29T14:17:08.495622+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "6af07903-0bfc-46b4-8d91-aaa76c6ec742", "result_sha256": "79df8b212c09549dcf04a2deccd3abbad991176981f89e9cfd78d56cf754a63c", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.00806, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "A" }, "group_id": "textalign-right-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-right-dark-1.png", "input_tokens": 2597, "key_errors": null, "latency_sec": 2.3561330360826105, "model_name": "pareto", "output_tokens": 209, "prediction": { "choice": "D" }, "prompt_sha256": "ee3ac587972345495a05631df96b48614d67cf083b60577be8181fd19c9043fd", "provider": "anthropic", "raw_prediction": "{\"choice\": \"D\"}", "recorded_at": "2026-09-29T14:18:06.604381+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-textalign-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "d9c34034-9aa2-47b4-9bf4-551f429ecd42", "result_sha256": "38c398d1ad510fdf296c56ff5ad6f21ba8d10c6ee69f2559cb6db1be00d090a6", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0152, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "A" }, "group_id": "textalign-justify-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-justify-light-0.png", "input_tokens": 2597, "key_errors": null, "latency_sec": 6.594838043907657, "model_name": "pareto", "output_tokens": 1161, "prediction": { "choice": "A" }, "prompt_sha256": "02fcd4a4c952217ee96fdbbca5e92af61005f02abe9611e017f571e23cb193e2", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-29T14:17:49.697642+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-013", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "84212912-8a3d-4941-a57e-ffa9e44644e8", "result_sha256": "8852d525cda3e20b8dc09bf1aaebefbe9e53a396b2fb46233ce0f50b655b7073", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.00764, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "C" }, "group_id": "textalign-justify-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-justify-light-1.png", "input_tokens": 2597, "key_errors": null, "latency_sec": 2.4986159331165254, "model_name": "pareto", "output_tokens": 153, "prediction": { "choice": "D" }, "prompt_sha256": "2e47c2a304900033c1e7233c922b86d826e6949f29106d3fa5c967cc727df497", "provider": "anthropic", "raw_prediction": "{\"choice\": \"D\"}", "recorded_at": "2026-09-29T14:17:09.985065+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-textalign-014", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "f4843eb3-be9d-4901-a6cb-f5200cdbc33d", "result_sha256": "cbf284fb07094056d45ab20f150b23b7f10d5f84e2e65449d0aa3c70d8636c09", "result": { "correct": false, "cost_estimated": false, "cost_usd": 0.008465, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "C" }, "group_id": "textalign-justify-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-justify-dark-0.png", "input_tokens": 2597, "key_errors": null, "latency_sec": 3.0027507601771504, "model_name": "pareto", "output_tokens": 263, "prediction": { "choice": "A" }, "prompt_sha256": "9a155e0b3c7bbbbff68050bc1a1e72dea59500ab480af46a3cd3b87e25de70bf", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-29T14:17:41.341207+00:00", "request_attempts": 1, "score": 0, "task_id": "layoutbench-textalign-015", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "35067642-5d71-4763-92a8-8772ec6babee", "result_sha256": "c8d3ea9202a6016c79f7fbd02e91ffda836084a1a9187cbbfc215fd033f77474", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0095525, "err": null, "error": null, "error_kind": null, "family": "textalign", "ground_truth": { "choice": "C" }, "group_id": "textalign-justify-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textalign-justify-dark-1.png", "input_tokens": 2597, "key_errors": null, "latency_sec": 5.388723507057875, "model_name": "pareto", "output_tokens": 408, "prediction": { "choice": "C" }, "prompt_sha256": "2e47c2a304900033c1e7233c922b86d826e6949f29106d3fa5c967cc727df497", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-29T14:17:07.470976+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textalign-016", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "5798ecfa-ef33-42d2-bcff-0c6277b3e1c5", "result_sha256": "2627c37df0dda58a41f76230e9651ecabb09a263144eb287ab59052cae4f21c6", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0074125, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "A" }, "group_id": "textcolumns-1-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-1-light-0.png", "input_tokens": 2545, "key_errors": null, "latency_sec": 2.0632782699540257, "model_name": "pareto", "output_tokens": 140, "prediction": { "choice": "A" }, "prompt_sha256": "7fb32e0fd4fc7490498a3b322716d502bc83af4be984e371808188e1a654a309", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-29T14:17:32.582321+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "769b7d28-1ec5-467e-8042-fb40ffe3d765", "result_sha256": "0bf2fdd34000f3a237fdeacce215de79dc13ce8834ccfcdf89188b6a92447e22", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00733, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "B" }, "group_id": "textcolumns-1-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-1-light-1.png", "input_tokens": 2545, "key_errors": null, "latency_sec": 1.9127576209139079, "model_name": "pareto", "output_tokens": 129, "prediction": { "choice": "B" }, "prompt_sha256": "9709598fa2c03afd3c8191b7a476b8aa1593f8488926311cbd6c9d5952354d67", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-29T14:17:46.577663+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "69858c0c-85d3-45cd-bcaa-30961879bc43", "result_sha256": "30142177781f9ea5a874e026c6c34eb7eaac658f4e1ad97fcfb2e6d22eb0968a", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0075325, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "B" }, "group_id": "textcolumns-1-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-1-dark-0.png", "input_tokens": 2545, "key_errors": null, "latency_sec": 2.103205655934289, "model_name": "pareto", "output_tokens": 156, "prediction": { "choice": "B" }, "prompt_sha256": "78899f74135f81158adab4e7446b61a7bda49f0ad810f8b42dad28a3640486f6", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-29T14:17:59.749607+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "878848c1-41e1-43e4-99dd-1cf1cca9a969", "result_sha256": "d931e14c42dcf212ae4f762a7561f7c9afb55fdbf9741c1480c2f00cbdbc9459", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0074275, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "B" }, "group_id": "textcolumns-1-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-1-dark-1.png", "input_tokens": 2545, "key_errors": null, "latency_sec": 2.1682536948937923, "model_name": "pareto", "output_tokens": 142, "prediction": { "choice": "B" }, "prompt_sha256": "9709598fa2c03afd3c8191b7a476b8aa1593f8488926311cbd6c9d5952354d67", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-29T14:17:38.665170+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "a112af1d-8549-4b5d-94c5-73436473e6ec", "result_sha256": "81609df8be786a75f63fc310aabed61b966a85033cef068a2ff2310f4ed1292c", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00661, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "C" }, "group_id": "textcolumns-2-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-2-light-0.png", "input_tokens": 2545, "key_errors": null, "latency_sec": 1.7989499571267515, "model_name": "pareto", "output_tokens": 33, "prediction": { "choice": "C" }, "prompt_sha256": "4888a14115f1b07fd0ac541d1950a02ffd9073681cd57eaacd9a4eb745a47694", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-29T14:17:50.958563+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "762788f4-46c0-4804-82cc-29636b49f4fc", "result_sha256": "35387998ed366860a2e96797e2fccf4ac6676d82f96228923c441fccd1f94279", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006595, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "B" }, "group_id": "textcolumns-2-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-2-light-1.png", "input_tokens": 2545, "key_errors": null, "latency_sec": 1.5949952921364456, "model_name": "pareto", "output_tokens": 31, "prediction": { "choice": "B" }, "prompt_sha256": "7ab8a26bbcac7d24004e4a74f0ea411de080ccc6cfed0989403acf58e56e0339", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-29T14:17:21.825276+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "2c6cceb9-7076-4882-adbe-1b8e6ee07ab0", "result_sha256": "22671387eacf8e83708194aff238c31297579364394c9a08f844f7abe4702e47", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006595, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "A" }, "group_id": "textcolumns-2-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-2-dark-0.png", "input_tokens": 2545, "key_errors": null, "latency_sec": 1.794996763812378, "model_name": "pareto", "output_tokens": 31, "prediction": { "choice": "A" }, "prompt_sha256": "78899f74135f81158adab4e7446b61a7bda49f0ad810f8b42dad28a3640486f6", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-29T14:17:58.984417+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "7ce0410b-4aa0-4437-8367-f506d28a13af", "result_sha256": "5ec7559a2e9142cacd272be5a73999b625f295e9d2b5b5b55967036b885c5d80", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00664, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "C" }, "group_id": "textcolumns-2-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-2-dark-1.png", "input_tokens": 2545, "key_errors": null, "latency_sec": 2.5256814828608185, "model_name": "pareto", "output_tokens": 37, "prediction": { "choice": "C" }, "prompt_sha256": "9709598fa2c03afd3c8191b7a476b8aa1593f8488926311cbd6c9d5952354d67", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-29T14:16:54.045429+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "a635abc3-f6c4-4874-8ede-dc4002eda393", "result_sha256": "39a9bf758dce2c76c41caca18916ca8addc3e8d403681ea4950e5c13fbabb779", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0067225, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "A" }, "group_id": "textcolumns-3-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-3-light-0.png", "input_tokens": 2545, "key_errors": null, "latency_sec": 1.9599044870119542, "model_name": "pareto", "output_tokens": 48, "prediction": { "choice": "A" }, "prompt_sha256": "7ab8a26bbcac7d24004e4a74f0ea411de080ccc6cfed0989403acf58e56e0339", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-29T14:17:12.693331+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "7d30a12f-8521-4107-88ab-27660b3947c0", "result_sha256": "1d7d031019d40d756ceb8ec8f6e50e9d93a713f2f5ce098b6b145840461aeead", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006835, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "A" }, "group_id": "textcolumns-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-3-light-1.png", "input_tokens": 2545, "key_errors": null, "latency_sec": 1.9135226719081402, "model_name": "pareto", "output_tokens": 63, "prediction": { "choice": "A" }, "prompt_sha256": "7ab8a26bbcac7d24004e4a74f0ea411de080ccc6cfed0989403acf58e56e0339", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-29T14:16:55.218339+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "a45df715-efe5-4ce4-baad-45d6555b5dea", "result_sha256": "8be7e6916d7b80df67eaeb6804edb233c3f819e07fd5e3a9275cfffa584ee57f", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006595, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "C" }, "group_id": "textcolumns-3-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-3-dark-0.png", "input_tokens": 2545, "key_errors": null, "latency_sec": 1.8905359450727701, "model_name": "pareto", "output_tokens": 31, "prediction": { "choice": "C" }, "prompt_sha256": "7fb32e0fd4fc7490498a3b322716d502bc83af4be984e371808188e1a654a309", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-29T14:17:44.029791+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "a01cee2f-7e3e-42f2-9dd2-513e2813f42d", "result_sha256": "d82597da8926f0feab9c834b97f8caa6bc0f81354512807b9f4b052d3ca4de16", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006745, "err": null, "error": null, "error_kind": null, "family": "textcolumns", "ground_truth": { "choice": "C" }, "group_id": "textcolumns-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/textcolumns-3-dark-1.png", "input_tokens": 2545, "key_errors": null, "latency_sec": 1.872590949991718, "model_name": "pareto", "output_tokens": 51, "prediction": { "choice": "C" }, "prompt_sha256": "78899f74135f81158adab4e7446b61a7bda49f0ad810f8b42dad28a3640486f6", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-29T14:17:44.632534+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-textcolumns-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "8d015355-529d-431b-91d6-121b6dd2e883", "result_sha256": "233caa1db9d68d96f31e184d3882f233a73564bbdb4a886544773b7261fda27f", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0067225, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-01", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-01.png", "input_tokens": 2566, "key_errors": null, "latency_sec": 1.8700051179621369, "model_name": "pareto", "output_tokens": 41, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-29T14:17:40.535488+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "8da1342f-e90e-402f-ae1b-efd1d3bd8703", "result_sha256": "ff7457da5ea0b5d222c9456d6df801ac6260107b03a92c8a302d1e5f8959d285", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0067, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-02", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-02.png", "input_tokens": 2566, "key_errors": null, "latency_sec": 1.7601971072144806, "model_name": "pareto", "output_tokens": 38, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-29T14:17:45.410614+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "0cdc1fb0-5e6a-4b97-93a9-779945b77604", "result_sha256": "0cef6fe730cf2755bfccd3961fa943b42cc02e32336ec5c9913dfe547e184c9a", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0067675, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-03", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-03.png", "input_tokens": 2566, "key_errors": null, "latency_sec": 1.827246906934306, "model_name": "pareto", "output_tokens": 47, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-29T14:17:53.219205+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "ff5ec6a6-71b9-45e9-8e79-53886f0d741d", "result_sha256": "b9aecf63a2e25578f4dc4d619aef7a8e82639f401a69aa22df78e0d5f9a0b79b", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0067525, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-04", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-04.png", "input_tokens": 2566, "key_errors": null, "latency_sec": 1.7966016689315438, "model_name": "pareto", "output_tokens": 45, "prediction": { "choice": "B" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-29T14:17:04.123399+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "0af64825-ea14-4b33-b8a8-ffe933f82f59", "result_sha256": "948e59febc499dba74dc1c2f4e973eb1d70c53cb915ca489d1ab8b82789af0d7", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0067225, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-05", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-05.png", "input_tokens": 2566, "key_errors": null, "latency_sec": 1.6422356590628624, "model_name": "pareto", "output_tokens": 41, "prediction": { "choice": "B" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-29T14:17:02.293549+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "981b910c-9031-4539-ada1-cd0c236ae4b5", "result_sha256": "782db790bcb9284cecb737b7a2d5f4894522e8fd09b58be85eb9650177f5f1ab", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0067225, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-06", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-06.png", "input_tokens": 2566, "key_errors": null, "latency_sec": 1.781356168910861, "model_name": "pareto", "output_tokens": 41, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-29T14:16:55.146899+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "5b15d4f2-43ce-4128-b967-c2c904694ccd", "result_sha256": "db328d637955bc0da84307bd7367a5bbe2b59ca2ddcff21587ca746f3b8c8902", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006775, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-07", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-07.png", "input_tokens": 2566, "key_errors": null, "latency_sec": 2.187021662015468, "model_name": "pareto", "output_tokens": 48, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-29T14:17:52.551247+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "4a32b2d6-a65d-4505-ab7e-36cb6c370217", "result_sha256": "faf281c4438dae2a9ca3fc838088889f187e8fa8dcf598f82c088df3e6e97f89", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0067525, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-08", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-08.png", "input_tokens": 2566, "key_errors": null, "latency_sec": 1.4852710850536823, "model_name": "pareto", "output_tokens": 45, "prediction": { "choice": "B" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-29T14:17:43.909242+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "52f9ca17-923c-4afe-9c55-11c9b3d97688", "result_sha256": "d9175981feebe62f53720828b7846ac9b83b2f3caad570c190c0c5d760bc7583", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0067, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-09", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-09.png", "input_tokens": 2566, "key_errors": null, "latency_sec": 1.7413938730023801, "model_name": "pareto", "output_tokens": 38, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-29T14:17:29.912096+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "a33f576c-049b-4d14-b04a-c84cbfde1500", "result_sha256": "eca9cc2c277f1dab8959d124bf442351578523381e712592990d78b116da0eb9", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0067525, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-10", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-10.png", "input_tokens": 2566, "key_errors": null, "latency_sec": 2.241855821106583, "model_name": "pareto", "output_tokens": 45, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-29T14:18:00.825674+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "f30e15b4-b4da-470b-b85e-ea7b57a2189f", "result_sha256": "2dcbb00680799536bab2a22d8ec92c5191f7dfb69f80d4ae974de92e729e301a", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0067525, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-11", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-11.png", "input_tokens": 2566, "key_errors": null, "latency_sec": 2.274386632023379, "model_name": "pareto", "output_tokens": 45, "prediction": { "choice": "B" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-29T14:16:54.004974+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "f2082d51-5bc1-400f-ad18-dc7218675055", "result_sha256": "a098fd0da6ee7b09677de4107c6adc967e32007e836848d7e7bfad15cda53af6", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0067525, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-12", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-12.png", "input_tokens": 2566, "key_errors": null, "latency_sec": 1.6304461141116917, "model_name": "pareto", "output_tokens": 45, "prediction": { "choice": "A" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-29T14:17:43.623784+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "8e73a5a9-7d8a-4413-88e3-0e1adba99f1a", "result_sha256": "aa21ccc3b0b99b2dfad5bdbf7c0da6d4cbc247edb21c76d0397f4400c30e249e", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0066775, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-13", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-13.png", "input_tokens": 2566, "key_errors": null, "latency_sec": 1.59511176077649, "model_name": "pareto", "output_tokens": 35, "prediction": { "choice": "A" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-29T14:18:04.204763+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-013", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "028bec69-346f-42ef-8ea2-f5519ced8217", "result_sha256": "55a7899dfe65a6c348aabb8d7e6cab664ad087a9e456047057176dee79b557d8", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00688, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-14", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-14.png", "input_tokens": 2566, "key_errors": null, "latency_sec": 1.6908708759583533, "model_name": "pareto", "output_tokens": 62, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-29T14:17:25.619836+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-014", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "6045a3a0-8144-452f-990c-30cad0d3f40e", "result_sha256": "4d7ba749725a4c8496511e59ad071e2d19e7104594185fa7b82b57ce4c193de5", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006685, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-15", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-15.png", "input_tokens": 2566, "key_errors": null, "latency_sec": 1.9638949721120298, "model_name": "pareto", "output_tokens": 36, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-29T14:17:48.541826+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-015", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "04611ff5-31a6-4c58-91f5-fdc3adad534b", "result_sha256": "4ba9042c7ffb6e30a76a1f83ccda552a932732145cc30d6017d10f8231a6370f", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006685, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-16", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-16.png", "input_tokens": 2566, "key_errors": null, "latency_sec": 1.7123219221830368, "model_name": "pareto", "output_tokens": 36, "prediction": { "choice": "A" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-29T14:17:33.679024+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-016", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "0db3fb03-3233-4776-b227-131ed4a932f3", "result_sha256": "95687f17707a167b1944e54be2027efb0ee33bb705d03ecf783eeac0aa31f919", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0068875, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-17", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-17.png", "input_tokens": 2566, "key_errors": null, "latency_sec": 1.9581228950992227, "model_name": "pareto", "output_tokens": 63, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-29T14:18:05.431804+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-017", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "a6679712-2b5d-492c-8d07-abd41a2c55b8", "result_sha256": "ffa1782f385eb18f36787dcca48fa419ef3e317e8e6c223efe17f5a214601863", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0067675, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-18", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-18.png", "input_tokens": 2566, "key_errors": null, "latency_sec": 1.6873109070584178, "model_name": "pareto", "output_tokens": 47, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-29T14:17:31.619401+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-018", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "1c89a56d-afc5-4ffc-9d9c-ceb91b9531fa", "result_sha256": "11f64c10e1649371207d2584b0c68561154f922cb1377e4eb164513915f151bc", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00667, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-19", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-19.png", "input_tokens": 2566, "key_errors": null, "latency_sec": 1.4752840739674866, "model_name": "pareto", "output_tokens": 34, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-29T14:17:47.602042+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-019", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "fbbc16f7-a657-4c8c-81a9-014ee735bb13", "result_sha256": "af13585f84972770fedeab2561f545e475f88511312620a3fbe7b34c29ae083a", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0067, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-20", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-20.png", "input_tokens": 2566, "key_errors": null, "latency_sec": 2.322535689920187, "model_name": "pareto", "output_tokens": 38, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-29T14:17:55.153438+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-020", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "8583f35a-68ca-4406-ad7d-330fee31b7dc", "result_sha256": "17b17d6de7072c32e3bf59ac82f2ed9248c3b880210a7de4b3dec25cf467c942", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0066775, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-21", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-21.png", "input_tokens": 2566, "key_errors": null, "latency_sec": 1.57088692788966, "model_name": "pareto", "output_tokens": 35, "prediction": { "choice": "A" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-29T14:16:59.499291+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-021", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "7c00c52c-9026-4e20-a8ed-e402d7629a78", "result_sha256": "b26cb995bc9f91914a2557d6708a2c4d90958b19ff2980bad8aab5d5ad30579f", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0066925, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-22", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-22.png", "input_tokens": 2566, "key_errors": null, "latency_sec": 1.5933829401619732, "model_name": "pareto", "output_tokens": 37, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-29T14:17:35.167911+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-022", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "399441a3-af7f-4f96-a9a4-54ce18b79c3f", "result_sha256": "e64a7999aed96336a3c52a1e693bdacb8f3f91623996340859b1f9ccba3b86cb", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006685, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "A" }, "group_id": "h-23", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-23.png", "input_tokens": 2566, "key_errors": null, "latency_sec": 1.653541567036882, "model_name": "pareto", "output_tokens": 36, "prediction": { "choice": "A" }, "prompt_sha256": "90e50f09ed7d57efcab7aba6471416a4bdc8cb107e32418092c5b602bdc15da6", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-29T14:17:42.854462+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-023", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "11579e47-39d8-4299-9b06-bdc5cd39883a", "result_sha256": "29c50cb3fa5d2480ec67d6898d1258ae6f42cf9124633eedf83074cf72fc47ab", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0067, "err": null, "error": null, "error_kind": null, "family": "topflow", "ground_truth": { "choice": "B" }, "group_id": "h-24", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/h-24.png", "input_tokens": 2566, "key_errors": null, "latency_sec": 1.385108107002452, "model_name": "pareto", "output_tokens": 38, "prediction": { "choice": "B" }, "prompt_sha256": "84269b77e267b7f9944b57d6e57a7fb5caa8600f5e9364c790073e8c2458c02e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-29T14:16:59.457682+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-topflow-024", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "499bcd73-4a4d-446b-9d25-b697a6539c0e", "result_sha256": "4b3f06d0efa29b137ecf11b9a65561242d2d78a94a114713c22624e5289b8d89", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0070225, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "B" }, "group_id": "widthcompare-first-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-first-light-0.png", "input_tokens": 2560, "key_errors": null, "latency_sec": 1.5687876418232918, "model_name": "pareto", "output_tokens": 83, "prediction": { "choice": "B" }, "prompt_sha256": "875f019d57875304ded0af8f075989118a0e4fcf9ca43c9eba1cda4f64a503a0", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-29T14:17:46.521437+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "fd46f4fd-362f-4ad2-a112-958346267fe7", "result_sha256": "0f828a29ed0d88fcea568e1155f5da5295e235b7810a00810559bf19af51e8a4", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00697, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "C" }, "group_id": "widthcompare-first-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-first-light-1.png", "input_tokens": 2560, "key_errors": null, "latency_sec": 1.610805595992133, "model_name": "pareto", "output_tokens": 76, "prediction": { "choice": "C" }, "prompt_sha256": "cd2c2f1f86d4ef1f19fa7f880569a29637d27eed1e41cdded418dc476e8498ed", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-29T14:17:43.723981+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "2a756d4b-8a56-45cb-b7ff-072a2735d8fa", "result_sha256": "5bc4a00f213947203a54209f39028830a9b0518c039cbe39468b42086466f852", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0071875, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "A" }, "group_id": "widthcompare-first-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-first-dark-0.png", "input_tokens": 2560, "key_errors": null, "latency_sec": 1.625844496069476, "model_name": "pareto", "output_tokens": 105, "prediction": { "choice": "A" }, "prompt_sha256": "756f6b3e431f583d190c4d469c2409153beb0d173c07c749e93a1af92d2e6f8e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-29T14:18:03.437865+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "d3625964-4eb7-44f7-8101-ba7b5d03c420", "result_sha256": "886b25343d0aa74bcc6493f6a7680ac3b142c11119b8f304fbabd035a9fae78e", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.007165, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "B" }, "group_id": "widthcompare-first-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-first-dark-1.png", "input_tokens": 2560, "key_errors": null, "latency_sec": 2.580907064024359, "model_name": "pareto", "output_tokens": 102, "prediction": { "choice": "B" }, "prompt_sha256": "875f019d57875304ded0af8f075989118a0e4fcf9ca43c9eba1cda4f64a503a0", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-29T14:17:49.130866+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "6d2e6698-3add-425e-b591-49ca6d04dcfc", "result_sha256": "6066fc7fd0003de36f52512a13c0c5b913cf1c770cc9f957d662eb7d3fc2c4dd", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.007045, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "B" }, "group_id": "widthcompare-last-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-last-light-0.png", "input_tokens": 2560, "key_errors": null, "latency_sec": 2.125079007819295, "model_name": "pareto", "output_tokens": 86, "prediction": { "choice": "B" }, "prompt_sha256": "756f6b3e431f583d190c4d469c2409153beb0d173c07c749e93a1af92d2e6f8e", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-29T14:17:38.313992+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "d7b78109-6646-402f-b0cb-ba5427791a75", "result_sha256": "54e12a66ed774202d097e094b09e1a5a340eef61a29473503a6dea143d18b69e", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0071875, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "A" }, "group_id": "widthcompare-last-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-last-light-1.png", "input_tokens": 2560, "key_errors": null, "latency_sec": 1.5955105340108275, "model_name": "pareto", "output_tokens": 105, "prediction": { "choice": "A" }, "prompt_sha256": "807b859493212c8c914db2b1d9395b00171e3c91df87b66b9663b6cc7b65c485", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-29T14:18:05.107540+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "e3ddc425-6e29-4720-889f-50e09a7f5a31", "result_sha256": "9faeccf1b237351f9f483947f12772d043b09aa59751bc24db262fa2527d3f79", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0070675, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "C" }, "group_id": "widthcompare-last-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-last-dark-0.png", "input_tokens": 2560, "key_errors": null, "latency_sec": 1.5063107998576015, "model_name": "pareto", "output_tokens": 89, "prediction": { "choice": "C" }, "prompt_sha256": "875f019d57875304ded0af8f075989118a0e4fcf9ca43c9eba1cda4f64a503a0", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-29T14:16:54.517053+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "d0afe608-9c15-4dec-8c95-96906bd47b40", "result_sha256": "f77addfcad876c15e164fddd99bfbdd454719360c655003f4fdaa6228f8e4bab", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0072175, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "A" }, "group_id": "widthcompare-last-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-last-dark-1.png", "input_tokens": 2560, "key_errors": null, "latency_sec": 1.801532695069909, "model_name": "pareto", "output_tokens": 109, "prediction": { "choice": "A" }, "prompt_sha256": "cd2c2f1f86d4ef1f19fa7f880569a29637d27eed1e41cdded418dc476e8498ed", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-29T14:16:59.670323+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "6b1767c4-159a-4f7e-89a9-754452d0d176", "result_sha256": "a612eec4185a48524ea384d25bfde6ee312db98e5b0e2f027083497eb9773729", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0077725, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "B" }, "group_id": "widthcompare-equal-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-equal-light-0.png", "input_tokens": 2560, "key_errors": null, "latency_sec": 2.2953475201502442, "model_name": "pareto", "output_tokens": 183, "prediction": { "choice": "B" }, "prompt_sha256": "cd2c2f1f86d4ef1f19fa7f880569a29637d27eed1e41cdded418dc476e8498ed", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-29T14:17:33.434037+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "a0208384-ef0c-4c49-b069-2ce0a704b38d", "result_sha256": "edcf4a51315584c0ad9b228a04ad327573046d67bb06b676bc394732812ee280", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0075925, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "A" }, "group_id": "widthcompare-equal-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-equal-light-1.png", "input_tokens": 2560, "key_errors": null, "latency_sec": 2.4131321778986603, "model_name": "pareto", "output_tokens": 159, "prediction": { "choice": "A" }, "prompt_sha256": "875f019d57875304ded0af8f075989118a0e4fcf9ca43c9eba1cda4f64a503a0", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-29T14:17:57.375358+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "380106bc-c026-4c6c-bd61-7b823b661b37", "result_sha256": "49b343747b9fa0fb0fd69cb6c5c7be4197662ef2695324c3181cb70ae4ca2b25", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00772, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "C" }, "group_id": "widthcompare-equal-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-equal-dark-0.png", "input_tokens": 2560, "key_errors": null, "latency_sec": 2.4650111000519246, "model_name": "pareto", "output_tokens": 176, "prediction": { "choice": "C" }, "prompt_sha256": "807b859493212c8c914db2b1d9395b00171e3c91df87b66b9663b6cc7b65c485", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-29T14:17:36.166156+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "c4096bf5-d28e-445f-9aca-32b9b5aff575", "result_sha256": "360b478332cefb16b86cf288477b4505c85a839dfeaad4cfa8e0b6396235b6de", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0076225, "err": null, "error": null, "error_kind": null, "family": "widthcompare", "ground_truth": { "choice": "C" }, "group_id": "widthcompare-equal-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/widthcompare-equal-dark-1.png", "input_tokens": 2560, "key_errors": null, "latency_sec": 1.8592016680631787, "model_name": "pareto", "output_tokens": 163, "prediction": { "choice": "C" }, "prompt_sha256": "807b859493212c8c914db2b1d9395b00171e3c91df87b66b9663b6cc7b65c485", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-29T14:16:55.872183+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-widthcompare-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "d61be90b-6daa-4ca6-ab0a-5426336bfc0a", "result_sha256": "2be923692fd7c3b7bc26b88628d5a3c64f0821ca79d9c155e99e2dcd18ac4314", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00798, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "B" }, "group_id": "wrapalign-left-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-left-light-0.png", "input_tokens": 2568, "key_errors": null, "latency_sec": 2.491852676961571, "model_name": "pareto", "output_tokens": 208, "prediction": { "choice": "B" }, "prompt_sha256": "eaad4b2f9e91b0501c9e0bbd973bee743f4eec05e7ad65bdddca7302eaa806eb", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-29T14:16:51.388566+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "7659aef3-e69c-4c9b-9d3b-c75910cb49ef", "result_sha256": "fd3f554f99ca65316a156993d61578892a27841a151012aaa16095549abb3691", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0074775, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "C" }, "group_id": "wrapalign-left-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-left-light-1.png", "input_tokens": 2568, "key_errors": null, "latency_sec": 1.8943017560523003, "model_name": "pareto", "output_tokens": 141, "prediction": { "choice": "C" }, "prompt_sha256": "3df695a5de14c72a5a5dd88e4cced5a400dc2307a605c97919c60104bfb7a338", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-29T14:17:35.350693+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "ff10ef06-3ce9-4312-a6f6-4320411298ef", "result_sha256": "34d43d150acb42ff5896209d377c5804471c9382906871d94f8b049c8b8c7192", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0084675, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "A" }, "group_id": "wrapalign-left-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-left-dark-0.png", "input_tokens": 2568, "key_errors": null, "latency_sec": 2.4887428351212293, "model_name": "pareto", "output_tokens": 273, "prediction": { "choice": "A" }, "prompt_sha256": "71d9f4106dcceb21abcbdbcb7978862ff824dca3aef4bc9d0cc134a9007d7be3", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-29T14:17:28.128891+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "74b9b61b-1c31-484d-9ae9-399dbbffced9", "result_sha256": "3cc2a9a385610064424ca9bd435bb9482011b9798e9b52cc93a2b8b083b8be07", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0078675, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "A" }, "group_id": "wrapalign-left-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-left-dark-1.png", "input_tokens": 2568, "key_errors": null, "latency_sec": 2.5824706580024213, "model_name": "pareto", "output_tokens": 193, "prediction": { "choice": "A" }, "prompt_sha256": "4f5e9513c600f9a4c3e41e06f8b5947480ecacc47063f687d8ace9fa210713f1", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-29T14:17:15.545943+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "08950d49-41fa-47b2-8912-7e2892d1a6c6", "result_sha256": "a663b1616bbfd8f908c4559607e37b1ac43d698098cd7537667e744476ea5304", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.007605, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "A" }, "group_id": "wrapalign-center-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-center-light-0.png", "input_tokens": 2568, "key_errors": null, "latency_sec": 2.156712754862383, "model_name": "pareto", "output_tokens": 158, "prediction": { "choice": "A" }, "prompt_sha256": "3df695a5de14c72a5a5dd88e4cced5a400dc2307a605c97919c60104bfb7a338", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-29T14:18:01.938903+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "43926545-fa95-4489-aa6e-762f62b90ee3", "result_sha256": "90a1d6675493805d0dd1d41e4a97a72310847920411b7b4ffdead2f834bae605", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00708, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "B" }, "group_id": "wrapalign-center-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-center-light-1.png", "input_tokens": 2568, "key_errors": null, "latency_sec": 1.6822827658616006, "model_name": "pareto", "output_tokens": 88, "prediction": { "choice": "B" }, "prompt_sha256": "71d9f4106dcceb21abcbdbcb7978862ff824dca3aef4bc9d0cc134a9007d7be3", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-29T14:17:03.153638+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "5b7049af-5df2-4d35-a24d-3d179643c1ad", "result_sha256": "cc8b2d45adf86d6c64b58f121a5448c0b7395ab05c0471fd87c92ec7410ebd0f", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0074625, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "B" }, "group_id": "wrapalign-center-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-center-dark-0.png", "input_tokens": 2568, "key_errors": null, "latency_sec": 1.6495219201315194, "model_name": "pareto", "output_tokens": 139, "prediction": { "choice": "B" }, "prompt_sha256": "6f9f40796910da778fa5f30bd2cb43da95875ca8d6fb47db39f42ffb70cbab76", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-29T14:16:58.062637+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "3906b3d4-86db-4a5f-b0a1-7e9a53ca755b", "result_sha256": "28001637b521ed5cee695bfb489b6897246292c4f70c0d8bd1f93a699395d7e9", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00756, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "C" }, "group_id": "wrapalign-center-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-center-dark-1.png", "input_tokens": 2568, "key_errors": null, "latency_sec": 2.9206901909783483, "model_name": "pareto", "output_tokens": 152, "prediction": { "choice": "C" }, "prompt_sha256": "4f5e9513c600f9a4c3e41e06f8b5947480ecacc47063f687d8ace9fa210713f1", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-29T14:16:57.858589+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "09c6092e-58ad-4c93-b036-ac068f7b849f", "result_sha256": "148d114a19ccc1af3e17353c308b1327ad6d8e606f462180f81439065e1c2a07", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00768, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "B" }, "group_id": "wrapalign-right-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-right-light-0.png", "input_tokens": 2568, "key_errors": null, "latency_sec": 3.2962478541303426, "model_name": "pareto", "output_tokens": 168, "prediction": { "choice": "B" }, "prompt_sha256": "3df695a5de14c72a5a5dd88e4cced5a400dc2307a605c97919c60104bfb7a338", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-29T14:17:55.076405+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "81f5cfe5-a502-409e-badd-0604db72d5d7", "result_sha256": "aca79426468471856d1dd774ef4b7511316f0aba05d05f869ac9fa30676638e1", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.008175, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "C" }, "group_id": "wrapalign-right-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-right-light-1.png", "input_tokens": 2568, "key_errors": null, "latency_sec": 2.1771751458290964, "model_name": "pareto", "output_tokens": 234, "prediction": { "choice": "C" }, "prompt_sha256": "be68205720b4630afe6748f490c25dd02332bab19b459ce5a69565f78da84e8a", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-29T14:17:01.458925+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "710ff21b-060d-4b4c-9ba9-498634f80ad6", "result_sha256": "e320ca9027927520055cea40751a538b020615719c3d3f02b7f8459d86f97b5e", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0086325, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "C" }, "group_id": "wrapalign-right-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-right-dark-0.png", "input_tokens": 2568, "key_errors": null, "latency_sec": 2.4915586579591036, "model_name": "pareto", "output_tokens": 295, "prediction": { "choice": "C" }, "prompt_sha256": "71d9f4106dcceb21abcbdbcb7978862ff824dca3aef4bc9d0cc134a9007d7be3", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-29T14:17:12.494397+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "cddf4388-7b19-4f8f-9cfc-96cd7f7d33e6", "result_sha256": "eaa931abd8e05c1da7f098626b0c71f36f8c4275c2173709294d067d2e262e75", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.008175, "err": null, "error": null, "error_kind": null, "family": "wrapalign", "ground_truth": { "choice": "A" }, "group_id": "wrapalign-right-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapalign-right-dark-1.png", "input_tokens": 2568, "key_errors": null, "latency_sec": 2.265217151027173, "model_name": "pareto", "output_tokens": 234, "prediction": { "choice": "A" }, "prompt_sha256": "6f9f40796910da778fa5f30bd2cb43da95875ca8d6fb47db39f42ffb70cbab76", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-29T14:18:03.559274+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapalign-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "df954a50-52e2-4ca9-962f-3c508af5e408", "result_sha256": "b800c21dc1babd75f5c7a9153b11a2561337e5a87c8d6fbe33a0a7a9481e1806", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006755, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "A" }, "group_id": "wrapcount-1-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-1-light-0.png", "input_tokens": 2546, "key_errors": null, "latency_sec": 1.5577019138727337, "model_name": "pareto", "output_tokens": 52, "prediction": { "choice": "A" }, "prompt_sha256": "15d8bd94924cd0eb8efdec17da21c6da4f97b10dd9a51e2d28e533f308413109", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-29T14:17:16.550247+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-001", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "b6fec9af-ee55-4447-87c2-80da8f7e0dda", "result_sha256": "8c994f250b679e9f235a4e3868be8276d9aec924dd3e1c5f79fc18f0cfca4ec3", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0067925, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "A" }, "group_id": "wrapcount-1-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-1-light-1.png", "input_tokens": 2546, "key_errors": null, "latency_sec": 1.5533016379922628, "model_name": "pareto", "output_tokens": 57, "prediction": { "choice": "A" }, "prompt_sha256": "d5a11da43aa9608fe90b20a74d3d14e9dbf3731c33be73ca93aad96c927ccc7d", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-29T14:16:56.403416+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-002", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "f790a5e0-27c0-44bf-ab08-5a4c8378c865", "result_sha256": "159d343c9926e79fd4906e1a730a42b74527a454008e81dcb85dcef5dddd30ce", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0067925, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "A" }, "group_id": "wrapcount-1-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-1-dark-0.png", "input_tokens": 2546, "key_errors": null, "latency_sec": 1.5618619991000742, "model_name": "pareto", "output_tokens": 57, "prediction": { "choice": "A" }, "prompt_sha256": "d5a11da43aa9608fe90b20a74d3d14e9dbf3731c33be73ca93aad96c927ccc7d", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-29T14:17:15.140654+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-003", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "831740c8-dd46-46ff-8a72-8ed337c1feb8", "result_sha256": "02eeb2f5ffb382b91ae7e87bd2eb4f83b63e53ee16602fda528d12d17c40daa7", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0075425, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "B" }, "group_id": "wrapcount-1-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-1-dark-1.png", "input_tokens": 2546, "key_errors": null, "latency_sec": 1.7209018000867218, "model_name": "pareto", "output_tokens": 157, "prediction": { "choice": "B" }, "prompt_sha256": "f426cc4a860aebf2f83ce2ed8bb35c244b740a3f0e0196bacf7e87e4c01825c0", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-29T14:17:59.080979+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-004", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "fec7ceb1-0412-47ab-a77a-53f366fb636b", "result_sha256": "30dd30a78cc54d1beb49ec81c38ff99cba49a97a8015b68ae099979c6111e3f8", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0068975, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "C" }, "group_id": "wrapcount-2-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-2-light-0.png", "input_tokens": 2546, "key_errors": null, "latency_sec": 1.5242184379603714, "model_name": "pareto", "output_tokens": 71, "prediction": { "choice": "C" }, "prompt_sha256": "e0ef52abca5ee7a42abae5e3fabccb2a95599a04ff3d41502af290e06d4711bd", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-29T14:17:23.369649+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-005", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "286a93fc-4b4f-4be9-b5e4-300bb6a42d61", "result_sha256": "37fb116fc0cd83b44803a3553b83cdf4ec9e68763e75c4745d31a5d9f54eaf20", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006845, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "C" }, "group_id": "wrapcount-2-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-2-light-1.png", "input_tokens": 2546, "key_errors": null, "latency_sec": 1.5227400579024106, "model_name": "pareto", "output_tokens": 64, "prediction": { "choice": "C" }, "prompt_sha256": "e0ef52abca5ee7a42abae5e3fabccb2a95599a04ff3d41502af290e06d4711bd", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-29T14:17:11.844755+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-006", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "84bea602-d614-4905-a602-ef258ef45b0c", "result_sha256": "362a41fc5e384f0eda543da64f68edb99d505cfd9608f961144ff2053a8fb894", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006845, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "B" }, "group_id": "wrapcount-2-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-2-dark-0.png", "input_tokens": 2546, "key_errors": null, "latency_sec": 1.5166279340628535, "model_name": "pareto", "output_tokens": 64, "prediction": { "choice": "B" }, "prompt_sha256": "d5a11da43aa9608fe90b20a74d3d14e9dbf3731c33be73ca93aad96c927ccc7d", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-29T14:16:55.570021+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-007", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "b1c4c090-5e65-4368-aacc-dbf7f80acd3d", "result_sha256": "c1a8ba0db2578f4918c27c14d11d2b6d152d1abfc8a5c75284662da266729f6f", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.00692, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "A" }, "group_id": "wrapcount-2-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-2-dark-1.png", "input_tokens": 2546, "key_errors": null, "latency_sec": 1.8786774198524654, "model_name": "pareto", "output_tokens": 74, "prediction": { "choice": "A" }, "prompt_sha256": "f426cc4a860aebf2f83ce2ed8bb35c244b740a3f0e0196bacf7e87e4c01825c0", "provider": "anthropic", "raw_prediction": "{\"choice\": \"A\"}", "recorded_at": "2026-09-29T14:16:54.929259+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-008", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "f5869d1f-aa70-484f-aea9-f8554c6da078", "result_sha256": "aee5e742e0ca8808bbd105edc1d711833d29db4776ba60b4d9233b8ad3423b00", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0068525, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "B" }, "group_id": "wrapcount-3-light-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-3-light-0.png", "input_tokens": 2546, "key_errors": null, "latency_sec": 1.508646213915199, "model_name": "pareto", "output_tokens": 65, "prediction": { "choice": "B" }, "prompt_sha256": "a879d0677f7b7d6bd13a35743cb1a9461192ed1362a649bede5b7da8c73785da", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-29T14:17:08.071506+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-009", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "fe0adaab-c37d-4522-a96b-465c43f17a90", "result_sha256": "1374a2822d1d4d6cc13a0e6f9ae639bfff238dd3e0c10f1a338d07f4ec9b5780", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.006845, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "C" }, "group_id": "wrapcount-3-light-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-3-light-1.png", "input_tokens": 2546, "key_errors": null, "latency_sec": 1.6703446209430695, "model_name": "pareto", "output_tokens": 64, "prediction": { "choice": "C" }, "prompt_sha256": "d5a11da43aa9608fe90b20a74d3d14e9dbf3731c33be73ca93aad96c927ccc7d", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-29T14:17:51.200882+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-010", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "e4174119-5ef8-4ec5-addf-5fd515a10fbd", "result_sha256": "72e8a6ddaea78bb736e38c71be81e79019b3b2a21ac1f99819e964be93a1d243", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0068525, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "C" }, "group_id": "wrapcount-3-dark-0", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-3-dark-0.png", "input_tokens": 2546, "key_errors": null, "latency_sec": 1.5159034670796245, "model_name": "pareto", "output_tokens": 65, "prediction": { "choice": "C" }, "prompt_sha256": "d5a11da43aa9608fe90b20a74d3d14e9dbf3731c33be73ca93aad96c927ccc7d", "provider": "anthropic", "raw_prediction": "{\"choice\": \"C\"}", "recorded_at": "2026-09-29T14:17:27.025873+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-011", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } }, { "model_id": "pareto", "status": "final", "included_in_comparison": true, "source_attempt_id": "f40db26f-baa2-4b98-b8ea-bb64b0bb2b4c", "result_sha256": "1717a50fb9c1139c71acf272f1cb191d50d80f1ee6a56a46dd9c8f84d3180a38", "result": { "correct": true, "cost_estimated": false, "cost_usd": 0.0068375, "err": null, "error": null, "error_kind": null, "family": "wrapcount", "ground_truth": { "choice": "B" }, "group_id": "wrapcount-3-dark-1", "image_path": "/mnt/disks/data/layoutbench/dataset/layoutbench-v0.3/wrapcount-3-dark-1.png", "input_tokens": 2546, "key_errors": null, "latency_sec": 1.7878159550018609, "model_name": "pareto", "output_tokens": 63, "prediction": { "choice": "B" }, "prompt_sha256": "15d8bd94924cd0eb8efdec17da21c6da4f97b10dd9a51e2d28e533f308413109", "provider": "anthropic", "raw_prediction": "{\"choice\": \"B\"}", "recorded_at": "2026-09-29T14:16:53.297271+00:00", "request_attempts": 1, "score": 100, "task_id": "layoutbench-wrapcount-012", "tight_score": null, "unmetered_attempts": 0, "valid": true, "within_bands": null } } ] }