{ "schema_version": 1, "dataset_git_commit": "70a514f06f9e156be480e2a0b28a1d3380467ace", "dataset_fingerprint": "fc531bdb2e7c9d10e2074519bd3e0d0f66bc8ec970d20f25cdb628054784c6de", "expected_task_count": 292, "run_id": "qualitative-20260917", "mock": false, "grading_version": "1", "release": "0.3.0", "evaluation_protocol": "deed0698dca65b5e24f3b06766d2d5344bbf0964e871167cebfc5e4aaba554fa", "finalized_at": "2026-09-17T12:10:52.016183+00:00", "source_updated_at": "2026-09-17T12:10:06.516568+00:00", "source_git_commit": "bd677173c19051bb9316d2d7075d2a931b8bacc3", "finalizer_git_commit": "bd677173c19051bb9316d2d7075d2a931b8bacc3", "finalizer_git_dirty": false, "artifact_type": "layoutbench-finalization", "publication_status": "finalized", "comparison": { "scope": "full", "count": 292, "task_ids": [ "layoutbench-blockalign-001", "layoutbench-blockalign-002", "layoutbench-blockalign-003", "layoutbench-blockalign-004", "layoutbench-blockalign-005", "layoutbench-blockalign-006", "layoutbench-blockalign-007", "layoutbench-blockalign-008", "layoutbench-blockalign-009", "layoutbench-blockalign-010", "layoutbench-blockalign-011", "layoutbench-blockalign-012", "layoutbench-crossalign-001", "layoutbench-crossalign-002", "layoutbench-crossalign-003", "layoutbench-crossalign-004", "layoutbench-crossalign-005", "layoutbench-crossalign-006", "layoutbench-crossalign-007", "layoutbench-crossalign-008", "layoutbench-crossalign-009", "layoutbench-crossalign-010", "layoutbench-crossalign-011", "layoutbench-crossalign-012", "layoutbench-crossalign-013", "layoutbench-crossalign-014", "layoutbench-crossalign-015", "layoutbench-crossalign-016", "layoutbench-direction-001", "layoutbench-direction-002", "layoutbench-direction-003", "layoutbench-direction-004", "layoutbench-direction-005", "layoutbench-direction-006", "layoutbench-direction-007", "layoutbench-direction-008", "layoutbench-distribution-001", "layoutbench-distribution-002", "layoutbench-distribution-003", "layoutbench-distribution-004", "layoutbench-distribution-005", "layoutbench-distribution-006", "layoutbench-distribution-007", "layoutbench-distribution-008", "layoutbench-distribution-009", "layoutbench-distribution-010", "layoutbench-distribution-011", "layoutbench-distribution-012", "layoutbench-distribution-013", "layoutbench-distribution-014", "layoutbench-distribution-015", "layoutbench-distribution-016", "layoutbench-distribution-017", "layoutbench-distribution-018", "layoutbench-distribution-019", "layoutbench-distribution-020", "layoutbench-distribution-021", "layoutbench-distribution-022", "layoutbench-distribution-023", "layoutbench-distribution-024", "layoutbench-gapcompare-001", "layoutbench-gapcompare-002", "layoutbench-gapcompare-003", "layoutbench-gapcompare-004", "layoutbench-gapcompare-005", "layoutbench-gapcompare-006", "layoutbench-gapcompare-007", "layoutbench-gapcompare-008", "layoutbench-gapcompare-009", "layoutbench-gapcompare-010", "layoutbench-gapcompare-011", "layoutbench-gapcompare-012", "layoutbench-gridcols-001", "layoutbench-gridcols-002", "layoutbench-gridcols-003", "layoutbench-gridcols-004", "layoutbench-gridcols-005", "layoutbench-gridcols-006", "layoutbench-gridcols-007", "layoutbench-gridcols-008", "layoutbench-gridcols-009", "layoutbench-gridcols-010", "layoutbench-gridcols-011", "layoutbench-gridcols-012", "layoutbench-gridgaps-001", "layoutbench-gridgaps-002", "layoutbench-gridgaps-003", "layoutbench-gridgaps-004", "layoutbench-gridgaps-005", "layoutbench-gridgaps-006", "layoutbench-gridgaps-007", "layoutbench-gridgaps-008", "layoutbench-gridgaps-009", "layoutbench-gridgaps-010", "layoutbench-gridgaps-011", "layoutbench-gridgaps-012", "layoutbench-gridrows-001", "layoutbench-gridrows-002", "layoutbench-gridrows-003", "layoutbench-gridrows-004", "layoutbench-gridrows-005", "layoutbench-gridrows-006", "layoutbench-gridrows-007", "layoutbench-gridrows-008", "layoutbench-gridrows-009", "layoutbench-gridrows-010", "layoutbench-gridrows-011", "layoutbench-gridrows-012", "layoutbench-gridspan-001", "layoutbench-gridspan-002", "layoutbench-gridspan-003", "layoutbench-gridspan-004", "layoutbench-gridspan-005", "layoutbench-gridspan-006", "layoutbench-gridspan-007", "layoutbench-gridspan-008", "layoutbench-gridspan-009", "layoutbench-gridspan-010", "layoutbench-gridspan-011", "layoutbench-gridspan-012", "layoutbench-gridtracks-001", "layoutbench-gridtracks-002", "layoutbench-gridtracks-003", "layoutbench-gridtracks-004", "layoutbench-gridtracks-005", "layoutbench-gridtracks-006", "layoutbench-gridtracks-007", "layoutbench-gridtracks-008", "layoutbench-gridtracks-009", "layoutbench-gridtracks-010", "layoutbench-gridtracks-011", "layoutbench-gridtracks-012", "layoutbench-groupgap-001", "layoutbench-groupgap-002", "layoutbench-groupgap-003", "layoutbench-groupgap-004", "layoutbench-groupgap-005", "layoutbench-groupgap-006", "layoutbench-groupgap-007", "layoutbench-groupgap-008", "layoutbench-groupgap-009", "layoutbench-groupgap-010", "layoutbench-groupgap-011", "layoutbench-groupgap-012", "layoutbench-nestedflow-001", "layoutbench-nestedflow-002", "layoutbench-nestedflow-003", "layoutbench-nestedflow-004", "layoutbench-nestedflow-005", "layoutbench-nestedflow-006", "layoutbench-nestedflow-007", "layoutbench-nestedflow-008", "layoutbench-nestedflow-009", "layoutbench-nestedflow-010", "layoutbench-nestedflow-011", "layoutbench-nestedflow-012", "layoutbench-nestedflow-013", "layoutbench-nestedflow-014", "layoutbench-nestedflow-015", "layoutbench-nestedflow-016", "layoutbench-nestedflow-017", "layoutbench-nestedflow-018", "layoutbench-nestedflow-019", "layoutbench-nestedflow-020", "layoutbench-nestedflow-021", "layoutbench-nestedflow-022", "layoutbench-nestedflow-023", "layoutbench-nestedflow-024", "layoutbench-nestedwrap-001", "layoutbench-nestedwrap-002", "layoutbench-nestedwrap-003", "layoutbench-nestedwrap-004", "layoutbench-nestedwrap-005", "layoutbench-nestedwrap-006", "layoutbench-nestedwrap-007", "layoutbench-nestedwrap-008", "layoutbench-nestedwrap-009", "layoutbench-nestedwrap-010", "layoutbench-nestedwrap-011", "layoutbench-nestedwrap-012", "layoutbench-nestedwrap-013", "layoutbench-nestedwrap-014", "layoutbench-nestedwrap-015", "layoutbench-nestedwrap-016", "layoutbench-nestedwrap-017", "layoutbench-nestedwrap-018", "layoutbench-nestedwrap-019", "layoutbench-nestedwrap-020", "layoutbench-nestedwrap-021", "layoutbench-nestedwrap-022", "layoutbench-nestedwrap-023", "layoutbench-nestedwrap-024", "layoutbench-padcompare-001", "layoutbench-padcompare-002", "layoutbench-padcompare-003", "layoutbench-padcompare-004", "layoutbench-padcompare-005", "layoutbench-padcompare-006", "layoutbench-padcompare-007", "layoutbench-padcompare-008", "layoutbench-padcompare-009", "layoutbench-padcompare-010", "layoutbench-padcompare-011", "layoutbench-padcompare-012", "layoutbench-textalign-001", "layoutbench-textalign-002", "layoutbench-textalign-003", "layoutbench-textalign-004", "layoutbench-textalign-005", "layoutbench-textalign-006", "layoutbench-textalign-007", "layoutbench-textalign-008", "layoutbench-textalign-009", "layoutbench-textalign-010", "layoutbench-textalign-011", "layoutbench-textalign-012", "layoutbench-textalign-013", "layoutbench-textalign-014", "layoutbench-textalign-015", "layoutbench-textalign-016", "layoutbench-textcolumns-001", "layoutbench-textcolumns-002", "layoutbench-textcolumns-003", "layoutbench-textcolumns-004", "layoutbench-textcolumns-005", "layoutbench-textcolumns-006", "layoutbench-textcolumns-007", "layoutbench-textcolumns-008", "layoutbench-textcolumns-009", "layoutbench-textcolumns-010", "layoutbench-textcolumns-011", "layoutbench-textcolumns-012", "layoutbench-topflow-001", "layoutbench-topflow-002", "layoutbench-topflow-003", "layoutbench-topflow-004", "layoutbench-topflow-005", "layoutbench-topflow-006", "layoutbench-topflow-007", "layoutbench-topflow-008", "layoutbench-topflow-009", "layoutbench-topflow-010", "layoutbench-topflow-011", "layoutbench-topflow-012", "layoutbench-topflow-013", "layoutbench-topflow-014", "layoutbench-topflow-015", "layoutbench-topflow-016", "layoutbench-topflow-017", "layoutbench-topflow-018", "layoutbench-topflow-019", "layoutbench-topflow-020", "layoutbench-topflow-021", "layoutbench-topflow-022", "layoutbench-topflow-023", "layoutbench-topflow-024", "layoutbench-widthcompare-001", "layoutbench-widthcompare-002", "layoutbench-widthcompare-003", "layoutbench-widthcompare-004", "layoutbench-widthcompare-005", "layoutbench-widthcompare-006", "layoutbench-widthcompare-007", "layoutbench-widthcompare-008", "layoutbench-widthcompare-009", "layoutbench-widthcompare-010", "layoutbench-widthcompare-011", "layoutbench-widthcompare-012", "layoutbench-wrapalign-001", "layoutbench-wrapalign-002", "layoutbench-wrapalign-003", "layoutbench-wrapalign-004", "layoutbench-wrapalign-005", "layoutbench-wrapalign-006", "layoutbench-wrapalign-007", "layoutbench-wrapalign-008", "layoutbench-wrapalign-009", "layoutbench-wrapalign-010", "layoutbench-wrapalign-011", "layoutbench-wrapalign-012", "layoutbench-wrapcount-001", "layoutbench-wrapcount-002", "layoutbench-wrapcount-003", "layoutbench-wrapcount-004", "layoutbench-wrapcount-005", "layoutbench-wrapcount-006", "layoutbench-wrapcount-007", "layoutbench-wrapcount-008", "layoutbench-wrapcount-009", "layoutbench-wrapcount-010", "layoutbench-wrapcount-011", "layoutbench-wrapcount-012" ], "cohort_fingerprint": "56e2fb761613c5caa49412d9ea2c156fe82287109203dd73d974c23290fb64da", "model_ids": [ "claude-fable-5-1", "claude-haiku-4-5-20251001", "claude-opus-5", "claude-sonnet-5", "gemini-3.1-pro-preview", "gemini-3.5-flash-lite", "gemini-3.8-flash", "gpt-5.6-luna", "gpt-5.6-sol", "gpt-5.6-terra", "gpt-6-astra", "muse-spark-1.2", "muse-spark-1.3" ] }, "artifact_sha256": { "attempts.jsonl": "3a8b413e9ff5f1285d4d6e659b5fec5692672593830e87bf248cbb142438a362", "final_results.json": "e5686b10f3040e44eed007cbf860a5170d9e0e97edd6b7aa222bac37d2f94938", "run.json": "1cce46d4940216d7c973603cd32c6d5eb6e856954c851aaf9df4a21b321a1e1f", "scorecard_claude-fable-5-1.json": "68e63e06b6cc01d44cd238445eb70a738af59d26835e85dd3af5e8a0c9b7989c", "scorecard_claude-haiku-4-5-20251001.json": "5211202f1b8a2d7ecc971550e30cc9e1646e0b6a7452f9a702952c5088b3eb74", "scorecard_claude-opus-5.json": "3b31acc2ae47dd4ea7d9ab9365432d97df9452abd99354f8d2455cea56a9ce96", "scorecard_claude-sonnet-5.json": "a6659e5059c5cc82e2caa4819a3ac2ed46a76df104fb61668f5b0708635a27ec", "scorecard_gemini-3.1-pro-preview.json": "0eddb2ef9306a19cd8002fbc9fa812f7d3e03f052c9d9a5383ef83701ba0cd43", "scorecard_gemini-3.5-flash-lite.json": "87ed57c3191768867a169eea406d16d479db248531c9f6bd36cb29fea21a363f", "scorecard_gemini-3.8-flash.json": "626f3afa7acf574c7a041acb931fc8532f6da2028cacaf12b9f0619d9ae2186b", "scorecard_gpt-5.6-luna.json": "2a40472cf15fe0a656d39b343f15afc623ed040731cd346843f76762af323e5b", "scorecard_gpt-5.6-sol.json": "ab3f639686326aa9fa8b6fefdd79b06ab140288720b06ef42bb7812c081e3a69", "scorecard_gpt-5.6-terra.json": "64155e964b5d4ec24d2dadbb5a785b37fa9c1dbc4358408337b2ac9fce282636", "scorecard_gpt-6-astra.json": "adb248511fe0ecdb6b3d9f37f9ce0af011a7d687efe9ee52a563d5ad0beb4b26", "scorecard_muse-spark-1.2.json": "2cb5f52cf51d296fe3aa02075c66ebd953938ff4c4943bfb32fb1b15ef749192", "scorecard_muse-spark-1.3.json": "dbda6d34e71856579238e796c42062c9b757121e0bfac774c6c7f46d91ba5b94", "state.sqlite3": "59000ca9eb1adc4191787731fca99880e98b75f2de8883b0e694149409686ee0", "summary.json": "2c7c16a93d814a76aa61ba74643905217186d2ad1a88fb19f1bfc028702d4363" } }