{
 "_note": "E12 patch sets, derived deterministically from E11's committed scores by scripts/01_derive_sets.py; no measurement of E12's own informs this.",
 "breakage_guard": {
  "belebele": [
   "belebele-133",
   "belebele-139",
   "belebele-150",
   "belebele-202",
   "belebele-213",
   "belebele-260",
   "belebele-262",
   "belebele-266",
   "belebele-289",
   "belebele-293",
   "belebele-299",
   "belebele-31",
   "belebele-326",
   "belebele-328",
   "belebele-330",
   "belebele-35",
   "belebele-4",
   "belebele-425",
   "belebele-450",
   "belebele-457",
   "belebele-47",
   "belebele-492",
   "belebele-522",
   "belebele-534",
   "belebele-555",
   "belebele-573",
   "belebele-578",
   "belebele-586",
   "belebele-593",
   "belebele-605",
   "belebele-615",
   "belebele-630",
   "belebele-637",
   "belebele-648",
   "belebele-678",
   "belebele-683",
   "belebele-685",
   "belebele-697",
   "belebele-720",
   "belebele-722",
   "belebele-742",
   "belebele-763",
   "belebele-769",
   "belebele-771",
   "belebele-795",
   "belebele-844",
   "belebele-859",
   "belebele-86",
   "belebele-869",
   "belebele-891"
  ],
  "mmlu": [
   "mmlu-10405",
   "mmlu-10541",
   "mmlu-10991",
   "mmlu-1105",
   "mmlu-11520",
   "mmlu-11687",
   "mmlu-11943",
   "mmlu-12772",
   "mmlu-12842",
   "mmlu-12963",
   "mmlu-13010",
   "mmlu-13078",
   "mmlu-13861",
   "mmlu-1517",
   "mmlu-1918",
   "mmlu-1945",
   "mmlu-2070",
   "mmlu-3356",
   "mmlu-3547",
   "mmlu-3623",
   "mmlu-3867",
   "mmlu-4620",
   "mmlu-4943",
   "mmlu-4970",
   "mmlu-5018",
   "mmlu-5164",
   "mmlu-5812",
   "mmlu-5853",
   "mmlu-5989",
   "mmlu-6013",
   "mmlu-631",
   "mmlu-6585",
   "mmlu-6593",
   "mmlu-6832",
   "mmlu-709",
   "mmlu-7108",
   "mmlu-7243",
   "mmlu-7403",
   "mmlu-7436",
   "mmlu-8030",
   "mmlu-8068",
   "mmlu-8074",
   "mmlu-8187",
   "mmlu-8484",
   "mmlu-8737",
   "mmlu-886",
   "mmlu-912",
   "mmlu-9526",
   "mmlu-9535",
   "mmlu-9782"
  ]
 },
 "counts": {
  "analyzed": {
   "belebele": 598,
   "mmlu": 589
  },
  "entity_spans": {
   "belebele": {
    "in_options": 21,
    "in_stem": 80,
    "n": 101
   },
   "mmlu": {
    "in_options": 43,
    "in_stem": 45,
    "n": 88
   }
  },
  "fwd": {
   "belebele": 151,
   "mmlu": 175
  },
  "pl_ok": {
   "belebele": 328,
   "mmlu": 202
  },
  "rev": {
   "belebele": 33,
   "mmlu": 49
  }
 },
 "endpoint": "correct_lennorm (E11 primary)",
 "entity_spans": {
  "belebele": {
   "belebele-102": {
    "en": [
     203,
     209
    ],
    "in_stem": true,
    "n_words": 1,
    "pl": [
     208,
     214
    ],
    "text": "system"
   },
   "belebele-118": {
    "en": [
     480,
     483
    ],
    "in_stem": true,
    "n_words": 2,
    "pl": [
     507,
     510
    ],
    "text": "3 2"
   },
   "belebele-125": {
    "en": [
     212,
     222
    ],
    "in_stem": true,
    "n_words": 2,
    "pl": [
     218,
     228
    ],
    "text": "John Smith"
   },
   "belebele-133": {
    "en": [
     254,
     263
    ],
    "in_stem": true,
    "n_words": 2,
    "pl": [
     223,
     232
    ],
    "text": "1000 1300"
   },
   "belebele-145": {
    "en": [
     17,
     21
    ],
    "in_stem": true,
    "n_words": 1,
    "pl": [
     17,
     21
    ],
    "text": "1956"
   },
   "belebele-153": {
    "en": [
     297,
     304
    ],
    "in_stem": true,
    "n_words": 4,
    "pl": [
     330,
     337
    ],
    "text": "4 2 3 9"
   },
   "belebele-17": {
    "en": [
     268,
     271
    ],
    "in_stem": true,
    "n_words": 2,
    "pl": [
     290,
     293
    ],
    "text": "1 1"
   },
   "belebele-177": {
    "en": [
     110,
     122
    ],
    "in_stem": true,
    "n_words": 2,
    "pl": [
     107,
     119
    ],
    "text": "Ilha Formosa"
   },
   "belebele-189": {
    "en": [
     254,
     266
    ],
    "in_stem": true,
    "n_words": 2,
    "pl": [
     469,
     481
    ],
    "text": "Robert Hooke"
   },
   "belebele-207": {
    "en": [
     62,
     66
    ],
    "in_stem": true,
    "n_words": 1,
    "pl": [
     63,
     67
    ],
    "text": "1839"
   },
   "belebele-208": {
    "en": [
     62,
     66
    ],
    "in_stem": true,
    "n_words": 1,
    "pl": [
     63,
     67
    ],
    "text": "1839"
   },
   "belebele-215": {
    "en": [
     494,
     519
    ],
    "in_stem": false,
    "n_words": 5,
    "pl": [
     519,
     544
    ],
    "text": "A Harappa B Santorini C"
   },
   "belebele-22": {
    "en": [
     484,
     494
    ],
    "in_stem": true,
    "n_words": 2,
    "pl": [
     516,
     526
    ],
    "text": "Toto 2004"
   },
   "belebele-227": {
    "en": [
     137,
     152
    ],
    "in_stem": true,
    "n_words": 2,
    "pl": [
     121,
     136
    ],
    "text": "Trinity College"
   },
   "belebele-23": {
    "en": [
     286,
     298
    ],
    "in_stem": true,
    "n_words": 2,
    "pl": [
     258,
     270
    ],
    "text": "Bremer 1998"
   },
   "belebele-24": {
    "en": [
     286,
     298
    ],
    "in_stem": true,
    "n_words": 2,
    "pl": [
     258,
     270
    ],
    "text": "Bremer 1998"
   },
   "belebele-240": {
    "en": [
     119,
     124
    ],
    "in_stem": true,
    "n_words": 2,
    "pl": [
     109,
     114
    ],
    "text": "70 km"
   },
   "belebele-247": {
    "en": [
     92,
     93
    ],
    "in_stem": true,
    "n_words": 1,
    "pl": [
     90,
     91
    ],
    "text": "8"
   },
   "belebele-272": {
    "en": [
     177,
     189
    ],
    "in_stem": true,
    "n_words": 2,
    "pl": [
     182,
     194
    ],
    "text": "Fort Capuzzo"
   },
   "belebele-280": {
    "en": [
     79,
     90
    ],
    "in_stem": true,
    "n_words": 2,
    "pl": [
     85,
     96
    ],
    "text": "Scott Kouns"
   },
   "belebele-283": {
    "en": [
     143,
     152
    ],
    "in_stem": true,
    "n_words": 2,
    "pl": [
     93,
     102
    ],
    "text": "Pago Pago"
   },
   "belebele-294": {
    "en": [
     24,
     38
    ],
    "in_stem": true,
    "n_words": 2,
    "pl": [
     0,
     14
    ],
    "text": "Jamie Siminoff"
   },
   "belebele-297": {
    "en": [
     26,
     35
    ],
    "in_stem": true,
    "n_words": 1,
    "pl": [
     17,
     26
    ],
    "text": "Amsterdam"
   },
   "belebele-298": {
    "en": [
     222,
     239
    ],
    "in_stem": true,
    "n_words": 3,
    "pl": [
     218,
     235
    ],
    "text": "David O Shannessy"
   },
   "belebele-316": {
    "en": [
     25,
     35
    ],
    "in_stem": true,
    "n_words": 1,
    "pl": [
     31,
     41
    ],
    "text": "Birmingham"
   },
   "belebele-333": {
    "en": [
     21,
     42
    ],
    "in_stem": true,
    "n_words": 2,
    "pl": [
     0,
     21
    ],
    "text": "Arnold Schwarzenegger"
   },
   "belebele-335": {
    "en": [
     368,
     377
    ],
    "in_stem": false,
    "n_words": 2,
    "pl": [
     363,
     372
    ],
    "text": "A Duvall"
   },
   "belebele-340": {
    "en": [
     508,
     548
    ],
    "in_stem": true,
    "n_words": 7,
    "pl": [
     485,
     525
    ],
    "text": "Extreme A Dustin Runnels B Xinhua C"
   },
   "belebele-343": {
    "en": [
     24,
     26
    ],
    "in_stem": true,
    "n_words": 1,
    "pl": [
     33,
     35
    ],
    "text": "10"
   },
   "belebele-345": {
    "en": [
     178,
     182
    ],
    "in_stem": true,
    "n_words": 1,
    "pl": [
     149,
     153
    ],
    "text": "Deng"
   },
   "belebele-349": {
    "en": [
     384,
     435
    ],
    "in_stem": false,
    "n_words": 9,
    "pl": [
     387,
     438
    ],
    "text": "A Medecines Sans Frontieres B Angola C Mangola D"
   },
   "belebele-358": {
    "en": [
     46,
     53
    ],
    "in_stem": true,
    "n_words": 1,
    "pl": [
     59,
     66
    ],
    "text": "program"
   },
   "belebele-384": {
    "en": [
     345,
     364
    ],
    "in_stem": true,
    "n_words": 4,
    "pl": [
     417,
     436
    ],
    "text": "JAS 39C Gripen A"
   },
   "belebele-392": {
    "en": [
     614,
     623
    ],
    "in_stem": false,
    "n_words": 2,
    "pl": [
     572,
     581
    ],
    "text": "Buffalo D"
   },
   "belebele-393": {
    "en": [
     557,
     566
    ],
    "in_stem": false,
    "n_words": 2,
    "pl": [
     551,
     560
    ],
    "text": "Buffalo B"
   },
   "belebele-395": {
    "en": [
     13,
     20
    ],
    "in_stem": true,
    "n_words": 2,
    "pl": [
     19,
     26
    ],
    "text": "120 160"
   },
   "belebele-409": {
    "en": [
     599,
     623
    ],
    "in_stem": true,
    "n_words": 3,
    "pl": [
     644,
     668
    ],
    "text": "Haiti Danielle Lantagne"
   },
   "belebele-413": {
    "en": [
     1,
     19
    ],
    "in_stem": true,
    "n_words": 2,
    "pl": [
     0,
     18
    ],
    "text": "Christopher Garcia"
   },
   "belebele-418": {
    "en": [
     288,
     305
    ],
    "in_stem": true,
    "n_words": 4,
    "pl": [
     271,
     288
    ],
    "text": "Rio Grande do Sul"
   },
   "belebele-423": {
    "en": [
     37,
     44
    ],
    "in_stem": true,
    "n_words": 2,
    "pl": [
     68,
     75
    ],
    "text": "1040 km"
   },
   "belebele-43": {
    "en": [
     299,
     321
    ],
    "in_stem": true,
    "n_words": 4,
    "pl": [
     280,
     302
    ],
    "text": "The President s Speech"
   },
   "belebele-434": {
    "en": [
     594,
     601
    ],
    "in_stem": false,
    "n_words": 3,
    "pl": [
     653,
     660
    ],
    "text": "HJR 3 B"
   },
   "belebele-435": {
    "en": [
     402,
     407
    ],
    "in_stem": true,
    "n_words": 2,
    "pl": [
     425,
     430
    ],
    "text": "HJR 3"
   },
   "belebele-436": {
    "en": [
     642,
     660
    ],
    "in_stem": false,
    "n_words": 4,
    "pl": [
     684,
     702
    ],
    "text": "C Deutsche Bank D"
   },
   "belebele-443": {
    "en": [
     142,
     152
    ],
    "in_stem": true,
    "n_words": 2,
    "pl": [
     157,
     167
    ],
    "text": "Lo Piccolo"
   },
   "belebele-449": {
    "en": [
     714,
     721
    ],
    "in_stem": false,
    "n_words": 2,
    "pl": [
     810,
     817
    ],
    "text": "Glynn C"
   },
   "belebele-452": {
    "en": [
     0,
     5
    ],
    "in_stem": true,
    "n_words": 1,
    "pl": [
     0,
     5
    ],
    "text": "Lodin"
   },
   "belebele-458": {
    "en": [
     112,
     135
    ],
    "in_stem": true,
    "n_words": 3,
    "pl": [
     111,
     134
    ],
    "text": "Hokuriku Electric Power"
   },
   "belebele-460": {
    "en": [
     126,
     130
    ],
    "in_stem": true,
    "n_words": 1,
    "pl": [
     258,
     262
    ],
    "text": "1967"
   },
   "belebele-463": {
    "en": [
     722,
     756
    ],
    "in_stem": false,
    "n_words": 7,
    "pl": [
     735,
     769
    ],
    "text": "A East Bay Community Law Center B"
   },
   "belebele-465": {
    "en": [
     543,
     574
    ],
    "in_stem": false,
    "n_words": 8,
    "pl": [
     622,
     653
    ],
    "text": "A 2220 B 2250 C 2243 D 2207"
   },
   "belebele-466": {
    "en": [
     178,
     184
    ],
    "in_stem": true,
    "n_words": 2,
    "pl": [
     163,
     169
    ],
    "text": "24 000"
   },
   "belebele-472": {
    "en": [
     383,
     404
    ],
    "in_stem": false,
    "n_words": 8,
    "pl": [
     416,
     437
    ],
    "text": "A 19 B 2 C 0 D 20"
   },
   "belebele-474": {
    "en": [
     30,
     41
    ],
    "in_stem": true,
    "n_words": 2,
    "pl": [
     10,
     21
    ],
    "text": "Mitt Romney"
   },
   "belebele-483": {
    "en": [
     517,
     530
    ],
    "in_stem": false,
    "n_words": 4,
    "pl": [
     509,
     522
    ],
    "text": "7 2 D Raonic"
   },
   "belebele-485": {
    "en": [
     11,
     21
    ],
    "in_stem": true,
    "n_words": 2,
    "pl": [
     28,
     38
    ],
    "text": "Naked News"
   },
   "belebele-488": {
    "en": [
     342,
     353
    ],
    "in_stem": false,
    "n_words": 3,
    "pl": [
     336,
     347
    ],
    "text": "Utah Jazz C"
   },
   "belebele-490": {
    "en": [
     534,
     624
    ],
    "in_stem": true,
    "n_words": 14,
    "pl": [
     572,
     662
    ],
    "text": "Atlanta Thrashers A Nicklas Backstrom B Jaromir Jagr C Mario Lemieux D Alex Ovechkin"
   },
   "belebele-496": {
    "en": [
     608,
     654
    ],
    "in_stem": false,
    "n_words": 6,
    "pl": [
     638,
     684
    ],
    "text": "The Sporting Shooters Association of Australia"
   },
   "belebele-50": {
    "en": [
     7,
     12
    ],
    "in_stem": true,
    "n_words": 1,
    "pl": [
     203,
     208
    ],
    "text": "style"
   },
   "belebele-506": {
    "en": [
     291,
     329
    ],
    "in_stem": false,
    "n_words": 8,
    "pl": [
     295,
     332
    ],
    "text": "B George W Bush C Gordon Johndroe D"
   },
   "belebele-507": {
    "en": [
     788,
     797
    ],
    "in_stem": false,
    "n_words": 3,
    "pl": [
     729,
     738
    ],
    "text": "B 1000 C"
   },
   "belebele-510": {
    "en": [
     365,
     377
    ],
    "in_stem": true,
    "n_words": 3,
    "pl": [
     436,
     448
    ],
    "text": "Ward A 20"
   },
   "belebele-514": {
    "en": [
     67,
     75
    ],
    "in_stem": true,
    "n_words": 1,
    "pl": [
     89,
     97
    ],
    "text": "Maryland"
   },
   "belebele-522": {
    "en": [
     545,
     563
    ],
    "in_stem": true,
    "n_words": 3,
    "pl": [
     571,
     589
    ],
    "text": "Karenia brevis A"
   },
   "belebele-524": {
    "en": [
     280,
     284
    ],
    "in_stem": true,
    "n_words": 2,
    "pl": [
     364,
     368
    ],
    "text": "6 34"
   },
   "belebele-525": {
    "en": [
     280,
     284
    ],
    "in_stem": true,
    "n_words": 2,
    "pl": [
     364,
     368
    ],
    "text": "6 34"
   },
   "belebele-537": {
    "en": [
     431,
     440
    ],
    "in_stem": false,
    "n_words": 2,
    "pl": [
     405,
     414
    ],
    "text": "D Saturn"
   },
   "belebele-547": {
    "en": [
     385,
     400
    ],
    "in_stem": true,
    "n_words": 2,
    "pl": [
     458,
     473
    ],
    "text": "Pamela Ferguson"
   },
   "belebele-569": {
    "en": [
     163,
     166
    ],
    "in_stem": true,
    "n_words": 2,
    "pl": [
     179,
     182
    ],
    "text": "6 5"
   },
   "belebele-576": {
    "en": [
     235,
     289
    ],
    "in_stem": true,
    "n_words": 8,
    "pl": [
     271,
     325
    ],
    "text": "Baba Kanjar Bhutha Kanjar Rampro Kanjar Gaza Kanjar"
   },
   "belebele-582": {
    "en": [
     31,
     40
    ],
    "in_stem": true,
    "n_words": 2,
    "pl": [
     193,
     202
    ],
    "text": "tie break"
   },
   "belebele-595": {
    "en": [
     15,
     23
    ],
    "in_stem": true,
    "n_words": 1,
    "pl": [
     18,
     26
    ],
    "text": "Danielle"
   },
   "belebele-600": {
    "en": [
     129,
     132
    ],
    "in_stem": true,
    "n_words": 1,
    "pl": [
     118,
     121
    ],
    "text": "300"
   },
   "belebele-601": {
    "en": [
     129,
     132
    ],
    "in_stem": true,
    "n_words": 1,
    "pl": [
     118,
     121
    ],
    "text": "300"
   },
   "belebele-608": {
    "en": [
     470,
     487
    ],
    "in_stem": false,
    "n_words": 3,
    "pl": [
     489,
     506
    ],
    "text": "Puerto Princesa B"
   },
   "belebele-617": {
    "en": [
     842,
     937
    ],
    "in_stem": false,
    "n_words": 12,
    "pl": [
     865,
     960
    ],
    "text": "A Petros Mantouvalos B Sakis Kechagioglou C George Nikolakopoulos D Makis Triantafylopoulos"
   },
   "belebele-620": {
    "en": [
     0,
     13
    ],
    "in_stem": true,
    "n_words": 2,
    "pl": [
     0,
     13
    ],
    "text": "Robin Uthappa"
   },
   "belebele-624": {
    "en": [
     94,
     103
    ],
    "in_stem": true,
    "n_words": 2,
    "pl": [
     56,
     65
    ],
    "text": "Amir Khan"
   },
   "belebele-632": {
    "en": [
     23,
     37
    ],
    "in_stem": true,
    "n_words": 2,
    "pl": [
     103,
     117
    ],
    "text": "Frequent Flyer"
   },
   "belebele-651": {
    "en": [
     675,
     687
    ],
    "in_stem": false,
    "n_words": 3,
    "pl": [
     747,
     759
    ],
    "text": "B Catalan C"
   },
   "belebele-666": {
    "en": [
     421,
     431
    ],
    "in_stem": true,
    "n_words": 4,
    "pl": [
     446,
     456
    ],
    "text": "56 64 km h"
   },
   "belebele-672": {
    "en": [
     236,
     247
    ],
    "in_stem": true,
    "n_words": 2,
    "pl": [
     229,
     240
    ],
    "text": "Casa Branca"
   },
   "belebele-681": {
    "en": [
     83,
     125
    ],
    "in_stem": true,
    "n_words": 4,
    "pl": [
     84,
     126
    ],
    "text": "Riomaggiore Manarola Corniglia Vernazza"
   },
   "belebele-684": {
    "en": [
     291,
     300
    ],
    "in_stem": true,
    "n_words": 4,
    "pl": [
     289,
     298
    ],
    "text": "5 mm 1 5"
   },
   "belebele-705": {
    "en": [
     735,
     744
    ],
    "in_stem": false,
    "n_words": 2,
    "pl": [
     901,
     910
    ],
    "text": "minimum B"
   },
   "belebele-708": {
    "en": [
     225,
     233
    ],
    "in_stem": true,
    "n_words": 2,
    "pl": [
     198,
     206
    ],
    "text": "FIC West"
   },
   "belebele-732": {
    "en": [
     526,
     535
    ],
    "in_stem": true,
    "n_words": 2,
    "pl": [
     648,
     657
    ],
    "text": "boda boda"
   },
   "belebele-741": {
    "en": [
     636,
     658
    ],
    "in_stem": true,
    "n_words": 5,
    "pl": [
     592,
     614
    ],
    "text": "Jardín de la Unión A"
   },
   "belebele-75": {
    "en": [
     51,
     57
    ],
    "in_stem": true,
    "n_words": 2,
    "pl": [
     73,
     79
    ],
    "text": "10 000"
   },
   "belebele-767": {
    "en": [
     70,
     73
    ],
    "in_stem": true,
    "n_words": 2,
    "pl": [
     78,
     81
    ],
    "text": "3 7"
   },
   "belebele-798": {
    "en": [
     223,
     233
    ],
    "in_stem": true,
    "n_words": 2,
    "pl": [
     174,
     184
    ],
    "text": "bien élevé"
   },
   "belebele-800": {
    "en": [
     0,
     9
    ],
    "in_stem": true,
    "n_words": 1,
    "pl": [
     2,
     11
    ],
    "text": "MetroRail"
   },
   "belebele-810": {
    "en": [
     0,
     7
    ],
    "in_stem": true,
    "n_words": 1,
    "pl": [
     7,
     14
    ],
    "text": "Cochamó"
   },
   "belebele-814": {
    "en": [
     106,
     126
    ],
    "in_stem": true,
    "n_words": 3,
    "pl": [
     105,
     125
    ],
    "text": "Global Running Tours"
   },
   "belebele-824": {
    "en": [
     722,
     733
    ],
    "in_stem": true,
    "n_words": 2,
    "pl": [
     761,
     772
    ],
    "text": "offline A"
   },
   "belebele-830": {
    "en": [
     4,
     23
    ],
    "in_stem": true,
    "n_words": 3,
    "pl": [
     466,
     485
    ],
    "text": "South Pole Traverse"
   },
   "belebele-842": {
    "en": [
     79,
     88
    ],
    "in_stem": true,
    "n_words": 3,
    "pl": [
     82,
     91
    ],
    "text": "80 km 50"
   },
   "belebele-847": {
    "en": [
     463,
     475
    ],
    "in_stem": true,
    "n_words": 3,
    "pl": [
     528,
     540
    ],
    "text": "Paro Nu 150"
   },
   "belebele-851": {
    "en": [
     141,
     148
    ],
    "in_stem": true,
    "n_words": 2,
    "pl": [
     146,
     153
    ],
    "text": "100 200"
   },
   "belebele-862": {
    "en": [
     115,
     121
    ],
    "in_stem": true,
    "n_words": 2,
    "pl": [
     111,
     117
    ],
    "text": "e mail"
   }
  },
  "mmlu": {
   "mmlu-10001": {
    "en": [
     70,
     94
    ],
    "in_stem": false,
    "n_words": 4,
    "pl": [
     67,
     91
    ],
    "text": "A act utilitarianism B"
   },
   "mmlu-10143": {
    "en": [
     540,
     555
    ],
    "in_stem": false,
    "n_words": 3,
    "pl": [
     588,
     603
    ],
    "text": "Homo erectus D"
   },
   "mmlu-10178": {
    "en": [
     214,
     226
    ],
    "in_stem": false,
    "n_words": 2,
    "pl": [
     233,
     245
    ],
    "text": "Mohanjo daro"
   },
   "mmlu-10380": {
    "en": [
     133,
     140
    ],
    "in_stem": true,
    "n_words": 2,
    "pl": [
     138,
     145
    ],
    "text": "150 000"
   },
   "mmlu-10563": {
    "en": [
     400,
     452
    ],
    "in_stem": false,
    "n_words": 14,
    "pl": [
     466,
     518
    ],
    "text": "A 2 005 000 B 1 975 000 C 1 945 000 D 1910000"
   },
   "mmlu-10786": {
    "en": [
     845,
     955
    ],
    "in_stem": true,
    "n_words": 15,
    "pl": [
     863,
     973
    ],
    "text": "an A equitable servitude B restrictive covenant C unenforceable restriction D easement for public use"
   },
   "mmlu-11373": {
    "en": [
     13,
     21
    ],
    "in_stem": true,
    "n_words": 1,
    "pl": [
     13,
     21
    ],
    "text": "operator"
   },
   "mmlu-1141": {
    "en": [
     198,
     257
    ],
    "in_stem": false,
    "n_words": 12,
    "pl": [
     204,
     263
    ],
    "text": "A True True B True False C False True D False False"
   },
   "mmlu-11421": {
    "en": [
     0,
     13
    ],
    "in_stem": true,
    "n_words": 2,
    "pl": [
     0,
     13
    ],
    "text": "Daylight Corp"
   },
   "mmlu-11463": {
    "en": [
     810,
     831
    ],
    "in_stem": true,
    "n_words": 3,
    "pl": [
     876,
     897
    ],
    "text": "Native American Power"
   },
   "mmlu-11516": {
    "en": [
     1490,
     1534
    ],
    "in_stem": false,
    "n_words": 12,
    "pl": [
     1516,
     1560
    ],
    "text": "B 10 000 00 C 18 000 00 D 24 000 00"
   },
   "mmlu-12136": {
    "en": [
     846,
     865
    ],
    "in_stem": false,
    "n_words": 4,
    "pl": [
     868,
     887
    ],
    "text": "Fair Housing Act C"
   },
   "mmlu-12165": {
    "en": [
     813,
     821
    ],
    "in_stem": true,
    "n_words": 4,
    "pl": [
     750,
     758
    ],
    "text": "5 x 5 cm"
   },
   "mmlu-12219": {
    "en": [
     402,
     410
    ],
    "in_stem": true,
    "n_words": 2,
    "pl": [
     426,
     434
    ],
    "text": "post hoc"
   },
   "mmlu-12286": {
    "en": [
     358,
     373
    ],
    "in_stem": true,
    "n_words": 6,
    "pl": [
     369,
     384
    ],
    "text": "39 4 C 102 9 F"
   },
   "mmlu-12343": {
    "en": [
     893,
     908
    ],
    "in_stem": true,
    "n_words": 6,
    "pl": [
     819,
     834
    ],
    "text": "37 8 C 100 0 F"
   },
   "mmlu-12459": {
    "en": [
     0,
     9
    ],
    "in_stem": true,
    "n_words": 2,
    "pl": [
     0,
     8
    ],
    "text": "Dr Jones"
   },
   "mmlu-12557": {
    "en": [
     14,
     15
    ],
    "in_stem": true,
    "n_words": 1,
    "pl": [
     11,
     12
    ],
    "text": "2"
   },
   "mmlu-12786": {
    "en": [
     159,
     168
    ],
    "in_stem": false,
    "n_words": 2,
    "pl": [
     157,
     166
    ],
    "text": "A Barter"
   },
   "mmlu-12870": {
    "en": [
     4,
     12
    ],
    "in_stem": true,
    "n_words": 1,
    "pl": [
     0,
     8
    ],
    "text": "Standard"
   },
   "mmlu-13063": {
    "en": [
     111,
     161
    ],
    "in_stem": false,
    "n_words": 8,
    "pl": [
     110,
     160
    ],
    "text": "A Enabling B Functional C Normative D Diffused"
   },
   "mmlu-13243": {
    "en": [
     41,
     42
    ],
    "in_stem": true,
    "n_words": 1,
    "pl": [
     36,
     37
    ],
    "text": "3"
   },
   "mmlu-13281": {
    "en": [
     242,
     252
    ],
    "in_stem": false,
    "n_words": 2,
    "pl": [
     257,
     267
    ],
    "text": "status quo"
   },
   "mmlu-13499": {
    "en": [
     7,
     11
    ],
    "in_stem": true,
    "n_words": 1,
    "pl": [
     44,
     48
    ],
    "text": "1990"
   },
   "mmlu-13576": {
    "en": [
     0,
     7
    ],
    "in_stem": true,
    "n_words": 1,
    "pl": [
     0,
     7
    ],
    "text": "Giselle"
   },
   "mmlu-13669": {
    "en": [
     105,
     108
    ],
    "in_stem": false,
    "n_words": 2,
    "pl": [
     88,
     91
    ],
    "text": "7 B"
   },
   "mmlu-13908": {
    "en": [
     79,
     118
    ],
    "in_stem": false,
    "n_words": 9,
    "pl": [
     91,
     130
    ],
    "text": "A Rumi B Rabi a C Fatimah D Khadija"
   },
   "mmlu-1445": {
    "en": [
     340,
     367
    ],
    "in_stem": false,
    "n_words": 12,
    "pl": [
     341,
     368
    ],
    "text": "A 1 N B 2 N C 3 N D 4 N"
   },
   "mmlu-1834": {
    "en": [
     48,
     77
    ],
    "in_stem": true,
    "n_words": 4,
    "pl": [
     39,
     68
    ],
    "text": "Dickey Fuller Engle Granger"
   },
   "mmlu-1935": {
    "en": [
     70,
     89
    ],
    "in_stem": false,
    "n_words": 8,
    "pl": [
     68,
     87
    ],
    "text": "A 3 B 6 C 8 D 9"
   },
   "mmlu-2022": {
    "en": [
     67,
     105
    ],
    "in_stem": false,
    "n_words": 12,
    "pl": [
     61,
     99
    ],
    "text": "A 23 mA B 40 mA C 10 mA D 60 mA"
   },
   "mmlu-2051": {
    "en": [
     66,
     100
    ],
    "in_stem": false,
    "n_words": 8,
    "pl": [
     71,
     105
    ],
    "text": "A MROM B PROM C EPROM D EEPROM"
   },
   "mmlu-2164": {
    "en": [
     138,
     163
    ],
    "in_stem": false,
    "n_words": 10,
    "pl": [
     122,
     147
    ],
    "text": "A 2 B 3 6 C 22 5 D 63"
   },
   "mmlu-2279": {
    "en": [
     186,
     229
    ],
    "in_stem": false,
    "n_words": 12,
    "pl": [
     228,
     271
    ],
    "text": "A 32 30 B 34 70 C 42 46 D 275 50"
   },
   "mmlu-2367": {
    "en": [
     312,
     354
    ],
    "in_stem": false,
    "n_words": 12,
    "pl": [
     285,
     327
    ],
    "text": "A 12 25 B 25 75 C 29 25 D 37 25"
   },
   "mmlu-2412": {
    "en": [
     105,
     114
    ],
    "in_stem": true,
    "n_words": 4,
    "pl": [
     115,
     124
    ],
    "text": "7 x 2 A"
   },
   "mmlu-2505": {
    "en": [
     143,
     162
    ],
    "in_stem": true,
    "n_words": 3,
    "pl": [
     147,
     166
    ],
    "text": "Minecraft C Emily"
   },
   "mmlu-3095": {
    "en": [
     180,
     231
    ],
    "in_stem": false,
    "n_words": 14,
    "pl": [
     171,
     222
    ],
    "text": "A KCl aq B Fe NO3 3 aq C MgSO4 aq D N2H4 aq"
   },
   "mmlu-3100": {
    "en": [
     114,
     134
    ],
    "in_stem": false,
    "n_words": 8,
    "pl": [
     116,
     136
    ],
    "text": "A 1 B 4 C 5 D 11"
   },
   "mmlu-3305": {
    "en": [
     878,
     889
    ],
    "in_stem": true,
    "n_words": 2,
    "pl": [
     846,
     857
    ],
    "text": "Elie Wiesel"
   },
   "mmlu-3356": {
    "en": [
     1824,
     1867
    ],
    "in_stem": true,
    "n_words": 7,
    "pl": [
     1804,
     1847
    ],
    "text": "The Chief Periods of European History 1885"
   },
   "mmlu-3408": {
    "en": [
     1863,
     1873
    ],
    "in_stem": true,
    "n_words": 2,
    "pl": [
     1796,
     1806
    ],
    "text": "émile Zola"
   },
   "mmlu-3562": {
    "en": [
     103,
     114
    ],
    "in_stem": false,
    "n_words": 2,
    "pl": [
     109,
     120
    ],
    "text": "B Oklahoma"
   },
   "mmlu-3774": {
    "en": [
     34,
     63
    ],
    "in_stem": true,
    "n_words": 3,
    "pl": [
     36,
     65
    ],
    "text": "Small Business Administration"
   },
   "mmlu-4168": {
    "en": [
     171,
     209
    ],
    "in_stem": false,
    "n_words": 10,
    "pl": [
     186,
     224
    ],
    "text": "A 190 B 900 C 1 000 D 1 900"
   },
   "mmlu-4870": {
    "en": [
     141,
     150
    ],
    "in_stem": true,
    "n_words": 3,
    "pl": [
     149,
     157
    ],
    "text": "8 A P"
   },
   "mmlu-5099": {
    "en": [
     92,
     96
    ],
    "in_stem": true,
    "n_words": 1,
    "pl": [
     95,
     99
    ],
    "text": "NASP"
   },
   "mmlu-5235": {
    "en": [
     0,
     8
    ],
    "in_stem": true,
    "n_words": 1,
    "pl": [
     0,
     8
    ],
    "text": "Carlotta"
   },
   "mmlu-5290": {
    "en": [
     193,
     201
    ],
    "in_stem": false,
    "n_words": 2,
    "pl": [
     180,
     188
    ],
    "text": "D Mania"
   },
   "mmlu-5429": {
    "en": [
     454,
     465
    ],
    "in_stem": false,
    "n_words": 2,
    "pl": [
     471,
     482
    ],
    "text": "test retest"
   },
   "mmlu-5457": {
    "en": [
     389,
     435
    ],
    "in_stem": false,
    "n_words": 12,
    "pl": [
     414,
     460
    ],
    "text": "A 22 500 B 23 700 C 25 500 D 27 300"
   },
   "mmlu-5521": {
    "en": [
     435,
     447
    ],
    "in_stem": true,
    "n_words": 4,
    "pl": [
     451,
     463
    ],
    "text": "H0 μ 1 25"
   },
   "mmlu-5739": {
    "en": [
     101,
     125
    ],
    "in_stem": true,
    "n_words": 4,
    "pl": [
     103,
     127
    ],
    "text": "Share Our Wealth Society"
   },
   "mmlu-5786": {
    "en": [
     1838,
     1854
    ],
    "in_stem": true,
    "n_words": 3,
    "pl": [
     1860,
     1876
    ],
    "text": "Susan B Anthony"
   },
   "mmlu-5855": {
    "en": [
     533,
     536
    ],
    "in_stem": true,
    "n_words": 2,
    "pl": [
     502,
     505
    ],
    "text": "1 4"
   },
   "mmlu-5989": {
    "en": [
     285,
     296
    ],
    "in_stem": true,
    "n_words": 2,
    "pl": [
     266,
     277
    ],
    "text": "Yin Kaishan"
   },
   "mmlu-5997": {
    "en": [
     651,
     667
    ],
    "in_stem": true,
    "n_words": 3,
    "pl": [
     659,
     675
    ],
    "text": "Robert K Massie"
   },
   "mmlu-6085": {
    "en": [
     112,
     124
    ],
    "in_stem": true,
    "n_words": 2,
    "pl": [
     106,
     118
    ],
    "text": "Joshua Drake"
   },
   "mmlu-6145": {
    "en": [
     13,
     26
    ],
    "in_stem": true,
    "n_words": 2,
    "pl": [
     11,
     24
    ],
    "text": "Lucia Pauling"
   },
   "mmlu-6157": {
    "en": [
     100,
     111
    ],
    "in_stem": false,
    "n_words": 3,
    "pl": [
     105,
     116
    ],
    "text": "C Alaska D"
   },
   "mmlu-623": {
    "en": [
     112,
     120
    ],
    "in_stem": false,
    "n_words": 3,
    "pl": [
     130,
     138
    ],
    "text": "C5 C6 B"
   },
   "mmlu-6335": {
    "en": [
     213,
     278
    ],
    "in_stem": false,
    "n_words": 13,
    "pl": [
     219,
     284
    ],
    "text": "A Sigmund Freud B John Money C B F Skinner D Havelock Ellis"
   },
   "mmlu-6417": {
    "en": [
     75,
     97
    ],
    "in_stem": false,
    "n_words": 8,
    "pl": [
     73,
     95
    ],
    "text": "A 5 B 10 C 15 D 20"
   },
   "mmlu-7087": {
    "en": [
     117,
     153
    ],
    "in_stem": false,
    "n_words": 7,
    "pl": [
     107,
     143
    ],
    "text": "A DMU B Firmographics C ACORN D"
   },
   "mmlu-7093": {
    "en": [
     4,
     7
    ],
    "in_stem": true,
    "n_words": 1,
    "pl": [
     14,
     17
    ],
    "text": "B2B"
   },
   "mmlu-7224": {
    "en": [
     0,
     6
    ],
    "in_stem": true,
    "n_words": 1,
    "pl": [
     0,
     6
    ],
    "text": "Robert"
   },
   "mmlu-7409": {
    "en": [
     95,
     166
    ],
    "in_stem": false,
    "n_words": 12,
    "pl": [
     109,
     180
    ],
    "text": "A Miss Jackie B Elizabeth Sanford C Geraldine Jones D Gladys Knight"
   },
   "mmlu-7658": {
    "en": [
     65,
     128
    ],
    "in_stem": false,
    "n_words": 12,
    "pl": [
     69,
     132
    ],
    "text": "A Laura Ashley B Helmut Lang C Donna Karan D Christian Dior"
   },
   "mmlu-7672": {
    "en": [
     46,
     103
    ],
    "in_stem": false,
    "n_words": 9,
    "pl": [
     42,
     99
    ],
    "text": "A millennium B computer code C catastrophe D thousand"
   },
   "mmlu-7675": {
    "en": [
     79,
     88
    ],
    "in_stem": false,
    "n_words": 2,
    "pl": [
     81,
     90
    ],
    "text": "William B"
   },
   "mmlu-7829": {
    "en": [
     79,
     93
    ],
    "in_stem": false,
    "n_words": 4,
    "pl": [
     90,
     104
    ],
    "text": "B tight end C"
   },
   "mmlu-7830": {
    "en": [
     62,
     76
    ],
    "in_stem": false,
    "n_words": 3,
    "pl": [
     78,
     92
    ],
    "text": "C orangutan D"
   },
   "mmlu-7856": {
    "en": [
     56,
     122
    ],
    "in_stem": false,
    "n_words": 12,
    "pl": [
     65,
     131
    ],
    "text": "A Earl B Grey Darjeeling C English Breakfast D Prince of Wales"
   },
   "mmlu-7930": {
    "en": [
     14,
     26
    ],
    "in_stem": true,
    "n_words": 2,
    "pl": [
     25,
     37
    ],
    "text": "Pepto Bismol"
   },
   "mmlu-8001": {
    "en": [
     68,
     101
    ],
    "in_stem": false,
    "n_words": 7,
    "pl": [
     69,
     102
    ],
    "text": "A Richie Rich B Charlie Brown C"
   },
   "mmlu-8112": {
    "en": [
     63,
     85
    ],
    "in_stem": false,
    "n_words": 4,
    "pl": [
     64,
     86
    ],
    "text": "C Mekong D Allegheny"
   },
   "mmlu-816": {
    "en": [
     273,
     281
    ],
    "in_stem": true,
    "n_words": 1,
    "pl": [
     235,
     243
    ],
    "text": "transfer"
   },
   "mmlu-8265": {
    "en": [
     66,
     80
    ],
    "in_stem": true,
    "n_words": 2,
    "pl": [
     68,
     82
    ],
    "text": "Stephen Macedo"
   },
   "mmlu-8297": {
    "en": [
     1,
     16
    ],
    "in_stem": true,
    "n_words": 2,
    "pl": [
     1,
     16
    ],
    "text": "Richardson Self"
   },
   "mmlu-83": {
    "en": [
     207,
     270
    ],
    "in_stem": true,
    "n_words": 13,
    "pl": [
     169,
     232
    ],
    "text": "6 A True True B False False C True False D False True"
   },
   "mmlu-8356": {
    "en": [
     30,
     36
    ],
    "in_stem": true,
    "n_words": 1,
    "pl": [
     36,
     42
    ],
    "text": "Hardin"
   },
   "mmlu-8503": {
    "en": [
     1,
     6
    ],
    "in_stem": true,
    "n_words": 1,
    "pl": [
     1,
     6
    ],
    "text": "Nagel"
   },
   "mmlu-9480": {
    "en": [
     84,
     117
    ],
    "in_stem": false,
    "n_words": 4,
    "pl": [
     76,
     109
    ],
    "text": "A Acrodermatitis enteropathica B"
   },
   "mmlu-9525": {
    "en": [
     106,
     138
    ],
    "in_stem": false,
    "n_words": 7,
    "pl": [
     102,
     134
    ],
    "text": "apoE B apoA apo a apoB apoC"
   },
   "mmlu-953": {
    "en": [
     31,
     153
    ],
    "in_stem": true,
    "n_words": 41,
    "pl": [
     29,
     151
    ],
    "text": "205Tl A ɣ 205TI 1 569 x 108 T 1 s 1 I ½ B 0 717 x 10 26 J T 1 C 0 827 x 10 26 J T 1 D 1 433 x 10 26 J T 1"
   },
   "mmlu-9530": {
    "en": [
     141,
     148
    ],
    "in_stem": false,
    "n_words": 2,
    "pl": [
     137,
     144
    ],
    "text": "NADP C"
   },
   "mmlu-9675": {
    "en": [
     52,
     67
    ],
    "in_stem": true,
    "n_words": 2,
    "pl": [
     44,
     59
    ],
    "text": "goitrogen A"
   },
   "mmlu-9876": {
    "en": [
     0,
     5
    ],
    "in_stem": true,
    "n_words": 1,
    "pl": [
     0,
     5
    ],
    "text": "Rawls"
   }
  }
 },
 "fwd": {
  "belebele": [
   "belebele-1",
   "belebele-12",
   "belebele-14",
   "belebele-17",
   "belebele-22",
   "belebele-23",
   "belebele-24",
   "belebele-38",
   "belebele-42",
   "belebele-43",
   "belebele-45",
   "belebele-50",
   "belebele-56",
   "belebele-75",
   "belebele-79",
   "belebele-82",
   "belebele-92",
   "belebele-98",
   "belebele-106",
   "belebele-109",
   "belebele-118",
   "belebele-125",
   "belebele-140",
   "belebele-145",
   "belebele-147",
   "belebele-153",
   "belebele-172",
   "belebele-177",
   "belebele-180",
   "belebele-184",
   "belebele-185",
   "belebele-190",
   "belebele-197",
   "belebele-207",
   "belebele-208",
   "belebele-215",
   "belebele-218",
   "belebele-226",
   "belebele-231",
   "belebele-237",
   "belebele-240",
   "belebele-247",
   "belebele-249",
   "belebele-251",
   "belebele-264",
   "belebele-283",
   "belebele-294",
   "belebele-297",
   "belebele-298",
   "belebele-316",
   "belebele-327",
   "belebele-333",
   "belebele-335",
   "belebele-340",
   "belebele-343",
   "belebele-345",
   "belebele-349",
   "belebele-372",
   "belebele-375",
   "belebele-376",
   "belebele-384",
   "belebele-392",
   "belebele-393",
   "belebele-409",
   "belebele-413",
   "belebele-418",
   "belebele-423",
   "belebele-427",
   "belebele-435",
   "belebele-436",
   "belebele-443",
   "belebele-449",
   "belebele-452",
   "belebele-458",
   "belebele-463",
   "belebele-466",
   "belebele-472",
   "belebele-474",
   "belebele-483",
   "belebele-485",
   "belebele-488",
   "belebele-490",
   "belebele-496",
   "belebele-507",
   "belebele-510",
   "belebele-513",
   "belebele-514",
   "belebele-524",
   "belebele-525",
   "belebele-547",
   "belebele-556",
   "belebele-569",
   "belebele-571",
   "belebele-582",
   "belebele-595",
   "belebele-600",
   "belebele-601",
   "belebele-608",
   "belebele-617",
   "belebele-620",
   "belebele-627",
   "belebele-632",
   "belebele-651",
   "belebele-652",
   "belebele-653",
   "belebele-657",
   "belebele-659",
   "belebele-660",
   "belebele-666",
   "belebele-668",
   "belebele-672",
   "belebele-684",
   "belebele-689",
   "belebele-695",
   "belebele-703",
   "belebele-705",
   "belebele-706",
   "belebele-708",
   "belebele-710",
   "belebele-712",
   "belebele-732",
   "belebele-736",
   "belebele-737",
   "belebele-738",
   "belebele-739",
   "belebele-750",
   "belebele-753",
   "belebele-756",
   "belebele-757",
   "belebele-760",
   "belebele-767",
   "belebele-772",
   "belebele-780",
   "belebele-783",
   "belebele-798",
   "belebele-800",
   "belebele-801",
   "belebele-804",
   "belebele-806",
   "belebele-810",
   "belebele-814",
   "belebele-821",
   "belebele-832",
   "belebele-833",
   "belebele-835",
   "belebele-847",
   "belebele-851",
   "belebele-855",
   "belebele-862",
   "belebele-877",
   "belebele-885"
  ],
  "mmlu": [
   "mmlu-83",
   "mmlu-352",
   "mmlu-561",
   "mmlu-614",
   "mmlu-736",
   "mmlu-1243",
   "mmlu-1329",
   "mmlu-1400",
   "mmlu-1445",
   "mmlu-1484",
   "mmlu-1498",
   "mmlu-1658",
   "mmlu-1751",
   "mmlu-1826",
   "mmlu-1935",
   "mmlu-1997",
   "mmlu-2279",
   "mmlu-2367",
   "mmlu-2412",
   "mmlu-2505",
   "mmlu-2672",
   "mmlu-2790",
   "mmlu-2800",
   "mmlu-3095",
   "mmlu-3408",
   "mmlu-3494",
   "mmlu-3518",
   "mmlu-3552",
   "mmlu-3562",
   "mmlu-3631",
   "mmlu-3661",
   "mmlu-3774",
   "mmlu-3789",
   "mmlu-3895",
   "mmlu-3959",
   "mmlu-3979",
   "mmlu-4041",
   "mmlu-4164",
   "mmlu-4168",
   "mmlu-4195",
   "mmlu-4200",
   "mmlu-4556",
   "mmlu-4665",
   "mmlu-4666",
   "mmlu-4899",
   "mmlu-4969",
   "mmlu-5011",
   "mmlu-5028",
   "mmlu-5099",
   "mmlu-5109",
   "mmlu-5235",
   "mmlu-5290",
   "mmlu-5393",
   "mmlu-5418",
   "mmlu-5429",
   "mmlu-5739",
   "mmlu-5786",
   "mmlu-5855",
   "mmlu-5997",
   "mmlu-6009",
   "mmlu-6030",
   "mmlu-6085",
   "mmlu-6145",
   "mmlu-6152",
   "mmlu-6157",
   "mmlu-6177",
   "mmlu-6195",
   "mmlu-6215",
   "mmlu-6231",
   "mmlu-6278",
   "mmlu-6321",
   "mmlu-6335",
   "mmlu-6417",
   "mmlu-6617",
   "mmlu-6665",
   "mmlu-6817",
   "mmlu-6923",
   "mmlu-6940",
   "mmlu-7014",
   "mmlu-7021",
   "mmlu-7068",
   "mmlu-7087",
   "mmlu-7089",
   "mmlu-7093",
   "mmlu-7224",
   "mmlu-7258",
   "mmlu-7266",
   "mmlu-7409",
   "mmlu-7410",
   "mmlu-7425",
   "mmlu-7658",
   "mmlu-7672",
   "mmlu-7675",
   "mmlu-7762",
   "mmlu-7774",
   "mmlu-7829",
   "mmlu-7830",
   "mmlu-7930",
   "mmlu-8112",
   "mmlu-8212",
   "mmlu-8265",
   "mmlu-8297",
   "mmlu-8356",
   "mmlu-8382",
   "mmlu-8503",
   "mmlu-8544",
   "mmlu-8556",
   "mmlu-8629",
   "mmlu-8656",
   "mmlu-8679",
   "mmlu-8708",
   "mmlu-8844",
   "mmlu-8886",
   "mmlu-9051",
   "mmlu-9073",
   "mmlu-9178",
   "mmlu-9242",
   "mmlu-9306",
   "mmlu-9465",
   "mmlu-9480",
   "mmlu-9525",
   "mmlu-9675",
   "mmlu-9704",
   "mmlu-9799",
   "mmlu-9876",
   "mmlu-10001",
   "mmlu-10138",
   "mmlu-10143",
   "mmlu-10158",
   "mmlu-10159",
   "mmlu-10178",
   "mmlu-10357",
   "mmlu-10380",
   "mmlu-10563",
   "mmlu-11064",
   "mmlu-11373",
   "mmlu-11421",
   "mmlu-11463",
   "mmlu-11516",
   "mmlu-11537",
   "mmlu-11656",
   "mmlu-11780",
   "mmlu-11783",
   "mmlu-11963",
   "mmlu-12040",
   "mmlu-12136",
   "mmlu-12165",
   "mmlu-12219",
   "mmlu-12286",
   "mmlu-12343",
   "mmlu-12459",
   "mmlu-12546",
   "mmlu-12557",
   "mmlu-12583",
   "mmlu-12726",
   "mmlu-12786",
   "mmlu-12794",
   "mmlu-12870",
   "mmlu-12893",
   "mmlu-12958",
   "mmlu-12976",
   "mmlu-13063",
   "mmlu-13243",
   "mmlu-13281",
   "mmlu-13331",
   "mmlu-13368",
   "mmlu-13372",
   "mmlu-13399",
   "mmlu-13499",
   "mmlu-13543",
   "mmlu-13576",
   "mmlu-13669",
   "mmlu-13727",
   "mmlu-13780",
   "mmlu-13908"
  ]
 },
 "irrelevant_map": {
  "belebele": {
   "belebele-1": "belebele-753",
   "belebele-106": "belebele-109",
   "belebele-109": "belebele-190",
   "belebele-118": "belebele-463",
   "belebele-12": "belebele-418",
   "belebele-125": "belebele-177",
   "belebele-14": "belebele-490",
   "belebele-140": "belebele-798",
   "belebele-145": "belebele-513",
   "belebele-147": "belebele-449",
   "belebele-153": "belebele-474",
   "belebele-17": "belebele-485",
   "belebele-172": "belebele-708",
   "belebele-177": "belebele-851",
   "belebele-180": "belebele-22",
   "belebele-184": "belebele-885",
   "belebele-185": "belebele-458",
   "belebele-190": "belebele-772",
   "belebele-197": "belebele-264",
   "belebele-207": "belebele-620",
   "belebele-208": "belebele-375",
   "belebele-215": "belebele-298",
   "belebele-218": "belebele-653",
   "belebele-22": "belebele-608",
   "belebele-226": "belebele-125",
   "belebele-23": "belebele-436",
   "belebele-231": "belebele-732",
   "belebele-237": "belebele-695",
   "belebele-24": "belebele-488",
   "belebele-240": "belebele-106",
   "belebele-247": "belebele-75",
   "belebele-249": "belebele-705",
   "belebele-251": "belebele-706",
   "belebele-264": "belebele-218",
   "belebele-283": "belebele-617",
   "belebele-294": "belebele-340",
   "belebele-297": "belebele-569",
   "belebele-298": "belebele-877",
   "belebele-316": "belebele-524",
   "belebele-327": "belebele-814",
   "belebele-333": "belebele-56",
   "belebele-335": "belebele-736",
   "belebele-340": "belebele-632",
   "belebele-343": "belebele-595",
   "belebele-345": "belebele-335",
   "belebele-349": "belebele-17",
   "belebele-372": "belebele-627",
   "belebele-375": "belebele-384",
   "belebele-376": "belebele-800",
   "belebele-38": "belebele-738",
   "belebele-384": "belebele-666",
   "belebele-392": "belebele-739",
   "belebele-393": "belebele-197",
   "belebele-409": "belebele-376",
   "belebele-413": "belebele-92",
   "belebele-418": "belebele-684",
   "belebele-42": "belebele-50",
   "belebele-423": "belebele-472",
   "belebele-427": "belebele-435",
   "belebele-43": "belebele-659",
   "belebele-435": "belebele-372",
   "belebele-436": "belebele-42",
   "belebele-443": "belebele-780",
   "belebele-449": "belebele-345",
   "belebele-45": "belebele-547",
   "belebele-452": "belebele-392",
   "belebele-458": "belebele-712",
   "belebele-463": "belebele-862",
   "belebele-466": "belebele-443",
   "belebele-472": "belebele-147",
   "belebele-474": "belebele-24",
   "belebele-483": "belebele-571",
   "belebele-485": "belebele-283",
   "belebele-488": "belebele-810",
   "belebele-490": "belebele-153",
   "belebele-496": "belebele-756",
   "belebele-50": "belebele-514",
   "belebele-507": "belebele-689",
   "belebele-510": "belebele-652",
   "belebele-513": "belebele-832",
   "belebele-514": "belebele-393",
   "belebele-524": "belebele-660",
   "belebele-525": "belebele-1",
   "belebele-547": "belebele-767",
   "belebele-556": "belebele-172",
   "belebele-56": "belebele-801",
   "belebele-569": "belebele-12",
   "belebele-571": "belebele-140",
   "belebele-582": "belebele-710",
   "belebele-595": "belebele-651",
   "belebele-600": "belebele-821",
   "belebele-601": "belebele-45",
   "belebele-608": "belebele-349",
   "belebele-617": "belebele-38",
   "belebele-620": "belebele-23",
   "belebele-627": "belebele-184",
   "belebele-632": "belebele-703",
   "belebele-651": "belebele-145",
   "belebele-652": "belebele-327",
   "belebele-653": "belebele-333",
   "belebele-657": "belebele-510",
   "belebele-659": "belebele-507",
   "belebele-660": "belebele-427",
   "belebele-666": "belebele-413",
   "belebele-668": "belebele-855",
   "belebele-672": "belebele-208",
   "belebele-684": "belebele-452",
   "belebele-689": "belebele-525",
   "belebele-695": "belebele-316",
   "belebele-703": "belebele-249",
   "belebele-705": "belebele-423",
   "belebele-706": "belebele-237",
   "belebele-708": "belebele-556",
   "belebele-710": "belebele-14",
   "belebele-712": "belebele-600",
   "belebele-732": "belebele-847",
   "belebele-736": "belebele-466",
   "belebele-737": "belebele-294",
   "belebele-738": "belebele-251",
   "belebele-739": "belebele-82",
   "belebele-75": "belebele-757",
   "belebele-750": "belebele-215",
   "belebele-753": "belebele-226",
   "belebele-756": "belebele-185",
   "belebele-757": "belebele-231",
   "belebele-760": "belebele-783",
   "belebele-767": "belebele-750",
   "belebele-772": "belebele-672",
   "belebele-780": "belebele-496",
   "belebele-783": "belebele-79",
   "belebele-79": "belebele-43",
   "belebele-798": "belebele-804",
   "belebele-800": "belebele-98",
   "belebele-801": "belebele-760",
   "belebele-804": "belebele-240",
   "belebele-806": "belebele-118",
   "belebele-810": "belebele-833",
   "belebele-814": "belebele-737",
   "belebele-82": "belebele-483",
   "belebele-821": "belebele-409",
   "belebele-832": "belebele-835",
   "belebele-833": "belebele-657",
   "belebele-835": "belebele-343",
   "belebele-847": "belebele-207",
   "belebele-851": "belebele-806",
   "belebele-855": "belebele-297",
   "belebele-862": "belebele-582",
   "belebele-877": "belebele-668",
   "belebele-885": "belebele-247",
   "belebele-92": "belebele-601",
   "belebele-98": "belebele-180"
  },
  "mmlu": {
   "mmlu-10001": "mmlu-12546",
   "mmlu-10138": "mmlu-12726",
   "mmlu-10143": "mmlu-10138",
   "mmlu-10158": "mmlu-5418",
   "mmlu-10159": "mmlu-1658",
   "mmlu-10178": "mmlu-614",
   "mmlu-10357": "mmlu-13331",
   "mmlu-10380": "mmlu-5011",
   "mmlu-10563": "mmlu-6665",
   "mmlu-11064": "mmlu-7409",
   "mmlu-11373": "mmlu-6321",
   "mmlu-11421": "mmlu-3562",
   "mmlu-11463": "mmlu-6085",
   "mmlu-11516": "mmlu-12976",
   "mmlu-11537": "mmlu-6231",
   "mmlu-11656": "mmlu-3518",
   "mmlu-11780": "mmlu-13780",
   "mmlu-11783": "mmlu-6817",
   "mmlu-11963": "mmlu-7068",
   "mmlu-12040": "mmlu-561",
   "mmlu-12136": "mmlu-12786",
   "mmlu-12165": "mmlu-5290",
   "mmlu-12219": "mmlu-7658",
   "mmlu-12286": "mmlu-12870",
   "mmlu-12343": "mmlu-6157",
   "mmlu-1243": "mmlu-1751",
   "mmlu-12459": "mmlu-8708",
   "mmlu-12546": "mmlu-4899",
   "mmlu-12557": "mmlu-4164",
   "mmlu-12583": "mmlu-11656",
   "mmlu-12726": "mmlu-7014",
   "mmlu-12786": "mmlu-4666",
   "mmlu-12794": "mmlu-4556",
   "mmlu-12870": "mmlu-7266",
   "mmlu-12893": "mmlu-6009",
   "mmlu-12958": "mmlu-7410",
   "mmlu-12976": "mmlu-1826",
   "mmlu-13063": "mmlu-3959",
   "mmlu-13243": "mmlu-8212",
   "mmlu-13281": "mmlu-2800",
   "mmlu-1329": "mmlu-11537",
   "mmlu-13331": "mmlu-12459",
   "mmlu-13368": "mmlu-5786",
   "mmlu-13372": "mmlu-4195",
   "mmlu-13399": "mmlu-11373",
   "mmlu-13499": "mmlu-7089",
   "mmlu-13543": "mmlu-7087",
   "mmlu-13576": "mmlu-11463",
   "mmlu-13669": "mmlu-11783",
   "mmlu-13727": "mmlu-7021",
   "mmlu-13780": "mmlu-8844",
   "mmlu-13908": "mmlu-13499",
   "mmlu-1400": "mmlu-9306",
   "mmlu-1445": "mmlu-6940",
   "mmlu-1484": "mmlu-9178",
   "mmlu-1498": "mmlu-13576",
   "mmlu-1658": "mmlu-1400",
   "mmlu-1751": "mmlu-10380",
   "mmlu-1826": "mmlu-6215",
   "mmlu-1935": "mmlu-10143",
   "mmlu-1997": "mmlu-8265",
   "mmlu-2279": "mmlu-7224",
   "mmlu-2367": "mmlu-3979",
   "mmlu-2412": "mmlu-8544",
   "mmlu-2505": "mmlu-83",
   "mmlu-2672": "mmlu-7093",
   "mmlu-2790": "mmlu-6417",
   "mmlu-2800": "mmlu-10158",
   "mmlu-3095": "mmlu-12136",
   "mmlu-3408": "mmlu-13243",
   "mmlu-3494": "mmlu-12343",
   "mmlu-3518": "mmlu-7830",
   "mmlu-352": "mmlu-11421",
   "mmlu-3552": "mmlu-1329",
   "mmlu-3562": "mmlu-7930",
   "mmlu-3631": "mmlu-8112",
   "mmlu-3661": "mmlu-5997",
   "mmlu-3774": "mmlu-6177",
   "mmlu-3789": "mmlu-9242",
   "mmlu-3895": "mmlu-9073",
   "mmlu-3959": "mmlu-9480",
   "mmlu-3979": "mmlu-4969",
   "mmlu-4041": "mmlu-12040",
   "mmlu-4164": "mmlu-736",
   "mmlu-4168": "mmlu-8382",
   "mmlu-4195": "mmlu-3895",
   "mmlu-4200": "mmlu-1445",
   "mmlu-4556": "mmlu-13727",
   "mmlu-4665": "mmlu-12165",
   "mmlu-4666": "mmlu-1484",
   "mmlu-4899": "mmlu-11963",
   "mmlu-4969": "mmlu-12286",
   "mmlu-5011": "mmlu-7425",
   "mmlu-5028": "mmlu-9525",
   "mmlu-5099": "mmlu-13399",
   "mmlu-5109": "mmlu-1935",
   "mmlu-5235": "mmlu-12219",
   "mmlu-5290": "mmlu-2412",
   "mmlu-5393": "mmlu-3552",
   "mmlu-5418": "mmlu-7762",
   "mmlu-5429": "mmlu-8297",
   "mmlu-561": "mmlu-8886",
   "mmlu-5739": "mmlu-11516",
   "mmlu-5786": "mmlu-12958",
   "mmlu-5855": "mmlu-3774",
   "mmlu-5997": "mmlu-6923",
   "mmlu-6009": "mmlu-3789",
   "mmlu-6030": "mmlu-12893",
   "mmlu-6085": "mmlu-9051",
   "mmlu-614": "mmlu-8356",
   "mmlu-6145": "mmlu-9799",
   "mmlu-6152": "mmlu-2672",
   "mmlu-6157": "mmlu-9876",
   "mmlu-6177": "mmlu-10159",
   "mmlu-6195": "mmlu-6278",
   "mmlu-6215": "mmlu-11064",
   "mmlu-6231": "mmlu-9675",
   "mmlu-6278": "mmlu-1498",
   "mmlu-6321": "mmlu-3095",
   "mmlu-6335": "mmlu-13669",
   "mmlu-6417": "mmlu-3661",
   "mmlu-6617": "mmlu-11780",
   "mmlu-6665": "mmlu-7774",
   "mmlu-6817": "mmlu-12583",
   "mmlu-6923": "mmlu-2790",
   "mmlu-6940": "mmlu-8556",
   "mmlu-7014": "mmlu-4665",
   "mmlu-7021": "mmlu-4168",
   "mmlu-7068": "mmlu-13368",
   "mmlu-7087": "mmlu-6030",
   "mmlu-7089": "mmlu-8656",
   "mmlu-7093": "mmlu-13063",
   "mmlu-7224": "mmlu-12557",
   "mmlu-7258": "mmlu-5099",
   "mmlu-7266": "mmlu-4041",
   "mmlu-736": "mmlu-1243",
   "mmlu-7409": "mmlu-13281",
   "mmlu-7410": "mmlu-7675",
   "mmlu-7425": "mmlu-3494",
   "mmlu-7658": "mmlu-2505",
   "mmlu-7672": "mmlu-352",
   "mmlu-7675": "mmlu-5109",
   "mmlu-7762": "mmlu-3408",
   "mmlu-7774": "mmlu-2367",
   "mmlu-7829": "mmlu-1997",
   "mmlu-7830": "mmlu-9704",
   "mmlu-7930": "mmlu-13372",
   "mmlu-8112": "mmlu-13543",
   "mmlu-8212": "mmlu-9465",
   "mmlu-8265": "mmlu-10357",
   "mmlu-8297": "mmlu-10001",
   "mmlu-83": "mmlu-7829",
   "mmlu-8356": "mmlu-8629",
   "mmlu-8382": "mmlu-13908",
   "mmlu-8503": "mmlu-5855",
   "mmlu-8544": "mmlu-6335",
   "mmlu-8556": "mmlu-5028",
   "mmlu-8629": "mmlu-12794",
   "mmlu-8656": "mmlu-6152",
   "mmlu-8679": "mmlu-7258",
   "mmlu-8708": "mmlu-10563",
   "mmlu-8844": "mmlu-8679",
   "mmlu-8886": "mmlu-6145",
   "mmlu-9051": "mmlu-7672",
   "mmlu-9073": "mmlu-5393",
   "mmlu-9178": "mmlu-4200",
   "mmlu-9242": "mmlu-5429",
   "mmlu-9306": "mmlu-2279",
   "mmlu-9465": "mmlu-5235",
   "mmlu-9480": "mmlu-8503",
   "mmlu-9525": "mmlu-5739",
   "mmlu-9675": "mmlu-6617",
   "mmlu-9704": "mmlu-3631",
   "mmlu-9799": "mmlu-6195",
   "mmlu-9876": "mmlu-10178"
  }
 },
 "p1_subsample": {
  "belebele": [
   "belebele-140",
   "belebele-177",
   "belebele-184",
   "belebele-185",
   "belebele-190",
   "belebele-23",
   "belebele-231",
   "belebele-237",
   "belebele-24",
   "belebele-240",
   "belebele-283",
   "belebele-297",
   "belebele-316",
   "belebele-327",
   "belebele-335",
   "belebele-345",
   "belebele-349",
   "belebele-384",
   "belebele-393",
   "belebele-413",
   "belebele-418",
   "belebele-427",
   "belebele-43",
   "belebele-45",
   "belebele-452",
   "belebele-485",
   "belebele-50",
   "belebele-507",
   "belebele-514",
   "belebele-524",
   "belebele-556",
   "belebele-569",
   "belebele-582",
   "belebele-601",
   "belebele-653",
   "belebele-659",
   "belebele-666",
   "belebele-684",
   "belebele-703",
   "belebele-706",
   "belebele-712",
   "belebele-736",
   "belebele-756",
   "belebele-772",
   "belebele-783",
   "belebele-79",
   "belebele-798",
   "belebele-801",
   "belebele-804",
   "belebele-832"
  ],
  "mmlu": [
   "mmlu-10178",
   "mmlu-11537",
   "mmlu-12165",
   "mmlu-12583",
   "mmlu-12893",
   "mmlu-12958",
   "mmlu-13281",
   "mmlu-13368",
   "mmlu-13399",
   "mmlu-1484",
   "mmlu-1498",
   "mmlu-1751",
   "mmlu-2279",
   "mmlu-2367",
   "mmlu-3494",
   "mmlu-3631",
   "mmlu-3895",
   "mmlu-4041",
   "mmlu-4168",
   "mmlu-4200",
   "mmlu-4556",
   "mmlu-4665",
   "mmlu-5418",
   "mmlu-5997",
   "mmlu-6030",
   "mmlu-6145",
   "mmlu-6152",
   "mmlu-6195",
   "mmlu-6665",
   "mmlu-6940",
   "mmlu-7093",
   "mmlu-7224",
   "mmlu-7266",
   "mmlu-7762",
   "mmlu-7829",
   "mmlu-7830",
   "mmlu-8112",
   "mmlu-8265",
   "mmlu-8297",
   "mmlu-8708",
   "mmlu-8886",
   "mmlu-9051",
   "mmlu-9178",
   "mmlu-9242",
   "mmlu-9306",
   "mmlu-9465",
   "mmlu-9480",
   "mmlu-9525",
   "mmlu-9675",
   "mmlu-9704"
  ]
 },
 "payload_sha256": "19bbaa128cb65794bae0587a76f14d47184169806660220159b9157f8a1df93f",
 "rev": {
  "belebele": [
   "belebele-31",
   "belebele-102",
   "belebele-116",
   "belebele-120",
   "belebele-133",
   "belebele-160",
   "belebele-189",
   "belebele-201",
   "belebele-213",
   "belebele-227",
   "belebele-272",
   "belebele-280",
   "belebele-358",
   "belebele-395",
   "belebele-434",
   "belebele-460",
   "belebele-465",
   "belebele-506",
   "belebele-522",
   "belebele-537",
   "belebele-576",
   "belebele-624",
   "belebele-641",
   "belebele-681",
   "belebele-722",
   "belebele-728",
   "belebele-741",
   "belebele-818",
   "belebele-824",
   "belebele-830",
   "belebele-842",
   "belebele-891",
   "belebele-896"
  ],
  "mmlu": [
   "mmlu-619",
   "mmlu-623",
   "mmlu-631",
   "mmlu-816",
   "mmlu-953",
   "mmlu-1105",
   "mmlu-1141",
   "mmlu-1158",
   "mmlu-1798",
   "mmlu-1834",
   "mmlu-2022",
   "mmlu-2051",
   "mmlu-2164",
   "mmlu-3100",
   "mmlu-3305",
   "mmlu-3356",
   "mmlu-3477",
   "mmlu-3623",
   "mmlu-3867",
   "mmlu-4596",
   "mmlu-4626",
   "mmlu-4870",
   "mmlu-5457",
   "mmlu-5521",
   "mmlu-5989",
   "mmlu-6975",
   "mmlu-7187",
   "mmlu-7243",
   "mmlu-7856",
   "mmlu-8001",
   "mmlu-8628",
   "mmlu-8737",
   "mmlu-8822",
   "mmlu-8934",
   "mmlu-8970",
   "mmlu-9149",
   "mmlu-9379",
   "mmlu-9498",
   "mmlu-9530",
   "mmlu-9581",
   "mmlu-10141",
   "mmlu-10786",
   "mmlu-10797",
   "mmlu-12654",
   "mmlu-12772",
   "mmlu-12842",
   "mmlu-13010",
   "mmlu-13078",
   "mmlu-13713"
  ]
 },
 "seed": 20260703,
 "source_sha256": {
  "data/belebele_en.jsonl": "db7d6cddffa0043dafc717b54523db4baac2023d4cbb8a1427437addc5181269",
  "data/belebele_pl.jsonl": "f60e5d4580f24cb8db66c6fbbd1342595671a7878d567c7f1a2cb9ee2e8e8082",
  "data/exclusions.json": "8c316fbe9764607fd3cbc2f518271f2be9b894b8e7f813e8a5fae1f0dcb96c4b",
  "data/mmlu_en.jsonl": "be09c7756aafa25cc1049a9cae2e9a4470588354fada2cac13b64092ac57be4f",
  "data/mmlu_pl.jsonl": "ee3b76d0e983cef9e0a55b40bb0f4bfdd86ee80b3b950595301630d305d86104",
  "results/scores_base.jsonl": "96c21a7f99d40851919297ea083923491f6910ee6749d622b66f8e9993149917"
 },
 "stem": {
  "mmlu-10001": false,
  "mmlu-10006": false,
  "mmlu-10039": false,
  "mmlu-10099": false,
  "mmlu-10138": false,
  "mmlu-10141": false,
  "mmlu-10143": false,
  "mmlu-10158": false,
  "mmlu-10159": false,
  "mmlu-10160": false,
  "mmlu-10178": false,
  "mmlu-10185": false,
  "mmlu-10206": false,
  "mmlu-10247": false,
  "mmlu-10255": false,
  "mmlu-10327": false,
  "mmlu-10357": false,
  "mmlu-10380": false,
  "mmlu-10384": false,
  "mmlu-10398": false,
  "mmlu-10405": false,
  "mmlu-10494": false,
  "mmlu-10496": false,
  "mmlu-10528": false,
  "mmlu-10541": false,
  "mmlu-10563": false,
  "mmlu-1058": true,
  "mmlu-10583": false,
  "mmlu-10620": false,
  "mmlu-10624": false,
  "mmlu-10697": false,
  "mmlu-10708": false,
  "mmlu-10733": false,
  "mmlu-10786": false,
  "mmlu-10797": false,
  "mmlu-10856": false,
  "mmlu-10860": false,
  "mmlu-10871": false,
  "mmlu-10901": false,
  "mmlu-10932": false,
  "mmlu-10950": false,
  "mmlu-10956": false,
  "mmlu-10991": false,
  "mmlu-1105": true,
  "mmlu-11064": false,
  "mmlu-11152": false,
  "mmlu-11158": false,
  "mmlu-11166": false,
  "mmlu-11184": false,
  "mmlu-11207": false,
  "mmlu-11208": false,
  "mmlu-1123": true,
  "mmlu-11290": false,
  "mmlu-11293": false,
  "mmlu-11357": false,
  "mmlu-11373": false,
  "mmlu-11400": false,
  "mmlu-1141": true,
  "mmlu-11421": false,
  "mmlu-11463": false,
  "mmlu-11496": false,
  "mmlu-11504": false,
  "mmlu-11508": false,
  "mmlu-11516": false,
  "mmlu-11520": false,
  "mmlu-11537": false,
  "mmlu-1158": true,
  "mmlu-11656": false,
  "mmlu-11687": false,
  "mmlu-11756": false,
  "mmlu-11771": false,
  "mmlu-11780": false,
  "mmlu-11783": false,
  "mmlu-11796": false,
  "mmlu-11805": false,
  "mmlu-11828": false,
  "mmlu-11867": false,
  "mmlu-11888": false,
  "mmlu-11928": false,
  "mmlu-11943": false,
  "mmlu-11963": false,
  "mmlu-11985": false,
  "mmlu-1199": false,
  "mmlu-12015": false,
  "mmlu-12024": false,
  "mmlu-12040": false,
  "mmlu-1205": false,
  "mmlu-12116": false,
  "mmlu-12119": false,
  "mmlu-12136": false,
  "mmlu-12143": false,
  "mmlu-12145": false,
  "mmlu-12158": false,
  "mmlu-12165": false,
  "mmlu-1221": false,
  "mmlu-12219": false,
  "mmlu-12286": false,
  "mmlu-12343": false,
  "mmlu-12400": false,
  "mmlu-1243": false,
  "mmlu-12459": false,
  "mmlu-12517": false,
  "mmlu-12518": false,
  "mmlu-12546": false,
  "mmlu-12557": false,
  "mmlu-1256": false,
  "mmlu-12577": false,
  "mmlu-12583": false,
  "mmlu-12608": false,
  "mmlu-12654": false,
  "mmlu-12660": false,
  "mmlu-12680": false,
  "mmlu-12726": false,
  "mmlu-12740": false,
  "mmlu-12763": false,
  "mmlu-12768": false,
  "mmlu-12772": false,
  "mmlu-12786": false,
  "mmlu-12794": false,
  "mmlu-12810": false,
  "mmlu-12815": false,
  "mmlu-12821": false,
  "mmlu-12842": false,
  "mmlu-12870": false,
  "mmlu-12893": false,
  "mmlu-12925": false,
  "mmlu-12958": false,
  "mmlu-12963": false,
  "mmlu-12976": false,
  "mmlu-13008": false,
  "mmlu-13010": false,
  "mmlu-13023": false,
  "mmlu-1304": false,
  "mmlu-13052": false,
  "mmlu-13063": false,
  "mmlu-13066": false,
  "mmlu-13078": false,
  "mmlu-13087": false,
  "mmlu-13122": false,
  "mmlu-13197": false,
  "mmlu-13243": false,
  "mmlu-13268": false,
  "mmlu-13281": false,
  "mmlu-1329": false,
  "mmlu-13331": false,
  "mmlu-13368": false,
  "mmlu-13372": false,
  "mmlu-13399": false,
  "mmlu-13484": false,
  "mmlu-13490": false,
  "mmlu-13499": false,
  "mmlu-135": true,
  "mmlu-13543": false,
  "mmlu-13562": false,
  "mmlu-13576": false,
  "mmlu-13613": false,
  "mmlu-13669": false,
  "mmlu-13713": false,
  "mmlu-13716": false,
  "mmlu-13727": false,
  "mmlu-13740": false,
  "mmlu-13767": false,
  "mmlu-13780": false,
  "mmlu-1379": true,
  "mmlu-13798": false,
  "mmlu-13805": false,
  "mmlu-13817": false,
  "mmlu-13842": false,
  "mmlu-13861": false,
  "mmlu-13892": false,
  "mmlu-13898": false,
  "mmlu-13908": false,
  "mmlu-13968": false,
  "mmlu-13971": false,
  "mmlu-13979": false,
  "mmlu-1400": true,
  "mmlu-14008": false,
  "mmlu-1445": true,
  "mmlu-1484": true,
  "mmlu-1497": true,
  "mmlu-1498": true,
  "mmlu-1517": true,
  "mmlu-1610": true,
  "mmlu-1613": true,
  "mmlu-1614": true,
  "mmlu-165": true,
  "mmlu-1658": true,
  "mmlu-1717": true,
  "mmlu-1737": true,
  "mmlu-1751": true,
  "mmlu-1798": true,
  "mmlu-1813": false,
  "mmlu-1826": false,
  "mmlu-1834": false,
  "mmlu-1841": false,
  "mmlu-1888": false,
  "mmlu-1918": false,
  "mmlu-1926": true,
  "mmlu-1935": true,
  "mmlu-1945": true,
  "mmlu-1960": true,
  "mmlu-1997": true,
  "mmlu-2022": true,
  "mmlu-2040": true,
  "mmlu-2043": true,
  "mmlu-2051": true,
  "mmlu-2070": true,
  "mmlu-2080": true,
  "mmlu-2097": true,
  "mmlu-2107": true,
  "mmlu-2134": true,
  "mmlu-2164": true,
  "mmlu-2169": true,
  "mmlu-2227": true,
  "mmlu-2265": true,
  "mmlu-2279": true,
  "mmlu-2300": true,
  "mmlu-2310": true,
  "mmlu-2314": true,
  "mmlu-2348": true,
  "mmlu-2367": true,
  "mmlu-2383": true,
  "mmlu-2384": true,
  "mmlu-2386": true,
  "mmlu-2412": true,
  "mmlu-2418": true,
  "mmlu-2424": true,
  "mmlu-2432": true,
  "mmlu-2505": false,
  "mmlu-2515": false,
  "mmlu-2534": false,
  "mmlu-2562": false,
  "mmlu-2583": false,
  "mmlu-2663": false,
  "mmlu-2672": true,
  "mmlu-2716": true,
  "mmlu-2721": true,
  "mmlu-2790": true,
  "mmlu-2800": true,
  "mmlu-2930": true,
  "mmlu-2945": true,
  "mmlu-2960": true,
  "mmlu-2961": true,
  "mmlu-2990": true,
  "mmlu-3009": true,
  "mmlu-3026": true,
  "mmlu-3075": true,
  "mmlu-3092": true,
  "mmlu-3095": true,
  "mmlu-3100": true,
  "mmlu-3136": true,
  "mmlu-3162": true,
  "mmlu-3199": true,
  "mmlu-3208": true,
  "mmlu-3250": true,
  "mmlu-3277": true,
  "mmlu-3305": false,
  "mmlu-3330": false,
  "mmlu-3339": false,
  "mmlu-3352": false,
  "mmlu-3356": false,
  "mmlu-3369": false,
  "mmlu-3408": false,
  "mmlu-3428": false,
  "mmlu-3433": false,
  "mmlu-3440": false,
  "mmlu-3477": false,
  "mmlu-3484": false,
  "mmlu-3494": false,
  "mmlu-3504": false,
  "mmlu-3508": false,
  "mmlu-3518": false,
  "mmlu-352": true,
  "mmlu-3542": false,
  "mmlu-3547": false,
  "mmlu-3550": false,
  "mmlu-3552": false,
  "mmlu-3562": false,
  "mmlu-3563": false,
  "mmlu-3581": false,
  "mmlu-3591": false,
  "mmlu-3623": false,
  "mmlu-3631": false,
  "mmlu-3661": false,
  "mmlu-3715": false,
  "mmlu-3720": false,
  "mmlu-3725": false,
  "mmlu-3774": false,
  "mmlu-3783": false,
  "mmlu-3784": false,
  "mmlu-3789": false,
  "mmlu-3867": false,
  "mmlu-3884": false,
  "mmlu-3895": false,
  "mmlu-3922": false,
  "mmlu-3959": false,
  "mmlu-3961": false,
  "mmlu-397": false,
  "mmlu-3979": false,
  "mmlu-3988": false,
  "mmlu-4041": false,
  "mmlu-4070": false,
  "mmlu-4164": false,
  "mmlu-4168": false,
  "mmlu-4172": false,
  "mmlu-418": false,
  "mmlu-4180": false,
  "mmlu-419": false,
  "mmlu-4195": false,
  "mmlu-4200": false,
  "mmlu-4207": false,
  "mmlu-4226": false,
  "mmlu-4245": true,
  "mmlu-4252": true,
  "mmlu-428": false,
  "mmlu-4409": true,
  "mmlu-4472": true,
  "mmlu-4523": false,
  "mmlu-4545": false,
  "mmlu-455": false,
  "mmlu-4556": false,
  "mmlu-4559": false,
  "mmlu-4589": false,
  "mmlu-4596": false,
  "mmlu-4620": false,
  "mmlu-4626": false,
  "mmlu-4665": false,
  "mmlu-4666": false,
  "mmlu-467": false,
  "mmlu-4686": false,
  "mmlu-4709": false,
  "mmlu-4733": false,
  "mmlu-4750": true,
  "mmlu-4756": true,
  "mmlu-4770": true,
  "mmlu-4792": true,
  "mmlu-484": false,
  "mmlu-4845": true,
  "mmlu-4870": true,
  "mmlu-4899": false,
  "mmlu-4943": false,
  "mmlu-4966": false,
  "mmlu-4969": false,
  "mmlu-4970": false,
  "mmlu-5011": false,
  "mmlu-5018": false,
  "mmlu-5019": false,
  "mmlu-5028": false,
  "mmlu-5099": false,
  "mmlu-5109": false,
  "mmlu-513": false,
  "mmlu-5164": false,
  "mmlu-5235": false,
  "mmlu-5242": false,
  "mmlu-5290": false,
  "mmlu-5385": false,
  "mmlu-5393": false,
  "mmlu-5405": false,
  "mmlu-5406": false,
  "mmlu-5418": false,
  "mmlu-5429": false,
  "mmlu-5430": false,
  "mmlu-5457": true,
  "mmlu-5471": true,
  "mmlu-5494": true,
  "mmlu-5521": true,
  "mmlu-5532": true,
  "mmlu-5581": true,
  "mmlu-5589": true,
  "mmlu-5597": true,
  "mmlu-561": false,
  "mmlu-565": false,
  "mmlu-566": false,
  "mmlu-5665": false,
  "mmlu-5691": false,
  "mmlu-5732": false,
  "mmlu-5739": false,
  "mmlu-5762": false,
  "mmlu-5786": false,
  "mmlu-5808": false,
  "mmlu-5812": false,
  "mmlu-582": false,
  "mmlu-585": false,
  "mmlu-5853": false,
  "mmlu-5855": false,
  "mmlu-5859": false,
  "mmlu-5868": false,
  "mmlu-5945": false,
  "mmlu-5984": false,
  "mmlu-5989": false,
  "mmlu-5997": false,
  "mmlu-6009": false,
  "mmlu-6013": false,
  "mmlu-6030": false,
  "mmlu-6085": false,
  "mmlu-614": false,
  "mmlu-6140": false,
  "mmlu-6145": false,
  "mmlu-6152": false,
  "mmlu-6157": false,
  "mmlu-6177": false,
  "mmlu-6181": false,
  "mmlu-619": false,
  "mmlu-6195": false,
  "mmlu-6200": false,
  "mmlu-6215": false,
  "mmlu-623": false,
  "mmlu-6231": false,
  "mmlu-6278": false,
  "mmlu-63": true,
  "mmlu-631": false,
  "mmlu-6321": false,
  "mmlu-6335": false,
  "mmlu-6378": false,
  "mmlu-6384": false,
  "mmlu-6391": false,
  "mmlu-6417": false,
  "mmlu-6546": false,
  "mmlu-657": false,
  "mmlu-658": false,
  "mmlu-6585": false,
  "mmlu-6587": false,
  "mmlu-6593": false,
  "mmlu-6617": false,
  "mmlu-666": false,
  "mmlu-6665": false,
  "mmlu-667": false,
  "mmlu-6742": false,
  "mmlu-6763": false,
  "mmlu-6817": false,
  "mmlu-6829": false,
  "mmlu-6832": false,
  "mmlu-6838": true,
  "mmlu-6923": true,
  "mmlu-6927": true,
  "mmlu-6940": true,
  "mmlu-6975": false,
  "mmlu-6978": false,
  "mmlu-7002": false,
  "mmlu-7014": false,
  "mmlu-7021": false,
  "mmlu-7047": false,
  "mmlu-7068": false,
  "mmlu-7082": false,
  "mmlu-7085": false,
  "mmlu-7087": false,
  "mmlu-7089": false,
  "mmlu-709": false,
  "mmlu-7093": false,
  "mmlu-7104": false,
  "mmlu-7108": false,
  "mmlu-7128": false,
  "mmlu-7187": false,
  "mmlu-7224": false,
  "mmlu-7243": false,
  "mmlu-7258": false,
  "mmlu-7266": false,
  "mmlu-7274": false,
  "mmlu-7335": false,
  "mmlu-736": false,
  "mmlu-7367": false,
  "mmlu-7379": false,
  "mmlu-7382": false,
  "mmlu-7403": false,
  "mmlu-7409": false,
  "mmlu-7410": false,
  "mmlu-7425": false,
  "mmlu-7426": false,
  "mmlu-7436": false,
  "mmlu-7465": false,
  "mmlu-7467": false,
  "mmlu-75": true,
  "mmlu-7507": false,
  "mmlu-7558": false,
  "mmlu-756": true,
  "mmlu-7577": false,
  "mmlu-759": true,
  "mmlu-7594": false,
  "mmlu-7658": false,
  "mmlu-7663": false,
  "mmlu-7672": false,
  "mmlu-7675": false,
  "mmlu-7733": false,
  "mmlu-7747": false,
  "mmlu-7762": false,
  "mmlu-7774": false,
  "mmlu-7791": false,
  "mmlu-7820": false,
  "mmlu-7829": false,
  "mmlu-7830": false,
  "mmlu-7856": false,
  "mmlu-7907": false,
  "mmlu-792": true,
  "mmlu-7930": false,
  "mmlu-7980": false,
  "mmlu-8001": false,
  "mmlu-8024": false,
  "mmlu-8030": false,
  "mmlu-8068": false,
  "mmlu-8074": false,
  "mmlu-8087": false,
  "mmlu-8091": false,
  "mmlu-8112": false,
  "mmlu-8136": false,
  "mmlu-8137": false,
  "mmlu-816": true,
  "mmlu-8168": false,
  "mmlu-8177": false,
  "mmlu-8184": false,
  "mmlu-8187": false,
  "mmlu-8205": false,
  "mmlu-8212": false,
  "mmlu-8215": false,
  "mmlu-8265": false,
  "mmlu-8278": false,
  "mmlu-8297": false,
  "mmlu-83": true,
  "mmlu-830": true,
  "mmlu-8356": false,
  "mmlu-8382": false,
  "mmlu-8414": false,
  "mmlu-8484": false,
  "mmlu-8500": false,
  "mmlu-8503": false,
  "mmlu-8520": false,
  "mmlu-8528": false,
  "mmlu-8544": false,
  "mmlu-8556": false,
  "mmlu-8628": false,
  "mmlu-8629": false,
  "mmlu-8632": false,
  "mmlu-8642": false,
  "mmlu-8654": false,
  "mmlu-8656": false,
  "mmlu-8665": false,
  "mmlu-8677": false,
  "mmlu-8679": false,
  "mmlu-8696": false,
  "mmlu-8708": false,
  "mmlu-8717": false,
  "mmlu-8737": false,
  "mmlu-8760": false,
  "mmlu-8769": false,
  "mmlu-8811": false,
  "mmlu-8822": false,
  "mmlu-8828": false,
  "mmlu-8844": false,
  "mmlu-886": true,
  "mmlu-8865": false,
  "mmlu-8874": false,
  "mmlu-8886": false,
  "mmlu-8927": false,
  "mmlu-8934": false,
  "mmlu-8953": false,
  "mmlu-8970": false,
  "mmlu-8989": false,
  "mmlu-9051": false,
  "mmlu-906": true,
  "mmlu-9073": false,
  "mmlu-9100": false,
  "mmlu-912": true,
  "mmlu-9149": false,
  "mmlu-9178": false,
  "mmlu-9196": false,
  "mmlu-9211": false,
  "mmlu-9242": false,
  "mmlu-9274": false,
  "mmlu-9289": false,
  "mmlu-9306": false,
  "mmlu-9307": false,
  "mmlu-934": true,
  "mmlu-9379": false,
  "mmlu-9399": false,
  "mmlu-9465": false,
  "mmlu-9480": false,
  "mmlu-9486": false,
  "mmlu-9498": false,
  "mmlu-9509": false,
  "mmlu-9510": false,
  "mmlu-9516": false,
  "mmlu-9520": false,
  "mmlu-9525": false,
  "mmlu-9526": false,
  "mmlu-953": true,
  "mmlu-9530": false,
  "mmlu-9535": false,
  "mmlu-9581": false,
  "mmlu-9640": false,
  "mmlu-965": true,
  "mmlu-9675": false,
  "mmlu-9690": false,
  "mmlu-9704": false,
  "mmlu-9725": false,
  "mmlu-9782": false,
  "mmlu-9799": false,
  "mmlu-9849": false,
  "mmlu-9876": false,
  "mmlu-9881": false,
  "mmlu-9899": false,
  "mmlu-9997": false
 }
}