{
 "B": 2000,
 "_generated_by": "scripts/03_analyze.py",
 "benchmarks": {
  "belebele": {
   "G": {
    "armA_500M": {
     "G": 0.17725752508361203,
     "ci95": [
      0.13776215205619385,
      0.2167528981110302
     ],
     "discordant_en_only": 135,
     "discordant_pl_only": 29,
     "n": 598,
     "sign_test_p": 1.4977681814745146e-17
    },
    "armB_500M": {
     "G": 0.1354515050167224,
     "ci95": [
      0.09557478117197746,
      0.17532822886146734
     ],
     "discordant_en_only": 120,
     "discordant_pl_only": 39,
     "n": 598,
     "sign_test_p": 8.658876808353174e-11
    },
    "base": {
     "G": 0.19732441471571907,
     "ci95": [
      0.1557739309192655,
      0.23887489851217264
     ],
     "discordant_en_only": 151,
     "discordant_pl_only": 33,
     "n": 598,
     "sign_test_p": 3.0906782823479846e-19
    }
   },
   "cells": {
    "armA_500M": {
     "en": {
      "acc": 0.6086956521739131,
      "floor": false,
      "n": 598,
      "wilson_ci95": [
       0.5690046837930285,
       0.6469990462503197
      ]
     },
     "en_raw_acc": 0.6086956521739131,
     "pl": {
      "acc": 0.431438127090301,
      "floor": false,
      "n": 598,
      "wilson_ci95": [
       0.3923043183185738,
       0.47144717503869893
      ]
     },
     "pl_raw_acc": 0.431438127090301
    },
    "armB_500M": {
     "en": {
      "acc": 0.6103678929765887,
      "floor": false,
      "n": 598,
      "wilson_ci95": [
       0.570696050163358,
       0.6486308141883493
      ]
     },
     "en_raw_acc": 0.6103678929765887,
     "pl": {
      "acc": 0.47491638795986624,
      "floor": false,
      "n": 598,
      "wilson_ci95": [
       0.43518012595041417,
       0.5149728594241979
      ]
     },
     "pl_raw_acc": 0.47491638795986624
    },
    "base": {
     "en": {
      "acc": 0.745819397993311,
      "floor": false,
      "n": 598,
      "wilson_ci95": [
       0.7094297572792087,
       0.7790709860495938
      ]
     },
     "en_raw_acc": 0.745819397993311,
     "pl": {
      "acc": 0.5484949832775919,
      "floor": false,
      "n": 598,
      "wilson_ci95": [
       0.5084262305673161,
       0.5879446643751007
      ]
     },
     "pl_raw_acc": 0.5484949832775919
    }
   },
   "contrasts": {
    "dG_CPT": {
     "ci95": [
      -0.07023411371237459,
      0.030142140468227337
     ],
     "est": -0.020066889632106955
    },
    "dG_TRANS": {
     "ci95": [
      -0.0903010033444816,
      0.010033444816053616
     ],
     "est": -0.04180602006688966
    },
    "dacc_EN_CPT": {
     "ci95": [
      -0.17391304347826086,
      -0.10200668896321063
     ],
     "est": -0.1371237458193979
    },
    "dacc_EN_TRANS": {
     "ci95": [
      -0.03846153846153855,
      0.041806020066889715
     ],
     "est": 0.0016722408026755842
    },
    "dacc_PL_CPT": {
     "ci95": [
      -0.16053511705685614,
      -0.07023411371237454
     ],
     "est": -0.11705685618729095
    },
    "dacc_PL_TRANS": {
     "ci95": [
      -0.0050167224080267525,
      0.09364548494983277
     ],
     "est": 0.043478260869565244
    }
   },
   "floor_cells": [],
   "n_pairs": 598
  },
  "mmlu": {
   "G": {
    "armA_500M": {
     "G": 0.11035653650254669,
     "ci95": [
      0.06962530091856584,
      0.15108777208652754
     ],
     "discordant_en_only": 111,
     "discordant_pl_only": 46,
     "n": 589,
     "sign_test_p": 2.215948407640851e-07
    },
    "armB_500M": {
     "G": 0.09507640067911714,
     "ci95": [
      0.05882592275038567,
      0.1313268786078486
     ],
     "discordant_en_only": 90,
     "discordant_pl_only": 34,
     "n": 589,
     "sign_test_p": 5.088330396963953e-07
    },
    "base": {
     "G": 0.21392190152801357,
     "ci95": [
      0.16721119546823815,
      0.260632607587789
     ],
     "discordant_en_only": 175,
     "discordant_pl_only": 49,
     "n": 589,
     "sign_test_p": 8.381452374481766e-18
    }
   },
   "cells": {
    "armA_500M": {
     "en": {
      "acc": 0.4533106960950764,
      "floor": false,
      "n": 589,
      "wilson_ci95": [
       0.4135395294718148,
       0.4936869318667556
      ]
     },
     "en_raw_acc": 0.4533106960950764,
     "pl": {
      "acc": 0.34295415959252973,
      "floor": false,
      "n": 589,
      "wilson_ci95": [
       0.30574667594233007,
       0.38219687583286127
      ]
     },
     "pl_raw_acc": 0.34295415959252973
    },
    "armB_500M": {
     "en": {
      "acc": 0.45840407470288624,
      "floor": false,
      "n": 589,
      "wilson_ci95": [
       0.41856380130612886,
       0.4987834097045974
      ]
     },
     "en_raw_acc": 0.45840407470288624,
     "pl": {
      "acc": 0.3633276740237691,
      "floor": false,
      "n": 589,
      "wilson_ci95": [
       0.32548754422952697,
       0.40293900623428813
      ]
     },
     "pl_raw_acc": 0.3633276740237691
    },
    "base": {
     "en": {
      "acc": 0.5568760611205433,
      "floor": false,
      "n": 589,
      "wilson_ci95": [
       0.5165186489456313,
       0.5964963890601104
      ]
     },
     "en_raw_acc": 0.5568760611205433,
     "pl": {
      "acc": 0.34295415959252973,
      "floor": false,
      "n": 589,
      "wilson_ci95": [
       0.30574667594233007,
       0.38219687583286127
      ]
     },
     "pl_raw_acc": 0.34295415959252973
    }
   },
   "contrasts": {
    "dG_CPT": {
     "ci95": [
      -0.15959252971137516,
      -0.0492359932088286
     ],
     "est": -0.10356536502546693
    },
    "dG_TRANS": {
     "ci95": [
      -0.06621392190152803,
      0.03569609507640045
     ],
     "est": -0.01528013582342952
    },
    "dacc_EN_CPT": {
     "ci95": [
      -0.14091680814940577,
      -0.06451612903225806
     ],
     "est": -0.10356536502546693
    },
    "dacc_EN_TRANS": {
     "ci95": [
      -0.0373514431239389,
      0.04923599320882849
     ],
     "est": 0.00509337860780984
    },
    "dacc_PL_CPT": {
     "ci95": [
      -0.04584040747028861,
      0.04244482173174874
     ],
     "est": 0.0
    },
    "dacc_PL_TRANS": {
     "ci95": [
      -0.035653650254668934,
      0.07304753820033935
     ],
     "est": 0.02037351443123936
    }
   },
   "floor_cells": [],
   "n_pairs": 589
  }
 },
 "exclusions_applied": {
  "belebele-401": "options B and D are the identical string '2' in both languages (source-inherited duplicate distractor), so scoring is ill-defined; PL question additionally contains a dittography ('ile os\u00f3b' repeated)",
  "belebele-431": "PL option B duplicates option A; the correct answer (EN gold B 'Its crash landing was a success') is absent from the Polish item \u2014 asymmetrically unanswerable in PL, would bias G against Polish",
  "mmlu-10932": "PL gold option B empty (text dropped by MT) \u2014 gold points at a blank",
  "mmlu-11400": "PL distractor option B empty (text dropped by MT) \u2014 PL choice set structurally degraded (gold A intact but option-set asymmetric)",
  "mmlu-2721": "cloze answer-leak: PL gold text ('macierzy mitochondrialnej') fused into the stem, option A empty \u2014 gold points at a blank and the answer is leaked",
  "mmlu-3484": "PL gold 'Centripetal' mistranslated as 'Od\u015brodkowa' (centrifugal), inverting the gold meaning AND duplicating option B \u2014 broken in PL",
  "mmlu-4523": "cloze shift: PL distractor A absorbed into stem, option A empty \u2014 PL choice set structurally degraded (gold D intact but option-set asymmetric)",
  "mmlu-5242": "cloze answer-leak: PL gold text ('limbic system') fused into the stem, option A empty \u2014 gold points at a blank and the answer is leaked",
  "mmlu-8632": "moral_scenarios MT collapse: all 4 PL options identical \u2014 unanswerable in PL",
  "mmlu-8642": "moral_scenarios MT collapse: all 4 PL options identical ('Niew\u0142a\u015bciwe, Niew\u0142a\u015bciwe') \u2014 unanswerable in PL",
  "mmlu-8927": "moral_scenarios MT collapse: all 4 PL options identical \u2014 unanswerable in PL",
  "mmlu-9211": "moral_scenarios MT collapse: all 4 PL options identical \u2014 unanswerable in PL",
  "mmlu-9307": "moral_scenarios MT collapse: all 4 PL options identical \u2014 unanswerable in PL"
 },
 "h4_mmlu_stem": {
  "n_nonstem": 478,
  "n_stem": 111,
  "per_model": {
   "armA_500M": {
    "G_nonstem": {
     "G": 0.1192468619246862,
     "ci95": [
      0.07210130907938604,
      0.16639241476998634
     ],
     "discordant_en_only": 98,
     "discordant_pl_only": 41,
     "n": 478,
     "sign_test_p": 1.470616773975362e-06
    },
    "G_stem": {
     "G": 0.07207207207207207,
     "ci95": [
      -0.0016320636209905537,
      0.1457762077651347
     ],
     "discordant_en_only": 13,
     "discordant_pl_only": 5,
     "n": 111,
     "sign_test_p": 0.09625244140625
    },
    "dd_stem_minus_nonstem": -0.04717478985261413
   },
   "armB_500M": {
    "G_nonstem": {
     "G": 0.100418410041841,
     "ci95": [
      0.05958345674969091,
      0.14125336333399108
     ],
     "discordant_en_only": 76,
     "discordant_pl_only": 28,
     "n": 478,
     "sign_test_p": 2.7367805899850523e-06
    },
    "G_stem": {
     "G": 0.07207207207207207,
     "ci95": [
      -0.005747349985105316,
      0.14989149412924946
     ],
     "discordant_en_only": 14,
     "discordant_pl_only": 6,
     "n": 111,
     "sign_test_p": 0.11531829833984375
    },
    "dd_stem_minus_nonstem": -0.02834633796976893
   },
   "base": {
    "G_nonstem": {
     "G": 0.2510460251046025,
     "ci95": [
      0.1988774009117803,
      0.3032146492974247
     ],
     "discordant_en_only": 156,
     "discordant_pl_only": 36,
     "n": 478,
     "sign_test_p": 5.265339191747883e-19
    },
    "G_stem": {
     "G": 0.05405405405405406,
     "ci95": [
      -0.04532344339833817,
      0.15343155150644627
     ],
     "discordant_en_only": 19,
     "discordant_pl_only": 13,
     "n": 111,
     "sign_test_p": 0.37708558747544885
    },
    "dd_stem_minus_nonstem": -0.19699197105054844
   }
  }
 },
 "primary": "correct_lennorm",
 "seed": 20260703,
 "verdicts": {
  "H1": {
   "G_base_belebele": 0.19732441471571907,
   "ci95": [
    0.1557739309192655,
    0.23887489851217264
   ],
   "confirmed": true,
   "effect_bin": "large"
  },
  "H2": {
   "access_gain": false,
   "dG_CPT_belebele": {
    "ci95": [
     -0.07023411371237459,
     0.030142140468227337
    ],
    "est": -0.020066889632106955
   },
   "dacc_EN": {
    "ci95": [
     -0.17391304347826086,
     -0.10200668896321063
    ],
    "est": -0.1371237458193979
   },
   "dacc_PL": {
    "ci95": [
     -0.16053511705685614,
     -0.07023411371237454
    ],
    "est": -0.11705685618729095
   },
   "narrows": false
  },
  "H3": {
   "dG_TRANS_belebele": {
    "ci95": [
     -0.0903010033444816,
     0.010033444816053616
    ],
    "est": -0.04180602006688966
   },
   "null_bin": false
  },
  "floor_demotions": {
   "belebele": [],
   "mmlu": []
  }
 }
}