{ "dataset_revision": "18193f187b92da67168c655c9973a165ed9593dd", "task_name": "MasakhaNEWSClassification", "mteb_version": "1.18.0", "scores": { "test": [ { "accuracy": 0.347872, "f1": 0.338868, "f1_weighted": 0.342049, "scores_per_experiment": [ { "accuracy": 0.319149, "f1": 0.308769, "f1_weighted": 0.314001 }, { "accuracy": 0.327128, "f1": 0.305988, "f1_weighted": 0.315422 }, { "accuracy": 0.369681, "f1": 0.349712, "f1_weighted": 0.35624 }, { "accuracy": 0.343085, "f1": 0.341253, "f1_weighted": 0.343349 }, { "accuracy": 0.340426, "f1": 0.341654, "f1_weighted": 0.343095 }, { "accuracy": 0.409574, "f1": 0.40828, "f1_weighted": 0.409315 }, { "accuracy": 0.359043, "f1": 0.356651, "f1_weighted": 0.353823 }, { "accuracy": 0.369681, "f1": 0.367046, "f1_weighted": 0.364763 }, { "accuracy": 0.343085, "f1": 0.311387, "f1_weighted": 0.319934 }, { "accuracy": 0.297872, "f1": 0.297936, "f1_weighted": 0.300543 } ], "main_score": 0.347872, "hf_subset": "amh", "languages": [ "amh-Ethi" ] }, { "accuracy": 0.806646, "f1": 0.799478, "f1_weighted": 0.805218, "scores_per_experiment": [ { "accuracy": 0.816456, "f1": 0.811345, "f1_weighted": 0.815425 }, { "accuracy": 0.81962, "f1": 0.812371, "f1_weighted": 0.816665 }, { "accuracy": 0.803797, "f1": 0.798412, "f1_weighted": 0.804405 }, { "accuracy": 0.78903, "f1": 0.781392, "f1_weighted": 0.787209 }, { "accuracy": 0.828059, "f1": 0.822948, "f1_weighted": 0.827682 }, { "accuracy": 0.798523, "f1": 0.790249, "f1_weighted": 0.795943 }, { "accuracy": 0.790084, "f1": 0.781683, "f1_weighted": 0.788461 }, { "accuracy": 0.796414, "f1": 0.788256, "f1_weighted": 0.795502 }, { "accuracy": 0.790084, "f1": 0.780111, "f1_weighted": 0.788244 }, { "accuracy": 0.834388, "f1": 0.82801, "f1_weighted": 0.832643 } ], "main_score": 0.806646, "hf_subset": "eng", "languages": [ "eng-Latn" ] }, { "accuracy": 0.753555, "f1": 0.714576, "f1_weighted": 0.757269, "scores_per_experiment": [ { "accuracy": 0.793839, "f1": 0.766869, "f1_weighted": 0.79004 }, { "accuracy": 0.770142, "f1": 0.724844, "f1_weighted": 0.771444 }, { "accuracy": 0.78673, "f1": 0.745795, "f1_weighted": 0.791227 }, { "accuracy": 0.701422, "f1": 0.668881, "f1_weighted": 0.703283 }, { "accuracy": 0.765403, "f1": 0.721535, "f1_weighted": 0.767616 }, { "accuracy": 0.758294, "f1": 0.711791, "f1_weighted": 0.763426 }, { "accuracy": 0.741706, "f1": 0.706639, "f1_weighted": 0.753777 }, { "accuracy": 0.746445, "f1": 0.708427, "f1_weighted": 0.743529 }, { "accuracy": 0.7891, "f1": 0.740266, "f1_weighted": 0.793076 }, { "accuracy": 0.682464, "f1": 0.650714, "f1_weighted": 0.695273 } ], "main_score": 0.753555, "hf_subset": "fra", "languages": [ "fra-Latn" ] }, { "accuracy": 0.646939, "f1": 0.62679, "f1_weighted": 0.642514, "scores_per_experiment": [ { "accuracy": 0.631083, "f1": 0.616246, "f1_weighted": 0.630539 }, { "accuracy": 0.634223, "f1": 0.613858, "f1_weighted": 0.632116 }, { "accuracy": 0.66248, "f1": 0.639131, "f1_weighted": 0.652593 }, { "accuracy": 0.635793, "f1": 0.613909, "f1_weighted": 0.629632 }, { "accuracy": 0.613815, "f1": 0.58847, "f1_weighted": 0.603054 }, { "accuracy": 0.682889, "f1": 0.663441, "f1_weighted": 0.679705 }, { "accuracy": 0.643642, "f1": 0.628727, "f1_weighted": 0.64743 }, { "accuracy": 0.687598, "f1": 0.675133, "f1_weighted": 0.686681 }, { "accuracy": 0.610675, "f1": 0.589811, "f1_weighted": 0.6093 }, { "accuracy": 0.66719, "f1": 0.63917, "f1_weighted": 0.654086 } ], "main_score": 0.646939, "hf_subset": "hau", "languages": [ "hau-Latn" ] }, { "accuracy": 0.606154, "f1": 0.587164, "f1_weighted": 0.597866, "scores_per_experiment": [ { "accuracy": 0.597436, "f1": 0.572067, "f1_weighted": 0.598838 }, { "accuracy": 0.661538, "f1": 0.632335, "f1_weighted": 0.654727 }, { "accuracy": 0.612821, "f1": 0.596571, "f1_weighted": 0.607181 }, { "accuracy": 0.638462, "f1": 0.624368, "f1_weighted": 0.625121 }, { "accuracy": 0.561538, "f1": 0.555232, "f1_weighted": 0.538813 }, { "accuracy": 0.625641, "f1": 0.611973, "f1_weighted": 0.623809 }, { "accuracy": 0.605128, "f1": 0.582202, "f1_weighted": 0.596392 }, { "accuracy": 0.582051, "f1": 0.552043, "f1_weighted": 0.556959 }, { "accuracy": 0.576923, "f1": 0.569257, "f1_weighted": 0.575813 }, { "accuracy": 0.6, "f1": 0.575586, "f1_weighted": 0.601007 } ], "main_score": 0.606154, "hf_subset": "ibo", "languages": [ "ibo-Latn" ] }, { "accuracy": 0.699429, "f1": 0.702038, "f1_weighted": 0.717446, "scores_per_experiment": [ { "accuracy": 0.668571, "f1": 0.699671, "f1_weighted": 0.709471 }, { "accuracy": 0.742857, "f1": 0.736268, "f1_weighted": 0.756027 }, { "accuracy": 0.691429, "f1": 0.698186, "f1_weighted": 0.721067 }, { "accuracy": 0.702857, "f1": 0.674758, "f1_weighted": 0.708045 }, { "accuracy": 0.68, "f1": 0.653556, "f1_weighted": 0.695682 }, { "accuracy": 0.685714, "f1": 0.693387, "f1_weighted": 0.700181 }, { "accuracy": 0.754286, "f1": 0.750904, "f1_weighted": 0.760954 }, { "accuracy": 0.605714, "f1": 0.634155, "f1_weighted": 0.628189 }, { "accuracy": 0.668571, "f1": 0.70063, "f1_weighted": 0.697832 }, { "accuracy": 0.794286, "f1": 0.778868, "f1_weighted": 0.797008 } ], "main_score": 0.699429, "hf_subset": "lin", "languages": [ "lin-Latn" ] }, { "accuracy": 0.589686, "f1": 0.55302, "f1_weighted": 0.605883, "scores_per_experiment": [ { "accuracy": 0.663677, "f1": 0.628942, "f1_weighted": 0.674076 }, { "accuracy": 0.565022, "f1": 0.520669, "f1_weighted": 0.584566 }, { "accuracy": 0.64574, "f1": 0.60671, "f1_weighted": 0.656899 }, { "accuracy": 0.618834, "f1": 0.584839, "f1_weighted": 0.639404 }, { "accuracy": 0.600897, "f1": 0.576719, "f1_weighted": 0.619767 }, { "accuracy": 0.488789, "f1": 0.460703, "f1_weighted": 0.505934 }, { "accuracy": 0.533632, "f1": 0.507691, "f1_weighted": 0.557781 }, { "accuracy": 0.565022, "f1": 0.514901, "f1_weighted": 0.582095 }, { "accuracy": 0.632287, "f1": 0.587945, "f1_weighted": 0.635887 }, { "accuracy": 0.58296, "f1": 0.541084, "f1_weighted": 0.602422 } ], "main_score": 0.589686, "hf_subset": "lug", "languages": [ "lug-Latn" ] }, { "accuracy": 0.609538, "f1": 0.543269, "f1_weighted": 0.622651, "scores_per_experiment": [ { "accuracy": 0.624615, "f1": 0.555896, "f1_weighted": 0.633994 }, { "accuracy": 0.630769, "f1": 0.553099, "f1_weighted": 0.637131 }, { "accuracy": 0.627692, "f1": 0.558468, "f1_weighted": 0.65331 }, { "accuracy": 0.64, "f1": 0.578374, "f1_weighted": 0.65857 }, { "accuracy": 0.556923, "f1": 0.488951, "f1_weighted": 0.576026 }, { "accuracy": 0.596923, "f1": 0.529158, "f1_weighted": 0.61899 }, { "accuracy": 0.56, "f1": 0.505754, "f1_weighted": 0.552949 }, { "accuracy": 0.64, "f1": 0.573883, "f1_weighted": 0.654136 }, { "accuracy": 0.618462, "f1": 0.548634, "f1_weighted": 0.624659 }, { "accuracy": 0.6, "f1": 0.540476, "f1_weighted": 0.616748 } ], "main_score": 0.609538, "hf_subset": "orm", "languages": [ "orm-Ethi" ] }, { "accuracy": 0.936066, "f1": 0.91112, "f1_weighted": 0.936554, "scores_per_experiment": [ { "accuracy": 0.937705, "f1": 0.904968, "f1_weighted": 0.93781 }, { "accuracy": 0.914754, "f1": 0.882793, "f1_weighted": 0.914906 }, { "accuracy": 0.95082, "f1": 0.928771, "f1_weighted": 0.951165 }, { "accuracy": 0.957377, "f1": 0.939355, "f1_weighted": 0.957452 }, { "accuracy": 0.944262, "f1": 0.937293, "f1_weighted": 0.944867 }, { "accuracy": 0.934426, "f1": 0.894811, "f1_weighted": 0.934217 }, { "accuracy": 0.940984, "f1": 0.913224, "f1_weighted": 0.942561 }, { "accuracy": 0.92459, "f1": 0.904061, "f1_weighted": 0.925699 }, { "accuracy": 0.931148, "f1": 0.912014, "f1_weighted": 0.931725 }, { "accuracy": 0.92459, "f1": 0.893911, "f1_weighted": 0.925142 } ], "main_score": 0.936066, "hf_subset": "pcm", "languages": [ "pcm-Latn" ] }, { "accuracy": 0.653106, "f1": 0.556462, "f1_weighted": 0.672741, "scores_per_experiment": [ { "accuracy": 0.586957, "f1": 0.515727, "f1_weighted": 0.590468 }, { "accuracy": 0.667702, "f1": 0.5739, "f1_weighted": 0.702783 }, { "accuracy": 0.68323, "f1": 0.526342, "f1_weighted": 0.68735 }, { "accuracy": 0.645963, "f1": 0.565902, "f1_weighted": 0.678271 }, { "accuracy": 0.642857, "f1": 0.525263, "f1_weighted": 0.654999 }, { "accuracy": 0.661491, "f1": 0.54779, "f1_weighted": 0.694095 }, { "accuracy": 0.649068, "f1": 0.577164, "f1_weighted": 0.663746 }, { "accuracy": 0.639752, "f1": 0.562085, "f1_weighted": 0.675202 }, { "accuracy": 0.627329, "f1": 0.552237, "f1_weighted": 0.645345 }, { "accuracy": 0.726708, "f1": 0.618211, "f1_weighted": 0.735147 } ], "main_score": 0.653106, "hf_subset": "run", "languages": [ "run-Latn" ] }, { "accuracy": 0.748509, "f1": 0.752408, "f1_weighted": 0.745002, "scores_per_experiment": [ { "accuracy": 0.772358, "f1": 0.777647, "f1_weighted": 0.770694 }, { "accuracy": 0.761518, "f1": 0.766039, "f1_weighted": 0.760251 }, { "accuracy": 0.726287, "f1": 0.732284, "f1_weighted": 0.721445 }, { "accuracy": 0.739837, "f1": 0.743189, "f1_weighted": 0.737103 }, { "accuracy": 0.794038, "f1": 0.796493, "f1_weighted": 0.79137 }, { "accuracy": 0.753388, "f1": 0.755767, "f1_weighted": 0.748856 }, { "accuracy": 0.764228, "f1": 0.770741, "f1_weighted": 0.764865 }, { "accuracy": 0.758808, "f1": 0.763854, "f1_weighted": 0.757962 }, { "accuracy": 0.723577, "f1": 0.722945, "f1_weighted": 0.713797 }, { "accuracy": 0.691057, "f1": 0.695127, "f1_weighted": 0.68368 } ], "main_score": 0.748509, "hf_subset": "sna", "languages": [ "sna-Latn" ] }, { "accuracy": 0.541837, "f1": 0.501604, "f1_weighted": 0.558454, "scores_per_experiment": [ { "accuracy": 0.52381, "f1": 0.504893, "f1_weighted": 0.536417 }, { "accuracy": 0.537415, "f1": 0.482128, "f1_weighted": 0.558096 }, { "accuracy": 0.585034, "f1": 0.554245, "f1_weighted": 0.598532 }, { "accuracy": 0.554422, "f1": 0.516969, "f1_weighted": 0.589406 }, { "accuracy": 0.52381, "f1": 0.477861, "f1_weighted": 0.54104 }, { "accuracy": 0.493197, "f1": 0.459654, "f1_weighted": 0.514123 }, { "accuracy": 0.591837, "f1": 0.523459, "f1_weighted": 0.611214 }, { "accuracy": 0.540816, "f1": 0.49489, "f1_weighted": 0.552421 }, { "accuracy": 0.486395, "f1": 0.453492, "f1_weighted": 0.494498 }, { "accuracy": 0.581633, "f1": 0.548453, "f1_weighted": 0.588794 } ], "main_score": 0.541837, "hf_subset": "som", "languages": [ "som-Latn" ] }, { "accuracy": 0.577941, "f1": 0.524049, "f1_weighted": 0.58375, "scores_per_experiment": [ { "accuracy": 0.55042, "f1": 0.479117, "f1_weighted": 0.554992 }, { "accuracy": 0.533613, "f1": 0.494033, "f1_weighted": 0.534289 }, { "accuracy": 0.592437, "f1": 0.560726, "f1_weighted": 0.600133 }, { "accuracy": 0.592437, "f1": 0.518362, "f1_weighted": 0.59888 }, { "accuracy": 0.602941, "f1": 0.545477, "f1_weighted": 0.609012 }, { "accuracy": 0.546218, "f1": 0.484309, "f1_weighted": 0.549016 }, { "accuracy": 0.605042, "f1": 0.54483, "f1_weighted": 0.615924 }, { "accuracy": 0.567227, "f1": 0.524616, "f1_weighted": 0.572499 }, { "accuracy": 0.615546, "f1": 0.566128, "f1_weighted": 0.626104 }, { "accuracy": 0.573529, "f1": 0.522895, "f1_weighted": 0.576654 } ], "main_score": 0.577941, "hf_subset": "swa", "languages": [ "swa-Latn" ] }, { "accuracy": 0.25625, "f1": 0.238002, "f1_weighted": 0.259118, "scores_per_experiment": [ { "accuracy": 0.183824, "f1": 0.198804, "f1_weighted": 0.171664 }, { "accuracy": 0.3125, "f1": 0.311395, "f1_weighted": 0.315276 }, { "accuracy": 0.25, "f1": 0.241998, "f1_weighted": 0.257042 }, { "accuracy": 0.220588, "f1": 0.210274, "f1_weighted": 0.224845 }, { "accuracy": 0.257353, "f1": 0.22269, "f1_weighted": 0.243125 }, { "accuracy": 0.238971, "f1": 0.223563, "f1_weighted": 0.256209 }, { "accuracy": 0.286765, "f1": 0.233519, "f1_weighted": 0.297292 }, { "accuracy": 0.290441, "f1": 0.253237, "f1_weighted": 0.299108 }, { "accuracy": 0.25, "f1": 0.233395, "f1_weighted": 0.255075 }, { "accuracy": 0.272059, "f1": 0.251141, "f1_weighted": 0.271547 } ], "main_score": 0.25625, "hf_subset": "tir", "languages": [ "tir-Ethi" ] }, { "accuracy": 0.677441, "f1": 0.562882, "f1_weighted": 0.693934, "scores_per_experiment": [ { "accuracy": 0.670034, "f1": 0.527529, "f1_weighted": 0.684521 }, { "accuracy": 0.639731, "f1": 0.500652, "f1_weighted": 0.65597 }, { "accuracy": 0.649832, "f1": 0.546153, "f1_weighted": 0.674104 }, { "accuracy": 0.686869, "f1": 0.574882, "f1_weighted": 0.694556 }, { "accuracy": 0.713805, "f1": 0.607638, "f1_weighted": 0.72514 }, { "accuracy": 0.693603, "f1": 0.579945, "f1_weighted": 0.701382 }, { "accuracy": 0.676768, "f1": 0.562496, "f1_weighted": 0.70686 }, { "accuracy": 0.653199, "f1": 0.521085, "f1_weighted": 0.677289 }, { "accuracy": 0.723906, "f1": 0.618049, "f1_weighted": 0.743918 }, { "accuracy": 0.666667, "f1": 0.59039, "f1_weighted": 0.675604 } ], "main_score": 0.677441, "hf_subset": "xho", "languages": [ "xho-Latn" ] }, { "accuracy": 0.668127, "f1": 0.666904, "f1_weighted": 0.671503, "scores_per_experiment": [ { "accuracy": 0.647202, "f1": 0.64668, "f1_weighted": 0.651938 }, { "accuracy": 0.673966, "f1": 0.676156, "f1_weighted": 0.678082 }, { "accuracy": 0.673966, "f1": 0.669388, "f1_weighted": 0.66992 }, { "accuracy": 0.686131, "f1": 0.691003, "f1_weighted": 0.696246 }, { "accuracy": 0.644769, "f1": 0.648279, "f1_weighted": 0.651738 }, { "accuracy": 0.625304, "f1": 0.623765, "f1_weighted": 0.632628 }, { "accuracy": 0.695864, "f1": 0.684743, "f1_weighted": 0.691633 }, { "accuracy": 0.703163, "f1": 0.69643, "f1_weighted": 0.703428 }, { "accuracy": 0.6618, "f1": 0.66246, "f1_weighted": 0.668136 }, { "accuracy": 0.6691, "f1": 0.670135, "f1_weighted": 0.671285 } ], "main_score": 0.668127, "hf_subset": "yor", "languages": [ "yor-Latn" ] } ] }, "evaluation_time": 37.32248044013977, "kg_co2_emissions": 0.001737962436827039 }