{ "dataset_revision": "18193f187b92da67168c655c9973a165ed9593dd", "task_name": "MasakhaNEWSClassification", "mteb_version": "1.18.0", "scores": { "test": [ { "accuracy": 0.337234, "f1": 0.322031, "f1_weighted": 0.326616, "scores_per_experiment": [ { "accuracy": 0.308511, "f1": 0.284763, "f1_weighted": 0.292872 }, { "accuracy": 0.31117, "f1": 0.293265, "f1_weighted": 0.302405 }, { "accuracy": 0.369681, "f1": 0.350483, "f1_weighted": 0.357052 }, { "accuracy": 0.284574, "f1": 0.27193, "f1_weighted": 0.277145 }, { "accuracy": 0.367021, "f1": 0.36288, "f1_weighted": 0.367269 }, { "accuracy": 0.388298, "f1": 0.375587, "f1_weighted": 0.376173 }, { "accuracy": 0.332447, "f1": 0.327384, "f1_weighted": 0.325189 }, { "accuracy": 0.321809, "f1": 0.320476, "f1_weighted": 0.319014 }, { "accuracy": 0.351064, "f1": 0.308543, "f1_weighted": 0.318851 }, { "accuracy": 0.337766, "f1": 0.325003, "f1_weighted": 0.330188 } ], "main_score": 0.337234, "hf_subset": "amh", "languages": [ "amh-Ethi" ] }, { "accuracy": 0.79557, "f1": 0.786745, "f1_weighted": 0.794241, "scores_per_experiment": [ { "accuracy": 0.798523, "f1": 0.790704, "f1_weighted": 0.797761 }, { "accuracy": 0.803797, "f1": 0.794441, "f1_weighted": 0.801376 }, { "accuracy": 0.792194, "f1": 0.785251, "f1_weighted": 0.792143 }, { "accuracy": 0.787975, "f1": 0.778064, "f1_weighted": 0.786648 }, { "accuracy": 0.814346, "f1": 0.806558, "f1_weighted": 0.812619 }, { "accuracy": 0.790084, "f1": 0.781561, "f1_weighted": 0.787828 }, { "accuracy": 0.775316, "f1": 0.765995, "f1_weighted": 0.774461 }, { "accuracy": 0.773207, "f1": 0.763796, "f1_weighted": 0.772324 }, { "accuracy": 0.785865, "f1": 0.7739, "f1_weighted": 0.783843 }, { "accuracy": 0.834388, "f1": 0.827178, "f1_weighted": 0.833408 } ], "main_score": 0.79557, "hf_subset": "eng", "languages": [ "eng-Latn" ] }, { "accuracy": 0.759953, "f1": 0.717702, "f1_weighted": 0.762786, "scores_per_experiment": [ { "accuracy": 0.78673, "f1": 0.736294, "f1_weighted": 0.783988 }, { "accuracy": 0.774882, "f1": 0.72922, "f1_weighted": 0.777748 }, { "accuracy": 0.800948, "f1": 0.762855, "f1_weighted": 0.801172 }, { "accuracy": 0.71564, "f1": 0.6856, "f1_weighted": 0.719905 }, { "accuracy": 0.765403, "f1": 0.719718, "f1_weighted": 0.770598 }, { "accuracy": 0.758294, "f1": 0.710794, "f1_weighted": 0.760024 }, { "accuracy": 0.760664, "f1": 0.714717, "f1_weighted": 0.76804 }, { "accuracy": 0.760664, "f1": 0.729039, "f1_weighted": 0.759167 }, { "accuracy": 0.765403, "f1": 0.711251, "f1_weighted": 0.766606 }, { "accuracy": 0.7109, "f1": 0.677531, "f1_weighted": 0.720613 } ], "main_score": 0.759953, "hf_subset": "fra", "languages": [ "fra-Latn" ] }, { "accuracy": 0.593407, "f1": 0.577673, "f1_weighted": 0.592688, "scores_per_experiment": [ { "accuracy": 0.579278, "f1": 0.569852, "f1_weighted": 0.583431 }, { "accuracy": 0.604396, "f1": 0.585155, "f1_weighted": 0.607778 }, { "accuracy": 0.626374, "f1": 0.60684, "f1_weighted": 0.6199 }, { "accuracy": 0.563579, "f1": 0.554047, "f1_weighted": 0.566158 }, { "accuracy": 0.572998, "f1": 0.546783, "f1_weighted": 0.567779 }, { "accuracy": 0.618524, "f1": 0.597831, "f1_weighted": 0.613854 }, { "accuracy": 0.549451, "f1": 0.54041, "f1_weighted": 0.552274 }, { "accuracy": 0.613815, "f1": 0.610224, "f1_weighted": 0.621392 }, { "accuracy": 0.583987, "f1": 0.558793, "f1_weighted": 0.574815 }, { "accuracy": 0.621664, "f1": 0.606795, "f1_weighted": 0.619499 } ], "main_score": 0.593407, "hf_subset": "hau", "languages": [ "hau-Latn" ] }, { "accuracy": 0.569231, "f1": 0.543173, "f1_weighted": 0.560041, "scores_per_experiment": [ { "accuracy": 0.551282, "f1": 0.523295, "f1_weighted": 0.554514 }, { "accuracy": 0.620513, "f1": 0.588125, "f1_weighted": 0.619356 }, { "accuracy": 0.569231, "f1": 0.543707, "f1_weighted": 0.556006 }, { "accuracy": 0.584615, "f1": 0.548212, "f1_weighted": 0.562458 }, { "accuracy": 0.548718, "f1": 0.538227, "f1_weighted": 0.537086 }, { "accuracy": 0.574359, "f1": 0.555932, "f1_weighted": 0.566297 }, { "accuracy": 0.569231, "f1": 0.53981, "f1_weighted": 0.557958 }, { "accuracy": 0.561538, "f1": 0.528873, "f1_weighted": 0.539511 }, { "accuracy": 0.541026, "f1": 0.522217, "f1_weighted": 0.54306 }, { "accuracy": 0.571795, "f1": 0.54333, "f1_weighted": 0.564168 } ], "main_score": 0.569231, "hf_subset": "ibo", "languages": [ "ibo-Latn" ] }, { "accuracy": 0.719429, "f1": 0.707166, "f1_weighted": 0.736257, "scores_per_experiment": [ { "accuracy": 0.645714, "f1": 0.677102, "f1_weighted": 0.687625 }, { "accuracy": 0.765714, "f1": 0.742817, "f1_weighted": 0.777671 }, { "accuracy": 0.674286, "f1": 0.684852, "f1_weighted": 0.702741 }, { "accuracy": 0.754286, "f1": 0.682757, "f1_weighted": 0.752288 }, { "accuracy": 0.754286, "f1": 0.699615, "f1_weighted": 0.762472 }, { "accuracy": 0.651429, "f1": 0.661607, "f1_weighted": 0.669334 }, { "accuracy": 0.76, "f1": 0.742348, "f1_weighted": 0.771045 }, { "accuracy": 0.708571, "f1": 0.704215, "f1_weighted": 0.72592 }, { "accuracy": 0.674286, "f1": 0.697401, "f1_weighted": 0.705291 }, { "accuracy": 0.805714, "f1": 0.778949, "f1_weighted": 0.808178 } ], "main_score": 0.719429, "hf_subset": "lin", "languages": [ "lin-Latn" ] }, { "accuracy": 0.585202, "f1": 0.546483, "f1_weighted": 0.601202, "scores_per_experiment": [ { "accuracy": 0.623318, "f1": 0.586367, "f1_weighted": 0.633347 }, { "accuracy": 0.538117, "f1": 0.46938, "f1_weighted": 0.554121 }, { "accuracy": 0.596413, "f1": 0.555043, "f1_weighted": 0.617878 }, { "accuracy": 0.627803, "f1": 0.596121, "f1_weighted": 0.645432 }, { "accuracy": 0.609865, "f1": 0.576233, "f1_weighted": 0.623728 }, { "accuracy": 0.520179, "f1": 0.500132, "f1_weighted": 0.536677 }, { "accuracy": 0.538117, "f1": 0.498624, "f1_weighted": 0.558997 }, { "accuracy": 0.569507, "f1": 0.525133, "f1_weighted": 0.578367 }, { "accuracy": 0.641256, "f1": 0.598568, "f1_weighted": 0.649943 }, { "accuracy": 0.587444, "f1": 0.559231, "f1_weighted": 0.613533 } ], "main_score": 0.585202, "hf_subset": "lug", "languages": [ "lug-Latn" ] }, { "accuracy": 0.594154, "f1": 0.525592, "f1_weighted": 0.610905, "scores_per_experiment": [ { "accuracy": 0.609231, "f1": 0.542634, "f1_weighted": 0.618382 }, { "accuracy": 0.612308, "f1": 0.540068, "f1_weighted": 0.624579 }, { "accuracy": 0.584615, "f1": 0.527811, "f1_weighted": 0.616787 }, { "accuracy": 0.624615, "f1": 0.563033, "f1_weighted": 0.6442 }, { "accuracy": 0.590769, "f1": 0.504072, "f1_weighted": 0.600395 }, { "accuracy": 0.578462, "f1": 0.524358, "f1_weighted": 0.602824 }, { "accuracy": 0.553846, "f1": 0.475496, "f1_weighted": 0.554292 }, { "accuracy": 0.606154, "f1": 0.53858, "f1_weighted": 0.624254 }, { "accuracy": 0.587692, "f1": 0.512499, "f1_weighted": 0.613378 }, { "accuracy": 0.593846, "f1": 0.527371, "f1_weighted": 0.609955 } ], "main_score": 0.594154, "hf_subset": "orm", "languages": [ "orm-Ethi" ] }, { "accuracy": 0.941639, "f1": 0.921995, "f1_weighted": 0.942489, "scores_per_experiment": [ { "accuracy": 0.95082, "f1": 0.941394, "f1_weighted": 0.950991 }, { "accuracy": 0.937705, "f1": 0.922432, "f1_weighted": 0.938419 }, { "accuracy": 0.957377, "f1": 0.933786, "f1_weighted": 0.957961 }, { "accuracy": 0.947541, "f1": 0.93247, "f1_weighted": 0.947723 }, { "accuracy": 0.944262, "f1": 0.933154, "f1_weighted": 0.944784 }, { "accuracy": 0.937705, "f1": 0.910894, "f1_weighted": 0.938286 }, { "accuracy": 0.940984, "f1": 0.912716, "f1_weighted": 0.942536 }, { "accuracy": 0.934426, "f1": 0.917652, "f1_weighted": 0.935464 }, { "accuracy": 0.934426, "f1": 0.913351, "f1_weighted": 0.935669 }, { "accuracy": 0.931148, "f1": 0.902102, "f1_weighted": 0.933053 } ], "main_score": 0.941639, "hf_subset": "pcm", "languages": [ "pcm-Latn" ] }, { "accuracy": 0.622671, "f1": 0.522395, "f1_weighted": 0.639933, "scores_per_experiment": [ { "accuracy": 0.565217, "f1": 0.475694, "f1_weighted": 0.570719 }, { "accuracy": 0.636646, "f1": 0.54673, "f1_weighted": 0.661432 }, { "accuracy": 0.639752, "f1": 0.508521, "f1_weighted": 0.648984 }, { "accuracy": 0.630435, "f1": 0.553206, "f1_weighted": 0.663335 }, { "accuracy": 0.645963, "f1": 0.51954, "f1_weighted": 0.649407 }, { "accuracy": 0.639752, "f1": 0.532354, "f1_weighted": 0.666219 }, { "accuracy": 0.60559, "f1": 0.517363, "f1_weighted": 0.618955 }, { "accuracy": 0.621118, "f1": 0.543481, "f1_weighted": 0.652901 }, { "accuracy": 0.60559, "f1": 0.499747, "f1_weighted": 0.611781 }, { "accuracy": 0.636646, "f1": 0.527314, "f1_weighted": 0.655597 } ], "main_score": 0.622671, "hf_subset": "run", "languages": [ "run-Latn" ] }, { "accuracy": 0.742276, "f1": 0.746025, "f1_weighted": 0.73806, "scores_per_experiment": [ { "accuracy": 0.745257, "f1": 0.750405, "f1_weighted": 0.742687 }, { "accuracy": 0.742547, "f1": 0.74478, "f1_weighted": 0.736865 }, { "accuracy": 0.731707, "f1": 0.739207, "f1_weighted": 0.728134 }, { "accuracy": 0.758808, "f1": 0.762035, "f1_weighted": 0.755955 }, { "accuracy": 0.775068, "f1": 0.779607, "f1_weighted": 0.773883 }, { "accuracy": 0.766938, "f1": 0.770942, "f1_weighted": 0.763142 }, { "accuracy": 0.742547, "f1": 0.748108, "f1_weighted": 0.741619 }, { "accuracy": 0.737127, "f1": 0.741851, "f1_weighted": 0.73564 }, { "accuracy": 0.742547, "f1": 0.740613, "f1_weighted": 0.731703 }, { "accuracy": 0.680217, "f1": 0.682707, "f1_weighted": 0.670974 } ], "main_score": 0.742276, "hf_subset": "sna", "languages": [ "sna-Latn" ] }, { "accuracy": 0.529592, "f1": 0.485928, "f1_weighted": 0.546305, "scores_per_experiment": [ { "accuracy": 0.5, "f1": 0.479643, "f1_weighted": 0.515069 }, { "accuracy": 0.55102, "f1": 0.501553, "f1_weighted": 0.580766 }, { "accuracy": 0.585034, "f1": 0.542716, "f1_weighted": 0.594006 }, { "accuracy": 0.554422, "f1": 0.505341, "f1_weighted": 0.578422 }, { "accuracy": 0.554422, "f1": 0.518378, "f1_weighted": 0.57974 }, { "accuracy": 0.452381, "f1": 0.407509, "f1_weighted": 0.477776 }, { "accuracy": 0.571429, "f1": 0.508555, "f1_weighted": 0.595907 }, { "accuracy": 0.5, "f1": 0.452843, "f1_weighted": 0.503181 }, { "accuracy": 0.5, "f1": 0.44002, "f1_weighted": 0.499538 }, { "accuracy": 0.527211, "f1": 0.502718, "f1_weighted": 0.538642 } ], "main_score": 0.529592, "hf_subset": "som", "languages": [ "som-Latn" ] }, { "accuracy": 0.547059, "f1": 0.490939, "f1_weighted": 0.552905, "scores_per_experiment": [ { "accuracy": 0.533613, "f1": 0.445033, "f1_weighted": 0.532363 }, { "accuracy": 0.512605, "f1": 0.463634, "f1_weighted": 0.516833 }, { "accuracy": 0.560924, "f1": 0.516761, "f1_weighted": 0.560778 }, { "accuracy": 0.544118, "f1": 0.483046, "f1_weighted": 0.558968 }, { "accuracy": 0.607143, "f1": 0.542998, "f1_weighted": 0.612619 }, { "accuracy": 0.5, "f1": 0.44435, "f1_weighted": 0.505165 }, { "accuracy": 0.554622, "f1": 0.512086, "f1_weighted": 0.564997 }, { "accuracy": 0.52521, "f1": 0.472159, "f1_weighted": 0.531307 }, { "accuracy": 0.598739, "f1": 0.551074, "f1_weighted": 0.610122 }, { "accuracy": 0.533613, "f1": 0.478254, "f1_weighted": 0.5359 } ], "main_score": 0.547059, "hf_subset": "swa", "languages": [ "swa-Latn" ] }, { "accuracy": 0.268382, "f1": 0.243193, "f1_weighted": 0.272061, "scores_per_experiment": [ { "accuracy": 0.235294, "f1": 0.233814, "f1_weighted": 0.234232 }, { "accuracy": 0.264706, "f1": 0.249383, "f1_weighted": 0.276391 }, { "accuracy": 0.194853, "f1": 0.199216, "f1_weighted": 0.181432 }, { "accuracy": 0.268382, "f1": 0.252983, "f1_weighted": 0.280396 }, { "accuracy": 0.341912, "f1": 0.295928, "f1_weighted": 0.350495 }, { "accuracy": 0.297794, "f1": 0.264856, "f1_weighted": 0.313933 }, { "accuracy": 0.297794, "f1": 0.246213, "f1_weighted": 0.293955 }, { "accuracy": 0.268382, "f1": 0.2345, "f1_weighted": 0.281343 }, { "accuracy": 0.238971, "f1": 0.213701, "f1_weighted": 0.232785 }, { "accuracy": 0.275735, "f1": 0.241332, "f1_weighted": 0.275646 } ], "main_score": 0.268382, "hf_subset": "tir", "languages": [ "tir-Ethi" ] }, { "accuracy": 0.654209, "f1": 0.544267, "f1_weighted": 0.674591, "scores_per_experiment": [ { "accuracy": 0.649832, "f1": 0.530145, "f1_weighted": 0.670108 }, { "accuracy": 0.606061, "f1": 0.49158, "f1_weighted": 0.629548 }, { "accuracy": 0.646465, "f1": 0.55012, "f1_weighted": 0.675798 }, { "accuracy": 0.619529, "f1": 0.478673, "f1_weighted": 0.634646 }, { "accuracy": 0.670034, "f1": 0.558126, "f1_weighted": 0.677227 }, { "accuracy": 0.670034, "f1": 0.568762, "f1_weighted": 0.681823 }, { "accuracy": 0.676768, "f1": 0.587805, "f1_weighted": 0.702995 }, { "accuracy": 0.646465, "f1": 0.529246, "f1_weighted": 0.67616 }, { "accuracy": 0.673401, "f1": 0.539347, "f1_weighted": 0.700685 }, { "accuracy": 0.683502, "f1": 0.608866, "f1_weighted": 0.696925 } ], "main_score": 0.654209, "hf_subset": "xho", "languages": [ "xho-Latn" ] }, { "accuracy": 0.633577, "f1": 0.630899, "f1_weighted": 0.635201, "scores_per_experiment": [ { "accuracy": 0.647202, "f1": 0.639716, "f1_weighted": 0.646172 }, { "accuracy": 0.652068, "f1": 0.645362, "f1_weighted": 0.655635 }, { "accuracy": 0.63747, "f1": 0.634532, "f1_weighted": 0.638248 }, { "accuracy": 0.664234, "f1": 0.671118, "f1_weighted": 0.676948 }, { "accuracy": 0.63017, "f1": 0.631098, "f1_weighted": 0.632289 }, { "accuracy": 0.632603, "f1": 0.624736, "f1_weighted": 0.633909 }, { "accuracy": 0.625304, "f1": 0.614996, "f1_weighted": 0.617702 }, { "accuracy": 0.625304, "f1": 0.62456, "f1_weighted": 0.622337 }, { "accuracy": 0.576642, "f1": 0.582574, "f1_weighted": 0.584735 }, { "accuracy": 0.644769, "f1": 0.640298, "f1_weighted": 0.644038 } ], "main_score": 0.633577, "hf_subset": "yor", "languages": [ "yor-Latn" ] } ] }, "evaluation_time": 34.18889307975769, "kg_co2_emissions": 0.001651823741352425 }