{ "dataset_revision": "18193f187b92da67168c655c9973a165ed9593dd", "task_name": "MasakhaNEWSClassification", "mteb_version": "1.18.0", "scores": { "test": [ { "accuracy": 0.330053, "f1": 0.31754, "f1_weighted": 0.322782, "scores_per_experiment": [ { "accuracy": 0.319149, "f1": 0.283499, "f1_weighted": 0.293862 }, { "accuracy": 0.319149, "f1": 0.303168, "f1_weighted": 0.311681 }, { "accuracy": 0.321809, "f1": 0.313904, "f1_weighted": 0.320258 }, { "accuracy": 0.276596, "f1": 0.262288, "f1_weighted": 0.261798 }, { "accuracy": 0.345745, "f1": 0.342114, "f1_weighted": 0.346767 }, { "accuracy": 0.345745, "f1": 0.342269, "f1_weighted": 0.34505 }, { "accuracy": 0.351064, "f1": 0.343343, "f1_weighted": 0.345914 }, { "accuracy": 0.359043, "f1": 0.350631, "f1_weighted": 0.355144 }, { "accuracy": 0.351064, "f1": 0.327739, "f1_weighted": 0.337393 }, { "accuracy": 0.31117, "f1": 0.306444, "f1_weighted": 0.309955 } ], "main_score": 0.330053, "hf_subset": "amh", "languages": [ "amh-Ethi" ] }, { "accuracy": 0.762553, "f1": 0.752144, "f1_weighted": 0.761015, "scores_per_experiment": [ { "accuracy": 0.775316, "f1": 0.767232, "f1_weighted": 0.775309 }, { "accuracy": 0.772152, "f1": 0.762605, "f1_weighted": 0.76891 }, { "accuracy": 0.766878, "f1": 0.757781, "f1_weighted": 0.766162 }, { "accuracy": 0.75211, "f1": 0.735788, "f1_weighted": 0.74817 }, { "accuracy": 0.7827, "f1": 0.775068, "f1_weighted": 0.780908 }, { "accuracy": 0.755274, "f1": 0.744213, "f1_weighted": 0.754143 }, { "accuracy": 0.745781, "f1": 0.734775, "f1_weighted": 0.74341 }, { "accuracy": 0.764768, "f1": 0.755812, "f1_weighted": 0.763777 }, { "accuracy": 0.707806, "f1": 0.694747, "f1_weighted": 0.708648 }, { "accuracy": 0.802743, "f1": 0.79342, "f1_weighted": 0.800712 } ], "main_score": 0.762553, "hf_subset": "eng", "languages": [ "eng-Latn" ] }, { "accuracy": 0.761611, "f1": 0.716318, "f1_weighted": 0.763251, "scores_per_experiment": [ { "accuracy": 0.829384, "f1": 0.77623, "f1_weighted": 0.827287 }, { "accuracy": 0.772512, "f1": 0.724362, "f1_weighted": 0.776101 }, { "accuracy": 0.796209, "f1": 0.756367, "f1_weighted": 0.80033 }, { "accuracy": 0.699052, "f1": 0.666452, "f1_weighted": 0.701619 }, { "accuracy": 0.772512, "f1": 0.721984, "f1_weighted": 0.775069 }, { "accuracy": 0.748815, "f1": 0.695045, "f1_weighted": 0.747405 }, { "accuracy": 0.763033, "f1": 0.713818, "f1_weighted": 0.774029 }, { "accuracy": 0.741706, "f1": 0.703396, "f1_weighted": 0.74099 }, { "accuracy": 0.777251, "f1": 0.733477, "f1_weighted": 0.769523 }, { "accuracy": 0.71564, "f1": 0.672051, "f1_weighted": 0.72016 } ], "main_score": 0.761611, "hf_subset": "fra", "languages": [ "fra-Latn" ] }, { "accuracy": 0.600785, "f1": 0.584528, "f1_weighted": 0.599924, "scores_per_experiment": [ { "accuracy": 0.55887, "f1": 0.551503, "f1_weighted": 0.559522 }, { "accuracy": 0.609105, "f1": 0.590063, "f1_weighted": 0.607217 }, { "accuracy": 0.591837, "f1": 0.564723, "f1_weighted": 0.580027 }, { "accuracy": 0.626374, "f1": 0.604688, "f1_weighted": 0.622253 }, { "accuracy": 0.580848, "f1": 0.566171, "f1_weighted": 0.581706 }, { "accuracy": 0.616954, "f1": 0.605345, "f1_weighted": 0.621641 }, { "accuracy": 0.604396, "f1": 0.595474, "f1_weighted": 0.614152 }, { "accuracy": 0.632653, "f1": 0.616434, "f1_weighted": 0.628752 }, { "accuracy": 0.577708, "f1": 0.574073, "f1_weighted": 0.587351 }, { "accuracy": 0.609105, "f1": 0.576803, "f1_weighted": 0.596623 } ], "main_score": 0.600785, "hf_subset": "hau", "languages": [ "hau-Latn" ] }, { "accuracy": 0.577436, "f1": 0.570158, "f1_weighted": 0.572558, "scores_per_experiment": [ { "accuracy": 0.582051, "f1": 0.559421, "f1_weighted": 0.581514 }, { "accuracy": 0.55641, "f1": 0.547687, "f1_weighted": 0.558379 }, { "accuracy": 0.579487, "f1": 0.586447, "f1_weighted": 0.573368 }, { "accuracy": 0.55641, "f1": 0.552814, "f1_weighted": 0.553349 }, { "accuracy": 0.546154, "f1": 0.553105, "f1_weighted": 0.531181 }, { "accuracy": 0.605128, "f1": 0.592173, "f1_weighted": 0.593662 }, { "accuracy": 0.612821, "f1": 0.591812, "f1_weighted": 0.605263 }, { "accuracy": 0.594872, "f1": 0.576381, "f1_weighted": 0.58246 }, { "accuracy": 0.548718, "f1": 0.564989, "f1_weighted": 0.549399 }, { "accuracy": 0.592308, "f1": 0.576753, "f1_weighted": 0.597005 } ], "main_score": 0.577436, "hf_subset": "ibo", "languages": [ "ibo-Latn" ] }, { "accuracy": 0.721714, "f1": 0.718837, "f1_weighted": 0.738484, "scores_per_experiment": [ { "accuracy": 0.702857, "f1": 0.714185, "f1_weighted": 0.738247 }, { "accuracy": 0.657143, "f1": 0.663154, "f1_weighted": 0.674764 }, { "accuracy": 0.697143, "f1": 0.721773, "f1_weighted": 0.724656 }, { "accuracy": 0.8, "f1": 0.78139, "f1_weighted": 0.80517 }, { "accuracy": 0.668571, "f1": 0.674938, "f1_weighted": 0.684944 }, { "accuracy": 0.697143, "f1": 0.668016, "f1_weighted": 0.713371 }, { "accuracy": 0.782857, "f1": 0.764913, "f1_weighted": 0.786603 }, { "accuracy": 0.731429, "f1": 0.726065, "f1_weighted": 0.750878 }, { "accuracy": 0.702857, "f1": 0.726644, "f1_weighted": 0.728044 }, { "accuracy": 0.777143, "f1": 0.74729, "f1_weighted": 0.778163 } ], "main_score": 0.721714, "hf_subset": "lin", "languages": [ "lin-Latn" ] }, { "accuracy": 0.587892, "f1": 0.546335, "f1_weighted": 0.612215, "scores_per_experiment": [ { "accuracy": 0.618834, "f1": 0.55781, "f1_weighted": 0.631276 }, { "accuracy": 0.493274, "f1": 0.44929, "f1_weighted": 0.536829 }, { "accuracy": 0.591928, "f1": 0.5679, "f1_weighted": 0.611436 }, { "accuracy": 0.605381, "f1": 0.558166, "f1_weighted": 0.635523 }, { "accuracy": 0.600897, "f1": 0.549936, "f1_weighted": 0.622518 }, { "accuracy": 0.565022, "f1": 0.519839, "f1_weighted": 0.589985 }, { "accuracy": 0.578475, "f1": 0.545132, "f1_weighted": 0.601925 }, { "accuracy": 0.591928, "f1": 0.53761, "f1_weighted": 0.608458 }, { "accuracy": 0.636771, "f1": 0.591092, "f1_weighted": 0.660875 }, { "accuracy": 0.596413, "f1": 0.586575, "f1_weighted": 0.623323 } ], "main_score": 0.587892, "hf_subset": "lug", "languages": [ "lug-Latn" ] }, { "accuracy": 0.547385, "f1": 0.494672, "f1_weighted": 0.553061, "scores_per_experiment": [ { "accuracy": 0.48, "f1": 0.458502, "f1_weighted": 0.487254 }, { "accuracy": 0.575385, "f1": 0.528849, "f1_weighted": 0.578938 }, { "accuracy": 0.569231, "f1": 0.522848, "f1_weighted": 0.593377 }, { "accuracy": 0.661538, "f1": 0.615282, "f1_weighted": 0.659647 }, { "accuracy": 0.572308, "f1": 0.52945, "f1_weighted": 0.597795 }, { "accuracy": 0.489231, "f1": 0.418749, "f1_weighted": 0.498914 }, { "accuracy": 0.52, "f1": 0.478956, "f1_weighted": 0.533972 }, { "accuracy": 0.649231, "f1": 0.576452, "f1_weighted": 0.65878 }, { "accuracy": 0.526154, "f1": 0.441781, "f1_weighted": 0.501792 }, { "accuracy": 0.430769, "f1": 0.375849, "f1_weighted": 0.420143 } ], "main_score": 0.547385, "hf_subset": "orm", "languages": [ "orm-Ethi" ] }, { "accuracy": 0.902623, "f1": 0.871772, "f1_weighted": 0.90411, "scores_per_experiment": [ { "accuracy": 0.921311, "f1": 0.881489, "f1_weighted": 0.920978 }, { "accuracy": 0.891803, "f1": 0.86357, "f1_weighted": 0.896137 }, { "accuracy": 0.918033, "f1": 0.890785, "f1_weighted": 0.918358 }, { "accuracy": 0.940984, "f1": 0.929732, "f1_weighted": 0.941106 }, { "accuracy": 0.908197, "f1": 0.883809, "f1_weighted": 0.909117 }, { "accuracy": 0.904918, "f1": 0.877741, "f1_weighted": 0.906217 }, { "accuracy": 0.881967, "f1": 0.837323, "f1_weighted": 0.886332 }, { "accuracy": 0.87541, "f1": 0.850092, "f1_weighted": 0.878073 }, { "accuracy": 0.904918, "f1": 0.873137, "f1_weighted": 0.906313 }, { "accuracy": 0.878689, "f1": 0.830046, "f1_weighted": 0.878472 } ], "main_score": 0.902623, "hf_subset": "pcm", "languages": [ "pcm-Latn" ] }, { "accuracy": 0.640683, "f1": 0.531035, "f1_weighted": 0.663379, "scores_per_experiment": [ { "accuracy": 0.555901, "f1": 0.480969, "f1_weighted": 0.588279 }, { "accuracy": 0.652174, "f1": 0.563022, "f1_weighted": 0.686258 }, { "accuracy": 0.645963, "f1": 0.507552, "f1_weighted": 0.651396 }, { "accuracy": 0.645963, "f1": 0.569649, "f1_weighted": 0.685457 }, { "accuracy": 0.65528, "f1": 0.543188, "f1_weighted": 0.681687 }, { "accuracy": 0.624224, "f1": 0.511193, "f1_weighted": 0.654982 }, { "accuracy": 0.677019, "f1": 0.552551, "f1_weighted": 0.698525 }, { "accuracy": 0.611801, "f1": 0.522834, "f1_weighted": 0.642013 }, { "accuracy": 0.670807, "f1": 0.523322, "f1_weighted": 0.676275 }, { "accuracy": 0.667702, "f1": 0.53607, "f1_weighted": 0.668917 } ], "main_score": 0.640683, "hf_subset": "run", "languages": [ "run-Latn" ] }, { "accuracy": 0.75122, "f1": 0.757188, "f1_weighted": 0.749271, "scores_per_experiment": [ { "accuracy": 0.810298, "f1": 0.815012, "f1_weighted": 0.81133 }, { "accuracy": 0.734417, "f1": 0.742186, "f1_weighted": 0.734723 }, { "accuracy": 0.737127, "f1": 0.743967, "f1_weighted": 0.735634 }, { "accuracy": 0.794038, "f1": 0.79816, "f1_weighted": 0.793122 }, { "accuracy": 0.788618, "f1": 0.793413, "f1_weighted": 0.786511 }, { "accuracy": 0.758808, "f1": 0.767981, "f1_weighted": 0.760609 }, { "accuracy": 0.712737, "f1": 0.717773, "f1_weighted": 0.706929 }, { "accuracy": 0.745257, "f1": 0.758031, "f1_weighted": 0.748313 }, { "accuracy": 0.704607, "f1": 0.700637, "f1_weighted": 0.689789 }, { "accuracy": 0.726287, "f1": 0.734723, "f1_weighted": 0.725752 } ], "main_score": 0.75122, "hf_subset": "sna", "languages": [ "sna-Latn" ] }, { "accuracy": 0.481633, "f1": 0.469838, "f1_weighted": 0.506886, "scores_per_experiment": [ { "accuracy": 0.445578, "f1": 0.429363, "f1_weighted": 0.467515 }, { "accuracy": 0.547619, "f1": 0.463607, "f1_weighted": 0.54935 }, { "accuracy": 0.513605, "f1": 0.488657, "f1_weighted": 0.530483 }, { "accuracy": 0.506803, "f1": 0.537586, "f1_weighted": 0.554427 }, { "accuracy": 0.435374, "f1": 0.45643, "f1_weighted": 0.479165 }, { "accuracy": 0.411565, "f1": 0.396386, "f1_weighted": 0.447485 }, { "accuracy": 0.465986, "f1": 0.467418, "f1_weighted": 0.513848 }, { "accuracy": 0.544218, "f1": 0.527106, "f1_weighted": 0.561815 }, { "accuracy": 0.408163, "f1": 0.411089, "f1_weighted": 0.413131 }, { "accuracy": 0.537415, "f1": 0.520742, "f1_weighted": 0.551636 } ], "main_score": 0.481633, "hf_subset": "som", "languages": [ "som-Latn" ] }, { "accuracy": 0.553782, "f1": 0.501059, "f1_weighted": 0.559329, "scores_per_experiment": [ { "accuracy": 0.529412, "f1": 0.462577, "f1_weighted": 0.537224 }, { "accuracy": 0.537815, "f1": 0.501185, "f1_weighted": 0.539313 }, { "accuracy": 0.558824, "f1": 0.500142, "f1_weighted": 0.55936 }, { "accuracy": 0.537815, "f1": 0.474266, "f1_weighted": 0.54528 }, { "accuracy": 0.548319, "f1": 0.488674, "f1_weighted": 0.553427 }, { "accuracy": 0.537815, "f1": 0.478484, "f1_weighted": 0.532544 }, { "accuracy": 0.554622, "f1": 0.513901, "f1_weighted": 0.563874 }, { "accuracy": 0.548319, "f1": 0.500766, "f1_weighted": 0.568335 }, { "accuracy": 0.62605, "f1": 0.586223, "f1_weighted": 0.63079 }, { "accuracy": 0.558824, "f1": 0.504375, "f1_weighted": 0.563143 } ], "main_score": 0.553782, "hf_subset": "swa", "languages": [ "swa-Latn" ] }, { "accuracy": 0.258456, "f1": 0.228124, "f1_weighted": 0.256736, "scores_per_experiment": [ { "accuracy": 0.194853, "f1": 0.184643, "f1_weighted": 0.182771 }, { "accuracy": 0.301471, "f1": 0.227761, "f1_weighted": 0.280765 }, { "accuracy": 0.264706, "f1": 0.222978, "f1_weighted": 0.262853 }, { "accuracy": 0.257353, "f1": 0.233392, "f1_weighted": 0.277359 }, { "accuracy": 0.235294, "f1": 0.20534, "f1_weighted": 0.240548 }, { "accuracy": 0.253676, "f1": 0.240948, "f1_weighted": 0.247559 }, { "accuracy": 0.327206, "f1": 0.294777, "f1_weighted": 0.342614 }, { "accuracy": 0.246324, "f1": 0.217487, "f1_weighted": 0.220552 }, { "accuracy": 0.246324, "f1": 0.230746, "f1_weighted": 0.246872 }, { "accuracy": 0.257353, "f1": 0.22317, "f1_weighted": 0.265465 } ], "main_score": 0.258456, "hf_subset": "tir", "languages": [ "tir-Ethi" ] }, { "accuracy": 0.631313, "f1": 0.536256, "f1_weighted": 0.653891, "scores_per_experiment": [ { "accuracy": 0.653199, "f1": 0.554815, "f1_weighted": 0.67526 }, { "accuracy": 0.589226, "f1": 0.494495, "f1_weighted": 0.613324 }, { "accuracy": 0.6633, "f1": 0.566979, "f1_weighted": 0.688219 }, { "accuracy": 0.602694, "f1": 0.52634, "f1_weighted": 0.624864 }, { "accuracy": 0.59596, "f1": 0.529187, "f1_weighted": 0.625425 }, { "accuracy": 0.612795, "f1": 0.522899, "f1_weighted": 0.638665 }, { "accuracy": 0.636364, "f1": 0.531457, "f1_weighted": 0.661267 }, { "accuracy": 0.619529, "f1": 0.507316, "f1_weighted": 0.646337 }, { "accuracy": 0.676768, "f1": 0.574134, "f1_weighted": 0.697697 }, { "accuracy": 0.6633, "f1": 0.554934, "f1_weighted": 0.667856 } ], "main_score": 0.631313, "hf_subset": "xho", "languages": [ "xho-Latn" ] }, { "accuracy": 0.669586, "f1": 0.668455, "f1_weighted": 0.671442, "scores_per_experiment": [ { "accuracy": 0.649635, "f1": 0.651692, "f1_weighted": 0.655294 }, { "accuracy": 0.6691, "f1": 0.667753, "f1_weighted": 0.670958 }, { "accuracy": 0.686131, "f1": 0.694854, "f1_weighted": 0.697434 }, { "accuracy": 0.664234, "f1": 0.667064, "f1_weighted": 0.670387 }, { "accuracy": 0.579075, "f1": 0.590414, "f1_weighted": 0.583744 }, { "accuracy": 0.676399, "f1": 0.666906, "f1_weighted": 0.674106 }, { "accuracy": 0.70073, "f1": 0.69677, "f1_weighted": 0.69666 }, { "accuracy": 0.710462, "f1": 0.695082, "f1_weighted": 0.702092 }, { "accuracy": 0.681265, "f1": 0.680012, "f1_weighted": 0.685902 }, { "accuracy": 0.678832, "f1": 0.674003, "f1_weighted": 0.677844 } ], "main_score": 0.669586, "hf_subset": "yor", "languages": [ "yor-Latn" ] } ] }, "evaluation_time": 125.83811569213867, "kg_co2_emissions": 0.010568726863401335 }