{ "dataset_revision": "8f95949846bb9e33c6aaf730ccfdb8fe6bcfb7a9", "task_name": "MultiHateClassification", "mteb_version": "1.18.0", "scores": { "test": [ { "accuracy": 0.6072, "f1": 0.571295, "f1_weighted": 0.613781, "ap": 0.365357, "ap_weighted": 0.365357, "scores_per_experiment": [ { "accuracy": 0.584, "f1": 0.55384, "f1_weighted": 0.596992, "ap": 0.348196, "ap_weighted": 0.348196 }, { "accuracy": 0.678, "f1": 0.634782, "f1_weighted": 0.681518, "ap": 0.406703, "ap_weighted": 0.406703 }, { "accuracy": 0.676, "f1": 0.616461, "f1_weighted": 0.672675, "ap": 0.389833, "ap_weighted": 0.389833 }, { "accuracy": 0.567, "f1": 0.544766, "f1_weighted": 0.582192, "ap": 0.345979, "ap_weighted": 0.345979 }, { "accuracy": 0.587, "f1": 0.564562, "f1_weighted": 0.601332, "ap": 0.358803, "ap_weighted": 0.358803 }, { "accuracy": 0.534, "f1": 0.52441, "f1_weighted": 0.549533, "ap": 0.342897, "ap_weighted": 0.342897 }, { "accuracy": 0.567, "f1": 0.555182, "f1_weighted": 0.582154, "ap": 0.360469, "ap_weighted": 0.360469 }, { "accuracy": 0.698, "f1": 0.589565, "f1_weighted": 0.668043, "ap": 0.378718, "ap_weighted": 0.378718 }, { "accuracy": 0.544, "f1": 0.533504, "f1_weighted": 0.559534, "ap": 0.347589, "ap_weighted": 0.347589 }, { "accuracy": 0.637, "f1": 0.595876, "f1_weighted": 0.643833, "ap": 0.374381, "ap_weighted": 0.374381 } ], "main_score": 0.6072, "hf_subset": "eng", "languages": [ "eng-Latn" ] }, { "accuracy": 0.5667, "f1": 0.523451, "f1_weighted": 0.575251, "ap": 0.320971, "ap_weighted": 0.320971, "scores_per_experiment": [ { "accuracy": 0.595, "f1": 0.527901, "f1_weighted": 0.599094, "ap": 0.313611, "ap_weighted": 0.313611 }, { "accuracy": 0.547, "f1": 0.52672, "f1_weighted": 0.565908, "ap": 0.325402, "ap_weighted": 0.325402 }, { "accuracy": 0.525, "f1": 0.496224, "f1_weighted": 0.544385, "ap": 0.30486, "ap_weighted": 0.30486 }, { "accuracy": 0.45, "f1": 0.449945, "f1_weighted": 0.447745, "ap": 0.318357, "ap_weighted": 0.318357 }, { "accuracy": 0.584, "f1": 0.550233, "f1_weighted": 0.599528, "ap": 0.332989, "ap_weighted": 0.332989 }, { "accuracy": 0.647, "f1": 0.560809, "f1_weighted": 0.638634, "ap": 0.332942, "ap_weighted": 0.332942 }, { "accuracy": 0.609, "f1": 0.53758, "f1_weighted": 0.610272, "ap": 0.318645, "ap_weighted": 0.318645 }, { "accuracy": 0.553, "f1": 0.538878, "f1_weighted": 0.571157, "ap": 0.3378, "ap_weighted": 0.3378 }, { "accuracy": 0.571, "f1": 0.525064, "f1_weighted": 0.584146, "ap": 0.314816, "ap_weighted": 0.314816 }, { "accuracy": 0.586, "f1": 0.521153, "f1_weighted": 0.591639, "ap": 0.310287, "ap_weighted": 0.310287 } ], "main_score": 0.5667, "hf_subset": "ita", "languages": [ "ita-Latn" ] }, { "accuracy": 0.5464, "f1": 0.527392, "f1_weighted": 0.55956, "ap": 0.333729, "ap_weighted": 0.333729, "scores_per_experiment": [ { "accuracy": 0.534, "f1": 0.524132, "f1_weighted": 0.551542, "ap": 0.332968, "ap_weighted": 0.332968 }, { "accuracy": 0.586, "f1": 0.531886, "f1_weighted": 0.59555, "ap": 0.317083, "ap_weighted": 0.317083 }, { "accuracy": 0.537, "f1": 0.529651, "f1_weighted": 0.553168, "ap": 0.340006, "ap_weighted": 0.340006 }, { "accuracy": 0.58, "f1": 0.54802, "f1_weighted": 0.59611, "ap": 0.332316, "ap_weighted": 0.332316 }, { "accuracy": 0.522, "f1": 0.514289, "f1_weighted": 0.538769, "ap": 0.329666, "ap_weighted": 0.329666 }, { "accuracy": 0.511, "f1": 0.507441, "f1_weighted": 0.524188, "ap": 0.332902, "ap_weighted": 0.332902 }, { "accuracy": 0.621, "f1": 0.574997, "f1_weighted": 0.630928, "ap": 0.345518, "ap_weighted": 0.345518 }, { "accuracy": 0.551, "f1": 0.527503, "f1_weighted": 0.56965, "ap": 0.323952, "ap_weighted": 0.323952 }, { "accuracy": 0.53, "f1": 0.524029, "f1_weighted": 0.545353, "ap": 0.338625, "ap_weighted": 0.338625 }, { "accuracy": 0.492, "f1": 0.491967, "f1_weighted": 0.490342, "ap": 0.344256, "ap_weighted": 0.344256 } ], "main_score": 0.5464, "hf_subset": "deu", "languages": [ "deu-Latn" ] }, { "accuracy": 0.5338, "f1": 0.507307, "f1_weighted": 0.547029, "ap": 0.319659, "ap_weighted": 0.319659, "scores_per_experiment": [ { "accuracy": 0.55, "f1": 0.534584, "f1_weighted": 0.568127, "ap": 0.33526, "ap_weighted": 0.33526 }, { "accuracy": 0.572, "f1": 0.534451, "f1_weighted": 0.586808, "ap": 0.323851, "ap_weighted": 0.323851 }, { "accuracy": 0.47, "f1": 0.469981, "f1_weighted": 0.47124, "ap": 0.327551, "ap_weighted": 0.327551 }, { "accuracy": 0.519, "f1": 0.514619, "f1_weighted": 0.532879, "ap": 0.336801, "ap_weighted": 0.336801 }, { "accuracy": 0.521, "f1": 0.479297, "f1_weighted": 0.537652, "ap": 0.29599, "ap_weighted": 0.29599 }, { "accuracy": 0.499, "f1": 0.4897, "f1_weighted": 0.51698, "ap": 0.314817, "ap_weighted": 0.314817 }, { "accuracy": 0.548, "f1": 0.488919, "f1_weighted": 0.557731, "ap": 0.298041, "ap_weighted": 0.298041 }, { "accuracy": 0.569, "f1": 0.527304, "f1_weighted": 0.582899, "ap": 0.318731, "ap_weighted": 0.318731 }, { "accuracy": 0.594, "f1": 0.544383, "f1_weighted": 0.603923, "ap": 0.326781, "ap_weighted": 0.326781 }, { "accuracy": 0.496, "f1": 0.489827, "f1_weighted": 0.51205, "ap": 0.318764, "ap_weighted": 0.318764 } ], "main_score": 0.5338, "hf_subset": "por", "languages": [ "por-Latn" ] }, { "accuracy": 0.5353, "f1": 0.513398, "f1_weighted": 0.550789, "ap": 0.321856, "ap_weighted": 0.321856, "scores_per_experiment": [ { "accuracy": 0.49, "f1": 0.488032, "f1_weighted": 0.500665, "ap": 0.325848, "ap_weighted": 0.325848 }, { "accuracy": 0.575, "f1": 0.535088, "f1_weighted": 0.589303, "ap": 0.322684, "ap_weighted": 0.322684 }, { "accuracy": 0.584, "f1": 0.552324, "f1_weighted": 0.599719, "ap": 0.335972, "ap_weighted": 0.335972 }, { "accuracy": 0.52, "f1": 0.491689, "f1_weighted": 0.539434, "ap": 0.303687, "ap_weighted": 0.303687 }, { "accuracy": 0.455, "f1": 0.449383, "f1_weighted": 0.471517, "ap": 0.298052, "ap_weighted": 0.298052 }, { "accuracy": 0.5, "f1": 0.493648, "f1_weighted": 0.516219, "ap": 0.320009, "ap_weighted": 0.320009 }, { "accuracy": 0.565, "f1": 0.533722, "f1_weighted": 0.581786, "ap": 0.324752, "ap_weighted": 0.324752 }, { "accuracy": 0.607, "f1": 0.55732, "f1_weighted": 0.616343, "ap": 0.333815, "ap_weighted": 0.333815 }, { "accuracy": 0.538, "f1": 0.522172, "f1_weighted": 0.556784, "ap": 0.326602, "ap_weighted": 0.326602 }, { "accuracy": 0.519, "f1": 0.510601, "f1_weighted": 0.536118, "ap": 0.327135, "ap_weighted": 0.327135 } ], "main_score": 0.5353, "hf_subset": "fra", "languages": [ "fra-Latn" ] }, { "accuracy": 0.5553, "f1": 0.523357, "f1_weighted": 0.569116, "ap": 0.320674, "ap_weighted": 0.320674, "scores_per_experiment": [ { "accuracy": 0.522, "f1": 0.502083, "f1_weighted": 0.542713, "ap": 0.308752, "ap_weighted": 0.308752 }, { "accuracy": 0.619, "f1": 0.570187, "f1_weighted": 0.629284, "ap": 0.338012, "ap_weighted": 0.338012 }, { "accuracy": 0.524, "f1": 0.516567, "f1_weighted": 0.541025, "ap": 0.328992, "ap_weighted": 0.328992 }, { "accuracy": 0.589, "f1": 0.539113, "f1_weighted": 0.600979, "ap": 0.318252, "ap_weighted": 0.318252 }, { "accuracy": 0.585, "f1": 0.555651, "f1_weighted": 0.602244, "ap": 0.335277, "ap_weighted": 0.335277 }, { "accuracy": 0.587, "f1": 0.548215, "f1_weighted": 0.602223, "ap": 0.326624, "ap_weighted": 0.326624 }, { "accuracy": 0.482, "f1": 0.478319, "f1_weighted": 0.496198, "ap": 0.312786, "ap_weighted": 0.312786 }, { "accuracy": 0.526, "f1": 0.519307, "f1_weighted": 0.542449, "ap": 0.331923, "ap_weighted": 0.331923 }, { "accuracy": 0.591, "f1": 0.496622, "f1_weighted": 0.585551, "ap": 0.294802, "ap_weighted": 0.294802 }, { "accuracy": 0.528, "f1": 0.507504, "f1_weighted": 0.548496, "ap": 0.311324, "ap_weighted": 0.311324 } ], "main_score": 0.5553, "hf_subset": "spa", "languages": [ "spa-Latn" ] }, { "accuracy": 0.5005, "f1": 0.482504, "f1_weighted": 0.501635, "ap": 0.321812, "ap_weighted": 0.321812, "scores_per_experiment": [ { "accuracy": 0.63, "f1": 0.560359, "f1_weighted": 0.63, "ap": 0.333519, "ap_weighted": 0.333519 }, { "accuracy": 0.631, "f1": 0.588018, "f1_weighted": 0.64098, "ap": 0.35655, "ap_weighted": 0.35655 }, { "accuracy": 0.531, "f1": 0.518836, "f1_weighted": 0.549285, "ap": 0.327846, "ap_weighted": 0.327846 }, { "accuracy": 0.504, "f1": 0.496254, "f1_weighted": 0.521115, "ap": 0.31958, "ap_weighted": 0.31958 }, { "accuracy": 0.448, "f1": 0.446006, "f1_weighted": 0.432776, "ap": 0.329396, "ap_weighted": 0.329396 }, { "accuracy": 0.413, "f1": 0.412999, "f1_weighted": 0.412766, "ap": 0.297365, "ap_weighted": 0.297365 }, { "accuracy": 0.494, "f1": 0.481708, "f1_weighted": 0.513475, "ap": 0.307171, "ap_weighted": 0.307171 }, { "accuracy": 0.483, "f1": 0.480527, "f1_weighted": 0.494793, "ap": 0.319961, "ap_weighted": 0.319961 }, { "accuracy": 0.368, "f1": 0.346838, "f1_weighted": 0.300045, "ap": 0.310756, "ap_weighted": 0.310756 }, { "accuracy": 0.503, "f1": 0.493493, "f1_weighted": 0.521111, "ap": 0.31597, "ap_weighted": 0.31597 } ], "main_score": 0.5005, "hf_subset": "ara", "languages": [ "ara-Arab" ] }, { "accuracy": 0.5454, "f1": 0.521759, "f1_weighted": 0.557884, "ap": 0.329008, "ap_weighted": 0.329008, "scores_per_experiment": [ { "accuracy": 0.514, "f1": 0.488806, "f1_weighted": 0.533746, "ap": 0.304283, "ap_weighted": 0.304283 }, { "accuracy": 0.558, "f1": 0.528028, "f1_weighted": 0.575127, "ap": 0.322797, "ap_weighted": 0.322797 }, { "accuracy": 0.508, "f1": 0.497875, "f1_weighted": 0.526111, "ap": 0.318444, "ap_weighted": 0.318444 }, { "accuracy": 0.444, "f1": 0.441854, "f1_weighted": 0.428151, "ap": 0.327882, "ap_weighted": 0.327882 }, { "accuracy": 0.486, "f1": 0.481989, "f1_weighted": 0.50004, "ap": 0.317966, "ap_weighted": 0.317966 }, { "accuracy": 0.611, "f1": 0.57661, "f1_weighted": 0.624394, "ap": 0.352019, "ap_weighted": 0.352019 }, { "accuracy": 0.573, "f1": 0.544776, "f1_weighted": 0.589663, "ap": 0.333543, "ap_weighted": 0.333543 }, { "accuracy": 0.559, "f1": 0.533222, "f1_weighted": 0.576661, "ap": 0.32769, "ap_weighted": 0.32769 }, { "accuracy": 0.64, "f1": 0.589086, "f1_weighted": 0.646364, "ap": 0.356446, "ap_weighted": 0.356446 }, { "accuracy": 0.561, "f1": 0.535339, "f1_weighted": 0.57858, "ap": 0.329015, "ap_weighted": 0.329015 } ], "main_score": 0.5454, "hf_subset": "hin", "languages": [ "hin-Deva" ] }, { "accuracy": 0.5311, "f1": 0.515513, "f1_weighted": 0.545074, "ap": 0.327081, "ap_weighted": 0.327081, "scores_per_experiment": [ { "accuracy": 0.561, "f1": 0.546468, "f1_weighted": 0.579266, "ap": 0.341072, "ap_weighted": 0.341072 }, { "accuracy": 0.583, "f1": 0.556371, "f1_weighted": 0.600282, "ap": 0.338721, "ap_weighted": 0.338721 }, { "accuracy": 0.509, "f1": 0.50508, "f1_weighted": 0.522875, "ap": 0.329414, "ap_weighted": 0.329414 }, { "accuracy": 0.552, "f1": 0.542514, "f1_weighted": 0.569128, "ap": 0.344271, "ap_weighted": 0.344271 }, { "accuracy": 0.536, "f1": 0.514594, "f1_weighted": 0.555775, "ap": 0.316215, "ap_weighted": 0.316215 }, { "accuracy": 0.538, "f1": 0.527074, "f1_weighted": 0.556115, "ap": 0.332267, "ap_weighted": 0.332267 }, { "accuracy": 0.438, "f1": 0.437991, "f1_weighted": 0.437083, "ap": 0.309147, "ap_weighted": 0.309147 }, { "accuracy": 0.465, "f1": 0.46491, "f1_weighted": 0.46772, "ap": 0.32076, "ap_weighted": 0.32076 }, { "accuracy": 0.542, "f1": 0.512498, "f1_weighted": 0.560948, "ap": 0.311192, "ap_weighted": 0.311192 }, { "accuracy": 0.587, "f1": 0.547633, "f1_weighted": 0.601546, "ap": 0.32775, "ap_weighted": 0.32775 } ], "main_score": 0.5311, "hf_subset": "pol", "languages": [ "pol-Latn" ] }, { "accuracy": 0.5297, "f1": 0.511029, "f1_weighted": 0.540251, "ap": 0.345133, "ap_weighted": 0.345133, "scores_per_experiment": [ { "accuracy": 0.531, "f1": 0.501568, "f1_weighted": 0.543475, "ap": 0.330472, "ap_weighted": 0.330472 }, { "accuracy": 0.452, "f1": 0.433226, "f1_weighted": 0.468917, "ap": 0.305511, "ap_weighted": 0.305511 }, { "accuracy": 0.522, "f1": 0.520273, "f1_weighted": 0.530232, "ap": 0.362205, "ap_weighted": 0.362205 }, { "accuracy": 0.467, "f1": 0.461285, "f1_weighted": 0.480484, "ap": 0.322713, "ap_weighted": 0.322713 }, { "accuracy": 0.543, "f1": 0.529234, "f1_weighted": 0.557088, "ap": 0.351753, "ap_weighted": 0.351753 }, { "accuracy": 0.464, "f1": 0.462432, "f1_weighted": 0.472476, "ap": 0.32962, "ap_weighted": 0.32962 }, { "accuracy": 0.603, "f1": 0.577648, "f1_weighted": 0.613451, "ap": 0.376936, "ap_weighted": 0.376936 }, { "accuracy": 0.627, "f1": 0.581726, "f1_weighted": 0.629339, "ap": 0.375363, "ap_weighted": 0.375363 }, { "accuracy": 0.534, "f1": 0.529846, "f1_weighted": 0.545137, "ap": 0.362634, "ap_weighted": 0.362634 }, { "accuracy": 0.554, "f1": 0.513047, "f1_weighted": 0.561908, "ap": 0.334124, "ap_weighted": 0.334124 } ], "main_score": 0.5297, "hf_subset": "cmn", "languages": [ "cmn-Hans" ] }, { "accuracy": 0.5493, "f1": 0.514187, "f1_weighted": 0.564157, "ap": 0.313383, "ap_weighted": 0.313383, "scores_per_experiment": [ { "accuracy": 0.568, "f1": 0.541596, "f1_weighted": 0.586043, "ap": 0.329357, "ap_weighted": 0.329357 }, { "accuracy": 0.515, "f1": 0.502421, "f1_weighted": 0.534383, "ap": 0.315805, "ap_weighted": 0.315805 }, { "accuracy": 0.63, "f1": 0.563628, "f1_weighted": 0.632383, "ap": 0.332879, "ap_weighted": 0.332879 }, { "accuracy": 0.513, "f1": 0.487031, "f1_weighted": 0.53366, "ap": 0.299878, "ap_weighted": 0.299878 }, { "accuracy": 0.477, "f1": 0.465125, "f1_weighted": 0.497323, "ap": 0.29721, "ap_weighted": 0.29721 }, { "accuracy": 0.499, "f1": 0.485668, "f1_weighted": 0.519122, "ap": 0.306158, "ap_weighted": 0.306158 }, { "accuracy": 0.598, "f1": 0.529316, "f1_weighted": 0.601956, "ap": 0.312315, "ap_weighted": 0.312315 }, { "accuracy": 0.581, "f1": 0.520581, "f1_weighted": 0.589339, "ap": 0.308484, "ap_weighted": 0.308484 }, { "accuracy": 0.566, "f1": 0.530772, "f1_weighted": 0.582714, "ap": 0.319127, "ap_weighted": 0.319127 }, { "accuracy": 0.546, "f1": 0.515733, "f1_weighted": 0.564644, "ap": 0.312618, "ap_weighted": 0.312618 } ], "main_score": 0.5493, "hf_subset": "nld", "languages": [ "nld-Latn" ] } ] }, "evaluation_time": 17.263341188430786, "kg_co2_emissions": 0.0004890434582188324 }