{ "dataset_revision": "4672e20407010da34463acc759c162ca9734bca6", "task_name": "MassiveIntentClassification", "mteb_version": "1.12.75-2.18.0", "scores": { "test": [ { "accuracy": 0.655279, "f1": 0.634081, "f1_weighted": 0.643452, "hf_subset": "sl", "languages": [ "slv-Latn" ], "main_score": 0.655279, "scores_per_experiment": [ { "accuracy": 0.669469, "f1": 0.645679, "f1_weighted": 0.657842 }, { "accuracy": 0.681574, "f1": 0.657967, "f1_weighted": 0.672757 }, { "accuracy": 0.631137, "f1": 0.615559, "f1_weighted": 0.622085 }, { "accuracy": 0.667115, "f1": 0.636127, "f1_weighted": 0.659011 }, { "accuracy": 0.649294, "f1": 0.628306, "f1_weighted": 0.633395 }, { "accuracy": 0.640215, "f1": 0.637659, "f1_weighted": 0.636694 }, { "accuracy": 0.661735, "f1": 0.635993, "f1_weighted": 0.648315 }, { "accuracy": 0.634163, "f1": 0.606909, "f1_weighted": 0.619538 }, { "accuracy": 0.637189, "f1": 0.62554, "f1_weighted": 0.609669 }, { "accuracy": 0.680901, "f1": 0.651066, "f1_weighted": 0.675215 } ], "mteb_version": "1.12.75" }, { "accuracy": 0.665266, "f1": 0.6426, "f1_weighted": 0.653442, "hf_subset": "ko", "languages": [ "kor-Kore" ], "main_score": 0.665266, "scores_per_experiment": [ { "accuracy": 0.69536, "f1": 0.666848, "f1_weighted": 0.685765 }, { "accuracy": 0.675521, "f1": 0.644675, "f1_weighted": 0.670101 }, { "accuracy": 0.661063, "f1": 0.635426, "f1_weighted": 0.650221 }, { "accuracy": 0.683591, "f1": 0.651173, "f1_weighted": 0.675028 }, { "accuracy": 0.657028, "f1": 0.634957, "f1_weighted": 0.642366 }, { "accuracy": 0.631809, "f1": 0.627352, "f1_weighted": 0.613357 }, { "accuracy": 0.666779, "f1": 0.647345, "f1_weighted": 0.65353 }, { "accuracy": 0.648285, "f1": 0.619517, "f1_weighted": 0.639852 }, { "accuracy": 0.663753, "f1": 0.646171, "f1_weighted": 0.644303 }, { "accuracy": 0.669469, "f1": 0.65254, "f1_weighted": 0.659897 } ], "mteb_version": "1.12.75" }, { "accuracy": 0.595326, "f1": 0.569019, "f1_weighted": 0.584594, "hf_subset": "tl", "languages": [ "tgl-Latn" ], "main_score": 0.595326, "scores_per_experiment": [ { "accuracy": 0.610289, "f1": 0.58276, "f1_weighted": 0.602851 }, { "accuracy": 0.631137, "f1": 0.598072, "f1_weighted": 0.628802 }, { "accuracy": 0.588097, "f1": 0.559968, "f1_weighted": 0.580732 }, { "accuracy": 0.620377, "f1": 0.581713, "f1_weighted": 0.618935 }, { "accuracy": 0.587088, "f1": 0.550035, "f1_weighted": 0.572381 }, { "accuracy": 0.56456, "f1": 0.557045, "f1_weighted": 0.558046 }, { "accuracy": 0.587424, "f1": 0.562, "f1_weighted": 0.570684 }, { "accuracy": 0.592468, "f1": 0.567425, "f1_weighted": 0.575667 }, { "accuracy": 0.581708, "f1": 0.55943, "f1_weighted": 0.566864 }, { "accuracy": 0.590114, "f1": 0.571742, "f1_weighted": 0.570979 } ], "mteb_version": "1.12.75" }, { "accuracy": 0.691493, "f1": 0.67303, "f1_weighted": 0.678942, "hf_subset": "sv", "languages": [ "swe-Latn" ], "main_score": 0.691493, "scores_per_experiment": [ { "accuracy": 0.706456, "f1": 0.685456, "f1_weighted": 0.696333 }, { "accuracy": 0.721587, "f1": 0.695822, "f1_weighted": 0.713797 }, { "accuracy": 0.680901, "f1": 0.661355, "f1_weighted": 0.670671 }, { "accuracy": 0.716543, "f1": 0.693498, "f1_weighted": 0.708392 }, { "accuracy": 0.679892, "f1": 0.658857, "f1_weighted": 0.658515 }, { "accuracy": 0.678547, "f1": 0.669399, "f1_weighted": 0.668706 }, { "accuracy": 0.693679, "f1": 0.67934, "f1_weighted": 0.684508 }, { "accuracy": 0.680901, "f1": 0.661582, "f1_weighted": 0.667647 }, { "accuracy": 0.658036, "f1": 0.652913, "f1_weighted": 0.63167 }, { "accuracy": 0.698386, "f1": 0.672074, "f1_weighted": 0.689184 } ], "mteb_version": "1.12.75" }, { "accuracy": 0.679489, "f1": 0.649362, "f1_weighted": 0.670481, "hf_subset": "pl", "languages": [ "pol-Latn" ], "main_score": 0.679489, "scores_per_experiment": [ { "accuracy": 0.693006, "f1": 0.670075, "f1_weighted": 0.682802 }, { "accuracy": 0.701076, "f1": 0.658747, "f1_weighted": 0.693145 }, { "accuracy": 0.681237, "f1": 0.64769, "f1_weighted": 0.673389 }, { "accuracy": 0.702085, "f1": 0.659805, "f1_weighted": 0.695484 }, { "accuracy": 0.671822, "f1": 0.642964, "f1_weighted": 0.657298 }, { "accuracy": 0.663753, "f1": 0.639506, "f1_weighted": 0.657587 }, { "accuracy": 0.678547, "f1": 0.654876, "f1_weighted": 0.66801 }, { "accuracy": 0.661063, "f1": 0.628151, "f1_weighted": 0.651193 }, { "accuracy": 0.661063, "f1": 0.642641, "f1_weighted": 0.649715 }, { "accuracy": 0.681237, "f1": 0.649166, "f1_weighted": 0.676189 } ], "mteb_version": "1.12.75" }, { "accuracy": 0.687592, "f1": 0.663272, "f1_weighted": 0.677254, "hf_subset": "ru", "languages": [ "rus-Cyrl" ], "main_score": 0.687592, "scores_per_experiment": [ { "accuracy": 0.713181, "f1": 0.689895, "f1_weighted": 0.705334 }, { "accuracy": 0.71688, "f1": 0.678757, "f1_weighted": 0.711326 }, { "accuracy": 0.682246, "f1": 0.655495, "f1_weighted": 0.674176 }, { "accuracy": 0.709818, "f1": 0.675916, "f1_weighted": 0.705105 }, { "accuracy": 0.681574, "f1": 0.660632, "f1_weighted": 0.665713 }, { "accuracy": 0.654674, "f1": 0.640523, "f1_weighted": 0.642903 }, { "accuracy": 0.675185, "f1": 0.661034, "f1_weighted": 0.661306 }, { "accuracy": 0.672159, "f1": 0.6391, "f1_weighted": 0.663974 }, { "accuracy": 0.662744, "f1": 0.649272, "f1_weighted": 0.640708 }, { "accuracy": 0.707465, "f1": 0.682101, "f1_weighted": 0.701993 } ], "mteb_version": "1.12.75" }, { "scores_per_experiment": [ { "accuracy": 0.728312, "f1": 0.699425, "f1_weighted": 0.718123, "precision": 0.682726, "precision_weighted": 0.772985, "recall": 0.798448, "recall_weighted": 0.728312, "ap": null, "ap_weighted": null }, { "accuracy": 0.738399, "f1": 0.707678, "f1_weighted": 0.73032, "precision": 0.685547, "precision_weighted": 0.773184, "recall": 0.807715, "recall_weighted": 0.738399, "ap": null, "ap_weighted": null }, { "accuracy": 0.70881, "f1": 0.679872, "f1_weighted": 0.695738, "precision": 0.653633, "precision_weighted": 0.728945, "recall": 0.790678, "recall_weighted": 0.70881, "ap": null, "ap_weighted": null }, { "accuracy": 0.72764, "f1": 0.70524, "f1_weighted": 0.717459, "precision": 0.693222, "precision_weighted": 0.784326, "recall": 0.803149, "recall_weighted": 0.72764, "ap": null, "ap_weighted": null }, { "accuracy": 0.704438, "f1": 0.684292, "f1_weighted": 0.685602, "precision": 0.654168, "precision_weighted": 0.714552, "recall": 0.793456, "recall_weighted": 0.704438, "ap": null, "ap_weighted": null }, { "accuracy": 0.68729, "f1": 0.679047, "f1_weighted": 0.677466, "precision": 0.665045, "precision_weighted": 0.737408, "recall": 0.788634, "recall_weighted": 0.68729, "ap": null, "ap_weighted": null }, { "accuracy": 0.704775, "f1": 0.696142, "f1_weighted": 0.69275, "precision": 0.682533, "precision_weighted": 0.769257, "recall": 0.807213, "recall_weighted": 0.704775, "ap": null, "ap_weighted": null }, { "accuracy": 0.706792, "f1": 0.682756, "f1_weighted": 0.694384, "precision": 0.66442, "precision_weighted": 0.753065, "recall": 0.790489, "recall_weighted": 0.706792, "ap": null, "ap_weighted": null }, { "accuracy": 0.679556, "f1": 0.675432, "f1_weighted": 0.655747, "precision": 0.668585, "precision_weighted": 0.749378, "recall": 0.782746, "recall_weighted": 0.679556, "ap": null, "ap_weighted": null }, { "accuracy": 0.721923, "f1": 0.703313, "f1_weighted": 0.7154, "precision": 0.691572, "precision_weighted": 0.778024, "recall": 0.800844, "recall_weighted": 0.721923, "ap": null, "ap_weighted": null } ], "accuracy": 0.710794, "f1": 0.69132, "f1_weighted": 0.698299, "precision": 0.674145, "precision_weighted": 0.756112, "recall": 0.796337, "recall_weighted": 0.710794, "ap": NaN, "ap_weighted": NaN, "main_score": 0.710794, "hf_subset": "en", "languages": [ "eng-Latn" ], "mteb_version": "2.18.0" }, { "scores_per_experiment": [ { "accuracy": 0.647276, "f1": 0.618636, "f1_weighted": 0.641741, "precision": 0.604537, "precision_weighted": 0.696874, "recall": 0.722682, "recall_weighted": 0.647276, "ap": null, "ap_weighted": null }, { "accuracy": 0.651984, "f1": 0.611171, "f1_weighted": 0.646868, "precision": 0.599406, "precision_weighted": 0.706705, "recall": 0.717629, "recall_weighted": 0.651984, "ap": null, "ap_weighted": null }, { "accuracy": 0.619368, "f1": 0.600835, "f1_weighted": 0.611676, "precision": 0.585351, "precision_weighted": 0.663483, "recall": 0.706253, "recall_weighted": 0.619368, "ap": null, "ap_weighted": null }, { "accuracy": 0.656019, "f1": 0.623314, "f1_weighted": 0.651207, "precision": 0.616608, "precision_weighted": 0.726078, "recall": 0.725294, "recall_weighted": 0.656019, "ap": null, "ap_weighted": null }, { "accuracy": 0.618359, "f1": 0.582892, "f1_weighted": 0.6026, "precision": 0.561923, "precision_weighted": 0.637991, "recall": 0.680787, "recall_weighted": 0.618359, "ap": null, "ap_weighted": null }, { "accuracy": 0.616005, "f1": 0.593015, "f1_weighted": 0.609053, "precision": 0.585044, "precision_weighted": 0.685481, "recall": 0.702812, "recall_weighted": 0.616005, "ap": null, "ap_weighted": null }, { "accuracy": 0.620377, "f1": 0.597767, "f1_weighted": 0.610647, "precision": 0.587995, "precision_weighted": 0.681843, "recall": 0.709406, "recall_weighted": 0.620377, "ap": null, "ap_weighted": null }, { "accuracy": 0.643914, "f1": 0.600322, "f1_weighted": 0.633914, "precision": 0.585176, "precision_weighted": 0.697155, "recall": 0.712086, "recall_weighted": 0.643914, "ap": null, "ap_weighted": null }, { "accuracy": 0.620713, "f1": 0.59794, "f1_weighted": 0.602408, "precision": 0.60477, "precision_weighted": 0.703955, "recall": 0.708382, "recall_weighted": 0.620713, "ap": null, "ap_weighted": null }, { "accuracy": 0.639206, "f1": 0.597677, "f1_weighted": 0.63132, "precision": 0.592767, "precision_weighted": 0.701932, "recall": 0.710413, "recall_weighted": 0.639206, "ap": null, "ap_weighted": null } ], "accuracy": 0.633322, "f1": 0.602357, "f1_weighted": 0.624143, "precision": 0.592358, "precision_weighted": 0.69015, "recall": 0.709575, "recall_weighted": 0.633322, "ap": NaN, "ap_weighted": NaN, "main_score": 0.633322, "hf_subset": "af", "languages": [ "afr-Latn" ], "mteb_version": "2.18.0" }, { "accuracy": 0.642367, "f1": 0.619945, "f1_weighted": 0.633969, "hf_subset": "az", "languages": [ "aze-Latn" ], "main_score": 0.642367, "scores_per_experiment": [ { "accuracy": 0.645259, "f1": 0.622583, "f1_weighted": 0.640336 }, { "accuracy": 0.66308, "f1": 0.634102, "f1_weighted": 0.656897 }, { "accuracy": 0.635844, "f1": 0.607464, "f1_weighted": 0.631915 }, { "accuracy": 0.664089, "f1": 0.634692, "f1_weighted": 0.657619 }, { "accuracy": 0.655683, "f1": 0.621491, "f1_weighted": 0.639847 }, { "accuracy": 0.614997, "f1": 0.610958, "f1_weighted": 0.608173 }, { "accuracy": 0.640888, "f1": 0.620601, "f1_weighted": 0.633204 }, { "accuracy": 0.631473, "f1": 0.612728, "f1_weighted": 0.627204 }, { "accuracy": 0.625084, "f1": 0.609798, "f1_weighted": 0.605373 }, { "accuracy": 0.647276, "f1": 0.625037, "f1_weighted": 0.639126 } ], "mteb_version": "1.12.75" }, { "accuracy": 0.674512, "f1": 0.650722, "f1_weighted": 0.664153, "hf_subset": "fr", "languages": [ "fra-Latn" ], "main_score": 0.674512, "scores_per_experiment": [ { "accuracy": 0.675185, "f1": 0.655315, "f1_weighted": 0.668099 }, { "accuracy": 0.691997, "f1": 0.662649, "f1_weighted": 0.688517 }, { "accuracy": 0.667451, "f1": 0.646778, "f1_weighted": 0.656272 }, { "accuracy": 0.683927, "f1": 0.654732, "f1_weighted": 0.675172 }, { "accuracy": 0.678884, "f1": 0.64963, "f1_weighted": 0.662101 }, { "accuracy": 0.642569, "f1": 0.628084, "f1_weighted": 0.631012 }, { "accuracy": 0.672159, "f1": 0.657037, "f1_weighted": 0.661125 }, { "accuracy": 0.677539, "f1": 0.642875, "f1_weighted": 0.669555 }, { "accuracy": 0.667451, "f1": 0.655895, "f1_weighted": 0.650315 }, { "accuracy": 0.687962, "f1": 0.654228, "f1_weighted": 0.679364 } ], "mteb_version": "1.12.75" }, { "accuracy": 0.629523, "f1": 0.601034, "f1_weighted": 0.619521, "hf_subset": "bn", "languages": [ "ben-Beng" ], "main_score": 0.629523, "scores_per_experiment": [ { "accuracy": 0.642905, "f1": 0.609529, "f1_weighted": 0.635901 }, { "accuracy": 0.651984, "f1": 0.608314, "f1_weighted": 0.646683 }, { "accuracy": 0.62811, "f1": 0.59956, "f1_weighted": 0.616021 }, { "accuracy": 0.628447, "f1": 0.597109, "f1_weighted": 0.624453 }, { "accuracy": 0.618023, "f1": 0.592592, "f1_weighted": 0.603344 }, { "accuracy": 0.602892, "f1": 0.590478, "f1_weighted": 0.591008 }, { "accuracy": 0.6308, "f1": 0.600524, "f1_weighted": 0.619828 }, { "accuracy": 0.606254, "f1": 0.580786, "f1_weighted": 0.593749 }, { "accuracy": 0.638198, "f1": 0.615152, "f1_weighted": 0.622418 }, { "accuracy": 0.647613, "f1": 0.616295, "f1_weighted": 0.641807 } ], "mteb_version": "1.12.75" }, { "accuracy": 0.675925, "f1": 0.655529, "f1_weighted": 0.662641, "hf_subset": "vi", "languages": [ "vie-Latn" ], "main_score": 0.675925, "scores_per_experiment": [ { "accuracy": 0.70612, "f1": 0.669085, "f1_weighted": 0.697809 }, { "accuracy": 0.685609, "f1": 0.662309, "f1_weighted": 0.675276 }, { "accuracy": 0.659045, "f1": 0.632195, "f1_weighted": 0.64605 }, { "accuracy": 0.684264, "f1": 0.657351, "f1_weighted": 0.674573 }, { "accuracy": 0.671822, "f1": 0.649677, "f1_weighted": 0.653414 }, { "accuracy": 0.665434, "f1": 0.65576, "f1_weighted": 0.651638 }, { "accuracy": 0.674512, "f1": 0.664086, "f1_weighted": 0.65751 }, { "accuracy": 0.662744, "f1": 0.636809, "f1_weighted": 0.651928 }, { "accuracy": 0.669805, "f1": 0.664445, "f1_weighted": 0.64981 }, { "accuracy": 0.679892, "f1": 0.663577, "f1_weighted": 0.668404 } ], "mteb_version": "1.12.75" }, { "scores_per_experiment": [ { "accuracy": 0.558507, "f1": 0.530986, "f1_weighted": 0.558737, "precision": 0.521498, "precision_weighted": 0.630376, "recall": 0.63751, "recall_weighted": 0.558507, "ap": null, "ap_weighted": null }, { "accuracy": 0.578346, "f1": 0.534767, "f1_weighted": 0.577381, "precision": 0.537072, "precision_weighted": 0.636608, "recall": 0.615318, "recall_weighted": 0.578346, "ap": null, "ap_weighted": null }, { "accuracy": 0.551782, "f1": 0.519602, "f1_weighted": 0.548162, "precision": 0.509185, "precision_weighted": 0.604443, "recall": 0.621883, "recall_weighted": 0.551782, "ap": null, "ap_weighted": null }, { "accuracy": 0.56187, "f1": 0.512005, "f1_weighted": 0.560591, "precision": 0.49876, "precision_weighted": 0.620377, "recall": 0.606222, "recall_weighted": 0.56187, "ap": null, "ap_weighted": null }, { "accuracy": 0.540686, "f1": 0.498256, "f1_weighted": 0.534242, "precision": 0.487427, "precision_weighted": 0.585416, "recall": 0.595262, "recall_weighted": 0.540686, "ap": null, "ap_weighted": null }, { "accuracy": 0.519166, "f1": 0.496142, "f1_weighted": 0.505124, "precision": 0.504366, "precision_weighted": 0.616301, "recall": 0.597975, "recall_weighted": 0.519166, "ap": null, "ap_weighted": null }, { "accuracy": 0.557835, "f1": 0.516439, "f1_weighted": 0.549559, "precision": 0.499706, "precision_weighted": 0.6043, "recall": 0.608789, "recall_weighted": 0.557835, "ap": null, "ap_weighted": null }, { "accuracy": 0.522865, "f1": 0.478775, "f1_weighted": 0.510872, "precision": 0.482441, "precision_weighted": 0.607152, "recall": 0.581887, "recall_weighted": 0.522865, "ap": null, "ap_weighted": null }, { "accuracy": 0.554808, "f1": 0.516968, "f1_weighted": 0.550997, "precision": 0.520396, "precision_weighted": 0.636855, "recall": 0.619525, "recall_weighted": 0.554808, "ap": null, "ap_weighted": null }, { "accuracy": 0.563215, "f1": 0.51827, "f1_weighted": 0.560974, "precision": 0.517193, "precision_weighted": 0.638311, "recall": 0.607067, "recall_weighted": 0.563215, "ap": null, "ap_weighted": null } ], "accuracy": 0.550908, "f1": 0.512221, "f1_weighted": 0.545664, "precision": 0.507804, "precision_weighted": 0.618014, "recall": 0.609144, "recall_weighted": 0.550908, "ap": NaN, "ap_weighted": NaN, "main_score": 0.550908, "hf_subset": "am", "languages": [ "amh-Ethi" ], "mteb_version": "2.18.0" }, { "accuracy": 0.672629, "f1": 0.641456, "f1_weighted": 0.665251, "hf_subset": "fi", "languages": [ "fin-Latn" ], "main_score": 0.672629, "scores_per_experiment": [ { "accuracy": 0.689644, "f1": 0.65762, "f1_weighted": 0.684739 }, { "accuracy": 0.685272, "f1": 0.645665, "f1_weighted": 0.680478 }, { "accuracy": 0.673504, "f1": 0.634753, "f1_weighted": 0.667049 }, { "accuracy": 0.696705, "f1": 0.658865, "f1_weighted": 0.695893 }, { "accuracy": 0.654674, "f1": 0.624641, "f1_weighted": 0.638829 }, { "accuracy": 0.648958, "f1": 0.631205, "f1_weighted": 0.643701 }, { "accuracy": 0.67115, "f1": 0.642636, "f1_weighted": 0.663529 }, { "accuracy": 0.668124, "f1": 0.625518, "f1_weighted": 0.659405 }, { "accuracy": 0.64694, "f1": 0.638241, "f1_weighted": 0.629315 }, { "accuracy": 0.691325, "f1": 0.655414, "f1_weighted": 0.689569 } ], "mteb_version": "1.12.75" }, { "accuracy": 0.669637, "f1": 0.636734, "f1_weighted": 0.660669, "hf_subset": "tr", "languages": [ "tur-Latn" ], "main_score": 0.669637, "scores_per_experiment": [ { "accuracy": 0.681237, "f1": 0.638404, "f1_weighted": 0.673877 }, { "accuracy": 0.686617, "f1": 0.647291, "f1_weighted": 0.677761 }, { "accuracy": 0.678211, "f1": 0.635003, "f1_weighted": 0.667792 }, { "accuracy": 0.662744, "f1": 0.625612, "f1_weighted": 0.657019 }, { "accuracy": 0.662744, "f1": 0.629871, "f1_weighted": 0.645937 }, { "accuracy": 0.653329, "f1": 0.633462, "f1_weighted": 0.651049 }, { "accuracy": 0.67384, "f1": 0.64531, "f1_weighted": 0.665682 }, { "accuracy": 0.660054, "f1": 0.630805, "f1_weighted": 0.654451 }, { "accuracy": 0.665434, "f1": 0.63797, "f1_weighted": 0.647129 }, { "accuracy": 0.672159, "f1": 0.643618, "f1_weighted": 0.665994 } ], "mteb_version": "1.12.75" }, { "accuracy": 0.662172, "f1": 0.643883, "f1_weighted": 0.650633, "hf_subset": "el", "languages": [ "ell-Grek" ], "main_score": 0.662172, "scores_per_experiment": [ { "accuracy": 0.676866, "f1": 0.653302, "f1_weighted": 0.667945 }, { "accuracy": 0.690989, "f1": 0.667157, "f1_weighted": 0.682717 }, { "accuracy": 0.659381, "f1": 0.636804, "f1_weighted": 0.646414 }, { "accuracy": 0.685609, "f1": 0.647477, "f1_weighted": 0.678654 }, { "accuracy": 0.643241, "f1": 0.626826, "f1_weighted": 0.625129 }, { "accuracy": 0.635508, "f1": 0.630575, "f1_weighted": 0.626327 }, { "accuracy": 0.668796, "f1": 0.662851, "f1_weighted": 0.654741 }, { "accuracy": 0.634163, "f1": 0.622712, "f1_weighted": 0.623541 }, { "accuracy": 0.637861, "f1": 0.6313, "f1_weighted": 0.61851 }, { "accuracy": 0.689307, "f1": 0.659831, "f1_weighted": 0.682353 } ], "mteb_version": "1.12.75" }, { "accuracy": 0.523201, "f1": 0.491512, "f1_weighted": 0.515495, "hf_subset": "jv", "languages": [ "jav-Latn" ], "main_score": 0.523201, "scores_per_experiment": [ { "accuracy": 0.536987, "f1": 0.513866, "f1_weighted": 0.538668 }, { "accuracy": 0.531271, "f1": 0.510846, "f1_weighted": 0.528939 }, { "accuracy": 0.525219, "f1": 0.489744, "f1_weighted": 0.517481 }, { "accuracy": 0.537996, "f1": 0.492662, "f1_weighted": 0.534041 }, { "accuracy": 0.518494, "f1": 0.477656, "f1_weighted": 0.50743 }, { "accuracy": 0.498655, "f1": 0.478617, "f1_weighted": 0.482138 }, { "accuracy": 0.520175, "f1": 0.488248, "f1_weighted": 0.510251 }, { "accuracy": 0.520175, "f1": 0.477777, "f1_weighted": 0.508741 }, { "accuracy": 0.536987, "f1": 0.500275, "f1_weighted": 0.528557 }, { "accuracy": 0.506052, "f1": 0.485429, "f1_weighted": 0.498702 } ], "mteb_version": "1.12.75" }, { "accuracy": 0.682582, "f1": 0.659336, "f1_weighted": 0.671189, "hf_subset": "nl", "languages": [ "nld-Latn" ], "main_score": 0.682582, "scores_per_experiment": [ { "accuracy": 0.699731, "f1": 0.67673, "f1_weighted": 0.689011 }, { "accuracy": 0.705783, "f1": 0.670422, "f1_weighted": 0.699742 }, { "accuracy": 0.674849, "f1": 0.653542, "f1_weighted": 0.66845 }, { "accuracy": 0.686954, "f1": 0.663994, "f1_weighted": 0.68075 }, { "accuracy": 0.672831, "f1": 0.649403, "f1_weighted": 0.655299 }, { "accuracy": 0.664761, "f1": 0.653688, "f1_weighted": 0.657383 }, { "accuracy": 0.690652, "f1": 0.660928, "f1_weighted": 0.677208 }, { "accuracy": 0.680565, "f1": 0.652324, "f1_weighted": 0.6658 }, { "accuracy": 0.655683, "f1": 0.641536, "f1_weighted": 0.6309 }, { "accuracy": 0.694015, "f1": 0.670795, "f1_weighted": 0.687349 } ], "mteb_version": "1.12.75" }, { "accuracy": 0.694418, "f1": 0.6696, "f1_weighted": 0.68556, "hf_subset": "fa", "languages": [ "fas-Arab" ], "main_score": 0.694418, "scores_per_experiment": [ { "accuracy": 0.712172, "f1": 0.687407, "f1_weighted": 0.706177 }, { "accuracy": 0.707128, "f1": 0.672318, "f1_weighted": 0.698543 }, { "accuracy": 0.687626, "f1": 0.658232, "f1_weighted": 0.673737 }, { "accuracy": 0.707465, "f1": 0.673353, "f1_weighted": 0.700677 }, { "accuracy": 0.677539, "f1": 0.66196, "f1_weighted": 0.662647 }, { "accuracy": 0.680565, "f1": 0.665243, "f1_weighted": 0.677121 }, { "accuracy": 0.707465, "f1": 0.687472, "f1_weighted": 0.697513 }, { "accuracy": 0.678547, "f1": 0.642832, "f1_weighted": 0.671096 }, { "accuracy": 0.695696, "f1": 0.679534, "f1_weighted": 0.68248 }, { "accuracy": 0.68998, "f1": 0.667649, "f1_weighted": 0.685606 } ], "mteb_version": "1.12.75" }, { "scores_per_experiment": [ { "accuracy": 0.595831, "f1": 0.565047, "f1_weighted": 0.596431, "precision": 0.563916, "precision_weighted": 0.667272, "recall": 0.665721, "recall_weighted": 0.595831, "ap": null, "ap_weighted": null }, { "accuracy": 0.585071, "f1": 0.547094, "f1_weighted": 0.581346, "precision": 0.544427, "precision_weighted": 0.647427, "recall": 0.639806, "recall_weighted": 0.585071, "ap": null, "ap_weighted": null }, { "accuracy": 0.555481, "f1": 0.517642, "f1_weighted": 0.552248, "precision": 0.515045, "precision_weighted": 0.642136, "recall": 0.60479, "recall_weighted": 0.555481, "ap": null, "ap_weighted": null }, { "accuracy": 0.584062, "f1": 0.557073, "f1_weighted": 0.585754, "precision": 0.556745, "precision_weighted": 0.67183, "recall": 0.64094, "recall_weighted": 0.584062, "ap": null, "ap_weighted": null }, { "accuracy": 0.544048, "f1": 0.507878, "f1_weighted": 0.52546, "precision": 0.517498, "precision_weighted": 0.636283, "recall": 0.630251, "recall_weighted": 0.544048, "ap": null, "ap_weighted": null }, { "accuracy": 0.544721, "f1": 0.537494, "f1_weighted": 0.539416, "precision": 0.545598, "precision_weighted": 0.653209, "recall": 0.632751, "recall_weighted": 0.544721, "ap": null, "ap_weighted": null }, { "accuracy": 0.572629, "f1": 0.537713, "f1_weighted": 0.562484, "precision": 0.531299, "precision_weighted": 0.630108, "recall": 0.646001, "recall_weighted": 0.572629, "ap": null, "ap_weighted": null }, { "accuracy": 0.566913, "f1": 0.533777, "f1_weighted": 0.559733, "precision": 0.534688, "precision_weighted": 0.636972, "recall": 0.622204, "recall_weighted": 0.566913, "ap": null, "ap_weighted": null }, { "accuracy": 0.558843, "f1": 0.5434, "f1_weighted": 0.536375, "precision": 0.548082, "precision_weighted": 0.629631, "recall": 0.656476, "recall_weighted": 0.558843, "ap": null, "ap_weighted": null }, { "accuracy": 0.574647, "f1": 0.539865, "f1_weighted": 0.569498, "precision": 0.545231, "precision_weighted": 0.673079, "recall": 0.64493, "recall_weighted": 0.574647, "ap": null, "ap_weighted": null } ], "accuracy": 0.568225, "f1": 0.538698, "f1_weighted": 0.560874, "precision": 0.540253, "precision_weighted": 0.648795, "recall": 0.638387, "recall_weighted": 0.568225, "ap": NaN, "ap_weighted": NaN, "main_score": 0.568225, "hf_subset": "sw", "languages": [ "swa-Latn" ], "mteb_version": "2.18.0" }, { "accuracy": 0.672596, "f1": 0.644748, "f1_weighted": 0.661874, "hf_subset": "da", "languages": [ "dan-Latn" ], "main_score": 0.672596, "scores_per_experiment": [ { "accuracy": 0.682919, "f1": 0.649876, "f1_weighted": 0.674413 }, { "accuracy": 0.699731, "f1": 0.666664, "f1_weighted": 0.692796 }, { "accuracy": 0.663753, "f1": 0.635695, "f1_weighted": 0.655325 }, { "accuracy": 0.688635, "f1": 0.646235, "f1_weighted": 0.681849 }, { "accuracy": 0.660054, "f1": 0.631856, "f1_weighted": 0.642998 }, { "accuracy": 0.658036, "f1": 0.647925, "f1_weighted": 0.649758 }, { "accuracy": 0.67922, "f1": 0.654461, "f1_weighted": 0.669823 }, { "accuracy": 0.657364, "f1": 0.633025, "f1_weighted": 0.644956 }, { "accuracy": 0.64156, "f1": 0.626041, "f1_weighted": 0.619766 }, { "accuracy": 0.694687, "f1": 0.655705, "f1_weighted": 0.687061 } ], "mteb_version": "1.12.75" }, { "accuracy": 0.557465, "f1": 0.51412, "f1_weighted": 0.545427, "hf_subset": "ar", "languages": [ "ara-Arab" ], "main_score": 0.557465, "scores_per_experiment": [ { "accuracy": 0.566913, "f1": 0.499604, "f1_weighted": 0.554939 }, { "accuracy": 0.583389, "f1": 0.545568, "f1_weighted": 0.585575 }, { "accuracy": 0.55918, "f1": 0.519179, "f1_weighted": 0.547617 }, { "accuracy": 0.555481, "f1": 0.517631, "f1_weighted": 0.547902 }, { "accuracy": 0.548756, "f1": 0.507883, "f1_weighted": 0.527409 }, { "accuracy": 0.524882, "f1": 0.500128, "f1_weighted": 0.509304 }, { "accuracy": 0.566241, "f1": 0.52113, "f1_weighted": 0.552082 }, { "accuracy": 0.544385, "f1": 0.491301, "f1_weighted": 0.531117 }, { "accuracy": 0.557498, "f1": 0.518559, "f1_weighted": 0.535494 }, { "accuracy": 0.567922, "f1": 0.520222, "f1_weighted": 0.562828 } ], "mteb_version": "1.12.75" }, { "accuracy": 0.670713, "f1": 0.6457, "f1_weighted": 0.660602, "hf_subset": "nb", "languages": [ "nob-Latn" ], "main_score": 0.670713, "scores_per_experiment": [ { "accuracy": 0.679556, "f1": 0.655709, "f1_weighted": 0.673766 }, { "accuracy": 0.695696, "f1": 0.665006, "f1_weighted": 0.689928 }, { "accuracy": 0.672831, "f1": 0.645956, "f1_weighted": 0.663098 }, { "accuracy": 0.683255, "f1": 0.653666, "f1_weighted": 0.672249 }, { "accuracy": 0.650639, "f1": 0.631124, "f1_weighted": 0.634023 }, { "accuracy": 0.652993, "f1": 0.638816, "f1_weighted": 0.646694 }, { "accuracy": 0.676866, "f1": 0.647313, "f1_weighted": 0.670971 }, { "accuracy": 0.661735, "f1": 0.633078, "f1_weighted": 0.653265 }, { "accuracy": 0.639879, "f1": 0.624161, "f1_weighted": 0.613673 }, { "accuracy": 0.693679, "f1": 0.662166, "f1_weighted": 0.688355 } ], "mteb_version": "1.12.75" }, { "accuracy": 0.650874, "f1": 0.623844, "f1_weighted": 0.640924, "hf_subset": "hi", "languages": [ "hin-Deva" ], "main_score": 0.650874, "scores_per_experiment": [ { "accuracy": 0.667115, "f1": 0.640189, "f1_weighted": 0.662645 }, { "accuracy": 0.687962, "f1": 0.649501, "f1_weighted": 0.683819 }, { "accuracy": 0.645931, "f1": 0.608631, "f1_weighted": 0.637634 }, { "accuracy": 0.659045, "f1": 0.620985, "f1_weighted": 0.655126 }, { "accuracy": 0.64694, "f1": 0.619052, "f1_weighted": 0.63027 }, { "accuracy": 0.616342, "f1": 0.609938, "f1_weighted": 0.601968 }, { "accuracy": 0.657028, "f1": 0.63761, "f1_weighted": 0.646954 }, { "accuracy": 0.632145, "f1": 0.603079, "f1_weighted": 0.623157 }, { "accuracy": 0.63887, "f1": 0.618471, "f1_weighted": 0.620118 }, { "accuracy": 0.657364, "f1": 0.630988, "f1_weighted": 0.647554 } ], "mteb_version": "1.12.75" }, { "accuracy": 0.63924, "f1": 0.606175, "f1_weighted": 0.633318, "hf_subset": "hy", "languages": [ "hye-Armn" ], "main_score": 0.63924, "scores_per_experiment": [ { "accuracy": 0.673167, "f1": 0.631102, "f1_weighted": 0.669466 }, { "accuracy": 0.6577, "f1": 0.619558, "f1_weighted": 0.654779 }, { "accuracy": 0.619032, "f1": 0.594215, "f1_weighted": 0.613842 }, { "accuracy": 0.651311, "f1": 0.603158, "f1_weighted": 0.650067 }, { "accuracy": 0.636853, "f1": 0.604257, "f1_weighted": 0.621074 }, { "accuracy": 0.608272, "f1": 0.591082, "f1_weighted": 0.608503 }, { "accuracy": 0.645259, "f1": 0.61642, "f1_weighted": 0.637556 }, { "accuracy": 0.622394, "f1": 0.578486, "f1_weighted": 0.618247 }, { "accuracy": 0.633826, "f1": 0.611209, "f1_weighted": 0.620986 }, { "accuracy": 0.644586, "f1": 0.612263, "f1_weighted": 0.638658 } ], "mteb_version": "1.12.75" }, { "accuracy": 0.692434, "f1": 0.662809, "f1_weighted": 0.680602, "hf_subset": "ja", "languages": [ "jpn-Jpan" ], "main_score": 0.692434, "scores_per_experiment": [ { "accuracy": 0.71419, "f1": 0.668832, "f1_weighted": 0.704918 }, { "accuracy": 0.716543, "f1": 0.690258, "f1_weighted": 0.7117 }, { "accuracy": 0.691325, "f1": 0.654384, "f1_weighted": 0.682308 }, { "accuracy": 0.705447, "f1": 0.670026, "f1_weighted": 0.696599 }, { "accuracy": 0.682582, "f1": 0.65435, "f1_weighted": 0.668642 }, { "accuracy": 0.68191, "f1": 0.662839, "f1_weighted": 0.666637 }, { "accuracy": 0.683591, "f1": 0.651512, "f1_weighted": 0.668324 }, { "accuracy": 0.668796, "f1": 0.647081, "f1_weighted": 0.651241 }, { "accuracy": 0.684936, "f1": 0.659634, "f1_weighted": 0.671968 }, { "accuracy": 0.695024, "f1": 0.669179, "f1_weighted": 0.683682 } ], "mteb_version": "1.12.75" }, { "accuracy": 0.626059, "f1": 0.59508, "f1_weighted": 0.615111, "hf_subset": "mn", "languages": [ "mon-Cyrl" ], "main_score": 0.626059, "scores_per_experiment": [ { "accuracy": 0.64156, "f1": 0.613761, "f1_weighted": 0.636427 }, { "accuracy": 0.652993, "f1": 0.619343, "f1_weighted": 0.647386 }, { "accuracy": 0.607599, "f1": 0.574471, "f1_weighted": 0.601874 }, { "accuracy": 0.653329, "f1": 0.607717, "f1_weighted": 0.646432 }, { "accuracy": 0.608944, "f1": 0.571488, "f1_weighted": 0.593004 }, { "accuracy": 0.603228, "f1": 0.584995, "f1_weighted": 0.58652 }, { "accuracy": 0.631137, "f1": 0.6028, "f1_weighted": 0.616456 }, { "accuracy": 0.611634, "f1": 0.585345, "f1_weighted": 0.597621 }, { "accuracy": 0.61197, "f1": 0.58914, "f1_weighted": 0.596426 }, { "accuracy": 0.638198, "f1": 0.601738, "f1_weighted": 0.62896 } ], "mteb_version": "1.12.75" }, { "accuracy": 0.690215, "f1": 0.66792, "f1_weighted": 0.680081, "hf_subset": "zh-CN", "languages": [ "cmo-Hans" ], "main_score": 0.690215, "scores_per_experiment": [ { "accuracy": 0.701748, "f1": 0.667887, "f1_weighted": 0.693155 }, { "accuracy": 0.717552, "f1": 0.69475, "f1_weighted": 0.710988 }, { "accuracy": 0.683927, "f1": 0.647827, "f1_weighted": 0.67622 }, { "accuracy": 0.694687, "f1": 0.673654, "f1_weighted": 0.689392 }, { "accuracy": 0.68998, "f1": 0.656875, "f1_weighted": 0.673059 }, { "accuracy": 0.677539, "f1": 0.659529, "f1_weighted": 0.669237 }, { "accuracy": 0.679892, "f1": 0.669022, "f1_weighted": 0.667033 }, { "accuracy": 0.67384, "f1": 0.658085, "f1_weighted": 0.661868 }, { "accuracy": 0.678211, "f1": 0.665947, "f1_weighted": 0.663727 }, { "accuracy": 0.704775, "f1": 0.685626, "f1_weighted": 0.696135 } ], "mteb_version": "1.12.75" }, { "accuracy": 0.645595, "f1": 0.640433, "f1_weighted": 0.626965, "hf_subset": "th", "languages": [ "tha-Thai" ], "main_score": 0.645595, "scores_per_experiment": [ { "accuracy": 0.656691, "f1": 0.649174, "f1_weighted": 0.636724 }, { "accuracy": 0.659718, "f1": 0.654997, "f1_weighted": 0.646472 }, { "accuracy": 0.647613, "f1": 0.635881, "f1_weighted": 0.632713 }, { "accuracy": 0.639543, "f1": 0.634121, "f1_weighted": 0.624159 }, { "accuracy": 0.641224, "f1": 0.630624, "f1_weighted": 0.613037 }, { "accuracy": 0.633826, "f1": 0.654096, "f1_weighted": 0.613257 }, { "accuracy": 0.64156, "f1": 0.644671, "f1_weighted": 0.620519 }, { "accuracy": 0.638198, "f1": 0.621017, "f1_weighted": 0.620754 }, { "accuracy": 0.63618, "f1": 0.626666, "f1_weighted": 0.615239 }, { "accuracy": 0.661399, "f1": 0.653079, "f1_weighted": 0.646777 } ], "mteb_version": "1.12.75" }, { "accuracy": 0.531675, "f1": 0.488733, "f1_weighted": 0.526023, "hf_subset": "cy", "languages": [ "cym-Latn" ], "main_score": 0.531675, "scores_per_experiment": [ { "accuracy": 0.536987, "f1": 0.502343, "f1_weighted": 0.535986 }, { "accuracy": 0.549765, "f1": 0.492052, "f1_weighted": 0.551766 }, { "accuracy": 0.542031, "f1": 0.50078, "f1_weighted": 0.539186 }, { "accuracy": 0.54304, "f1": 0.493167, "f1_weighted": 0.540026 }, { "accuracy": 0.525891, "f1": 0.474585, "f1_weighted": 0.512492 }, { "accuracy": 0.510087, "f1": 0.48381, "f1_weighted": 0.508018 }, { "accuracy": 0.534633, "f1": 0.499544, "f1_weighted": 0.528102 }, { "accuracy": 0.531271, "f1": 0.492621, "f1_weighted": 0.520222 }, { "accuracy": 0.507061, "f1": 0.468136, "f1_weighted": 0.491944 }, { "accuracy": 0.535978, "f1": 0.480288, "f1_weighted": 0.532492 } ], "mteb_version": "1.12.75" }, { "accuracy": 0.628547, "f1": 0.596724, "f1_weighted": 0.620804, "hf_subset": "te", "languages": [ "tel-Telu" ], "main_score": 0.628547, "scores_per_experiment": [ { "accuracy": 0.654338, "f1": 0.622328, "f1_weighted": 0.650365 }, { "accuracy": 0.648285, "f1": 0.602517, "f1_weighted": 0.646793 }, { "accuracy": 0.625084, "f1": 0.596457, "f1_weighted": 0.615085 }, { "accuracy": 0.6308, "f1": 0.597281, "f1_weighted": 0.627737 }, { "accuracy": 0.612643, "f1": 0.574805, "f1_weighted": 0.601062 }, { "accuracy": 0.602892, "f1": 0.592408, "f1_weighted": 0.592585 }, { "accuracy": 0.623739, "f1": 0.596817, "f1_weighted": 0.619968 }, { "accuracy": 0.622058, "f1": 0.581366, "f1_weighted": 0.614701 }, { "accuracy": 0.62004, "f1": 0.596311, "f1_weighted": 0.599084 }, { "accuracy": 0.645595, "f1": 0.606945, "f1_weighted": 0.640656 } ], "mteb_version": "1.12.75" }, { "accuracy": 0.552118, "f1": 0.529556, "f1_weighted": 0.537547, "hf_subset": "ka", "languages": [ "kat-Geor" ], "main_score": 0.552118, "scores_per_experiment": [ { "accuracy": 0.568931, "f1": 0.542571, "f1_weighted": 0.556407 }, { "accuracy": 0.575319, "f1": 0.556008, "f1_weighted": 0.565359 }, { "accuracy": 0.541358, "f1": 0.519903, "f1_weighted": 0.527795 }, { "accuracy": 0.547747, "f1": 0.528701, "f1_weighted": 0.535437 }, { "accuracy": 0.530599, "f1": 0.506313, "f1_weighted": 0.50531 }, { "accuracy": 0.523537, "f1": 0.5184, "f1_weighted": 0.510544 }, { "accuracy": 0.568594, "f1": 0.54797, "f1_weighted": 0.555962 }, { "accuracy": 0.548756, "f1": 0.51437, "f1_weighted": 0.540242 }, { "accuracy": 0.56187, "f1": 0.537121, "f1_weighted": 0.540508 }, { "accuracy": 0.554472, "f1": 0.524208, "f1_weighted": 0.537906 } ], "mteb_version": "1.12.75" }, { "accuracy": 0.650941, "f1": 0.631186, "f1_weighted": 0.637987, "hf_subset": "ro", "languages": [ "ron-Latn" ], "main_score": 0.650941, "scores_per_experiment": [ { "accuracy": 0.670814, "f1": 0.648342, "f1_weighted": 0.661602 }, { "accuracy": 0.652993, "f1": 0.627355, "f1_weighted": 0.638641 }, { "accuracy": 0.645931, "f1": 0.631038, "f1_weighted": 0.636965 }, { "accuracy": 0.656355, "f1": 0.63775, "f1_weighted": 0.650338 }, { "accuracy": 0.655683, "f1": 0.626146, "f1_weighted": 0.636751 }, { "accuracy": 0.631473, "f1": 0.618281, "f1_weighted": 0.621476 }, { "accuracy": 0.650639, "f1": 0.630216, "f1_weighted": 0.635261 }, { "accuracy": 0.642569, "f1": 0.620885, "f1_weighted": 0.627907 }, { "accuracy": 0.636516, "f1": 0.636662, "f1_weighted": 0.610224 }, { "accuracy": 0.666443, "f1": 0.635187, "f1_weighted": 0.660705 } ], "mteb_version": "1.12.75" }, { "accuracy": 0.668964, "f1": 0.64997, "f1_weighted": 0.655823, "hf_subset": "id", "languages": [ "ind-Latn" ], "main_score": 0.668964, "scores_per_experiment": [ { "accuracy": 0.67922, "f1": 0.658544, "f1_weighted": 0.666086 }, { "accuracy": 0.69536, "f1": 0.67878, "f1_weighted": 0.688657 }, { "accuracy": 0.655683, "f1": 0.634378, "f1_weighted": 0.643824 }, { "accuracy": 0.696369, "f1": 0.669999, "f1_weighted": 0.68725 }, { "accuracy": 0.64694, "f1": 0.621346, "f1_weighted": 0.627254 }, { "accuracy": 0.642233, "f1": 0.63697, "f1_weighted": 0.629423 }, { "accuracy": 0.662071, "f1": 0.649803, "f1_weighted": 0.64878 }, { "accuracy": 0.668796, "f1": 0.642488, "f1_weighted": 0.655476 }, { "accuracy": 0.660054, "f1": 0.64636, "f1_weighted": 0.636526 }, { "accuracy": 0.682919, "f1": 0.661026, "f1_weighted": 0.674951 } ], "mteb_version": "1.12.75" }, { "accuracy": 0.657431, "f1": 0.623333, "f1_weighted": 0.646472, "hf_subset": "de", "languages": [ "deu-Latn" ], "main_score": 0.657431, "scores_per_experiment": [ { "accuracy": 0.66846, "f1": 0.638298, "f1_weighted": 0.660568 }, { "accuracy": 0.675521, "f1": 0.639754, "f1_weighted": 0.666506 }, { "accuracy": 0.652993, "f1": 0.623166, "f1_weighted": 0.641785 }, { "accuracy": 0.672159, "f1": 0.635817, "f1_weighted": 0.665387 }, { "accuracy": 0.654001, "f1": 0.616035, "f1_weighted": 0.634622 }, { "accuracy": 0.637861, "f1": 0.611362, "f1_weighted": 0.632112 }, { "accuracy": 0.656691, "f1": 0.612894, "f1_weighted": 0.645988 }, { "accuracy": 0.64963, "f1": 0.611881, "f1_weighted": 0.63795 }, { "accuracy": 0.643241, "f1": 0.622198, "f1_weighted": 0.625166 }, { "accuracy": 0.663753, "f1": 0.621929, "f1_weighted": 0.654633 } ], "mteb_version": "1.12.75" }, { "accuracy": 0.649025, "f1": 0.629536, "f1_weighted": 0.638782, "hf_subset": "lv", "languages": [ "lav-Latn" ], "main_score": 0.649025, "scores_per_experiment": [ { "accuracy": 0.669805, "f1": 0.649882, "f1_weighted": 0.662536 }, { "accuracy": 0.681237, "f1": 0.661064, "f1_weighted": 0.677101 }, { "accuracy": 0.641224, "f1": 0.612783, "f1_weighted": 0.630335 }, { "accuracy": 0.662071, "f1": 0.640376, "f1_weighted": 0.657984 }, { "accuracy": 0.638534, "f1": 0.615233, "f1_weighted": 0.624118 }, { "accuracy": 0.611634, "f1": 0.608583, "f1_weighted": 0.600721 }, { "accuracy": 0.659045, "f1": 0.634195, "f1_weighted": 0.646777 }, { "accuracy": 0.626429, "f1": 0.609909, "f1_weighted": 0.613224 }, { "accuracy": 0.631473, "f1": 0.62694, "f1_weighted": 0.61465 }, { "accuracy": 0.668796, "f1": 0.636395, "f1_weighted": 0.660369 } ], "mteb_version": "1.12.75" }, { "accuracy": 0.641594, "f1": 0.633453, "f1_weighted": 0.626059, "hf_subset": "zh-TW", "languages": [ "cmo-Hant" ], "main_score": 0.641594, "scores_per_experiment": [ { "accuracy": 0.671822, "f1": 0.660167, "f1_weighted": 0.665387 }, { "accuracy": 0.66577, "f1": 0.65024, "f1_weighted": 0.653798 }, { "accuracy": 0.645259, "f1": 0.622416, "f1_weighted": 0.633604 }, { "accuracy": 0.624412, "f1": 0.629131, "f1_weighted": 0.608813 }, { "accuracy": 0.639206, "f1": 0.622959, "f1_weighted": 0.617287 }, { "accuracy": 0.617014, "f1": 0.618289, "f1_weighted": 0.596082 }, { "accuracy": 0.634499, "f1": 0.625618, "f1_weighted": 0.611846 }, { "accuracy": 0.64963, "f1": 0.64158, "f1_weighted": 0.639793 }, { "accuracy": 0.625757, "f1": 0.628178, "f1_weighted": 0.603186 }, { "accuracy": 0.642569, "f1": 0.635948, "f1_weighted": 0.630791 } ], "mteb_version": "1.12.75" }, { "accuracy": 0.648016, "f1": 0.610291, "f1_weighted": 0.638888, "hf_subset": "sq", "languages": [ "sqi-Latn" ], "main_score": 0.648016, "scores_per_experiment": [ { "accuracy": 0.662408, "f1": 0.629292, "f1_weighted": 0.653616 }, { "accuracy": 0.677202, "f1": 0.635383, "f1_weighted": 0.674593 }, { "accuracy": 0.637861, "f1": 0.595568, "f1_weighted": 0.631062 }, { "accuracy": 0.6577, "f1": 0.605893, "f1_weighted": 0.652567 }, { "accuracy": 0.634499, "f1": 0.596035, "f1_weighted": 0.617218 }, { "accuracy": 0.621385, "f1": 0.595351, "f1_weighted": 0.615987 }, { "accuracy": 0.647276, "f1": 0.614907, "f1_weighted": 0.637957 }, { "accuracy": 0.640215, "f1": 0.596135, "f1_weighted": 0.631919 }, { "accuracy": 0.640888, "f1": 0.614747, "f1_weighted": 0.62114 }, { "accuracy": 0.660726, "f1": 0.619603, "f1_weighted": 0.65282 } ], "mteb_version": "1.12.75" }, { "accuracy": 0.599966, "f1": 0.569884, "f1_weighted": 0.594092, "hf_subset": "my", "languages": [ "mya-Mymr" ], "main_score": 0.599966, "scores_per_experiment": [ { "accuracy": 0.634499, "f1": 0.603433, "f1_weighted": 0.628546 }, { "accuracy": 0.618023, "f1": 0.581425, "f1_weighted": 0.614705 }, { "accuracy": 0.606927, "f1": 0.571474, "f1_weighted": 0.599508 }, { "accuracy": 0.603564, "f1": 0.566533, "f1_weighted": 0.603801 }, { "accuracy": 0.587761, "f1": 0.548882, "f1_weighted": 0.575565 }, { "accuracy": 0.588097, "f1": 0.571133, "f1_weighted": 0.586238 }, { "accuracy": 0.61466, "f1": 0.584406, "f1_weighted": 0.606704 }, { "accuracy": 0.588097, "f1": 0.557362, "f1_weighted": 0.582239 }, { "accuracy": 0.588769, "f1": 0.568438, "f1_weighted": 0.581334 }, { "accuracy": 0.569267, "f1": 0.545756, "f1_weighted": 0.562278 } ], "mteb_version": "1.12.75" }, { "accuracy": 0.670545, "f1": 0.650862, "f1_weighted": 0.661068, "hf_subset": "it", "languages": [ "ita-Latn" ], "main_score": 0.670545, "scores_per_experiment": [ { "accuracy": 0.676866, "f1": 0.653031, "f1_weighted": 0.670816 }, { "accuracy": 0.684936, "f1": 0.671244, "f1_weighted": 0.676489 }, { "accuracy": 0.668124, "f1": 0.641298, "f1_weighted": 0.65979 }, { "accuracy": 0.665434, "f1": 0.643615, "f1_weighted": 0.661903 }, { "accuracy": 0.672831, "f1": 0.648429, "f1_weighted": 0.657742 }, { "accuracy": 0.657364, "f1": 0.645432, "f1_weighted": 0.6476 }, { "accuracy": 0.68998, "f1": 0.677383, "f1_weighted": 0.681373 }, { "accuracy": 0.643914, "f1": 0.624733, "f1_weighted": 0.628122 }, { "accuracy": 0.658036, "f1": 0.645678, "f1_weighted": 0.646746 }, { "accuracy": 0.687962, "f1": 0.657778, "f1_weighted": 0.680096 } ], "mteb_version": "1.12.75" }, { "accuracy": 0.65158, "f1": 0.622396, "f1_weighted": 0.644508, "hf_subset": "he", "languages": [ "heb-Hebr" ], "main_score": 0.65158, "scores_per_experiment": [ { "accuracy": 0.653329, "f1": 0.628351, "f1_weighted": 0.647454 }, { "accuracy": 0.680229, "f1": 0.647065, "f1_weighted": 0.675313 }, { "accuracy": 0.646268, "f1": 0.617769, "f1_weighted": 0.638223 }, { "accuracy": 0.671822, "f1": 0.634064, "f1_weighted": 0.66864 }, { "accuracy": 0.628447, "f1": 0.597321, "f1_weighted": 0.614573 }, { "accuracy": 0.639879, "f1": 0.608288, "f1_weighted": 0.638073 }, { "accuracy": 0.653665, "f1": 0.622644, "f1_weighted": 0.648555 }, { "accuracy": 0.626093, "f1": 0.60344, "f1_weighted": 0.616055 }, { "accuracy": 0.634499, "f1": 0.613944, "f1_weighted": 0.621851 }, { "accuracy": 0.681574, "f1": 0.651079, "f1_weighted": 0.67634 } ], "mteb_version": "1.12.75" }, { "accuracy": 0.510558, "f1": 0.484333, "f1_weighted": 0.504191, "hf_subset": "km", "languages": [ "khm-Khmr" ], "main_score": 0.510558, "scores_per_experiment": [ { "accuracy": 0.541695, "f1": 0.515648, "f1_weighted": 0.538359 }, { "accuracy": 0.542367, "f1": 0.499263, "f1_weighted": 0.541769 }, { "accuracy": 0.502354, "f1": 0.47694, "f1_weighted": 0.49238 }, { "accuracy": 0.51076, "f1": 0.483244, "f1_weighted": 0.504812 }, { "accuracy": 0.52152, "f1": 0.495406, "f1_weighted": 0.51333 }, { "accuracy": 0.472764, "f1": 0.469684, "f1_weighted": 0.467417 }, { "accuracy": 0.493275, "f1": 0.474805, "f1_weighted": 0.482577 }, { "accuracy": 0.512777, "f1": 0.477865, "f1_weighted": 0.516145 }, { "accuracy": 0.495965, "f1": 0.466113, "f1_weighted": 0.484899 }, { "accuracy": 0.512105, "f1": 0.484368, "f1_weighted": 0.500223 } ], "mteb_version": "1.12.75" }, { "accuracy": 0.653497, "f1": 0.633045, "f1_weighted": 0.644174, "hf_subset": "ms", "languages": [ "msa-Latn" ], "main_score": 0.653497, "scores_per_experiment": [ { "accuracy": 0.689307, "f1": 0.661725, "f1_weighted": 0.684268 }, { "accuracy": 0.684936, "f1": 0.66667, "f1_weighted": 0.679691 }, { "accuracy": 0.639206, "f1": 0.620429, "f1_weighted": 0.631373 }, { "accuracy": 0.654338, "f1": 0.633286, "f1_weighted": 0.649778 }, { "accuracy": 0.64425, "f1": 0.611612, "f1_weighted": 0.628918 }, { "accuracy": 0.612643, "f1": 0.610535, "f1_weighted": 0.603794 }, { "accuracy": 0.650975, "f1": 0.636021, "f1_weighted": 0.640245 }, { "accuracy": 0.642233, "f1": 0.619158, "f1_weighted": 0.63018 }, { "accuracy": 0.643578, "f1": 0.625149, "f1_weighted": 0.624348 }, { "accuracy": 0.673504, "f1": 0.645867, "f1_weighted": 0.669146 } ], "mteb_version": "1.12.75" }, { "accuracy": 0.650269, "f1": 0.629996, "f1_weighted": 0.638038, "hf_subset": "hu", "languages": [ "hun-Latn" ], "main_score": 0.650269, "scores_per_experiment": [ { "accuracy": 0.664425, "f1": 0.636791, "f1_weighted": 0.652621 }, { "accuracy": 0.672495, "f1": 0.647705, "f1_weighted": 0.664108 }, { "accuracy": 0.642569, "f1": 0.636504, "f1_weighted": 0.631294 }, { "accuracy": 0.658373, "f1": 0.633277, "f1_weighted": 0.646184 }, { "accuracy": 0.641896, "f1": 0.610749, "f1_weighted": 0.625655 }, { "accuracy": 0.638198, "f1": 0.622958, "f1_weighted": 0.628825 }, { "accuracy": 0.643241, "f1": 0.633646, "f1_weighted": 0.632231 }, { "accuracy": 0.64156, "f1": 0.610797, "f1_weighted": 0.629962 }, { "accuracy": 0.630128, "f1": 0.624284, "f1_weighted": 0.607589 }, { "accuracy": 0.669805, "f1": 0.643244, "f1_weighted": 0.661914 } ], "mteb_version": "1.12.75" }, { "accuracy": 0.616375, "f1": 0.585028, "f1_weighted": 0.605117, "hf_subset": "is", "languages": [ "isl-Latn" ], "main_score": 0.616375, "scores_per_experiment": [ { "accuracy": 0.634163, "f1": 0.605178, "f1_weighted": 0.627488 }, { "accuracy": 0.650303, "f1": 0.616546, "f1_weighted": 0.641398 }, { "accuracy": 0.600538, "f1": 0.573494, "f1_weighted": 0.588278 }, { "accuracy": 0.637861, "f1": 0.59799, "f1_weighted": 0.633002 }, { "accuracy": 0.616678, "f1": 0.567186, "f1_weighted": 0.602613 }, { "accuracy": 0.574983, "f1": 0.56332, "f1_weighted": 0.564765 }, { "accuracy": 0.625757, "f1": 0.602732, "f1_weighted": 0.616403 }, { "accuracy": 0.595158, "f1": 0.55516, "f1_weighted": 0.575219 }, { "accuracy": 0.604237, "f1": 0.576007, "f1_weighted": 0.584142 }, { "accuracy": 0.624075, "f1": 0.592665, "f1_weighted": 0.61786 } ], "mteb_version": "1.12.75" }, { "accuracy": 0.616644, "f1": 0.585414, "f1_weighted": 0.604774, "hf_subset": "kn", "languages": [ "kan-Knda" ], "main_score": 0.616644, "scores_per_experiment": [ { "accuracy": 0.649966, "f1": 0.616931, "f1_weighted": 0.647001 }, { "accuracy": 0.643241, "f1": 0.600321, "f1_weighted": 0.637783 }, { "accuracy": 0.605582, "f1": 0.574223, "f1_weighted": 0.591275 }, { "accuracy": 0.626429, "f1": 0.596514, "f1_weighted": 0.621022 }, { "accuracy": 0.598857, "f1": 0.564931, "f1_weighted": 0.571633 }, { "accuracy": 0.585407, "f1": 0.567138, "f1_weighted": 0.573028 }, { "accuracy": 0.611634, "f1": 0.584964, "f1_weighted": 0.602161 }, { "accuracy": 0.607599, "f1": 0.56517, "f1_weighted": 0.597168 }, { "accuracy": 0.6039, "f1": 0.586607, "f1_weighted": 0.581508 }, { "accuracy": 0.633826, "f1": 0.597339, "f1_weighted": 0.625163 } ], "mteb_version": "1.12.75" }, { "accuracy": 0.646738, "f1": 0.617287, "f1_weighted": 0.638061, "hf_subset": "ml", "languages": [ "mal-Mlym" ], "main_score": 0.646738, "scores_per_experiment": [ { "accuracy": 0.682919, "f1": 0.651181, "f1_weighted": 0.67869 }, { "accuracy": 0.656019, "f1": 0.624346, "f1_weighted": 0.653061 }, { "accuracy": 0.635171, "f1": 0.603951, "f1_weighted": 0.627691 }, { "accuracy": 0.6577, "f1": 0.621776, "f1_weighted": 0.65165 }, { "accuracy": 0.654001, "f1": 0.621771, "f1_weighted": 0.639079 }, { "accuracy": 0.602555, "f1": 0.594228, "f1_weighted": 0.5949 }, { "accuracy": 0.648621, "f1": 0.613406, "f1_weighted": 0.639839 }, { "accuracy": 0.63618, "f1": 0.592616, "f1_weighted": 0.628501 }, { "accuracy": 0.623403, "f1": 0.614677, "f1_weighted": 0.60507 }, { "accuracy": 0.670814, "f1": 0.634919, "f1_weighted": 0.662133 } ], "mteb_version": "1.12.75" }, { "accuracy": 0.603968, "f1": 0.574021, "f1_weighted": 0.595161, "hf_subset": "ta", "languages": [ "tam-Taml" ], "main_score": 0.603968, "scores_per_experiment": [ { "accuracy": 0.625757, "f1": 0.59197, "f1_weighted": 0.620602 }, { "accuracy": 0.609953, "f1": 0.578858, "f1_weighted": 0.604136 }, { "accuracy": 0.596167, "f1": 0.562906, "f1_weighted": 0.59235 }, { "accuracy": 0.60121, "f1": 0.57576, "f1_weighted": 0.595579 }, { "accuracy": 0.594822, "f1": 0.554734, "f1_weighted": 0.576275 }, { "accuracy": 0.593141, "f1": 0.575081, "f1_weighted": 0.589209 }, { "accuracy": 0.61197, "f1": 0.588776, "f1_weighted": 0.603097 }, { "accuracy": 0.587088, "f1": 0.554513, "f1_weighted": 0.572886 }, { "accuracy": 0.605582, "f1": 0.577245, "f1_weighted": 0.592427 }, { "accuracy": 0.613988, "f1": 0.580369, "f1_weighted": 0.605047 } ], "mteb_version": "1.12.75" }, { "accuracy": 0.631675, "f1": 0.606502, "f1_weighted": 0.62386, "hf_subset": "ur", "languages": [ "urd-Arab" ], "main_score": 0.631675, "scores_per_experiment": [ { "accuracy": 0.649966, "f1": 0.619613, "f1_weighted": 0.645891 }, { "accuracy": 0.655683, "f1": 0.619472, "f1_weighted": 0.649656 }, { "accuracy": 0.622394, "f1": 0.593919, "f1_weighted": 0.614339 }, { "accuracy": 0.6577, "f1": 0.631294, "f1_weighted": 0.65638 }, { "accuracy": 0.608272, "f1": 0.584768, "f1_weighted": 0.593059 }, { "accuracy": 0.606254, "f1": 0.599696, "f1_weighted": 0.595612 }, { "accuracy": 0.631137, "f1": 0.617103, "f1_weighted": 0.626938 }, { "accuracy": 0.605245, "f1": 0.585213, "f1_weighted": 0.596006 }, { "accuracy": 0.635844, "f1": 0.607717, "f1_weighted": 0.622562 }, { "accuracy": 0.64425, "f1": 0.606228, "f1_weighted": 0.638156 } ], "mteb_version": "1.12.75" }, { "scores_per_experiment": [ { "accuracy": 0.68191, "f1": 0.661943, "f1_weighted": 0.675563, "precision": 0.642134, "precision_weighted": 0.734417, "recall": 0.757933, "recall_weighted": 0.68191, "ap": null, "ap_weighted": null }, { "accuracy": 0.697041, "f1": 0.669649, "f1_weighted": 0.692823, "precision": 0.64806, "precision_weighted": 0.744601, "recall": 0.758623, "recall_weighted": 0.697041, "ap": null, "ap_weighted": null }, { "accuracy": 0.670814, "f1": 0.650411, "f1_weighted": 0.663627, "precision": 0.629856, "precision_weighted": 0.735625, "recall": 0.751259, "recall_weighted": 0.670814, "ap": null, "ap_weighted": null }, { "accuracy": 0.695696, "f1": 0.664606, "f1_weighted": 0.688559, "precision": 0.646166, "precision_weighted": 0.752951, "recall": 0.749182, "recall_weighted": 0.695696, "ap": null, "ap_weighted": null }, { "accuracy": 0.676194, "f1": 0.6465, "f1_weighted": 0.662177, "precision": 0.628329, "precision_weighted": 0.73128, "recall": 0.76571, "recall_weighted": 0.676194, "ap": null, "ap_weighted": null }, { "accuracy": 0.652993, "f1": 0.640369, "f1_weighted": 0.642213, "precision": 0.628997, "precision_weighted": 0.740142, "recall": 0.746757, "recall_weighted": 0.652993, "ap": null, "ap_weighted": null }, { "accuracy": 0.678547, "f1": 0.660756, "f1_weighted": 0.672051, "precision": 0.653976, "precision_weighted": 0.747442, "recall": 0.770139, "recall_weighted": 0.678547, "ap": null, "ap_weighted": null }, { "accuracy": 0.655346, "f1": 0.633002, "f1_weighted": 0.644149, "precision": 0.614528, "precision_weighted": 0.712617, "recall": 0.733816, "recall_weighted": 0.655346, "ap": null, "ap_weighted": null }, { "accuracy": 0.659381, "f1": 0.656156, "f1_weighted": 0.645844, "precision": 0.651161, "precision_weighted": 0.738885, "recall": 0.766038, "recall_weighted": 0.659381, "ap": null, "ap_weighted": null }, { "accuracy": 0.694351, "f1": 0.673545, "f1_weighted": 0.690227, "precision": 0.658424, "precision_weighted": 0.757977, "recall": 0.781044, "recall_weighted": 0.694351, "ap": null, "ap_weighted": null } ], "accuracy": 0.676227, "f1": 0.655694, "f1_weighted": 0.667723, "precision": 0.640163, "precision_weighted": 0.739594, "recall": 0.75805, "recall_weighted": 0.676227, "ap": NaN, "ap_weighted": NaN, "main_score": 0.676227, "hf_subset": "pt", "languages": [ "por-Latn" ], "mteb_version": "2.18.0" }, { "accuracy": 0.670377, "f1": 0.652941, "f1_weighted": 0.661172, "hf_subset": "es", "languages": [ "spa-Latn" ], "main_score": 0.670377, "scores_per_experiment": [ { "accuracy": 0.685945, "f1": 0.664735, "f1_weighted": 0.680918 }, { "accuracy": 0.69536, "f1": 0.673013, "f1_weighted": 0.686313 }, { "accuracy": 0.662408, "f1": 0.650796, "f1_weighted": 0.652038 }, { "accuracy": 0.678547, "f1": 0.65257, "f1_weighted": 0.669958 }, { "accuracy": 0.669805, "f1": 0.647459, "f1_weighted": 0.657061 }, { "accuracy": 0.651984, "f1": 0.641127, "f1_weighted": 0.646849 }, { "accuracy": 0.667451, "f1": 0.650144, "f1_weighted": 0.658129 }, { "accuracy": 0.653329, "f1": 0.636203, "f1_weighted": 0.644706 }, { "accuracy": 0.658373, "f1": 0.65615, "f1_weighted": 0.639373 }, { "accuracy": 0.680565, "f1": 0.657211, "f1_weighted": 0.67638 } ], "mteb_version": "1.12.75" } ], "validation": [ { "accuracy": 0.653123, "f1": 0.612472, "f1_weighted": 0.639092, "hf_subset": "sl", "languages": [ "slv-Latn" ], "main_score": 0.653123, "scores_per_experiment": [ { "accuracy": 0.659124, "f1": 0.622499, "f1_weighted": 0.645259 }, { "accuracy": 0.674373, "f1": 0.634303, "f1_weighted": 0.663734 }, { "accuracy": 0.633546, "f1": 0.600306, "f1_weighted": 0.624301 }, { "accuracy": 0.660108, "f1": 0.602425, "f1_weighted": 0.646112 }, { "accuracy": 0.665027, "f1": 0.619334, "f1_weighted": 0.648942 }, { "accuracy": 0.648795, "f1": 0.622483, "f1_weighted": 0.636891 }, { "accuracy": 0.656665, "f1": 0.615339, "f1_weighted": 0.642729 }, { "accuracy": 0.61879, "f1": 0.582366, "f1_weighted": 0.604731 }, { "accuracy": 0.631087, "f1": 0.595147, "f1_weighted": 0.601107 }, { "accuracy": 0.683719, "f1": 0.630516, "f1_weighted": 0.677111 } ], "mteb_version": "1.12.75" }, { "accuracy": 0.674127, "f1": 0.628237, "f1_weighted": 0.6603, "hf_subset": "ko", "languages": [ "kor-Kore" ], "main_score": 0.674127, "scores_per_experiment": [ { "accuracy": 0.688637, "f1": 0.650593, "f1_weighted": 0.675878 }, { "accuracy": 0.691589, "f1": 0.628998, "f1_weighted": 0.685016 }, { "accuracy": 0.675848, "f1": 0.621202, "f1_weighted": 0.663594 }, { "accuracy": 0.68667, "f1": 0.638692, "f1_weighted": 0.673365 }, { "accuracy": 0.669454, "f1": 0.621727, "f1_weighted": 0.652446 }, { "accuracy": 0.639941, "f1": 0.617139, "f1_weighted": 0.615958 }, { "accuracy": 0.667978, "f1": 0.617018, "f1_weighted": 0.650571 }, { "accuracy": 0.655681, "f1": 0.607533, "f1_weighted": 0.647242 }, { "accuracy": 0.673881, "f1": 0.64, "f1_weighted": 0.654312 }, { "accuracy": 0.691589, "f1": 0.639473, "f1_weighted": 0.684621 } ], "mteb_version": "1.12.75" }, { "accuracy": 0.60487, "f1": 0.563608, "f1_weighted": 0.596159, "hf_subset": "tl", "languages": [ "tgl-Latn" ], "main_score": 0.60487, "scores_per_experiment": [ { "accuracy": 0.594196, "f1": 0.555488, "f1_weighted": 0.587247 }, { "accuracy": 0.632071, "f1": 0.579329, "f1_weighted": 0.628961 }, { "accuracy": 0.614855, "f1": 0.570957, "f1_weighted": 0.609227 }, { "accuracy": 0.620758, "f1": 0.562213, "f1_weighted": 0.617124 }, { "accuracy": 0.611412, "f1": 0.561432, "f1_weighted": 0.602382 }, { "accuracy": 0.580915, "f1": 0.552662, "f1_weighted": 0.576177 }, { "accuracy": 0.587801, "f1": 0.556649, "f1_weighted": 0.573268 }, { "accuracy": 0.600098, "f1": 0.555464, "f1_weighted": 0.584057 }, { "accuracy": 0.596655, "f1": 0.572507, "f1_weighted": 0.587046 }, { "accuracy": 0.609936, "f1": 0.569376, "f1_weighted": 0.596098 } ], "mteb_version": "1.12.75" }, { "accuracy": 0.694147, "f1": 0.657565, "f1_weighted": 0.68031, "hf_subset": "sv", "languages": [ "swe-Latn" ], "main_score": 0.694147, "scores_per_experiment": [ { "accuracy": 0.710772, "f1": 0.68128, "f1_weighted": 0.702213 }, { "accuracy": 0.718642, "f1": 0.675898, "f1_weighted": 0.708575 }, { "accuracy": 0.692081, "f1": 0.640996, "f1_weighted": 0.682608 }, { "accuracy": 0.71028, "f1": 0.666737, "f1_weighted": 0.698521 }, { "accuracy": 0.697491, "f1": 0.665144, "f1_weighted": 0.67744 }, { "accuracy": 0.677324, "f1": 0.652497, "f1_weighted": 0.664736 }, { "accuracy": 0.687654, "f1": 0.651319, "f1_weighted": 0.673488 }, { "accuracy": 0.670438, "f1": 0.635546, "f1_weighted": 0.654865 }, { "accuracy": 0.6606, "f1": 0.642514, "f1_weighted": 0.632714 }, { "accuracy": 0.716183, "f1": 0.663718, "f1_weighted": 0.707942 } ], "mteb_version": "1.12.75" }, { "accuracy": 0.679734, "f1": 0.631997, "f1_weighted": 0.669321, "hf_subset": "pl", "languages": [ "pol-Latn" ], "main_score": 0.679734, "scores_per_experiment": [ { "accuracy": 0.689129, "f1": 0.641472, "f1_weighted": 0.682007 }, { "accuracy": 0.696508, "f1": 0.642751, "f1_weighted": 0.688486 }, { "accuracy": 0.692081, "f1": 0.635204, "f1_weighted": 0.682338 }, { "accuracy": 0.689621, "f1": 0.626081, "f1_weighted": 0.682572 }, { "accuracy": 0.682243, "f1": 0.631209, "f1_weighted": 0.664361 }, { "accuracy": 0.67634, "f1": 0.631446, "f1_weighted": 0.663939 }, { "accuracy": 0.67093, "f1": 0.626991, "f1_weighted": 0.662129 }, { "accuracy": 0.654206, "f1": 0.621399, "f1_weighted": 0.64317 }, { "accuracy": 0.657157, "f1": 0.624917, "f1_weighted": 0.641063 }, { "accuracy": 0.689129, "f1": 0.638499, "f1_weighted": 0.683149 } ], "mteb_version": "1.12.75" }, { "accuracy": 0.684309, "f1": 0.642544, "f1_weighted": 0.672439, "hf_subset": "ru", "languages": [ "rus-Cyrl" ], "main_score": 0.684309, "scores_per_experiment": [ { "accuracy": 0.694048, "f1": 0.657343, "f1_weighted": 0.688534 }, { "accuracy": 0.708805, "f1": 0.654568, "f1_weighted": 0.701519 }, { "accuracy": 0.703886, "f1": 0.65685, "f1_weighted": 0.693359 }, { "accuracy": 0.706837, "f1": 0.652876, "f1_weighted": 0.698797 }, { "accuracy": 0.677816, "f1": 0.627656, "f1_weighted": 0.660592 }, { "accuracy": 0.656665, "f1": 0.623604, "f1_weighted": 0.643261 }, { "accuracy": 0.667486, "f1": 0.634653, "f1_weighted": 0.649873 }, { "accuracy": 0.661092, "f1": 0.626348, "f1_weighted": 0.649551 }, { "accuracy": 0.657649, "f1": 0.632461, "f1_weighted": 0.635279 }, { "accuracy": 0.708805, "f1": 0.659082, "f1_weighted": 0.703626 } ], "mteb_version": "1.12.75" }, { "scores_per_experiment": [ { "accuracy": 0.715691, "f1": 0.679712, "f1_weighted": 0.707159, "precision": 0.660692, "precision_weighted": 0.75928, "recall": 0.773231, "recall_weighted": 0.715691, "ap": null, "ap_weighted": null }, { "accuracy": 0.736842, "f1": 0.691426, "f1_weighted": 0.726532, "precision": 0.671716, "precision_weighted": 0.757377, "recall": 0.778139, "recall_weighted": 0.736842, "ap": null, "ap_weighted": null }, { "accuracy": 0.719626, "f1": 0.66177, "f1_weighted": 0.706892, "precision": 0.645298, "precision_weighted": 0.771807, "recall": 0.763245, "recall_weighted": 0.719626, "ap": null, "ap_weighted": null }, { "accuracy": 0.723069, "f1": 0.670832, "f1_weighted": 0.711448, "precision": 0.653903, "precision_weighted": 0.775495, "recall": 0.772168, "recall_weighted": 0.723069, "ap": null, "ap_weighted": null }, { "accuracy": 0.71028, "f1": 0.679761, "f1_weighted": 0.692249, "precision": 0.652322, "precision_weighted": 0.715787, "recall": 0.779727, "recall_weighted": 0.71028, "ap": null, "ap_weighted": null }, { "accuracy": 0.713232, "f1": 0.68881, "f1_weighted": 0.700173, "precision": 0.668355, "precision_weighted": 0.751555, "recall": 0.789512, "recall_weighted": 0.713232, "ap": null, "ap_weighted": null }, { "accuracy": 0.698967, "f1": 0.684091, "f1_weighted": 0.683175, "precision": 0.666419, "precision_weighted": 0.732414, "recall": 0.785572, "recall_weighted": 0.698967, "ap": null, "ap_weighted": null }, { "accuracy": 0.693556, "f1": 0.650114, "f1_weighted": 0.681325, "precision": 0.643995, "precision_weighted": 0.749115, "recall": 0.745405, "recall_weighted": 0.693556, "ap": null, "ap_weighted": null }, { "accuracy": 0.679784, "f1": 0.654319, "f1_weighted": 0.65551, "precision": 0.650492, "precision_weighted": 0.747123, "recall": 0.761364, "recall_weighted": 0.679784, "ap": null, "ap_weighted": null }, { "accuracy": 0.740777, "f1": 0.696154, "f1_weighted": 0.731929, "precision": 0.676289, "precision_weighted": 0.77759, "recall": 0.784205, "recall_weighted": 0.740777, "ap": null, "ap_weighted": null } ], "accuracy": 0.713182, "f1": 0.675699, "f1_weighted": 0.699639, "precision": 0.658948, "precision_weighted": 0.753754, "recall": 0.773257, "recall_weighted": 0.713182, "ap": NaN, "ap_weighted": NaN, "main_score": 0.713182, "hf_subset": "en", "languages": [ "eng-Latn" ], "mteb_version": "2.18.0" }, { "scores_per_experiment": [ { "accuracy": 0.651746, "f1": 0.596072, "f1_weighted": 0.647232, "precision": 0.584751, "precision_weighted": 0.703265, "recall": 0.689412, "recall_weighted": 0.651746, "ap": null, "ap_weighted": null }, { "accuracy": 0.6606, "f1": 0.598533, "f1_weighted": 0.655217, "precision": 0.585067, "precision_weighted": 0.695282, "recall": 0.68663, "recall_weighted": 0.6606, "ap": null, "ap_weighted": null }, { "accuracy": 0.64486, "f1": 0.595194, "f1_weighted": 0.63864, "precision": 0.584166, "precision_weighted": 0.693644, "recall": 0.69122, "recall_weighted": 0.64486, "ap": null, "ap_weighted": null }, { "accuracy": 0.653222, "f1": 0.597967, "f1_weighted": 0.64647, "precision": 0.590997, "precision_weighted": 0.694648, "recall": 0.687473, "recall_weighted": 0.653222, "ap": null, "ap_weighted": null }, { "accuracy": 0.639449, "f1": 0.577378, "f1_weighted": 0.627171, "precision": 0.565781, "precision_weighted": 0.661771, "recall": 0.672067, "recall_weighted": 0.639449, "ap": null, "ap_weighted": null }, { "accuracy": 0.636498, "f1": 0.600569, "f1_weighted": 0.627454, "precision": 0.586249, "precision_weighted": 0.683308, "recall": 0.698147, "recall_weighted": 0.636498, "ap": null, "ap_weighted": null }, { "accuracy": 0.611904, "f1": 0.573418, "f1_weighted": 0.602643, "precision": 0.555789, "precision_weighted": 0.644585, "recall": 0.675417, "recall_weighted": 0.611904, "ap": null, "ap_weighted": null }, { "accuracy": 0.627644, "f1": 0.566373, "f1_weighted": 0.62132, "precision": 0.562326, "precision_weighted": 0.685199, "recall": 0.65714, "recall_weighted": 0.627644, "ap": null, "ap_weighted": null }, { "accuracy": 0.607477, "f1": 0.573867, "f1_weighted": 0.587704, "precision": 0.577385, "precision_weighted": 0.666897, "recall": 0.679212, "recall_weighted": 0.607477, "ap": null, "ap_weighted": null }, { "accuracy": 0.648795, "f1": 0.577581, "f1_weighted": 0.643454, "precision": 0.573213, "precision_weighted": 0.693759, "recall": 0.667408, "recall_weighted": 0.648795, "ap": null, "ap_weighted": null } ], "accuracy": 0.638219, "f1": 0.585695, "f1_weighted": 0.629731, "precision": 0.576572, "precision_weighted": 0.682236, "recall": 0.680412, "recall_weighted": 0.638219, "ap": NaN, "ap_weighted": NaN, "main_score": 0.638219, "hf_subset": "af", "languages": [ "afr-Latn" ], "mteb_version": "2.18.0" }, { "accuracy": 0.642007, "f1": 0.60165, "f1_weighted": 0.631278, "hf_subset": "az", "languages": [ "aze-Latn" ], "main_score": 0.642007, "scores_per_experiment": [ { "accuracy": 0.655681, "f1": 0.61054, "f1_weighted": 0.645705 }, { "accuracy": 0.654697, "f1": 0.605517, "f1_weighted": 0.645272 }, { "accuracy": 0.637973, "f1": 0.588672, "f1_weighted": 0.6325 }, { "accuracy": 0.664043, "f1": 0.613792, "f1_weighted": 0.653121 }, { "accuracy": 0.651746, "f1": 0.608989, "f1_weighted": 0.634263 }, { "accuracy": 0.620266, "f1": 0.598752, "f1_weighted": 0.611477 }, { "accuracy": 0.639449, "f1": 0.603412, "f1_weighted": 0.631068 }, { "accuracy": 0.620266, "f1": 0.585262, "f1_weighted": 0.607904 }, { "accuracy": 0.623709, "f1": 0.605055, "f1_weighted": 0.605904 }, { "accuracy": 0.652238, "f1": 0.596509, "f1_weighted": 0.645571 } ], "mteb_version": "1.12.75" }, { "accuracy": 0.674963, "f1": 0.626629, "f1_weighted": 0.663585, "hf_subset": "fr", "languages": [ "fra-Latn" ], "main_score": 0.674963, "scores_per_experiment": [ { "accuracy": 0.673881, "f1": 0.624765, "f1_weighted": 0.665993 }, { "accuracy": 0.689621, "f1": 0.646362, "f1_weighted": 0.683895 }, { "accuracy": 0.6788, "f1": 0.626482, "f1_weighted": 0.667621 }, { "accuracy": 0.689129, "f1": 0.627519, "f1_weighted": 0.67845 }, { "accuracy": 0.680767, "f1": 0.623659, "f1_weighted": 0.663183 }, { "accuracy": 0.653222, "f1": 0.60863, "f1_weighted": 0.641784 }, { "accuracy": 0.671422, "f1": 0.632181, "f1_weighted": 0.661265 }, { "accuracy": 0.6606, "f1": 0.610217, "f1_weighted": 0.652998 }, { "accuracy": 0.65273, "f1": 0.623797, "f1_weighted": 0.632806 }, { "accuracy": 0.699459, "f1": 0.642679, "f1_weighted": 0.687857 } ], "mteb_version": "1.12.75" }, { "accuracy": 0.633546, "f1": 0.588167, "f1_weighted": 0.623748, "hf_subset": "bn", "languages": [ "ben-Beng" ], "main_score": 0.633546, "scores_per_experiment": [ { "accuracy": 0.645844, "f1": 0.600274, "f1_weighted": 0.637574 }, { "accuracy": 0.649287, "f1": 0.592383, "f1_weighted": 0.6451 }, { "accuracy": 0.628628, "f1": 0.583814, "f1_weighted": 0.618693 }, { "accuracy": 0.633546, "f1": 0.587725, "f1_weighted": 0.626687 }, { "accuracy": 0.635514, "f1": 0.594174, "f1_weighted": 0.621141 }, { "accuracy": 0.617806, "f1": 0.589361, "f1_weighted": 0.604229 }, { "accuracy": 0.63453, "f1": 0.585822, "f1_weighted": 0.622866 }, { "accuracy": 0.606985, "f1": 0.559054, "f1_weighted": 0.601488 }, { "accuracy": 0.631579, "f1": 0.594459, "f1_weighted": 0.614462 }, { "accuracy": 0.651746, "f1": 0.594606, "f1_weighted": 0.645238 } ], "mteb_version": "1.12.75" }, { "accuracy": 0.676636, "f1": 0.634369, "f1_weighted": 0.664073, "hf_subset": "vi", "languages": [ "vie-Latn" ], "main_score": 0.676636, "scores_per_experiment": [ { "accuracy": 0.700443, "f1": 0.653582, "f1_weighted": 0.694473 }, { "accuracy": 0.693064, "f1": 0.64575, "f1_weighted": 0.686064 }, { "accuracy": 0.671422, "f1": 0.618897, "f1_weighted": 0.659089 }, { "accuracy": 0.700443, "f1": 0.64518, "f1_weighted": 0.691799 }, { "accuracy": 0.677324, "f1": 0.629191, "f1_weighted": 0.662589 }, { "accuracy": 0.672405, "f1": 0.64235, "f1_weighted": 0.654213 }, { "accuracy": 0.671422, "f1": 0.635984, "f1_weighted": 0.653657 }, { "accuracy": 0.643384, "f1": 0.600364, "f1_weighted": 0.633564 }, { "accuracy": 0.662568, "f1": 0.636579, "f1_weighted": 0.643756 }, { "accuracy": 0.673881, "f1": 0.635817, "f1_weighted": 0.661528 } ], "mteb_version": "1.12.75" }, { "scores_per_experiment": [ { "accuracy": 0.565667, "f1": 0.530829, "f1_weighted": 0.567065, "precision": 0.524535, "precision_weighted": 0.635668, "recall": 0.63464, "recall_weighted": 0.565667, "ap": null, "ap_weighted": null }, { "accuracy": 0.587309, "f1": 0.525588, "f1_weighted": 0.586111, "precision": 0.530123, "precision_weighted": 0.642554, "recall": 0.612316, "recall_weighted": 0.587309, "ap": null, "ap_weighted": null }, { "accuracy": 0.562715, "f1": 0.50786, "f1_weighted": 0.555613, "precision": 0.502907, "precision_weighted": 0.612742, "recall": 0.603829, "recall_weighted": 0.562715, "ap": null, "ap_weighted": null }, { "accuracy": 0.576488, "f1": 0.506239, "f1_weighted": 0.575911, "precision": 0.495217, "precision_weighted": 0.627874, "recall": 0.604333, "recall_weighted": 0.576488, "ap": null, "ap_weighted": null }, { "accuracy": 0.555829, "f1": 0.501892, "f1_weighted": 0.55259, "precision": 0.501346, "precision_weighted": 0.623041, "recall": 0.599082, "recall_weighted": 0.555829, "ap": null, "ap_weighted": null }, { "accuracy": 0.543532, "f1": 0.506449, "f1_weighted": 0.530741, "precision": 0.519327, "precision_weighted": 0.633534, "recall": 0.607445, "recall_weighted": 0.543532, "ap": null, "ap_weighted": null }, { "accuracy": 0.562715, "f1": 0.515612, "f1_weighted": 0.556439, "precision": 0.510413, "precision_weighted": 0.611107, "recall": 0.607878, "recall_weighted": 0.562715, "ap": null, "ap_weighted": null }, { "accuracy": 0.532218, "f1": 0.473399, "f1_weighted": 0.516775, "precision": 0.469026, "precision_weighted": 0.580947, "recall": 0.579061, "recall_weighted": 0.532218, "ap": null, "ap_weighted": null }, { "accuracy": 0.549926, "f1": 0.511832, "f1_weighted": 0.546984, "precision": 0.530625, "precision_weighted": 0.639163, "recall": 0.611926, "recall_weighted": 0.549926, "ap": null, "ap_weighted": null }, { "accuracy": 0.57698, "f1": 0.521096, "f1_weighted": 0.57454, "precision": 0.522233, "precision_weighted": 0.633459, "recall": 0.604369, "recall_weighted": 0.57698, "ap": null, "ap_weighted": null } ], "accuracy": 0.561338, "f1": 0.51008, "f1_weighted": 0.556277, "precision": 0.510575, "precision_weighted": 0.624009, "recall": 0.606488, "recall_weighted": 0.561338, "ap": NaN, "ap_weighted": NaN, "main_score": 0.561338, "hf_subset": "am", "languages": [ "amh-Ethi" ], "mteb_version": "2.18.0" }, { "accuracy": 0.674029, "f1": 0.626521, "f1_weighted": 0.663963, "hf_subset": "fi", "languages": [ "fin-Latn" ], "main_score": 0.674029, "scores_per_experiment": [ { "accuracy": 0.697983, "f1": 0.650018, "f1_weighted": 0.693585 }, { "accuracy": 0.676832, "f1": 0.617097, "f1_weighted": 0.67015 }, { "accuracy": 0.67634, "f1": 0.611536, "f1_weighted": 0.666664 }, { "accuracy": 0.69454, "f1": 0.642957, "f1_weighted": 0.68702 }, { "accuracy": 0.661092, "f1": 0.61943, "f1_weighted": 0.645206 }, { "accuracy": 0.657157, "f1": 0.618881, "f1_weighted": 0.648783 }, { "accuracy": 0.670438, "f1": 0.620808, "f1_weighted": 0.658629 }, { "accuracy": 0.658633, "f1": 0.612712, "f1_weighted": 0.647403 }, { "accuracy": 0.654206, "f1": 0.629482, "f1_weighted": 0.634273 }, { "accuracy": 0.693064, "f1": 0.642287, "f1_weighted": 0.687919 } ], "mteb_version": "1.12.75" }, { "accuracy": 0.677964, "f1": 0.631413, "f1_weighted": 0.66873, "hf_subset": "tr", "languages": [ "tur-Latn" ], "main_score": 0.677964, "scores_per_experiment": [ { "accuracy": 0.684702, "f1": 0.62694, "f1_weighted": 0.677721 }, { "accuracy": 0.695524, "f1": 0.640054, "f1_weighted": 0.686276 }, { "accuracy": 0.699951, "f1": 0.647408, "f1_weighted": 0.691633 }, { "accuracy": 0.671913, "f1": 0.621946, "f1_weighted": 0.662451 }, { "accuracy": 0.6788, "f1": 0.630304, "f1_weighted": 0.662826 }, { "accuracy": 0.671422, "f1": 0.634034, "f1_weighted": 0.665178 }, { "accuracy": 0.669946, "f1": 0.630797, "f1_weighted": 0.659879 }, { "accuracy": 0.664043, "f1": 0.6202, "f1_weighted": 0.656635 }, { "accuracy": 0.660108, "f1": 0.62376, "f1_weighted": 0.6461 }, { "accuracy": 0.683227, "f1": 0.638688, "f1_weighted": 0.678599 } ], "mteb_version": "1.12.75" }, { "accuracy": 0.66729, "f1": 0.630523, "f1_weighted": 0.653892, "hf_subset": "el", "languages": [ "ell-Grek" ], "main_score": 0.66729, "scores_per_experiment": [ { "accuracy": 0.672405, "f1": 0.633799, "f1_weighted": 0.662147 }, { "accuracy": 0.698967, "f1": 0.656214, "f1_weighted": 0.690341 }, { "accuracy": 0.673389, "f1": 0.617327, "f1_weighted": 0.663159 }, { "accuracy": 0.685194, "f1": 0.632859, "f1_weighted": 0.671969 }, { "accuracy": 0.651746, "f1": 0.611645, "f1_weighted": 0.632874 }, { "accuracy": 0.658141, "f1": 0.638776, "f1_weighted": 0.641705 }, { "accuracy": 0.666995, "f1": 0.633133, "f1_weighted": 0.654017 }, { "accuracy": 0.635022, "f1": 0.605749, "f1_weighted": 0.621117 }, { "accuracy": 0.638465, "f1": 0.621004, "f1_weighted": 0.621066 }, { "accuracy": 0.692573, "f1": 0.654726, "f1_weighted": 0.680527 } ], "mteb_version": "1.12.75" }, { "accuracy": 0.531087, "f1": 0.485187, "f1_weighted": 0.524974, "hf_subset": "jv", "languages": [ "jav-Latn" ], "main_score": 0.531087, "scores_per_experiment": [ { "accuracy": 0.513527, "f1": 0.485451, "f1_weighted": 0.520896 }, { "accuracy": 0.551894, "f1": 0.512161, "f1_weighted": 0.554562 }, { "accuracy": 0.547467, "f1": 0.490186, "f1_weighted": 0.543391 }, { "accuracy": 0.548451, "f1": 0.500144, "f1_weighted": 0.541105 }, { "accuracy": 0.530251, "f1": 0.477156, "f1_weighted": 0.51651 }, { "accuracy": 0.515494, "f1": 0.489817, "f1_weighted": 0.505986 }, { "accuracy": 0.512051, "f1": 0.466904, "f1_weighted": 0.503636 }, { "accuracy": 0.525824, "f1": 0.463181, "f1_weighted": 0.515742 }, { "accuracy": 0.53517, "f1": 0.487962, "f1_weighted": 0.522964 }, { "accuracy": 0.530743, "f1": 0.478905, "f1_weighted": 0.524951 } ], "mteb_version": "1.12.75" }, { "accuracy": 0.683965, "f1": 0.641404, "f1_weighted": 0.669957, "hf_subset": "nl", "languages": [ "nld-Latn" ], "main_score": 0.683965, "scores_per_experiment": [ { "accuracy": 0.702902, "f1": 0.660618, "f1_weighted": 0.691415 }, { "accuracy": 0.697, "f1": 0.648585, "f1_weighted": 0.688853 }, { "accuracy": 0.691097, "f1": 0.647912, "f1_weighted": 0.680906 }, { "accuracy": 0.692081, "f1": 0.638306, "f1_weighted": 0.681491 }, { "accuracy": 0.681751, "f1": 0.638225, "f1_weighted": 0.664912 }, { "accuracy": 0.6788, "f1": 0.647528, "f1_weighted": 0.663171 }, { "accuracy": 0.684211, "f1": 0.645075, "f1_weighted": 0.668721 }, { "accuracy": 0.667486, "f1": 0.617849, "f1_weighted": 0.650437 }, { "accuracy": 0.647811, "f1": 0.624049, "f1_weighted": 0.622789 }, { "accuracy": 0.696508, "f1": 0.645889, "f1_weighted": 0.68687 } ], "mteb_version": "1.12.75" }, { "accuracy": 0.696311, "f1": 0.654469, "f1_weighted": 0.685603, "hf_subset": "fa", "languages": [ "fas-Arab" ], "main_score": 0.696311, "scores_per_experiment": [ { "accuracy": 0.706345, "f1": 0.663959, "f1_weighted": 0.702207 }, { "accuracy": 0.714215, "f1": 0.665258, "f1_weighted": 0.705516 }, { "accuracy": 0.714215, "f1": 0.661435, "f1_weighted": 0.701497 }, { "accuracy": 0.703394, "f1": 0.64621, "f1_weighted": 0.69619 }, { "accuracy": 0.689621, "f1": 0.650752, "f1_weighted": 0.674949 }, { "accuracy": 0.680275, "f1": 0.655396, "f1_weighted": 0.667565 }, { "accuracy": 0.708805, "f1": 0.672034, "f1_weighted": 0.696608 }, { "accuracy": 0.672897, "f1": 0.624528, "f1_weighted": 0.662141 }, { "accuracy": 0.673881, "f1": 0.646951, "f1_weighted": 0.657639 }, { "accuracy": 0.699459, "f1": 0.658172, "f1_weighted": 0.691719 } ], "mteb_version": "1.12.75" }, { "scores_per_experiment": [ { "accuracy": 0.587801, "f1": 0.539456, "f1_weighted": 0.582601, "precision": 0.533291, "precision_weighted": 0.650533, "recall": 0.646044, "recall_weighted": 0.587801, "ap": null, "ap_weighted": null }, { "accuracy": 0.577964, "f1": 0.511856, "f1_weighted": 0.577571, "precision": 0.516871, "precision_weighted": 0.655845, "recall": 0.610446, "recall_weighted": 0.577964, "ap": null, "ap_weighted": null }, { "accuracy": 0.564683, "f1": 0.504724, "f1_weighted": 0.561815, "precision": 0.495147, "precision_weighted": 0.630267, "recall": 0.607085, "recall_weighted": 0.564683, "ap": null, "ap_weighted": null }, { "accuracy": 0.577472, "f1": 0.531136, "f1_weighted": 0.576647, "precision": 0.531713, "precision_weighted": 0.657054, "recall": 0.628647, "recall_weighted": 0.577472, "ap": null, "ap_weighted": null }, { "accuracy": 0.55091, "f1": 0.504482, "f1_weighted": 0.532166, "precision": 0.502882, "precision_weighted": 0.616073, "recall": 0.620632, "recall_weighted": 0.55091, "ap": null, "ap_weighted": null }, { "accuracy": 0.553369, "f1": 0.532914, "f1_weighted": 0.545593, "precision": 0.536585, "precision_weighted": 0.638799, "recall": 0.626257, "recall_weighted": 0.553369, "ap": null, "ap_weighted": null }, { "accuracy": 0.563207, "f1": 0.513934, "f1_weighted": 0.547069, "precision": 0.515529, "precision_weighted": 0.651318, "recall": 0.62306, "recall_weighted": 0.563207, "ap": null, "ap_weighted": null }, { "accuracy": 0.56124, "f1": 0.509284, "f1_weighted": 0.550391, "precision": 0.508339, "precision_weighted": 0.617716, "recall": 0.612641, "recall_weighted": 0.56124, "ap": null, "ap_weighted": null }, { "accuracy": 0.55878, "f1": 0.516565, "f1_weighted": 0.542423, "precision": 0.5286, "precision_weighted": 0.646078, "recall": 0.61903, "recall_weighted": 0.55878, "ap": null, "ap_weighted": null }, { "accuracy": 0.594196, "f1": 0.552516, "f1_weighted": 0.58874, "precision": 0.551393, "precision_weighted": 0.669038, "recall": 0.649814, "recall_weighted": 0.594196, "ap": null, "ap_weighted": null } ], "accuracy": 0.568962, "f1": 0.521687, "f1_weighted": 0.560502, "precision": 0.522035, "precision_weighted": 0.643272, "recall": 0.624365, "recall_weighted": 0.568962, "ap": NaN, "ap_weighted": NaN, "main_score": 0.568962, "hf_subset": "sw", "languages": [ "swa-Latn" ], "mteb_version": "2.18.0" }, { "accuracy": 0.67639, "f1": 0.631721, "f1_weighted": 0.664164, "hf_subset": "da", "languages": [ "dan-Latn" ], "main_score": 0.67639, "scores_per_experiment": [ { "accuracy": 0.688146, "f1": 0.639371, "f1_weighted": 0.678124 }, { "accuracy": 0.701426, "f1": 0.646692, "f1_weighted": 0.695045 }, { "accuracy": 0.669946, "f1": 0.616352, "f1_weighted": 0.660321 }, { "accuracy": 0.694048, "f1": 0.632169, "f1_weighted": 0.686991 }, { "accuracy": 0.664535, "f1": 0.619198, "f1_weighted": 0.647636 }, { "accuracy": 0.672897, "f1": 0.645252, "f1_weighted": 0.659504 }, { "accuracy": 0.670438, "f1": 0.629495, "f1_weighted": 0.657235 }, { "accuracy": 0.646827, "f1": 0.61275, "f1_weighted": 0.635215 }, { "accuracy": 0.645844, "f1": 0.622244, "f1_weighted": 0.619239 }, { "accuracy": 0.709788, "f1": 0.653686, "f1_weighted": 0.702327 } ], "mteb_version": "1.12.75" }, { "accuracy": 0.561436, "f1": 0.504006, "f1_weighted": 0.547209, "hf_subset": "ar", "languages": [ "ara-Arab" ], "main_score": 0.561436, "scores_per_experiment": [ { "accuracy": 0.571569, "f1": 0.505585, "f1_weighted": 0.561047 }, { "accuracy": 0.578947, "f1": 0.516559, "f1_weighted": 0.572711 }, { "accuracy": 0.561731, "f1": 0.500798, "f1_weighted": 0.543625 }, { "accuracy": 0.549926, "f1": 0.496326, "f1_weighted": 0.542619 }, { "accuracy": 0.559272, "f1": 0.502237, "f1_weighted": 0.53602 }, { "accuracy": 0.547467, "f1": 0.50959, "f1_weighted": 0.531616 }, { "accuracy": 0.565667, "f1": 0.507454, "f1_weighted": 0.552672 }, { "accuracy": 0.539597, "f1": 0.47447, "f1_weighted": 0.520886 }, { "accuracy": 0.563207, "f1": 0.518462, "f1_weighted": 0.546299 }, { "accuracy": 0.57698, "f1": 0.508577, "f1_weighted": 0.564592 } ], "mteb_version": "1.12.75" }, { "accuracy": 0.670044, "f1": 0.624463, "f1_weighted": 0.659445, "hf_subset": "nb", "languages": [ "nob-Latn" ], "main_score": 0.670044, "scores_per_experiment": [ { "accuracy": 0.676832, "f1": 0.639426, "f1_weighted": 0.671784 }, { "accuracy": 0.693064, "f1": 0.641582, "f1_weighted": 0.682539 }, { "accuracy": 0.669454, "f1": 0.61683, "f1_weighted": 0.660843 }, { "accuracy": 0.68667, "f1": 0.632081, "f1_weighted": 0.676757 }, { "accuracy": 0.655189, "f1": 0.615382, "f1_weighted": 0.64085 }, { "accuracy": 0.669946, "f1": 0.634932, "f1_weighted": 0.663927 }, { "accuracy": 0.661092, "f1": 0.61222, "f1_weighted": 0.649058 }, { "accuracy": 0.653222, "f1": 0.608685, "f1_weighted": 0.644259 }, { "accuracy": 0.63699, "f1": 0.604453, "f1_weighted": 0.609703 }, { "accuracy": 0.697983, "f1": 0.639041, "f1_weighted": 0.694726 } ], "mteb_version": "1.12.75" }, { "accuracy": 0.654107, "f1": 0.609712, "f1_weighted": 0.643954, "hf_subset": "hi", "languages": [ "hin-Deva" ], "main_score": 0.654107, "scores_per_experiment": [ { "accuracy": 0.66847, "f1": 0.620569, "f1_weighted": 0.665035 }, { "accuracy": 0.671422, "f1": 0.618662, "f1_weighted": 0.66688 }, { "accuracy": 0.659616, "f1": 0.603714, "f1_weighted": 0.651248 }, { "accuracy": 0.661092, "f1": 0.60114, "f1_weighted": 0.651934 }, { "accuracy": 0.662076, "f1": 0.614886, "f1_weighted": 0.643681 }, { "accuracy": 0.622725, "f1": 0.602032, "f1_weighted": 0.60878 }, { "accuracy": 0.654206, "f1": 0.623241, "f1_weighted": 0.644064 }, { "accuracy": 0.641417, "f1": 0.592278, "f1_weighted": 0.633568 }, { "accuracy": 0.643876, "f1": 0.615551, "f1_weighted": 0.628727 }, { "accuracy": 0.656173, "f1": 0.605049, "f1_weighted": 0.645624 } ], "mteb_version": "1.12.75" }, { "accuracy": 0.641761, "f1": 0.594936, "f1_weighted": 0.632561, "hf_subset": "hy", "languages": [ "hye-Armn" ], "main_score": 0.641761, "scores_per_experiment": [ { "accuracy": 0.646335, "f1": 0.590692, "f1_weighted": 0.639991 }, { "accuracy": 0.653222, "f1": 0.604492, "f1_weighted": 0.65048 }, { "accuracy": 0.654206, "f1": 0.59776, "f1_weighted": 0.645314 }, { "accuracy": 0.655189, "f1": 0.588458, "f1_weighted": 0.647473 }, { "accuracy": 0.638465, "f1": 0.5941, "f1_weighted": 0.621497 }, { "accuracy": 0.62912, "f1": 0.593817, "f1_weighted": 0.624502 }, { "accuracy": 0.643384, "f1": 0.6067, "f1_weighted": 0.631333 }, { "accuracy": 0.614363, "f1": 0.561949, "f1_weighted": 0.605405 }, { "accuracy": 0.631087, "f1": 0.60216, "f1_weighted": 0.618278 }, { "accuracy": 0.652238, "f1": 0.609229, "f1_weighted": 0.64134 } ], "mteb_version": "1.12.75" }, { "accuracy": 0.700148, "f1": 0.656397, "f1_weighted": 0.686463, "hf_subset": "ja", "languages": [ "jpn-Jpan" ], "main_score": 0.700148, "scores_per_experiment": [ { "accuracy": 0.721102, "f1": 0.669141, "f1_weighted": 0.713754 }, { "accuracy": 0.727988, "f1": 0.680465, "f1_weighted": 0.721807 }, { "accuracy": 0.705362, "f1": 0.649756, "f1_weighted": 0.692254 }, { "accuracy": 0.701918, "f1": 0.644854, "f1_weighted": 0.68877 }, { "accuracy": 0.685194, "f1": 0.646374, "f1_weighted": 0.667691 }, { "accuracy": 0.689621, "f1": 0.659909, "f1_weighted": 0.670239 }, { "accuracy": 0.685194, "f1": 0.638927, "f1_weighted": 0.670993 }, { "accuracy": 0.680767, "f1": 0.644152, "f1_weighted": 0.660759 }, { "accuracy": 0.690113, "f1": 0.656843, "f1_weighted": 0.675984 }, { "accuracy": 0.714215, "f1": 0.673547, "f1_weighted": 0.702375 } ], "mteb_version": "1.12.75" }, { "accuracy": 0.622528, "f1": 0.580135, "f1_weighted": 0.610355, "hf_subset": "mn", "languages": [ "mon-Cyrl" ], "main_score": 0.622528, "scores_per_experiment": [ { "accuracy": 0.624693, "f1": 0.588494, "f1_weighted": 0.616305 }, { "accuracy": 0.639449, "f1": 0.593337, "f1_weighted": 0.635759 }, { "accuracy": 0.613871, "f1": 0.568578, "f1_weighted": 0.607138 }, { "accuracy": 0.651746, "f1": 0.596502, "f1_weighted": 0.64542 }, { "accuracy": 0.619774, "f1": 0.571126, "f1_weighted": 0.603427 }, { "accuracy": 0.599115, "f1": 0.565766, "f1_weighted": 0.576486 }, { "accuracy": 0.627644, "f1": 0.589413, "f1_weighted": 0.611115 }, { "accuracy": 0.614363, "f1": 0.566042, "f1_weighted": 0.600546 }, { "accuracy": 0.606001, "f1": 0.577682, "f1_weighted": 0.588196 }, { "accuracy": 0.628628, "f1": 0.584407, "f1_weighted": 0.619159 } ], "mteb_version": "1.12.75" }, { "accuracy": 0.69577, "f1": 0.661623, "f1_weighted": 0.683145, "hf_subset": "zh-CN", "languages": [ "cmo-Hans" ], "main_score": 0.69577, "scores_per_experiment": [ { "accuracy": 0.715691, "f1": 0.675434, "f1_weighted": 0.709005 }, { "accuracy": 0.725529, "f1": 0.686736, "f1_weighted": 0.712972 }, { "accuracy": 0.692573, "f1": 0.647263, "f1_weighted": 0.681231 }, { "accuracy": 0.710772, "f1": 0.671881, "f1_weighted": 0.701532 }, { "accuracy": 0.703394, "f1": 0.655755, "f1_weighted": 0.688588 }, { "accuracy": 0.686178, "f1": 0.654095, "f1_weighted": 0.67142 }, { "accuracy": 0.673881, "f1": 0.644223, "f1_weighted": 0.658029 }, { "accuracy": 0.67634, "f1": 0.650192, "f1_weighted": 0.668374 }, { "accuracy": 0.665519, "f1": 0.655198, "f1_weighted": 0.645546 }, { "accuracy": 0.707821, "f1": 0.675453, "f1_weighted": 0.694757 } ], "mteb_version": "1.12.75" }, { "accuracy": 0.646778, "f1": 0.617531, "f1_weighted": 0.628148, "hf_subset": "th", "languages": [ "tha-Thai" ], "main_score": 0.646778, "scores_per_experiment": [ { "accuracy": 0.664535, "f1": 0.639119, "f1_weighted": 0.645618 }, { "accuracy": 0.6606, "f1": 0.628293, "f1_weighted": 0.644247 }, { "accuracy": 0.651746, "f1": 0.616173, "f1_weighted": 0.639534 }, { "accuracy": 0.636498, "f1": 0.604015, "f1_weighted": 0.623495 }, { "accuracy": 0.654697, "f1": 0.616873, "f1_weighted": 0.629808 }, { "accuracy": 0.647319, "f1": 0.630619, "f1_weighted": 0.627405 }, { "accuracy": 0.640925, "f1": 0.614349, "f1_weighted": 0.619185 }, { "accuracy": 0.622725, "f1": 0.598267, "f1_weighted": 0.601275 }, { "accuracy": 0.616331, "f1": 0.594528, "f1_weighted": 0.593725 }, { "accuracy": 0.672405, "f1": 0.633078, "f1_weighted": 0.657191 } ], "mteb_version": "1.12.75" }, { "accuracy": 0.529218, "f1": 0.470574, "f1_weighted": 0.526869, "hf_subset": "cy", "languages": [ "cym-Latn" ], "main_score": 0.529218, "scores_per_experiment": [ { "accuracy": 0.549434, "f1": 0.493611, "f1_weighted": 0.547024 }, { "accuracy": 0.540089, "f1": 0.463704, "f1_weighted": 0.544366 }, { "accuracy": 0.541564, "f1": 0.486489, "f1_weighted": 0.542503 }, { "accuracy": 0.530743, "f1": 0.469076, "f1_weighted": 0.531371 }, { "accuracy": 0.537629, "f1": 0.470683, "f1_weighted": 0.533878 }, { "accuracy": 0.510084, "f1": 0.470276, "f1_weighted": 0.512824 }, { "accuracy": 0.518446, "f1": 0.461122, "f1_weighted": 0.515442 }, { "accuracy": 0.523364, "f1": 0.462978, "f1_weighted": 0.516484 }, { "accuracy": 0.496803, "f1": 0.453397, "f1_weighted": 0.483768 }, { "accuracy": 0.544024, "f1": 0.474408, "f1_weighted": 0.541025 } ], "mteb_version": "1.12.75" }, { "accuracy": 0.635071, "f1": 0.589646, "f1_weighted": 0.627095, "hf_subset": "te", "languages": [ "tel-Telu" ], "main_score": 0.635071, "scores_per_experiment": [ { "accuracy": 0.645844, "f1": 0.596525, "f1_weighted": 0.639267 }, { "accuracy": 0.661092, "f1": 0.60178, "f1_weighted": 0.661527 }, { "accuracy": 0.645352, "f1": 0.592135, "f1_weighted": 0.639987 }, { "accuracy": 0.637973, "f1": 0.593499, "f1_weighted": 0.635331 }, { "accuracy": 0.623217, "f1": 0.57524, "f1_weighted": 0.61251 }, { "accuracy": 0.616331, "f1": 0.595967, "f1_weighted": 0.604546 }, { "accuracy": 0.622725, "f1": 0.584062, "f1_weighted": 0.614903 }, { "accuracy": 0.631579, "f1": 0.582362, "f1_weighted": 0.621694 }, { "accuracy": 0.612887, "f1": 0.580452, "f1_weighted": 0.591509 }, { "accuracy": 0.653714, "f1": 0.594442, "f1_weighted": 0.649679 } ], "mteb_version": "1.12.75" }, { "accuracy": 0.549139, "f1": 0.520206, "f1_weighted": 0.53561, "hf_subset": "ka", "languages": [ "kat-Geor" ], "main_score": 0.549139, "scores_per_experiment": [ { "accuracy": 0.551894, "f1": 0.521485, "f1_weighted": 0.543956 }, { "accuracy": 0.570585, "f1": 0.533103, "f1_weighted": 0.566557 }, { "accuracy": 0.543532, "f1": 0.517016, "f1_weighted": 0.530352 }, { "accuracy": 0.541564, "f1": 0.501474, "f1_weighted": 0.528545 }, { "accuracy": 0.53517, "f1": 0.498122, "f1_weighted": 0.512303 }, { "accuracy": 0.53517, "f1": 0.527923, "f1_weighted": 0.51972 }, { "accuracy": 0.563699, "f1": 0.543765, "f1_weighted": 0.550746 }, { "accuracy": 0.545991, "f1": 0.517083, "f1_weighted": 0.535948 }, { "accuracy": 0.55091, "f1": 0.518292, "f1_weighted": 0.529976 }, { "accuracy": 0.552878, "f1": 0.523801, "f1_weighted": 0.537992 } ], "mteb_version": "1.12.75" }, { "accuracy": 0.659911, "f1": 0.619308, "f1_weighted": 0.647691, "hf_subset": "ro", "languages": [ "ron-Latn" ], "main_score": 0.659911, "scores_per_experiment": [ { "accuracy": 0.66847, "f1": 0.624561, "f1_weighted": 0.662432 }, { "accuracy": 0.670438, "f1": 0.621509, "f1_weighted": 0.661268 }, { "accuracy": 0.658633, "f1": 0.618341, "f1_weighted": 0.65136 }, { "accuracy": 0.662076, "f1": 0.618132, "f1_weighted": 0.651611 }, { "accuracy": 0.665027, "f1": 0.619655, "f1_weighted": 0.645761 }, { "accuracy": 0.65273, "f1": 0.623623, "f1_weighted": 0.642696 }, { "accuracy": 0.6606, "f1": 0.622635, "f1_weighted": 0.646963 }, { "accuracy": 0.641909, "f1": 0.605856, "f1_weighted": 0.626392 }, { "accuracy": 0.647319, "f1": 0.620521, "f1_weighted": 0.621759 }, { "accuracy": 0.671913, "f1": 0.618248, "f1_weighted": 0.666669 } ], "mteb_version": "1.12.75" }, { "accuracy": 0.671913, "f1": 0.631019, "f1_weighted": 0.658166, "hf_subset": "id", "languages": [ "ind-Latn" ], "main_score": 0.671913, "scores_per_experiment": [ { "accuracy": 0.676832, "f1": 0.644024, "f1_weighted": 0.664219 }, { "accuracy": 0.701918, "f1": 0.653948, "f1_weighted": 0.69739 }, { "accuracy": 0.666503, "f1": 0.611972, "f1_weighted": 0.654796 }, { "accuracy": 0.690605, "f1": 0.646596, "f1_weighted": 0.681622 }, { "accuracy": 0.667978, "f1": 0.622639, "f1_weighted": 0.647325 }, { "accuracy": 0.65273, "f1": 0.623089, "f1_weighted": 0.633632 }, { "accuracy": 0.671913, "f1": 0.626765, "f1_weighted": 0.655007 }, { "accuracy": 0.650271, "f1": 0.618329, "f1_weighted": 0.636454 }, { "accuracy": 0.651254, "f1": 0.626713, "f1_weighted": 0.630952 }, { "accuracy": 0.689129, "f1": 0.636118, "f1_weighted": 0.68026 } ], "mteb_version": "1.12.75" }, { "accuracy": 0.658633, "f1": 0.612808, "f1_weighted": 0.646722, "hf_subset": "de", "languages": [ "deu-Latn" ], "main_score": 0.658633, "scores_per_experiment": [ { "accuracy": 0.662076, "f1": 0.615678, "f1_weighted": 0.655848 }, { "accuracy": 0.677816, "f1": 0.619501, "f1_weighted": 0.669273 }, { "accuracy": 0.665027, "f1": 0.62259, "f1_weighted": 0.652439 }, { "accuracy": 0.670438, "f1": 0.625217, "f1_weighted": 0.659367 }, { "accuracy": 0.665027, "f1": 0.616657, "f1_weighted": 0.650233 }, { "accuracy": 0.646827, "f1": 0.610179, "f1_weighted": 0.638563 }, { "accuracy": 0.655681, "f1": 0.60346, "f1_weighted": 0.64219 }, { "accuracy": 0.636498, "f1": 0.588598, "f1_weighted": 0.619792 }, { "accuracy": 0.638957, "f1": 0.609176, "f1_weighted": 0.622529 }, { "accuracy": 0.667978, "f1": 0.61702, "f1_weighted": 0.656986 } ], "mteb_version": "1.12.75" }, { "accuracy": 0.64427, "f1": 0.600307, "f1_weighted": 0.634435, "hf_subset": "lv", "languages": [ "lav-Latn" ], "main_score": 0.64427, "scores_per_experiment": [ { "accuracy": 0.6606, "f1": 0.604844, "f1_weighted": 0.656003 }, { "accuracy": 0.671422, "f1": 0.625916, "f1_weighted": 0.667928 }, { "accuracy": 0.647319, "f1": 0.602571, "f1_weighted": 0.637716 }, { "accuracy": 0.65273, "f1": 0.606633, "f1_weighted": 0.645126 }, { "accuracy": 0.636498, "f1": 0.592943, "f1_weighted": 0.625742 }, { "accuracy": 0.625184, "f1": 0.588754, "f1_weighted": 0.614971 }, { "accuracy": 0.652238, "f1": 0.604138, "f1_weighted": 0.640734 }, { "accuracy": 0.616331, "f1": 0.576565, "f1_weighted": 0.601578 }, { "accuracy": 0.610428, "f1": 0.579141, "f1_weighted": 0.592848 }, { "accuracy": 0.669946, "f1": 0.621565, "f1_weighted": 0.661698 } ], "mteb_version": "1.12.75" }, { "accuracy": 0.645007, "f1": 0.624162, "f1_weighted": 0.626455, "hf_subset": "zh-TW", "languages": [ "cmo-Hant" ], "main_score": 0.645007, "scores_per_experiment": [ { "accuracy": 0.664535, "f1": 0.640049, "f1_weighted": 0.654955 }, { "accuracy": 0.667486, "f1": 0.647207, "f1_weighted": 0.6543 }, { "accuracy": 0.658141, "f1": 0.628551, "f1_weighted": 0.643126 }, { "accuracy": 0.628136, "f1": 0.612143, "f1_weighted": 0.609389 }, { "accuracy": 0.647811, "f1": 0.615471, "f1_weighted": 0.621879 }, { "accuracy": 0.63699, "f1": 0.619574, "f1_weighted": 0.611237 }, { "accuracy": 0.631087, "f1": 0.613866, "f1_weighted": 0.604257 }, { "accuracy": 0.63699, "f1": 0.622785, "f1_weighted": 0.624854 }, { "accuracy": 0.620266, "f1": 0.601052, "f1_weighted": 0.597234 }, { "accuracy": 0.658633, "f1": 0.640919, "f1_weighted": 0.643323 } ], "mteb_version": "1.12.75" }, { "accuracy": 0.662666, "f1": 0.618323, "f1_weighted": 0.650915, "hf_subset": "sq", "languages": [ "sqi-Latn" ], "main_score": 0.662666, "scores_per_experiment": [ { "accuracy": 0.67093, "f1": 0.626195, "f1_weighted": 0.663789 }, { "accuracy": 0.697983, "f1": 0.642676, "f1_weighted": 0.69122 }, { "accuracy": 0.645352, "f1": 0.589093, "f1_weighted": 0.635128 }, { "accuracy": 0.672405, "f1": 0.617406, "f1_weighted": 0.663617 }, { "accuracy": 0.6424, "f1": 0.600421, "f1_weighted": 0.623887 }, { "accuracy": 0.640925, "f1": 0.615137, "f1_weighted": 0.632212 }, { "accuracy": 0.664535, "f1": 0.62825, "f1_weighted": 0.65148 }, { "accuracy": 0.652238, "f1": 0.607583, "f1_weighted": 0.641108 }, { "accuracy": 0.646335, "f1": 0.610207, "f1_weighted": 0.622356 }, { "accuracy": 0.693556, "f1": 0.64626, "f1_weighted": 0.684355 } ], "mteb_version": "1.12.75" }, { "accuracy": 0.596409, "f1": 0.550569, "f1_weighted": 0.591194, "hf_subset": "my", "languages": [ "mya-Mymr" ], "main_score": 0.596409, "scores_per_experiment": [ { "accuracy": 0.617314, "f1": 0.574362, "f1_weighted": 0.611629 }, { "accuracy": 0.611904, "f1": 0.557964, "f1_weighted": 0.611459 }, { "accuracy": 0.608952, "f1": 0.553723, "f1_weighted": 0.604297 }, { "accuracy": 0.586818, "f1": 0.538259, "f1_weighted": 0.583184 }, { "accuracy": 0.601574, "f1": 0.542609, "f1_weighted": 0.594103 }, { "accuracy": 0.577472, "f1": 0.551892, "f1_weighted": 0.571651 }, { "accuracy": 0.607969, "f1": 0.56833, "f1_weighted": 0.602472 }, { "accuracy": 0.581899, "f1": 0.529133, "f1_weighted": 0.578177 }, { "accuracy": 0.589277, "f1": 0.560512, "f1_weighted": 0.583375 }, { "accuracy": 0.580915, "f1": 0.528906, "f1_weighted": 0.571589 } ], "mteb_version": "1.12.75" }, { "accuracy": 0.678751, "f1": 0.648232, "f1_weighted": 0.666848, "hf_subset": "it", "languages": [ "ita-Latn" ], "main_score": 0.678751, "scores_per_experiment": [ { "accuracy": 0.686178, "f1": 0.659338, "f1_weighted": 0.679758 }, { "accuracy": 0.693064, "f1": 0.666939, "f1_weighted": 0.683131 }, { "accuracy": 0.681751, "f1": 0.644971, "f1_weighted": 0.670168 }, { "accuracy": 0.669454, "f1": 0.638802, "f1_weighted": 0.661181 }, { "accuracy": 0.686178, "f1": 0.642993, "f1_weighted": 0.669774 }, { "accuracy": 0.674373, "f1": 0.652801, "f1_weighted": 0.663747 }, { "accuracy": 0.687162, "f1": 0.655175, "f1_weighted": 0.67442 }, { "accuracy": 0.651254, "f1": 0.629758, "f1_weighted": 0.631192 }, { "accuracy": 0.664043, "f1": 0.639734, "f1_weighted": 0.652882 }, { "accuracy": 0.694048, "f1": 0.651811, "f1_weighted": 0.682226 } ], "mteb_version": "1.12.75" }, { "accuracy": 0.661043, "f1": 0.608118, "f1_weighted": 0.651246, "hf_subset": "he", "languages": [ "heb-Hebr" ], "main_score": 0.661043, "scores_per_experiment": [ { "accuracy": 0.647811, "f1": 0.602347, "f1_weighted": 0.64218 }, { "accuracy": 0.696016, "f1": 0.634484, "f1_weighted": 0.688475 }, { "accuracy": 0.673389, "f1": 0.619061, "f1_weighted": 0.66264 }, { "accuracy": 0.666995, "f1": 0.613394, "f1_weighted": 0.659427 }, { "accuracy": 0.651746, "f1": 0.59259, "f1_weighted": 0.634093 }, { "accuracy": 0.652238, "f1": 0.596255, "f1_weighted": 0.647462 }, { "accuracy": 0.656173, "f1": 0.596558, "f1_weighted": 0.645728 }, { "accuracy": 0.637482, "f1": 0.593633, "f1_weighted": 0.628098 }, { "accuracy": 0.641417, "f1": 0.602462, "f1_weighted": 0.622934 }, { "accuracy": 0.687162, "f1": 0.630397, "f1_weighted": 0.681427 } ], "mteb_version": "1.12.75" }, { "accuracy": 0.501033, "f1": 0.472749, "f1_weighted": 0.496818, "hf_subset": "km", "languages": [ "khm-Khmr" ], "main_score": 0.501033, "scores_per_experiment": [ { "accuracy": 0.520413, "f1": 0.4951, "f1_weighted": 0.521853 }, { "accuracy": 0.518938, "f1": 0.473504, "f1_weighted": 0.521829 }, { "accuracy": 0.488933, "f1": 0.460762, "f1_weighted": 0.4823 }, { "accuracy": 0.502705, "f1": 0.471186, "f1_weighted": 0.504009 }, { "accuracy": 0.519429, "f1": 0.47922, "f1_weighted": 0.507944 }, { "accuracy": 0.466306, "f1": 0.457003, "f1_weighted": 0.454937 }, { "accuracy": 0.477619, "f1": 0.461564, "f1_weighted": 0.468131 }, { "accuracy": 0.508116, "f1": 0.476039, "f1_weighted": 0.512469 }, { "accuracy": 0.497295, "f1": 0.479013, "f1_weighted": 0.49489 }, { "accuracy": 0.510576, "f1": 0.474103, "f1_weighted": 0.499821 } ], "mteb_version": "1.12.75" }, { "accuracy": 0.646926, "f1": 0.610076, "f1_weighted": 0.635752, "hf_subset": "ms", "languages": [ "msa-Latn" ], "main_score": 0.646926, "scores_per_experiment": [ { "accuracy": 0.668962, "f1": 0.62731, "f1_weighted": 0.662022 }, { "accuracy": 0.671913, "f1": 0.627285, "f1_weighted": 0.667785 }, { "accuracy": 0.646827, "f1": 0.603692, "f1_weighted": 0.638847 }, { "accuracy": 0.658141, "f1": 0.615991, "f1_weighted": 0.648846 }, { "accuracy": 0.651746, "f1": 0.600397, "f1_weighted": 0.636581 }, { "accuracy": 0.614855, "f1": 0.600151, "f1_weighted": 0.600537 }, { "accuracy": 0.637482, "f1": 0.606286, "f1_weighted": 0.627464 }, { "accuracy": 0.62666, "f1": 0.595399, "f1_weighted": 0.613705 }, { "accuracy": 0.623709, "f1": 0.601096, "f1_weighted": 0.600434 }, { "accuracy": 0.668962, "f1": 0.623156, "f1_weighted": 0.6613 } ], "mteb_version": "1.12.75" }, { "accuracy": 0.64909, "f1": 0.614341, "f1_weighted": 0.63607, "hf_subset": "hu", "languages": [ "hun-Latn" ], "main_score": 0.64909, "scores_per_experiment": [ { "accuracy": 0.659124, "f1": 0.621017, "f1_weighted": 0.646771 }, { "accuracy": 0.675357, "f1": 0.638038, "f1_weighted": 0.667965 }, { "accuracy": 0.643384, "f1": 0.612011, "f1_weighted": 0.632033 }, { "accuracy": 0.657649, "f1": 0.613968, "f1_weighted": 0.643697 }, { "accuracy": 0.651746, "f1": 0.613562, "f1_weighted": 0.636847 }, { "accuracy": 0.658633, "f1": 0.62915, "f1_weighted": 0.648915 }, { "accuracy": 0.630103, "f1": 0.608527, "f1_weighted": 0.615992 }, { "accuracy": 0.622233, "f1": 0.583307, "f1_weighted": 0.611401 }, { "accuracy": 0.62666, "f1": 0.599942, "f1_weighted": 0.602447 }, { "accuracy": 0.666011, "f1": 0.623893, "f1_weighted": 0.654633 } ], "mteb_version": "1.12.75" }, { "accuracy": 0.6212, "f1": 0.567369, "f1_weighted": 0.610359, "hf_subset": "is", "languages": [ "isl-Latn" ], "main_score": 0.6212, "scores_per_experiment": [ { "accuracy": 0.62912, "f1": 0.575002, "f1_weighted": 0.621772 }, { "accuracy": 0.647811, "f1": 0.588723, "f1_weighted": 0.634888 }, { "accuracy": 0.605017, "f1": 0.5502, "f1_weighted": 0.595484 }, { "accuracy": 0.634038, "f1": 0.570833, "f1_weighted": 0.62615 }, { "accuracy": 0.636498, "f1": 0.579647, "f1_weighted": 0.62326 }, { "accuracy": 0.588785, "f1": 0.552272, "f1_weighted": 0.575075 }, { "accuracy": 0.62666, "f1": 0.574585, "f1_weighted": 0.620096 }, { "accuracy": 0.593212, "f1": 0.541088, "f1_weighted": 0.573652 }, { "accuracy": 0.611412, "f1": 0.56457, "f1_weighted": 0.598722 }, { "accuracy": 0.639449, "f1": 0.576774, "f1_weighted": 0.634486 } ], "mteb_version": "1.12.75" }, { "accuracy": 0.62184, "f1": 0.572636, "f1_weighted": 0.608719, "hf_subset": "kn", "languages": [ "kan-Knda" ], "main_score": 0.62184, "scores_per_experiment": [ { "accuracy": 0.635022, "f1": 0.572648, "f1_weighted": 0.629228 }, { "accuracy": 0.648795, "f1": 0.591352, "f1_weighted": 0.643926 }, { "accuracy": 0.61879, "f1": 0.563541, "f1_weighted": 0.608044 }, { "accuracy": 0.627152, "f1": 0.581267, "f1_weighted": 0.619989 }, { "accuracy": 0.602558, "f1": 0.553463, "f1_weighted": 0.578019 }, { "accuracy": 0.603542, "f1": 0.561664, "f1_weighted": 0.589392 }, { "accuracy": 0.617314, "f1": 0.576483, "f1_weighted": 0.601909 }, { "accuracy": 0.614855, "f1": 0.560955, "f1_weighted": 0.599886 }, { "accuracy": 0.609444, "f1": 0.58041, "f1_weighted": 0.585229 }, { "accuracy": 0.640925, "f1": 0.584583, "f1_weighted": 0.631564 } ], "mteb_version": "1.12.75" }, { "accuracy": 0.64545, "f1": 0.597255, "f1_weighted": 0.635379, "hf_subset": "ml", "languages": [ "mal-Mlym" ], "main_score": 0.64545, "scores_per_experiment": [ { "accuracy": 0.673389, "f1": 0.630873, "f1_weighted": 0.666665 }, { "accuracy": 0.65273, "f1": 0.596067, "f1_weighted": 0.649585 }, { "accuracy": 0.645352, "f1": 0.588457, "f1_weighted": 0.637426 }, { "accuracy": 0.660108, "f1": 0.607777, "f1_weighted": 0.651722 }, { "accuracy": 0.657649, "f1": 0.605468, "f1_weighted": 0.646588 }, { "accuracy": 0.603542, "f1": 0.568718, "f1_weighted": 0.591704 }, { "accuracy": 0.640433, "f1": 0.586424, "f1_weighted": 0.628686 }, { "accuracy": 0.630103, "f1": 0.579907, "f1_weighted": 0.619869 }, { "accuracy": 0.620758, "f1": 0.589488, "f1_weighted": 0.602094 }, { "accuracy": 0.670438, "f1": 0.61937, "f1_weighted": 0.65945 } ], "mteb_version": "1.12.75" }, { "accuracy": 0.600344, "f1": 0.561384, "f1_weighted": 0.590267, "hf_subset": "ta", "languages": [ "tam-Taml" ], "main_score": 0.600344, "scores_per_experiment": [ { "accuracy": 0.603542, "f1": 0.565601, "f1_weighted": 0.597836 }, { "accuracy": 0.601574, "f1": 0.558033, "f1_weighted": 0.594123 }, { "accuracy": 0.612395, "f1": 0.571232, "f1_weighted": 0.609969 }, { "accuracy": 0.592228, "f1": 0.547289, "f1_weighted": 0.583121 }, { "accuracy": 0.61879, "f1": 0.577406, "f1_weighted": 0.600858 }, { "accuracy": 0.590753, "f1": 0.56157, "f1_weighted": 0.582854 }, { "accuracy": 0.596163, "f1": 0.57108, "f1_weighted": 0.583658 }, { "accuracy": 0.579439, "f1": 0.542521, "f1_weighted": 0.561474 }, { "accuracy": 0.590261, "f1": 0.562611, "f1_weighted": 0.578534 }, { "accuracy": 0.618298, "f1": 0.5565, "f1_weighted": 0.610246 } ], "mteb_version": "1.12.75" }, { "accuracy": 0.630644, "f1": 0.585112, "f1_weighted": 0.621122, "hf_subset": "ur", "languages": [ "urd-Arab" ], "main_score": 0.630644, "scores_per_experiment": [ { "accuracy": 0.64486, "f1": 0.598652, "f1_weighted": 0.638869 }, { "accuracy": 0.655681, "f1": 0.602442, "f1_weighted": 0.651128 }, { "accuracy": 0.622233, "f1": 0.565114, "f1_weighted": 0.617553 }, { "accuracy": 0.655189, "f1": 0.600232, "f1_weighted": 0.64815 }, { "accuracy": 0.605017, "f1": 0.557898, "f1_weighted": 0.588715 }, { "accuracy": 0.61879, "f1": 0.595746, "f1_weighted": 0.60466 }, { "accuracy": 0.623217, "f1": 0.579512, "f1_weighted": 0.613645 }, { "accuracy": 0.604033, "f1": 0.564056, "f1_weighted": 0.590821 }, { "accuracy": 0.637482, "f1": 0.598464, "f1_weighted": 0.626776 }, { "accuracy": 0.639941, "f1": 0.589006, "f1_weighted": 0.630903 } ], "mteb_version": "1.12.75" }, { "scores_per_experiment": [ { "accuracy": 0.679784, "f1": 0.643719, "f1_weighted": 0.674762, "precision": 0.62691, "precision_weighted": 0.728925, "recall": 0.738688, "recall_weighted": 0.679784, "ap": null, "ap_weighted": null }, { "accuracy": 0.709297, "f1": 0.658021, "f1_weighted": 0.704261, "precision": 0.641019, "precision_weighted": 0.753824, "recall": 0.743351, "recall_weighted": 0.709297, "ap": null, "ap_weighted": null }, { "accuracy": 0.679784, "f1": 0.641644, "f1_weighted": 0.669558, "precision": 0.613671, "precision_weighted": 0.699838, "recall": 0.737271, "recall_weighted": 0.679784, "ap": null, "ap_weighted": null }, { "accuracy": 0.688146, "f1": 0.63668, "f1_weighted": 0.676745, "precision": 0.62969, "precision_weighted": 0.752496, "recall": 0.72772, "recall_weighted": 0.688146, "ap": null, "ap_weighted": null }, { "accuracy": 0.680275, "f1": 0.637716, "f1_weighted": 0.66477, "precision": 0.619402, "precision_weighted": 0.711702, "recall": 0.736321, "recall_weighted": 0.680275, "ap": null, "ap_weighted": null }, { "accuracy": 0.669454, "f1": 0.640129, "f1_weighted": 0.656014, "precision": 0.633059, "precision_weighted": 0.732335, "recall": 0.737567, "recall_weighted": 0.669454, "ap": null, "ap_weighted": null }, { "accuracy": 0.672897, "f1": 0.633125, "f1_weighted": 0.661467, "precision": 0.630628, "precision_weighted": 0.75471, "recall": 0.734342, "recall_weighted": 0.672897, "ap": null, "ap_weighted": null }, { "accuracy": 0.651746, "f1": 0.621292, "f1_weighted": 0.64124, "precision": 0.622909, "precision_weighted": 0.721557, "recall": 0.712474, "recall_weighted": 0.651746, "ap": null, "ap_weighted": null }, { "accuracy": 0.659124, "f1": 0.637422, "f1_weighted": 0.641157, "precision": 0.63031, "precision_weighted": 0.73072, "recall": 0.742225, "recall_weighted": 0.659124, "ap": null, "ap_weighted": null }, { "accuracy": 0.696508, "f1": 0.652921, "f1_weighted": 0.689154, "precision": 0.6458, "precision_weighted": 0.752992, "recall": 0.741201, "recall_weighted": 0.696508, "ap": null, "ap_weighted": null } ], "accuracy": 0.678701, "f1": 0.640267, "f1_weighted": 0.667913, "precision": 0.62934, "precision_weighted": 0.73391, "recall": 0.735116, "recall_weighted": 0.678701, "ap": NaN, "ap_weighted": NaN, "main_score": 0.678701, "hf_subset": "pt", "languages": [ "por-Latn" ], "mteb_version": "2.18.0" }, { "accuracy": 0.673389, "f1": 0.638039, "f1_weighted": 0.661986, "hf_subset": "es", "languages": [ "spa-Latn" ], "main_score": 0.673389, "scores_per_experiment": [ { "accuracy": 0.685194, "f1": 0.649211, "f1_weighted": 0.679569 }, { "accuracy": 0.696016, "f1": 0.655343, "f1_weighted": 0.68387 }, { "accuracy": 0.671422, "f1": 0.633481, "f1_weighted": 0.661914 }, { "accuracy": 0.673881, "f1": 0.629832, "f1_weighted": 0.660411 }, { "accuracy": 0.679292, "f1": 0.639464, "f1_weighted": 0.662924 }, { "accuracy": 0.673389, "f1": 0.636876, "f1_weighted": 0.667014 }, { "accuracy": 0.652238, "f1": 0.622136, "f1_weighted": 0.642309 }, { "accuracy": 0.655681, "f1": 0.629437, "f1_weighted": 0.645407 }, { "accuracy": 0.659124, "f1": 0.642879, "f1_weighted": 0.638569 }, { "accuracy": 0.687654, "f1": 0.641732, "f1_weighted": 0.677874 } ], "mteb_version": "1.12.75" } ] }, "evaluation_time": 5685.548665425682, "kg_co2_emissions": 1.1238586200524652, "date": 1784174882.608298, "evaluation_phases": [ { "name": "Data loading", "start": 5284.960953558679, "end": 5285.804046618636, "split": "", "subset": "" }, { "name": "Dataset transform", "start": 5285.804069441685, "end": 5285.804079931637, "split": "", "subset": "" }, { "name": "Encoding training samples", "start": 5288.810265491658, "end": 5292.337757057685, "split": "validation", "subset": "af" }, { "name": "Encoding test samples", "start": 5292.4684333216865, "end": 5294.46249568864, "split": "validation", "subset": "af" }, { "name": "Running experiment (0/10)", "start": 5294.463049610669, "end": 5297.375295786653, "split": "validation", "subset": "af" }, { "name": "Running experiment (1/10)", "start": 5297.375671523681, "end": 5300.568286336667, "split": "validation", "subset": "af" }, { "name": "Running experiment (2/10)", "start": 5300.568705915648, "end": 5303.182847592689, "split": "validation", "subset": "af" }, { "name": "Running experiment (3/10)", "start": 5303.183286668675, "end": 5306.318764736643, "split": "validation", "subset": "af" }, { "name": "Running experiment (4/10)", "start": 5306.319226705644, "end": 5309.498552184668, "split": "validation", "subset": "af" }, { "name": "Running experiment (5/10)", "start": 5309.49905959866, "end": 5312.606071595685, "split": "validation", "subset": "af" }, { "name": "Running experiment (6/10)", "start": 5312.60647051665, "end": 5315.590073532658, "split": "validation", "subset": "af" }, { "name": "Running experiment (7/10)", "start": 5315.5921377466875, "end": 5318.594686291646, "split": "validation", "subset": "af" }, { "name": "Running experiment (8/10)", "start": 5318.596460309636, "end": 5320.81766099768, "split": "validation", "subset": "af" }, { "name": "Running experiment (9/10)", "start": 5320.818716403679, "end": 5323.327313022688, "split": "validation", "subset": "af" }, { "name": "Scoring", "start": 5294.4625239916495, "end": 5323.327321648656, "split": "validation", "subset": "af" }, { "name": "Encoding training samples", "start": 5326.445629289665, "end": 5330.20296246768, "split": "validation", "subset": "am" }, { "name": "Encoding test samples", "start": 5330.34021340264, "end": 5332.583911190683, "split": "validation", "subset": "am" }, { "name": "Running experiment (0/10)", "start": 5332.584393698664, "end": 5335.487665328663, "split": "validation", "subset": "am" }, { "name": "Running experiment (1/10)", "start": 5335.488066342659, "end": 5338.291141016642, "split": "validation", "subset": "am" }, { "name": "Running experiment (2/10)", "start": 5338.291533134645, "end": 5340.9269780096365, "split": "validation", "subset": "am" }, { "name": "Running experiment (3/10)", "start": 5340.927437744685, "end": 5343.710639431665, "split": "validation", "subset": "am" }, { "name": "Running experiment (4/10)", "start": 5343.711044744647, "end": 5346.102775475651, "split": "validation", "subset": "am" }, { "name": "Running experiment (5/10)", "start": 5346.103182120656, "end": 5348.552118699648, "split": "validation", "subset": "am" }, { "name": "Running experiment (6/10)", "start": 5348.552600846684, "end": 5351.132995153661, "split": "validation", "subset": "am" }, { "name": "Running experiment (7/10)", "start": 5351.133430161688, "end": 5354.214783953677, "split": "validation", "subset": "am" }, { "name": "Running experiment (8/10)", "start": 5354.215199044673, "end": 5357.433192849683, "split": "validation", "subset": "am" }, { "name": "Running experiment (9/10)", "start": 5357.43362857966, "end": 5360.321463279659, "split": "validation", "subset": "am" }, { "name": "Scoring", "start": 5332.583939784672, "end": 5360.321469441638, "split": "validation", "subset": "am" }, { "name": "Encoding training samples", "start": 5363.295870705682, "end": 5366.477333213668, "split": "validation", "subset": "en" }, { "name": "Encoding test samples", "start": 5366.609564995684, "end": 5368.537248795677, "split": "validation", "subset": "en" }, { "name": "Running experiment (0/10)", "start": 5368.53787876165, "end": 5370.893355267646, "split": "validation", "subset": "en" }, { "name": "Running experiment (1/10)", "start": 5370.893772512674, "end": 5374.004039867665, "split": "validation", "subset": "en" }, { "name": "Running experiment (2/10)", "start": 5374.004432606685, "end": 5376.864658060658, "split": "validation", "subset": "en" }, { "name": "Running experiment (3/10)", "start": 5376.865064875688, "end": 5380.3659993156325, "split": "validation", "subset": "en" }, { "name": "Running experiment (4/10)", "start": 5380.366397955688, "end": 5383.651773303689, "split": "validation", "subset": "en" }, { "name": "Running experiment (5/10)", "start": 5383.652156233671, "end": 5386.3706818916835, "split": "validation", "subset": "en" }, { "name": "Running experiment (6/10)", "start": 5386.371312267671, "end": 5388.696612146683, "split": "validation", "subset": "en" }, { "name": "Running experiment (7/10)", "start": 5388.6969917316455, "end": 5391.323528854642, "split": "validation", "subset": "en" }, { "name": "Running experiment (8/10)", "start": 5391.323967299657, "end": 5394.984392108687, "split": "validation", "subset": "en" }, { "name": "Running experiment (9/10)", "start": 5394.984774126671, "end": 5398.288650764676, "split": "validation", "subset": "en" }, { "name": "Scoring", "start": 5368.537273402675, "end": 5398.288659069687, "split": "validation", "subset": "en" }, { "name": "Encoding training samples", "start": 5401.377846342686, "end": 5404.654987172689, "split": "validation", "subset": "pt" }, { "name": "Encoding test samples", "start": 5404.792802812648, "end": 5406.801252785663, "split": "validation", "subset": "pt" }, { "name": "Running experiment (0/10)", "start": 5406.80228405667, "end": 5409.8380764106405, "split": "validation", "subset": "pt" }, { "name": "Running experiment (1/10)", "start": 5409.839059479651, "end": 5413.065179357654, "split": "validation", "subset": "pt" }, { "name": "Running experiment (2/10)", "start": 5413.065847925667, "end": 5416.5880924466765, "split": "validation", "subset": "pt" }, { "name": "Running experiment (3/10)", "start": 5416.588539217657, "end": 5420.182173688663, "split": "validation", "subset": "pt" }, { "name": "Running experiment (4/10)", "start": 5420.182666565641, "end": 5423.952170162636, "split": "validation", "subset": "pt" }, { "name": "Running experiment (5/10)", "start": 5423.9526150986785, "end": 5427.571689716657, "split": "validation", "subset": "pt" }, { "name": "Running experiment (6/10)", "start": 5427.57219219167, "end": 5430.505205343652, "split": "validation", "subset": "pt" }, { "name": "Running experiment (7/10)", "start": 5430.505630914646, "end": 5433.483755949652, "split": "validation", "subset": "pt" }, { "name": "Running experiment (8/10)", "start": 5433.4841613826575, "end": 5436.610790195642, "split": "validation", "subset": "pt" }, { "name": "Running experiment (9/10)", "start": 5436.611198683677, "end": 5439.938118112681, "split": "validation", "subset": "pt" }, { "name": "Scoring", "start": 5406.801274005673, "end": 5439.938148560643, "split": "validation", "subset": "pt" }, { "name": "Encoding training samples", "start": 5442.883463633654, "end": 5446.320032366668, "split": "validation", "subset": "sw" }, { "name": "Encoding test samples", "start": 5446.45383718668, "end": 5448.428152701643, "split": "validation", "subset": "sw" }, { "name": "Running experiment (0/10)", "start": 5448.428682609636, "end": 5451.982491570641, "split": "validation", "subset": "sw" }, { "name": "Running experiment (1/10)", "start": 5451.982891102671, "end": 5454.9892988756765, "split": "validation", "subset": "sw" }, { "name": "Running experiment (2/10)", "start": 5454.989681605657, "end": 5457.2389627356315, "split": "validation", "subset": "sw" }, { "name": "Running experiment (3/10)", "start": 5457.239395670651, "end": 5460.272797838668, "split": "validation", "subset": "sw" }, { "name": "Running experiment (4/10)", "start": 5460.273192189634, "end": 5462.627626728674, "split": "validation", "subset": "sw" }, { "name": "Running experiment (5/10)", "start": 5462.62822290964, "end": 5464.623146593687, "split": "validation", "subset": "sw" }, { "name": "Running experiment (6/10)", "start": 5464.623533451639, "end": 5467.659994154645, "split": "validation", "subset": "sw" }, { "name": "Running experiment (7/10)", "start": 5467.660490929673, "end": 5470.56370195467, "split": "validation", "subset": "sw" }, { "name": "Running experiment (8/10)", "start": 5470.564141361683, "end": 5472.935397322639, "split": "validation", "subset": "sw" }, { "name": "Running experiment (9/10)", "start": 5472.935848731664, "end": 5476.154868907644, "split": "validation", "subset": "sw" }, { "name": "Scoring", "start": 5448.428192527674, "end": 5476.154876722663, "split": "validation", "subset": "sw" }, { "name": "Encoding training samples", "start": 5479.169470887689, "end": 5482.628466447641, "split": "test", "subset": "af" }, { "name": "Encoding test samples", "start": 5482.817777855671, "end": 5485.667440314661, "split": "test", "subset": "af" }, { "name": "Running experiment (0/10)", "start": 5485.667959040671, "end": 5489.451290710655, "split": "test", "subset": "af" }, { "name": "Running experiment (1/10)", "start": 5489.451684481639, "end": 5492.296203480684, "split": "test", "subset": "af" }, { "name": "Running experiment (2/10)", "start": 5492.298170430644, "end": 5495.6369273706805, "split": "test", "subset": "af" }, { "name": "Running experiment (3/10)", "start": 5495.638581431645, "end": 5497.7543553096475, "split": "test", "subset": "af" }, { "name": "Running experiment (4/10)", "start": 5497.7554007856525, "end": 5501.057592940633, "split": "test", "subset": "af" }, { "name": "Running experiment (5/10)", "start": 5501.05883861467, "end": 5504.960235056642, "split": "test", "subset": "af" }, { "name": "Running experiment (6/10)", "start": 5504.961063776689, "end": 5507.800040547678, "split": "test", "subset": "af" }, { "name": "Running experiment (7/10)", "start": 5507.800803562684, "end": 5510.898251362669, "split": "test", "subset": "af" }, { "name": "Running experiment (8/10)", "start": 5510.898858725675, "end": 5513.662945355638, "split": "test", "subset": "af" }, { "name": "Running experiment (9/10)", "start": 5513.663468108687, "end": 5516.9303865476395, "split": "test", "subset": "af" }, { "name": "Scoring", "start": 5485.667459871678, "end": 5516.930392488663, "split": "test", "subset": "af" }, { "name": "Encoding training samples", "start": 5520.021752254688, "end": 5523.780234802689, "split": "test", "subset": "am" }, { "name": "Encoding test samples", "start": 5523.976093038684, "end": 5527.162181530672, "split": "test", "subset": "am" }, { "name": "Running experiment (0/10)", "start": 5527.162730113661, "end": 5530.548104620655, "split": "test", "subset": "am" }, { "name": "Running experiment (1/10)", "start": 5530.548488211643, "end": 5534.233977447671, "split": "test", "subset": "am" }, { "name": "Running experiment (2/10)", "start": 5534.234501774656, "end": 5537.387183563667, "split": "test", "subset": "am" }, { "name": "Running experiment (3/10)", "start": 5537.387682361645, "end": 5539.772299640637, "split": "test", "subset": "am" }, { "name": "Running experiment (4/10)", "start": 5539.772813507647, "end": 5542.406145920686, "split": "test", "subset": "am" }, { "name": "Running experiment (5/10)", "start": 5542.406557916664, "end": 5545.094950331666, "split": "test", "subset": "am" }, { "name": "Running experiment (6/10)", "start": 5545.095352387638, "end": 5548.525515257672, "split": "test", "subset": "am" }, { "name": "Running experiment (7/10)", "start": 5548.525930088654, "end": 5551.263512423669, "split": "test", "subset": "am" }, { "name": "Running experiment (8/10)", "start": 5551.263939456665, "end": 5554.303869748663, "split": "test", "subset": "am" }, { "name": "Running experiment (9/10)", "start": 5554.3042821746785, "end": 5557.166006693675, "split": "test", "subset": "am" }, { "name": "Scoring", "start": 5527.162213170668, "end": 5557.166014007642, "split": "test", "subset": "am" }, { "name": "Encoding training samples", "start": 5560.167955085635, "end": 5563.311039564665, "split": "test", "subset": "en" }, { "name": "Encoding test samples", "start": 5563.5016292656655, "end": 5566.161801178649, "split": "test", "subset": "en" }, { "name": "Running experiment (0/10)", "start": 5566.162338669645, "end": 5569.385481592675, "split": "test", "subset": "en" }, { "name": "Running experiment (1/10)", "start": 5569.386123671662, "end": 5572.9158397516585, "split": "test", "subset": "en" }, { "name": "Running experiment (2/10)", "start": 5572.916234243661, "end": 5576.351623364666, "split": "test", "subset": "en" }, { "name": "Running experiment (3/10)", "start": 5576.353689982672, "end": 5579.582329287659, "split": "test", "subset": "en" }, { "name": "Running experiment (4/10)", "start": 5579.5840803416795, "end": 5583.077304535662, "split": "test", "subset": "en" }, { "name": "Running experiment (5/10)", "start": 5583.07836216467, "end": 5585.191166442644, "split": "test", "subset": "en" }, { "name": "Running experiment (6/10)", "start": 5585.192267863662, "end": 5588.639676979685, "split": "test", "subset": "en" }, { "name": "Running experiment (7/10)", "start": 5588.64056272665, "end": 5591.808161266672, "split": "test", "subset": "en" }, { "name": "Running experiment (8/10)", "start": 5591.808877163683, "end": 5595.25204751367, "split": "test", "subset": "en" }, { "name": "Running experiment (9/10)", "start": 5595.252900819643, "end": 5598.481476584682, "split": "test", "subset": "en" }, { "name": "Scoring", "start": 5566.161821756687, "end": 5598.481492033636, "split": "test", "subset": "en" }, { "name": "Encoding training samples", "start": 5601.524453177641, "end": 5604.781383059686, "split": "test", "subset": "pt" }, { "name": "Encoding test samples", "start": 5604.972909262637, "end": 5607.762131965661, "split": "test", "subset": "pt" }, { "name": "Running experiment (0/10)", "start": 5607.762622227659, "end": 5611.249342013674, "split": "test", "subset": "pt" }, { "name": "Running experiment (1/10)", "start": 5611.249792290677, "end": 5615.157854215649, "split": "test", "subset": "pt" }, { "name": "Running experiment (2/10)", "start": 5615.158297108661, "end": 5618.702074386645, "split": "test", "subset": "pt" }, { "name": "Running experiment (3/10)", "start": 5618.702463117661, "end": 5623.014068674645, "split": "test", "subset": "pt" }, { "name": "Running experiment (4/10)", "start": 5623.014443900669, "end": 5626.962145104655, "split": "test", "subset": "pt" }, { "name": "Running experiment (5/10)", "start": 5626.962567839655, "end": 5630.762379853637, "split": "test", "subset": "pt" }, { "name": "Running experiment (6/10)", "start": 5630.762855117675, "end": 5634.511525904643, "split": "test", "subset": "pt" }, { "name": "Running experiment (7/10)", "start": 5634.511915116687, "end": 5638.12492251466, "split": "test", "subset": "pt" }, { "name": "Running experiment (8/10)", "start": 5638.125328037655, "end": 5641.950872053683, "split": "test", "subset": "pt" }, { "name": "Running experiment (9/10)", "start": 5641.951246607641, "end": 5645.963978250686, "split": "test", "subset": "pt" }, { "name": "Scoring", "start": 5607.762151952658, "end": 5645.963984001661, "split": "test", "subset": "pt" }, { "name": "Encoding training samples", "start": 5649.325075351633, "end": 5652.745494992647, "split": "test", "subset": "sw" }, { "name": "Encoding test samples", "start": 5652.940871426661, "end": 5655.831157278677, "split": "test", "subset": "sw" }, { "name": "Running experiment (0/10)", "start": 5655.831628405664, "end": 5658.701091601688, "split": "test", "subset": "sw" }, { "name": "Running experiment (1/10)", "start": 5658.701488968683, "end": 5661.486539670674, "split": "test", "subset": "sw" }, { "name": "Running experiment (2/10)", "start": 5661.487014965678, "end": 5665.082798698684, "split": "test", "subset": "sw" }, { "name": "Running experiment (3/10)", "start": 5665.084753646632, "end": 5667.985378390644, "split": "test", "subset": "sw" }, { "name": "Running experiment (4/10)", "start": 5667.987139162666, "end": 5670.854158741655, "split": "test", "subset": "sw" }, { "name": "Running experiment (5/10)", "start": 5670.8553733366425, "end": 5673.419497365656, "split": "test", "subset": "sw" }, { "name": "Running experiment (6/10)", "start": 5673.420644182654, "end": 5676.705960299645, "split": "test", "subset": "sw" }, { "name": "Running experiment (7/10)", "start": 5676.706855282653, "end": 5679.70619313966, "split": "test", "subset": "sw" }, { "name": "Running experiment (8/10)", "start": 5679.7069558846415, "end": 5682.345482936653, "split": "test", "subset": "sw" }, { "name": "Running experiment (9/10)", "start": 5682.346075000649, "end": 5685.474616538675, "split": "test", "subset": "sw" }, { "name": "Scoring", "start": 5655.831176394655, "end": 5685.474628230673, "split": "test", "subset": "sw" } ] }