{ "dataset_revision": "ae001d0e6b1228650b7bd1c2c65fb50ad11a8aba", "task_name": "MTOPIntentClassification", "mteb_version": "1.18.0", "scores": { "validation": [ { "accuracy": 0.601566, "f1": 0.418504, "f1_weighted": 0.626789, "scores_per_experiment": [ { "accuracy": 0.565996, "f1": 0.397279, "f1_weighted": 0.598765 }, { "accuracy": 0.602237, "f1": 0.41513, "f1_weighted": 0.62457 }, { "accuracy": 0.608501, "f1": 0.422107, "f1_weighted": 0.63183 }, { "accuracy": 0.612528, "f1": 0.425321, "f1_weighted": 0.633384 }, { "accuracy": 0.617002, "f1": 0.416006, "f1_weighted": 0.647991 }, { "accuracy": 0.589709, "f1": 0.41728, "f1_weighted": 0.618669 }, { "accuracy": 0.595078, "f1": 0.420755, "f1_weighted": 0.622148 }, { "accuracy": 0.611633, "f1": 0.418993, "f1_weighted": 0.636876 }, { "accuracy": 0.599105, "f1": 0.420113, "f1_weighted": 0.617878 }, { "accuracy": 0.61387, "f1": 0.432058, "f1_weighted": 0.635782 } ], "main_score": 0.601566, "hf_subset": "en", "languages": [ "eng-Latn" ] }, { "accuracy": 0.448099, "f1": 0.260266, "f1_weighted": 0.505191, "scores_per_experiment": [ { "accuracy": 0.464463, "f1": 0.263449, "f1_weighted": 0.521426 }, { "accuracy": 0.457851, "f1": 0.264825, "f1_weighted": 0.512197 }, { "accuracy": 0.460055, "f1": 0.277581, "f1_weighted": 0.525807 }, { "accuracy": 0.403306, "f1": 0.246484, "f1_weighted": 0.45412 }, { "accuracy": 0.451791, "f1": 0.245663, "f1_weighted": 0.505171 }, { "accuracy": 0.41157, "f1": 0.243965, "f1_weighted": 0.468629 }, { "accuracy": 0.478788, "f1": 0.262343, "f1_weighted": 0.537091 }, { "accuracy": 0.460055, "f1": 0.283243, "f1_weighted": 0.513476 }, { "accuracy": 0.440771, "f1": 0.252492, "f1_weighted": 0.502163 }, { "accuracy": 0.452342, "f1": 0.262613, "f1_weighted": 0.511835 } ], "main_score": 0.448099, "hf_subset": "de", "languages": [ "deu-Latn" ] }, { "accuracy": 0.361428, "f1": 0.224965, "f1_weighted": 0.387833, "scores_per_experiment": [ { "accuracy": 0.29666, "f1": 0.197992, "f1_weighted": 0.31514 }, { "accuracy": 0.394892, "f1": 0.237153, "f1_weighted": 0.437081 }, { "accuracy": 0.385724, "f1": 0.226452, "f1_weighted": 0.406423 }, { "accuracy": 0.335298, "f1": 0.229197, "f1_weighted": 0.361753 }, { "accuracy": 0.358874, "f1": 0.214962, "f1_weighted": 0.372444 }, { "accuracy": 0.35167, "f1": 0.216158, "f1_weighted": 0.389947 }, { "accuracy": 0.40537, "f1": 0.246638, "f1_weighted": 0.424464 }, { "accuracy": 0.352325, "f1": 0.244255, "f1_weighted": 0.38459 }, { "accuracy": 0.345121, "f1": 0.221295, "f1_weighted": 0.365674 }, { "accuracy": 0.388343, "f1": 0.215542, "f1_weighted": 0.420813 } ], "main_score": 0.361428, "hf_subset": "es", "languages": [ "spa-Latn" ] }, { "accuracy": 0.373494, "f1": 0.238009, "f1_weighted": 0.414752, "scores_per_experiment": [ { "accuracy": 0.372226, "f1": 0.244992, "f1_weighted": 0.431034 }, { "accuracy": 0.31896, "f1": 0.225624, "f1_weighted": 0.351214 }, { "accuracy": 0.415346, "f1": 0.242224, "f1_weighted": 0.4714 }, { "accuracy": 0.407102, "f1": 0.238799, "f1_weighted": 0.441415 }, { "accuracy": 0.395688, "f1": 0.251491, "f1_weighted": 0.439936 }, { "accuracy": 0.422321, "f1": 0.263948, "f1_weighted": 0.473632 }, { "accuracy": 0.3513, "f1": 0.224818, "f1_weighted": 0.389596 }, { "accuracy": 0.336081, "f1": 0.23426, "f1_weighted": 0.368749 }, { "accuracy": 0.398224, "f1": 0.250192, "f1_weighted": 0.439132 }, { "accuracy": 0.317692, "f1": 0.203743, "f1_weighted": 0.34141 } ], "main_score": 0.373494, "hf_subset": "fr", "languages": [ "fra-Latn" ] }, { "accuracy": 0.037575, "f1": 0.010679, "f1_weighted": 0.029092, "scores_per_experiment": [ { "accuracy": 0.017396, "f1": 0.00618, "f1_weighted": 0.010201 }, { "accuracy": 0.068091, "f1": 0.019805, "f1_weighted": 0.059702 }, { "accuracy": 0.04175, "f1": 0.010435, "f1_weighted": 0.027126 }, { "accuracy": 0.013419, "f1": 0.006068, "f1_weighted": 0.004364 }, { "accuracy": 0.040258, "f1": 0.011862, "f1_weighted": 0.036763 }, { "accuracy": 0.044235, "f1": 0.010239, "f1_weighted": 0.029347 }, { "accuracy": 0.057157, "f1": 0.013591, "f1_weighted": 0.052936 }, { "accuracy": 0.034294, "f1": 0.006818, "f1_weighted": 0.020758 }, { "accuracy": 0.028827, "f1": 0.014232, "f1_weighted": 0.03323 }, { "accuracy": 0.030318, "f1": 0.007561, "f1_weighted": 0.016493 } ], "main_score": 0.037575, "hf_subset": "hi", "languages": [ "hin-Deva" ] }, { "accuracy": 0.038659, "f1": 0.012141, "f1_weighted": 0.033883, "scores_per_experiment": [ { "accuracy": 0.014961, "f1": 0.005701, "f1_weighted": 0.015521 }, { "accuracy": 0.056254, "f1": 0.016111, "f1_weighted": 0.039587 }, { "accuracy": 0.016756, "f1": 0.006689, "f1_weighted": 0.013704 }, { "accuracy": 0.056254, "f1": 0.013578, "f1_weighted": 0.035077 }, { "accuracy": 0.014363, "f1": 0.011428, "f1_weighted": 0.015857 }, { "accuracy": 0.043088, "f1": 0.014366, "f1_weighted": 0.047308 }, { "accuracy": 0.073609, "f1": 0.016187, "f1_weighted": 0.076878 }, { "accuracy": 0.031718, "f1": 0.012233, "f1_weighted": 0.033429 }, { "accuracy": 0.055057, "f1": 0.012609, "f1_weighted": 0.039297 }, { "accuracy": 0.024536, "f1": 0.012505, "f1_weighted": 0.02217 } ], "main_score": 0.038659, "hf_subset": "th", "languages": [ "tha-Thai" ] } ], "test": [ { "accuracy": 0.585659, "f1": 0.421039, "f1_weighted": 0.610889, "scores_per_experiment": [ { "accuracy": 0.55814, "f1": 0.39616, "f1_weighted": 0.590014 }, { "accuracy": 0.593707, "f1": 0.410889, "f1_weighted": 0.618698 }, { "accuracy": 0.582535, "f1": 0.427909, "f1_weighted": 0.609285 }, { "accuracy": 0.603055, "f1": 0.424047, "f1_weighted": 0.621469 }, { "accuracy": 0.603283, "f1": 0.437687, "f1_weighted": 0.633875 }, { "accuracy": 0.565663, "f1": 0.40679, "f1_weighted": 0.590013 }, { "accuracy": 0.581167, "f1": 0.423922, "f1_weighted": 0.607485 }, { "accuracy": 0.578887, "f1": 0.429257, "f1_weighted": 0.603507 }, { "accuracy": 0.595303, "f1": 0.429175, "f1_weighted": 0.619222 }, { "accuracy": 0.594847, "f1": 0.424559, "f1_weighted": 0.615319 } ], "main_score": 0.585659, "hf_subset": "en", "languages": [ "eng-Latn" ] }, { "accuracy": 0.454945, "f1": 0.278385, "f1_weighted": 0.510407, "scores_per_experiment": [ { "accuracy": 0.4745, "f1": 0.29095, "f1_weighted": 0.534401 }, { "accuracy": 0.465765, "f1": 0.290832, "f1_weighted": 0.514282 }, { "accuracy": 0.452804, "f1": 0.26858, "f1_weighted": 0.513214 }, { "accuracy": 0.413356, "f1": 0.279523, "f1_weighted": 0.460469 }, { "accuracy": 0.45703, "f1": 0.265372, "f1_weighted": 0.512883 }, { "accuracy": 0.4204, "f1": 0.282225, "f1_weighted": 0.480475 }, { "accuracy": 0.497605, "f1": 0.281286, "f1_weighted": 0.553322 }, { "accuracy": 0.457875, "f1": 0.272568, "f1_weighted": 0.508735 }, { "accuracy": 0.444914, "f1": 0.269067, "f1_weighted": 0.501497 }, { "accuracy": 0.465201, "f1": 0.283443, "f1_weighted": 0.524792 } ], "main_score": 0.454945, "hf_subset": "de", "languages": [ "deu-Latn" ] }, { "accuracy": 0.355237, "f1": 0.222585, "f1_weighted": 0.379417, "scores_per_experiment": [ { "accuracy": 0.285857, "f1": 0.210892, "f1_weighted": 0.301947 }, { "accuracy": 0.375584, "f1": 0.221885, "f1_weighted": 0.406897 }, { "accuracy": 0.389927, "f1": 0.236147, "f1_weighted": 0.40794 }, { "accuracy": 0.323549, "f1": 0.196269, "f1_weighted": 0.354739 }, { "accuracy": 0.364243, "f1": 0.220847, "f1_weighted": 0.374952 }, { "accuracy": 0.351234, "f1": 0.226423, "f1_weighted": 0.385325 }, { "accuracy": 0.414276, "f1": 0.245677, "f1_weighted": 0.43858 }, { "accuracy": 0.334556, "f1": 0.225439, "f1_weighted": 0.362573 }, { "accuracy": 0.343896, "f1": 0.224822, "f1_weighted": 0.360558 }, { "accuracy": 0.369246, "f1": 0.217443, "f1_weighted": 0.400662 } ], "main_score": 0.355237, "hf_subset": "es", "languages": [ "spa-Latn" ] }, { "accuracy": 0.366395, "f1": 0.242859, "f1_weighted": 0.401344, "scores_per_experiment": [ { "accuracy": 0.362042, "f1": 0.243467, "f1_weighted": 0.410002 }, { "accuracy": 0.318196, "f1": 0.232076, "f1_weighted": 0.34479 }, { "accuracy": 0.406201, "f1": 0.237554, "f1_weighted": 0.453743 }, { "accuracy": 0.384904, "f1": 0.249825, "f1_weighted": 0.410609 }, { "accuracy": 0.390542, "f1": 0.240072, "f1_weighted": 0.434772 }, { "accuracy": 0.420608, "f1": 0.271017, "f1_weighted": 0.472042 }, { "accuracy": 0.354526, "f1": 0.236616, "f1_weighted": 0.379575 }, { "accuracy": 0.337614, "f1": 0.24849, "f1_weighted": 0.361962 }, { "accuracy": 0.381773, "f1": 0.257122, "f1_weighted": 0.415915 }, { "accuracy": 0.307548, "f1": 0.21235, "f1_weighted": 0.330031 } ], "main_score": 0.366395, "hf_subset": "fr", "languages": [ "fra-Latn" ] }, { "accuracy": 0.037469, "f1": 0.011804, "f1_weighted": 0.026643, "scores_per_experiment": [ { "accuracy": 0.017569, "f1": 0.014638, "f1_weighted": 0.009312 }, { "accuracy": 0.063464, "f1": 0.019347, "f1_weighted": 0.050239 }, { "accuracy": 0.045536, "f1": 0.015833, "f1_weighted": 0.026129 }, { "accuracy": 0.018645, "f1": 0.006541, "f1_weighted": 0.007443 }, { "accuracy": 0.032987, "f1": 0.008329, "f1_weighted": 0.030337 }, { "accuracy": 0.050556, "f1": 0.015107, "f1_weighted": 0.03302 }, { "accuracy": 0.050197, "f1": 0.009979, "f1_weighted": 0.048579 }, { "accuracy": 0.036931, "f1": 0.007135, "f1_weighted": 0.016699 }, { "accuracy": 0.027608, "f1": 0.015621, "f1_weighted": 0.02994 }, { "accuracy": 0.031194, "f1": 0.005513, "f1_weighted": 0.014734 } ], "main_score": 0.037469, "hf_subset": "hi", "languages": [ "hin-Deva" ] }, { "accuracy": 0.045136, "f1": 0.012926, "f1_weighted": 0.03557, "scores_per_experiment": [ { "accuracy": 0.02604, "f1": 0.008742, "f1_weighted": 0.021589 }, { "accuracy": 0.066546, "f1": 0.016753, "f1_weighted": 0.041065 }, { "accuracy": 0.021338, "f1": 0.018165, "f1_weighted": 0.012252 }, { "accuracy": 0.066184, "f1": 0.016805, "f1_weighted": 0.037308 }, { "accuracy": 0.019168, "f1": 0.010249, "f1_weighted": 0.015913 }, { "accuracy": 0.039421, "f1": 0.009912, "f1_weighted": 0.041163 }, { "accuracy": 0.080651, "f1": 0.014761, "f1_weighted": 0.082978 }, { "accuracy": 0.041953, "f1": 0.010408, "f1_weighted": 0.040051 }, { "accuracy": 0.058951, "f1": 0.016011, "f1_weighted": 0.038215 }, { "accuracy": 0.031103, "f1": 0.00745, "f1_weighted": 0.025165 } ], "main_score": 0.045136, "hf_subset": "th", "languages": [ "tha-Thai" ] } ] }, "evaluation_time": 176.90019345283508, "kg_co2_emissions": 0.005140753979208663 }