{ "dataset_revision": "d80d48c1eb48d3562165c59d59d0034df9fff0bf", "task_name": "MTOPDomainClassification", "mteb_version": "1.18.0", "scores": { "validation": [ { "accuracy": 0.905727, "f1": 0.905524, "f1_weighted": 0.905428, "scores_per_experiment": [ { "accuracy": 0.910962, "f1": 0.907622, "f1_weighted": 0.910839 }, { "accuracy": 0.909172, "f1": 0.910908, "f1_weighted": 0.90889 }, { "accuracy": 0.895749, "f1": 0.895716, "f1_weighted": 0.8955 }, { "accuracy": 0.915884, "f1": 0.915971, "f1_weighted": 0.915543 }, { "accuracy": 0.917226, "f1": 0.915326, "f1_weighted": 0.916974 }, { "accuracy": 0.904698, "f1": 0.903138, "f1_weighted": 0.903642 }, { "accuracy": 0.898881, "f1": 0.900118, "f1_weighted": 0.898156 }, { "accuracy": 0.902013, "f1": 0.902141, "f1_weighted": 0.901923 }, { "accuracy": 0.900671, "f1": 0.902097, "f1_weighted": 0.900401 }, { "accuracy": 0.902013, "f1": 0.9022, "f1_weighted": 0.902409 } ], "main_score": 0.905727, "hf_subset": "en", "languages": [ "eng-Latn" ] }, { "accuracy": 0.817521, "f1": 0.805651, "f1_weighted": 0.819529, "scores_per_experiment": [ { "accuracy": 0.819284, "f1": 0.808051, "f1_weighted": 0.821009 }, { "accuracy": 0.777961, "f1": 0.772104, "f1_weighted": 0.77873 }, { "accuracy": 0.812121, "f1": 0.793115, "f1_weighted": 0.812058 }, { "accuracy": 0.830303, "f1": 0.816332, "f1_weighted": 0.830894 }, { "accuracy": 0.811019, "f1": 0.793389, "f1_weighted": 0.813873 }, { "accuracy": 0.814876, "f1": 0.80358, "f1_weighted": 0.814336 }, { "accuracy": 0.843526, "f1": 0.831815, "f1_weighted": 0.846366 }, { "accuracy": 0.791185, "f1": 0.78392, "f1_weighted": 0.798514 }, { "accuracy": 0.833609, "f1": 0.824911, "f1_weighted": 0.836443 }, { "accuracy": 0.841322, "f1": 0.829291, "f1_weighted": 0.843063 } ], "main_score": 0.817521, "hf_subset": "de", "languages": [ "deu-Latn" ] }, { "accuracy": 0.849705, "f1": 0.851768, "f1_weighted": 0.849208, "scores_per_experiment": [ { "accuracy": 0.861166, "f1": 0.86292, "f1_weighted": 0.860433 }, { "accuracy": 0.826457, "f1": 0.831407, "f1_weighted": 0.824403 }, { "accuracy": 0.859856, "f1": 0.865878, "f1_weighted": 0.860141 }, { "accuracy": 0.838245, "f1": 0.838077, "f1_weighted": 0.838208 }, { "accuracy": 0.841519, "f1": 0.844691, "f1_weighted": 0.840464 }, { "accuracy": 0.869679, "f1": 0.874141, "f1_weighted": 0.869036 }, { "accuracy": 0.851343, "f1": 0.853524, "f1_weighted": 0.850633 }, { "accuracy": 0.861166, "f1": 0.864502, "f1_weighted": 0.861794 }, { "accuracy": 0.835625, "f1": 0.835008, "f1_weighted": 0.835641 }, { "accuracy": 0.851997, "f1": 0.847533, "f1_weighted": 0.851328 } ], "main_score": 0.849705, "hf_subset": "es", "languages": [ "spa-Latn" ] }, { "accuracy": 0.882815, "f1": 0.882618, "f1_weighted": 0.882549, "scores_per_experiment": [ { "accuracy": 0.871909, "f1": 0.872487, "f1_weighted": 0.871019 }, { "accuracy": 0.882055, "f1": 0.882099, "f1_weighted": 0.882484 }, { "accuracy": 0.875713, "f1": 0.875774, "f1_weighted": 0.875256 }, { "accuracy": 0.882689, "f1": 0.88277, "f1_weighted": 0.881741 }, { "accuracy": 0.883957, "f1": 0.881889, "f1_weighted": 0.884288 }, { "accuracy": 0.90298, "f1": 0.904308, "f1_weighted": 0.90264 }, { "accuracy": 0.897907, "f1": 0.894619, "f1_weighted": 0.89812 }, { "accuracy": 0.87825, "f1": 0.88314, "f1_weighted": 0.877509 }, { "accuracy": 0.884591, "f1": 0.886932, "f1_weighted": 0.884674 }, { "accuracy": 0.868104, "f1": 0.862164, "f1_weighted": 0.867757 } ], "main_score": 0.882815, "hf_subset": "fr", "languages": [ "fra-Latn" ] }, { "accuracy": 0.538469, "f1": 0.530621, "f1_weighted": 0.539968, "scores_per_experiment": [ { "accuracy": 0.542744, "f1": 0.536203, "f1_weighted": 0.544835 }, { "accuracy": 0.549702, "f1": 0.529146, "f1_weighted": 0.542377 }, { "accuracy": 0.539761, "f1": 0.543241, "f1_weighted": 0.547391 }, { "accuracy": 0.528827, "f1": 0.518914, "f1_weighted": 0.52935 }, { "accuracy": 0.523857, "f1": 0.526227, "f1_weighted": 0.530919 }, { "accuracy": 0.508946, "f1": 0.498524, "f1_weighted": 0.51122 }, { "accuracy": 0.550696, "f1": 0.544861, "f1_weighted": 0.549147 }, { "accuracy": 0.535288, "f1": 0.519466, "f1_weighted": 0.533289 }, { "accuracy": 0.551193, "f1": 0.548238, "f1_weighted": 0.561526 }, { "accuracy": 0.553678, "f1": 0.541391, "f1_weighted": 0.549621 } ], "main_score": 0.538469, "hf_subset": "hi", "languages": [ "hin-Deva" ] }, { "accuracy": 0.422801, "f1": 0.411153, "f1_weighted": 0.425479, "scores_per_experiment": [ { "accuracy": 0.456014, "f1": 0.439041, "f1_weighted": 0.450043 }, { "accuracy": 0.442849, "f1": 0.432648, "f1_weighted": 0.453846 }, { "accuracy": 0.381209, "f1": 0.370621, "f1_weighted": 0.384211 }, { "accuracy": 0.439856, "f1": 0.428452, "f1_weighted": 0.441339 }, { "accuracy": 0.415919, "f1": 0.408334, "f1_weighted": 0.4244 }, { "accuracy": 0.446439, "f1": 0.441192, "f1_weighted": 0.454083 }, { "accuracy": 0.38839, "f1": 0.376072, "f1_weighted": 0.390436 }, { "accuracy": 0.400958, "f1": 0.385882, "f1_weighted": 0.403639 }, { "accuracy": 0.419509, "f1": 0.406937, "f1_weighted": 0.424123 }, { "accuracy": 0.436864, "f1": 0.422357, "f1_weighted": 0.428667 } ], "main_score": 0.422801, "hf_subset": "th", "languages": [ "tha-Thai" ] } ], "test": [ { "accuracy": 0.900638, "f1": 0.898637, "f1_weighted": 0.900739, "scores_per_experiment": [ { "accuracy": 0.899681, "f1": 0.896426, "f1_weighted": 0.899642 }, { "accuracy": 0.912221, "f1": 0.911539, "f1_weighted": 0.912292 }, { "accuracy": 0.897629, "f1": 0.89546, "f1_weighted": 0.897874 }, { "accuracy": 0.914045, "f1": 0.910286, "f1_weighted": 0.914551 }, { "accuracy": 0.904013, "f1": 0.900907, "f1_weighted": 0.904581 }, { "accuracy": 0.906749, "f1": 0.902905, "f1_weighted": 0.905872 }, { "accuracy": 0.889421, "f1": 0.89004, "f1_weighted": 0.888682 }, { "accuracy": 0.901733, "f1": 0.898884, "f1_weighted": 0.901945 }, { "accuracy": 0.892613, "f1": 0.893681, "f1_weighted": 0.892386 }, { "accuracy": 0.888281, "f1": 0.886243, "f1_weighted": 0.889561 } ], "main_score": 0.900638, "hf_subset": "en", "languages": [ "eng-Latn" ] }, { "accuracy": 0.809073, "f1": 0.796323, "f1_weighted": 0.81201, "scores_per_experiment": [ { "accuracy": 0.813187, "f1": 0.804981, "f1_weighted": 0.815462 }, { "accuracy": 0.774866, "f1": 0.76626, "f1_weighted": 0.774605 }, { "accuracy": 0.809806, "f1": 0.792019, "f1_weighted": 0.810712 }, { "accuracy": 0.808397, "f1": 0.791998, "f1_weighted": 0.811372 }, { "accuracy": 0.805297, "f1": 0.787246, "f1_weighted": 0.808987 }, { "accuracy": 0.81854, "f1": 0.805561, "f1_weighted": 0.819055 }, { "accuracy": 0.81854, "f1": 0.803391, "f1_weighted": 0.822655 }, { "accuracy": 0.779374, "f1": 0.771337, "f1_weighted": 0.789067 }, { "accuracy": 0.823049, "f1": 0.814811, "f1_weighted": 0.825665 }, { "accuracy": 0.839673, "f1": 0.825629, "f1_weighted": 0.84252 } ], "main_score": 0.809073, "hf_subset": "de", "languages": [ "deu-Latn" ] }, { "accuracy": 0.836458, "f1": 0.835556, "f1_weighted": 0.836356, "scores_per_experiment": [ { "accuracy": 0.852235, "f1": 0.850733, "f1_weighted": 0.852039 }, { "accuracy": 0.822882, "f1": 0.824673, "f1_weighted": 0.820981 }, { "accuracy": 0.848232, "f1": 0.848407, "f1_weighted": 0.848996 }, { "accuracy": 0.832221, "f1": 0.828082, "f1_weighted": 0.832228 }, { "accuracy": 0.828886, "f1": 0.826356, "f1_weighted": 0.82945 }, { "accuracy": 0.852235, "f1": 0.853711, "f1_weighted": 0.851798 }, { "accuracy": 0.829219, "f1": 0.829176, "f1_weighted": 0.828249 }, { "accuracy": 0.83489, "f1": 0.83778, "f1_weighted": 0.835695 }, { "accuracy": 0.829887, "f1": 0.82983, "f1_weighted": 0.83022 }, { "accuracy": 0.833889, "f1": 0.826811, "f1_weighted": 0.833901 } ], "main_score": 0.836458, "hf_subset": "es", "languages": [ "spa-Latn" ] }, { "accuracy": 0.88409, "f1": 0.882255, "f1_weighted": 0.884412, "scores_per_experiment": [ { "accuracy": 0.88005, "f1": 0.877325, "f1_weighted": 0.879827 }, { "accuracy": 0.888819, "f1": 0.88518, "f1_weighted": 0.889851 }, { "accuracy": 0.883808, "f1": 0.886344, "f1_weighted": 0.883833 }, { "accuracy": 0.88005, "f1": 0.881067, "f1_weighted": 0.879785 }, { "accuracy": 0.876918, "f1": 0.878266, "f1_weighted": 0.877368 }, { "accuracy": 0.900094, "f1": 0.897241, "f1_weighted": 0.90001 }, { "accuracy": 0.892264, "f1": 0.886859, "f1_weighted": 0.893315 }, { "accuracy": 0.885374, "f1": 0.887957, "f1_weighted": 0.884834 }, { "accuracy": 0.887253, "f1": 0.885346, "f1_weighted": 0.887981 }, { "accuracy": 0.86627, "f1": 0.856967, "f1_weighted": 0.867314 } ], "main_score": 0.88409, "hf_subset": "fr", "languages": [ "fra-Latn" ] }, { "accuracy": 0.54593, "f1": 0.527505, "f1_weighted": 0.550073, "scores_per_experiment": [ { "accuracy": 0.541413, "f1": 0.526704, "f1_weighted": 0.547193 }, { "accuracy": 0.560774, "f1": 0.527914, "f1_weighted": 0.554481 }, { "accuracy": 0.573324, "f1": 0.561084, "f1_weighted": 0.580886 }, { "accuracy": 0.530656, "f1": 0.505286, "f1_weighted": 0.528738 }, { "accuracy": 0.528863, "f1": 0.51976, "f1_weighted": 0.538497 }, { "accuracy": 0.528505, "f1": 0.511948, "f1_weighted": 0.535754 }, { "accuracy": 0.55683, "f1": 0.539875, "f1_weighted": 0.563427 }, { "accuracy": 0.541054, "f1": 0.514496, "f1_weighted": 0.538272 }, { "accuracy": 0.53962, "f1": 0.528839, "f1_weighted": 0.551137 }, { "accuracy": 0.558265, "f1": 0.539145, "f1_weighted": 0.56235 } ], "main_score": 0.54593, "hf_subset": "hi", "languages": [ "hin-Deva" ] }, { "accuracy": 0.431899, "f1": 0.413205, "f1_weighted": 0.43801, "scores_per_experiment": [ { "accuracy": 0.444846, "f1": 0.419737, "f1_weighted": 0.44142 }, { "accuracy": 0.465099, "f1": 0.447372, "f1_weighted": 0.478135 }, { "accuracy": 0.396745, "f1": 0.386373, "f1_weighted": 0.401191 }, { "accuracy": 0.461844, "f1": 0.439907, "f1_weighted": 0.467529 }, { "accuracy": 0.420253, "f1": 0.410948, "f1_weighted": 0.432906 }, { "accuracy": 0.469439, "f1": 0.453006, "f1_weighted": 0.483262 }, { "accuracy": 0.396383, "f1": 0.380788, "f1_weighted": 0.401601 }, { "accuracy": 0.414105, "f1": 0.391247, "f1_weighted": 0.421087 }, { "accuracy": 0.424231, "f1": 0.402147, "f1_weighted": 0.433073 }, { "accuracy": 0.42604, "f1": 0.400523, "f1_weighted": 0.4199 } ], "main_score": 0.431899, "hf_subset": "th", "languages": [ "tha-Thai" ] } ] }, "evaluation_time": 207.9346740245819, "kg_co2_emissions": 0.009971172218411203 }