{ "dataset_revision": "8a04d940a42cd40658986fdd8e3da561533a3646", "task_name": "PawsXPairClassification", "mteb_version": "1.18.0", "scores": { "test": [ { "similarity_accuracy": 0.5755, "similarity_accuracy_threshold": 0.995312, "similarity_f1": 0.622905, "similarity_f1_threshold": 0.749775, "similarity_precision": 0.453022, "similarity_recall": 0.996648, "similarity_ap": 0.510274, "cosine_accuracy": 0.5755, "cosine_accuracy_threshold": 0.995312, "cosine_f1": 0.622905, "cosine_f1_threshold": 0.749775, "cosine_precision": 0.453022, "cosine_recall": 0.996648, "cosine_ap": 0.510939, "manhattan_accuracy": 0.575, "manhattan_accuracy_threshold": 1.523386, "manhattan_f1": 0.623031, "manhattan_f1_threshold": 8.854254, "manhattan_precision": 0.453619, "manhattan_recall": 0.994413, "manhattan_ap": 0.511616, "euclidean_accuracy": 0.5755, "euclidean_accuracy_threshold": 0.096829, "euclidean_f1": 0.622905, "euclidean_f1_threshold": 0.707416, "euclidean_precision": 0.453022, "euclidean_recall": 0.996648, "euclidean_ap": 0.510939, "dot_accuracy": 0.5755, "dot_accuracy_threshold": 0.995312, "dot_f1": 0.622905, "dot_f1_threshold": 0.749775, "dot_precision": 0.453022, "dot_recall": 0.996648, "dot_ap": 0.509184, "max_accuracy": 0.5755, "max_f1": 0.623031, "max_precision": 0.453619, "max_recall": 0.996648, "max_ap": 0.511616, "main_score": 0.511616, "hf_subset": "de", "languages": [ "deu-Latn" ] }, { "similarity_accuracy": 0.613, "similarity_accuracy_threshold": 0.997776, "similarity_f1": 0.624656, "similarity_f1_threshold": 0.898008, "similarity_precision": 0.454181, "similarity_recall": 1.0, "similarity_ap": 0.599927, "cosine_accuracy": 0.613, "cosine_accuracy_threshold": 0.997776, "cosine_f1": 0.624656, "cosine_f1_threshold": 0.898008, "cosine_precision": 0.454181, "cosine_recall": 1.0, "cosine_ap": 0.59992, "manhattan_accuracy": 0.6125, "manhattan_accuracy_threshold": 1.315934, "manhattan_f1": 0.624871, "manhattan_f1_threshold": 6.703775, "manhattan_precision": 0.454409, "manhattan_recall": 1.0, "manhattan_ap": 0.599398, "euclidean_accuracy": 0.613, "euclidean_accuracy_threshold": 0.0667, "euclidean_f1": 0.624656, "euclidean_f1_threshold": 0.451436, "euclidean_precision": 0.454181, "euclidean_recall": 1.0, "euclidean_ap": 0.599919, "dot_accuracy": 0.613, "dot_accuracy_threshold": 0.997776, "dot_f1": 0.624656, "dot_f1_threshold": 0.898008, "dot_precision": 0.454181, "dot_recall": 1.0, "dot_ap": 0.599923, "max_accuracy": 0.613, "max_f1": 0.624871, "max_precision": 0.454409, "max_recall": 1.0, "max_ap": 0.599927, "main_score": 0.599927, "hf_subset": "en", "languages": [ "eng-Latn" ] }, { "similarity_accuracy": 0.575, "similarity_accuracy_threshold": 0.996646, "similarity_f1": 0.624441, "similarity_f1_threshold": 0.83103, "similarity_precision": 0.453954, "similarity_recall": 1.0, "similarity_ap": 0.526792, "cosine_accuracy": 0.575, "cosine_accuracy_threshold": 0.996646, "cosine_f1": 0.624441, "cosine_f1_threshold": 0.83103, "cosine_precision": 0.453954, "cosine_recall": 1.0, "cosine_ap": 0.52679, "manhattan_accuracy": 0.5755, "manhattan_accuracy_threshold": 1.306165, "manhattan_f1": 0.624441, "manhattan_f1_threshold": 9.257932, "manhattan_precision": 0.453954, "manhattan_recall": 1.0, "manhattan_ap": 0.526671, "euclidean_accuracy": 0.575, "euclidean_accuracy_threshold": 0.081904, "euclidean_f1": 0.624441, "euclidean_f1_threshold": 0.580918, "euclidean_precision": 0.453954, "euclidean_recall": 1.0, "euclidean_ap": 0.52679, "dot_accuracy": 0.575, "dot_accuracy_threshold": 0.996646, "dot_f1": 0.624441, "dot_f1_threshold": 0.83103, "dot_precision": 0.453954, "dot_recall": 1.0, "dot_ap": 0.526788, "max_accuracy": 0.5755, "max_f1": 0.624441, "max_precision": 0.453954, "max_recall": 1.0, "max_ap": 0.526792, "main_score": 0.526792, "hf_subset": "es", "languages": [ "spa-Latn" ] }, { "similarity_accuracy": 0.587, "similarity_accuracy_threshold": 0.991722, "similarity_f1": 0.624913, "similarity_f1_threshold": 0.793979, "similarity_precision": 0.454454, "similarity_recall": 1.0, "similarity_ap": 0.547397, "cosine_accuracy": 0.587, "cosine_accuracy_threshold": 0.991722, "cosine_f1": 0.624913, "cosine_f1_threshold": 0.793979, "cosine_precision": 0.454454, "cosine_recall": 1.0, "cosine_ap": 0.547887, "manhattan_accuracy": 0.5875, "manhattan_accuracy_threshold": 1.900486, "manhattan_f1": 0.624913, "manhattan_f1_threshold": 9.934288, "manhattan_precision": 0.454454, "manhattan_recall": 1.0, "manhattan_ap": 0.548621, "euclidean_accuracy": 0.587, "euclidean_accuracy_threshold": 0.128674, "euclidean_f1": 0.624913, "euclidean_f1_threshold": 0.640954, "euclidean_precision": 0.454454, "euclidean_recall": 1.0, "euclidean_ap": 0.547889, "dot_accuracy": 0.587, "dot_accuracy_threshold": 0.991721, "dot_f1": 0.624913, "dot_f1_threshold": 0.793979, "dot_precision": 0.454454, "dot_recall": 1.0, "dot_ap": 0.547643, "max_accuracy": 0.5875, "max_f1": 0.624913, "max_precision": 0.454454, "max_recall": 1.0, "max_ap": 0.548621, "main_score": 0.548621, "hf_subset": "fr", "languages": [ "fra-Latn" ] }, { "similarity_accuracy": 0.571, "similarity_accuracy_threshold": 0.997291, "similarity_f1": 0.624601, "similarity_f1_threshold": 0.795385, "similarity_precision": 0.454592, "similarity_recall": 0.997735, "similarity_ap": 0.48799, "cosine_accuracy": 0.571, "cosine_accuracy_threshold": 0.997291, "cosine_f1": 0.624601, "cosine_f1_threshold": 0.795385, "cosine_precision": 0.454592, "cosine_recall": 0.997735, "cosine_ap": 0.488013, "manhattan_accuracy": 0.571, "manhattan_accuracy_threshold": 1.143143, "manhattan_f1": 0.62429, "manhattan_f1_threshold": 9.574088, "manhattan_precision": 0.454734, "manhattan_recall": 0.99547, "manhattan_ap": 0.488412, "euclidean_accuracy": 0.571, "euclidean_accuracy_threshold": 0.073598, "euclidean_f1": 0.624601, "euclidean_f1_threshold": 0.63971, "euclidean_precision": 0.454592, "euclidean_recall": 0.997735, "euclidean_ap": 0.488013, "dot_accuracy": 0.571, "dot_accuracy_threshold": 0.997291, "dot_f1": 0.624601, "dot_f1_threshold": 0.795385, "dot_precision": 0.454592, "dot_recall": 0.997735, "dot_ap": 0.488806, "max_accuracy": 0.571, "max_f1": 0.624601, "max_precision": 0.454734, "max_recall": 0.997735, "max_ap": 0.488806, "main_score": 0.488806, "hf_subset": "ja", "languages": [ "jpn-Hira" ] }, { "similarity_accuracy": 0.5755, "similarity_accuracy_threshold": 0.994194, "similarity_f1": 0.622669, "similarity_f1_threshold": 0.762456, "similarity_precision": 0.458774, "similarity_recall": 0.96875, "similarity_ap": 0.512201, "cosine_accuracy": 0.5755, "cosine_accuracy_threshold": 0.994194, "cosine_f1": 0.622669, "cosine_f1_threshold": 0.762456, "cosine_precision": 0.458774, "cosine_recall": 0.96875, "cosine_ap": 0.512427, "manhattan_accuracy": 0.574, "manhattan_accuracy_threshold": 1.662139, "manhattan_f1": 0.621536, "manhattan_f1_threshold": 11.450926, "manhattan_precision": 0.453197, "manhattan_recall": 0.988839, "manhattan_ap": 0.51251, "euclidean_accuracy": 0.5755, "euclidean_accuracy_threshold": 0.107757, "euclidean_f1": 0.622669, "euclidean_f1_threshold": 0.689265, "euclidean_precision": 0.458774, "euclidean_recall": 0.96875, "euclidean_ap": 0.512427, "dot_accuracy": 0.5755, "dot_accuracy_threshold": 0.994194, "dot_f1": 0.622669, "dot_f1_threshold": 0.762456, "dot_precision": 0.458774, "dot_recall": 0.96875, "dot_ap": 0.513547, "max_accuracy": 0.5755, "max_f1": 0.622669, "max_precision": 0.458774, "max_recall": 0.988839, "max_ap": 0.513547, "main_score": 0.513547, "hf_subset": "ko", "languages": [ "kor-Hang" ] }, { "similarity_accuracy": 0.591, "similarity_accuracy_threshold": 0.998855, "similarity_f1": 0.622129, "similarity_f1_threshold": 0.741714, "similarity_precision": 0.451515, "similarity_recall": 1.0, "similarity_ap": 0.537755, "cosine_accuracy": 0.591, "cosine_accuracy_threshold": 0.998854, "cosine_f1": 0.622129, "cosine_f1_threshold": 0.741714, "cosine_precision": 0.451515, "cosine_recall": 1.0, "cosine_ap": 0.536186, "manhattan_accuracy": 0.591, "manhattan_accuracy_threshold": 0.745728, "manhattan_f1": 0.622129, "manhattan_f1_threshold": 10.935993, "manhattan_precision": 0.451515, "manhattan_recall": 1.0, "manhattan_ap": 0.536517, "euclidean_accuracy": 0.591, "euclidean_accuracy_threshold": 0.047865, "euclidean_f1": 0.622129, "euclidean_f1_threshold": 0.718724, "euclidean_precision": 0.451515, "euclidean_recall": 1.0, "euclidean_ap": 0.536185, "dot_accuracy": 0.591, "dot_accuracy_threshold": 0.998855, "dot_f1": 0.622129, "dot_f1_threshold": 0.741714, "dot_precision": 0.451515, "dot_recall": 1.0, "dot_ap": 0.534632, "max_accuracy": 0.591, "max_f1": 0.622129, "max_precision": 0.451515, "max_recall": 1.0, "max_ap": 0.537755, "main_score": 0.537755, "hf_subset": "zh", "languages": [ "cmn-Hans" ] } ], "validation": [ { "similarity_accuracy": 0.599, "similarity_accuracy_threshold": 0.99788, "similarity_f1": 0.598919, "similarity_f1_threshold": 0.777573, "similarity_precision": 0.427469, "similarity_recall": 1.0, "similarity_ap": 0.475662, "cosine_accuracy": 0.599, "cosine_accuracy_threshold": 0.99788, "cosine_f1": 0.598919, "cosine_f1_threshold": 0.777573, "cosine_precision": 0.427469, "cosine_recall": 1.0, "cosine_ap": 0.472176, "manhattan_accuracy": 0.5985, "manhattan_accuracy_threshold": 0.72621, "manhattan_f1": 0.598919, "manhattan_f1_threshold": 10.306824, "manhattan_precision": 0.427469, "manhattan_recall": 1.0, "manhattan_ap": 0.472638, "euclidean_accuracy": 0.599, "euclidean_accuracy_threshold": 0.065118, "euclidean_f1": 0.598919, "euclidean_f1_threshold": 0.666916, "euclidean_precision": 0.427469, "euclidean_recall": 1.0, "euclidean_ap": 0.472177, "dot_accuracy": 0.599, "dot_accuracy_threshold": 0.99788, "dot_f1": 0.598919, "dot_f1_threshold": 0.777573, "dot_precision": 0.427469, "dot_recall": 1.0, "dot_ap": 0.474714, "max_accuracy": 0.599, "max_f1": 0.598919, "max_precision": 0.427469, "max_recall": 1.0, "max_ap": 0.475662, "main_score": 0.475662, "hf_subset": "de", "languages": [ "deu-Latn" ] }, { "similarity_accuracy": 0.62, "similarity_accuracy_threshold": 0.996843, "similarity_f1": 0.618944, "similarity_f1_threshold": 0.987467, "similarity_precision": 0.496499, "similarity_recall": 0.821553, "similarity_ap": 0.576784, "cosine_accuracy": 0.62, "cosine_accuracy_threshold": 0.996843, "cosine_f1": 0.618944, "cosine_f1_threshold": 0.987467, "cosine_precision": 0.496499, "cosine_recall": 0.821553, "cosine_ap": 0.576791, "manhattan_accuracy": 0.62, "manhattan_accuracy_threshold": 1.244105, "manhattan_f1": 0.618357, "manhattan_f1_threshold": 2.432063, "manhattan_precision": 0.497878, "manhattan_recall": 0.815759, "manhattan_ap": 0.577491, "euclidean_accuracy": 0.62, "euclidean_accuracy_threshold": 0.079462, "euclidean_f1": 0.618944, "euclidean_f1_threshold": 0.158322, "euclidean_precision": 0.496499, "euclidean_recall": 0.821553, "euclidean_ap": 0.576791, "dot_accuracy": 0.62, "dot_accuracy_threshold": 0.996843, "dot_f1": 0.618944, "dot_f1_threshold": 0.987467, "dot_precision": 0.496499, "dot_recall": 0.821553, "dot_ap": 0.5768, "max_accuracy": 0.62, "max_f1": 0.618944, "max_precision": 0.497878, "max_recall": 0.821553, "max_ap": 0.577491, "main_score": 0.577491, "hf_subset": "en", "languages": [ "eng-Latn" ] }, { "similarity_accuracy": 0.5985, "similarity_accuracy_threshold": 0.997422, "similarity_f1": 0.603639, "similarity_f1_threshold": 0.855369, "similarity_precision": 0.432515, "similarity_recall": 0.998819, "similarity_ap": 0.500464, "cosine_accuracy": 0.5985, "cosine_accuracy_threshold": 0.997422, "cosine_f1": 0.603639, "cosine_f1_threshold": 0.855369, "cosine_precision": 0.432515, "cosine_recall": 0.998819, "cosine_ap": 0.499753, "manhattan_accuracy": 0.5985, "manhattan_accuracy_threshold": 1.115182, "manhattan_f1": 0.604003, "manhattan_f1_threshold": 8.011074, "manhattan_precision": 0.433111, "manhattan_recall": 0.997639, "manhattan_ap": 0.499812, "euclidean_accuracy": 0.5985, "euclidean_accuracy_threshold": 0.071805, "euclidean_f1": 0.603639, "euclidean_f1_threshold": 0.537829, "euclidean_precision": 0.432515, "euclidean_recall": 0.998819, "euclidean_ap": 0.499753, "dot_accuracy": 0.5985, "dot_accuracy_threshold": 0.997422, "dot_f1": 0.603639, "dot_f1_threshold": 0.855369, "dot_precision": 0.432515, "dot_recall": 0.998819, "dot_ap": 0.500426, "max_accuracy": 0.5985, "max_f1": 0.604003, "max_precision": 0.433111, "max_recall": 0.998819, "max_ap": 0.500464, "main_score": 0.500464, "hf_subset": "es", "languages": [ "spa-Latn" ] }, { "similarity_accuracy": 0.61, "similarity_accuracy_threshold": 0.996322, "similarity_f1": 0.603721, "similarity_f1_threshold": 0.858229, "similarity_precision": 0.432378, "similarity_recall": 1.0, "similarity_ap": 0.551966, "cosine_accuracy": 0.61, "cosine_accuracy_threshold": 0.996322, "cosine_f1": 0.603721, "cosine_f1_threshold": 0.858229, "cosine_precision": 0.432378, "cosine_recall": 1.0, "cosine_ap": 0.553183, "manhattan_accuracy": 0.6105, "manhattan_accuracy_threshold": 1.277849, "manhattan_f1": 0.603721, "manhattan_f1_threshold": 8.212933, "manhattan_precision": 0.432378, "manhattan_recall": 1.0, "manhattan_ap": 0.553444, "euclidean_accuracy": 0.61, "euclidean_accuracy_threshold": 0.085768, "euclidean_f1": 0.603721, "euclidean_f1_threshold": 0.532435, "euclidean_precision": 0.432378, "euclidean_recall": 1.0, "euclidean_ap": 0.553182, "dot_accuracy": 0.61, "dot_accuracy_threshold": 0.996322, "dot_f1": 0.603721, "dot_f1_threshold": 0.858229, "dot_precision": 0.432378, "dot_recall": 1.0, "dot_ap": 0.552982, "max_accuracy": 0.6105, "max_f1": 0.603721, "max_precision": 0.432378, "max_recall": 1.0, "max_ap": 0.553444, "main_score": 0.553444, "hf_subset": "fr", "languages": [ "fra-Latn" ] }, { "similarity_accuracy": 0.595, "similarity_accuracy_threshold": 0.995102, "similarity_f1": 0.60376, "similarity_f1_threshold": 0.868977, "similarity_precision": 0.436715, "similarity_recall": 0.977752, "similarity_ap": 0.507885, "cosine_accuracy": 0.595, "cosine_accuracy_threshold": 0.995102, "cosine_f1": 0.60376, "cosine_f1_threshold": 0.868977, "cosine_precision": 0.436715, "cosine_recall": 0.977752, "cosine_ap": 0.506808, "manhattan_accuracy": 0.5955, "manhattan_accuracy_threshold": 1.50482, "manhattan_f1": 0.604432, "manhattan_f1_threshold": 7.856647, "manhattan_precision": 0.438125, "manhattan_recall": 0.974239, "manhattan_ap": 0.506545, "euclidean_accuracy": 0.595, "euclidean_accuracy_threshold": 0.098972, "euclidean_f1": 0.60376, "euclidean_f1_threshold": 0.511904, "euclidean_precision": 0.436715, "euclidean_recall": 0.977752, "euclidean_ap": 0.506809, "dot_accuracy": 0.595, "dot_accuracy_threshold": 0.995102, "dot_f1": 0.60376, "dot_f1_threshold": 0.868977, "dot_precision": 0.436715, "dot_recall": 0.977752, "dot_ap": 0.505423, "max_accuracy": 0.5955, "max_f1": 0.604432, "max_precision": 0.438125, "max_recall": 0.977752, "max_ap": 0.507885, "main_score": 0.507885, "hf_subset": "ja", "languages": [ "jpn-Hira" ] }, { "similarity_accuracy": 0.591, "similarity_accuracy_threshold": 0.995367, "similarity_f1": 0.594614, "similarity_f1_threshold": 0.719607, "similarity_precision": 0.42331, "similarity_recall": 0.99881, "similarity_ap": 0.460981, "cosine_accuracy": 0.591, "cosine_accuracy_threshold": 0.995367, "cosine_f1": 0.594614, "cosine_f1_threshold": 0.719607, "cosine_precision": 0.42331, "cosine_recall": 0.99881, "cosine_ap": 0.462112, "manhattan_accuracy": 0.5905, "manhattan_accuracy_threshold": 1.13396, "manhattan_f1": 0.594825, "manhattan_f1_threshold": 11.586039, "manhattan_precision": 0.423523, "manhattan_recall": 0.99881, "manhattan_ap": 0.46218, "euclidean_accuracy": 0.591, "euclidean_accuracy_threshold": 0.096264, "euclidean_f1": 0.594614, "euclidean_f1_threshold": 0.748841, "euclidean_precision": 0.42331, "euclidean_recall": 0.99881, "euclidean_ap": 0.462112, "dot_accuracy": 0.591, "dot_accuracy_threshold": 0.995367, "dot_f1": 0.594614, "dot_f1_threshold": 0.719607, "dot_precision": 0.42331, "dot_recall": 0.99881, "dot_ap": 0.464187, "max_accuracy": 0.591, "max_f1": 0.594825, "max_precision": 0.423523, "max_recall": 0.99881, "max_ap": 0.464187, "main_score": 0.464187, "hf_subset": "ko", "languages": [ "kor-Hang" ] }, { "similarity_accuracy": 0.601, "similarity_accuracy_threshold": 0.999981, "similarity_f1": 0.602196, "similarity_f1_threshold": 0.790576, "similarity_precision": 0.431472, "similarity_recall": 0.996483, "similarity_ap": 0.494599, "cosine_accuracy": 0.601, "cosine_accuracy_threshold": 0.999981, "cosine_f1": 0.602196, "cosine_f1_threshold": 0.790576, "cosine_precision": 0.431472, "cosine_recall": 0.996483, "cosine_ap": 0.495208, "manhattan_accuracy": 0.6015, "manhattan_accuracy_threshold": 0.389959, "manhattan_f1": 0.60241, "manhattan_f1_threshold": 9.907649, "manhattan_precision": 0.431691, "manhattan_recall": 0.996483, "manhattan_ap": 0.495438, "euclidean_accuracy": 0.601, "euclidean_accuracy_threshold": 0.00433, "euclidean_f1": 0.602196, "euclidean_f1_threshold": 0.647177, "euclidean_precision": 0.431472, "euclidean_recall": 0.996483, "euclidean_ap": 0.495207, "dot_accuracy": 0.601, "dot_accuracy_threshold": 0.999981, "dot_f1": 0.602196, "dot_f1_threshold": 0.790576, "dot_precision": 0.431472, "dot_recall": 0.996483, "dot_ap": 0.498537, "max_accuracy": 0.6015, "max_f1": 0.60241, "max_precision": 0.431691, "max_recall": 0.996483, "max_ap": 0.498537, "main_score": 0.498537, "hf_subset": "zh", "languages": [ "cmn-Hans" ] } ] }, "evaluation_time": 7.66953706741333, "kg_co2_emissions": 0.0003409919553931515 }