{ "dataset_revision": "8a04d940a42cd40658986fdd8e3da561533a3646", "task_name": "PawsXPairClassification", "mteb_version": "1.18.0", "scores": { "test": [ { "similarity_accuracy": 0.5745, "similarity_accuracy_threshold": 0.987412, "similarity_f1": 0.624476, "similarity_f1_threshold": 0.645495, "similarity_precision": 0.454453, "similarity_recall": 0.997765, "similarity_ap": 0.509867, "cosine_accuracy": 0.5745, "cosine_accuracy_threshold": 0.987412, "cosine_f1": 0.624476, "cosine_f1_threshold": 0.645495, "cosine_precision": 0.454453, "cosine_recall": 0.997765, "cosine_ap": 0.513769, "manhattan_accuracy": 0.5745, "manhattan_accuracy_threshold": 9.116299, "manhattan_f1": 0.624476, "manhattan_f1_threshold": 50.220467, "manhattan_precision": 0.454453, "manhattan_recall": 0.997765, "manhattan_ap": 0.514454, "euclidean_accuracy": 0.5745, "euclidean_accuracy_threshold": 0.444832, "euclidean_f1": 0.624476, "euclidean_f1_threshold": 2.374301, "euclidean_precision": 0.454453, "euclidean_recall": 0.997765, "euclidean_ap": 0.513404, "dot_accuracy": 0.563, "dot_accuracy_threshold": 8.246034, "dot_f1": 0.625, "dot_f1_threshold": 4.735727, "dot_precision": 0.454545, "dot_recall": 1.0, "dot_ap": 0.484842, "max_accuracy": 0.5745, "max_f1": 0.625, "max_precision": 0.454545, "max_recall": 1.0, "max_ap": 0.514454, "main_score": 0.514454, "hf_subset": "de", "languages": [ "deu-Latn" ] }, { "similarity_accuracy": 0.6055, "similarity_accuracy_threshold": 0.980977, "similarity_f1": 0.624569, "similarity_f1_threshold": 0.848837, "similarity_precision": 0.454545, "similarity_recall": 0.997795, "similarity_ap": 0.566601, "cosine_accuracy": 0.6055, "cosine_accuracy_threshold": 0.980977, "cosine_f1": 0.624569, "cosine_f1_threshold": 0.848837, "cosine_precision": 0.454545, "cosine_recall": 0.997795, "cosine_ap": 0.5666, "manhattan_accuracy": 0.607, "manhattan_accuracy_threshold": 11.377908, "manhattan_f1": 0.624697, "manhattan_f1_threshold": 31.796532, "manhattan_precision": 0.455141, "manhattan_recall": 0.99559, "manhattan_ap": 0.569317, "euclidean_accuracy": 0.603, "euclidean_accuracy_threshold": 0.519968, "euclidean_f1": 0.624656, "euclidean_f1_threshold": 1.78214, "euclidean_precision": 0.454181, "euclidean_recall": 1.0, "euclidean_ap": 0.566418, "dot_accuracy": 0.559, "dot_accuracy_threshold": 8.042445, "dot_f1": 0.623752, "dot_f1_threshold": 5.818357, "dot_precision": 0.453453, "dot_recall": 0.998897, "dot_ap": 0.492891, "max_accuracy": 0.607, "max_f1": 0.624697, "max_precision": 0.455141, "max_recall": 1.0, "max_ap": 0.569317, "main_score": 0.569317, "hf_subset": "en", "languages": [ "eng-Latn" ] }, { "similarity_accuracy": 0.5785, "similarity_accuracy_threshold": 0.987587, "similarity_f1": 0.624441, "similarity_f1_threshold": 0.585239, "similarity_precision": 0.453954, "similarity_recall": 1.0, "similarity_ap": 0.528474, "cosine_accuracy": 0.5785, "cosine_accuracy_threshold": 0.987587, "cosine_f1": 0.624441, "cosine_f1_threshold": 0.585239, "cosine_precision": 0.453954, "cosine_recall": 1.0, "cosine_ap": 0.528472, "manhattan_accuracy": 0.5785, "manhattan_accuracy_threshold": 9.591961, "manhattan_f1": 0.624441, "manhattan_f1_threshold": 53.401688, "manhattan_precision": 0.453954, "manhattan_recall": 1.0, "manhattan_ap": 0.529672, "euclidean_accuracy": 0.578, "euclidean_accuracy_threshold": 0.436486, "euclidean_f1": 0.624441, "euclidean_f1_threshold": 2.505038, "euclidean_precision": 0.453954, "euclidean_recall": 1.0, "euclidean_ap": 0.528839, "dot_accuracy": 0.5555, "dot_accuracy_threshold": 8.136959, "dot_f1": 0.624226, "dot_f1_threshold": 2.27632, "dot_precision": 0.453727, "dot_recall": 1.0, "dot_ap": 0.476967, "max_accuracy": 0.5785, "max_f1": 0.624441, "max_precision": 0.453954, "max_recall": 1.0, "max_ap": 0.529672, "main_score": 0.529672, "hf_subset": "es", "languages": [ "spa-Latn" ] }, { "similarity_accuracy": 0.586, "similarity_accuracy_threshold": 0.978725, "similarity_f1": 0.624957, "similarity_f1_threshold": 0.769597, "similarity_precision": 0.45542, "similarity_recall": 0.99557, "similarity_ap": 0.548491, "cosine_accuracy": 0.586, "cosine_accuracy_threshold": 0.978725, "cosine_f1": 0.624957, "cosine_f1_threshold": 0.769597, "cosine_precision": 0.45542, "cosine_recall": 0.99557, "cosine_ap": 0.548365, "manhattan_accuracy": 0.586, "manhattan_accuracy_threshold": 12.763716, "manhattan_f1": 0.625477, "manhattan_f1_threshold": 41.818108, "manhattan_precision": 0.455511, "manhattan_recall": 0.997785, "manhattan_ap": 0.549279, "euclidean_accuracy": 0.5855, "euclidean_accuracy_threshold": 0.588405, "euclidean_f1": 0.62526, "euclidean_f1_threshold": 1.986486, "euclidean_precision": 0.45528, "euclidean_recall": 0.997785, "euclidean_ap": 0.548296, "dot_accuracy": 0.5605, "dot_accuracy_threshold": 8.139194, "dot_f1": 0.625694, "dot_f1_threshold": 5.838549, "dot_precision": 0.455741, "dot_recall": 0.997785, "dot_ap": 0.490735, "max_accuracy": 0.586, "max_f1": 0.625694, "max_precision": 0.455741, "max_recall": 0.997785, "max_ap": 0.549279, "main_score": 0.549279, "hf_subset": "fr", "languages": [ "fra-Latn" ] }, { "similarity_accuracy": 0.571, "similarity_accuracy_threshold": 0.988341, "similarity_f1": 0.624103, "similarity_f1_threshold": 0.827647, "similarity_precision": 0.456693, "similarity_recall": 0.985277, "similarity_ap": 0.483883, "cosine_accuracy": 0.571, "cosine_accuracy_threshold": 0.988341, "cosine_f1": 0.624103, "cosine_f1_threshold": 0.827647, "cosine_precision": 0.456693, "cosine_recall": 0.985277, "cosine_ap": 0.482887, "manhattan_accuracy": 0.5705, "manhattan_accuracy_threshold": 7.546754, "manhattan_f1": 0.623794, "manhattan_f1_threshold": 37.579575, "manhattan_precision": 0.455637, "manhattan_recall": 0.988675, "manhattan_ap": 0.48258, "euclidean_accuracy": 0.5715, "euclidean_accuracy_threshold": 0.460986, "euclidean_f1": 0.623762, "euclidean_f1_threshold": 2.986195, "euclidean_precision": 0.45347, "euclidean_recall": 0.998867, "euclidean_ap": 0.482136, "dot_accuracy": 0.564, "dot_accuracy_threshold": 8.811785, "dot_f1": 0.62536, "dot_f1_threshold": 7.136352, "dot_precision": 0.458531, "dot_recall": 0.983012, "dot_ap": 0.471932, "max_accuracy": 0.5715, "max_f1": 0.62536, "max_precision": 0.458531, "max_recall": 0.998867, "max_ap": 0.483883, "main_score": 0.483883, "hf_subset": "ja", "languages": [ "jpn-Hira" ] }, { "similarity_accuracy": 0.564, "similarity_accuracy_threshold": 0.98317, "similarity_f1": 0.625702, "similarity_f1_threshold": 0.69301, "similarity_precision": 0.456455, "similarity_recall": 0.99442, "similarity_ap": 0.49713, "cosine_accuracy": 0.564, "cosine_accuracy_threshold": 0.98317, "cosine_f1": 0.625702, "cosine_f1_threshold": 0.69301, "cosine_precision": 0.456455, "cosine_recall": 0.99442, "cosine_ap": 0.497629, "manhattan_accuracy": 0.566, "manhattan_accuracy_threshold": 12.233444, "manhattan_f1": 0.625483, "manhattan_f1_threshold": 47.93018, "manhattan_precision": 0.456221, "manhattan_recall": 0.99442, "manhattan_ap": 0.498739, "euclidean_accuracy": 0.5645, "euclidean_accuracy_threshold": 0.546778, "euclidean_f1": 0.625659, "euclidean_f1_threshold": 2.203503, "euclidean_precision": 0.456644, "euclidean_recall": 0.993304, "euclidean_ap": 0.498573, "dot_accuracy": 0.5545, "dot_accuracy_threshold": 9.209051, "dot_f1": 0.62535, "dot_f1_threshold": 5.377058, "dot_precision": 0.455612, "dot_recall": 0.996652, "dot_ap": 0.462889, "max_accuracy": 0.566, "max_f1": 0.625702, "max_precision": 0.456644, "max_recall": 0.996652, "max_ap": 0.498739, "main_score": 0.498739, "hf_subset": "ko", "languages": [ "kor-Hang" ] }, { "similarity_accuracy": 0.5885, "similarity_accuracy_threshold": 0.998014, "similarity_f1": 0.622997, "similarity_f1_threshold": 0.673448, "similarity_precision": 0.452429, "similarity_recall": 1.0, "similarity_ap": 0.534309, "cosine_accuracy": 0.5885, "cosine_accuracy_threshold": 0.998014, "cosine_f1": 0.622997, "cosine_f1_threshold": 0.673448, "cosine_precision": 0.452429, "cosine_recall": 1.0, "cosine_ap": 0.536499, "manhattan_accuracy": 0.589, "manhattan_accuracy_threshold": 4.617087, "manhattan_f1": 0.623214, "manhattan_f1_threshold": 50.72044, "manhattan_precision": 0.452658, "manhattan_recall": 1.0, "manhattan_ap": 0.536775, "euclidean_accuracy": 0.588, "euclidean_accuracy_threshold": 0.202201, "euclidean_f1": 0.622997, "euclidean_f1_threshold": 2.476406, "euclidean_precision": 0.452429, "euclidean_recall": 1.0, "euclidean_ap": 0.536128, "dot_accuracy": 0.5575, "dot_accuracy_threshold": 10.713001, "dot_f1": 0.622905, "dot_f1_threshold": 6.61228, "dot_precision": 0.452792, "dot_recall": 0.997763, "dot_ap": 0.478608, "max_accuracy": 0.589, "max_f1": 0.623214, "max_precision": 0.452792, "max_recall": 1.0, "max_ap": 0.536775, "main_score": 0.536775, "hf_subset": "zh", "languages": [ "cmn-Hans" ] } ], "validation": [ { "similarity_accuracy": 0.596, "similarity_accuracy_threshold": 0.997386, "similarity_f1": 0.600073, "similarity_f1_threshold": 0.766006, "similarity_precision": 0.431125, "similarity_recall": 0.986763, "similarity_ap": 0.459774, "cosine_accuracy": 0.596, "cosine_accuracy_threshold": 0.997386, "cosine_f1": 0.600073, "cosine_f1_threshold": 0.766006, "cosine_precision": 0.431125, "cosine_recall": 0.986763, "cosine_ap": 0.463171, "manhattan_accuracy": 0.5965, "manhattan_accuracy_threshold": 4.719241, "manhattan_f1": 0.599854, "manhattan_f1_threshold": 42.736607, "manhattan_precision": 0.430214, "manhattan_recall": 0.990373, "manhattan_ap": 0.463295, "euclidean_accuracy": 0.596, "euclidean_accuracy_threshold": 0.211849, "euclidean_f1": 0.599782, "euclidean_f1_threshold": 2.038617, "euclidean_precision": 0.429688, "euclidean_recall": 0.99278, "euclidean_ap": 0.462798, "dot_accuracy": 0.5895, "dot_accuracy_threshold": 8.122007, "dot_f1": 0.600293, "dot_f1_threshold": 5.989973, "dot_precision": 0.431352, "dot_recall": 0.986763, "dot_ap": 0.439356, "max_accuracy": 0.5965, "max_f1": 0.600293, "max_precision": 0.431352, "max_recall": 0.99278, "max_ap": 0.463295, "main_score": 0.463295, "hf_subset": "de", "languages": [ "deu-Latn" ] }, { "similarity_accuracy": 0.62, "similarity_accuracy_threshold": 0.982611, "similarity_f1": 0.608624, "similarity_f1_threshold": 0.946543, "similarity_precision": 0.471374, "similarity_recall": 0.858633, "similarity_ap": 0.568465, "cosine_accuracy": 0.62, "cosine_accuracy_threshold": 0.982611, "cosine_f1": 0.608624, "cosine_f1_threshold": 0.946542, "cosine_precision": 0.471374, "cosine_recall": 0.858633, "cosine_ap": 0.568475, "manhattan_accuracy": 0.621, "manhattan_accuracy_threshold": 12.38467, "manhattan_f1": 0.608383, "manhattan_f1_threshold": 20.763882, "manhattan_precision": 0.464068, "manhattan_recall": 0.882966, "manhattan_ap": 0.573693, "euclidean_accuracy": 0.6205, "euclidean_accuracy_threshold": 0.524827, "euclidean_f1": 0.608356, "euclidean_f1_threshold": 1.00871, "euclidean_precision": 0.458775, "euclidean_recall": 0.902665, "euclidean_ap": 0.571206, "dot_accuracy": 0.5685, "dot_accuracy_threshold": 9.502129, "dot_f1": 0.605048, "dot_f1_threshold": 6.756976, "dot_precision": 0.43641, "dot_recall": 0.986095, "dot_ap": 0.444965, "max_accuracy": 0.621, "max_f1": 0.608624, "max_precision": 0.471374, "max_recall": 0.986095, "max_ap": 0.573693, "main_score": 0.573693, "hf_subset": "en", "languages": [ "eng-Latn" ] }, { "similarity_accuracy": 0.6, "similarity_accuracy_threshold": 0.983171, "similarity_f1": 0.603062, "similarity_f1_threshold": 0.722694, "similarity_precision": 0.431702, "similarity_recall": 1.0, "similarity_ap": 0.506459, "cosine_accuracy": 0.6, "cosine_accuracy_threshold": 0.983171, "cosine_f1": 0.603062, "cosine_f1_threshold": 0.722694, "cosine_precision": 0.431702, "cosine_recall": 1.0, "cosine_ap": 0.505813, "manhattan_accuracy": 0.5995, "manhattan_accuracy_threshold": 11.758725, "manhattan_f1": 0.602847, "manhattan_f1_threshold": 60.121349, "manhattan_precision": 0.431482, "manhattan_recall": 1.0, "manhattan_ap": 0.50714, "euclidean_accuracy": 0.5985, "euclidean_accuracy_threshold": 0.518211, "euclidean_f1": 0.602847, "euclidean_f1_threshold": 2.867202, "euclidean_precision": 0.431482, "euclidean_recall": 1.0, "euclidean_ap": 0.506029, "dot_accuracy": 0.5805, "dot_accuracy_threshold": 8.378374, "dot_f1": 0.603276, "dot_f1_threshold": 5.67449, "dot_precision": 0.431922, "dot_recall": 1.0, "dot_ap": 0.454313, "max_accuracy": 0.6, "max_f1": 0.603276, "max_precision": 0.431922, "max_recall": 1.0, "max_ap": 0.50714, "main_score": 0.50714, "hf_subset": "es", "languages": [ "spa-Latn" ] }, { "similarity_accuracy": 0.604, "similarity_accuracy_threshold": 0.978243, "similarity_f1": 0.604027, "similarity_f1_threshold": 0.815162, "similarity_precision": 0.43379, "similarity_recall": 0.994186, "similarity_ap": 0.543962, "cosine_accuracy": 0.604, "cosine_accuracy_threshold": 0.978243, "cosine_f1": 0.604027, "cosine_f1_threshold": 0.815162, "cosine_precision": 0.43379, "cosine_recall": 0.994186, "cosine_ap": 0.545642, "manhattan_accuracy": 0.6045, "manhattan_accuracy_threshold": 12.635983, "manhattan_f1": 0.60447, "manhattan_f1_threshold": 34.821896, "manhattan_precision": 0.434916, "manhattan_recall": 0.990698, "manhattan_ap": 0.547508, "euclidean_accuracy": 0.603, "euclidean_accuracy_threshold": 0.58457, "euclidean_f1": 0.604881, "euclidean_f1_threshold": 1.663841, "euclidean_precision": 0.434672, "euclidean_recall": 0.994186, "euclidean_ap": 0.54634, "dot_accuracy": 0.576, "dot_accuracy_threshold": 8.403912, "dot_f1": 0.603613, "dot_f1_threshold": 6.306687, "dot_precision": 0.43403, "dot_recall": 0.990698, "dot_ap": 0.461741, "max_accuracy": 0.6045, "max_f1": 0.604881, "max_precision": 0.434916, "max_recall": 0.994186, "max_ap": 0.547508, "main_score": 0.547508, "hf_subset": "fr", "languages": [ "fra-Latn" ] }, { "similarity_accuracy": 0.604, "similarity_accuracy_threshold": 0.986981, "similarity_f1": 0.603534, "similarity_f1_threshold": 0.716978, "similarity_precision": 0.432186, "similarity_recall": 1.0, "similarity_ap": 0.528383, "cosine_accuracy": 0.604, "cosine_accuracy_threshold": 0.986981, "cosine_f1": 0.603534, "cosine_f1_threshold": 0.716978, "cosine_precision": 0.432186, "cosine_recall": 1.0, "cosine_ap": 0.528918, "manhattan_accuracy": 0.6045, "manhattan_accuracy_threshold": 10.275486, "manhattan_f1": 0.603534, "manhattan_f1_threshold": 46.09124, "manhattan_precision": 0.432186, "manhattan_recall": 1.0, "manhattan_ap": 0.529229, "euclidean_accuracy": 0.605, "euclidean_accuracy_threshold": 0.388395, "euclidean_f1": 0.603747, "euclidean_f1_threshold": 2.119401, "euclidean_precision": 0.432405, "euclidean_recall": 1.0, "euclidean_ap": 0.528307, "dot_accuracy": 0.5775, "dot_accuracy_threshold": 9.389141, "dot_f1": 0.604189, "dot_f1_threshold": 6.74228, "dot_precision": 0.43352, "dot_recall": 0.996487, "dot_ap": 0.467308, "max_accuracy": 0.605, "max_f1": 0.604189, "max_precision": 0.43352, "max_recall": 1.0, "max_ap": 0.529229, "main_score": 0.529229, "hf_subset": "ja", "languages": [ "jpn-Hira" ] }, { "similarity_accuracy": 0.5885, "similarity_accuracy_threshold": 0.993121, "similarity_f1": 0.597652, "similarity_f1_threshold": 0.440332, "similarity_precision": 0.42618, "similarity_recall": 1.0, "similarity_ap": 0.454328, "cosine_accuracy": 0.5885, "cosine_accuracy_threshold": 0.993121, "cosine_f1": 0.597652, "cosine_f1_threshold": 0.440332, "cosine_precision": 0.42618, "cosine_recall": 1.0, "cosine_ap": 0.456551, "manhattan_accuracy": 0.589, "manhattan_accuracy_threshold": 7.506373, "manhattan_f1": 0.597652, "manhattan_f1_threshold": 68.203064, "manhattan_precision": 0.42618, "manhattan_recall": 1.0, "manhattan_ap": 0.457345, "euclidean_accuracy": 0.5885, "euclidean_accuracy_threshold": 0.347221, "euclidean_f1": 0.597652, "euclidean_f1_threshold": 3.261828, "euclidean_precision": 0.42618, "euclidean_recall": 1.0, "euclidean_ap": 0.457075, "dot_accuracy": 0.5795, "dot_accuracy_threshold": 10.614189, "dot_f1": 0.597652, "dot_f1_threshold": 4.113975, "dot_precision": 0.42618, "dot_recall": 1.0, "dot_ap": 0.425526, "max_accuracy": 0.589, "max_f1": 0.597652, "max_precision": 0.42618, "max_recall": 1.0, "max_ap": 0.457345, "main_score": 0.457345, "hf_subset": "ko", "languages": [ "kor-Hang" ] }, { "similarity_accuracy": 0.6015, "similarity_accuracy_threshold": 0.99752, "similarity_f1": 0.601339, "similarity_f1_threshold": 0.513773, "similarity_precision": 0.42994, "similarity_recall": 1.0, "similarity_ap": 0.50673, "cosine_accuracy": 0.6015, "cosine_accuracy_threshold": 0.99752, "cosine_f1": 0.601339, "cosine_f1_threshold": 0.513773, "cosine_precision": 0.42994, "cosine_recall": 1.0, "cosine_ap": 0.5105, "manhattan_accuracy": 0.602, "manhattan_accuracy_threshold": 4.270779, "manhattan_f1": 0.601339, "manhattan_f1_threshold": 64.484924, "manhattan_precision": 0.42994, "manhattan_recall": 1.0, "manhattan_ap": 0.509926, "euclidean_accuracy": 0.602, "euclidean_accuracy_threshold": 0.200803, "euclidean_f1": 0.601339, "euclidean_f1_threshold": 3.070917, "euclidean_precision": 0.42994, "euclidean_recall": 1.0, "euclidean_ap": 0.509263, "dot_accuracy": 0.5765, "dot_accuracy_threshold": 10.565512, "dot_f1": 0.601339, "dot_f1_threshold": 4.984882, "dot_precision": 0.42994, "dot_recall": 1.0, "dot_ap": 0.468221, "max_accuracy": 0.602, "max_f1": 0.601339, "max_precision": 0.42994, "max_recall": 1.0, "max_ap": 0.5105, "main_score": 0.5105, "hf_subset": "zh", "languages": [ "cmn-Hans" ] } ] }, "evaluation_time": 22.140340566635132, "kg_co2_emissions": 0.0015402861565590928 }