{ "dataset_revision": "558da352e7dba3ed3229fc4922aef2ebaff0a90b", "task_name": "PawsXPairClassification", "mteb_version": "2.12.30", "scores": { "test": [ { "similarity_accuracy": 0.5925, "similarity_f1": 0.623031, "similarity_precision": 0.453619, "similarity_recall": 0.994413, "similarity_ap": 0.560136, "cosine_accuracy": 0.5925, "cosine_f1": 0.623031, "cosine_precision": 0.453619, "cosine_recall": 0.994413, "cosine_ap": 0.557504, "manhattan_accuracy": 0.592, "manhattan_f1": 0.623031, "manhattan_precision": 0.453619, "manhattan_recall": 0.994413, "manhattan_ap": 0.559236, "euclidean_accuracy": 0.5925, "euclidean_f1": 0.623031, "euclidean_precision": 0.453619, "euclidean_recall": 0.994413, "euclidean_ap": 0.559114, "dot_accuracy": 0.5925, "dot_f1": 0.623031, "dot_precision": 0.453619, "dot_recall": 0.994413, "dot_ap": 0.560821, "max_accuracy": 0.5925, "max_f1": 0.623031, "max_precision": 0.453619, "max_recall": 0.994413, "max_ap": 0.560821, "main_score": 0.560821, "hf_subset": "de", "languages": [ "deu-Latn" ], "mteb_version": "2.12.30" }, { "similarity_accuracy": 0.6185, "similarity_f1": 0.626045, "similarity_precision": 0.457506, "similarity_recall": 0.99118, "similarity_ap": 0.603344, "cosine_accuracy": 0.6185, "cosine_f1": 0.626045, "cosine_precision": 0.457506, "cosine_recall": 0.99118, "cosine_ap": 0.603345, "manhattan_accuracy": 0.6195, "manhattan_f1": 0.626087, "manhattan_precision": 0.457317, "manhattan_recall": 0.992282, "manhattan_ap": 0.603415, "euclidean_accuracy": 0.6185, "euclidean_f1": 0.626045, "euclidean_precision": 0.457506, "euclidean_recall": 0.99118, "euclidean_ap": 0.603345, "dot_accuracy": 0.6185, "dot_f1": 0.626045, "dot_precision": 0.457506, "dot_recall": 0.99118, "dot_ap": 0.603344, "max_accuracy": 0.6195, "max_f1": 0.626087, "max_precision": 0.457506, "max_recall": 0.992282, "max_ap": 0.603415, "main_score": 0.603415, "hf_subset": "en", "languages": [ "eng-Latn" ], "mteb_version": "2.12.30" }, { "similarity_accuracy": 0.588, "similarity_f1": 0.624441, "similarity_precision": 0.453954, "similarity_recall": 1.0, "similarity_ap": 0.56379, "cosine_accuracy": 0.588, "cosine_f1": 0.624441, "cosine_precision": 0.453954, "cosine_recall": 1.0, "cosine_ap": 0.56379, "manhattan_accuracy": 0.588, "manhattan_f1": 0.624441, "manhattan_precision": 0.453954, "manhattan_recall": 1.0, "manhattan_ap": 0.563971, "euclidean_accuracy": 0.588, "euclidean_f1": 0.624441, "euclidean_precision": 0.453954, "euclidean_recall": 1.0, "euclidean_ap": 0.56379, "dot_accuracy": 0.588, "dot_f1": 0.624441, "dot_precision": 0.453954, "dot_recall": 1.0, "dot_ap": 0.563792, "max_accuracy": 0.588, "max_f1": 0.624441, "max_precision": 0.453954, "max_recall": 1.0, "max_ap": 0.563971, "main_score": 0.563971, "hf_subset": "es", "languages": [ "spa-Latn" ], "mteb_version": "2.12.30" }, { "similarity_accuracy": 0.605, "similarity_f1": 0.624913, "similarity_precision": 0.454454, "similarity_recall": 1.0, "similarity_ap": 0.582228, "cosine_accuracy": 0.605, "cosine_f1": 0.624913, "cosine_precision": 0.454454, "cosine_recall": 1.0, "cosine_ap": 0.581632, "manhattan_accuracy": 0.6055, "manhattan_f1": 0.624913, "manhattan_precision": 0.454454, "manhattan_recall": 1.0, "manhattan_ap": 0.581664, "euclidean_accuracy": 0.605, "euclidean_f1": 0.624913, "euclidean_precision": 0.454454, "euclidean_recall": 1.0, "euclidean_ap": 0.581632, "dot_accuracy": 0.605, "dot_f1": 0.624913, "dot_precision": 0.454454, "dot_recall": 1.0, "dot_ap": 0.581613, "max_accuracy": 0.6055, "max_f1": 0.624913, "max_precision": 0.454454, "max_recall": 1.0, "max_ap": 0.582228, "main_score": 0.582228, "hf_subset": "fr", "languages": [ "fra-Latn" ], "mteb_version": "2.12.30" }, { "similarity_accuracy": 0.5725, "similarity_f1": 0.624601, "similarity_precision": 0.454592, "similarity_recall": 0.997735, "similarity_ap": 0.508023, "cosine_accuracy": 0.5725, "cosine_f1": 0.624601, "cosine_precision": 0.454592, "cosine_recall": 0.997735, "cosine_ap": 0.508189, "manhattan_accuracy": 0.5735, "manhattan_f1": 0.624204, "manhattan_precision": 0.453937, "manhattan_recall": 0.998867, "manhattan_ap": 0.50884, "euclidean_accuracy": 0.5725, "euclidean_f1": 0.624601, "euclidean_precision": 0.454592, "euclidean_recall": 0.997735, "euclidean_ap": 0.508801, "dot_accuracy": 0.5725, "dot_f1": 0.624601, "dot_precision": 0.454592, "dot_recall": 0.997735, "dot_ap": 0.506967, "max_accuracy": 0.5735, "max_f1": 0.624601, "max_precision": 0.454592, "max_recall": 0.998867, "max_ap": 0.50884, "main_score": 0.50884, "hf_subset": "ja", "languages": [ "jpn-Hira" ], "mteb_version": "2.12.30" }, { "similarity_accuracy": 0.5725, "similarity_f1": 0.625439, "similarity_precision": 0.45641, "similarity_recall": 0.993304, "similarity_ap": 0.513588, "cosine_accuracy": 0.5725, "cosine_f1": 0.625439, "cosine_precision": 0.45641, "cosine_recall": 0.993304, "cosine_ap": 0.514857, "manhattan_accuracy": 0.5725, "manhattan_f1": 0.625788, "manhattan_precision": 0.456078, "manhattan_recall": 0.996652, "manhattan_ap": 0.514714, "euclidean_accuracy": 0.5725, "euclidean_f1": 0.625439, "euclidean_precision": 0.45641, "euclidean_recall": 0.993304, "euclidean_ap": 0.514857, "dot_accuracy": 0.5725, "dot_f1": 0.625439, "dot_precision": 0.45641, "dot_recall": 0.993304, "dot_ap": 0.517219, "max_accuracy": 0.5725, "max_f1": 0.625788, "max_precision": 0.45641, "max_recall": 0.996652, "max_ap": 0.517219, "main_score": 0.517219, "hf_subset": "ko", "languages": [ "kor-Hang" ], "mteb_version": "2.12.30" }, { "similarity_accuracy": 0.5925, "similarity_f1": 0.62278, "similarity_precision": 0.4522, "similarity_recall": 1.0, "similarity_ap": 0.564836, "cosine_accuracy": 0.5925, "cosine_f1": 0.62278, "cosine_precision": 0.4522, "cosine_recall": 1.0, "cosine_ap": 0.563158, "manhattan_accuracy": 0.5925, "manhattan_f1": 0.622601, "manhattan_precision": 0.45625, "manhattan_recall": 0.979866, "manhattan_ap": 0.56609, "euclidean_accuracy": 0.5925, "euclidean_f1": 0.62278, "euclidean_precision": 0.4522, "euclidean_recall": 1.0, "euclidean_ap": 0.566199, "dot_accuracy": 0.5925, "dot_f1": 0.62278, "dot_precision": 0.4522, "dot_recall": 1.0, "dot_ap": 0.562387, "max_accuracy": 0.5925, "max_f1": 0.62278, "max_precision": 0.45625, "max_recall": 1.0, "max_ap": 0.566199, "main_score": 0.566199, "hf_subset": "zh", "languages": [ "cmn-Hans" ], "mteb_version": "2.12.30" } ], "validation": [ { "similarity_accuracy": 0.6085, "similarity_f1": 0.599135, "similarity_precision": 0.427689, "similarity_recall": 1.0, "similarity_ap": 0.512449, "cosine_accuracy": 0.6085, "cosine_f1": 0.599135, "cosine_precision": 0.427689, "cosine_recall": 1.0, "cosine_ap": 0.506711, "manhattan_accuracy": 0.6085, "manhattan_f1": 0.599135, "manhattan_precision": 0.427689, "manhattan_recall": 1.0, "manhattan_ap": 0.512192, "euclidean_accuracy": 0.6085, "euclidean_f1": 0.599135, "euclidean_precision": 0.427689, "euclidean_recall": 1.0, "euclidean_ap": 0.512011, "dot_accuracy": 0.6085, "dot_f1": 0.599135, "dot_precision": 0.427689, "dot_recall": 1.0, "dot_ap": 0.506459, "max_accuracy": 0.6085, "max_f1": 0.599135, "max_precision": 0.427689, "max_recall": 1.0, "max_ap": 0.512449, "main_score": 0.512449, "hf_subset": "de", "languages": [ "deu-Latn" ], "mteb_version": "2.12.30" }, { "similarity_accuracy": 0.635, "similarity_f1": 0.612389, "similarity_precision": 0.495347, "similarity_recall": 0.801854, "similarity_ap": 0.582518, "cosine_accuracy": 0.635, "cosine_f1": 0.612389, "cosine_precision": 0.495347, "cosine_recall": 0.801854, "cosine_ap": 0.582517, "manhattan_accuracy": 0.635, "manhattan_f1": 0.613074, "manhattan_precision": 0.49536, "manhattan_recall": 0.804171, "manhattan_ap": 0.582517, "euclidean_accuracy": 0.635, "euclidean_f1": 0.612389, "euclidean_precision": 0.495347, "euclidean_recall": 0.801854, "euclidean_ap": 0.582517, "dot_accuracy": 0.635, "dot_f1": 0.612389, "dot_precision": 0.495347, "dot_recall": 0.801854, "dot_ap": 0.58252, "max_accuracy": 0.635, "max_f1": 0.613074, "max_precision": 0.49536, "max_recall": 0.804171, "max_ap": 0.58252, "main_score": 0.58252, "hf_subset": "en", "languages": [ "eng-Latn" ], "mteb_version": "2.12.30" }, { "similarity_accuracy": 0.615, "similarity_f1": 0.603639, "similarity_precision": 0.432515, "similarity_recall": 0.998819, "similarity_ap": 0.547292, "cosine_accuracy": 0.615, "cosine_f1": 0.603639, "cosine_precision": 0.432515, "cosine_recall": 0.998819, "cosine_ap": 0.547171, "manhattan_accuracy": 0.6165, "manhattan_f1": 0.603639, "manhattan_precision": 0.432515, "manhattan_recall": 0.998819, "manhattan_ap": 0.547441, "euclidean_accuracy": 0.615, "euclidean_f1": 0.603639, "euclidean_precision": 0.432515, "euclidean_recall": 0.998819, "euclidean_ap": 0.547173, "dot_accuracy": 0.615, "dot_f1": 0.603639, "dot_precision": 0.432515, "dot_recall": 0.998819, "dot_ap": 0.548183, "max_accuracy": 0.6165, "max_f1": 0.603639, "max_precision": 0.432515, "max_recall": 0.998819, "max_ap": 0.548183, "main_score": 0.548183, "hf_subset": "es", "languages": [ "spa-Latn" ], "mteb_version": "2.12.30" }, { "similarity_accuracy": 0.621, "similarity_f1": 0.602874, "similarity_precision": 0.43151, "similarity_recall": 1.0, "similarity_ap": 0.576638, "cosine_accuracy": 0.621, "cosine_f1": 0.602874, "cosine_precision": 0.43151, "cosine_recall": 1.0, "cosine_ap": 0.576521, "manhattan_accuracy": 0.62, "manhattan_f1": 0.602874, "manhattan_precision": 0.43151, "manhattan_recall": 1.0, "manhattan_ap": 0.576525, "euclidean_accuracy": 0.621, "euclidean_f1": 0.602874, "euclidean_precision": 0.43151, "euclidean_recall": 1.0, "euclidean_ap": 0.576521, "dot_accuracy": 0.621, "dot_f1": 0.602874, "dot_precision": 0.43151, "dot_recall": 1.0, "dot_ap": 0.577238, "max_accuracy": 0.621, "max_f1": 0.602874, "max_precision": 0.43151, "max_recall": 1.0, "max_ap": 0.577238, "main_score": 0.577238, "hf_subset": "fr", "languages": [ "fra-Latn" ], "mteb_version": "2.12.30" }, { "similarity_accuracy": 0.601, "similarity_f1": 0.602188, "similarity_precision": 0.431026, "similarity_recall": 0.998829, "similarity_ap": 0.533727, "cosine_accuracy": 0.601, "cosine_f1": 0.602188, "cosine_precision": 0.431026, "cosine_recall": 0.998829, "cosine_ap": 0.532784, "manhattan_accuracy": 0.602, "manhattan_f1": 0.602188, "manhattan_precision": 0.431026, "manhattan_recall": 0.998829, "manhattan_ap": 0.532846, "euclidean_accuracy": 0.601, "euclidean_f1": 0.602188, "euclidean_precision": 0.431026, "euclidean_recall": 0.998829, "euclidean_ap": 0.532784, "dot_accuracy": 0.601, "dot_f1": 0.602188, "dot_precision": 0.431026, "dot_recall": 0.998829, "dot_ap": 0.532983, "max_accuracy": 0.602, "max_f1": 0.602188, "max_precision": 0.431026, "max_recall": 0.998829, "max_ap": 0.533727, "main_score": 0.533727, "hf_subset": "ja", "languages": [ "jpn-Hira" ], "mteb_version": "2.12.30" }, { "similarity_accuracy": 0.5955, "similarity_f1": 0.596278, "similarity_precision": 0.426305, "similarity_recall": 0.991667, "similarity_ap": 0.481645, "cosine_accuracy": 0.5955, "cosine_f1": 0.596278, "cosine_precision": 0.426305, "cosine_recall": 0.991667, "cosine_ap": 0.480515, "manhattan_accuracy": 0.5955, "manhattan_f1": 0.596278, "manhattan_precision": 0.426305, "manhattan_recall": 0.991667, "manhattan_ap": 0.481228, "euclidean_accuracy": 0.5955, "euclidean_f1": 0.596278, "euclidean_precision": 0.426305, "euclidean_recall": 0.991667, "euclidean_ap": 0.480938, "dot_accuracy": 0.5955, "dot_f1": 0.596278, "dot_precision": 0.426305, "dot_recall": 0.991667, "dot_ap": 0.482265, "max_accuracy": 0.5955, "max_f1": 0.596278, "max_precision": 0.426305, "max_recall": 0.991667, "max_ap": 0.482265, "main_score": 0.482265, "hf_subset": "ko", "languages": [ "kor-Hang" ], "mteb_version": "2.12.30" }, { "similarity_accuracy": 0.6085, "similarity_f1": 0.601483, "similarity_precision": 0.430303, "similarity_recall": 0.998828, "similarity_ap": 0.527052, "cosine_accuracy": 0.6085, "cosine_f1": 0.601483, "cosine_precision": 0.430303, "cosine_recall": 0.998828, "cosine_ap": 0.52547, "manhattan_accuracy": 0.6085, "manhattan_f1": 0.601483, "manhattan_precision": 0.430303, "manhattan_recall": 0.998828, "manhattan_ap": 0.525524, "euclidean_accuracy": 0.6085, "euclidean_f1": 0.601483, "euclidean_precision": 0.430303, "euclidean_recall": 0.998828, "euclidean_ap": 0.52547, "dot_accuracy": 0.6085, "dot_f1": 0.601483, "dot_precision": 0.430303, "dot_recall": 0.998828, "dot_ap": 0.522965, "max_accuracy": 0.6085, "max_f1": 0.601483, "max_precision": 0.430303, "max_recall": 0.998828, "max_ap": 0.527052, "main_score": 0.527052, "hf_subset": "zh", "languages": [ "cmn-Hans" ], "mteb_version": "2.12.30" } ] }, "evaluation_time": 242.49617628590204, "kg_co2_emissions": null, "date": 1785199005.409176, "evaluation_phases": [ { "name": "Data loading", "start": 9.828014299273491e-06, "end": 132.22386265313253, "split": "", "subset": "" }, { "name": "Dataset transform", "start": 132.22388126305304, "end": 132.22388819605112, "split": "", "subset": "" }, { "name": "Encoding samples 1", "start": 132.2299218759872, "end": 136.01950367097743, "split": "test", "subset": "de" }, { "name": "Encoding samples 2", "start": 136.01953364908695, "end": 139.84981481707655, "split": "test", "subset": "de" }, { "name": "Scoring", "start": 139.84983489196748, "end": 139.88336693798192, "split": "test", "subset": "de" }, { "name": "Encoding samples 1", "start": 139.94705561199225, "end": 143.73428240208887, "split": "test", "subset": "en" }, { "name": "Encoding samples 2", "start": 143.73430673894472, "end": 147.5506625820417, "split": "test", "subset": "en" }, { "name": "Scoring", "start": 147.5506824750919, "end": 147.57262456300668, "split": "test", "subset": "en" }, { "name": "Encoding samples 1", "start": 147.63254764094017, "end": 151.48584075691178, "split": "test", "subset": "es" }, { "name": "Encoding samples 2", "start": 151.48585973703302, "end": 155.28949746699072, "split": "test", "subset": "es" }, { "name": "Scoring", "start": 155.28951464802958, "end": 155.31163218594156, "split": "test", "subset": "es" }, { "name": "Encoding samples 1", "start": 155.37225613603368, "end": 159.33226078702137, "split": "test", "subset": "fr" }, { "name": "Encoding samples 2", "start": 159.3322815541178, "end": 163.18829614901915, "split": "test", "subset": "fr" }, { "name": "Scoring", "start": 163.18831346998923, "end": 163.20546387205832, "split": "test", "subset": "fr" }, { "name": "Encoding samples 1", "start": 163.24031823291443, "end": 167.290238074027, "split": "test", "subset": "ja" }, { "name": "Encoding samples 2", "start": 167.29027852695435, "end": 171.48514193110168, "split": "test", "subset": "ja" }, { "name": "Scoring", "start": 171.4851687720511, "end": 171.5072263421025, "split": "test", "subset": "ja" }, { "name": "Encoding samples 1", "start": 171.56904522608966, "end": 175.6336431170348, "split": "test", "subset": "ko" }, { "name": "Encoding samples 2", "start": 175.63366393791512, "end": 179.7366068949923, "split": "test", "subset": "ko" }, { "name": "Scoring", "start": 179.7366369299125, "end": 179.75882265297696, "split": "test", "subset": "ko" }, { "name": "Encoding samples 1", "start": 179.8206603960134, "end": 183.93058663699776, "split": "test", "subset": "zh" }, { "name": "Encoding samples 2", "start": 183.9306062089745, "end": 187.76258796500042, "split": "test", "subset": "zh" }, { "name": "Scoring", "start": 187.7626099300105, "end": 187.78452587802894, "split": "test", "subset": "zh" }, { "name": "Encoding samples 1", "start": 187.84644675604068, "end": 191.64978155004792, "split": "validation", "subset": "de" }, { "name": "Encoding samples 2", "start": 191.6498134729918, "end": 195.41779736895114, "split": "validation", "subset": "de" }, { "name": "Scoring", "start": 195.41781948995776, "end": 195.43494333396666, "split": "validation", "subset": "de" }, { "name": "Encoding samples 1", "start": 195.47949016210623, "end": 199.2553539860528, "split": "validation", "subset": "en" }, { "name": "Encoding samples 2", "start": 199.2553929660935, "end": 203.05978650902398, "split": "validation", "subset": "en" }, { "name": "Scoring", "start": 203.05980870104395, "end": 203.09130691294558, "split": "validation", "subset": "en" }, { "name": "Encoding samples 1", "start": 203.15468233893625, "end": 206.93164622713812, "split": "validation", "subset": "es" }, { "name": "Encoding samples 2", "start": 206.93167461897247, "end": 210.6956344449427, "split": "validation", "subset": "es" }, { "name": "Scoring", "start": 210.69565124204382, "end": 210.7200353750959, "split": "validation", "subset": "es" }, { "name": "Encoding samples 1", "start": 210.78235473111272, "end": 214.7062670381274, "split": "validation", "subset": "fr" }, { "name": "Encoding samples 2", "start": 214.70628104591742, "end": 218.5094589809887, "split": "validation", "subset": "fr" }, { "name": "Scoring", "start": 218.50948335090652, "end": 218.53077535703778, "split": "validation", "subset": "fr" }, { "name": "Encoding samples 1", "start": 218.59169070096686, "end": 222.71385875996202, "split": "validation", "subset": "ja" }, { "name": "Encoding samples 2", "start": 222.7138750529848, "end": 226.87912855902687, "split": "validation", "subset": "ja" }, { "name": "Scoring", "start": 226.87914784695022, "end": 226.90086876112036, "split": "validation", "subset": "ja" }, { "name": "Encoding samples 1", "start": 226.96174903097562, "end": 231.00980397197418, "split": "validation", "subset": "ko" }, { "name": "Encoding samples 2", "start": 231.0098152779974, "end": 234.90504091302864, "split": "validation", "subset": "ko" }, { "name": "Scoring", "start": 234.9050553070847, "end": 234.921986797126, "split": "validation", "subset": "ko" }, { "name": "Encoding samples 1", "start": 234.95798717602156, "end": 238.6191616069991, "split": "validation", "subset": "zh" }, { "name": "Encoding samples 2", "start": 238.61918975412846, "end": 242.44434419693425, "split": "validation", "subset": "zh" }, { "name": "Scoring", "start": 242.4443715219386, "end": 242.46145072206855, "split": "validation", "subset": "zh" } ] }