{ "dataset_revision": "558da352e7dba3ed3229fc4922aef2ebaff0a90b", "task_name": "PawsXPairClassification", "mteb_version": "2.18.0", "scores": { "test": [ { "similarity_accuracy": 0.6415, "similarity_f1": 0.623094, "similarity_precision": 0.461538, "similarity_recall": 0.958659, "similarity_ap": 0.632216, "cosine_accuracy": 0.6415, "cosine_f1": 0.623094, "cosine_precision": 0.461538, "cosine_recall": 0.958659, "cosine_ap": 0.631111, "manhattan_accuracy": 0.6425, "manhattan_f1": 0.623204, "manhattan_precision": 0.454035, "manhattan_recall": 0.993296, "manhattan_ap": 0.630398, "euclidean_accuracy": 0.6415, "euclidean_f1": 0.623094, "euclidean_precision": 0.461538, "euclidean_recall": 0.958659, "euclidean_ap": 0.631052, "dot_accuracy": 0.648, "dot_f1": 0.623094, "dot_precision": 0.461538, "dot_recall": 0.958659, "dot_ap": 0.628368, "max_accuracy": 0.648, "max_f1": 0.623204, "max_precision": 0.461538, "max_recall": 0.993296, "max_ap": 0.632216, "main_score": 0.632216, "hf_subset": "de", "languages": [ "deu-Latn" ], "mteb_version": "2.18.0" }, { "similarity_accuracy": 0.695, "similarity_f1": 0.662743, "similarity_precision": 0.562212, "similarity_recall": 0.807056, "similarity_ap": 0.710576, "cosine_accuracy": 0.695, "cosine_f1": 0.662743, "cosine_precision": 0.562212, "cosine_recall": 0.807056, "cosine_ap": 0.710577, "manhattan_accuracy": 0.694, "manhattan_f1": 0.663953, "manhattan_precision": 0.562883, "manhattan_recall": 0.809261, "manhattan_ap": 0.710349, "euclidean_accuracy": 0.695, "euclidean_f1": 0.662174, "euclidean_precision": 0.558757, "euclidean_recall": 0.812569, "euclidean_ap": 0.710559, "dot_accuracy": 0.693, "dot_f1": 0.662455, "dot_precision": 0.560733, "dot_recall": 0.809261, "dot_ap": 0.709496, "max_accuracy": 0.695, "max_f1": 0.663953, "max_precision": 0.562883, "max_recall": 0.812569, "max_ap": 0.710577, "main_score": 0.710577, "hf_subset": "en", "languages": [ "eng-Latn" ], "mteb_version": "2.18.0" }, { "similarity_accuracy": 0.64, "similarity_f1": 0.625088, "similarity_precision": 0.458141, "similarity_recall": 0.983462, "similarity_ap": 0.64014, "cosine_accuracy": 0.64, "cosine_f1": 0.625088, "cosine_precision": 0.458141, "cosine_recall": 0.983462, "cosine_ap": 0.64014, "manhattan_accuracy": 0.642, "manhattan_f1": 0.625475, "manhattan_precision": 0.455276, "manhattan_recall": 0.998897, "manhattan_ap": 0.640307, "euclidean_accuracy": 0.6395, "euclidean_f1": 0.62535, "euclidean_precision": 0.458184, "euclidean_recall": 0.984564, "euclidean_ap": 0.640079, "dot_accuracy": 0.641, "dot_f1": 0.62513, "dot_precision": 0.4556, "dot_recall": 0.99559, "dot_ap": 0.64108, "max_accuracy": 0.642, "max_f1": 0.625475, "max_precision": 0.458184, "max_recall": 0.998897, "max_ap": 0.64108, "main_score": 0.64108, "hf_subset": "es", "languages": [ "spa-Latn" ], "mteb_version": "2.18.0" }, { "similarity_accuracy": 0.6565, "similarity_f1": 0.626686, "similarity_precision": 0.461097, "similarity_recall": 0.977852, "similarity_ap": 0.661767, "cosine_accuracy": 0.6565, "cosine_f1": 0.626686, "cosine_precision": 0.461097, "cosine_recall": 0.977852, "cosine_ap": 0.661962, "manhattan_accuracy": 0.6565, "manhattan_f1": 0.626506, "manhattan_precision": 0.460657, "manhattan_recall": 0.978959, "manhattan_ap": 0.661254, "euclidean_accuracy": 0.656, "euclidean_f1": 0.626686, "euclidean_precision": 0.461097, "euclidean_recall": 0.977852, "euclidean_ap": 0.661953, "dot_accuracy": 0.653, "dot_f1": 0.626557, "dot_precision": 0.4617, "dot_recall": 0.974529, "dot_ap": 0.662458, "max_accuracy": 0.6565, "max_f1": 0.626686, "max_precision": 0.4617, "max_recall": 0.978959, "max_ap": 0.662458, "main_score": 0.662458, "hf_subset": "fr", "languages": [ "fra-Latn" ], "mteb_version": "2.18.0" }, { "similarity_accuracy": 0.59, "similarity_f1": 0.624204, "similarity_precision": 0.453937, "similarity_recall": 0.998867, "similarity_ap": 0.535632, "cosine_accuracy": 0.59, "cosine_f1": 0.624204, "cosine_precision": 0.453937, "cosine_recall": 0.998867, "cosine_ap": 0.536352, "manhattan_accuracy": 0.589, "manhattan_f1": 0.624204, "manhattan_precision": 0.453937, "manhattan_recall": 0.998867, "manhattan_ap": 0.536109, "euclidean_accuracy": 0.5895, "euclidean_f1": 0.624204, "euclidean_precision": 0.453937, "euclidean_recall": 0.998867, "euclidean_ap": 0.536322, "dot_accuracy": 0.591, "dot_f1": 0.624204, "dot_precision": 0.453937, "dot_recall": 0.998867, "dot_ap": 0.536771, "max_accuracy": 0.591, "max_f1": 0.624204, "max_precision": 0.453937, "max_recall": 0.998867, "max_ap": 0.536771, "main_score": 0.536771, "hf_subset": "ja", "languages": [ "jpn-Hira" ], "mteb_version": "2.18.0" }, { "similarity_accuracy": 0.583, "similarity_f1": 0.62548, "similarity_precision": 0.455053, "similarity_recall": 1.0, "similarity_ap": 0.536912, "cosine_accuracy": 0.583, "cosine_f1": 0.62548, "cosine_precision": 0.455053, "cosine_recall": 1.0, "cosine_ap": 0.536872, "manhattan_accuracy": 0.582, "manhattan_f1": 0.62548, "manhattan_precision": 0.455053, "manhattan_recall": 1.0, "manhattan_ap": 0.536873, "euclidean_accuracy": 0.583, "euclidean_f1": 0.62548, "euclidean_precision": 0.455053, "euclidean_recall": 1.0, "euclidean_ap": 0.536866, "dot_accuracy": 0.5815, "dot_f1": 0.62548, "dot_precision": 0.455053, "dot_recall": 1.0, "dot_ap": 0.539007, "max_accuracy": 0.583, "max_f1": 0.62548, "max_precision": 0.455053, "max_recall": 1.0, "max_ap": 0.539007, "main_score": 0.539007, "hf_subset": "ko", "languages": [ "kor-Hang" ], "mteb_version": "2.18.0" }, { "similarity_accuracy": 0.619, "similarity_f1": 0.622563, "similarity_precision": 0.451972, "similarity_recall": 1.0, "similarity_ap": 0.605326, "cosine_accuracy": 0.619, "cosine_f1": 0.622563, "cosine_precision": 0.451972, "cosine_recall": 1.0, "cosine_ap": 0.605727, "manhattan_accuracy": 0.6185, "manhattan_f1": 0.622563, "manhattan_precision": 0.451972, "manhattan_recall": 1.0, "manhattan_ap": 0.605708, "euclidean_accuracy": 0.619, "euclidean_f1": 0.622563, "euclidean_precision": 0.451972, "euclidean_recall": 1.0, "euclidean_ap": 0.605708, "dot_accuracy": 0.62, "dot_f1": 0.622563, "dot_precision": 0.451972, "dot_recall": 1.0, "dot_ap": 0.603879, "max_accuracy": 0.62, "max_f1": 0.622563, "max_precision": 0.451972, "max_recall": 1.0, "max_ap": 0.605727, "main_score": 0.605727, "hf_subset": "zh", "languages": [ "cmn-Hans" ], "mteb_version": "2.18.0" } ], "validation": [ { "similarity_accuracy": 0.6395, "similarity_f1": 0.599711, "similarity_precision": 0.428498, "similarity_recall": 0.998797, "similarity_ap": 0.556565, "cosine_accuracy": 0.6395, "cosine_f1": 0.599711, "cosine_precision": 0.428498, "cosine_recall": 0.998797, "cosine_ap": 0.557304, "manhattan_accuracy": 0.638, "manhattan_f1": 0.599567, "manhattan_precision": 0.42813, "manhattan_recall": 1.0, "manhattan_ap": 0.557219, "euclidean_accuracy": 0.6395, "euclidean_f1": 0.599711, "euclidean_precision": 0.428498, "euclidean_recall": 0.998797, "euclidean_ap": 0.557228, "dot_accuracy": 0.6375, "dot_f1": 0.599711, "dot_precision": 0.428498, "dot_recall": 0.998797, "dot_ap": 0.562447, "max_accuracy": 0.6395, "max_f1": 0.599711, "max_precision": 0.428498, "max_recall": 1.0, "max_ap": 0.562447, "main_score": 0.562447, "hf_subset": "de", "languages": [ "deu-Latn" ], "mteb_version": "2.18.0" }, { "similarity_accuracy": 0.6955, "similarity_f1": 0.648706, "similarity_precision": 0.596117, "similarity_recall": 0.711472, "similarity_ap": 0.677976, "cosine_accuracy": 0.6955, "cosine_f1": 0.648706, "cosine_precision": 0.596117, "cosine_recall": 0.711472, "cosine_ap": 0.677978, "manhattan_accuracy": 0.6935, "manhattan_f1": 0.649077, "manhattan_precision": 0.59593, "manhattan_recall": 0.71263, "manhattan_ap": 0.67777, "euclidean_accuracy": 0.6965, "euclidean_f1": 0.649104, "euclidean_precision": 0.595169, "euclidean_recall": 0.713789, "euclidean_ap": 0.677957, "dot_accuracy": 0.6925, "dot_f1": 0.649419, "dot_precision": 0.596508, "dot_recall": 0.71263, "dot_ap": 0.673999, "max_accuracy": 0.6965, "max_f1": 0.649419, "max_precision": 0.596508, "max_recall": 0.713789, "max_ap": 0.677978, "main_score": 0.677978, "hf_subset": "en", "languages": [ "eng-Latn" ], "mteb_version": "2.18.0" }, { "similarity_accuracy": 0.6545, "similarity_f1": 0.606104, "similarity_precision": 0.435501, "similarity_recall": 0.996458, "similarity_ap": 0.6182, "cosine_accuracy": 0.6545, "cosine_f1": 0.606104, "cosine_precision": 0.435501, "cosine_recall": 0.996458, "cosine_ap": 0.61836, "manhattan_accuracy": 0.653, "manhattan_f1": 0.605776, "manhattan_precision": 0.436297, "manhattan_recall": 0.990555, "manhattan_ap": 0.617959, "euclidean_accuracy": 0.6545, "euclidean_f1": 0.606104, "euclidean_precision": 0.435501, "euclidean_recall": 0.996458, "euclidean_ap": 0.618357, "dot_accuracy": 0.6535, "dot_f1": 0.605603, "dot_precision": 0.435209, "dot_recall": 0.995277, "dot_ap": 0.615132, "max_accuracy": 0.6545, "max_f1": 0.606104, "max_precision": 0.436297, "max_recall": 0.996458, "max_ap": 0.61836, "main_score": 0.61836, "hf_subset": "es", "languages": [ "spa-Latn" ], "mteb_version": "2.18.0" }, { "similarity_accuracy": 0.671, "similarity_f1": 0.610641, "similarity_precision": 0.533449, "similarity_recall": 0.713953, "similarity_ap": 0.659619, "cosine_accuracy": 0.671, "cosine_f1": 0.610641, "cosine_precision": 0.533449, "cosine_recall": 0.713953, "cosine_ap": 0.660076, "manhattan_accuracy": 0.6715, "manhattan_f1": 0.610537, "manhattan_precision": 0.529462, "manhattan_recall": 0.72093, "manhattan_ap": 0.659851, "euclidean_accuracy": 0.671, "euclidean_f1": 0.610672, "euclidean_precision": 0.530928, "euclidean_recall": 0.718605, "euclidean_ap": 0.660111, "dot_accuracy": 0.672, "dot_f1": 0.609626, "dot_precision": 0.52381, "dot_recall": 0.72907, "dot_ap": 0.659905, "max_accuracy": 0.672, "max_f1": 0.610672, "max_precision": 0.533449, "max_recall": 0.72907, "max_ap": 0.660111, "main_score": 0.660111, "hf_subset": "fr", "languages": [ "fra-Latn" ], "mteb_version": "2.18.0" }, { "similarity_accuracy": 0.6255, "similarity_f1": 0.60212, "similarity_precision": 0.431174, "similarity_recall": 0.997658, "similarity_ap": 0.5743, "cosine_accuracy": 0.6255, "cosine_f1": 0.60212, "cosine_precision": 0.431174, "cosine_recall": 0.997658, "cosine_ap": 0.575608, "manhattan_accuracy": 0.625, "manhattan_f1": 0.601983, "manhattan_precision": 0.431472, "manhattan_recall": 0.995316, "manhattan_ap": 0.57571, "euclidean_accuracy": 0.6255, "euclidean_f1": 0.60212, "euclidean_precision": 0.431174, "euclidean_recall": 0.997658, "euclidean_ap": 0.57557, "dot_accuracy": 0.6255, "dot_f1": 0.60212, "dot_precision": 0.431174, "dot_recall": 0.997658, "dot_ap": 0.577667, "max_accuracy": 0.6255, "max_f1": 0.60212, "max_precision": 0.431472, "max_recall": 0.997658, "max_ap": 0.577667, "main_score": 0.577667, "hf_subset": "ja", "languages": [ "jpn-Hira" ], "mteb_version": "2.18.0" }, { "similarity_accuracy": 0.6195, "similarity_f1": 0.596529, "similarity_precision": 0.428349, "similarity_recall": 0.982143, "similarity_ap": 0.507649, "cosine_accuracy": 0.6195, "cosine_f1": 0.596529, "cosine_precision": 0.428349, "cosine_recall": 0.982143, "cosine_ap": 0.506369, "manhattan_accuracy": 0.619, "manhattan_f1": 0.596669, "manhattan_precision": 0.42872, "manhattan_recall": 0.980952, "manhattan_ap": 0.506536, "euclidean_accuracy": 0.619, "euclidean_f1": 0.596529, "euclidean_precision": 0.428349, "euclidean_recall": 0.982143, "euclidean_ap": 0.506347, "dot_accuracy": 0.6175, "dot_f1": 0.596529, "dot_precision": 0.428349, "dot_recall": 0.982143, "dot_ap": 0.505437, "max_accuracy": 0.6195, "max_f1": 0.596669, "max_precision": 0.42872, "max_recall": 0.982143, "max_ap": 0.507649, "main_score": 0.507649, "hf_subset": "ko", "languages": [ "kor-Hang" ], "mteb_version": "2.18.0" }, { "similarity_accuracy": 0.617, "similarity_f1": 0.601764, "similarity_precision": 0.430373, "similarity_recall": 1.0, "similarity_ap": 0.559716, "cosine_accuracy": 0.617, "cosine_f1": 0.601764, "cosine_precision": 0.430373, "cosine_recall": 1.0, "cosine_ap": 0.560303, "manhattan_accuracy": 0.6165, "manhattan_f1": 0.601764, "manhattan_precision": 0.430373, "manhattan_recall": 1.0, "manhattan_ap": 0.560192, "euclidean_accuracy": 0.617, "euclidean_f1": 0.601764, "euclidean_precision": 0.430373, "euclidean_recall": 1.0, "euclidean_ap": 0.560307, "dot_accuracy": 0.616, "dot_f1": 0.601764, "dot_precision": 0.430373, "dot_recall": 1.0, "dot_ap": 0.563612, "max_accuracy": 0.617, "max_f1": 0.601764, "max_precision": 0.430373, "max_recall": 1.0, "max_ap": 0.563612, "main_score": 0.563612, "hf_subset": "zh", "languages": [ "cmn-Hans" ], "mteb_version": "2.18.0" } ] }, "evaluation_time": 722.0668402537704, "kg_co2_emissions": null, "date": 1785368020.108128, "evaluation_phases": [ { "name": "Data loading", "start": 4.552304744720459e-06, "end": 7.0820949748158455, "split": "", "subset": "" }, { "name": "Dataset transform", "start": 7.082104787230492, "end": 7.082107305526733, "split": "", "subset": "" }, { "name": "Encoding samples 1", "start": 7.090980388224125, "end": 32.34516243636608, "split": "test", "subset": "de" }, { "name": "Encoding samples 2", "start": 32.34522105008364, "end": 57.53475929796696, "split": "test", "subset": "de" }, { "name": "Scoring", "start": 57.534782879054546, "end": 57.69727285206318, "split": "test", "subset": "de" }, { "name": "Encoding samples 1", "start": 57.75227329134941, "end": 83.34019578248262, "split": "test", "subset": "en" }, { "name": "Encoding samples 2", "start": 83.34024676680565, "end": 108.70852465182543, "split": "test", "subset": "en" }, { "name": "Scoring", "start": 108.70855731517076, "end": 108.81873634457588, "split": "test", "subset": "en" }, { "name": "Encoding samples 1", "start": 108.88539209216833, "end": 134.3343731239438, "split": "test", "subset": "es" }, { "name": "Encoding samples 2", "start": 134.33438970148563, "end": 159.81804107874632, "split": "test", "subset": "es" }, { "name": "Scoring", "start": 159.81806648522615, "end": 159.91421289741993, "split": "test", "subset": "es" }, { "name": "Encoding samples 1", "start": 160.38801631331444, "end": 185.91818902641535, "split": "test", "subset": "fr" }, { "name": "Encoding samples 2", "start": 185.91825184226036, "end": 211.38256082683802, "split": "test", "subset": "fr" }, { "name": "Scoring", "start": 211.38259229809046, "end": 211.51846216619015, "split": "test", "subset": "fr" }, { "name": "Encoding samples 1", "start": 211.790193118155, "end": 237.20645006000996, "split": "test", "subset": "ja" }, { "name": "Encoding samples 2", "start": 237.20646622031927, "end": 262.74422542750835, "split": "test", "subset": "ja" }, { "name": "Scoring", "start": 262.7442393004894, "end": 262.8220177292824, "split": "test", "subset": "ja" }, { "name": "Encoding samples 1", "start": 262.8727720975876, "end": 288.41766241937876, "split": "test", "subset": "ko" }, { "name": "Encoding samples 2", "start": 288.41767900437117, "end": 314.20426443964243, "split": "test", "subset": "ko" }, { "name": "Scoring", "start": 314.20428091287613, "end": 314.3188491612673, "split": "test", "subset": "ko" }, { "name": "Encoding samples 1", "start": 314.3767549917102, "end": 339.59033035486937, "split": "test", "subset": "zh" }, { "name": "Encoding samples 2", "start": 339.5903450772166, "end": 364.97871516644955, "split": "test", "subset": "zh" }, { "name": "Scoring", "start": 364.9787452891469, "end": 365.1166500598192, "split": "test", "subset": "zh" }, { "name": "Encoding samples 1", "start": 365.16957726329565, "end": 390.26379316300154, "split": "validation", "subset": "de" }, { "name": "Encoding samples 2", "start": 390.263818718493, "end": 415.5333357229829, "split": "validation", "subset": "de" }, { "name": "Scoring", "start": 415.53335332125425, "end": 415.63563350588083, "split": "validation", "subset": "de" }, { "name": "Encoding samples 1", "start": 415.73304560780525, "end": 441.13524609804153, "split": "validation", "subset": "en" }, { "name": "Encoding samples 2", "start": 441.13527539372444, "end": 466.4590546488762, "split": "validation", "subset": "en" }, { "name": "Scoring", "start": 466.45906821638346, "end": 466.61099330335855, "split": "validation", "subset": "en" }, { "name": "Encoding samples 1", "start": 466.66883589327335, "end": 491.839285582304, "split": "validation", "subset": "es" }, { "name": "Encoding samples 2", "start": 491.8393008932471, "end": 516.9870511516929, "split": "validation", "subset": "es" }, { "name": "Scoring", "start": 516.9870652332902, "end": 517.1164532005787, "split": "validation", "subset": "es" }, { "name": "Encoding samples 1", "start": 517.1758000180125, "end": 542.5282067880034, "split": "validation", "subset": "fr" }, { "name": "Encoding samples 2", "start": 542.5282218903303, "end": 567.8841849491, "split": "validation", "subset": "fr" }, { "name": "Scoring", "start": 567.8841984272003, "end": 568.0184906646609, "split": "validation", "subset": "fr" }, { "name": "Encoding samples 1", "start": 568.0710633918643, "end": 593.6508400663733, "split": "validation", "subset": "ja" }, { "name": "Encoding samples 2", "start": 593.6508586704731, "end": 619.2436277270317, "split": "validation", "subset": "ja" }, { "name": "Scoring", "start": 619.2436419650912, "end": 619.3899345621467, "split": "validation", "subset": "ja" }, { "name": "Encoding samples 1", "start": 619.4486864283681, "end": 645.0236921682954, "split": "validation", "subset": "ko" }, { "name": "Encoding samples 2", "start": 645.0237073153257, "end": 670.5069700405002, "split": "validation", "subset": "ko" }, { "name": "Scoring", "start": 670.5069831460714, "end": 670.6244450956583, "split": "validation", "subset": "ko" }, { "name": "Encoding samples 1", "start": 670.6841503307223, "end": 696.1277118325233, "split": "validation", "subset": "zh" }, { "name": "Encoding samples 2", "start": 696.1277482658625, "end": 721.861158490181, "split": "validation", "subset": "zh" }, { "name": "Scoring", "start": 721.8611769527197, "end": 722.0128759220243, "split": "validation", "subset": "zh" } ] }