{ "dataset_revision": "29afa2569dcedaaa2fe6a3dcfebab33d28b82e8c", "task_name": "STSBenchmarkMultilingualSTS", "mteb_version": "1.18.0", "scores": { "dev": [ { "pearson": 0.695737, "spearman": 0.698098, "cosine_pearson": 0.695737, "cosine_spearman": 0.698099, "manhattan_pearson": 0.691706, "manhattan_spearman": 0.703991, "euclidean_pearson": 0.692174, "euclidean_spearman": 0.705204, "main_score": 0.698099, "hf_subset": "de", "languages": [ "deu-Latn" ] }, { "pearson": 0.698872, "spearman": 0.743574, "cosine_pearson": 0.698872, "cosine_spearman": 0.743574, "manhattan_pearson": 0.790812, "manhattan_spearman": 0.791593, "euclidean_pearson": 0.790448, "euclidean_spearman": 0.791216, "main_score": 0.743574, "hf_subset": "en", "languages": [ "eng-Latn" ] }, { "pearson": 0.629523, "spearman": 0.634149, "cosine_pearson": 0.629523, "cosine_spearman": 0.634149, "manhattan_pearson": 0.648533, "manhattan_spearman": 0.654762, "euclidean_pearson": 0.649111, "euclidean_spearman": 0.655941, "main_score": 0.634149, "hf_subset": "pt", "languages": [ "por-Latn" ] }, { "pearson": 0.661887, "spearman": 0.66586, "cosine_pearson": 0.661887, "cosine_spearman": 0.66586, "manhattan_pearson": 0.682499, "manhattan_spearman": 0.686238, "euclidean_pearson": 0.683125, "euclidean_spearman": 0.687477, "main_score": 0.66586, "hf_subset": "es", "languages": [ "spa-Latn" ] }, { "pearson": 0.689153, "spearman": 0.692721, "cosine_pearson": 0.689153, "cosine_spearman": 0.69272, "manhattan_pearson": 0.699662, "manhattan_spearman": 0.707651, "euclidean_pearson": 0.701659, "euclidean_spearman": 0.709503, "main_score": 0.69272, "hf_subset": "fr", "languages": [ "fra-Latn" ] }, { "pearson": 0.61784, "spearman": 0.634008, "cosine_pearson": 0.61784, "cosine_spearman": 0.634008, "manhattan_pearson": 0.624125, "manhattan_spearman": 0.632011, "euclidean_pearson": 0.622616, "euclidean_spearman": 0.631778, "main_score": 0.634008, "hf_subset": "pl", "languages": [ "pol-Latn" ] }, { "pearson": 0.649493, "spearman": 0.660471, "cosine_pearson": 0.649493, "cosine_spearman": 0.660471, "manhattan_pearson": 0.663561, "manhattan_spearman": 0.672665, "euclidean_pearson": 0.662808, "euclidean_spearman": 0.672994, "main_score": 0.660471, "hf_subset": "nl", "languages": [ "nld-Latn" ] }, { "pearson": 0.671717, "spearman": 0.676205, "cosine_pearson": 0.671717, "cosine_spearman": 0.676205, "manhattan_pearson": 0.682192, "manhattan_spearman": 0.690102, "euclidean_pearson": 0.683741, "euclidean_spearman": 0.691773, "main_score": 0.676205, "hf_subset": "it", "languages": [ "ita-Latn" ] }, { "pearson": 0.459934, "spearman": 0.474539, "cosine_pearson": 0.459934, "cosine_spearman": 0.474534, "manhattan_pearson": 0.49194, "manhattan_spearman": 0.473095, "euclidean_pearson": 0.492654, "euclidean_spearman": 0.47474, "main_score": 0.474534, "hf_subset": "ru", "languages": [ "rus-Cyrl" ] }, { "pearson": 0.415857, "spearman": 0.427848, "cosine_pearson": 0.415857, "cosine_spearman": 0.427737, "manhattan_pearson": 0.438109, "manhattan_spearman": 0.43419, "euclidean_pearson": 0.439728, "euclidean_spearman": 0.436198, "main_score": 0.427737, "hf_subset": "zh", "languages": [ "cmn-Hans" ] } ], "test": [ { "pearson": 0.610911, "spearman": 0.600205, "cosine_pearson": 0.610911, "cosine_spearman": 0.600206, "manhattan_pearson": 0.613946, "manhattan_spearman": 0.60379, "euclidean_pearson": 0.615348, "euclidean_spearman": 0.605599, "main_score": 0.600206, "hf_subset": "de", "languages": [ "deu-Latn" ] }, { "pearson": 0.754121, "spearman": 0.76722, "cosine_pearson": 0.754121, "cosine_spearman": 0.76722, "manhattan_pearson": 0.798987, "manhattan_spearman": 0.79361, "euclidean_pearson": 0.799793, "euclidean_spearman": 0.793635, "main_score": 0.76722, "hf_subset": "en", "languages": [ "eng-Latn" ] }, { "pearson": 0.552039, "spearman": 0.554906, "cosine_pearson": 0.552039, "cosine_spearman": 0.554904, "manhattan_pearson": 0.57587, "manhattan_spearman": 0.573625, "euclidean_pearson": 0.577827, "euclidean_spearman": 0.576227, "main_score": 0.554904, "hf_subset": "pt", "languages": [ "por-Latn" ] }, { "pearson": 0.581988, "spearman": 0.589484, "cosine_pearson": 0.581988, "cosine_spearman": 0.58948, "manhattan_pearson": 0.608498, "manhattan_spearman": 0.610113, "euclidean_pearson": 0.608379, "euclidean_spearman": 0.610376, "main_score": 0.58948, "hf_subset": "es", "languages": [ "spa-Latn" ] }, { "pearson": 0.602032, "spearman": 0.601147, "cosine_pearson": 0.602032, "cosine_spearman": 0.601164, "manhattan_pearson": 0.619956, "manhattan_spearman": 0.61629, "euclidean_pearson": 0.620558, "euclidean_spearman": 0.617746, "main_score": 0.601164, "hf_subset": "fr", "languages": [ "fra-Latn" ] }, { "pearson": 0.576816, "spearman": 0.575318, "cosine_pearson": 0.576816, "cosine_spearman": 0.575329, "manhattan_pearson": 0.57848, "manhattan_spearman": 0.571059, "euclidean_pearson": 0.579378, "euclidean_spearman": 0.572711, "main_score": 0.575329, "hf_subset": "pl", "languages": [ "pol-Latn" ] }, { "pearson": 0.561952, "spearman": 0.557943, "cosine_pearson": 0.561952, "cosine_spearman": 0.557938, "manhattan_pearson": 0.576225, "manhattan_spearman": 0.566359, "euclidean_pearson": 0.579533, "euclidean_spearman": 0.570181, "main_score": 0.557938, "hf_subset": "nl", "languages": [ "nld-Latn" ] }, { "pearson": 0.593288, "spearman": 0.584301, "cosine_pearson": 0.593288, "cosine_spearman": 0.584304, "manhattan_pearson": 0.604041, "manhattan_spearman": 0.602091, "euclidean_pearson": 0.603221, "euclidean_spearman": 0.600671, "main_score": 0.584304, "hf_subset": "it", "languages": [ "ita-Latn" ] }, { "pearson": 0.454076, "spearman": 0.465111, "cosine_pearson": 0.454076, "cosine_spearman": 0.465114, "manhattan_pearson": 0.475075, "manhattan_spearman": 0.462958, "euclidean_pearson": 0.475109, "euclidean_spearman": 0.463201, "main_score": 0.465114, "hf_subset": "ru", "languages": [ "rus-Cyrl" ] }, { "pearson": 0.326738, "spearman": 0.366942, "cosine_pearson": 0.326738, "cosine_spearman": 0.366769, "manhattan_pearson": 0.357655, "manhattan_spearman": 0.367556, "euclidean_pearson": 0.358615, "euclidean_spearman": 0.368944, "main_score": 0.366769, "hf_subset": "zh", "languages": [ "cmn-Hans" ] } ] }, "evaluation_time": 5.231090784072876, "kg_co2_emissions": 0.0001740896187340386 }