{ "dataset_revision": "a54635a274ef2835721b7cbe3eb27483b9ec964b", "task_name": "Vidore3TelecomRetrieval", "mteb_version": "2.22.2", "scores": { "test": [ { "ndcg_at_1": 0.69971, "ndcg_at_3": 0.68401, "ndcg_at_5": 0.69129, "ndcg_at_10": 0.7132, "ndcg_at_20": 0.74311, "ndcg_at_100": 0.7843, "ndcg_at_1000": 0.79288, "map_at_1": 0.32132, "map_at_3": 0.49629, "map_at_5": 0.55008, "map_at_10": 0.59923, "map_at_20": 0.62535, "map_at_100": 0.6456, "map_at_1000": 0.64693, "recall_at_1": 0.32132, "recall_at_3": 0.5446, "recall_at_5": 0.64074, "recall_at_10": 0.74463, "recall_at_20": 0.83686, "recall_at_100": 0.95933, "recall_at_1000": 0.9973, "accuracy": 0.32132, "precision_at_1": 0.72126, "precision_at_3": 0.50766, "precision_at_5": 0.3908, "precision_at_10": 0.25833, "precision_at_20": 0.15905, "precision_at_100": 0.04187, "precision_at_1000": 0.00447, "mrr_at_1": 0.721264, "mrr_at_3": 0.803161, "mrr_at_5": 0.813075, "mrr_at_10": 0.815787, "mrr_at_20": 0.817803, "mrr_at_100": 0.817939, "mrr_at_1000": 0.817961, "nauc_ndcg_at_1_max": 0.021335, "nauc_ndcg_at_1_std": -0.095868, "nauc_ndcg_at_1_diff1": 0.385765, "nauc_ndcg_at_3_max": -0.070494, "nauc_ndcg_at_3_std": -0.144784, "nauc_ndcg_at_3_diff1": 0.331704, "nauc_ndcg_at_5_max": -0.033555, "nauc_ndcg_at_5_std": -0.131027, "nauc_ndcg_at_5_diff1": 0.358376, "nauc_ndcg_at_10_max": -0.021259, "nauc_ndcg_at_10_std": -0.133386, "nauc_ndcg_at_10_diff1": 0.378072, "nauc_ndcg_at_20_max": 0.00599, "nauc_ndcg_at_20_std": -0.098985, "nauc_ndcg_at_20_diff1": 0.378825, "nauc_ndcg_at_100_max": 0.021166, "nauc_ndcg_at_100_std": -0.060415, "nauc_ndcg_at_100_diff1": 0.371788, "nauc_ndcg_at_1000_max": 0.004531, "nauc_ndcg_at_1000_std": -0.079504, "nauc_ndcg_at_1000_diff1": 0.368494, "nauc_map_at_1_max": -0.017388, "nauc_map_at_1_std": -0.149148, "nauc_map_at_1_diff1": 0.451607, "nauc_map_at_3_max": -0.023588, "nauc_map_at_3_std": -0.153197, "nauc_map_at_3_diff1": 0.373612, "nauc_map_at_5_max": -0.015214, "nauc_map_at_5_std": -0.144389, "nauc_map_at_5_diff1": 0.367465, "nauc_map_at_10_max": -0.016212, "nauc_map_at_10_std": -0.143447, "nauc_map_at_10_diff1": 0.363082, "nauc_map_at_20_max": -0.002104, "nauc_map_at_20_std": -0.121132, "nauc_map_at_20_diff1": 0.364024, "nauc_map_at_100_max": 0.008585, "nauc_map_at_100_std": -0.099496, "nauc_map_at_100_diff1": 0.361944, "nauc_map_at_1000_max": 0.006917, "nauc_map_at_1000_std": -0.101099, "nauc_map_at_1000_diff1": 0.361695, "nauc_recall_at_1_max": -0.017388, "nauc_recall_at_1_std": -0.149148, "nauc_recall_at_1_diff1": 0.451607, "nauc_recall_at_3_max": -0.045007, "nauc_recall_at_3_std": -0.163669, "nauc_recall_at_3_diff1": 0.343178, "nauc_recall_at_5_max": -0.016961, "nauc_recall_at_5_std": -0.150747, "nauc_recall_at_5_diff1": 0.321166, "nauc_recall_at_10_max": -0.011463, "nauc_recall_at_10_std": -0.151618, "nauc_recall_at_10_diff1": 0.341737, "nauc_recall_at_20_max": 0.084284, "nauc_recall_at_20_std": -0.024013, "nauc_recall_at_20_diff1": 0.331484, "nauc_recall_at_100_max": 0.372226, "nauc_recall_at_100_std": 0.38365, "nauc_recall_at_100_diff1": 0.359231, "nauc_recall_at_1000_max": 0.460028, "nauc_recall_at_1000_std": 0.636453, "nauc_recall_at_1000_diff1": -0.125166, "nauc_precision_at_1_max": -0.012016, "nauc_precision_at_1_std": -0.122338, "nauc_precision_at_1_diff1": 0.386431, "nauc_precision_at_3_max": -0.029429, "nauc_precision_at_3_std": 0.00401, "nauc_precision_at_3_diff1": -0.036353, "nauc_precision_at_5_max": -0.002943, "nauc_precision_at_5_std": 0.058373, "nauc_precision_at_5_diff1": -0.095165, "nauc_precision_at_10_max": 0.022761, "nauc_precision_at_10_std": 0.103923, "nauc_precision_at_10_diff1": -0.140952, "nauc_precision_at_20_max": 0.05696, "nauc_precision_at_20_std": 0.176158, "nauc_precision_at_20_diff1": -0.186518, "nauc_precision_at_100_max": 0.044136, "nauc_precision_at_100_std": 0.192302, "nauc_precision_at_100_diff1": -0.227064, "nauc_precision_at_1000_max": 0.008401, "nauc_precision_at_1000_std": 0.149265, "nauc_precision_at_1000_diff1": -0.238835, "nauc_mrr_at_1_max": -0.012016, "nauc_mrr_at_1_std": -0.122338, "nauc_mrr_at_1_diff1": 0.386431, "nauc_mrr_at_3_max": -0.067969, "nauc_mrr_at_3_std": -0.138141, "nauc_mrr_at_3_diff1": 0.355183, "nauc_mrr_at_5_max": -0.04825, "nauc_mrr_at_5_std": -0.130102, "nauc_mrr_at_5_diff1": 0.362124, "nauc_mrr_at_10_max": -0.054165, "nauc_mrr_at_10_std": -0.137334, "nauc_mrr_at_10_diff1": 0.365124, "nauc_mrr_at_20_max": -0.049936, "nauc_mrr_at_20_std": -0.132755, "nauc_mrr_at_20_diff1": 0.365292, "nauc_mrr_at_100_max": -0.050186, "nauc_mrr_at_100_std": -0.13298, "nauc_mrr_at_100_diff1": 0.364833, "nauc_mrr_at_1000_max": -0.050303, "nauc_mrr_at_1000_std": -0.133121, "nauc_mrr_at_1000_diff1": 0.364759, "hit_rate_at_1": 0.72126, "hit_rate_at_3": 0.89943, "hit_rate_at_5": 0.94253, "hit_rate_at_10": 0.96264, "hit_rate_at_20": 0.98851, "hit_rate_at_100": 0.99425, "hit_rate_at_1000": 1.0, "main_score": 0.7132, "hf_subset": "french", "languages": [ "fra-Latn" ], "mteb_version": "2.22.2" }, { "ndcg_at_1": 0.68822, "ndcg_at_3": 0.67727, "ndcg_at_5": 0.68565, "ndcg_at_10": 0.70689, "ndcg_at_20": 0.74068, "ndcg_at_100": 0.78209, "ndcg_at_1000": 0.78926, "map_at_1": 0.31452, "map_at_3": 0.492, "map_at_5": 0.54728, "map_at_10": 0.5965, "map_at_20": 0.62531, "map_at_100": 0.64554, "map_at_1000": 0.64683, "recall_at_1": 0.31452, "recall_at_3": 0.5437, "recall_at_5": 0.63622, "recall_at_10": 0.73936, "recall_at_20": 0.84207, "recall_at_100": 0.96558, "recall_at_1000": 0.9973, "accuracy": 0.31452, "precision_at_1": 0.71839, "precision_at_3": 0.50096, "precision_at_5": 0.38966, "precision_at_10": 0.25833, "precision_at_20": 0.16207, "precision_at_100": 0.04227, "precision_at_1000": 0.00447, "mrr_at_1": 0.718391, "mrr_at_3": 0.800287, "mrr_at_5": 0.809195, "mrr_at_10": 0.813046, "mrr_at_20": 0.81429, "mrr_at_100": 0.814695, "mrr_at_1000": 0.81472, "nauc_ndcg_at_1_max": 0.099104, "nauc_ndcg_at_1_std": -0.040915, "nauc_ndcg_at_1_diff1": 0.484068, "nauc_ndcg_at_3_max": 0.003776, "nauc_ndcg_at_3_std": -0.123084, "nauc_ndcg_at_3_diff1": 0.365875, "nauc_ndcg_at_5_max": 0.019941, "nauc_ndcg_at_5_std": -0.123725, "nauc_ndcg_at_5_diff1": 0.364606, "nauc_ndcg_at_10_max": 0.035434, "nauc_ndcg_at_10_std": -0.120213, "nauc_ndcg_at_10_diff1": 0.380864, "nauc_ndcg_at_20_max": 0.053597, "nauc_ndcg_at_20_std": -0.095219, "nauc_ndcg_at_20_diff1": 0.389735, "nauc_ndcg_at_100_max": 0.081331, "nauc_ndcg_at_100_std": -0.042998, "nauc_ndcg_at_100_diff1": 0.393226, "nauc_ndcg_at_1000_max": 0.06901, "nauc_ndcg_at_1000_std": -0.055113, "nauc_ndcg_at_1000_diff1": 0.397519, "nauc_map_at_1_max": 0.012916, "nauc_map_at_1_std": -0.130457, "nauc_map_at_1_diff1": 0.475093, "nauc_map_at_3_max": 0.015383, "nauc_map_at_3_std": -0.149995, "nauc_map_at_3_diff1": 0.373937, "nauc_map_at_5_max": 0.018672, "nauc_map_at_5_std": -0.151853, "nauc_map_at_5_diff1": 0.361447, "nauc_map_at_10_max": 0.023761, "nauc_map_at_10_std": -0.144681, "nauc_map_at_10_diff1": 0.35505, "nauc_map_at_20_max": 0.039255, "nauc_map_at_20_std": -0.119586, "nauc_map_at_20_diff1": 0.357034, "nauc_map_at_100_max": 0.055163, "nauc_map_at_100_std": -0.092138, "nauc_map_at_100_diff1": 0.357883, "nauc_map_at_1000_max": 0.053833, "nauc_map_at_1000_std": -0.093031, "nauc_map_at_1000_diff1": 0.358712, "nauc_recall_at_1_max": 0.012916, "nauc_recall_at_1_std": -0.130457, "nauc_recall_at_1_diff1": 0.475093, "nauc_recall_at_3_max": -0.02148, "nauc_recall_at_3_std": -0.181389, "nauc_recall_at_3_diff1": 0.308965, "nauc_recall_at_5_max": 0.006724, "nauc_recall_at_5_std": -0.171373, "nauc_recall_at_5_diff1": 0.280719, "nauc_recall_at_10_max": 0.018217, "nauc_recall_at_10_std": -0.168259, "nauc_recall_at_10_diff1": 0.273727, "nauc_recall_at_20_max": 0.08382, "nauc_recall_at_20_std": -0.077614, "nauc_recall_at_20_diff1": 0.292317, "nauc_recall_at_100_max": 0.416235, "nauc_recall_at_100_std": 0.369835, "nauc_recall_at_100_diff1": 0.201656, "nauc_recall_at_1000_max": 0.529596, "nauc_recall_at_1000_std": 0.710639, "nauc_recall_at_1000_diff1": -0.046865, "nauc_precision_at_1_max": 0.077816, "nauc_precision_at_1_std": -0.045856, "nauc_precision_at_1_diff1": 0.485778, "nauc_precision_at_3_max": 0.032645, "nauc_precision_at_3_std": 0.016005, "nauc_precision_at_3_diff1": 0.003661, "nauc_precision_at_5_max": 0.022814, "nauc_precision_at_5_std": 0.04987, "nauc_precision_at_5_diff1": -0.095765, "nauc_precision_at_10_max": 0.050896, "nauc_precision_at_10_std": 0.122268, "nauc_precision_at_10_diff1": -0.133912, "nauc_precision_at_20_max": 0.078813, "nauc_precision_at_20_std": 0.205951, "nauc_precision_at_20_diff1": -0.175455, "nauc_precision_at_100_max": 0.071901, "nauc_precision_at_100_std": 0.243372, "nauc_precision_at_100_diff1": -0.198092, "nauc_precision_at_1000_max": 0.040093, "nauc_precision_at_1000_std": 0.209334, "nauc_precision_at_1000_diff1": -0.197768, "nauc_mrr_at_1_max": 0.077816, "nauc_mrr_at_1_std": -0.045856, "nauc_mrr_at_1_diff1": 0.485778, "nauc_mrr_at_3_max": 0.043693, "nauc_mrr_at_3_std": -0.063488, "nauc_mrr_at_3_diff1": 0.479383, "nauc_mrr_at_5_max": 0.061728, "nauc_mrr_at_5_std": -0.049555, "nauc_mrr_at_5_diff1": 0.479129, "nauc_mrr_at_10_max": 0.049968, "nauc_mrr_at_10_std": -0.060652, "nauc_mrr_at_10_diff1": 0.479941, "nauc_mrr_at_20_max": 0.054449, "nauc_mrr_at_20_std": -0.056393, "nauc_mrr_at_20_diff1": 0.481945, "nauc_mrr_at_100_max": 0.055908, "nauc_mrr_at_100_std": -0.055103, "nauc_mrr_at_100_diff1": 0.481209, "nauc_mrr_at_1000_max": 0.055774, "nauc_mrr_at_1000_std": -0.055254, "nauc_mrr_at_1000_diff1": 0.481184, "hit_rate_at_1": 0.71839, "hit_rate_at_3": 0.89943, "hit_rate_at_5": 0.93678, "hit_rate_at_10": 0.96552, "hit_rate_at_20": 0.98276, "hit_rate_at_100": 0.99425, "hit_rate_at_1000": 1.0, "main_score": 0.70689, "hf_subset": "spanish", "languages": [ "spa-Latn" ], "mteb_version": "2.22.2" }, { "ndcg_at_1": 0.70546, "ndcg_at_3": 0.68161, "ndcg_at_5": 0.69477, "ndcg_at_10": 0.71837, "ndcg_at_20": 0.75186, "ndcg_at_100": 0.79033, "ndcg_at_1000": 0.79806, "map_at_1": 0.32424, "map_at_3": 0.4951, "map_at_5": 0.55208, "map_at_10": 0.60531, "map_at_20": 0.63567, "map_at_100": 0.65509, "map_at_1000": 0.65656, "recall_at_1": 0.32424, "recall_at_3": 0.54564, "recall_at_5": 0.64054, "recall_at_10": 0.7519, "recall_at_20": 0.85228, "recall_at_100": 0.96551, "recall_at_1000": 0.99779, "accuracy": 0.32424, "precision_at_1": 0.73276, "precision_at_3": 0.50287, "precision_at_5": 0.3954, "precision_at_10": 0.26293, "precision_at_20": 0.16451, "precision_at_100": 0.04198, "precision_at_1000": 0.00447, "mrr_at_1": 0.732759, "mrr_at_3": 0.809387, "mrr_at_5": 0.819444, "mrr_at_10": 0.822636, "mrr_at_20": 0.823521, "mrr_at_100": 0.823816, "mrr_at_1000": 0.823825, "nauc_ndcg_at_1_max": 0.059458, "nauc_ndcg_at_1_std": -0.089864, "nauc_ndcg_at_1_diff1": 0.453519, "nauc_ndcg_at_3_max": 0.03226, "nauc_ndcg_at_3_std": -0.119174, "nauc_ndcg_at_3_diff1": 0.374759, "nauc_ndcg_at_5_max": 0.040989, "nauc_ndcg_at_5_std": -0.113104, "nauc_ndcg_at_5_diff1": 0.378071, "nauc_ndcg_at_10_max": 0.054297, "nauc_ndcg_at_10_std": -0.120866, "nauc_ndcg_at_10_diff1": 0.386538, "nauc_ndcg_at_20_max": 0.08154, "nauc_ndcg_at_20_std": -0.087577, "nauc_ndcg_at_20_diff1": 0.390679, "nauc_ndcg_at_100_max": 0.086083, "nauc_ndcg_at_100_std": -0.058089, "nauc_ndcg_at_100_diff1": 0.393319, "nauc_ndcg_at_1000_max": 0.072922, "nauc_ndcg_at_1000_std": -0.068218, "nauc_ndcg_at_1000_diff1": 0.393669, "nauc_map_at_1_max": 0.005324, "nauc_map_at_1_std": -0.136468, "nauc_map_at_1_diff1": 0.457658, "nauc_map_at_3_max": 0.052245, "nauc_map_at_3_std": -0.129253, "nauc_map_at_3_diff1": 0.393837, "nauc_map_at_5_max": 0.05324, "nauc_map_at_5_std": -0.127135, "nauc_map_at_5_diff1": 0.367024, "nauc_map_at_10_max": 0.050937, "nauc_map_at_10_std": -0.128189, "nauc_map_at_10_diff1": 0.35389, "nauc_map_at_20_max": 0.065943, "nauc_map_at_20_std": -0.104329, "nauc_map_at_20_diff1": 0.35294, "nauc_map_at_100_max": 0.074462, "nauc_map_at_100_std": -0.085412, "nauc_map_at_100_diff1": 0.351718, "nauc_map_at_1000_max": 0.072864, "nauc_map_at_1000_std": -0.086361, "nauc_map_at_1000_diff1": 0.351769, "nauc_recall_at_1_max": 0.005324, "nauc_recall_at_1_std": -0.136468, "nauc_recall_at_1_diff1": 0.457658, "nauc_recall_at_3_max": 0.052411, "nauc_recall_at_3_std": -0.142644, "nauc_recall_at_3_diff1": 0.359484, "nauc_recall_at_5_max": 0.064869, "nauc_recall_at_5_std": -0.134795, "nauc_recall_at_5_diff1": 0.328182, "nauc_recall_at_10_max": 0.07591, "nauc_recall_at_10_std": -0.142991, "nauc_recall_at_10_diff1": 0.298823, "nauc_recall_at_20_max": 0.178263, "nauc_recall_at_20_std": -0.032503, "nauc_recall_at_20_diff1": 0.292198, "nauc_recall_at_100_max": 0.387861, "nauc_recall_at_100_std": 0.265873, "nauc_recall_at_100_diff1": 0.317904, "nauc_recall_at_1000_max": 0.623597, "nauc_recall_at_1000_std": 0.531915, "nauc_recall_at_1000_diff1": 0.42713, "nauc_precision_at_1_max": 0.04169, "nauc_precision_at_1_std": -0.10349, "nauc_precision_at_1_diff1": 0.454918, "nauc_precision_at_3_max": 0.064273, "nauc_precision_at_3_std": 0.033133, "nauc_precision_at_3_diff1": -0.007004, "nauc_precision_at_5_max": 0.021647, "nauc_precision_at_5_std": 0.055655, "nauc_precision_at_5_diff1": -0.124593, "nauc_precision_at_10_max": 0.017503, "nauc_precision_at_10_std": 0.087526, "nauc_precision_at_10_diff1": -0.188056, "nauc_precision_at_20_max": 0.041899, "nauc_precision_at_20_std": 0.166473, "nauc_precision_at_20_diff1": -0.218375, "nauc_precision_at_100_max": 0.027711, "nauc_precision_at_100_std": 0.192457, "nauc_precision_at_100_diff1": -0.238734, "nauc_precision_at_1000_max": -0.008584, "nauc_precision_at_1000_std": 0.158045, "nauc_precision_at_1000_diff1": -0.241197, "nauc_mrr_at_1_max": 0.04169, "nauc_mrr_at_1_std": -0.10349, "nauc_mrr_at_1_diff1": 0.454918, "nauc_mrr_at_3_max": 0.029523, "nauc_mrr_at_3_std": -0.107202, "nauc_mrr_at_3_diff1": 0.440865, "nauc_mrr_at_5_max": 0.043985, "nauc_mrr_at_5_std": -0.095847, "nauc_mrr_at_5_diff1": 0.457953, "nauc_mrr_at_10_max": 0.037635, "nauc_mrr_at_10_std": -0.103769, "nauc_mrr_at_10_diff1": 0.458802, "nauc_mrr_at_20_max": 0.039439, "nauc_mrr_at_20_std": -0.101378, "nauc_mrr_at_20_diff1": 0.457548, "nauc_mrr_at_100_max": 0.039108, "nauc_mrr_at_100_std": -0.101325, "nauc_mrr_at_100_diff1": 0.457073, "nauc_mrr_at_1000_max": 0.039053, "nauc_mrr_at_1000_std": -0.101387, "nauc_mrr_at_1000_diff1": 0.457042, "hit_rate_at_1": 0.73276, "hit_rate_at_3": 0.9023, "hit_rate_at_5": 0.9454, "hit_rate_at_10": 0.97126, "hit_rate_at_20": 0.98563, "hit_rate_at_100": 0.99713, "hit_rate_at_1000": 1.0, "main_score": 0.71837, "hf_subset": "english", "languages": [ "eng-Latn" ], "mteb_version": "2.22.2" }, { "ndcg_at_1": 0.68822, "ndcg_at_3": 0.68931, "ndcg_at_5": 0.69282, "ndcg_at_10": 0.71651, "ndcg_at_20": 0.74946, "ndcg_at_100": 0.78947, "ndcg_at_1000": 0.79584, "map_at_1": 0.31986, "map_at_3": 0.50113, "map_at_5": 0.55358, "map_at_10": 0.60685, "map_at_20": 0.63519, "map_at_100": 0.65532, "map_at_1000": 0.65638, "recall_at_1": 0.31986, "recall_at_3": 0.55129, "recall_at_5": 0.63973, "recall_at_10": 0.74909, "recall_at_20": 0.85237, "recall_at_100": 0.96969, "recall_at_1000": 0.9973, "accuracy": 0.31986, "precision_at_1": 0.71552, "precision_at_3": 0.51628, "precision_at_5": 0.39425, "precision_at_10": 0.26351, "precision_at_20": 0.16336, "precision_at_100": 0.04253, "precision_at_1000": 0.00447, "mrr_at_1": 0.715517, "mrr_at_3": 0.798851, "mrr_at_5": 0.806897, "mrr_at_10": 0.810582, "mrr_at_20": 0.811946, "mrr_at_100": 0.812276, "mrr_at_1000": 0.812299, "nauc_ndcg_at_1_max": -0.003771, "nauc_ndcg_at_1_std": -0.153553, "nauc_ndcg_at_1_diff1": 0.444235, "nauc_ndcg_at_3_max": -0.065346, "nauc_ndcg_at_3_std": -0.184478, "nauc_ndcg_at_3_diff1": 0.350342, "nauc_ndcg_at_5_max": -0.02347, "nauc_ndcg_at_5_std": -0.163015, "nauc_ndcg_at_5_diff1": 0.377618, "nauc_ndcg_at_10_max": -0.03193, "nauc_ndcg_at_10_std": -0.18308, "nauc_ndcg_at_10_diff1": 0.396087, "nauc_ndcg_at_20_max": 0.010225, "nauc_ndcg_at_20_std": -0.147869, "nauc_ndcg_at_20_diff1": 0.400733, "nauc_ndcg_at_100_max": 0.017389, "nauc_ndcg_at_100_std": -0.120825, "nauc_ndcg_at_100_diff1": 0.402717, "nauc_ndcg_at_1000_max": 0.003976, "nauc_ndcg_at_1000_std": -0.129046, "nauc_ndcg_at_1000_diff1": 0.395207, "nauc_map_at_1_max": -0.038268, "nauc_map_at_1_std": -0.18297, "nauc_map_at_1_diff1": 0.458999, "nauc_map_at_3_max": -0.035173, "nauc_map_at_3_std": -0.197172, "nauc_map_at_3_diff1": 0.378548, "nauc_map_at_5_max": -0.013356, "nauc_map_at_5_std": -0.181212, "nauc_map_at_5_diff1": 0.384922, "nauc_map_at_10_max": -0.017635, "nauc_map_at_10_std": -0.184589, "nauc_map_at_10_diff1": 0.385758, "nauc_map_at_20_max": 0.001421, "nauc_map_at_20_std": -0.163135, "nauc_map_at_20_diff1": 0.387767, "nauc_map_at_100_max": 0.012866, "nauc_map_at_100_std": -0.14368, "nauc_map_at_100_diff1": 0.38775, "nauc_map_at_1000_max": 0.011324, "nauc_map_at_1000_std": -0.144535, "nauc_map_at_1000_diff1": 0.3868, "nauc_recall_at_1_max": -0.038268, "nauc_recall_at_1_std": -0.18297, "nauc_recall_at_1_diff1": 0.458999, "nauc_recall_at_3_max": -0.070359, "nauc_recall_at_3_std": -0.220434, "nauc_recall_at_3_diff1": 0.317478, "nauc_recall_at_5_max": -0.008716, "nauc_recall_at_5_std": -0.178428, "nauc_recall_at_5_diff1": 0.326593, "nauc_recall_at_10_max": -0.032542, "nauc_recall_at_10_std": -0.203161, "nauc_recall_at_10_diff1": 0.339745, "nauc_recall_at_20_max": 0.123022, "nauc_recall_at_20_std": -0.065343, "nauc_recall_at_20_diff1": 0.321555, "nauc_recall_at_100_max": 0.389338, "nauc_recall_at_100_std": 0.225887, "nauc_recall_at_100_diff1": 0.456307, "nauc_recall_at_1000_max": 0.460028, "nauc_recall_at_1000_std": 0.558, "nauc_recall_at_1000_diff1": 0.043578, "nauc_precision_at_1_max": -0.036236, "nauc_precision_at_1_std": -0.18553, "nauc_precision_at_1_diff1": 0.43504, "nauc_precision_at_3_max": -0.008723, "nauc_precision_at_3_std": -0.005861, "nauc_precision_at_3_diff1": -0.019704, "nauc_precision_at_5_max": 0.033194, "nauc_precision_at_5_std": 0.065021, "nauc_precision_at_5_diff1": -0.069091, "nauc_precision_at_10_max": 0.042934, "nauc_precision_at_10_std": 0.110607, "nauc_precision_at_10_diff1": -0.120826, "nauc_precision_at_20_max": 0.080219, "nauc_precision_at_20_std": 0.178025, "nauc_precision_at_20_diff1": -0.182139, "nauc_precision_at_100_max": 0.060724, "nauc_precision_at_100_std": 0.198742, "nauc_precision_at_100_diff1": -0.225545, "nauc_precision_at_1000_max": 0.032048, "nauc_precision_at_1000_std": 0.175065, "nauc_precision_at_1000_diff1": -0.244146, "nauc_mrr_at_1_max": -0.036236, "nauc_mrr_at_1_std": -0.18553, "nauc_mrr_at_1_diff1": 0.43504, "nauc_mrr_at_3_max": -0.092096, "nauc_mrr_at_3_std": -0.21294, "nauc_mrr_at_3_diff1": 0.392661, "nauc_mrr_at_5_max": -0.074824, "nauc_mrr_at_5_std": -0.200491, "nauc_mrr_at_5_diff1": 0.405628, "nauc_mrr_at_10_max": -0.069834, "nauc_mrr_at_10_std": -0.197838, "nauc_mrr_at_10_diff1": 0.415957, "nauc_mrr_at_20_max": -0.066309, "nauc_mrr_at_20_std": -0.19495, "nauc_mrr_at_20_diff1": 0.414021, "nauc_mrr_at_100_max": -0.066914, "nauc_mrr_at_100_std": -0.195596, "nauc_mrr_at_100_diff1": 0.413471, "nauc_mrr_at_1000_max": -0.067057, "nauc_mrr_at_1000_std": -0.195757, "nauc_mrr_at_1000_diff1": 0.413418, "hit_rate_at_1": 0.71552, "hit_rate_at_3": 0.9023, "hit_rate_at_5": 0.93678, "hit_rate_at_10": 0.96264, "hit_rate_at_20": 0.98276, "hit_rate_at_100": 0.99425, "hit_rate_at_1000": 1.0, "main_score": 0.71651, "hf_subset": "german", "languages": [ "deu-Latn" ], "mteb_version": "2.22.2" }, { "ndcg_at_1": 0.69253, "ndcg_at_3": 0.68119, "ndcg_at_5": 0.69154, "ndcg_at_10": 0.71392, "ndcg_at_20": 0.74494, "ndcg_at_100": 0.78574, "ndcg_at_1000": 0.79312, "map_at_1": 0.31539, "map_at_3": 0.48985, "map_at_5": 0.54893, "map_at_10": 0.60046, "map_at_20": 0.62849, "map_at_100": 0.6486, "map_at_1000": 0.64983, "recall_at_1": 0.31539, "recall_at_3": 0.54354, "recall_at_5": 0.64203, "recall_at_10": 0.75198, "recall_at_20": 0.84626, "recall_at_100": 0.96471, "recall_at_1000": 0.99679, "accuracy": 0.31539, "precision_at_1": 0.72414, "precision_at_3": 0.50479, "precision_at_5": 0.39598, "precision_at_10": 0.26063, "precision_at_20": 0.16293, "precision_at_100": 0.04221, "precision_at_1000": 0.00446, "mrr_at_1": 0.724138, "mrr_at_3": 0.811782, "mrr_at_5": 0.816954, "mrr_at_10": 0.820714, "mrr_at_20": 0.821538, "mrr_at_100": 0.822063, "mrr_at_1000": 0.822093, "nauc_ndcg_at_1_max": 0.011451, "nauc_ndcg_at_1_std": -0.118171, "nauc_ndcg_at_1_diff1": 0.451703, "nauc_ndcg_at_3_max": -0.084062, "nauc_ndcg_at_3_std": -0.186375, "nauc_ndcg_at_3_diff1": 0.349534, "nauc_ndcg_at_5_max": -0.040672, "nauc_ndcg_at_5_std": -0.157503, "nauc_ndcg_at_5_diff1": 0.350662, "nauc_ndcg_at_10_max": -0.037358, "nauc_ndcg_at_10_std": -0.166684, "nauc_ndcg_at_10_diff1": 0.381187, "nauc_ndcg_at_20_max": -0.021483, "nauc_ndcg_at_20_std": -0.144118, "nauc_ndcg_at_20_diff1": 0.388242, "nauc_ndcg_at_100_max": 0.002245, "nauc_ndcg_at_100_std": -0.101218, "nauc_ndcg_at_100_diff1": 0.385347, "nauc_ndcg_at_1000_max": -0.011141, "nauc_ndcg_at_1000_std": -0.117129, "nauc_ndcg_at_1000_diff1": 0.388667, "nauc_map_at_1_max": -0.0333, "nauc_map_at_1_std": -0.156941, "nauc_map_at_1_diff1": 0.446952, "nauc_map_at_3_max": -0.045282, "nauc_map_at_3_std": -0.190364, "nauc_map_at_3_diff1": 0.382593, "nauc_map_at_5_max": -0.031034, "nauc_map_at_5_std": -0.178911, "nauc_map_at_5_diff1": 0.368265, "nauc_map_at_10_max": -0.039102, "nauc_map_at_10_std": -0.18341, "nauc_map_at_10_diff1": 0.361879, "nauc_map_at_20_max": -0.026247, "nauc_map_at_20_std": -0.162196, "nauc_map_at_20_diff1": 0.363689, "nauc_map_at_100_max": -0.010198, "nauc_map_at_100_std": -0.138041, "nauc_map_at_100_diff1": 0.363579, "nauc_map_at_1000_max": -0.01185, "nauc_map_at_1000_std": -0.139746, "nauc_map_at_1000_diff1": 0.363936, "nauc_recall_at_1_max": -0.0333, "nauc_recall_at_1_std": -0.156941, "nauc_recall_at_1_diff1": 0.446952, "nauc_recall_at_3_max": -0.071092, "nauc_recall_at_3_std": -0.20576, "nauc_recall_at_3_diff1": 0.330735, "nauc_recall_at_5_max": -0.012248, "nauc_recall_at_5_std": -0.161566, "nauc_recall_at_5_diff1": 0.292422, "nauc_recall_at_10_max": -0.008787, "nauc_recall_at_10_std": -0.150824, "nauc_recall_at_10_diff1": 0.292841, "nauc_recall_at_20_max": 0.025302, "nauc_recall_at_20_std": -0.103099, "nauc_recall_at_20_diff1": 0.310957, "nauc_recall_at_100_max": 0.342508, "nauc_recall_at_100_std": 0.34571, "nauc_recall_at_100_diff1": 0.234095, "nauc_recall_at_1000_max": 0.545957, "nauc_recall_at_1000_std": 0.681195, "nauc_recall_at_1000_diff1": -0.131844, "nauc_precision_at_1_max": -0.031518, "nauc_precision_at_1_std": -0.16497, "nauc_precision_at_1_diff1": 0.437125, "nauc_precision_at_3_max": -0.029511, "nauc_precision_at_3_std": -0.02563, "nauc_precision_at_3_diff1": -0.003813, "nauc_precision_at_5_max": 0.014448, "nauc_precision_at_5_std": 0.049564, "nauc_precision_at_5_diff1": -0.086025, "nauc_precision_at_10_max": 0.027316, "nauc_precision_at_10_std": 0.092236, "nauc_precision_at_10_diff1": -0.126046, "nauc_precision_at_20_max": 0.062937, "nauc_precision_at_20_std": 0.173133, "nauc_precision_at_20_diff1": -0.166004, "nauc_precision_at_100_max": 0.071982, "nauc_precision_at_100_std": 0.209843, "nauc_precision_at_100_diff1": -0.20614, "nauc_precision_at_1000_max": 0.040356, "nauc_precision_at_1000_std": 0.172189, "nauc_precision_at_1000_diff1": -0.209114, "nauc_mrr_at_1_max": -0.031518, "nauc_mrr_at_1_std": -0.16497, "nauc_mrr_at_1_diff1": 0.437125, "nauc_mrr_at_3_max": -0.0883, "nauc_mrr_at_3_std": -0.194975, "nauc_mrr_at_3_diff1": 0.413431, "nauc_mrr_at_5_max": -0.059976, "nauc_mrr_at_5_std": -0.171665, "nauc_mrr_at_5_diff1": 0.420245, "nauc_mrr_at_10_max": -0.063275, "nauc_mrr_at_10_std": -0.176021, "nauc_mrr_at_10_diff1": 0.422474, "nauc_mrr_at_20_max": -0.063272, "nauc_mrr_at_20_std": -0.17569, "nauc_mrr_at_20_diff1": 0.42154, "nauc_mrr_at_100_max": -0.062749, "nauc_mrr_at_100_std": -0.175976, "nauc_mrr_at_100_diff1": 0.422191, "nauc_mrr_at_1000_max": -0.062952, "nauc_mrr_at_1000_std": -0.176201, "nauc_mrr_at_1000_diff1": 0.422155, "hit_rate_at_1": 0.72414, "hit_rate_at_3": 0.91379, "hit_rate_at_5": 0.93678, "hit_rate_at_10": 0.96552, "hit_rate_at_20": 0.97701, "hit_rate_at_100": 0.99425, "hit_rate_at_1000": 1.0, "main_score": 0.71392, "hf_subset": "italian", "languages": [ "ita-Latn" ], "mteb_version": "2.22.2" }, { "ndcg_at_1": 0.70115, "ndcg_at_3": 0.69131, "ndcg_at_5": 0.69345, "ndcg_at_10": 0.71295, "ndcg_at_20": 0.74888, "ndcg_at_100": 0.78873, "ndcg_at_1000": 0.79559, "map_at_1": 0.31582, "map_at_3": 0.4958, "map_at_5": 0.55032, "map_at_10": 0.601, "map_at_20": 0.63132, "map_at_100": 0.65121, "map_at_1000": 0.65241, "recall_at_1": 0.31582, "recall_at_3": 0.55094, "recall_at_5": 0.64032, "recall_at_10": 0.73963, "recall_at_20": 0.84894, "recall_at_100": 0.96685, "recall_at_1000": 0.99747, "accuracy": 0.31582, "precision_at_1": 0.72701, "precision_at_3": 0.51437, "precision_at_5": 0.39368, "precision_at_10": 0.26034, "precision_at_20": 0.16408, "precision_at_100": 0.04241, "precision_at_1000": 0.00447, "mrr_at_1": 0.727011, "mrr_at_3": 0.813218, "mrr_at_5": 0.817241, "mrr_at_10": 0.819836, "mrr_at_20": 0.822274, "mrr_at_100": 0.822542, "mrr_at_1000": 0.82256, "nauc_ndcg_at_1_max": 0.004623, "nauc_ndcg_at_1_std": -0.085161, "nauc_ndcg_at_1_diff1": 0.421917, "nauc_ndcg_at_3_max": -0.018373, "nauc_ndcg_at_3_std": -0.117437, "nauc_ndcg_at_3_diff1": 0.320846, "nauc_ndcg_at_5_max": -0.00117, "nauc_ndcg_at_5_std": -0.11636, "nauc_ndcg_at_5_diff1": 0.367547, "nauc_ndcg_at_10_max": 0.00422, "nauc_ndcg_at_10_std": -0.110408, "nauc_ndcg_at_10_diff1": 0.375915, "nauc_ndcg_at_20_max": 0.044455, "nauc_ndcg_at_20_std": -0.069911, "nauc_ndcg_at_20_diff1": 0.390876, "nauc_ndcg_at_100_max": 0.052835, "nauc_ndcg_at_100_std": -0.039468, "nauc_ndcg_at_100_diff1": 0.369927, "nauc_ndcg_at_1000_max": 0.041622, "nauc_ndcg_at_1000_std": -0.050677, "nauc_ndcg_at_1000_diff1": 0.376688, "nauc_map_at_1_max": -0.025265, "nauc_map_at_1_std": -0.132993, "nauc_map_at_1_diff1": 0.477321, "nauc_map_at_3_max": -0.014144, "nauc_map_at_3_std": -0.146444, "nauc_map_at_3_diff1": 0.362281, "nauc_map_at_5_max": -0.005541, "nauc_map_at_5_std": -0.145351, "nauc_map_at_5_diff1": 0.374632, "nauc_map_at_10_max": 0.000324, "nauc_map_at_10_std": -0.13057, "nauc_map_at_10_diff1": 0.359316, "nauc_map_at_20_max": 0.026818, "nauc_map_at_20_std": -0.099854, "nauc_map_at_20_diff1": 0.365847, "nauc_map_at_100_max": 0.039301, "nauc_map_at_100_std": -0.078215, "nauc_map_at_100_diff1": 0.357324, "nauc_map_at_1000_max": 0.038307, "nauc_map_at_1000_std": -0.078947, "nauc_map_at_1000_diff1": 0.358289, "nauc_recall_at_1_max": -0.025265, "nauc_recall_at_1_std": -0.132993, "nauc_recall_at_1_diff1": 0.477321, "nauc_recall_at_3_max": -0.035674, "nauc_recall_at_3_std": -0.168207, "nauc_recall_at_3_diff1": 0.301662, "nauc_recall_at_5_max": -0.010964, "nauc_recall_at_5_std": -0.160812, "nauc_recall_at_5_diff1": 0.325222, "nauc_recall_at_10_max": -0.004299, "nauc_recall_at_10_std": -0.135371, "nauc_recall_at_10_diff1": 0.30134, "nauc_recall_at_20_max": 0.13577, "nauc_recall_at_20_std": 0.006995, "nauc_recall_at_20_diff1": 0.33957, "nauc_recall_at_100_max": 0.350378, "nauc_recall_at_100_std": 0.341067, "nauc_recall_at_100_diff1": 0.142058, "nauc_recall_at_1000_max": 0.060552, "nauc_recall_at_1000_std": 0.522399, "nauc_recall_at_1000_diff1": -0.40277, "nauc_precision_at_1_max": -0.016537, "nauc_precision_at_1_std": -0.10915, "nauc_precision_at_1_diff1": 0.431122, "nauc_precision_at_3_max": 0.042988, "nauc_precision_at_3_std": 0.038318, "nauc_precision_at_3_diff1": -0.082481, "nauc_precision_at_5_max": 0.055935, "nauc_precision_at_5_std": 0.078635, "nauc_precision_at_5_diff1": -0.09414, "nauc_precision_at_10_max": 0.070493, "nauc_precision_at_10_std": 0.136863, "nauc_precision_at_10_diff1": -0.157669, "nauc_precision_at_20_max": 0.139753, "nauc_precision_at_20_std": 0.232587, "nauc_precision_at_20_diff1": -0.193595, "nauc_precision_at_100_max": 0.105722, "nauc_precision_at_100_std": 0.233875, "nauc_precision_at_100_diff1": -0.248051, "nauc_precision_at_1000_max": 0.076913, "nauc_precision_at_1000_std": 0.202578, "nauc_precision_at_1000_diff1": -0.24013, "nauc_mrr_at_1_max": -0.016537, "nauc_mrr_at_1_std": -0.10915, "nauc_mrr_at_1_diff1": 0.431122, "nauc_mrr_at_3_max": -0.036986, "nauc_mrr_at_3_std": -0.121453, "nauc_mrr_at_3_diff1": 0.433344, "nauc_mrr_at_5_max": -0.018068, "nauc_mrr_at_5_std": -0.102344, "nauc_mrr_at_5_diff1": 0.430818, "nauc_mrr_at_10_max": -0.025765, "nauc_mrr_at_10_std": -0.110007, "nauc_mrr_at_10_diff1": 0.438396, "nauc_mrr_at_20_max": -0.024357, "nauc_mrr_at_20_std": -0.109813, "nauc_mrr_at_20_diff1": 0.435088, "nauc_mrr_at_100_max": -0.024293, "nauc_mrr_at_100_std": -0.110228, "nauc_mrr_at_100_diff1": 0.436135, "nauc_mrr_at_1000_max": -0.024391, "nauc_mrr_at_1000_std": -0.110335, "nauc_mrr_at_1000_diff1": 0.436108, "hit_rate_at_1": 0.72701, "hit_rate_at_3": 0.91954, "hit_rate_at_5": 0.93678, "hit_rate_at_10": 0.95402, "hit_rate_at_20": 0.98563, "hit_rate_at_100": 0.99425, "hit_rate_at_1000": 1.0, "main_score": 0.71295, "hf_subset": "portuguese", "languages": [ "por-Latn" ], "mteb_version": "2.22.2" } ] }, "evaluation_time": 3452.667266201228, "kg_co2_emissions": null, "date": 1791048335.935561, "evaluation_phases": [ { "name": "Data loading", "start": 4.887022078037262e-05, "end": 36.56975330039859, "split": "", "subset": "" }, { "name": "Dataset transform", "start": 36.569812981411815, "end": 36.56981660053134, "split": "", "subset": "" }, { "name": "Encoding corpus", "start": 36.57911009900272, "end": 36.57911657914519, "split": "test", "subset": "french" }, { "name": "Encoding queries", "start": 36.579159419983625, "end": 695.9215273372829, "split": "test", "subset": "french" }, { "name": "Scoring", "start": 695.9217877406627, "end": 696.2846037484705, "split": "test", "subset": "french" }, { "name": "Encoding corpus", "start": 696.3163363002241, "end": 696.3163411002606, "split": "test", "subset": "spanish" }, { "name": "Encoding queries", "start": 696.316375920549, "end": 1250.415532629937, "split": "test", "subset": "spanish" }, { "name": "Scoring", "start": 1250.41580215469, "end": 1250.7646419610828, "split": "test", "subset": "spanish" }, { "name": "Encoding corpus", "start": 1250.7945761997253, "end": 1250.7945802491158, "split": "test", "subset": "english" }, { "name": "Encoding queries", "start": 1250.7946132216603, "end": 1803.9154271092266, "split": "test", "subset": "english" }, { "name": "Scoring", "start": 1803.9156979732215, "end": 1804.267286900431, "split": "test", "subset": "english" }, { "name": "Encoding corpus", "start": 1804.2985470294952, "end": 1804.2985511887819, "split": "test", "subset": "german" }, { "name": "Encoding queries", "start": 1804.2985833771527, "end": 2355.633011840284, "split": "test", "subset": "german" }, { "name": "Scoring", "start": 2355.633307514712, "end": 2355.989472080022, "split": "test", "subset": "german" }, { "name": "Encoding corpus", "start": 2356.0262081008404, "end": 2356.0262128617615, "split": "test", "subset": "italian" }, { "name": "Encoding queries", "start": 2356.0262458417565, "end": 2907.534093907103, "split": "test", "subset": "italian" }, { "name": "Scoring", "start": 2907.5343705117702, "end": 2907.9160477072, "split": "test", "subset": "italian" }, { "name": "Encoding corpus", "start": 2907.954280104488, "end": 2907.954284604639, "split": "test", "subset": "portuguese" }, { "name": "Encoding queries", "start": 2907.9543167371303, "end": 3452.2555844467133, "split": "test", "subset": "portuguese" }, { "name": "Scoring", "start": 3452.255855690688, "end": 3452.637757163495, "split": "test", "subset": "portuguese" } ] }