{ "dataset_revision": "a54635a274ef2835721b7cbe3eb27483b9ec964b", "task_name": "Vidore3TelecomRetrieval", "mteb_version": "2.13.1", "scores": { "test": [ { "ndcg_at_1": 0.69828, "ndcg_at_3": 0.67294, "ndcg_at_5": 0.67841, "ndcg_at_10": 0.70518, "ndcg_at_20": 0.73774, "ndcg_at_100": 0.77696, "ndcg_at_1000": 0.78483, "map_at_1": 0.31306, "map_at_3": 0.48262, "map_at_5": 0.53319, "map_at_10": 0.58649, "map_at_20": 0.61546, "map_at_100": 0.63574, "map_at_1000": 0.63696, "recall_at_1": 0.31306, "recall_at_3": 0.52856, "recall_at_5": 0.6189, "recall_at_10": 0.74122, "recall_at_20": 0.84488, "recall_at_100": 0.96122, "recall_at_1000": 0.99599, "accuracy": 0.31306, "precision_at_1": 0.72414, "precision_at_3": 0.50096, "precision_at_5": 0.38908, "precision_at_10": 0.26034, "precision_at_20": 0.16178, "precision_at_100": 0.04204, "precision_at_1000": 0.00446, "mrr_at_1": 0.724138, "mrr_at_3": 0.792146, "mrr_at_5": 0.802634, "mrr_at_10": 0.80899, "mrr_at_20": 0.810334, "mrr_at_100": 0.810505, "mrr_at_1000": 0.810545, "nauc_ndcg_at_1_max": -0.07386, "nauc_ndcg_at_1_std": -0.126977, "nauc_ndcg_at_1_diff1": 0.424431, "nauc_ndcg_at_3_max": -0.004882, "nauc_ndcg_at_3_std": -0.113408, "nauc_ndcg_at_3_diff1": 0.332585, "nauc_ndcg_at_5_max": -0.000146, "nauc_ndcg_at_5_std": -0.134719, "nauc_ndcg_at_5_diff1": 0.326436, "nauc_ndcg_at_10_max": 0.004521, "nauc_ndcg_at_10_std": -0.139, "nauc_ndcg_at_10_diff1": 0.353314, "nauc_ndcg_at_20_max": 0.031112, "nauc_ndcg_at_20_std": -0.108138, "nauc_ndcg_at_20_diff1": 0.359548, "nauc_ndcg_at_100_max": 0.045522, "nauc_ndcg_at_100_std": -0.060229, "nauc_ndcg_at_100_diff1": 0.368218, "nauc_ndcg_at_1000_max": 0.025429, "nauc_ndcg_at_1000_std": -0.081035, "nauc_ndcg_at_1000_diff1": 0.365275, "nauc_map_at_1_max": -0.007635, "nauc_map_at_1_std": -0.162374, "nauc_map_at_1_diff1": 0.435117, "nauc_map_at_3_max": 0.037338, "nauc_map_at_3_std": -0.139599, "nauc_map_at_3_diff1": 0.319795, "nauc_map_at_5_max": 0.035128, "nauc_map_at_5_std": -0.146319, "nauc_map_at_5_diff1": 0.309858, "nauc_map_at_10_max": 0.020151, "nauc_map_at_10_std": -0.1548, "nauc_map_at_10_diff1": 0.314248, "nauc_map_at_20_max": 0.033566, "nauc_map_at_20_std": -0.133591, "nauc_map_at_20_diff1": 0.3155, "nauc_map_at_100_max": 0.040263, "nauc_map_at_100_std": -0.112696, "nauc_map_at_100_diff1": 0.319466, "nauc_map_at_1000_max": 0.038701, "nauc_map_at_1000_std": -0.114241, "nauc_map_at_1000_diff1": 0.319395, "nauc_recall_at_1_max": -0.007635, "nauc_recall_at_1_std": -0.162374, "nauc_recall_at_1_diff1": 0.435117, "nauc_recall_at_3_max": 0.048324, "nauc_recall_at_3_std": -0.145621, "nauc_recall_at_3_diff1": 0.285464, "nauc_recall_at_5_max": 0.034703, "nauc_recall_at_5_std": -0.158021, "nauc_recall_at_5_diff1": 0.235179, "nauc_recall_at_10_max": 0.021869, "nauc_recall_at_10_std": -0.178403, "nauc_recall_at_10_diff1": 0.272453, "nauc_recall_at_20_max": 0.108362, "nauc_recall_at_20_std": -0.069195, "nauc_recall_at_20_diff1": 0.264508, "nauc_recall_at_100_max": 0.467787, "nauc_recall_at_100_std": 0.466386, "nauc_recall_at_100_diff1": 0.337757, "nauc_recall_at_1000_max": 0.435262, "nauc_recall_at_1000_std": 0.749172, "nauc_recall_at_1000_diff1": 0.121064, "nauc_precision_at_1_max": -0.116547, "nauc_precision_at_1_std": -0.142925, "nauc_precision_at_1_diff1": 0.402791, "nauc_precision_at_3_max": 0.003463, "nauc_precision_at_3_std": 0.028526, "nauc_precision_at_3_diff1": -0.043826, "nauc_precision_at_5_max": -0.017705, "nauc_precision_at_5_std": 0.040803, "nauc_precision_at_5_diff1": -0.112596, "nauc_precision_at_10_max": -0.02099, "nauc_precision_at_10_std": 0.074713, "nauc_precision_at_10_diff1": -0.134039, "nauc_precision_at_20_max": 0.013554, "nauc_precision_at_20_std": 0.144563, "nauc_precision_at_20_diff1": -0.154206, "nauc_precision_at_100_max": 0.005778, "nauc_precision_at_100_std": 0.17908, "nauc_precision_at_100_diff1": -0.168588, "nauc_precision_at_1000_max": -0.027638, "nauc_precision_at_1000_std": 0.13891, "nauc_precision_at_1000_diff1": -0.180588, "nauc_mrr_at_1_max": -0.116547, "nauc_mrr_at_1_std": -0.142925, "nauc_mrr_at_1_diff1": 0.402791, "nauc_mrr_at_3_max": -0.083353, "nauc_mrr_at_3_std": -0.106961, "nauc_mrr_at_3_diff1": 0.403069, "nauc_mrr_at_5_max": -0.08392, "nauc_mrr_at_5_std": -0.099531, "nauc_mrr_at_5_diff1": 0.390799, "nauc_mrr_at_10_max": -0.088706, "nauc_mrr_at_10_std": -0.106099, "nauc_mrr_at_10_diff1": 0.398675, "nauc_mrr_at_20_max": -0.086017, "nauc_mrr_at_20_std": -0.103235, "nauc_mrr_at_20_diff1": 0.398901, "nauc_mrr_at_100_max": -0.086365, "nauc_mrr_at_100_std": -0.103137, "nauc_mrr_at_100_diff1": 0.398433, "nauc_mrr_at_1000_max": -0.086595, "nauc_mrr_at_1000_std": -0.103395, "nauc_mrr_at_1000_diff1": 0.398349, "hit_rate_at_1": 0.72414, "hit_rate_at_3": 0.87356, "hit_rate_at_5": 0.91954, "hit_rate_at_10": 0.96839, "hit_rate_at_20": 0.98851, "hit_rate_at_100": 0.99425, "hit_rate_at_1000": 1.0, "main_score": 0.70518, "hf_subset": "french", "languages": [ "fra-Latn" ], "mteb_version": "2.13.1" }, { "ndcg_at_1": 0.69828, "ndcg_at_3": 0.67266, "ndcg_at_5": 0.6816, "ndcg_at_10": 0.70289, "ndcg_at_20": 0.73827, "ndcg_at_100": 0.77724, "ndcg_at_1000": 0.78467, "map_at_1": 0.31234, "map_at_3": 0.48132, "map_at_5": 0.5359, "map_at_10": 0.58591, "map_at_20": 0.61646, "map_at_100": 0.6363, "map_at_1000": 0.63753, "recall_at_1": 0.31234, "recall_at_3": 0.53023, "recall_at_5": 0.62742, "recall_at_10": 0.7366, "recall_at_20": 0.85015, "recall_at_100": 0.96534, "recall_at_1000": 0.99729, "accuracy": 0.31234, "precision_at_1": 0.72989, "precision_at_3": 0.49904, "precision_at_5": 0.3931, "precision_at_10": 0.25977, "precision_at_20": 0.16307, "precision_at_100": 0.04213, "precision_at_1000": 0.00447, "mrr_at_1": 0.729885, "mrr_at_3": 0.79454, "mrr_at_5": 0.806178, "mrr_at_10": 0.811209, "mrr_at_20": 0.813314, "mrr_at_100": 0.813527, "mrr_at_1000": 0.813536, "nauc_ndcg_at_1_max": -0.000804, "nauc_ndcg_at_1_std": -0.030713, "nauc_ndcg_at_1_diff1": 0.416976, "nauc_ndcg_at_3_max": 0.004161, "nauc_ndcg_at_3_std": -0.072919, "nauc_ndcg_at_3_diff1": 0.3184, "nauc_ndcg_at_5_max": 0.024346, "nauc_ndcg_at_5_std": -0.089947, "nauc_ndcg_at_5_diff1": 0.318248, "nauc_ndcg_at_10_max": 0.014127, "nauc_ndcg_at_10_std": -0.100951, "nauc_ndcg_at_10_diff1": 0.324978, "nauc_ndcg_at_20_max": 0.039195, "nauc_ndcg_at_20_std": -0.069839, "nauc_ndcg_at_20_diff1": 0.341857, "nauc_ndcg_at_100_max": 0.060736, "nauc_ndcg_at_100_std": -0.021593, "nauc_ndcg_at_100_diff1": 0.348836, "nauc_ndcg_at_1000_max": 0.04545, "nauc_ndcg_at_1000_std": -0.033557, "nauc_ndcg_at_1000_diff1": 0.345565, "nauc_map_at_1_max": -0.00137, "nauc_map_at_1_std": -0.138803, "nauc_map_at_1_diff1": 0.439286, "nauc_map_at_3_max": 0.025594, "nauc_map_at_3_std": -0.123989, "nauc_map_at_3_diff1": 0.321625, "nauc_map_at_5_max": 0.031362, "nauc_map_at_5_std": -0.129304, "nauc_map_at_5_diff1": 0.318111, "nauc_map_at_10_max": 0.022207, "nauc_map_at_10_std": -0.132546, "nauc_map_at_10_diff1": 0.308798, "nauc_map_at_20_max": 0.039271, "nauc_map_at_20_std": -0.105142, "nauc_map_at_20_diff1": 0.309852, "nauc_map_at_100_max": 0.052866, "nauc_map_at_100_std": -0.079536, "nauc_map_at_100_diff1": 0.31257, "nauc_map_at_1000_max": 0.051583, "nauc_map_at_1000_std": -0.080338, "nauc_map_at_1000_diff1": 0.311988, "nauc_recall_at_1_max": -0.00137, "nauc_recall_at_1_std": -0.138803, "nauc_recall_at_1_diff1": 0.439286, "nauc_recall_at_3_max": 0.013452, "nauc_recall_at_3_std": -0.136519, "nauc_recall_at_3_diff1": 0.273334, "nauc_recall_at_5_max": 0.031851, "nauc_recall_at_5_std": -0.152023, "nauc_recall_at_5_diff1": 0.248738, "nauc_recall_at_10_max": 0.008854, "nauc_recall_at_10_std": -0.172133, "nauc_recall_at_10_diff1": 0.230627, "nauc_recall_at_20_max": 0.085219, "nauc_recall_at_20_std": -0.064672, "nauc_recall_at_20_diff1": 0.257672, "nauc_recall_at_100_max": 0.410584, "nauc_recall_at_100_std": 0.334751, "nauc_recall_at_100_diff1": 0.282438, "nauc_recall_at_1000_max": 0.480055, "nauc_recall_at_1000_std": 0.706286, "nauc_recall_at_1000_diff1": 0.020514, "nauc_precision_at_1_max": -0.033935, "nauc_precision_at_1_std": -0.047432, "nauc_precision_at_1_diff1": 0.389032, "nauc_precision_at_3_max": 0.020936, "nauc_precision_at_3_std": 0.04908, "nauc_precision_at_3_diff1": -0.042506, "nauc_precision_at_5_max": 0.021283, "nauc_precision_at_5_std": 0.072488, "nauc_precision_at_5_diff1": -0.105788, "nauc_precision_at_10_max": 0.024138, "nauc_precision_at_10_std": 0.117178, "nauc_precision_at_10_diff1": -0.145695, "nauc_precision_at_20_max": 0.065532, "nauc_precision_at_20_std": 0.191734, "nauc_precision_at_20_diff1": -0.165602, "nauc_precision_at_100_max": 0.060332, "nauc_precision_at_100_std": 0.219437, "nauc_precision_at_100_diff1": -0.174182, "nauc_precision_at_1000_max": 0.025036, "nauc_precision_at_1000_std": 0.182006, "nauc_precision_at_1000_diff1": -0.190394, "nauc_mrr_at_1_max": -0.033935, "nauc_mrr_at_1_std": -0.047432, "nauc_mrr_at_1_diff1": 0.389032, "nauc_mrr_at_3_max": -0.055341, "nauc_mrr_at_3_std": -0.044591, "nauc_mrr_at_3_diff1": 0.373425, "nauc_mrr_at_5_max": -0.036429, "nauc_mrr_at_5_std": -0.037365, "nauc_mrr_at_5_diff1": 0.369423, "nauc_mrr_at_10_max": -0.038383, "nauc_mrr_at_10_std": -0.0305, "nauc_mrr_at_10_diff1": 0.377281, "nauc_mrr_at_20_max": -0.039093, "nauc_mrr_at_20_std": -0.031831, "nauc_mrr_at_20_diff1": 0.375853, "nauc_mrr_at_100_max": -0.039852, "nauc_mrr_at_100_std": -0.032883, "nauc_mrr_at_100_diff1": 0.376, "nauc_mrr_at_1000_max": -0.0399, "nauc_mrr_at_1000_std": -0.032938, "nauc_mrr_at_1000_diff1": 0.375974, "hit_rate_at_1": 0.72989, "hit_rate_at_3": 0.87356, "hit_rate_at_5": 0.92241, "hit_rate_at_10": 0.95977, "hit_rate_at_20": 0.98851, "hit_rate_at_100": 0.99713, "hit_rate_at_1000": 1.0, "main_score": 0.70289, "hf_subset": "spanish", "languages": [ "spa-Latn" ], "mteb_version": "2.13.1" }, { "ndcg_at_1": 0.71839, "ndcg_at_3": 0.68942, "ndcg_at_5": 0.69687, "ndcg_at_10": 0.72534, "ndcg_at_20": 0.75702, "ndcg_at_100": 0.79361, "ndcg_at_1000": 0.79962, "map_at_1": 0.31967, "map_at_3": 0.49566, "map_at_5": 0.55183, "map_at_10": 0.61011, "map_at_20": 0.63966, "map_at_100": 0.6587, "map_at_1000": 0.65964, "recall_at_1": 0.31967, "recall_at_3": 0.54171, "recall_at_5": 0.64051, "recall_at_10": 0.76382, "recall_at_20": 0.86483, "recall_at_100": 0.97032, "recall_at_1000": 0.99693, "accuracy": 0.31967, "precision_at_1": 0.74425, "precision_at_3": 0.51724, "precision_at_5": 0.40057, "precision_at_10": 0.27241, "precision_at_20": 0.16853, "precision_at_100": 0.04267, "precision_at_1000": 0.00447, "mrr_at_1": 0.744253, "mrr_at_3": 0.810345, "mrr_at_5": 0.818678, "mrr_at_10": 0.822843, "mrr_at_20": 0.824888, "mrr_at_100": 0.825114, "mrr_at_1000": 0.825126, "nauc_ndcg_at_1_max": 0.058952, "nauc_ndcg_at_1_std": -0.076294, "nauc_ndcg_at_1_diff1": 0.447232, "nauc_ndcg_at_3_max": 0.088526, "nauc_ndcg_at_3_std": -0.084953, "nauc_ndcg_at_3_diff1": 0.357829, "nauc_ndcg_at_5_max": 0.074882, "nauc_ndcg_at_5_std": -0.118285, "nauc_ndcg_at_5_diff1": 0.383735, "nauc_ndcg_at_10_max": 0.091576, "nauc_ndcg_at_10_std": -0.10583, "nauc_ndcg_at_10_diff1": 0.413698, "nauc_ndcg_at_20_max": 0.135738, "nauc_ndcg_at_20_std": -0.072169, "nauc_ndcg_at_20_diff1": 0.415618, "nauc_ndcg_at_100_max": 0.130127, "nauc_ndcg_at_100_std": -0.035552, "nauc_ndcg_at_100_diff1": 0.404535, "nauc_ndcg_at_1000_max": 0.114045, "nauc_ndcg_at_1000_std": -0.04729, "nauc_ndcg_at_1000_diff1": 0.402471, "nauc_map_at_1_max": 0.054404, "nauc_map_at_1_std": -0.147502, "nauc_map_at_1_diff1": 0.486582, "nauc_map_at_3_max": 0.122095, "nauc_map_at_3_std": -0.124948, "nauc_map_at_3_diff1": 0.38069, "nauc_map_at_5_max": 0.097123, "nauc_map_at_5_std": -0.153792, "nauc_map_at_5_diff1": 0.37475, "nauc_map_at_10_max": 0.097776, "nauc_map_at_10_std": -0.144528, "nauc_map_at_10_diff1": 0.380531, "nauc_map_at_20_max": 0.118927, "nauc_map_at_20_std": -0.117214, "nauc_map_at_20_diff1": 0.377215, "nauc_map_at_100_max": 0.12044, "nauc_map_at_100_std": -0.096048, "nauc_map_at_100_diff1": 0.371385, "nauc_map_at_1000_max": 0.118976, "nauc_map_at_1000_std": -0.096903, "nauc_map_at_1000_diff1": 0.370847, "nauc_recall_at_1_max": 0.054404, "nauc_recall_at_1_std": -0.147502, "nauc_recall_at_1_diff1": 0.486582, "nauc_recall_at_3_max": 0.114347, "nauc_recall_at_3_std": -0.132619, "nauc_recall_at_3_diff1": 0.323791, "nauc_recall_at_5_max": 0.080679, "nauc_recall_at_5_std": -0.186871, "nauc_recall_at_5_diff1": 0.317523, "nauc_recall_at_10_max": 0.094862, "nauc_recall_at_10_std": -0.159267, "nauc_recall_at_10_diff1": 0.361762, "nauc_recall_at_20_max": 0.240989, "nauc_recall_at_20_std": -0.021033, "nauc_recall_at_20_diff1": 0.369344, "nauc_recall_at_100_max": 0.554135, "nauc_recall_at_100_std": 0.402701, "nauc_recall_at_100_diff1": 0.318743, "nauc_recall_at_1000_max": 0.740616, "nauc_recall_at_1000_std": 0.895772, "nauc_recall_at_1000_diff1": 0.319404, "nauc_precision_at_1_max": 0.020185, "nauc_precision_at_1_std": -0.101872, "nauc_precision_at_1_diff1": 0.444187, "nauc_precision_at_3_max": 0.053926, "nauc_precision_at_3_std": 0.052197, "nauc_precision_at_3_diff1": -0.058094, "nauc_precision_at_5_max": -0.030049, "nauc_precision_at_5_std": 0.03768, "nauc_precision_at_5_diff1": -0.123674, "nauc_precision_at_10_max": -0.037061, "nauc_precision_at_10_std": 0.105882, "nauc_precision_at_10_diff1": -0.164148, "nauc_precision_at_20_max": -0.001047, "nauc_precision_at_20_std": 0.179738, "nauc_precision_at_20_diff1": -0.212351, "nauc_precision_at_100_max": -0.041408, "nauc_precision_at_100_std": 0.206462, "nauc_precision_at_100_diff1": -0.249796, "nauc_precision_at_1000_max": -0.072835, "nauc_precision_at_1000_std": 0.178463, "nauc_precision_at_1000_diff1": -0.262314, "nauc_mrr_at_1_max": 0.020185, "nauc_mrr_at_1_std": -0.101872, "nauc_mrr_at_1_diff1": 0.444187, "nauc_mrr_at_3_max": 0.027441, "nauc_mrr_at_3_std": -0.054141, "nauc_mrr_at_3_diff1": 0.420407, "nauc_mrr_at_5_max": 0.035434, "nauc_mrr_at_5_std": -0.060918, "nauc_mrr_at_5_diff1": 0.42493, "nauc_mrr_at_10_max": 0.029803, "nauc_mrr_at_10_std": -0.065461, "nauc_mrr_at_10_diff1": 0.431577, "nauc_mrr_at_20_max": 0.033652, "nauc_mrr_at_20_std": -0.062897, "nauc_mrr_at_20_diff1": 0.432652, "nauc_mrr_at_100_max": 0.032532, "nauc_mrr_at_100_std": -0.064267, "nauc_mrr_at_100_diff1": 0.432506, "nauc_mrr_at_1000_max": 0.032468, "nauc_mrr_at_1000_std": -0.064326, "nauc_mrr_at_1000_diff1": 0.43253, "hit_rate_at_1": 0.74425, "hit_rate_at_3": 0.89368, "hit_rate_at_5": 0.93103, "hit_rate_at_10": 0.95977, "hit_rate_at_20": 0.98851, "hit_rate_at_100": 0.99713, "hit_rate_at_1000": 1.0, "main_score": 0.72534, "hf_subset": "english", "languages": [ "eng-Latn" ], "mteb_version": "2.13.1" }, { "ndcg_at_1": 0.70977, "ndcg_at_3": 0.68069, "ndcg_at_5": 0.68821, "ndcg_at_10": 0.71394, "ndcg_at_20": 0.74397, "ndcg_at_100": 0.78599, "ndcg_at_1000": 0.79338, "map_at_1": 0.32287, "map_at_3": 0.48958, "map_at_5": 0.54464, "map_at_10": 0.60007, "map_at_20": 0.62701, "map_at_100": 0.64772, "map_at_1000": 0.64884, "recall_at_1": 0.32287, "recall_at_3": 0.53393, "recall_at_5": 0.6257, "recall_at_10": 0.7421, "recall_at_20": 0.84054, "recall_at_100": 0.9649, "recall_at_1000": 0.99775, "accuracy": 0.32287, "precision_at_1": 0.73563, "precision_at_3": 0.50383, "precision_at_5": 0.39195, "precision_at_10": 0.26322, "precision_at_20": 0.16149, "precision_at_100": 0.04236, "precision_at_1000": 0.00447, "mrr_at_1": 0.735632, "mrr_at_3": 0.802682, "mrr_at_5": 0.809722, "mrr_at_10": 0.814453, "mrr_at_20": 0.816081, "mrr_at_100": 0.816726, "mrr_at_1000": 0.816747, "nauc_ndcg_at_1_max": -0.011708, "nauc_ndcg_at_1_std": -0.033829, "nauc_ndcg_at_1_diff1": 0.384207, "nauc_ndcg_at_3_max": 0.014683, "nauc_ndcg_at_3_std": -0.059219, "nauc_ndcg_at_3_diff1": 0.304571, "nauc_ndcg_at_5_max": 0.036996, "nauc_ndcg_at_5_std": -0.07145, "nauc_ndcg_at_5_diff1": 0.335963, "nauc_ndcg_at_10_max": 0.019643, "nauc_ndcg_at_10_std": -0.098331, "nauc_ndcg_at_10_diff1": 0.348444, "nauc_ndcg_at_20_max": 0.042398, "nauc_ndcg_at_20_std": -0.062934, "nauc_ndcg_at_20_diff1": 0.362259, "nauc_ndcg_at_100_max": 0.059955, "nauc_ndcg_at_100_std": -0.019279, "nauc_ndcg_at_100_diff1": 0.350898, "nauc_ndcg_at_1000_max": 0.038163, "nauc_ndcg_at_1000_std": -0.037117, "nauc_ndcg_at_1000_diff1": 0.340987, "nauc_map_at_1_max": -0.018906, "nauc_map_at_1_std": -0.155829, "nauc_map_at_1_diff1": 0.441211, "nauc_map_at_3_max": 0.023672, "nauc_map_at_3_std": -0.128134, "nauc_map_at_3_diff1": 0.329817, "nauc_map_at_5_max": 0.035335, "nauc_map_at_5_std": -0.122566, "nauc_map_at_5_diff1": 0.331, "nauc_map_at_10_max": 0.031263, "nauc_map_at_10_std": -0.123858, "nauc_map_at_10_diff1": 0.329985, "nauc_map_at_20_max": 0.04196, "nauc_map_at_20_std": -0.098967, "nauc_map_at_20_diff1": 0.333427, "nauc_map_at_100_max": 0.052376, "nauc_map_at_100_std": -0.075071, "nauc_map_at_100_diff1": 0.329059, "nauc_map_at_1000_max": 0.050305, "nauc_map_at_1000_std": -0.076368, "nauc_map_at_1000_diff1": 0.327846, "nauc_recall_at_1_max": -0.018906, "nauc_recall_at_1_std": -0.155829, "nauc_recall_at_1_diff1": 0.441211, "nauc_recall_at_3_max": 0.037413, "nauc_recall_at_3_std": -0.131138, "nauc_recall_at_3_diff1": 0.300242, "nauc_recall_at_5_max": 0.061052, "nauc_recall_at_5_std": -0.13091, "nauc_recall_at_5_diff1": 0.306247, "nauc_recall_at_10_max": 0.047244, "nauc_recall_at_10_std": -0.149237, "nauc_recall_at_10_diff1": 0.337592, "nauc_recall_at_20_max": 0.120228, "nauc_recall_at_20_std": -0.01935, "nauc_recall_at_20_diff1": 0.372133, "nauc_recall_at_100_max": 0.508239, "nauc_recall_at_100_std": 0.442742, "nauc_recall_at_100_diff1": 0.415588, "nauc_recall_at_1000_max": 0.507759, "nauc_recall_at_1000_std": 0.66443, "nauc_recall_at_1000_diff1": -0.245105, "nauc_precision_at_1_max": -0.050727, "nauc_precision_at_1_std": -0.038929, "nauc_precision_at_1_diff1": 0.373375, "nauc_precision_at_3_max": 0.029007, "nauc_precision_at_3_std": 0.078311, "nauc_precision_at_3_diff1": -0.090334, "nauc_precision_at_5_max": 0.045339, "nauc_precision_at_5_std": 0.106691, "nauc_precision_at_5_diff1": -0.120988, "nauc_precision_at_10_max": 0.028822, "nauc_precision_at_10_std": 0.124224, "nauc_precision_at_10_diff1": -0.141018, "nauc_precision_at_20_max": 0.047309, "nauc_precision_at_20_std": 0.194851, "nauc_precision_at_20_diff1": -0.166263, "nauc_precision_at_100_max": 0.03678, "nauc_precision_at_100_std": 0.219226, "nauc_precision_at_100_diff1": -0.206126, "nauc_precision_at_1000_max": -0.003495, "nauc_precision_at_1000_std": 0.179493, "nauc_precision_at_1000_diff1": -0.226847, "nauc_mrr_at_1_max": -0.050727, "nauc_mrr_at_1_std": -0.038929, "nauc_mrr_at_1_diff1": 0.373375, "nauc_mrr_at_3_max": -0.044942, "nauc_mrr_at_3_std": -0.018998, "nauc_mrr_at_3_diff1": 0.353034, "nauc_mrr_at_5_max": -0.035, "nauc_mrr_at_5_std": -0.01865, "nauc_mrr_at_5_diff1": 0.358786, "nauc_mrr_at_10_max": -0.0409, "nauc_mrr_at_10_std": -0.024114, "nauc_mrr_at_10_diff1": 0.363172, "nauc_mrr_at_20_max": -0.03975, "nauc_mrr_at_20_std": -0.021184, "nauc_mrr_at_20_diff1": 0.363511, "nauc_mrr_at_100_max": -0.040718, "nauc_mrr_at_100_std": -0.022255, "nauc_mrr_at_100_diff1": 0.361864, "nauc_mrr_at_1000_max": -0.040831, "nauc_mrr_at_1000_std": -0.022366, "nauc_mrr_at_1000_diff1": 0.36184, "hit_rate_at_1": 0.73563, "hit_rate_at_3": 0.88793, "hit_rate_at_5": 0.91954, "hit_rate_at_10": 0.95402, "hit_rate_at_20": 0.97701, "hit_rate_at_100": 0.99713, "hit_rate_at_1000": 1.0, "main_score": 0.71394, "hf_subset": "german", "languages": [ "deu-Latn" ], "mteb_version": "2.13.1" }, { "ndcg_at_1": 0.69828, "ndcg_at_3": 0.66795, "ndcg_at_5": 0.676, "ndcg_at_10": 0.70465, "ndcg_at_20": 0.73755, "ndcg_at_100": 0.77775, "ndcg_at_1000": 0.78518, "map_at_1": 0.31051, "map_at_3": 0.47526, "map_at_5": 0.53187, "map_at_10": 0.58663, "map_at_20": 0.61623, "map_at_100": 0.63643, "map_at_1000": 0.63763, "recall_at_1": 0.31051, "recall_at_3": 0.51704, "recall_at_5": 0.61636, "recall_at_10": 0.73917, "recall_at_20": 0.84682, "recall_at_100": 0.96503, "recall_at_1000": 0.99697, "accuracy": 0.31051, "precision_at_1": 0.72126, "precision_at_3": 0.49808, "precision_at_5": 0.38736, "precision_at_10": 0.26092, "precision_at_20": 0.16236, "precision_at_100": 0.04221, "precision_at_1000": 0.00447, "mrr_at_1": 0.721264, "mrr_at_3": 0.788314, "mrr_at_5": 0.79909, "mrr_at_10": 0.805387, "mrr_at_20": 0.807136, "mrr_at_100": 0.807553, "mrr_at_1000": 0.807569, "nauc_ndcg_at_1_max": -0.014386, "nauc_ndcg_at_1_std": -0.052326, "nauc_ndcg_at_1_diff1": 0.386907, "nauc_ndcg_at_3_max": -0.016726, "nauc_ndcg_at_3_std": -0.065165, "nauc_ndcg_at_3_diff1": 0.274829, "nauc_ndcg_at_5_max": -0.013597, "nauc_ndcg_at_5_std": -0.103372, "nauc_ndcg_at_5_diff1": 0.285584, "nauc_ndcg_at_10_max": -0.020091, "nauc_ndcg_at_10_std": -0.128171, "nauc_ndcg_at_10_diff1": 0.290581, "nauc_ndcg_at_20_max": 0.010336, "nauc_ndcg_at_20_std": -0.084648, "nauc_ndcg_at_20_diff1": 0.316251, "nauc_ndcg_at_100_max": 0.025129, "nauc_ndcg_at_100_std": -0.044704, "nauc_ndcg_at_100_diff1": 0.313176, "nauc_ndcg_at_1000_max": 0.011402, "nauc_ndcg_at_1000_std": -0.058313, "nauc_ndcg_at_1000_diff1": 0.309113, "nauc_map_at_1_max": -0.001008, "nauc_map_at_1_std": -0.130173, "nauc_map_at_1_diff1": 0.425745, "nauc_map_at_3_max": 0.015909, "nauc_map_at_3_std": -0.122249, "nauc_map_at_3_diff1": 0.30391, "nauc_map_at_5_max": 0.013181, "nauc_map_at_5_std": -0.135656, "nauc_map_at_5_diff1": 0.291842, "nauc_map_at_10_max": 0.002356, "nauc_map_at_10_std": -0.147592, "nauc_map_at_10_diff1": 0.283568, "nauc_map_at_20_max": 0.013955, "nauc_map_at_20_std": -0.120837, "nauc_map_at_20_diff1": 0.287302, "nauc_map_at_100_max": 0.023068, "nauc_map_at_100_std": -0.096638, "nauc_map_at_100_diff1": 0.285066, "nauc_map_at_1000_max": 0.021847, "nauc_map_at_1000_std": -0.09778, "nauc_map_at_1000_diff1": 0.284407, "nauc_recall_at_1_max": -0.001008, "nauc_recall_at_1_std": -0.130173, "nauc_recall_at_1_diff1": 0.425745, "nauc_recall_at_3_max": 0.006985, "nauc_recall_at_3_std": -0.114204, "nauc_recall_at_3_diff1": 0.255655, "nauc_recall_at_5_max": 0.005963, "nauc_recall_at_5_std": -0.150127, "nauc_recall_at_5_diff1": 0.229411, "nauc_recall_at_10_max": -0.028153, "nauc_recall_at_10_std": -0.203419, "nauc_recall_at_10_diff1": 0.204381, "nauc_recall_at_20_max": 0.060087, "nauc_recall_at_20_std": -0.052969, "nauc_recall_at_20_diff1": 0.268766, "nauc_recall_at_100_max": 0.317019, "nauc_recall_at_100_std": 0.331957, "nauc_recall_at_100_diff1": 0.25587, "nauc_recall_at_1000_max": 0.291516, "nauc_recall_at_1000_std": 0.723434, "nauc_recall_at_1000_diff1": -0.194537, "nauc_precision_at_1_max": -0.047615, "nauc_precision_at_1_std": -0.07133, "nauc_precision_at_1_diff1": 0.382267, "nauc_precision_at_3_max": -0.005849, "nauc_precision_at_3_std": 0.056318, "nauc_precision_at_3_diff1": -0.074898, "nauc_precision_at_5_max": -0.019287, "nauc_precision_at_5_std": 0.042715, "nauc_precision_at_5_diff1": -0.108142, "nauc_precision_at_10_max": -0.012348, "nauc_precision_at_10_std": 0.079152, "nauc_precision_at_10_diff1": -0.150258, "nauc_precision_at_20_max": 0.032367, "nauc_precision_at_20_std": 0.168784, "nauc_precision_at_20_diff1": -0.169927, "nauc_precision_at_100_max": 0.023929, "nauc_precision_at_100_std": 0.197302, "nauc_precision_at_100_diff1": -0.19623, "nauc_precision_at_1000_max": -0.005743, "nauc_precision_at_1000_std": 0.161165, "nauc_precision_at_1000_diff1": -0.207367, "nauc_mrr_at_1_max": -0.047615, "nauc_mrr_at_1_std": -0.07133, "nauc_mrr_at_1_diff1": 0.382267, "nauc_mrr_at_3_max": -0.053057, "nauc_mrr_at_3_std": -0.033918, "nauc_mrr_at_3_diff1": 0.352161, "nauc_mrr_at_5_max": -0.053262, "nauc_mrr_at_5_std": -0.043059, "nauc_mrr_at_5_diff1": 0.361271, "nauc_mrr_at_10_max": -0.057506, "nauc_mrr_at_10_std": -0.053168, "nauc_mrr_at_10_diff1": 0.362694, "nauc_mrr_at_20_max": -0.052939, "nauc_mrr_at_20_std": -0.045852, "nauc_mrr_at_20_diff1": 0.365479, "nauc_mrr_at_100_max": -0.053899, "nauc_mrr_at_100_std": -0.048132, "nauc_mrr_at_100_diff1": 0.365027, "nauc_mrr_at_1000_max": -0.05397, "nauc_mrr_at_1000_std": -0.048202, "nauc_mrr_at_1000_diff1": 0.364994, "hit_rate_at_1": 0.72126, "hit_rate_at_3": 0.86782, "hit_rate_at_5": 0.91379, "hit_rate_at_10": 0.9569, "hit_rate_at_20": 0.98276, "hit_rate_at_100": 0.99713, "hit_rate_at_1000": 1.0, "main_score": 0.70465, "hf_subset": "italian", "languages": [ "ita-Latn" ], "mteb_version": "2.13.1" }, { "ndcg_at_1": 0.71264, "ndcg_at_3": 0.67768, "ndcg_at_5": 0.6862, "ndcg_at_10": 0.71413, "ndcg_at_20": 0.74533, "ndcg_at_100": 0.78319, "ndcg_at_1000": 0.79065, "map_at_1": 0.31441, "map_at_3": 0.48057, "map_at_5": 0.53709, "map_at_10": 0.59369, "map_at_20": 0.62268, "map_at_100": 0.64205, "map_at_1000": 0.64315, "recall_at_1": 0.31441, "recall_at_3": 0.52503, "recall_at_5": 0.624, "recall_at_10": 0.75027, "recall_at_20": 0.85426, "recall_at_100": 0.96359, "recall_at_1000": 0.99729, "accuracy": 0.31441, "precision_at_1": 0.73563, "precision_at_3": 0.50287, "precision_at_5": 0.39425, "precision_at_10": 0.26638, "precision_at_20": 0.16494, "precision_at_100": 0.04221, "precision_at_1000": 0.00447, "mrr_at_1": 0.735632, "mrr_at_3": 0.802203, "mrr_at_5": 0.811686, "mrr_at_10": 0.816669, "mrr_at_20": 0.818531, "mrr_at_100": 0.818834, "mrr_at_1000": 0.818867, "nauc_ndcg_at_1_max": -0.027419, "nauc_ndcg_at_1_std": -0.07556, "nauc_ndcg_at_1_diff1": 0.333682, "nauc_ndcg_at_3_max": -0.041579, "nauc_ndcg_at_3_std": -0.137649, "nauc_ndcg_at_3_diff1": 0.26628, "nauc_ndcg_at_5_max": -0.005714, "nauc_ndcg_at_5_std": -0.128474, "nauc_ndcg_at_5_diff1": 0.270826, "nauc_ndcg_at_10_max": -0.018033, "nauc_ndcg_at_10_std": -0.157186, "nauc_ndcg_at_10_diff1": 0.272409, "nauc_ndcg_at_20_max": 0.015136, "nauc_ndcg_at_20_std": -0.120034, "nauc_ndcg_at_20_diff1": 0.29202, "nauc_ndcg_at_100_max": 0.043407, "nauc_ndcg_at_100_std": -0.065315, "nauc_ndcg_at_100_diff1": 0.29094, "nauc_ndcg_at_1000_max": 0.023114, "nauc_ndcg_at_1000_std": -0.084631, "nauc_ndcg_at_1000_diff1": 0.284999, "nauc_map_at_1_max": -0.027035, "nauc_map_at_1_std": -0.177363, "nauc_map_at_1_diff1": 0.420623, "nauc_map_at_3_max": -0.013778, "nauc_map_at_3_std": -0.172929, "nauc_map_at_3_diff1": 0.311293, "nauc_map_at_5_max": 7.4e-05, "nauc_map_at_5_std": -0.165312, "nauc_map_at_5_diff1": 0.294392, "nauc_map_at_10_max": -0.007103, "nauc_map_at_10_std": -0.173523, "nauc_map_at_10_diff1": 0.270481, "nauc_map_at_20_max": 0.01488, "nauc_map_at_20_std": -0.145742, "nauc_map_at_20_diff1": 0.273374, "nauc_map_at_100_max": 0.029716, "nauc_map_at_100_std": -0.119868, "nauc_map_at_100_diff1": 0.2743, "nauc_map_at_1000_max": 0.028012, "nauc_map_at_1000_std": -0.12135, "nauc_map_at_1000_diff1": 0.27348, "nauc_recall_at_1_max": -0.027035, "nauc_recall_at_1_std": -0.177363, "nauc_recall_at_1_diff1": 0.420623, "nauc_recall_at_3_max": -0.021568, "nauc_recall_at_3_std": -0.179972, "nauc_recall_at_3_diff1": 0.274483, "nauc_recall_at_5_max": -0.005652, "nauc_recall_at_5_std": -0.173007, "nauc_recall_at_5_diff1": 0.228602, "nauc_recall_at_10_max": -0.032363, "nauc_recall_at_10_std": -0.219352, "nauc_recall_at_10_diff1": 0.197668, "nauc_recall_at_20_max": 0.062152, "nauc_recall_at_20_std": -0.10134, "nauc_recall_at_20_diff1": 0.252798, "nauc_recall_at_100_max": 0.477179, "nauc_recall_at_100_std": 0.438891, "nauc_recall_at_100_diff1": 0.27378, "nauc_recall_at_1000_max": 0.480055, "nauc_recall_at_1000_std": 0.706286, "nauc_recall_at_1000_diff1": -0.150923, "nauc_precision_at_1_max": -0.055637, "nauc_precision_at_1_std": -0.083, "nauc_precision_at_1_diff1": 0.33076, "nauc_precision_at_3_max": -0.015466, "nauc_precision_at_3_std": 0.004771, "nauc_precision_at_3_diff1": -0.083474, "nauc_precision_at_5_max": 0.03504, "nauc_precision_at_5_std": 0.079409, "nauc_precision_at_5_diff1": -0.128062, "nauc_precision_at_10_max": 0.031496, "nauc_precision_at_10_std": 0.111235, "nauc_precision_at_10_diff1": -0.190748, "nauc_precision_at_20_max": 0.093019, "nauc_precision_at_20_std": 0.196889, "nauc_precision_at_20_diff1": -0.187944, "nauc_precision_at_100_max": 0.09391, "nauc_precision_at_100_std": 0.238657, "nauc_precision_at_100_diff1": -0.194759, "nauc_precision_at_1000_max": 0.051651, "nauc_precision_at_1000_std": 0.194132, "nauc_precision_at_1000_diff1": -0.207719, "nauc_mrr_at_1_max": -0.055637, "nauc_mrr_at_1_std": -0.083, "nauc_mrr_at_1_diff1": 0.33076, "nauc_mrr_at_3_max": -0.082354, "nauc_mrr_at_3_std": -0.088306, "nauc_mrr_at_3_diff1": 0.309442, "nauc_mrr_at_5_max": -0.059231, "nauc_mrr_at_5_std": -0.074164, "nauc_mrr_at_5_diff1": 0.307049, "nauc_mrr_at_10_max": -0.060308, "nauc_mrr_at_10_std": -0.083311, "nauc_mrr_at_10_diff1": 0.317252, "nauc_mrr_at_20_max": -0.060858, "nauc_mrr_at_20_std": -0.077823, "nauc_mrr_at_20_diff1": 0.31496, "nauc_mrr_at_100_max": -0.061487, "nauc_mrr_at_100_std": -0.078359, "nauc_mrr_at_100_diff1": 0.315014, "nauc_mrr_at_1000_max": -0.061695, "nauc_mrr_at_1000_std": -0.078574, "nauc_mrr_at_1000_diff1": 0.314924, "hit_rate_at_1": 0.73563, "hit_rate_at_3": 0.88218, "hit_rate_at_5": 0.92241, "hit_rate_at_10": 0.95977, "hit_rate_at_20": 0.98563, "hit_rate_at_100": 0.99425, "hit_rate_at_1000": 1.0, "main_score": 0.71413, "hf_subset": "portuguese", "languages": [ "por-Latn" ], "mteb_version": "2.13.1" } ] }, "evaluation_time": 56020.66130520124, "kg_co2_emissions": null, "date": 1783029335.567064, "evaluation_phases": [ { "name": "Data loading", "start": 4.144012928009033e-05, "end": 38.256029040087014, "split": "", "subset": "" }, { "name": "Dataset transform", "start": 38.25607448001392, "end": 38.256076449994, "split": "", "subset": "" }, { "name": "Encoding queries and documents", "start": 38.269814134109765, "end": 5652.892628813162, "split": "test", "subset": "italian" }, { "name": "Scoring", "start": 5652.8930498461705, "end": 5653.201427583117, "split": "test", "subset": "italian" }, { "name": "Encoding queries and documents", "start": 5653.247074790066, "end": 10492.527706969995, "split": "test", "subset": "portuguese" }, { "name": "Scoring", "start": 10492.527924982132, "end": 10492.831951234024, "split": "test", "subset": "portuguese" } ] }