{ "dataset_revision": "237ed4f43c7fb3c4df07ec4e9dd0a4366be555b0", "task_name": "Vidore3PharmaceuticalsRetrieval", "mteb_version": "2.18.6", "scores": { "test": [ { "ndcg_at_1": 0.64286, "ndcg_at_3": 0.64978, "ndcg_at_5": 0.64982, "ndcg_at_10": 0.67249, "ndcg_at_20": 0.70744, "ndcg_at_100": 0.75376, "ndcg_at_1000": 0.76769, "map_at_1": 0.33723, "map_at_3": 0.47051, "map_at_5": 0.50985, "map_at_10": 0.5576, "map_at_20": 0.59102, "map_at_100": 0.61321, "map_at_1000": 0.61529, "recall_at_1": 0.33723, "recall_at_3": 0.52759, "recall_at_5": 0.59763, "recall_at_10": 0.70628, "recall_at_20": 0.80939, "recall_at_100": 0.93981, "recall_at_1000": 0.99709, "accuracy": 0.33723, "precision_at_1": 0.66484, "precision_at_3": 0.43407, "precision_at_5": 0.33956, "precision_at_10": 0.23654, "precision_at_20": 0.15646, "precision_at_100": 0.04269, "precision_at_1000": 0.00474, "mrr_at_1": 0.664835, "mrr_at_3": 0.760073, "mrr_at_5": 0.768315, "mrr_at_10": 0.77392, "mrr_at_20": 0.775905, "mrr_at_100": 0.7763, "mrr_at_1000": 0.776327, "nauc_ndcg_at_1_max": 0.044218, "nauc_ndcg_at_1_std": -0.238814, "nauc_ndcg_at_1_diff1": 0.602836, "nauc_ndcg_at_3_max": 0.039743, "nauc_ndcg_at_3_std": -0.274349, "nauc_ndcg_at_3_diff1": 0.521313, "nauc_ndcg_at_5_max": 0.051285, "nauc_ndcg_at_5_std": -0.269257, "nauc_ndcg_at_5_diff1": 0.536249, "nauc_ndcg_at_10_max": 0.029327, "nauc_ndcg_at_10_std": -0.324043, "nauc_ndcg_at_10_diff1": 0.537104, "nauc_ndcg_at_20_max": 0.024891, "nauc_ndcg_at_20_std": -0.331463, "nauc_ndcg_at_20_diff1": 0.553847, "nauc_ndcg_at_100_max": 0.054916, "nauc_ndcg_at_100_std": -0.283349, "nauc_ndcg_at_100_diff1": 0.540029, "nauc_ndcg_at_1000_max": 0.070403, "nauc_ndcg_at_1000_std": -0.266792, "nauc_ndcg_at_1000_diff1": 0.54221, "nauc_map_at_1_max": 0.026935, "nauc_map_at_1_std": -0.245665, "nauc_map_at_1_diff1": 0.628798, "nauc_map_at_3_max": 0.037476, "nauc_map_at_3_std": -0.303723, "nauc_map_at_3_diff1": 0.556442, "nauc_map_at_5_max": 0.03795, "nauc_map_at_5_std": -0.299731, "nauc_map_at_5_diff1": 0.556206, "nauc_map_at_10_max": 0.029596, "nauc_map_at_10_std": -0.311039, "nauc_map_at_10_diff1": 0.533186, "nauc_map_at_20_max": 0.025251, "nauc_map_at_20_std": -0.314092, "nauc_map_at_20_diff1": 0.528903, "nauc_map_at_100_max": 0.032951, "nauc_map_at_100_std": -0.301888, "nauc_map_at_100_diff1": 0.52055, "nauc_map_at_1000_max": 0.035504, "nauc_map_at_1000_std": -0.298653, "nauc_map_at_1000_diff1": 0.520591, "nauc_recall_at_1_max": 0.026935, "nauc_recall_at_1_std": -0.245665, "nauc_recall_at_1_diff1": 0.628798, "nauc_recall_at_3_max": 0.025557, "nauc_recall_at_3_std": -0.31611, "nauc_recall_at_3_diff1": 0.498559, "nauc_recall_at_5_max": 0.007836, "nauc_recall_at_5_std": -0.314126, "nauc_recall_at_5_diff1": 0.49082, "nauc_recall_at_10_max": -0.03924, "nauc_recall_at_10_std": -0.379577, "nauc_recall_at_10_diff1": 0.438252, "nauc_recall_at_20_max": -0.097905, "nauc_recall_at_20_std": -0.447128, "nauc_recall_at_20_diff1": 0.479284, "nauc_recall_at_100_max": -0.072224, "nauc_recall_at_100_std": -0.370037, "nauc_recall_at_100_diff1": 0.427127, "nauc_recall_at_1000_max": 0.193492, "nauc_recall_at_1000_std": -0.244328, "nauc_recall_at_1000_diff1": 0.476893, "nauc_precision_at_1_max": 0.053837, "nauc_precision_at_1_std": -0.258096, "nauc_precision_at_1_diff1": 0.619036, "nauc_precision_at_3_max": 0.041432, "nauc_precision_at_3_std": -0.077013, "nauc_precision_at_3_diff1": -0.027238, "nauc_precision_at_5_max": 0.031789, "nauc_precision_at_5_std": 0.035715, "nauc_precision_at_5_diff1": -0.140595, "nauc_precision_at_10_max": 0.003153, "nauc_precision_at_10_std": 0.068312, "nauc_precision_at_10_diff1": -0.261356, "nauc_precision_at_20_max": 0.007616, "nauc_precision_at_20_std": 0.119323, "nauc_precision_at_20_diff1": -0.28341, "nauc_precision_at_100_max": 0.064896, "nauc_precision_at_100_std": 0.246644, "nauc_precision_at_100_diff1": -0.346137, "nauc_precision_at_1000_max": 0.07856, "nauc_precision_at_1000_std": 0.280068, "nauc_precision_at_1000_diff1": -0.363602, "nauc_mrr_at_1_max": 0.053837, "nauc_mrr_at_1_std": -0.258096, "nauc_mrr_at_1_diff1": 0.619036, "nauc_mrr_at_3_max": 0.104192, "nauc_mrr_at_3_std": -0.293039, "nauc_mrr_at_3_diff1": 0.606072, "nauc_mrr_at_5_max": 0.081838, "nauc_mrr_at_5_std": -0.294902, "nauc_mrr_at_5_diff1": 0.597188, "nauc_mrr_at_10_max": 0.081929, "nauc_mrr_at_10_std": -0.29555, "nauc_mrr_at_10_diff1": 0.596402, "nauc_mrr_at_20_max": 0.0807, "nauc_mrr_at_20_std": -0.29111, "nauc_mrr_at_20_diff1": 0.599571, "nauc_mrr_at_100_max": 0.080697, "nauc_mrr_at_100_std": -0.290834, "nauc_mrr_at_100_diff1": 0.599166, "nauc_mrr_at_1000_max": 0.080906, "nauc_mrr_at_1000_std": -0.290679, "nauc_mrr_at_1000_diff1": 0.599266, "hit_rate_at_1": 0.66484, "hit_rate_at_3": 0.87363, "hit_rate_at_5": 0.91209, "hit_rate_at_10": 0.9533, "hit_rate_at_20": 0.98077, "hit_rate_at_100": 0.99451, "hit_rate_at_1000": 1.0, "main_score": 0.67249, "hf_subset": "french", "languages": [ "fra-Latn" ], "mteb_version": "2.18.6" }, { "ndcg_at_1": 0.65659, "ndcg_at_3": 0.65111, "ndcg_at_5": 0.65407, "ndcg_at_10": 0.67885, "ndcg_at_20": 0.71047, "ndcg_at_100": 0.75757, "ndcg_at_1000": 0.7719, "map_at_1": 0.34805, "map_at_3": 0.47774, "map_at_5": 0.51569, "map_at_10": 0.56558, "map_at_20": 0.59667, "map_at_100": 0.61929, "map_at_1000": 0.62129, "recall_at_1": 0.34805, "recall_at_3": 0.53185, "recall_at_5": 0.5977, "recall_at_10": 0.70979, "recall_at_20": 0.80457, "recall_at_100": 0.93786, "recall_at_1000": 0.99764, "accuracy": 0.34805, "precision_at_1": 0.67582, "precision_at_3": 0.43132, "precision_at_5": 0.33901, "precision_at_10": 0.23874, "precision_at_20": 0.15549, "precision_at_100": 0.04275, "precision_at_1000": 0.00474, "mrr_at_1": 0.675824, "mrr_at_3": 0.767857, "mrr_at_5": 0.775549, "mrr_at_10": 0.781608, "mrr_at_20": 0.783364, "mrr_at_100": 0.783815, "mrr_at_1000": 0.783849, "nauc_ndcg_at_1_max": 0.06123, "nauc_ndcg_at_1_std": -0.24972, "nauc_ndcg_at_1_diff1": 0.605646, "nauc_ndcg_at_3_max": 0.075366, "nauc_ndcg_at_3_std": -0.258909, "nauc_ndcg_at_3_diff1": 0.527229, "nauc_ndcg_at_5_max": 0.092956, "nauc_ndcg_at_5_std": -0.259258, "nauc_ndcg_at_5_diff1": 0.534664, "nauc_ndcg_at_10_max": 0.079078, "nauc_ndcg_at_10_std": -0.305349, "nauc_ndcg_at_10_diff1": 0.545973, "nauc_ndcg_at_20_max": 0.083571, "nauc_ndcg_at_20_std": -0.310849, "nauc_ndcg_at_20_diff1": 0.551201, "nauc_ndcg_at_100_max": 0.103642, "nauc_ndcg_at_100_std": -0.265223, "nauc_ndcg_at_100_diff1": 0.537417, "nauc_ndcg_at_1000_max": 0.110093, "nauc_ndcg_at_1000_std": -0.257811, "nauc_ndcg_at_1000_diff1": 0.54473, "nauc_map_at_1_max": 0.069776, "nauc_map_at_1_std": -0.236841, "nauc_map_at_1_diff1": 0.620468, "nauc_map_at_3_max": 0.08492, "nauc_map_at_3_std": -0.293453, "nauc_map_at_3_diff1": 0.564839, "nauc_map_at_5_max": 0.089535, "nauc_map_at_5_std": -0.293295, "nauc_map_at_5_diff1": 0.558913, "nauc_map_at_10_max": 0.080118, "nauc_map_at_10_std": -0.307932, "nauc_map_at_10_diff1": 0.543013, "nauc_map_at_20_max": 0.07969, "nauc_map_at_20_std": -0.307712, "nauc_map_at_20_diff1": 0.536404, "nauc_map_at_100_max": 0.084573, "nauc_map_at_100_std": -0.295219, "nauc_map_at_100_diff1": 0.528551, "nauc_map_at_1000_max": 0.086152, "nauc_map_at_1000_std": -0.292952, "nauc_map_at_1000_diff1": 0.528504, "nauc_recall_at_1_max": 0.069776, "nauc_recall_at_1_std": -0.236841, "nauc_recall_at_1_diff1": 0.620468, "nauc_recall_at_3_max": 0.078985, "nauc_recall_at_3_std": -0.289045, "nauc_recall_at_3_diff1": 0.506702, "nauc_recall_at_5_max": 0.078232, "nauc_recall_at_5_std": -0.295899, "nauc_recall_at_5_diff1": 0.494238, "nauc_recall_at_10_max": 0.034418, "nauc_recall_at_10_std": -0.346035, "nauc_recall_at_10_diff1": 0.453007, "nauc_recall_at_20_max": 0.017513, "nauc_recall_at_20_std": -0.393486, "nauc_recall_at_20_diff1": 0.462665, "nauc_recall_at_100_max": 0.064423, "nauc_recall_at_100_std": -0.258432, "nauc_recall_at_100_diff1": 0.382659, "nauc_recall_at_1000_max": -0.027368, "nauc_recall_at_1000_std": -0.525667, "nauc_recall_at_1000_diff1": 0.385641, "nauc_precision_at_1_max": 0.054729, "nauc_precision_at_1_std": -0.281564, "nauc_precision_at_1_diff1": 0.622944, "nauc_precision_at_3_max": 0.04082, "nauc_precision_at_3_std": -0.076946, "nauc_precision_at_3_diff1": -0.020216, "nauc_precision_at_5_max": 0.02073, "nauc_precision_at_5_std": 0.009797, "nauc_precision_at_5_diff1": -0.137589, "nauc_precision_at_10_max": -0.017028, "nauc_precision_at_10_std": 0.050428, "nauc_precision_at_10_diff1": -0.24811, "nauc_precision_at_20_max": -0.015652, "nauc_precision_at_20_std": 0.111239, "nauc_precision_at_20_diff1": -0.295354, "nauc_precision_at_100_max": 0.018289, "nauc_precision_at_100_std": 0.235486, "nauc_precision_at_100_diff1": -0.355052, "nauc_precision_at_1000_max": 0.018936, "nauc_precision_at_1000_std": 0.259135, "nauc_precision_at_1000_diff1": -0.373454, "nauc_mrr_at_1_max": 0.054729, "nauc_mrr_at_1_std": -0.281564, "nauc_mrr_at_1_diff1": 0.622944, "nauc_mrr_at_3_max": 0.105276, "nauc_mrr_at_3_std": -0.275341, "nauc_mrr_at_3_diff1": 0.594179, "nauc_mrr_at_5_max": 0.093976, "nauc_mrr_at_5_std": -0.291085, "nauc_mrr_at_5_diff1": 0.59502, "nauc_mrr_at_10_max": 0.091994, "nauc_mrr_at_10_std": -0.283548, "nauc_mrr_at_10_diff1": 0.599091, "nauc_mrr_at_20_max": 0.088774, "nauc_mrr_at_20_std": -0.285254, "nauc_mrr_at_20_diff1": 0.599034, "nauc_mrr_at_100_max": 0.088627, "nauc_mrr_at_100_std": -0.284646, "nauc_mrr_at_100_diff1": 0.59899, "nauc_mrr_at_1000_max": 0.088846, "nauc_mrr_at_1000_std": -0.284493, "nauc_mrr_at_1000_diff1": 0.599058, "hit_rate_at_1": 0.67582, "hit_rate_at_3": 0.87363, "hit_rate_at_5": 0.90934, "hit_rate_at_10": 0.95604, "hit_rate_at_20": 0.98077, "hit_rate_at_100": 0.99451, "hit_rate_at_1000": 1.0, "main_score": 0.67885, "hf_subset": "spanish", "languages": [ "spa-Latn" ], "mteb_version": "2.18.6" }, { "ndcg_at_1": 0.6717, "ndcg_at_3": 0.66036, "ndcg_at_5": 0.66456, "ndcg_at_10": 0.68866, "ndcg_at_20": 0.72183, "ndcg_at_100": 0.76734, "ndcg_at_1000": 0.78061, "map_at_1": 0.35991, "map_at_3": 0.48994, "map_at_5": 0.52943, "map_at_10": 0.58068, "map_at_20": 0.61336, "map_at_100": 0.63465, "map_at_1000": 0.63665, "recall_at_1": 0.35991, "recall_at_3": 0.53594, "recall_at_5": 0.60498, "recall_at_10": 0.71555, "recall_at_20": 0.81311, "recall_at_100": 0.94397, "recall_at_1000": 0.99764, "accuracy": 0.35991, "precision_at_1": 0.69505, "precision_at_3": 0.43864, "precision_at_5": 0.34451, "precision_at_10": 0.24258, "precision_at_20": 0.15838, "precision_at_100": 0.04297, "precision_at_1000": 0.00474, "mrr_at_1": 0.695055, "mrr_at_3": 0.772894, "mrr_at_5": 0.780723, "mrr_at_10": 0.786583, "mrr_at_20": 0.7889, "mrr_at_100": 0.789337, "mrr_at_1000": 0.789367, "nauc_ndcg_at_1_max": 0.093937, "nauc_ndcg_at_1_std": -0.280589, "nauc_ndcg_at_1_diff1": 0.581499, "nauc_ndcg_at_3_max": 0.125949, "nauc_ndcg_at_3_std": -0.272898, "nauc_ndcg_at_3_diff1": 0.530026, "nauc_ndcg_at_5_max": 0.130401, "nauc_ndcg_at_5_std": -0.263149, "nauc_ndcg_at_5_diff1": 0.525784, "nauc_ndcg_at_10_max": 0.136076, "nauc_ndcg_at_10_std": -0.294972, "nauc_ndcg_at_10_diff1": 0.548574, "nauc_ndcg_at_20_max": 0.128181, "nauc_ndcg_at_20_std": -0.294201, "nauc_ndcg_at_20_diff1": 0.560426, "nauc_ndcg_at_100_max": 0.142447, "nauc_ndcg_at_100_std": -0.261505, "nauc_ndcg_at_100_diff1": 0.539911, "nauc_ndcg_at_1000_max": 0.14783, "nauc_ndcg_at_1000_std": -0.259646, "nauc_ndcg_at_1000_diff1": 0.539727, "nauc_map_at_1_max": 0.134803, "nauc_map_at_1_std": -0.217625, "nauc_map_at_1_diff1": 0.611368, "nauc_map_at_3_max": 0.164127, "nauc_map_at_3_std": -0.277007, "nauc_map_at_3_diff1": 0.571249, "nauc_map_at_5_max": 0.150306, "nauc_map_at_5_std": -0.279008, "nauc_map_at_5_diff1": 0.562677, "nauc_map_at_10_max": 0.137173, "nauc_map_at_10_std": -0.292881, "nauc_map_at_10_diff1": 0.545055, "nauc_map_at_20_max": 0.127467, "nauc_map_at_20_std": -0.29421, "nauc_map_at_20_diff1": 0.538774, "nauc_map_at_100_max": 0.130521, "nauc_map_at_100_std": -0.285465, "nauc_map_at_100_diff1": 0.52842, "nauc_map_at_1000_max": 0.132011, "nauc_map_at_1000_std": -0.283716, "nauc_map_at_1000_diff1": 0.527688, "nauc_recall_at_1_max": 0.134803, "nauc_recall_at_1_std": -0.217625, "nauc_recall_at_1_diff1": 0.611368, "nauc_recall_at_3_max": 0.171621, "nauc_recall_at_3_std": -0.268762, "nauc_recall_at_3_diff1": 0.540225, "nauc_recall_at_5_max": 0.133731, "nauc_recall_at_5_std": -0.276591, "nauc_recall_at_5_diff1": 0.502807, "nauc_recall_at_10_max": 0.10266, "nauc_recall_at_10_std": -0.315967, "nauc_recall_at_10_diff1": 0.485734, "nauc_recall_at_20_max": 0.06872, "nauc_recall_at_20_std": -0.320908, "nauc_recall_at_20_diff1": 0.511777, "nauc_recall_at_100_max": 0.107059, "nauc_recall_at_100_std": -0.185603, "nauc_recall_at_100_diff1": 0.452682, "nauc_recall_at_1000_max": -0.019781, "nauc_recall_at_1000_std": -0.431432, "nauc_recall_at_1000_diff1": 0.28978, "nauc_precision_at_1_max": 0.08099, "nauc_precision_at_1_std": -0.311371, "nauc_precision_at_1_diff1": 0.589849, "nauc_precision_at_3_max": 0.011132, "nauc_precision_at_3_std": -0.111698, "nauc_precision_at_3_diff1": -0.017597, "nauc_precision_at_5_max": -0.057558, "nauc_precision_at_5_std": -0.013872, "nauc_precision_at_5_diff1": -0.170067, "nauc_precision_at_10_max": -0.093592, "nauc_precision_at_10_std": 0.01641, "nauc_precision_at_10_diff1": -0.261212, "nauc_precision_at_20_max": -0.09949, "nauc_precision_at_20_std": 0.077119, "nauc_precision_at_20_diff1": -0.2996, "nauc_precision_at_100_max": -0.061532, "nauc_precision_at_100_std": 0.193726, "nauc_precision_at_100_diff1": -0.379036, "nauc_precision_at_1000_max": -0.055623, "nauc_precision_at_1000_std": 0.210552, "nauc_precision_at_1000_diff1": -0.398771, "nauc_mrr_at_1_max": 0.08099, "nauc_mrr_at_1_std": -0.311371, "nauc_mrr_at_1_diff1": 0.589849, "nauc_mrr_at_3_max": 0.141661, "nauc_mrr_at_3_std": -0.310319, "nauc_mrr_at_3_diff1": 0.583916, "nauc_mrr_at_5_max": 0.127061, "nauc_mrr_at_5_std": -0.313292, "nauc_mrr_at_5_diff1": 0.5704, "nauc_mrr_at_10_max": 0.120065, "nauc_mrr_at_10_std": -0.320669, "nauc_mrr_at_10_diff1": 0.577144, "nauc_mrr_at_20_max": 0.122524, "nauc_mrr_at_20_std": -0.314864, "nauc_mrr_at_20_diff1": 0.578635, "nauc_mrr_at_100_max": 0.122544, "nauc_mrr_at_100_std": -0.31515, "nauc_mrr_at_100_diff1": 0.578414, "nauc_mrr_at_1000_max": 0.122605, "nauc_mrr_at_1000_std": -0.315084, "nauc_mrr_at_1000_diff1": 0.578468, "hit_rate_at_1": 0.69505, "hit_rate_at_3": 0.86813, "hit_rate_at_5": 0.90385, "hit_rate_at_10": 0.9478, "hit_rate_at_20": 0.98077, "hit_rate_at_100": 0.99451, "hit_rate_at_1000": 1.0, "main_score": 0.68866, "hf_subset": "english", "languages": [ "eng-Latn" ], "mteb_version": "2.18.6" }, { "ndcg_at_1": 0.64835, "ndcg_at_3": 0.64555, "ndcg_at_5": 0.65038, "ndcg_at_10": 0.67422, "ndcg_at_20": 0.7061, "ndcg_at_100": 0.75469, "ndcg_at_1000": 0.76828, "map_at_1": 0.34275, "map_at_3": 0.47438, "map_at_5": 0.51359, "map_at_10": 0.56407, "map_at_20": 0.59415, "map_at_100": 0.61658, "map_at_1000": 0.61853, "recall_at_1": 0.34275, "recall_at_3": 0.52804, "recall_at_5": 0.59959, "recall_at_10": 0.70605, "recall_at_20": 0.79798, "recall_at_100": 0.94017, "recall_at_1000": 0.99629, "accuracy": 0.34275, "precision_at_1": 0.66758, "precision_at_3": 0.42766, "precision_at_5": 0.33516, "precision_at_10": 0.23791, "precision_at_20": 0.1544, "precision_at_100": 0.0425, "precision_at_1000": 0.00471, "mrr_at_1": 0.667582, "mrr_at_3": 0.755952, "mrr_at_5": 0.764881, "mrr_at_10": 0.769459, "mrr_at_20": 0.77228, "mrr_at_100": 0.773002, "mrr_at_1000": 0.773022, "nauc_ndcg_at_1_max": 0.021263, "nauc_ndcg_at_1_std": -0.275713, "nauc_ndcg_at_1_diff1": 0.564973, "nauc_ndcg_at_3_max": 0.025751, "nauc_ndcg_at_3_std": -0.273943, "nauc_ndcg_at_3_diff1": 0.502961, "nauc_ndcg_at_5_max": 0.045296, "nauc_ndcg_at_5_std": -0.272539, "nauc_ndcg_at_5_diff1": 0.506927, "nauc_ndcg_at_10_max": 0.032409, "nauc_ndcg_at_10_std": -0.313117, "nauc_ndcg_at_10_diff1": 0.520293, "nauc_ndcg_at_20_max": 0.033805, "nauc_ndcg_at_20_std": -0.310508, "nauc_ndcg_at_20_diff1": 0.525274, "nauc_ndcg_at_100_max": 0.050664, "nauc_ndcg_at_100_std": -0.268653, "nauc_ndcg_at_100_diff1": 0.504764, "nauc_ndcg_at_1000_max": 0.065652, "nauc_ndcg_at_1000_std": -0.260959, "nauc_ndcg_at_1000_diff1": 0.509823, "nauc_map_at_1_max": 0.045619, "nauc_map_at_1_std": -0.227414, "nauc_map_at_1_diff1": 0.607897, "nauc_map_at_3_max": 0.049729, "nauc_map_at_3_std": -0.286341, "nauc_map_at_3_diff1": 0.557457, "nauc_map_at_5_max": 0.056284, "nauc_map_at_5_std": -0.287349, "nauc_map_at_5_diff1": 0.548691, "nauc_map_at_10_max": 0.040702, "nauc_map_at_10_std": -0.306916, "nauc_map_at_10_diff1": 0.530398, "nauc_map_at_20_max": 0.038076, "nauc_map_at_20_std": -0.304282, "nauc_map_at_20_diff1": 0.520689, "nauc_map_at_100_max": 0.039028, "nauc_map_at_100_std": -0.292844, "nauc_map_at_100_diff1": 0.508699, "nauc_map_at_1000_max": 0.041506, "nauc_map_at_1000_std": -0.290477, "nauc_map_at_1000_diff1": 0.508501, "nauc_recall_at_1_max": 0.045619, "nauc_recall_at_1_std": -0.227414, "nauc_recall_at_1_diff1": 0.607897, "nauc_recall_at_3_max": 0.034909, "nauc_recall_at_3_std": -0.290151, "nauc_recall_at_3_diff1": 0.49728, "nauc_recall_at_5_max": 0.027844, "nauc_recall_at_5_std": -0.302895, "nauc_recall_at_5_diff1": 0.470362, "nauc_recall_at_10_max": -0.015393, "nauc_recall_at_10_std": -0.363121, "nauc_recall_at_10_diff1": 0.435926, "nauc_recall_at_20_max": -0.048835, "nauc_recall_at_20_std": -0.375837, "nauc_recall_at_20_diff1": 0.429657, "nauc_recall_at_100_max": -0.079776, "nauc_recall_at_100_std": -0.263757, "nauc_recall_at_100_diff1": 0.336194, "nauc_recall_at_1000_max": 0.016926, "nauc_recall_at_1000_std": -0.218456, "nauc_recall_at_1000_diff1": 0.324336, "nauc_precision_at_1_max": 0.00797, "nauc_precision_at_1_std": -0.301055, "nauc_precision_at_1_diff1": 0.585315, "nauc_precision_at_3_max": 0.00314, "nauc_precision_at_3_std": -0.096584, "nauc_precision_at_3_diff1": -0.007702, "nauc_precision_at_5_max": 0.026883, "nauc_precision_at_5_std": -8.7e-05, "nauc_precision_at_5_diff1": -0.134281, "nauc_precision_at_10_max": -0.028941, "nauc_precision_at_10_std": 0.028531, "nauc_precision_at_10_diff1": -0.241177, "nauc_precision_at_20_max": -0.026846, "nauc_precision_at_20_std": 0.090879, "nauc_precision_at_20_diff1": -0.289899, "nauc_precision_at_100_max": 0.013186, "nauc_precision_at_100_std": 0.214667, "nauc_precision_at_100_diff1": -0.373792, "nauc_precision_at_1000_max": 0.031825, "nauc_precision_at_1000_std": 0.236014, "nauc_precision_at_1000_diff1": -0.388291, "nauc_mrr_at_1_max": 0.00797, "nauc_mrr_at_1_std": -0.301055, "nauc_mrr_at_1_diff1": 0.585315, "nauc_mrr_at_3_max": 0.065831, "nauc_mrr_at_3_std": -0.291147, "nauc_mrr_at_3_diff1": 0.563915, "nauc_mrr_at_5_max": 0.0501, "nauc_mrr_at_5_std": -0.297088, "nauc_mrr_at_5_diff1": 0.556769, "nauc_mrr_at_10_max": 0.051145, "nauc_mrr_at_10_std": -0.295753, "nauc_mrr_at_10_diff1": 0.559362, "nauc_mrr_at_20_max": 0.050655, "nauc_mrr_at_20_std": -0.291211, "nauc_mrr_at_20_diff1": 0.560792, "nauc_mrr_at_100_max": 0.049904, "nauc_mrr_at_100_std": -0.292738, "nauc_mrr_at_100_diff1": 0.561184, "nauc_mrr_at_1000_max": 0.050038, "nauc_mrr_at_1000_std": -0.29264, "nauc_mrr_at_1000_diff1": 0.561243, "hit_rate_at_1": 0.66758, "hit_rate_at_3": 0.86264, "hit_rate_at_5": 0.9011, "hit_rate_at_10": 0.93407, "hit_rate_at_20": 0.97253, "hit_rate_at_100": 0.99451, "hit_rate_at_1000": 1.0, "main_score": 0.67422, "hf_subset": "german", "languages": [ "deu-Latn" ], "mteb_version": "2.18.6" }, { "ndcg_at_1": 0.6511, "ndcg_at_3": 0.65058, "ndcg_at_5": 0.65371, "ndcg_at_10": 0.67835, "ndcg_at_20": 0.71238, "ndcg_at_100": 0.7573, "ndcg_at_1000": 0.77189, "map_at_1": 0.35209, "map_at_3": 0.47838, "map_at_5": 0.51682, "map_at_10": 0.56754, "map_at_20": 0.59982, "map_at_100": 0.62085, "map_at_1000": 0.62297, "recall_at_1": 0.35209, "recall_at_3": 0.52991, "recall_at_5": 0.59735, "recall_at_10": 0.70996, "recall_at_20": 0.80994, "recall_at_100": 0.93738, "recall_at_1000": 0.99688, "accuracy": 0.35209, "precision_at_1": 0.67582, "precision_at_3": 0.43315, "precision_at_5": 0.33791, "precision_at_10": 0.23846, "precision_at_20": 0.15659, "precision_at_100": 0.04239, "precision_at_1000": 0.00473, "mrr_at_1": 0.675824, "mrr_at_3": 0.767857, "mrr_at_5": 0.774038, "mrr_at_10": 0.780691, "mrr_at_20": 0.782963, "mrr_at_100": 0.783279, "mrr_at_1000": 0.783303, "nauc_ndcg_at_1_max": 0.009986, "nauc_ndcg_at_1_std": -0.220512, "nauc_ndcg_at_1_diff1": 0.606037, "nauc_ndcg_at_3_max": 0.018246, "nauc_ndcg_at_3_std": -0.259639, "nauc_ndcg_at_3_diff1": 0.509131, "nauc_ndcg_at_5_max": 0.036002, "nauc_ndcg_at_5_std": -0.247438, "nauc_ndcg_at_5_diff1": 0.521609, "nauc_ndcg_at_10_max": 0.006386, "nauc_ndcg_at_10_std": -0.300393, "nauc_ndcg_at_10_diff1": 0.53793, "nauc_ndcg_at_20_max": 0.006916, "nauc_ndcg_at_20_std": -0.305939, "nauc_ndcg_at_20_diff1": 0.547279, "nauc_ndcg_at_100_max": 0.032348, "nauc_ndcg_at_100_std": -0.255681, "nauc_ndcg_at_100_diff1": 0.529764, "nauc_ndcg_at_1000_max": 0.044141, "nauc_ndcg_at_1000_std": -0.246184, "nauc_ndcg_at_1000_diff1": 0.533874, "nauc_map_at_1_max": 0.024143, "nauc_map_at_1_std": -0.220444, "nauc_map_at_1_diff1": 0.62575, "nauc_map_at_3_max": 0.023022, "nauc_map_at_3_std": -0.284816, "nauc_map_at_3_diff1": 0.558002, "nauc_map_at_5_max": 0.027586, "nauc_map_at_5_std": -0.279761, "nauc_map_at_5_diff1": 0.554062, "nauc_map_at_10_max": 0.011127, "nauc_map_at_10_std": -0.297445, "nauc_map_at_10_diff1": 0.5368, "nauc_map_at_20_max": 0.007832, "nauc_map_at_20_std": -0.298807, "nauc_map_at_20_diff1": 0.526969, "nauc_map_at_100_max": 0.013667, "nauc_map_at_100_std": -0.285164, "nauc_map_at_100_diff1": 0.51652, "nauc_map_at_1000_max": 0.016082, "nauc_map_at_1000_std": -0.282639, "nauc_map_at_1000_diff1": 0.516421, "nauc_recall_at_1_max": 0.024143, "nauc_recall_at_1_std": -0.220444, "nauc_recall_at_1_diff1": 0.62575, "nauc_recall_at_3_max": 0.020639, "nauc_recall_at_3_std": -0.298531, "nauc_recall_at_3_diff1": 0.513134, "nauc_recall_at_5_max": 0.011905, "nauc_recall_at_5_std": -0.301584, "nauc_recall_at_5_diff1": 0.495722, "nauc_recall_at_10_max": -0.048581, "nauc_recall_at_10_std": -0.36114, "nauc_recall_at_10_diff1": 0.463063, "nauc_recall_at_20_max": -0.083795, "nauc_recall_at_20_std": -0.424476, "nauc_recall_at_20_diff1": 0.481555, "nauc_recall_at_100_max": -0.048835, "nauc_recall_at_100_std": -0.289904, "nauc_recall_at_100_diff1": 0.418224, "nauc_recall_at_1000_max": -0.076521, "nauc_recall_at_1000_std": -0.521687, "nauc_recall_at_1000_diff1": 0.367571, "nauc_precision_at_1_max": 0.00473, "nauc_precision_at_1_std": -0.249285, "nauc_precision_at_1_diff1": 0.622003, "nauc_precision_at_3_max": 0.003061, "nauc_precision_at_3_std": -0.095809, "nauc_precision_at_3_diff1": -0.059314, "nauc_precision_at_5_max": 0.018171, "nauc_precision_at_5_std": 0.02124, "nauc_precision_at_5_diff1": -0.15808, "nauc_precision_at_10_max": -0.027112, "nauc_precision_at_10_std": 0.041866, "nauc_precision_at_10_diff1": -0.258715, "nauc_precision_at_20_max": -0.013079, "nauc_precision_at_20_std": 0.097055, "nauc_precision_at_20_diff1": -0.30009, "nauc_precision_at_100_max": 0.041173, "nauc_precision_at_100_std": 0.226739, "nauc_precision_at_100_diff1": -0.369903, "nauc_precision_at_1000_max": 0.055545, "nauc_precision_at_1000_std": 0.249843, "nauc_precision_at_1000_diff1": -0.38327, "nauc_mrr_at_1_max": 0.00473, "nauc_mrr_at_1_std": -0.249285, "nauc_mrr_at_1_diff1": 0.622003, "nauc_mrr_at_3_max": 0.047668, "nauc_mrr_at_3_std": -0.281163, "nauc_mrr_at_3_diff1": 0.593563, "nauc_mrr_at_5_max": 0.033459, "nauc_mrr_at_5_std": -0.291207, "nauc_mrr_at_5_diff1": 0.592153, "nauc_mrr_at_10_max": 0.030865, "nauc_mrr_at_10_std": -0.283425, "nauc_mrr_at_10_diff1": 0.597673, "nauc_mrr_at_20_max": 0.028984, "nauc_mrr_at_20_std": -0.283522, "nauc_mrr_at_20_diff1": 0.59793, "nauc_mrr_at_100_max": 0.029454, "nauc_mrr_at_100_std": -0.282717, "nauc_mrr_at_100_diff1": 0.598092, "nauc_mrr_at_1000_max": 0.029611, "nauc_mrr_at_1000_std": -0.282599, "nauc_mrr_at_1000_diff1": 0.598133, "hit_rate_at_1": 0.67582, "hit_rate_at_3": 0.87637, "hit_rate_at_5": 0.90385, "hit_rate_at_10": 0.95055, "hit_rate_at_20": 0.98352, "hit_rate_at_100": 0.99451, "hit_rate_at_1000": 1.0, "main_score": 0.67835, "hf_subset": "italian", "languages": [ "ita-Latn" ], "mteb_version": "2.18.6" }, { "ndcg_at_1": 0.64011, "ndcg_at_3": 0.65407, "ndcg_at_5": 0.65681, "ndcg_at_10": 0.68033, "ndcg_at_20": 0.71072, "ndcg_at_100": 0.75892, "ndcg_at_1000": 0.77292, "map_at_1": 0.34902, "map_at_3": 0.48092, "map_at_5": 0.52, "map_at_10": 0.57002, "map_at_20": 0.60088, "map_at_100": 0.62361, "map_at_1000": 0.62567, "recall_at_1": 0.34902, "recall_at_3": 0.53397, "recall_at_5": 0.60381, "recall_at_10": 0.71077, "recall_at_20": 0.80094, "recall_at_100": 0.9404, "recall_at_1000": 0.99764, "accuracy": 0.34902, "precision_at_1": 0.65934, "precision_at_3": 0.43956, "precision_at_5": 0.34396, "precision_at_10": 0.24038, "precision_at_20": 0.15536, "precision_at_100": 0.04269, "precision_at_1000": 0.00474, "mrr_at_1": 0.659341, "mrr_at_3": 0.7587, "mrr_at_5": 0.765842, "mrr_at_10": 0.771987, "mrr_at_20": 0.773691, "mrr_at_100": 0.774176, "mrr_at_1000": 0.77418, "nauc_ndcg_at_1_max": 0.065434, "nauc_ndcg_at_1_std": -0.281763, "nauc_ndcg_at_1_diff1": 0.644433, "nauc_ndcg_at_3_max": 0.064795, "nauc_ndcg_at_3_std": -0.276242, "nauc_ndcg_at_3_diff1": 0.545738, "nauc_ndcg_at_5_max": 0.096817, "nauc_ndcg_at_5_std": -0.256185, "nauc_ndcg_at_5_diff1": 0.553347, "nauc_ndcg_at_10_max": 0.075442, "nauc_ndcg_at_10_std": -0.304101, "nauc_ndcg_at_10_diff1": 0.5587, "nauc_ndcg_at_20_max": 0.085062, "nauc_ndcg_at_20_std": -0.3031, "nauc_ndcg_at_20_diff1": 0.565093, "nauc_ndcg_at_100_max": 0.111906, "nauc_ndcg_at_100_std": -0.254265, "nauc_ndcg_at_100_diff1": 0.559325, "nauc_ndcg_at_1000_max": 0.118717, "nauc_ndcg_at_1000_std": -0.249358, "nauc_ndcg_at_1000_diff1": 0.563302, "nauc_map_at_1_max": 0.098194, "nauc_map_at_1_std": -0.220614, "nauc_map_at_1_diff1": 0.61793, "nauc_map_at_3_max": 0.100382, "nauc_map_at_3_std": -0.287476, "nauc_map_at_3_diff1": 0.569392, "nauc_map_at_5_max": 0.107908, "nauc_map_at_5_std": -0.278473, "nauc_map_at_5_diff1": 0.564317, "nauc_map_at_10_max": 0.090707, "nauc_map_at_10_std": -0.295496, "nauc_map_at_10_diff1": 0.544737, "nauc_map_at_20_max": 0.090741, "nauc_map_at_20_std": -0.293191, "nauc_map_at_20_diff1": 0.539379, "nauc_map_at_100_max": 0.096791, "nauc_map_at_100_std": -0.279082, "nauc_map_at_100_diff1": 0.533731, "nauc_map_at_1000_max": 0.098663, "nauc_map_at_1000_std": -0.276826, "nauc_map_at_1000_diff1": 0.533617, "nauc_recall_at_1_max": 0.098194, "nauc_recall_at_1_std": -0.220614, "nauc_recall_at_1_diff1": 0.61793, "nauc_recall_at_3_max": 0.090138, "nauc_recall_at_3_std": -0.2874, "nauc_recall_at_3_diff1": 0.51955, "nauc_recall_at_5_max": 0.085762, "nauc_recall_at_5_std": -0.282295, "nauc_recall_at_5_diff1": 0.4974, "nauc_recall_at_10_max": 0.023231, "nauc_recall_at_10_std": -0.342465, "nauc_recall_at_10_diff1": 0.456283, "nauc_recall_at_20_max": 0.016396, "nauc_recall_at_20_std": -0.36915, "nauc_recall_at_20_diff1": 0.458405, "nauc_recall_at_100_max": 0.073446, "nauc_recall_at_100_std": -0.217024, "nauc_recall_at_100_diff1": 0.431265, "nauc_recall_at_1000_max": -0.020509, "nauc_recall_at_1000_std": -0.558272, "nauc_recall_at_1000_diff1": 0.457244, "nauc_precision_at_1_max": 0.059381, "nauc_precision_at_1_std": -0.31109, "nauc_precision_at_1_diff1": 0.668424, "nauc_precision_at_3_max": -0.000649, "nauc_precision_at_3_std": -0.105624, "nauc_precision_at_3_diff1": -0.016717, "nauc_precision_at_5_max": 0.006887, "nauc_precision_at_5_std": 0.014127, "nauc_precision_at_5_diff1": -0.134728, "nauc_precision_at_10_max": -0.047965, "nauc_precision_at_10_std": 0.043032, "nauc_precision_at_10_diff1": -0.243549, "nauc_precision_at_20_max": -0.028971, "nauc_precision_at_20_std": 0.105896, "nauc_precision_at_20_diff1": -0.277453, "nauc_precision_at_100_max": 0.021214, "nauc_precision_at_100_std": 0.235838, "nauc_precision_at_100_diff1": -0.343335, "nauc_precision_at_1000_max": 0.021898, "nauc_precision_at_1000_std": 0.251024, "nauc_precision_at_1000_diff1": -0.363233, "nauc_mrr_at_1_max": 0.059381, "nauc_mrr_at_1_std": -0.31109, "nauc_mrr_at_1_diff1": 0.668424, "nauc_mrr_at_3_max": 0.116814, "nauc_mrr_at_3_std": -0.297883, "nauc_mrr_at_3_diff1": 0.648118, "nauc_mrr_at_5_max": 0.09633, "nauc_mrr_at_5_std": -0.312192, "nauc_mrr_at_5_diff1": 0.641368, "nauc_mrr_at_10_max": 0.093866, "nauc_mrr_at_10_std": -0.307326, "nauc_mrr_at_10_diff1": 0.647165, "nauc_mrr_at_20_max": 0.093374, "nauc_mrr_at_20_std": -0.305795, "nauc_mrr_at_20_diff1": 0.647444, "nauc_mrr_at_100_max": 0.093743, "nauc_mrr_at_100_std": -0.305033, "nauc_mrr_at_100_diff1": 0.647185, "nauc_mrr_at_1000_max": 0.093723, "nauc_mrr_at_1000_std": -0.305062, "nauc_mrr_at_1000_diff1": 0.647187, "hit_rate_at_1": 0.65934, "hit_rate_at_3": 0.87912, "hit_rate_at_5": 0.91209, "hit_rate_at_10": 0.95604, "hit_rate_at_20": 0.98077, "hit_rate_at_100": 0.99725, "hit_rate_at_1000": 1.0, "main_score": 0.68033, "hf_subset": "portuguese", "languages": [ "por-Latn" ], "mteb_version": "2.18.6" } ] }, "evaluation_time": 5816.3873386411, "kg_co2_emissions": null, "date": 1785031578.997696, "evaluation_phases": [ { "name": "Data loading", "start": 1.6266945749521255e-05, "end": 26.67837639292702, "split": "", "subset": "" }, { "name": "Dataset transform", "start": 26.678434194996953, "end": 26.6784397540614, "split": "", "subset": "" }, { "name": "Encoding queries and documents", "start": 26.68970058299601, "end": 1181.0699114836752, "split": "test", "subset": "french" }, { "name": "Scoring", "start": 1181.0701518999413, "end": 1181.630832691677, "split": "test", "subset": "french" }, { "name": "Encoding queries and documents", "start": 1181.6603546929546, "end": 2590.285706026014, "split": "test", "subset": "spanish" }, { "name": "Scoring", "start": 2590.2858266709372, "end": 2590.7031109887175, "split": "test", "subset": "spanish" }, { "name": "Encoding queries and documents", "start": 2590.7338419146836, "end": 3862.1960915429518, "split": "test", "subset": "english" }, { "name": "Scoring", "start": 3862.1961821289733, "end": 3862.50557737099, "split": "test", "subset": "english" }, { "name": "Encoding queries and documents", "start": 3862.537031712942, "end": 4615.360616014805, "split": "test", "subset": "german" }, { "name": "Scoring", "start": 4615.360703513026, "end": 4615.6664060927, "split": "test", "subset": "german" }, { "name": "Encoding queries and documents", "start": 4615.688725729939, "end": 5271.233584980946, "split": "test", "subset": "italian" }, { "name": "Scoring", "start": 5271.233664043713, "end": 5271.537373113912, "split": "test", "subset": "italian" }, { "name": "Encoding queries and documents", "start": 5271.564200275112, "end": 5816.054263813887, "split": "test", "subset": "portuguese" }, { "name": "Scoring", "start": 5816.054336919915, "end": 5816.37303925911, "split": "test", "subset": "portuguese" } ] }