{ "dataset_revision": "237ed4f43c7fb3c4df07ec4e9dd0a4366be555b0", "task_name": "Vidore3PharmaceuticalsRetrieval", "mteb_version": "2.24.0", "scores": { "test": [ { "ndcg_at_1": 0.64973, "ndcg_at_3": 0.63619, "ndcg_at_5": 0.64449, "ndcg_at_10": 0.66511, "ndcg_at_20": 0.69202, "ndcg_at_100": 0.73828, "ndcg_at_1000": 0.75683, "map_at_1": 0.34269, "map_at_3": 0.46332, "map_at_5": 0.50329, "map_at_10": 0.54885, "map_at_20": 0.57493, "map_at_100": 0.59426, "map_at_1000": 0.59696, "recall_at_1": 0.34269, "recall_at_3": 0.51216, "recall_at_5": 0.59565, "recall_at_10": 0.69829, "recall_at_20": 0.77982, "recall_at_100": 0.91829, "recall_at_1000": 0.9938, "accuracy": 0.34269, "precision_at_1": 0.67582, "precision_at_3": 0.42216, "precision_at_5": 0.32967, "precision_at_10": 0.22912, "precision_at_20": 0.1467, "precision_at_100": 0.04044, "precision_at_1000": 0.0047, "mrr_at_1": 0.675824, "mrr_at_3": 0.760989, "mrr_at_5": 0.775687, "mrr_at_10": 0.780813, "mrr_at_20": 0.782114, "mrr_at_100": 0.782398, "mrr_at_1000": 0.782415, "nauc_ndcg_at_1_max": 0.05688, "nauc_ndcg_at_1_std": -0.062158, "nauc_ndcg_at_1_diff1": 0.51072, "nauc_ndcg_at_3_max": 0.044741, "nauc_ndcg_at_3_std": -0.124645, "nauc_ndcg_at_3_diff1": 0.462411, "nauc_ndcg_at_5_max": 0.000744, "nauc_ndcg_at_5_std": -0.1628, "nauc_ndcg_at_5_diff1": 0.466852, "nauc_ndcg_at_10_max": -0.029014, "nauc_ndcg_at_10_std": -0.18193, "nauc_ndcg_at_10_diff1": 0.474767, "nauc_ndcg_at_20_max": -0.029509, "nauc_ndcg_at_20_std": -0.182371, "nauc_ndcg_at_20_diff1": 0.476093, "nauc_ndcg_at_100_max": -0.031995, "nauc_ndcg_at_100_std": -0.166992, "nauc_ndcg_at_100_diff1": 0.462968, "nauc_ndcg_at_1000_max": 6.9e-05, "nauc_ndcg_at_1000_std": -0.139303, "nauc_ndcg_at_1000_diff1": 0.464909, "nauc_map_at_1_max": -0.025813, "nauc_map_at_1_std": -0.115243, "nauc_map_at_1_diff1": 0.546421, "nauc_map_at_3_max": 0.016316, "nauc_map_at_3_std": -0.140731, "nauc_map_at_3_diff1": 0.516599, "nauc_map_at_5_max": -0.000902, "nauc_map_at_5_std": -0.154032, "nauc_map_at_5_diff1": 0.50194, "nauc_map_at_10_max": -0.012143, "nauc_map_at_10_std": -0.155508, "nauc_map_at_10_diff1": 0.481088, "nauc_map_at_20_max": -0.014542, "nauc_map_at_20_std": -0.166716, "nauc_map_at_20_diff1": 0.467488, "nauc_map_at_100_max": -0.017951, "nauc_map_at_100_std": -0.167962, "nauc_map_at_100_diff1": 0.45924, "nauc_map_at_1000_max": -0.014371, "nauc_map_at_1000_std": -0.164319, "nauc_map_at_1000_diff1": 0.458929, "nauc_recall_at_1_max": -0.025813, "nauc_recall_at_1_std": -0.115243, "nauc_recall_at_1_diff1": 0.546421, "nauc_recall_at_3_max": 0.008136, "nauc_recall_at_3_std": -0.16307, "nauc_recall_at_3_diff1": 0.479316, "nauc_recall_at_5_max": -0.029993, "nauc_recall_at_5_std": -0.190875, "nauc_recall_at_5_diff1": 0.444645, "nauc_recall_at_10_max": -0.074182, "nauc_recall_at_10_std": -0.216521, "nauc_recall_at_10_diff1": 0.410984, "nauc_recall_at_20_max": -0.12464, "nauc_recall_at_20_std": -0.2729, "nauc_recall_at_20_diff1": 0.386078, "nauc_recall_at_100_max": -0.281292, "nauc_recall_at_100_std": -0.322637, "nauc_recall_at_100_diff1": 0.371137, "nauc_recall_at_1000_max": 0.193326, "nauc_recall_at_1000_std": 0.437362, "nauc_recall_at_1000_diff1": 0.735256, "nauc_precision_at_1_max": 0.053559, "nauc_precision_at_1_std": -0.074696, "nauc_precision_at_1_diff1": 0.52341, "nauc_precision_at_3_max": 0.118361, "nauc_precision_at_3_std": -0.018128, "nauc_precision_at_3_diff1": 0.006033, "nauc_precision_at_5_max": 0.021958, "nauc_precision_at_5_std": -0.020207, "nauc_precision_at_5_diff1": -0.167476, "nauc_precision_at_10_max": -0.012533, "nauc_precision_at_10_std": -0.013335, "nauc_precision_at_10_diff1": -0.264591, "nauc_precision_at_20_max": 0.006877, "nauc_precision_at_20_std": 0.006248, "nauc_precision_at_20_diff1": -0.303373, "nauc_precision_at_100_max": 0.031058, "nauc_precision_at_100_std": 0.086276, "nauc_precision_at_100_diff1": -0.363458, "nauc_precision_at_1000_max": 0.082824, "nauc_precision_at_1000_std": 0.142806, "nauc_precision_at_1000_diff1": -0.379062, "nauc_mrr_at_1_max": 0.053559, "nauc_mrr_at_1_std": -0.074696, "nauc_mrr_at_1_diff1": 0.52341, "nauc_mrr_at_3_max": 0.07109, "nauc_mrr_at_3_std": -0.092555, "nauc_mrr_at_3_diff1": 0.503725, "nauc_mrr_at_5_max": 0.076171, "nauc_mrr_at_5_std": -0.086706, "nauc_mrr_at_5_diff1": 0.514482, "nauc_mrr_at_10_max": 0.071197, "nauc_mrr_at_10_std": -0.087799, "nauc_mrr_at_10_diff1": 0.514183, "nauc_mrr_at_20_max": 0.067221, "nauc_mrr_at_20_std": -0.089032, "nauc_mrr_at_20_diff1": 0.512993, "nauc_mrr_at_100_max": 0.068174, "nauc_mrr_at_100_std": -0.088071, "nauc_mrr_at_100_diff1": 0.513236, "nauc_mrr_at_1000_max": 0.06809, "nauc_mrr_at_1000_std": -0.08817, "nauc_mrr_at_1000_diff1": 0.513217, "hit_rate_at_1": 0.67582, "hit_rate_at_3": 0.85989, "hit_rate_at_5": 0.92308, "hit_rate_at_10": 0.96154, "hit_rate_at_20": 0.98077, "hit_rate_at_100": 0.99451, "hit_rate_at_1000": 0.99725, "main_score": 0.66511, "hf_subset": "french", "languages": [ "fra-Latn" ], "mteb_version": "2.24.0" }, { "ndcg_at_1": 0.64148, "ndcg_at_3": 0.64412, "ndcg_at_5": 0.64376, "ndcg_at_10": 0.66222, "ndcg_at_20": 0.68828, "ndcg_at_100": 0.73666, "ndcg_at_1000": 0.75498, "map_at_1": 0.33473, "map_at_3": 0.46471, "map_at_5": 0.50215, "map_at_10": 0.54558, "map_at_20": 0.57102, "map_at_100": 0.59201, "map_at_1000": 0.59468, "recall_at_1": 0.33473, "recall_at_3": 0.52253, "recall_at_5": 0.59443, "recall_at_10": 0.69638, "recall_at_20": 0.77685, "recall_at_100": 0.91848, "recall_at_1000": 0.99495, "accuracy": 0.33473, "precision_at_1": 0.66758, "precision_at_3": 0.43498, "precision_at_5": 0.33077, "precision_at_10": 0.22995, "precision_at_20": 0.14615, "precision_at_100": 0.04066, "precision_at_1000": 0.00471, "mrr_at_1": 0.667582, "mrr_at_3": 0.762363, "mrr_at_5": 0.770604, "mrr_at_10": 0.775956, "mrr_at_20": 0.777688, "mrr_at_100": 0.77841, "mrr_at_1000": 0.778413, "nauc_ndcg_at_1_max": 0.031616, "nauc_ndcg_at_1_std": -0.142292, "nauc_ndcg_at_1_diff1": 0.58025, "nauc_ndcg_at_3_max": 0.019501, "nauc_ndcg_at_3_std": -0.166021, "nauc_ndcg_at_3_diff1": 0.501682, "nauc_ndcg_at_5_max": 0.007177, "nauc_ndcg_at_5_std": -0.191831, "nauc_ndcg_at_5_diff1": 0.50692, "nauc_ndcg_at_10_max": -0.030581, "nauc_ndcg_at_10_std": -0.223129, "nauc_ndcg_at_10_diff1": 0.506898, "nauc_ndcg_at_20_max": -0.019074, "nauc_ndcg_at_20_std": -0.198575, "nauc_ndcg_at_20_diff1": 0.517034, "nauc_ndcg_at_100_max": -0.031137, "nauc_ndcg_at_100_std": -0.200852, "nauc_ndcg_at_100_diff1": 0.501983, "nauc_ndcg_at_1000_max": -0.0063, "nauc_ndcg_at_1000_std": -0.179095, "nauc_ndcg_at_1000_diff1": 0.508379, "nauc_map_at_1_max": -0.006296, "nauc_map_at_1_std": -0.1327, "nauc_map_at_1_diff1": 0.60354, "nauc_map_at_3_max": 0.020594, "nauc_map_at_3_std": -0.164821, "nauc_map_at_3_diff1": 0.551513, "nauc_map_at_5_max": 0.014919, "nauc_map_at_5_std": -0.177954, "nauc_map_at_5_diff1": 0.533254, "nauc_map_at_10_max": -0.01129, "nauc_map_at_10_std": -0.199162, "nauc_map_at_10_diff1": 0.509863, "nauc_map_at_20_max": -0.011355, "nauc_map_at_20_std": -0.200433, "nauc_map_at_20_diff1": 0.50437, "nauc_map_at_100_max": -0.0186, "nauc_map_at_100_std": -0.207099, "nauc_map_at_100_diff1": 0.493392, "nauc_map_at_1000_max": -0.016283, "nauc_map_at_1000_std": -0.203962, "nauc_map_at_1000_diff1": 0.49354, "nauc_recall_at_1_max": -0.006296, "nauc_recall_at_1_std": -0.1327, "nauc_recall_at_1_diff1": 0.60354, "nauc_recall_at_3_max": 0.023684, "nauc_recall_at_3_std": -0.167705, "nauc_recall_at_3_diff1": 0.494651, "nauc_recall_at_5_max": 0.001576, "nauc_recall_at_5_std": -0.195611, "nauc_recall_at_5_diff1": 0.457145, "nauc_recall_at_10_max": -0.079898, "nauc_recall_at_10_std": -0.263037, "nauc_recall_at_10_diff1": 0.407965, "nauc_recall_at_20_max": -0.089382, "nauc_recall_at_20_std": -0.238813, "nauc_recall_at_20_diff1": 0.416937, "nauc_recall_at_100_max": -0.251416, "nauc_recall_at_100_std": -0.296491, "nauc_recall_at_100_diff1": 0.335521, "nauc_recall_at_1000_max": -0.012445, "nauc_recall_at_1000_std": 0.176037, "nauc_recall_at_1000_diff1": 0.576634, "nauc_precision_at_1_max": 0.018182, "nauc_precision_at_1_std": -0.166999, "nauc_precision_at_1_diff1": 0.58792, "nauc_precision_at_3_max": 0.03776, "nauc_precision_at_3_std": -0.086805, "nauc_precision_at_3_diff1": -0.015404, "nauc_precision_at_5_max": 0.005468, "nauc_precision_at_5_std": -0.076173, "nauc_precision_at_5_diff1": -0.161381, "nauc_precision_at_10_max": -0.063031, "nauc_precision_at_10_std": -0.085114, "nauc_precision_at_10_diff1": -0.273334, "nauc_precision_at_20_max": -0.029151, "nauc_precision_at_20_std": -0.015232, "nauc_precision_at_20_diff1": -0.307248, "nauc_precision_at_100_max": -0.023015, "nauc_precision_at_100_std": 0.05364, "nauc_precision_at_100_diff1": -0.374488, "nauc_precision_at_1000_max": 0.021165, "nauc_precision_at_1000_std": 0.109755, "nauc_precision_at_1000_diff1": -0.383226, "nauc_mrr_at_1_max": 0.018182, "nauc_mrr_at_1_std": -0.166999, "nauc_mrr_at_1_diff1": 0.58792, "nauc_mrr_at_3_max": 0.048074, "nauc_mrr_at_3_std": -0.147097, "nauc_mrr_at_3_diff1": 0.555474, "nauc_mrr_at_5_max": 0.04532, "nauc_mrr_at_5_std": -0.148318, "nauc_mrr_at_5_diff1": 0.562662, "nauc_mrr_at_10_max": 0.038474, "nauc_mrr_at_10_std": -0.156273, "nauc_mrr_at_10_diff1": 0.561664, "nauc_mrr_at_20_max": 0.037871, "nauc_mrr_at_20_std": -0.153258, "nauc_mrr_at_20_diff1": 0.560704, "nauc_mrr_at_100_max": 0.037927, "nauc_mrr_at_100_std": -0.153714, "nauc_mrr_at_100_diff1": 0.56181, "nauc_mrr_at_1000_max": 0.037922, "nauc_mrr_at_1000_std": -0.153729, "nauc_mrr_at_1000_diff1": 0.56181, "hit_rate_at_1": 0.66758, "hit_rate_at_3": 0.87363, "hit_rate_at_5": 0.90934, "hit_rate_at_10": 0.9478, "hit_rate_at_20": 0.97253, "hit_rate_at_100": 0.99725, "hit_rate_at_1000": 1.0, "main_score": 0.66222, "hf_subset": "spanish", "languages": [ "spa-Latn" ], "mteb_version": "2.24.0" }, { "ndcg_at_1": 0.65522, "ndcg_at_3": 0.66106, "ndcg_at_5": 0.6632, "ndcg_at_10": 0.68039, "ndcg_at_20": 0.70723, "ndcg_at_100": 0.75453, "ndcg_at_1000": 0.77065, "map_at_1": 0.34482, "map_at_3": 0.48153, "map_at_5": 0.52011, "map_at_10": 0.56739, "map_at_20": 0.59497, "map_at_100": 0.61645, "map_at_1000": 0.61877, "recall_at_1": 0.34482, "recall_at_3": 0.53488, "recall_at_5": 0.61157, "recall_at_10": 0.71167, "recall_at_20": 0.79584, "recall_at_100": 0.93078, "recall_at_1000": 0.9977, "accuracy": 0.34482, "precision_at_1": 0.68132, "precision_at_3": 0.44963, "precision_at_5": 0.34451, "precision_at_10": 0.23819, "precision_at_20": 0.15165, "precision_at_100": 0.0419, "precision_at_1000": 0.00473, "mrr_at_1": 0.681319, "mrr_at_3": 0.770147, "mrr_at_5": 0.784158, "mrr_at_10": 0.787079, "mrr_at_20": 0.788858, "mrr_at_100": 0.789469, "mrr_at_1000": 0.789497, "nauc_ndcg_at_1_max": -0.013888, "nauc_ndcg_at_1_std": -0.187124, "nauc_ndcg_at_1_diff1": 0.610672, "nauc_ndcg_at_3_max": -0.004411, "nauc_ndcg_at_3_std": -0.196155, "nauc_ndcg_at_3_diff1": 0.518555, "nauc_ndcg_at_5_max": 0.016009, "nauc_ndcg_at_5_std": -0.196775, "nauc_ndcg_at_5_diff1": 0.526319, "nauc_ndcg_at_10_max": -0.000786, "nauc_ndcg_at_10_std": -0.210733, "nauc_ndcg_at_10_diff1": 0.54292, "nauc_ndcg_at_20_max": 0.002183, "nauc_ndcg_at_20_std": -0.203394, "nauc_ndcg_at_20_diff1": 0.555764, "nauc_ndcg_at_100_max": -0.026489, "nauc_ndcg_at_100_std": -0.21119, "nauc_ndcg_at_100_diff1": 0.551843, "nauc_ndcg_at_1000_max": -0.00145, "nauc_ndcg_at_1000_std": -0.193141, "nauc_ndcg_at_1000_diff1": 0.551417, "nauc_map_at_1_max": 0.035182, "nauc_map_at_1_std": -0.130279, "nauc_map_at_1_diff1": 0.632721, "nauc_map_at_3_max": 0.061733, "nauc_map_at_3_std": -0.164351, "nauc_map_at_3_diff1": 0.561012, "nauc_map_at_5_max": 0.058876, "nauc_map_at_5_std": -0.167846, "nauc_map_at_5_diff1": 0.549358, "nauc_map_at_10_max": 0.028883, "nauc_map_at_10_std": -0.178744, "nauc_map_at_10_diff1": 0.532426, "nauc_map_at_20_max": 0.019049, "nauc_map_at_20_std": -0.191415, "nauc_map_at_20_diff1": 0.530124, "nauc_map_at_100_max": 0.007263, "nauc_map_at_100_std": -0.198662, "nauc_map_at_100_diff1": 0.525614, "nauc_map_at_1000_max": 0.010749, "nauc_map_at_1000_std": -0.19534, "nauc_map_at_1000_diff1": 0.525859, "nauc_recall_at_1_max": 0.035182, "nauc_recall_at_1_std": -0.130279, "nauc_recall_at_1_diff1": 0.632721, "nauc_recall_at_3_max": 0.071257, "nauc_recall_at_3_std": -0.160331, "nauc_recall_at_3_diff1": 0.510582, "nauc_recall_at_5_max": 0.074035, "nauc_recall_at_5_std": -0.167287, "nauc_recall_at_5_diff1": 0.483318, "nauc_recall_at_10_max": 0.006808, "nauc_recall_at_10_std": -0.209154, "nauc_recall_at_10_diff1": 0.482263, "nauc_recall_at_20_max": -0.002938, "nauc_recall_at_20_std": -0.20981, "nauc_recall_at_20_diff1": 0.493571, "nauc_recall_at_100_max": -0.248595, "nauc_recall_at_100_std": -0.310074, "nauc_recall_at_100_diff1": 0.495557, "nauc_recall_at_1000_max": -0.411263, "nauc_recall_at_1000_std": -0.409877, "nauc_recall_at_1000_diff1": 0.688494, "nauc_precision_at_1_max": -0.029484, "nauc_precision_at_1_std": -0.211142, "nauc_precision_at_1_diff1": 0.626151, "nauc_precision_at_3_max": -0.036762, "nauc_precision_at_3_std": -0.109095, "nauc_precision_at_3_diff1": -0.042172, "nauc_precision_at_5_max": -0.049325, "nauc_precision_at_5_std": -0.058737, "nauc_precision_at_5_diff1": -0.190514, "nauc_precision_at_10_max": -0.102612, "nauc_precision_at_10_std": -0.052204, "nauc_precision_at_10_diff1": -0.263502, "nauc_precision_at_20_max": -0.092927, "nauc_precision_at_20_std": -0.017552, "nauc_precision_at_20_diff1": -0.283019, "nauc_precision_at_100_max": -0.086507, "nauc_precision_at_100_std": 0.050663, "nauc_precision_at_100_diff1": -0.339381, "nauc_precision_at_1000_max": -0.029547, "nauc_precision_at_1000_std": 0.1037, "nauc_precision_at_1000_diff1": -0.367713, "nauc_mrr_at_1_max": -0.029484, "nauc_mrr_at_1_std": -0.211142, "nauc_mrr_at_1_diff1": 0.626151, "nauc_mrr_at_3_max": -0.004176, "nauc_mrr_at_3_std": -0.217675, "nauc_mrr_at_3_diff1": 0.617448, "nauc_mrr_at_5_max": 0.001181, "nauc_mrr_at_5_std": -0.218448, "nauc_mrr_at_5_diff1": 0.620555, "nauc_mrr_at_10_max": -0.005977, "nauc_mrr_at_10_std": -0.219344, "nauc_mrr_at_10_diff1": 0.619449, "nauc_mrr_at_20_max": -0.002102, "nauc_mrr_at_20_std": -0.213305, "nauc_mrr_at_20_diff1": 0.619745, "nauc_mrr_at_100_max": -0.00338, "nauc_mrr_at_100_std": -0.213722, "nauc_mrr_at_100_diff1": 0.619646, "nauc_mrr_at_1000_max": -0.003519, "nauc_mrr_at_1000_std": -0.213897, "nauc_mrr_at_1000_diff1": 0.619655, "hit_rate_at_1": 0.68132, "hit_rate_at_3": 0.87088, "hit_rate_at_5": 0.93132, "hit_rate_at_10": 0.95055, "hit_rate_at_20": 0.97527, "hit_rate_at_100": 0.99451, "hit_rate_at_1000": 1.0, "main_score": 0.68039, "hf_subset": "english", "languages": [ "eng-Latn" ], "mteb_version": "2.24.0" }, { "ndcg_at_1": 0.62363, "ndcg_at_3": 0.63333, "ndcg_at_5": 0.63254, "ndcg_at_10": 0.6514, "ndcg_at_20": 0.68185, "ndcg_at_100": 0.72788, "ndcg_at_1000": 0.74797, "map_at_1": 0.33448, "map_at_3": 0.46265, "map_at_5": 0.49541, "map_at_10": 0.53886, "map_at_20": 0.56638, "map_at_100": 0.58565, "map_at_1000": 0.58846, "recall_at_1": 0.33448, "recall_at_3": 0.51589, "recall_at_5": 0.58102, "recall_at_10": 0.68074, "recall_at_20": 0.77554, "recall_at_100": 0.90992, "recall_at_1000": 0.99468, "accuracy": 0.33448, "precision_at_1": 0.6511, "precision_at_3": 0.42582, "precision_at_5": 0.32253, "precision_at_10": 0.22253, "precision_at_20": 0.14382, "precision_at_100": 0.03992, "precision_at_1000": 0.00469, "mrr_at_1": 0.651099, "mrr_at_3": 0.749542, "mrr_at_5": 0.760119, "mrr_at_10": 0.764787, "mrr_at_20": 0.766902, "mrr_at_100": 0.767395, "mrr_at_1000": 0.767443, "nauc_ndcg_at_1_max": -0.034708, "nauc_ndcg_at_1_std": -0.132504, "nauc_ndcg_at_1_diff1": 0.522071, "nauc_ndcg_at_3_max": -0.052999, "nauc_ndcg_at_3_std": -0.160415, "nauc_ndcg_at_3_diff1": 0.468308, "nauc_ndcg_at_5_max": -0.070912, "nauc_ndcg_at_5_std": -0.18405, "nauc_ndcg_at_5_diff1": 0.459984, "nauc_ndcg_at_10_max": -0.082321, "nauc_ndcg_at_10_std": -0.194204, "nauc_ndcg_at_10_diff1": 0.471623, "nauc_ndcg_at_20_max": -0.082418, "nauc_ndcg_at_20_std": -0.193722, "nauc_ndcg_at_20_diff1": 0.482782, "nauc_ndcg_at_100_max": -0.100282, "nauc_ndcg_at_100_std": -0.18952, "nauc_ndcg_at_100_diff1": 0.471408, "nauc_ndcg_at_1000_max": -0.074922, "nauc_ndcg_at_1000_std": -0.166562, "nauc_ndcg_at_1000_diff1": 0.472939, "nauc_map_at_1_max": -0.045891, "nauc_map_at_1_std": -0.136786, "nauc_map_at_1_diff1": 0.563953, "nauc_map_at_3_max": -0.033628, "nauc_map_at_3_std": -0.16815, "nauc_map_at_3_diff1": 0.513321, "nauc_map_at_5_max": -0.044088, "nauc_map_at_5_std": -0.178317, "nauc_map_at_5_diff1": 0.493464, "nauc_map_at_10_max": -0.059834, "nauc_map_at_10_std": -0.187299, "nauc_map_at_10_diff1": 0.474003, "nauc_map_at_20_max": -0.064787, "nauc_map_at_20_std": -0.194863, "nauc_map_at_20_diff1": 0.466912, "nauc_map_at_100_max": -0.074327, "nauc_map_at_100_std": -0.200649, "nauc_map_at_100_diff1": 0.460081, "nauc_map_at_1000_max": -0.071508, "nauc_map_at_1000_std": -0.196838, "nauc_map_at_1000_diff1": 0.459574, "nauc_recall_at_1_max": -0.045891, "nauc_recall_at_1_std": -0.136786, "nauc_recall_at_1_diff1": 0.563953, "nauc_recall_at_3_max": -0.03241, "nauc_recall_at_3_std": -0.161147, "nauc_recall_at_3_diff1": 0.478411, "nauc_recall_at_5_max": -0.057584, "nauc_recall_at_5_std": -0.184412, "nauc_recall_at_5_diff1": 0.437065, "nauc_recall_at_10_max": -0.074098, "nauc_recall_at_10_std": -0.190185, "nauc_recall_at_10_diff1": 0.403712, "nauc_recall_at_20_max": -0.109977, "nauc_recall_at_20_std": -0.231965, "nauc_recall_at_20_diff1": 0.417169, "nauc_recall_at_100_max": -0.294933, "nauc_recall_at_100_std": -0.277317, "nauc_recall_at_100_diff1": 0.361449, "nauc_recall_at_1000_max": -0.018596, "nauc_recall_at_1000_std": 0.107322, "nauc_recall_at_1000_diff1": 0.500055, "nauc_precision_at_1_max": -0.041334, "nauc_precision_at_1_std": -0.153995, "nauc_precision_at_1_diff1": 0.522299, "nauc_precision_at_3_max": 0.004225, "nauc_precision_at_3_std": -0.053129, "nauc_precision_at_3_diff1": 0.002835, "nauc_precision_at_5_max": -0.032298, "nauc_precision_at_5_std": -0.047277, "nauc_precision_at_5_diff1": -0.147413, "nauc_precision_at_10_max": -0.058908, "nauc_precision_at_10_std": -0.043454, "nauc_precision_at_10_diff1": -0.236125, "nauc_precision_at_20_max": -0.045311, "nauc_precision_at_20_std": -0.013603, "nauc_precision_at_20_diff1": -0.266621, "nauc_precision_at_100_max": -0.02599, "nauc_precision_at_100_std": 0.066708, "nauc_precision_at_100_diff1": -0.331186, "nauc_precision_at_1000_max": 0.031469, "nauc_precision_at_1000_std": 0.12394, "nauc_precision_at_1000_diff1": -0.357934, "nauc_mrr_at_1_max": -0.041334, "nauc_mrr_at_1_std": -0.153995, "nauc_mrr_at_1_diff1": 0.522299, "nauc_mrr_at_3_max": -0.007444, "nauc_mrr_at_3_std": -0.119131, "nauc_mrr_at_3_diff1": 0.523182, "nauc_mrr_at_5_max": -0.021796, "nauc_mrr_at_5_std": -0.126336, "nauc_mrr_at_5_diff1": 0.516466, "nauc_mrr_at_10_max": -0.021176, "nauc_mrr_at_10_std": -0.123864, "nauc_mrr_at_10_diff1": 0.514338, "nauc_mrr_at_20_max": -0.021455, "nauc_mrr_at_20_std": -0.123636, "nauc_mrr_at_20_diff1": 0.51633, "nauc_mrr_at_100_max": -0.021612, "nauc_mrr_at_100_std": -0.124032, "nauc_mrr_at_100_diff1": 0.516469, "nauc_mrr_at_1000_max": -0.021781, "nauc_mrr_at_1000_std": -0.124275, "nauc_mrr_at_1000_diff1": 0.516438, "hit_rate_at_1": 0.6511, "hit_rate_at_3": 0.85989, "hit_rate_at_5": 0.90659, "hit_rate_at_10": 0.93956, "hit_rate_at_20": 0.96978, "hit_rate_at_100": 0.99176, "hit_rate_at_1000": 1.0, "main_score": 0.6514, "hf_subset": "german", "languages": [ "deu-Latn" ], "mteb_version": "2.24.0" }, { "ndcg_at_1": 0.6456, "ndcg_at_3": 0.64213, "ndcg_at_5": 0.64539, "ndcg_at_10": 0.6652, "ndcg_at_20": 0.69135, "ndcg_at_100": 0.73711, "ndcg_at_1000": 0.75614, "map_at_1": 0.33807, "map_at_3": 0.46881, "map_at_5": 0.50525, "map_at_10": 0.55008, "map_at_20": 0.57636, "map_at_100": 0.59586, "map_at_1000": 0.5985, "recall_at_1": 0.33807, "recall_at_3": 0.52355, "recall_at_5": 0.59642, "recall_at_10": 0.69978, "recall_at_20": 0.78078, "recall_at_100": 0.91476, "recall_at_1000": 0.99565, "accuracy": 0.33807, "precision_at_1": 0.67308, "precision_at_3": 0.43315, "precision_at_5": 0.33297, "precision_at_10": 0.23132, "precision_at_20": 0.14725, "precision_at_100": 0.04052, "precision_at_1000": 0.00471, "mrr_at_1": 0.673077, "mrr_at_3": 0.764652, "mrr_at_5": 0.775641, "mrr_at_10": 0.78074, "mrr_at_20": 0.781825, "mrr_at_100": 0.78237, "mrr_at_1000": 0.782375, "nauc_ndcg_at_1_max": -0.032664, "nauc_ndcg_at_1_std": -0.146356, "nauc_ndcg_at_1_diff1": 0.554779, "nauc_ndcg_at_3_max": -0.047434, "nauc_ndcg_at_3_std": -0.15375, "nauc_ndcg_at_3_diff1": 0.49307, "nauc_ndcg_at_5_max": -0.0798, "nauc_ndcg_at_5_std": -0.196332, "nauc_ndcg_at_5_diff1": 0.507745, "nauc_ndcg_at_10_max": -0.098573, "nauc_ndcg_at_10_std": -0.192206, "nauc_ndcg_at_10_diff1": 0.516968, "nauc_ndcg_at_20_max": -0.097878, "nauc_ndcg_at_20_std": -0.192398, "nauc_ndcg_at_20_diff1": 0.524781, "nauc_ndcg_at_100_max": -0.106149, "nauc_ndcg_at_100_std": -0.181442, "nauc_ndcg_at_100_diff1": 0.507298, "nauc_ndcg_at_1000_max": -0.079069, "nauc_ndcg_at_1000_std": -0.161953, "nauc_ndcg_at_1000_diff1": 0.510272, "nauc_map_at_1_max": -0.035215, "nauc_map_at_1_std": -0.100745, "nauc_map_at_1_diff1": 0.59083, "nauc_map_at_3_max": -0.045507, "nauc_map_at_3_std": -0.153227, "nauc_map_at_3_diff1": 0.530239, "nauc_map_at_5_max": -0.05569, "nauc_map_at_5_std": -0.167433, "nauc_map_at_5_diff1": 0.527479, "nauc_map_at_10_max": -0.076764, "nauc_map_at_10_std": -0.17307, "nauc_map_at_10_diff1": 0.515978, "nauc_map_at_20_max": -0.083849, "nauc_map_at_20_std": -0.184098, "nauc_map_at_20_diff1": 0.506874, "nauc_map_at_100_max": -0.090451, "nauc_map_at_100_std": -0.186352, "nauc_map_at_100_diff1": 0.496455, "nauc_map_at_1000_max": -0.087481, "nauc_map_at_1000_std": -0.183119, "nauc_map_at_1000_diff1": 0.496488, "nauc_recall_at_1_max": -0.035215, "nauc_recall_at_1_std": -0.100745, "nauc_recall_at_1_diff1": 0.59083, "nauc_recall_at_3_max": -0.065924, "nauc_recall_at_3_std": -0.173054, "nauc_recall_at_3_diff1": 0.4787, "nauc_recall_at_5_max": -0.094489, "nauc_recall_at_5_std": -0.207841, "nauc_recall_at_5_diff1": 0.475202, "nauc_recall_at_10_max": -0.137386, "nauc_recall_at_10_std": -0.208912, "nauc_recall_at_10_diff1": 0.458144, "nauc_recall_at_20_max": -0.169223, "nauc_recall_at_20_std": -0.257036, "nauc_recall_at_20_diff1": 0.471386, "nauc_recall_at_100_max": -0.320848, "nauc_recall_at_100_std": -0.266633, "nauc_recall_at_100_diff1": 0.389285, "nauc_recall_at_1000_max": -0.385278, "nauc_recall_at_1000_std": -0.12675, "nauc_recall_at_1000_diff1": 0.610622, "nauc_precision_at_1_max": -0.053423, "nauc_precision_at_1_std": -0.181957, "nauc_precision_at_1_diff1": 0.554633, "nauc_precision_at_3_max": 0.02056, "nauc_precision_at_3_std": -0.061027, "nauc_precision_at_3_diff1": -0.017843, "nauc_precision_at_5_max": -0.034535, "nauc_precision_at_5_std": -0.080262, "nauc_precision_at_5_diff1": -0.15406, "nauc_precision_at_10_max": -0.047443, "nauc_precision_at_10_std": -0.03938, "nauc_precision_at_10_diff1": -0.248346, "nauc_precision_at_20_max": -0.021824, "nauc_precision_at_20_std": -0.018285, "nauc_precision_at_20_diff1": -0.281056, "nauc_precision_at_100_max": 0.009545, "nauc_precision_at_100_std": 0.060948, "nauc_precision_at_100_diff1": -0.358838, "nauc_precision_at_1000_max": 0.064521, "nauc_precision_at_1000_std": 0.109291, "nauc_precision_at_1000_diff1": -0.380447, "nauc_mrr_at_1_max": -0.053423, "nauc_mrr_at_1_std": -0.181957, "nauc_mrr_at_1_diff1": 0.554633, "nauc_mrr_at_3_max": -0.022862, "nauc_mrr_at_3_std": -0.160301, "nauc_mrr_at_3_diff1": 0.540522, "nauc_mrr_at_5_max": -0.047936, "nauc_mrr_at_5_std": -0.17831, "nauc_mrr_at_5_diff1": 0.542711, "nauc_mrr_at_10_max": -0.037301, "nauc_mrr_at_10_std": -0.166524, "nauc_mrr_at_10_diff1": 0.543335, "nauc_mrr_at_20_max": -0.042116, "nauc_mrr_at_20_std": -0.170729, "nauc_mrr_at_20_diff1": 0.543525, "nauc_mrr_at_100_max": -0.041607, "nauc_mrr_at_100_std": -0.169878, "nauc_mrr_at_100_diff1": 0.543632, "nauc_mrr_at_1000_max": -0.041619, "nauc_mrr_at_1000_std": -0.169912, "nauc_mrr_at_1000_diff1": 0.543626, "hit_rate_at_1": 0.67308, "hit_rate_at_3": 0.87637, "hit_rate_at_5": 0.92582, "hit_rate_at_10": 0.96154, "hit_rate_at_20": 0.97527, "hit_rate_at_100": 0.99725, "hit_rate_at_1000": 1.0, "main_score": 0.6652, "hf_subset": "italian", "languages": [ "ita-Latn" ], "mteb_version": "2.24.0" }, { "ndcg_at_1": 0.64835, "ndcg_at_3": 0.64758, "ndcg_at_5": 0.64522, "ndcg_at_10": 0.66566, "ndcg_at_20": 0.69031, "ndcg_at_100": 0.73865, "ndcg_at_1000": 0.75643, "map_at_1": 0.33815, "map_at_3": 0.46846, "map_at_5": 0.5038, "map_at_10": 0.54963, "map_at_20": 0.57518, "map_at_100": 0.59571, "map_at_1000": 0.59826, "recall_at_1": 0.33815, "recall_at_3": 0.52513, "recall_at_5": 0.59083, "recall_at_10": 0.7005, "recall_at_20": 0.77761, "recall_at_100": 0.9195, "recall_at_1000": 0.99299, "accuracy": 0.33815, "precision_at_1": 0.67857, "precision_at_3": 0.43681, "precision_at_5": 0.33352, "precision_at_10": 0.23159, "precision_at_20": 0.14684, "precision_at_100": 0.0408, "precision_at_1000": 0.00471, "mrr_at_1": 0.678571, "mrr_at_3": 0.766941, "mrr_at_5": 0.777106, "mrr_at_10": 0.783229, "mrr_at_20": 0.784542, "mrr_at_100": 0.785105, "mrr_at_1000": 0.785105, "nauc_ndcg_at_1_max": -0.036133, "nauc_ndcg_at_1_std": -0.117571, "nauc_ndcg_at_1_diff1": 0.572075, "nauc_ndcg_at_3_max": -0.021704, "nauc_ndcg_at_3_std": -0.171552, "nauc_ndcg_at_3_diff1": 0.507818, "nauc_ndcg_at_5_max": -0.043998, "nauc_ndcg_at_5_std": -0.19925, "nauc_ndcg_at_5_diff1": 0.494196, "nauc_ndcg_at_10_max": -0.072522, "nauc_ndcg_at_10_std": -0.211847, "nauc_ndcg_at_10_diff1": 0.499973, "nauc_ndcg_at_20_max": -0.063395, "nauc_ndcg_at_20_std": -0.189174, "nauc_ndcg_at_20_diff1": 0.495807, "nauc_ndcg_at_100_max": -0.072868, "nauc_ndcg_at_100_std": -0.192616, "nauc_ndcg_at_100_diff1": 0.488679, "nauc_ndcg_at_1000_max": -0.047124, "nauc_ndcg_at_1000_std": -0.16655, "nauc_ndcg_at_1000_diff1": 0.498168, "nauc_map_at_1_max": -0.037869, "nauc_map_at_1_std": -0.112959, "nauc_map_at_1_diff1": 0.585826, "nauc_map_at_3_max": -0.001016, "nauc_map_at_3_std": -0.16442, "nauc_map_at_3_diff1": 0.55949, "nauc_map_at_5_max": -0.01897, "nauc_map_at_5_std": -0.18083, "nauc_map_at_5_diff1": 0.532361, "nauc_map_at_10_max": -0.04594, "nauc_map_at_10_std": -0.190555, "nauc_map_at_10_diff1": 0.51082, "nauc_map_at_20_max": -0.050645, "nauc_map_at_20_std": -0.191, "nauc_map_at_20_diff1": 0.496057, "nauc_map_at_100_max": -0.054292, "nauc_map_at_100_std": -0.195747, "nauc_map_at_100_diff1": 0.48933, "nauc_map_at_1000_max": -0.051375, "nauc_map_at_1000_std": -0.192101, "nauc_map_at_1000_diff1": 0.48965, "nauc_recall_at_1_max": -0.037869, "nauc_recall_at_1_std": -0.112959, "nauc_recall_at_1_diff1": 0.585826, "nauc_recall_at_3_max": 0.026502, "nauc_recall_at_3_std": -0.165901, "nauc_recall_at_3_diff1": 0.512138, "nauc_recall_at_5_max": -0.012701, "nauc_recall_at_5_std": -0.194179, "nauc_recall_at_5_diff1": 0.461089, "nauc_recall_at_10_max": -0.087013, "nauc_recall_at_10_std": -0.234643, "nauc_recall_at_10_diff1": 0.417832, "nauc_recall_at_20_max": -0.102724, "nauc_recall_at_20_std": -0.216811, "nauc_recall_at_20_diff1": 0.381044, "nauc_recall_at_100_max": -0.274334, "nauc_recall_at_100_std": -0.316542, "nauc_recall_at_100_diff1": 0.30322, "nauc_recall_at_1000_max": 0.051282, "nauc_recall_at_1000_std": 0.440589, "nauc_recall_at_1000_diff1": 0.459351, "nauc_precision_at_1_max": -0.044598, "nauc_precision_at_1_std": -0.147174, "nauc_precision_at_1_diff1": 0.584551, "nauc_precision_at_3_max": 0.027038, "nauc_precision_at_3_std": -0.073898, "nauc_precision_at_3_diff1": 0.005619, "nauc_precision_at_5_max": -0.020702, "nauc_precision_at_5_std": -0.072228, "nauc_precision_at_5_diff1": -0.174607, "nauc_precision_at_10_max": -0.061034, "nauc_precision_at_10_std": -0.045192, "nauc_precision_at_10_diff1": -0.267815, "nauc_precision_at_20_max": -0.024861, "nauc_precision_at_20_std": 0.021589, "nauc_precision_at_20_diff1": -0.308854, "nauc_precision_at_100_max": -0.002679, "nauc_precision_at_100_std": 0.080477, "nauc_precision_at_100_diff1": -0.355997, "nauc_precision_at_1000_max": 0.047665, "nauc_precision_at_1000_std": 0.136821, "nauc_precision_at_1000_diff1": -0.359145, "nauc_mrr_at_1_max": -0.044598, "nauc_mrr_at_1_std": -0.147174, "nauc_mrr_at_1_diff1": 0.584551, "nauc_mrr_at_3_max": 0.011546, "nauc_mrr_at_3_std": -0.151429, "nauc_mrr_at_3_diff1": 0.567306, "nauc_mrr_at_5_max": 0.004338, "nauc_mrr_at_5_std": -0.155036, "nauc_mrr_at_5_diff1": 0.568761, "nauc_mrr_at_10_max": -0.00523, "nauc_mrr_at_10_std": -0.158805, "nauc_mrr_at_10_diff1": 0.568942, "nauc_mrr_at_20_max": -0.00564, "nauc_mrr_at_20_std": -0.158049, "nauc_mrr_at_20_diff1": 0.569436, "nauc_mrr_at_100_max": -0.004835, "nauc_mrr_at_100_std": -0.157027, "nauc_mrr_at_100_diff1": 0.57021, "nauc_mrr_at_1000_max": -0.004835, "nauc_mrr_at_1000_std": -0.157027, "nauc_mrr_at_1000_diff1": 0.57021, "hit_rate_at_1": 0.67857, "hit_rate_at_3": 0.87088, "hit_rate_at_5": 0.91484, "hit_rate_at_10": 0.95879, "hit_rate_at_20": 0.97802, "hit_rate_at_100": 0.99725, "hit_rate_at_1000": 0.99725, "main_score": 0.66566, "hf_subset": "portuguese", "languages": [ "por-Latn" ], "mteb_version": "2.24.0" } ] }, "evaluation_time": 925.3140473030508, "kg_co2_emissions": null, "date": 1791446684.929269, "evaluation_phases": [ { "name": "Data loading", "start": 2.8633512556552887e-05, "end": 53.79506474919617, "split": "", "subset": "" }, { "name": "Dataset transform", "start": 53.79512018337846, "end": 53.80062533356249, "split": "", "subset": "" }, { "name": "Encoding corpus", "start": 53.810342838056386, "end": 53.81035186257213, "split": "test", "subset": "french" }, { "name": "Encoding queries", "start": 53.81042049638927, "end": 199.22063869796693, "split": "test", "subset": "french" }, { "name": "Scoring", "start": 199.2212721966207, "end": 199.50814275816083, "split": "test", "subset": "french" }, { "name": "Encoding corpus", "start": 199.52392416726798, "end": 199.52392903529108, "split": "test", "subset": "spanish" }, { "name": "Encoding queries", "start": 199.52395848929882, "end": 343.810662670061, "split": "test", "subset": "spanish" }, { "name": "Scoring", "start": 343.81126094423234, "end": 344.0979017652571, "split": "test", "subset": "spanish" }, { "name": "Encoding corpus", "start": 344.1134840827435, "end": 344.1134956004098, "split": "test", "subset": "english" }, { "name": "Encoding queries", "start": 344.11352908983827, "end": 488.22289595846087, "split": "test", "subset": "english" }, { "name": "Scoring", "start": 488.2234354130924, "end": 488.5017244545743, "split": "test", "subset": "english" }, { "name": "Encoding corpus", "start": 488.51690018642694, "end": 488.51690454315394, "split": "test", "subset": "german" }, { "name": "Encoding queries", "start": 488.516932124272, "end": 633.9084328478202, "split": "test", "subset": "german" }, { "name": "Scoring", "start": 633.908926460892, "end": 634.2009703470394, "split": "test", "subset": "german" }, { "name": "Encoding corpus", "start": 634.2167543666437, "end": 634.2167584719136, "split": "test", "subset": "italian" }, { "name": "Encoding queries", "start": 634.2167858146131, "end": 779.3743772907183, "split": "test", "subset": "italian" }, { "name": "Scoring", "start": 779.3751165131107, "end": 779.706349151209, "split": "test", "subset": "italian" }, { "name": "Encoding corpus", "start": 779.7239382499829, "end": 779.7239429084584, "split": "test", "subset": "portuguese" }, { "name": "Encoding queries", "start": 779.7239725226536, "end": 924.9915958289057, "split": "test", "subset": "portuguese" }, { "name": "Scoring", "start": 924.9920196980238, "end": 925.3011500379071, "split": "test", "subset": "portuguese" } ] }