{ "dataset_revision": "948c9fb4d61518c042a59aea5ac3e3d528cea19b", "task_name": "Vidore3PhysicsRetrieval.v2", "mteb_version": "2.13.1", "scores": { "test": [ { "ndcg_at_1": 0.43212, "ndcg_at_3": 0.43995, "ndcg_at_5": 0.44792, "ndcg_at_10": 0.47184, "ndcg_at_20": 0.51949, "ndcg_at_100": 0.60429, "ndcg_at_1000": 0.63053, "map_at_1": 0.16725, "map_at_3": 0.2511, "map_at_5": 0.28417, "map_at_10": 0.33222, "map_at_20": 0.3735, "map_at_100": 0.41214, "map_at_1000": 0.41706, "recall_at_1": 0.16725, "recall_at_3": 0.30432, "recall_at_5": 0.37515, "recall_at_10": 0.49175, "recall_at_20": 0.6333, "recall_at_100": 0.88041, "recall_at_1000": 0.99438, "accuracy": 0.16725, "precision_at_1": 0.46689, "precision_at_3": 0.3532, "precision_at_5": 0.30397, "precision_at_10": 0.23974, "precision_at_20": 0.17848, "precision_at_100": 0.05987, "precision_at_1000": 0.00716, "mrr_at_1": 0.466887, "mrr_at_3": 0.582781, "mrr_at_5": 0.604801, "mrr_at_10": 0.613387, "mrr_at_20": 0.617582, "mrr_at_100": 0.619271, "mrr_at_1000": 0.619338, "nauc_ndcg_at_1_max": 0.033028, "nauc_ndcg_at_1_std": -0.211212, "nauc_ndcg_at_1_diff1": 0.233104, "nauc_ndcg_at_3_max": 0.060047, "nauc_ndcg_at_3_std": -0.273398, "nauc_ndcg_at_3_diff1": 0.229066, "nauc_ndcg_at_5_max": 0.09896, "nauc_ndcg_at_5_std": -0.289264, "nauc_ndcg_at_5_diff1": 0.223844, "nauc_ndcg_at_10_max": 0.103275, "nauc_ndcg_at_10_std": -0.325948, "nauc_ndcg_at_10_diff1": 0.232419, "nauc_ndcg_at_20_max": 0.103074, "nauc_ndcg_at_20_std": -0.319204, "nauc_ndcg_at_20_diff1": 0.228288, "nauc_ndcg_at_100_max": 0.136092, "nauc_ndcg_at_100_std": -0.232747, "nauc_ndcg_at_100_diff1": 0.245944, "nauc_ndcg_at_1000_max": 0.118739, "nauc_ndcg_at_1000_std": -0.234044, "nauc_ndcg_at_1000_diff1": 0.248855, "nauc_map_at_1_max": 0.014358, "nauc_map_at_1_std": -0.386766, "nauc_map_at_1_diff1": 0.261496, "nauc_map_at_3_max": -0.002944, "nauc_map_at_3_std": -0.412696, "nauc_map_at_3_diff1": 0.239034, "nauc_map_at_5_max": 0.008253, "nauc_map_at_5_std": -0.389349, "nauc_map_at_5_diff1": 0.225673, "nauc_map_at_10_max": 0.038831, "nauc_map_at_10_std": -0.375991, "nauc_map_at_10_diff1": 0.223092, "nauc_map_at_20_max": 0.066688, "nauc_map_at_20_std": -0.345196, "nauc_map_at_20_diff1": 0.22446, "nauc_map_at_100_max": 0.086155, "nauc_map_at_100_std": -0.302656, "nauc_map_at_100_diff1": 0.231229, "nauc_map_at_1000_max": 0.087656, "nauc_map_at_1000_std": -0.298416, "nauc_map_at_1000_diff1": 0.233091, "nauc_recall_at_1_max": 0.014358, "nauc_recall_at_1_std": -0.386766, "nauc_recall_at_1_diff1": 0.261496, "nauc_recall_at_3_max": 0.00757, "nauc_recall_at_3_std": -0.401954, "nauc_recall_at_3_diff1": 0.210631, "nauc_recall_at_5_max": 0.040848, "nauc_recall_at_5_std": -0.357184, "nauc_recall_at_5_diff1": 0.167968, "nauc_recall_at_10_max": 0.071322, "nauc_recall_at_10_std": -0.347499, "nauc_recall_at_10_diff1": 0.156019, "nauc_recall_at_20_max": 0.104202, "nauc_recall_at_20_std": -0.311514, "nauc_recall_at_20_diff1": 0.146787, "nauc_recall_at_100_max": 0.271586, "nauc_recall_at_100_std": -0.029892, "nauc_recall_at_100_diff1": 0.161752, "nauc_recall_at_1000_max": 0.634912, "nauc_recall_at_1000_std": 0.468094, "nauc_recall_at_1000_diff1": 0.274593, "nauc_precision_at_1_max": 0.03979, "nauc_precision_at_1_std": -0.199358, "nauc_precision_at_1_diff1": 0.23001, "nauc_precision_at_3_max": 0.059225, "nauc_precision_at_3_std": -0.04521, "nauc_precision_at_3_diff1": 0.10326, "nauc_precision_at_5_max": 0.122187, "nauc_precision_at_5_std": 0.043491, "nauc_precision_at_5_diff1": 0.075339, "nauc_precision_at_10_max": 0.157753, "nauc_precision_at_10_std": 0.117382, "nauc_precision_at_10_diff1": 0.059688, "nauc_precision_at_20_max": 0.169282, "nauc_precision_at_20_std": 0.218258, "nauc_precision_at_20_diff1": 0.041416, "nauc_precision_at_100_max": 0.154157, "nauc_precision_at_100_std": 0.379737, "nauc_precision_at_100_diff1": 0.008506, "nauc_precision_at_1000_max": 0.110389, "nauc_precision_at_1000_std": 0.375868, "nauc_precision_at_1000_diff1": -0.014886, "nauc_mrr_at_1_max": 0.03979, "nauc_mrr_at_1_std": -0.199358, "nauc_mrr_at_1_diff1": 0.23001, "nauc_mrr_at_3_max": 0.082028, "nauc_mrr_at_3_std": -0.184185, "nauc_mrr_at_3_diff1": 0.225951, "nauc_mrr_at_5_max": 0.105206, "nauc_mrr_at_5_std": -0.19506, "nauc_mrr_at_5_diff1": 0.232428, "nauc_mrr_at_10_max": 0.093029, "nauc_mrr_at_10_std": -0.193911, "nauc_mrr_at_10_diff1": 0.230183, "nauc_mrr_at_20_max": 0.092077, "nauc_mrr_at_20_std": -0.19355, "nauc_mrr_at_20_diff1": 0.229841, "nauc_mrr_at_100_max": 0.092129, "nauc_mrr_at_100_std": -0.192012, "nauc_mrr_at_100_diff1": 0.229578, "nauc_mrr_at_1000_max": 0.091968, "nauc_mrr_at_1000_std": -0.192122, "nauc_mrr_at_1000_diff1": 0.229546, "hit_rate_at_1": 0.46689, "hit_rate_at_3": 0.72185, "hit_rate_at_5": 0.81788, "hit_rate_at_10": 0.88079, "hit_rate_at_20": 0.93709, "hit_rate_at_100": 0.98344, "hit_rate_at_1000": 1.0, "main_score": 0.47184, "hf_subset": "english", "languages": [ "eng-Latn" ], "mteb_version": "2.13.1" }, { "ndcg_at_1": 0.41556, "ndcg_at_3": 0.42176, "ndcg_at_5": 0.43483, "ndcg_at_10": 0.4631, "ndcg_at_20": 0.50846, "ndcg_at_100": 0.59313, "ndcg_at_1000": 0.61965, "map_at_1": 0.15067, "map_at_3": 0.23603, "map_at_5": 0.27126, "map_at_10": 0.32096, "map_at_20": 0.36131, "map_at_100": 0.39968, "map_at_1000": 0.4046, "recall_at_1": 0.15067, "recall_at_3": 0.29665, "recall_at_5": 0.37367, "recall_at_10": 0.49465, "recall_at_20": 0.63383, "recall_at_100": 0.88277, "recall_at_1000": 0.99576, "accuracy": 0.15067, "precision_at_1": 0.45033, "precision_at_3": 0.34437, "precision_at_5": 0.29868, "precision_at_10": 0.24073, "precision_at_20": 0.17765, "precision_at_100": 0.05987, "precision_at_1000": 0.00717, "mrr_at_1": 0.450331, "mrr_at_3": 0.56457, "mrr_at_5": 0.587252, "mrr_at_10": 0.596011, "mrr_at_20": 0.600912, "mrr_at_100": 0.602401, "mrr_at_1000": 0.602465, "nauc_ndcg_at_1_max": 0.052707, "nauc_ndcg_at_1_std": -0.126358, "nauc_ndcg_at_1_diff1": 0.294267, "nauc_ndcg_at_3_max": 0.10997, "nauc_ndcg_at_3_std": -0.218366, "nauc_ndcg_at_3_diff1": 0.283641, "nauc_ndcg_at_5_max": 0.134846, "nauc_ndcg_at_5_std": -0.233405, "nauc_ndcg_at_5_diff1": 0.273932, "nauc_ndcg_at_10_max": 0.129105, "nauc_ndcg_at_10_std": -0.268431, "nauc_ndcg_at_10_diff1": 0.268252, "nauc_ndcg_at_20_max": 0.14056, "nauc_ndcg_at_20_std": -0.258726, "nauc_ndcg_at_20_diff1": 0.255097, "nauc_ndcg_at_100_max": 0.153173, "nauc_ndcg_at_100_std": -0.191678, "nauc_ndcg_at_100_diff1": 0.273756, "nauc_ndcg_at_1000_max": 0.13125, "nauc_ndcg_at_1000_std": -0.184162, "nauc_ndcg_at_1000_diff1": 0.284518, "nauc_map_at_1_max": 0.034739, "nauc_map_at_1_std": -0.344797, "nauc_map_at_1_diff1": 0.413046, "nauc_map_at_3_max": 0.0341, "nauc_map_at_3_std": -0.362916, "nauc_map_at_3_diff1": 0.332582, "nauc_map_at_5_max": 0.04335, "nauc_map_at_5_std": -0.343537, "nauc_map_at_5_diff1": 0.312227, "nauc_map_at_10_max": 0.064639, "nauc_map_at_10_std": -0.329884, "nauc_map_at_10_diff1": 0.287635, "nauc_map_at_20_max": 0.090738, "nauc_map_at_20_std": -0.298258, "nauc_map_at_20_diff1": 0.27889, "nauc_map_at_100_max": 0.103563, "nauc_map_at_100_std": -0.258848, "nauc_map_at_100_diff1": 0.281636, "nauc_map_at_1000_max": 0.103575, "nauc_map_at_1000_std": -0.254948, "nauc_map_at_1000_diff1": 0.28384, "nauc_recall_at_1_max": 0.034739, "nauc_recall_at_1_std": -0.344797, "nauc_recall_at_1_diff1": 0.413046, "nauc_recall_at_3_max": 0.061393, "nauc_recall_at_3_std": -0.338177, "nauc_recall_at_3_diff1": 0.26674, "nauc_recall_at_5_max": 0.082578, "nauc_recall_at_5_std": -0.302976, "nauc_recall_at_5_diff1": 0.218152, "nauc_recall_at_10_max": 0.105858, "nauc_recall_at_10_std": -0.295145, "nauc_recall_at_10_diff1": 0.165969, "nauc_recall_at_20_max": 0.156541, "nauc_recall_at_20_std": -0.253949, "nauc_recall_at_20_diff1": 0.108165, "nauc_recall_at_100_max": 0.323834, "nauc_recall_at_100_std": -0.087821, "nauc_recall_at_100_diff1": 0.127873, "nauc_recall_at_1000_max": 0.771478, "nauc_recall_at_1000_std": 0.629591, "nauc_recall_at_1000_diff1": 0.460853, "nauc_precision_at_1_max": 0.054511, "nauc_precision_at_1_std": -0.117145, "nauc_precision_at_1_diff1": 0.281866, "nauc_precision_at_3_max": 0.104956, "nauc_precision_at_3_std": -0.038286, "nauc_precision_at_3_diff1": 0.097141, "nauc_precision_at_5_max": 0.130365, "nauc_precision_at_5_std": 0.042465, "nauc_precision_at_5_diff1": 0.033333, "nauc_precision_at_10_max": 0.115066, "nauc_precision_at_10_std": 0.112723, "nauc_precision_at_10_diff1": -0.015969, "nauc_precision_at_20_max": 0.137933, "nauc_precision_at_20_std": 0.205238, "nauc_precision_at_20_diff1": -0.033062, "nauc_precision_at_100_max": 0.091935, "nauc_precision_at_100_std": 0.328267, "nauc_precision_at_100_diff1": -0.062646, "nauc_precision_at_1000_max": 0.044962, "nauc_precision_at_1000_std": 0.332903, "nauc_precision_at_1000_diff1": -0.070867, "nauc_mrr_at_1_max": 0.054511, "nauc_mrr_at_1_std": -0.117145, "nauc_mrr_at_1_diff1": 0.281866, "nauc_mrr_at_3_max": 0.128176, "nauc_mrr_at_3_std": -0.131647, "nauc_mrr_at_3_diff1": 0.281038, "nauc_mrr_at_5_max": 0.132052, "nauc_mrr_at_5_std": -0.142482, "nauc_mrr_at_5_diff1": 0.273756, "nauc_mrr_at_10_max": 0.12698, "nauc_mrr_at_10_std": -0.136331, "nauc_mrr_at_10_diff1": 0.273687, "nauc_mrr_at_20_max": 0.125341, "nauc_mrr_at_20_std": -0.134913, "nauc_mrr_at_20_diff1": 0.273481, "nauc_mrr_at_100_max": 0.123685, "nauc_mrr_at_100_std": -0.134526, "nauc_mrr_at_100_diff1": 0.274407, "nauc_mrr_at_1000_max": 0.123536, "nauc_mrr_at_1000_std": -0.13456, "nauc_mrr_at_1000_diff1": 0.274377, "hit_rate_at_1": 0.45033, "hit_rate_at_3": 0.71192, "hit_rate_at_5": 0.81126, "hit_rate_at_10": 0.87417, "hit_rate_at_20": 0.9404, "hit_rate_at_100": 0.98675, "hit_rate_at_1000": 1.0, "main_score": 0.4631, "hf_subset": "italian", "languages": [ "ita-Latn" ], "mteb_version": "2.13.1" }, { "ndcg_at_1": 0.40894, "ndcg_at_3": 0.42141, "ndcg_at_5": 0.42387, "ndcg_at_10": 0.45475, "ndcg_at_20": 0.50237, "ndcg_at_100": 0.58917, "ndcg_at_1000": 0.61564, "map_at_1": 0.15113, "map_at_3": 0.23742, "map_at_5": 0.26479, "map_at_10": 0.315, "map_at_20": 0.35555, "map_at_100": 0.39407, "map_at_1000": 0.39918, "recall_at_1": 0.15113, "recall_at_3": 0.29943, "recall_at_5": 0.35932, "recall_at_10": 0.48313, "recall_at_20": 0.62504, "recall_at_100": 0.883, "recall_at_1000": 0.99603, "accuracy": 0.15113, "precision_at_1": 0.44702, "precision_at_3": 0.34216, "precision_at_5": 0.28411, "precision_at_10": 0.23245, "precision_at_20": 0.1745, "precision_at_100": 0.05967, "precision_at_1000": 0.00717, "mrr_at_1": 0.44702, "mrr_at_3": 0.565673, "mrr_at_5": 0.58521, "mrr_at_10": 0.594599, "mrr_at_20": 0.599829, "mrr_at_100": 0.601144, "mrr_at_1000": 0.601205, "nauc_ndcg_at_1_max": 0.047848, "nauc_ndcg_at_1_std": -0.185769, "nauc_ndcg_at_1_diff1": 0.314802, "nauc_ndcg_at_3_max": 0.11622, "nauc_ndcg_at_3_std": -0.251889, "nauc_ndcg_at_3_diff1": 0.332283, "nauc_ndcg_at_5_max": 0.155243, "nauc_ndcg_at_5_std": -0.249967, "nauc_ndcg_at_5_diff1": 0.313424, "nauc_ndcg_at_10_max": 0.160697, "nauc_ndcg_at_10_std": -0.302526, "nauc_ndcg_at_10_diff1": 0.293083, "nauc_ndcg_at_20_max": 0.169466, "nauc_ndcg_at_20_std": -0.284187, "nauc_ndcg_at_20_diff1": 0.285827, "nauc_ndcg_at_100_max": 0.185111, "nauc_ndcg_at_100_std": -0.209598, "nauc_ndcg_at_100_diff1": 0.310274, "nauc_ndcg_at_1000_max": 0.167152, "nauc_ndcg_at_1000_std": -0.206149, "nauc_ndcg_at_1000_diff1": 0.313534, "nauc_map_at_1_max": 0.062842, "nauc_map_at_1_std": -0.358749, "nauc_map_at_1_diff1": 0.404877, "nauc_map_at_3_max": 0.069036, "nauc_map_at_3_std": -0.374932, "nauc_map_at_3_diff1": 0.344711, "nauc_map_at_5_max": 0.07981, "nauc_map_at_5_std": -0.358644, "nauc_map_at_5_diff1": 0.327757, "nauc_map_at_10_max": 0.10305, "nauc_map_at_10_std": -0.357229, "nauc_map_at_10_diff1": 0.306485, "nauc_map_at_20_max": 0.128138, "nauc_map_at_20_std": -0.32405, "nauc_map_at_20_diff1": 0.305795, "nauc_map_at_100_max": 0.14494, "nauc_map_at_100_std": -0.281371, "nauc_map_at_100_diff1": 0.317496, "nauc_map_at_1000_max": 0.145577, "nauc_map_at_1000_std": -0.277229, "nauc_map_at_1000_diff1": 0.318903, "nauc_recall_at_1_max": 0.062842, "nauc_recall_at_1_std": -0.358749, "nauc_recall_at_1_diff1": 0.404877, "nauc_recall_at_3_max": 0.079153, "nauc_recall_at_3_std": -0.341812, "nauc_recall_at_3_diff1": 0.282422, "nauc_recall_at_5_max": 0.105861, "nauc_recall_at_5_std": -0.314381, "nauc_recall_at_5_diff1": 0.244085, "nauc_recall_at_10_max": 0.135505, "nauc_recall_at_10_std": -0.325118, "nauc_recall_at_10_diff1": 0.177673, "nauc_recall_at_20_max": 0.172348, "nauc_recall_at_20_std": -0.268826, "nauc_recall_at_20_diff1": 0.171801, "nauc_recall_at_100_max": 0.313475, "nauc_recall_at_100_std": -0.072525, "nauc_recall_at_100_diff1": 0.211883, "nauc_recall_at_1000_max": 0.528599, "nauc_recall_at_1000_std": 0.3843, "nauc_recall_at_1000_diff1": 0.212859, "nauc_precision_at_1_max": 0.047065, "nauc_precision_at_1_std": -0.167323, "nauc_precision_at_1_diff1": 0.293782, "nauc_precision_at_3_max": 0.059338, "nauc_precision_at_3_std": -0.072415, "nauc_precision_at_3_diff1": 0.164659, "nauc_precision_at_5_max": 0.128716, "nauc_precision_at_5_std": 0.057209, "nauc_precision_at_5_diff1": 0.080923, "nauc_precision_at_10_max": 0.133626, "nauc_precision_at_10_std": 0.096553, "nauc_precision_at_10_diff1": 0.022875, "nauc_precision_at_20_max": 0.163278, "nauc_precision_at_20_std": 0.214938, "nauc_precision_at_20_diff1": 0.014251, "nauc_precision_at_100_max": 0.115381, "nauc_precision_at_100_std": 0.356998, "nauc_precision_at_100_diff1": -0.012274, "nauc_precision_at_1000_max": 0.068904, "nauc_precision_at_1000_std": 0.355382, "nauc_precision_at_1000_diff1": -0.03815, "nauc_mrr_at_1_max": 0.047065, "nauc_mrr_at_1_std": -0.167323, "nauc_mrr_at_1_diff1": 0.293782, "nauc_mrr_at_3_max": 0.101654, "nauc_mrr_at_3_std": -0.171303, "nauc_mrr_at_3_diff1": 0.29087, "nauc_mrr_at_5_max": 0.124469, "nauc_mrr_at_5_std": -0.162751, "nauc_mrr_at_5_diff1": 0.284798, "nauc_mrr_at_10_max": 0.117837, "nauc_mrr_at_10_std": -0.1714, "nauc_mrr_at_10_diff1": 0.285191, "nauc_mrr_at_20_max": 0.113753, "nauc_mrr_at_20_std": -0.174234, "nauc_mrr_at_20_diff1": 0.283925, "nauc_mrr_at_100_max": 0.113578, "nauc_mrr_at_100_std": -0.172971, "nauc_mrr_at_100_diff1": 0.285666, "nauc_mrr_at_1000_max": 0.113433, "nauc_mrr_at_1000_std": -0.172999, "nauc_mrr_at_1000_diff1": 0.285567, "hit_rate_at_1": 0.44702, "hit_rate_at_3": 0.71523, "hit_rate_at_5": 0.80132, "hit_rate_at_10": 0.86755, "hit_rate_at_20": 0.94371, "hit_rate_at_100": 0.98675, "hit_rate_at_1000": 1.0, "main_score": 0.45475, "hf_subset": "german", "languages": [ "deu-Latn" ], "mteb_version": "2.13.1" }, { "ndcg_at_1": 0.39735, "ndcg_at_3": 0.42376, "ndcg_at_5": 0.43848, "ndcg_at_10": 0.4596, "ndcg_at_20": 0.5058, "ndcg_at_100": 0.59385, "ndcg_at_1000": 0.61965, "map_at_1": 0.15207, "map_at_3": 0.23985, "map_at_5": 0.2751, "map_at_10": 0.32129, "map_at_20": 0.36151, "map_at_100": 0.40093, "map_at_1000": 0.40578, "recall_at_1": 0.15207, "recall_at_3": 0.29854, "recall_at_5": 0.37532, "recall_at_10": 0.48474, "recall_at_20": 0.62779, "recall_at_100": 0.88329, "recall_at_1000": 0.99591, "accuracy": 0.15207, "precision_at_1": 0.42384, "precision_at_3": 0.34879, "precision_at_5": 0.30265, "precision_at_10": 0.23609, "precision_at_20": 0.17384, "precision_at_100": 0.0599, "precision_at_1000": 0.00717, "mrr_at_1": 0.423841, "mrr_at_3": 0.549117, "mrr_at_5": 0.57511, "mrr_at_10": 0.582628, "mrr_at_20": 0.587551, "mrr_at_100": 0.58918, "mrr_at_1000": 0.589232, "nauc_ndcg_at_1_max": 0.069774, "nauc_ndcg_at_1_std": -0.205918, "nauc_ndcg_at_1_diff1": 0.318893, "nauc_ndcg_at_3_max": 0.111761, "nauc_ndcg_at_3_std": -0.258169, "nauc_ndcg_at_3_diff1": 0.284003, "nauc_ndcg_at_5_max": 0.147713, "nauc_ndcg_at_5_std": -0.264133, "nauc_ndcg_at_5_diff1": 0.262741, "nauc_ndcg_at_10_max": 0.150963, "nauc_ndcg_at_10_std": -0.285316, "nauc_ndcg_at_10_diff1": 0.26506, "nauc_ndcg_at_20_max": 0.150305, "nauc_ndcg_at_20_std": -0.279289, "nauc_ndcg_at_20_diff1": 0.241651, "nauc_ndcg_at_100_max": 0.175072, "nauc_ndcg_at_100_std": -0.206956, "nauc_ndcg_at_100_diff1": 0.263665, "nauc_ndcg_at_1000_max": 0.157412, "nauc_ndcg_at_1000_std": -0.208409, "nauc_ndcg_at_1000_diff1": 0.284179, "nauc_map_at_1_max": 0.061971, "nauc_map_at_1_std": -0.384874, "nauc_map_at_1_diff1": 0.394941, "nauc_map_at_3_max": 0.038066, "nauc_map_at_3_std": -0.401182, "nauc_map_at_3_diff1": 0.312986, "nauc_map_at_5_max": 0.047633, "nauc_map_at_5_std": -0.375889, "nauc_map_at_5_diff1": 0.288206, "nauc_map_at_10_max": 0.079541, "nauc_map_at_10_std": -0.353908, "nauc_map_at_10_diff1": 0.270359, "nauc_map_at_20_max": 0.104383, "nauc_map_at_20_std": -0.323305, "nauc_map_at_20_diff1": 0.263801, "nauc_map_at_100_max": 0.128257, "nauc_map_at_100_std": -0.279941, "nauc_map_at_100_diff1": 0.267761, "nauc_map_at_1000_max": 0.129273, "nauc_map_at_1000_std": -0.276261, "nauc_map_at_1000_diff1": 0.270923, "nauc_recall_at_1_max": 0.061971, "nauc_recall_at_1_std": -0.384874, "nauc_recall_at_1_diff1": 0.394941, "nauc_recall_at_3_max": 0.05255, "nauc_recall_at_3_std": -0.377265, "nauc_recall_at_3_diff1": 0.239974, "nauc_recall_at_5_max": 0.081428, "nauc_recall_at_5_std": -0.327795, "nauc_recall_at_5_diff1": 0.186193, "nauc_recall_at_10_max": 0.128475, "nauc_recall_at_10_std": -0.295835, "nauc_recall_at_10_diff1": 0.153203, "nauc_recall_at_20_max": 0.143312, "nauc_recall_at_20_std": -0.253728, "nauc_recall_at_20_diff1": 0.076192, "nauc_recall_at_100_max": 0.289864, "nauc_recall_at_100_std": -0.043964, "nauc_recall_at_100_diff1": 0.017868, "nauc_recall_at_1000_max": 0.56908, "nauc_recall_at_1000_std": 0.523724, "nauc_recall_at_1000_diff1": 0.108829, "nauc_precision_at_1_max": 0.085204, "nauc_precision_at_1_std": -0.198928, "nauc_precision_at_1_diff1": 0.333211, "nauc_precision_at_3_max": 0.099721, "nauc_precision_at_3_std": -0.042102, "nauc_precision_at_3_diff1": 0.13338, "nauc_precision_at_5_max": 0.141968, "nauc_precision_at_5_std": 0.048784, "nauc_precision_at_5_diff1": 0.042694, "nauc_precision_at_10_max": 0.175331, "nauc_precision_at_10_std": 0.130223, "nauc_precision_at_10_diff1": 0.023873, "nauc_precision_at_20_max": 0.182479, "nauc_precision_at_20_std": 0.226588, "nauc_precision_at_20_diff1": -0.01049, "nauc_precision_at_100_max": 0.154823, "nauc_precision_at_100_std": 0.362979, "nauc_precision_at_100_diff1": -0.022348, "nauc_precision_at_1000_max": 0.100936, "nauc_precision_at_1000_std": 0.347872, "nauc_precision_at_1000_diff1": -0.013353, "nauc_mrr_at_1_max": 0.085204, "nauc_mrr_at_1_std": -0.198928, "nauc_mrr_at_1_diff1": 0.333211, "nauc_mrr_at_3_max": 0.137429, "nauc_mrr_at_3_std": -0.184763, "nauc_mrr_at_3_diff1": 0.317406, "nauc_mrr_at_5_max": 0.147273, "nauc_mrr_at_5_std": -0.188258, "nauc_mrr_at_5_diff1": 0.316544, "nauc_mrr_at_10_max": 0.141277, "nauc_mrr_at_10_std": -0.189375, "nauc_mrr_at_10_diff1": 0.321538, "nauc_mrr_at_20_max": 0.141302, "nauc_mrr_at_20_std": -0.18856, "nauc_mrr_at_20_diff1": 0.319972, "nauc_mrr_at_100_max": 0.139695, "nauc_mrr_at_100_std": -0.188209, "nauc_mrr_at_100_diff1": 0.320191, "nauc_mrr_at_1000_max": 0.139566, "nauc_mrr_at_1000_std": -0.188292, "nauc_mrr_at_1000_diff1": 0.320204, "hit_rate_at_1": 0.42384, "hit_rate_at_3": 0.70199, "hit_rate_at_5": 0.81457, "hit_rate_at_10": 0.87086, "hit_rate_at_20": 0.93709, "hit_rate_at_100": 0.98675, "hit_rate_at_1000": 1.0, "main_score": 0.4596, "hf_subset": "portuguese", "languages": [ "por-Latn" ], "mteb_version": "2.13.1" }, { "ndcg_at_1": 0.38245, "ndcg_at_3": 0.41626, "ndcg_at_5": 0.43426, "ndcg_at_10": 0.45581, "ndcg_at_20": 0.50645, "ndcg_at_100": 0.59151, "ndcg_at_1000": 0.61801, "map_at_1": 0.15151, "map_at_3": 0.23841, "map_at_5": 0.27398, "map_at_10": 0.31954, "map_at_20": 0.36197, "map_at_100": 0.40025, "map_at_1000": 0.40523, "recall_at_1": 0.15151, "recall_at_3": 0.29897, "recall_at_5": 0.38086, "recall_at_10": 0.48134, "recall_at_20": 0.63368, "recall_at_100": 0.88103, "recall_at_1000": 0.99609, "accuracy": 0.15151, "precision_at_1": 0.41391, "precision_at_3": 0.34106, "precision_at_5": 0.29669, "precision_at_10": 0.23675, "precision_at_20": 0.175, "precision_at_100": 0.0598, "precision_at_1000": 0.00717, "mrr_at_1": 0.413907, "mrr_at_3": 0.540287, "mrr_at_5": 0.566777, "mrr_at_10": 0.573135, "mrr_at_20": 0.579227, "mrr_at_100": 0.580477, "mrr_at_1000": 0.580532, "nauc_ndcg_at_1_max": 0.014338, "nauc_ndcg_at_1_std": -0.201002, "nauc_ndcg_at_1_diff1": 0.363793, "nauc_ndcg_at_3_max": 0.09692, "nauc_ndcg_at_3_std": -0.227143, "nauc_ndcg_at_3_diff1": 0.328065, "nauc_ndcg_at_5_max": 0.110621, "nauc_ndcg_at_5_std": -0.242166, "nauc_ndcg_at_5_diff1": 0.304037, "nauc_ndcg_at_10_max": 0.104492, "nauc_ndcg_at_10_std": -0.267741, "nauc_ndcg_at_10_diff1": 0.299203, "nauc_ndcg_at_20_max": 0.111875, "nauc_ndcg_at_20_std": -0.262064, "nauc_ndcg_at_20_diff1": 0.291169, "nauc_ndcg_at_100_max": 0.136969, "nauc_ndcg_at_100_std": -0.189395, "nauc_ndcg_at_100_diff1": 0.318843, "nauc_ndcg_at_1000_max": 0.119691, "nauc_ndcg_at_1000_std": -0.187701, "nauc_ndcg_at_1000_diff1": 0.319967, "nauc_map_at_1_max": 0.020742, "nauc_map_at_1_std": -0.358296, "nauc_map_at_1_diff1": 0.441496, "nauc_map_at_3_max": 0.028412, "nauc_map_at_3_std": -0.367566, "nauc_map_at_3_diff1": 0.362505, "nauc_map_at_5_max": 0.024173, "nauc_map_at_5_std": -0.349036, "nauc_map_at_5_diff1": 0.335496, "nauc_map_at_10_max": 0.048202, "nauc_map_at_10_std": -0.329976, "nauc_map_at_10_diff1": 0.320641, "nauc_map_at_20_max": 0.071615, "nauc_map_at_20_std": -0.299953, "nauc_map_at_20_diff1": 0.314974, "nauc_map_at_100_max": 0.09305, "nauc_map_at_100_std": -0.256735, "nauc_map_at_100_diff1": 0.319918, "nauc_map_at_1000_max": 0.094499, "nauc_map_at_1000_std": -0.25277, "nauc_map_at_1000_diff1": 0.320764, "nauc_recall_at_1_max": 0.020742, "nauc_recall_at_1_std": -0.358296, "nauc_recall_at_1_diff1": 0.441496, "nauc_recall_at_3_max": 0.078993, "nauc_recall_at_3_std": -0.328173, "nauc_recall_at_3_diff1": 0.291791, "nauc_recall_at_5_max": 0.05365, "nauc_recall_at_5_std": -0.300907, "nauc_recall_at_5_diff1": 0.217322, "nauc_recall_at_10_max": 0.087393, "nauc_recall_at_10_std": -0.288497, "nauc_recall_at_10_diff1": 0.194817, "nauc_recall_at_20_max": 0.122313, "nauc_recall_at_20_std": -0.256558, "nauc_recall_at_20_diff1": 0.144774, "nauc_recall_at_100_max": 0.287362, "nauc_recall_at_100_std": -0.024589, "nauc_recall_at_100_diff1": 0.186382, "nauc_recall_at_1000_max": 0.702091, "nauc_recall_at_1000_std": 0.527497, "nauc_recall_at_1000_diff1": 0.355375, "nauc_precision_at_1_max": 0.04284, "nauc_precision_at_1_std": -0.165269, "nauc_precision_at_1_diff1": 0.361698, "nauc_precision_at_3_max": 0.092848, "nauc_precision_at_3_std": -0.018557, "nauc_precision_at_3_diff1": 0.140038, "nauc_precision_at_5_max": 0.119608, "nauc_precision_at_5_std": 0.068696, "nauc_precision_at_5_diff1": 0.043903, "nauc_precision_at_10_max": 0.120631, "nauc_precision_at_10_std": 0.132165, "nauc_precision_at_10_diff1": 0.007442, "nauc_precision_at_20_max": 0.151949, "nauc_precision_at_20_std": 0.221382, "nauc_precision_at_20_diff1": -0.020138, "nauc_precision_at_100_max": 0.13467, "nauc_precision_at_100_std": 0.361462, "nauc_precision_at_100_diff1": -0.059871, "nauc_precision_at_1000_max": 0.091841, "nauc_precision_at_1000_std": 0.355942, "nauc_precision_at_1000_diff1": -0.091098, "nauc_mrr_at_1_max": 0.04284, "nauc_mrr_at_1_std": -0.165269, "nauc_mrr_at_1_diff1": 0.361698, "nauc_mrr_at_3_max": 0.120974, "nauc_mrr_at_3_std": -0.138238, "nauc_mrr_at_3_diff1": 0.34641, "nauc_mrr_at_5_max": 0.119579, "nauc_mrr_at_5_std": -0.148544, "nauc_mrr_at_5_diff1": 0.337185, "nauc_mrr_at_10_max": 0.108653, "nauc_mrr_at_10_std": -0.15244, "nauc_mrr_at_10_diff1": 0.338677, "nauc_mrr_at_20_max": 0.11203, "nauc_mrr_at_20_std": -0.148379, "nauc_mrr_at_20_diff1": 0.338634, "nauc_mrr_at_100_max": 0.110797, "nauc_mrr_at_100_std": -0.148363, "nauc_mrr_at_100_diff1": 0.33924, "nauc_mrr_at_1000_max": 0.110644, "nauc_mrr_at_1000_std": -0.148479, "nauc_mrr_at_1000_diff1": 0.339272, "hit_rate_at_1": 0.41391, "hit_rate_at_3": 0.70199, "hit_rate_at_5": 0.81457, "hit_rate_at_10": 0.86424, "hit_rate_at_20": 0.94371, "hit_rate_at_100": 0.98675, "hit_rate_at_1000": 1.0, "main_score": 0.45581, "hf_subset": "spanish", "languages": [ "spa-Latn" ], "mteb_version": "2.13.1" }, { "ndcg_at_1": 0.41887, "ndcg_at_3": 0.42914, "ndcg_at_5": 0.44051, "ndcg_at_10": 0.47026, "ndcg_at_20": 0.51317, "ndcg_at_100": 0.5988, "ndcg_at_1000": 0.62438, "map_at_1": 0.15858, "map_at_3": 0.23978, "map_at_5": 0.27606, "map_at_10": 0.32682, "map_at_20": 0.36694, "map_at_100": 0.40546, "map_at_1000": 0.41035, "recall_at_1": 0.15858, "recall_at_3": 0.29882, "recall_at_5": 0.3758, "recall_at_10": 0.50036, "recall_at_20": 0.63635, "recall_at_100": 0.88541, "recall_at_1000": 0.99581, "accuracy": 0.15858, "precision_at_1": 0.45033, "precision_at_3": 0.34879, "precision_at_5": 0.30199, "precision_at_10": 0.24503, "precision_at_20": 0.17699, "precision_at_100": 0.06007, "precision_at_1000": 0.00717, "mrr_at_1": 0.450331, "mrr_at_3": 0.567881, "mrr_at_5": 0.587914, "mrr_at_10": 0.598384, "mrr_at_20": 0.602399, "mrr_at_100": 0.603843, "mrr_at_1000": 0.603899, "nauc_ndcg_at_1_max": 0.121049, "nauc_ndcg_at_1_std": -0.129726, "nauc_ndcg_at_1_diff1": 0.257239, "nauc_ndcg_at_3_max": 0.157882, "nauc_ndcg_at_3_std": -0.227943, "nauc_ndcg_at_3_diff1": 0.265296, "nauc_ndcg_at_5_max": 0.167988, "nauc_ndcg_at_5_std": -0.238015, "nauc_ndcg_at_5_diff1": 0.248225, "nauc_ndcg_at_10_max": 0.169255, "nauc_ndcg_at_10_std": -0.25491, "nauc_ndcg_at_10_diff1": 0.227029, "nauc_ndcg_at_20_max": 0.173023, "nauc_ndcg_at_20_std": -0.2497, "nauc_ndcg_at_20_diff1": 0.216667, "nauc_ndcg_at_100_max": 0.197413, "nauc_ndcg_at_100_std": -0.17591, "nauc_ndcg_at_100_diff1": 0.238909, "nauc_ndcg_at_1000_max": 0.185073, "nauc_ndcg_at_1000_std": -0.172498, "nauc_ndcg_at_1000_diff1": 0.247994, "nauc_map_at_1_max": 0.052854, "nauc_map_at_1_std": -0.322046, "nauc_map_at_1_diff1": 0.329403, "nauc_map_at_3_max": 0.043812, "nauc_map_at_3_std": -0.361817, "nauc_map_at_3_diff1": 0.28208, "nauc_map_at_5_max": 0.051226, "nauc_map_at_5_std": -0.336775, "nauc_map_at_5_diff1": 0.262482, "nauc_map_at_10_max": 0.086486, "nauc_map_at_10_std": -0.320498, "nauc_map_at_10_diff1": 0.240847, "nauc_map_at_20_max": 0.11489, "nauc_map_at_20_std": -0.288744, "nauc_map_at_20_diff1": 0.239463, "nauc_map_at_100_max": 0.137317, "nauc_map_at_100_std": -0.246591, "nauc_map_at_100_diff1": 0.244216, "nauc_map_at_1000_max": 0.13909, "nauc_map_at_1000_std": -0.242388, "nauc_map_at_1000_diff1": 0.245977, "nauc_recall_at_1_max": 0.052854, "nauc_recall_at_1_std": -0.322046, "nauc_recall_at_1_diff1": 0.329403, "nauc_recall_at_3_max": 0.052158, "nauc_recall_at_3_std": -0.354099, "nauc_recall_at_3_diff1": 0.22808, "nauc_recall_at_5_max": 0.069071, "nauc_recall_at_5_std": -0.301661, "nauc_recall_at_5_diff1": 0.187662, "nauc_recall_at_10_max": 0.120494, "nauc_recall_at_10_std": -0.270832, "nauc_recall_at_10_diff1": 0.116379, "nauc_recall_at_20_max": 0.154171, "nauc_recall_at_20_std": -0.243657, "nauc_recall_at_20_diff1": 0.074412, "nauc_recall_at_100_max": 0.284871, "nauc_recall_at_100_std": -0.022425, "nauc_recall_at_100_diff1": 0.056425, "nauc_recall_at_1000_max": 0.558264, "nauc_recall_at_1000_std": 0.495322, "nauc_recall_at_1000_diff1": 0.066912, "nauc_precision_at_1_max": 0.140246, "nauc_precision_at_1_std": -0.096637, "nauc_precision_at_1_diff1": 0.268781, "nauc_precision_at_3_max": 0.1675, "nauc_precision_at_3_std": -0.046983, "nauc_precision_at_3_diff1": 0.15671, "nauc_precision_at_5_max": 0.17658, "nauc_precision_at_5_std": 0.032023, "nauc_precision_at_5_diff1": 0.077165, "nauc_precision_at_10_max": 0.207438, "nauc_precision_at_10_std": 0.119968, "nauc_precision_at_10_diff1": 0.008533, "nauc_precision_at_20_max": 0.217177, "nauc_precision_at_20_std": 0.214797, "nauc_precision_at_20_diff1": -0.000267, "nauc_precision_at_100_max": 0.179579, "nauc_precision_at_100_std": 0.350754, "nauc_precision_at_100_diff1": -0.009016, "nauc_precision_at_1000_max": 0.130689, "nauc_precision_at_1000_std": 0.34922, "nauc_precision_at_1000_diff1": -0.021998, "nauc_mrr_at_1_max": 0.140246, "nauc_mrr_at_1_std": -0.096637, "nauc_mrr_at_1_diff1": 0.268781, "nauc_mrr_at_3_max": 0.200876, "nauc_mrr_at_3_std": -0.121806, "nauc_mrr_at_3_diff1": 0.281078, "nauc_mrr_at_5_max": 0.212089, "nauc_mrr_at_5_std": -0.117233, "nauc_mrr_at_5_diff1": 0.280974, "nauc_mrr_at_10_max": 0.210154, "nauc_mrr_at_10_std": -0.111388, "nauc_mrr_at_10_diff1": 0.275323, "nauc_mrr_at_20_max": 0.208321, "nauc_mrr_at_20_std": -0.111746, "nauc_mrr_at_20_diff1": 0.271828, "nauc_mrr_at_100_max": 0.206641, "nauc_mrr_at_100_std": -0.111608, "nauc_mrr_at_100_diff1": 0.273005, "nauc_mrr_at_1000_max": 0.206493, "nauc_mrr_at_1000_std": -0.111683, "nauc_mrr_at_1000_diff1": 0.273009, "hit_rate_at_1": 0.45033, "hit_rate_at_3": 0.72185, "hit_rate_at_5": 0.80795, "hit_rate_at_10": 0.88411, "hit_rate_at_20": 0.9404, "hit_rate_at_100": 0.98675, "hit_rate_at_1000": 1.0, "main_score": 0.47026, "hf_subset": "french", "languages": [ "fra-Latn" ], "mteb_version": "2.13.1" } ] }, "evaluation_time": 3463.095775971189, "kg_co2_emissions": null, "date": 1782143242.1808, "evaluation_phases": [ { "name": "Data loading", "start": 6.983242928981781e-05, "end": 72.607676663436, "split": "", "subset": "" }, { "name": "Dataset transform", "start": 72.60772663820535, "end": 72.6077304566279, "split": "", "subset": "" }, { "name": "Encoding queries and documents", "start": 72.62122993264347, "end": 646.9154388317838, "split": "test", "subset": "italian" }, { "name": "Scoring", "start": 646.9156701909378, "end": 647.2107972446829, "split": "test", "subset": "italian" }, { "name": "Encoding queries and documents", "start": 647.2554397573695, "end": 1206.8742012586445, "split": "test", "subset": "german" }, { "name": "Scoring", "start": 1206.874459608458, "end": 1207.1638729954138, "split": "test", "subset": "german" }, { "name": "Encoding queries and documents", "start": 1207.1967919794843, "end": 1767.3272046456113, "split": "test", "subset": "portuguese" }, { "name": "Scoring", "start": 1767.3274743668735, "end": 1767.616190421395, "split": "test", "subset": "portuguese" }, { "name": "Encoding queries and documents", "start": 1767.6510382434353, "end": 2320.847398303449, "split": "test", "subset": "spanish" }, { "name": "Scoring", "start": 2320.847649369389, "end": 2321.1405290318653, "split": "test", "subset": "spanish" }, { "name": "Encoding queries and documents", "start": 2321.1771001433954, "end": 2868.421861823648, "split": "test", "subset": "french" }, { "name": "Scoring", "start": 2868.4221301423386, "end": 2868.7158617191017, "split": "test", "subset": "french" } ] }