{ "dataset_revision": "4d42cf85d8b8117935c04698b5b7fd3c70c8c9f4", "task_name": "Vidore3ComputerScienceRetrieval.v2", "mteb_version": "2.13.1", "scores": { "test": [ { "ndcg_at_1": 0.02326, "ndcg_at_3": 0.01091, "ndcg_at_5": 0.00985, "ndcg_at_10": 0.00867, "ndcg_at_20": 0.01309, "ndcg_at_100": 0.03791, "ndcg_at_1000": 0.16992, "map_at_1": 0.00324, "map_at_3": 0.00324, "map_at_5": 0.00358, "map_at_10": 0.00397, "map_at_20": 0.00488, "map_at_100": 0.00758, "map_at_1000": 0.01288, "recall_at_1": 0.00324, "recall_at_3": 0.00324, "recall_at_5": 0.00411, "recall_at_10": 0.00631, "recall_at_20": 0.01746, "recall_at_100": 0.11391, "recall_at_1000": 0.85849, "accuracy": 0.00324, "precision_at_1": 0.02326, "precision_at_3": 0.00775, "precision_at_5": 0.00651, "precision_at_10": 0.00465, "precision_at_20": 0.00558, "precision_at_100": 0.0054, "precision_at_1000": 0.00415, "mrr_at_1": 0.023256, "mrr_at_3": 0.023256, "mrr_at_5": 0.024419, "mrr_at_10": 0.025581, "mrr_at_20": 0.029307, "mrr_at_100": 0.034123, "mrr_at_1000": 0.036961, "nauc_ndcg_at_1_max": -0.02993, "nauc_ndcg_at_1_std": -0.332198, "nauc_ndcg_at_1_diff1": 0.323768, "nauc_ndcg_at_3_max": -0.02993, "nauc_ndcg_at_3_std": -0.332198, "nauc_ndcg_at_3_diff1": 0.323768, "nauc_ndcg_at_5_max": 0.032366, "nauc_ndcg_at_5_std": -0.236222, "nauc_ndcg_at_5_diff1": 0.39293, "nauc_ndcg_at_10_max": -0.076843, "nauc_ndcg_at_10_std": -0.288409, "nauc_ndcg_at_10_diff1": 0.319537, "nauc_ndcg_at_20_max": -0.10057, "nauc_ndcg_at_20_std": -0.146551, "nauc_ndcg_at_20_diff1": 0.266491, "nauc_ndcg_at_100_max": -0.068304, "nauc_ndcg_at_100_std": -0.022539, "nauc_ndcg_at_100_diff1": 0.201079, "nauc_ndcg_at_1000_max": 0.065888, "nauc_ndcg_at_1000_std": 0.031832, "nauc_ndcg_at_1000_diff1": 0.191146, "nauc_map_at_1_max": -0.193893, "nauc_map_at_1_std": -0.351551, "nauc_map_at_1_diff1": 0.202382, "nauc_map_at_3_max": -0.193893, "nauc_map_at_3_std": -0.351551, "nauc_map_at_3_diff1": 0.202382, "nauc_map_at_5_max": -0.1268, "nauc_map_at_5_std": -0.307105, "nauc_map_at_5_diff1": 0.279811, "nauc_map_at_10_max": -0.129422, "nauc_map_at_10_std": -0.316424, "nauc_map_at_10_diff1": 0.283716, "nauc_map_at_20_max": -0.127634, "nauc_map_at_20_std": -0.248212, "nauc_map_at_20_diff1": 0.291969, "nauc_map_at_100_max": -0.093407, "nauc_map_at_100_std": -0.161217, "nauc_map_at_100_diff1": 0.270141, "nauc_map_at_1000_max": -0.049371, "nauc_map_at_1000_std": -0.100371, "nauc_map_at_1000_diff1": 0.270217, "nauc_recall_at_1_max": -0.193893, "nauc_recall_at_1_std": -0.351551, "nauc_recall_at_1_diff1": 0.202382, "nauc_recall_at_3_max": -0.193893, "nauc_recall_at_3_std": -0.351551, "nauc_recall_at_3_diff1": 0.202382, "nauc_recall_at_5_max": -0.024939, "nauc_recall_at_5_std": -0.215223, "nauc_recall_at_5_diff1": 0.37211, "nauc_recall_at_10_max": -0.135069, "nauc_recall_at_10_std": -0.308314, "nauc_recall_at_10_diff1": 0.261245, "nauc_recall_at_20_max": -0.140836, "nauc_recall_at_20_std": -0.018025, "nauc_recall_at_20_diff1": 0.257139, "nauc_recall_at_100_max": -0.096479, "nauc_recall_at_100_std": 0.045561, "nauc_recall_at_100_diff1": 0.137699, "nauc_recall_at_1000_max": 0.01884, "nauc_recall_at_1000_std": 0.197166, "nauc_recall_at_1000_diff1": 0.113411, "nauc_precision_at_1_max": -0.02993, "nauc_precision_at_1_std": -0.332198, "nauc_precision_at_1_diff1": 0.323768, "nauc_precision_at_3_max": -0.02993, "nauc_precision_at_3_std": -0.332198, "nauc_precision_at_3_diff1": 0.323768, "nauc_precision_at_5_max": 0.167653, "nauc_precision_at_5_std": -0.123435, "nauc_precision_at_5_diff1": 0.516977, "nauc_precision_at_10_max": 0.04343, "nauc_precision_at_10_std": -0.220409, "nauc_precision_at_10_diff1": 0.410126, "nauc_precision_at_20_max": -0.03928, "nauc_precision_at_20_std": -0.080269, "nauc_precision_at_20_diff1": 0.317371, "nauc_precision_at_100_max": 0.032103, "nauc_precision_at_100_std": 0.051545, "nauc_precision_at_100_diff1": 0.247105, "nauc_precision_at_1000_max": 0.146835, "nauc_precision_at_1000_std": 0.036405, "nauc_precision_at_1000_diff1": 0.029317, "nauc_mrr_at_1_max": -0.02993, "nauc_mrr_at_1_std": -0.332198, "nauc_mrr_at_1_diff1": 0.323768, "nauc_mrr_at_3_max": -0.02993, "nauc_mrr_at_3_std": -0.332198, "nauc_mrr_at_3_diff1": 0.323768, "nauc_mrr_at_5_max": 0.019114, "nauc_mrr_at_5_std": -0.26876, "nauc_mrr_at_5_diff1": 0.355969, "nauc_mrr_at_10_max": -0.005902, "nauc_mrr_at_10_std": -0.282384, "nauc_mrr_at_10_diff1": 0.328026, "nauc_mrr_at_20_max": -0.011078, "nauc_mrr_at_20_std": -0.233844, "nauc_mrr_at_20_diff1": 0.296113, "nauc_mrr_at_100_max": -0.013048, "nauc_mrr_at_100_std": -0.202089, "nauc_mrr_at_100_diff1": 0.274526, "nauc_mrr_at_1000_max": -0.009569, "nauc_mrr_at_1000_std": -0.205987, "nauc_mrr_at_1000_diff1": 0.277779, "hit_rate_at_1": 0.02326, "hit_rate_at_3": 0.02326, "hit_rate_at_5": 0.02791, "hit_rate_at_10": 0.03721, "hit_rate_at_20": 0.08837, "hit_rate_at_100": 0.33023, "hit_rate_at_1000": 0.9814, "main_score": 0.00867, "hf_subset": "french", "languages": [ "fra-Latn" ], "mteb_version": "2.13.1" }, { "ndcg_at_1": 0.02326, "ndcg_at_3": 0.01166, "ndcg_at_5": 0.01023, "ndcg_at_10": 0.01151, "ndcg_at_20": 0.01419, "ndcg_at_100": 0.04224, "ndcg_at_1000": 0.17483, "map_at_1": 0.00324, "map_at_3": 0.00355, "map_at_5": 0.00386, "map_at_10": 0.00488, "map_at_20": 0.00554, "map_at_100": 0.0087, "map_at_1000": 0.01396, "recall_at_1": 0.00324, "recall_at_3": 0.00417, "recall_at_5": 0.00572, "recall_at_10": 0.01001, "recall_at_20": 0.01703, "recall_at_100": 0.13049, "recall_at_1000": 0.87454, "accuracy": 0.00324, "precision_at_1": 0.02326, "precision_at_3": 0.0093, "precision_at_5": 0.00651, "precision_at_10": 0.00605, "precision_at_20": 0.00558, "precision_at_100": 0.00591, "precision_at_1000": 0.00421, "mrr_at_1": 0.023256, "mrr_at_3": 0.024806, "mrr_at_5": 0.025736, "mrr_at_10": 0.02761, "mrr_at_20": 0.030072, "mrr_at_100": 0.035662, "mrr_at_1000": 0.038683, "nauc_ndcg_at_1_max": 0.136605, "nauc_ndcg_at_1_std": -0.356944, "nauc_ndcg_at_1_diff1": 0.442149, "nauc_ndcg_at_3_max": 0.091674, "nauc_ndcg_at_3_std": -0.370425, "nauc_ndcg_at_3_diff1": 0.382495, "nauc_ndcg_at_5_max": -0.024909, "nauc_ndcg_at_5_std": -0.405403, "nauc_ndcg_at_5_diff1": 0.265084, "nauc_ndcg_at_10_max": -0.099934, "nauc_ndcg_at_10_std": -0.344871, "nauc_ndcg_at_10_diff1": 0.15671, "nauc_ndcg_at_20_max": -0.109947, "nauc_ndcg_at_20_std": -0.215458, "nauc_ndcg_at_20_diff1": 0.154654, "nauc_ndcg_at_100_max": -0.062299, "nauc_ndcg_at_100_std": 0.007718, "nauc_ndcg_at_100_diff1": 0.115871, "nauc_ndcg_at_1000_max": 0.061757, "nauc_ndcg_at_1000_std": 0.002064, "nauc_ndcg_at_1000_diff1": 0.209262, "nauc_map_at_1_max": -0.083466, "nauc_map_at_1_std": -0.415786, "nauc_map_at_1_diff1": 0.232903, "nauc_map_at_3_max": -0.125886, "nauc_map_at_3_std": -0.429141, "nauc_map_at_3_diff1": 0.169331, "nauc_map_at_5_max": -0.161482, "nauc_map_at_5_std": -0.440349, "nauc_map_at_5_diff1": 0.130417, "nauc_map_at_10_max": -0.131979, "nauc_map_at_10_std": -0.384647, "nauc_map_at_10_diff1": 0.139552, "nauc_map_at_20_max": -0.116468, "nauc_map_at_20_std": -0.303474, "nauc_map_at_20_diff1": 0.162833, "nauc_map_at_100_max": -0.099801, "nauc_map_at_100_std": -0.177107, "nauc_map_at_100_diff1": 0.153534, "nauc_map_at_1000_max": -0.056, "nauc_map_at_1000_std": -0.130901, "nauc_map_at_1000_diff1": 0.185979, "nauc_recall_at_1_max": -0.083466, "nauc_recall_at_1_std": -0.415786, "nauc_recall_at_1_diff1": 0.232903, "nauc_recall_at_3_max": -0.191779, "nauc_recall_at_3_std": -0.449888, "nauc_recall_at_3_diff1": 0.070579, "nauc_recall_at_5_max": -0.293958, "nauc_recall_at_5_std": -0.482058, "nauc_recall_at_5_diff1": -0.033874, "nauc_recall_at_10_max": -0.196438, "nauc_recall_at_10_std": -0.33377, "nauc_recall_at_10_diff1": 0.018283, "nauc_recall_at_20_max": -0.167049, "nauc_recall_at_20_std": -0.049654, "nauc_recall_at_20_diff1": 0.089167, "nauc_recall_at_100_max": -0.112009, "nauc_recall_at_100_std": 0.129421, "nauc_recall_at_100_diff1": -0.009562, "nauc_recall_at_1000_max": -0.008204, "nauc_recall_at_1000_std": 0.226119, "nauc_recall_at_1000_diff1": 0.102458, "nauc_precision_at_1_max": 0.136605, "nauc_precision_at_1_std": -0.356944, "nauc_precision_at_1_diff1": 0.442149, "nauc_precision_at_3_max": 0.019089, "nauc_precision_at_3_std": -0.392202, "nauc_precision_at_3_diff1": 0.286126, "nauc_precision_at_5_max": -0.064852, "nauc_precision_at_5_std": -0.417387, "nauc_precision_at_5_diff1": 0.200319, "nauc_precision_at_10_max": 0.056165, "nauc_precision_at_10_std": -0.20587, "nauc_precision_at_10_diff1": 0.241284, "nauc_precision_at_20_max": 0.031597, "nauc_precision_at_20_std": -0.035821, "nauc_precision_at_20_diff1": 0.273979, "nauc_precision_at_100_max": 0.103483, "nauc_precision_at_100_std": 0.119251, "nauc_precision_at_100_diff1": 0.266627, "nauc_precision_at_1000_max": 0.181452, "nauc_precision_at_1000_std": -0.006465, "nauc_precision_at_1000_diff1": 0.212937, "nauc_mrr_at_1_max": 0.136605, "nauc_mrr_at_1_std": -0.356944, "nauc_mrr_at_1_diff1": 0.442149, "nauc_mrr_at_3_max": 0.092537, "nauc_mrr_at_3_std": -0.370166, "nauc_mrr_at_3_diff1": 0.38364, "nauc_mrr_at_5_max": 0.068644, "nauc_mrr_at_5_std": -0.377334, "nauc_mrr_at_5_diff1": 0.358405, "nauc_mrr_at_10_max": 0.069889, "nauc_mrr_at_10_std": -0.34079, "nauc_mrr_at_10_diff1": 0.338802, "nauc_mrr_at_20_max": 0.05638, "nauc_mrr_at_20_std": -0.289605, "nauc_mrr_at_20_diff1": 0.317826, "nauc_mrr_at_100_max": 0.052917, "nauc_mrr_at_100_std": -0.245364, "nauc_mrr_at_100_diff1": 0.295263, "nauc_mrr_at_1000_max": 0.052537, "nauc_mrr_at_1000_std": -0.251915, "nauc_mrr_at_1000_diff1": 0.299328, "hit_rate_at_1": 0.02326, "hit_rate_at_3": 0.02791, "hit_rate_at_5": 0.03256, "hit_rate_at_10": 0.04651, "hit_rate_at_20": 0.08372, "hit_rate_at_100": 0.33488, "hit_rate_at_1000": 0.98605, "main_score": 0.01151, "hf_subset": "spanish", "languages": [ "spa-Latn" ], "mteb_version": "2.13.1" }, { "ndcg_at_1": 0.05349, "ndcg_at_3": 0.04361, "ndcg_at_5": 0.04297, "ndcg_at_10": 0.05007, "ndcg_at_20": 0.06272, "ndcg_at_100": 0.10674, "ndcg_at_1000": 0.2283, "map_at_1": 0.01494, "map_at_3": 0.02315, "map_at_5": 0.02458, "map_at_10": 0.02815, "map_at_20": 0.03131, "map_at_100": 0.03673, "map_at_1000": 0.04299, "recall_at_1": 0.01494, "recall_at_3": 0.03073, "recall_at_5": 0.0357, "recall_at_10": 0.05528, "recall_at_20": 0.08692, "recall_at_100": 0.26305, "recall_at_1000": 0.92244, "accuracy": 0.01494, "precision_at_1": 0.05581, "precision_at_3": 0.03256, "precision_at_5": 0.02605, "precision_at_10": 0.02326, "precision_at_20": 0.0207, "precision_at_100": 0.01107, "precision_at_1000": 0.00448, "mrr_at_1": 0.055814, "mrr_at_3": 0.069767, "mrr_at_5": 0.074651, "mrr_at_10": 0.084193, "mrr_at_20": 0.090479, "mrr_at_100": 0.098481, "mrr_at_1000": 0.100356, "nauc_ndcg_at_1_max": -0.092534, "nauc_ndcg_at_1_std": -0.094609, "nauc_ndcg_at_1_diff1": 0.229781, "nauc_ndcg_at_3_max": -0.157119, "nauc_ndcg_at_3_std": -0.085717, "nauc_ndcg_at_3_diff1": 0.200345, "nauc_ndcg_at_5_max": -0.157169, "nauc_ndcg_at_5_std": -0.062654, "nauc_ndcg_at_5_diff1": 0.2203, "nauc_ndcg_at_10_max": -0.174939, "nauc_ndcg_at_10_std": -0.024556, "nauc_ndcg_at_10_diff1": 0.270012, "nauc_ndcg_at_20_max": -0.140523, "nauc_ndcg_at_20_std": -0.032164, "nauc_ndcg_at_20_diff1": 0.301644, "nauc_ndcg_at_100_max": -0.204777, "nauc_ndcg_at_100_std": -0.016543, "nauc_ndcg_at_100_diff1": 0.199262, "nauc_ndcg_at_1000_max": -0.047384, "nauc_ndcg_at_1000_std": -0.035582, "nauc_ndcg_at_1000_diff1": 0.281205, "nauc_map_at_1_max": -0.219033, "nauc_map_at_1_std": -0.051951, "nauc_map_at_1_diff1": 0.138093, "nauc_map_at_3_max": -0.233064, "nauc_map_at_3_std": -0.039437, "nauc_map_at_3_diff1": 0.123872, "nauc_map_at_5_max": -0.207706, "nauc_map_at_5_std": -0.032132, "nauc_map_at_5_diff1": 0.152397, "nauc_map_at_10_max": -0.199879, "nauc_map_at_10_std": -0.015679, "nauc_map_at_10_diff1": 0.196922, "nauc_map_at_20_max": -0.186114, "nauc_map_at_20_std": -0.014152, "nauc_map_at_20_diff1": 0.228324, "nauc_map_at_100_max": -0.198017, "nauc_map_at_100_std": -0.00887, "nauc_map_at_100_diff1": 0.216294, "nauc_map_at_1000_max": -0.178026, "nauc_map_at_1000_std": -0.005656, "nauc_map_at_1000_diff1": 0.227367, "nauc_recall_at_1_max": -0.219033, "nauc_recall_at_1_std": -0.051951, "nauc_recall_at_1_diff1": 0.138093, "nauc_recall_at_3_max": -0.215759, "nauc_recall_at_3_std": -0.014173, "nauc_recall_at_3_diff1": 0.139803, "nauc_recall_at_5_max": -0.170729, "nauc_recall_at_5_std": -0.001004, "nauc_recall_at_5_diff1": 0.192857, "nauc_recall_at_10_max": -0.175521, "nauc_recall_at_10_std": 0.041761, "nauc_recall_at_10_diff1": 0.300165, "nauc_recall_at_20_max": -0.125747, "nauc_recall_at_20_std": 0.019048, "nauc_recall_at_20_diff1": 0.337471, "nauc_recall_at_100_max": -0.259151, "nauc_recall_at_100_std": 0.055129, "nauc_recall_at_100_diff1": 0.029956, "nauc_recall_at_1000_max": 0.016412, "nauc_recall_at_1000_std": 0.169863, "nauc_recall_at_1000_diff1": 0.302879, "nauc_precision_at_1_max": -0.109262, "nauc_precision_at_1_std": -0.114354, "nauc_precision_at_1_diff1": 0.217345, "nauc_precision_at_3_max": -0.152568, "nauc_precision_at_3_std": -0.116593, "nauc_precision_at_3_diff1": 0.212442, "nauc_precision_at_5_max": -0.081102, "nauc_precision_at_5_std": -0.066169, "nauc_precision_at_5_diff1": 0.301761, "nauc_precision_at_10_max": -0.071421, "nauc_precision_at_10_std": 0.034796, "nauc_precision_at_10_diff1": 0.432897, "nauc_precision_at_20_max": 0.013414, "nauc_precision_at_20_std": 0.031203, "nauc_precision_at_20_diff1": 0.466247, "nauc_precision_at_100_max": -0.035128, "nauc_precision_at_100_std": 0.02107, "nauc_precision_at_100_diff1": 0.281467, "nauc_precision_at_1000_max": 0.273781, "nauc_precision_at_1000_std": -0.050724, "nauc_precision_at_1000_diff1": 0.25284, "nauc_mrr_at_1_max": -0.109262, "nauc_mrr_at_1_std": -0.114354, "nauc_mrr_at_1_diff1": 0.217345, "nauc_mrr_at_3_max": -0.117999, "nauc_mrr_at_3_std": -0.080121, "nauc_mrr_at_3_diff1": 0.220259, "nauc_mrr_at_5_max": -0.112254, "nauc_mrr_at_5_std": -0.064695, "nauc_mrr_at_5_diff1": 0.227906, "nauc_mrr_at_10_max": -0.117401, "nauc_mrr_at_10_std": -0.0325, "nauc_mrr_at_10_diff1": 0.270737, "nauc_mrr_at_20_max": -0.105791, "nauc_mrr_at_20_std": -0.032827, "nauc_mrr_at_20_diff1": 0.266693, "nauc_mrr_at_100_max": -0.112531, "nauc_mrr_at_100_std": -0.035624, "nauc_mrr_at_100_diff1": 0.252235, "nauc_mrr_at_1000_max": -0.111332, "nauc_mrr_at_1000_std": -0.035493, "nauc_mrr_at_1000_diff1": 0.253068, "hit_rate_at_1": 0.05581, "hit_rate_at_3": 0.08837, "hit_rate_at_5": 0.11163, "hit_rate_at_10": 0.17674, "hit_rate_at_20": 0.26977, "hit_rate_at_100": 0.62326, "hit_rate_at_1000": 0.9907, "main_score": 0.05007, "hf_subset": "english", "languages": [ "eng-Latn" ], "mteb_version": "2.13.1" }, { "ndcg_at_1": 0.0186, "ndcg_at_3": 0.01223, "ndcg_at_5": 0.01068, "ndcg_at_10": 0.01202, "ndcg_at_20": 0.01388, "ndcg_at_100": 0.0385, "ndcg_at_1000": 0.17158, "map_at_1": 0.00285, "map_at_3": 0.00353, "map_at_5": 0.00392, "map_at_10": 0.00494, "map_at_20": 0.00535, "map_at_100": 0.00781, "map_at_1000": 0.01312, "recall_at_1": 0.00285, "recall_at_3": 0.00452, "recall_at_5": 0.00556, "recall_at_10": 0.01136, "recall_at_20": 0.01779, "recall_at_100": 0.11056, "recall_at_1000": 0.86342, "accuracy": 0.00285, "precision_at_1": 0.0186, "precision_at_3": 0.01085, "precision_at_5": 0.00837, "precision_at_10": 0.00744, "precision_at_20": 0.00488, "precision_at_100": 0.00558, "precision_at_1000": 0.00418, "mrr_at_1": 0.018605, "mrr_at_3": 0.024806, "mrr_at_5": 0.025969, "mrr_at_10": 0.028029, "mrr_at_20": 0.02948, "mrr_at_100": 0.034892, "mrr_at_1000": 0.037812, "nauc_ndcg_at_1_max": 0.029914, "nauc_ndcg_at_1_std": -0.31265, "nauc_ndcg_at_1_diff1": 0.689042, "nauc_ndcg_at_3_max": 0.053418, "nauc_ndcg_at_3_std": -0.329527, "nauc_ndcg_at_3_diff1": 0.680955, "nauc_ndcg_at_5_max": 0.031601, "nauc_ndcg_at_5_std": -0.296639, "nauc_ndcg_at_5_diff1": 0.651144, "nauc_ndcg_at_10_max": -0.142078, "nauc_ndcg_at_10_std": -0.273886, "nauc_ndcg_at_10_diff1": 0.604253, "nauc_ndcg_at_20_max": -0.13595, "nauc_ndcg_at_20_std": -0.172258, "nauc_ndcg_at_20_diff1": 0.54151, "nauc_ndcg_at_100_max": -0.151353, "nauc_ndcg_at_100_std": -0.043427, "nauc_ndcg_at_100_diff1": 0.358024, "nauc_ndcg_at_1000_max": -0.084536, "nauc_ndcg_at_1000_std": -0.087054, "nauc_ndcg_at_1000_diff1": 0.366993, "nauc_map_at_1_max": -0.194697, "nauc_map_at_1_std": -0.403666, "nauc_map_at_1_diff1": 0.466217, "nauc_map_at_3_max": -0.169535, "nauc_map_at_3_std": -0.410216, "nauc_map_at_3_diff1": 0.502225, "nauc_map_at_5_max": -0.135608, "nauc_map_at_5_std": -0.363507, "nauc_map_at_5_diff1": 0.52915, "nauc_map_at_10_max": -0.162358, "nauc_map_at_10_std": -0.323799, "nauc_map_at_10_diff1": 0.554771, "nauc_map_at_20_max": -0.161726, "nauc_map_at_20_std": -0.27137, "nauc_map_at_20_diff1": 0.527028, "nauc_map_at_100_max": -0.153105, "nauc_map_at_100_std": -0.174185, "nauc_map_at_100_diff1": 0.463039, "nauc_map_at_1000_max": -0.129737, "nauc_map_at_1000_std": -0.150421, "nauc_map_at_1000_diff1": 0.467736, "nauc_recall_at_1_max": -0.194697, "nauc_recall_at_1_std": -0.403666, "nauc_recall_at_1_diff1": 0.466217, "nauc_recall_at_3_max": -0.170204, "nauc_recall_at_3_std": -0.42519, "nauc_recall_at_3_diff1": 0.535547, "nauc_recall_at_5_max": -0.099323, "nauc_recall_at_5_std": -0.322432, "nauc_recall_at_5_diff1": 0.558995, "nauc_recall_at_10_max": -0.227077, "nauc_recall_at_10_std": -0.212529, "nauc_recall_at_10_diff1": 0.602604, "nauc_recall_at_20_max": -0.188919, "nauc_recall_at_20_std": -0.020967, "nauc_recall_at_20_diff1": 0.468644, "nauc_recall_at_100_max": -0.198626, "nauc_recall_at_100_std": 0.063755, "nauc_recall_at_100_diff1": 0.182831, "nauc_recall_at_1000_max": -0.310581, "nauc_recall_at_1000_std": -0.119803, "nauc_recall_at_1000_diff1": 0.200631, "nauc_precision_at_1_max": 0.029914, "nauc_precision_at_1_std": -0.31265, "nauc_precision_at_1_diff1": 0.689042, "nauc_precision_at_3_max": 0.030754, "nauc_precision_at_3_std": -0.350894, "nauc_precision_at_3_diff1": 0.676717, "nauc_precision_at_5_max": 0.070782, "nauc_precision_at_5_std": -0.244633, "nauc_precision_at_5_diff1": 0.67336, "nauc_precision_at_10_max": -0.077178, "nauc_precision_at_10_std": -0.206845, "nauc_precision_at_10_diff1": 0.576229, "nauc_precision_at_20_max": -0.038539, "nauc_precision_at_20_std": -0.077308, "nauc_precision_at_20_diff1": 0.476196, "nauc_precision_at_100_max": -0.121396, "nauc_precision_at_100_std": 0.06227, "nauc_precision_at_100_diff1": 0.351077, "nauc_precision_at_1000_max": 0.04656, "nauc_precision_at_1000_std": -0.0074, "nauc_precision_at_1000_diff1": 0.128185, "nauc_mrr_at_1_max": 0.029914, "nauc_mrr_at_1_std": -0.31265, "nauc_mrr_at_1_diff1": 0.689042, "nauc_mrr_at_3_max": 0.044225, "nauc_mrr_at_3_std": -0.329752, "nauc_mrr_at_3_diff1": 0.681935, "nauc_mrr_at_5_max": 0.056717, "nauc_mrr_at_5_std": -0.300515, "nauc_mrr_at_5_diff1": 0.665873, "nauc_mrr_at_10_max": 0.017255, "nauc_mrr_at_10_std": -0.295647, "nauc_mrr_at_10_diff1": 0.641174, "nauc_mrr_at_20_max": 0.019132, "nauc_mrr_at_20_std": -0.263318, "nauc_mrr_at_20_diff1": 0.616062, "nauc_mrr_at_100_max": -0.000373, "nauc_mrr_at_100_std": -0.2274, "nauc_mrr_at_100_diff1": 0.577383, "nauc_mrr_at_1000_max": 0.00442, "nauc_mrr_at_1000_std": -0.234141, "nauc_mrr_at_1000_diff1": 0.583454, "hit_rate_at_1": 0.0186, "hit_rate_at_3": 0.03256, "hit_rate_at_5": 0.03721, "hit_rate_at_10": 0.05581, "hit_rate_at_20": 0.07907, "hit_rate_at_100": 0.34419, "hit_rate_at_1000": 0.98605, "main_score": 0.01202, "hf_subset": "german", "languages": [ "deu-Latn" ], "mteb_version": "2.13.1" }, { "ndcg_at_1": 0.02326, "ndcg_at_3": 0.01498, "ndcg_at_5": 0.01496, "ndcg_at_10": 0.01769, "ndcg_at_20": 0.0204, "ndcg_at_100": 0.05156, "ndcg_at_1000": 0.17905, "map_at_1": 0.00324, "map_at_3": 0.00441, "map_at_5": 0.00529, "map_at_10": 0.00673, "map_at_20": 0.00763, "map_at_100": 0.01112, "map_at_1000": 0.01638, "recall_at_1": 0.00324, "recall_at_3": 0.00676, "recall_at_5": 0.01011, "recall_at_10": 0.01923, "recall_at_20": 0.02737, "recall_at_100": 0.14447, "recall_at_1000": 0.86645, "accuracy": 0.00324, "precision_at_1": 0.02326, "precision_at_3": 0.0124, "precision_at_5": 0.01209, "precision_at_10": 0.01163, "precision_at_20": 0.00814, "precision_at_100": 0.00702, "precision_at_1000": 0.0042, "mrr_at_1": 0.023256, "mrr_at_3": 0.027907, "mrr_at_5": 0.032093, "mrr_at_10": 0.036083, "mrr_at_20": 0.037238, "mrr_at_100": 0.04334, "mrr_at_1000": 0.046004, "nauc_ndcg_at_1_max": 0.096601, "nauc_ndcg_at_1_std": -0.343024, "nauc_ndcg_at_1_diff1": 0.727696, "nauc_ndcg_at_3_max": 0.005535, "nauc_ndcg_at_3_std": -0.352078, "nauc_ndcg_at_3_diff1": 0.587886, "nauc_ndcg_at_5_max": 0.022293, "nauc_ndcg_at_5_std": -0.330633, "nauc_ndcg_at_5_diff1": 0.555547, "nauc_ndcg_at_10_max": -0.003373, "nauc_ndcg_at_10_std": -0.206843, "nauc_ndcg_at_10_diff1": 0.469578, "nauc_ndcg_at_20_max": 0.002385, "nauc_ndcg_at_20_std": -0.188525, "nauc_ndcg_at_20_diff1": 0.484819, "nauc_ndcg_at_100_max": -0.015849, "nauc_ndcg_at_100_std": -0.124962, "nauc_ndcg_at_100_diff1": 0.421885, "nauc_ndcg_at_1000_max": 0.069467, "nauc_ndcg_at_1000_std": -0.059197, "nauc_ndcg_at_1000_diff1": 0.400699, "nauc_map_at_1_max": -0.101127, "nauc_map_at_1_std": -0.41842, "nauc_map_at_1_diff1": 0.58993, "nauc_map_at_3_max": -0.186418, "nauc_map_at_3_std": -0.423888, "nauc_map_at_3_diff1": 0.483347, "nauc_map_at_5_max": -0.085043, "nauc_map_at_5_std": -0.398524, "nauc_map_at_5_diff1": 0.500655, "nauc_map_at_10_max": -0.062072, "nauc_map_at_10_std": -0.280783, "nauc_map_at_10_diff1": 0.464668, "nauc_map_at_20_max": -0.054493, "nauc_map_at_20_std": -0.26788, "nauc_map_at_20_diff1": 0.483632, "nauc_map_at_100_max": -0.049299, "nauc_map_at_100_std": -0.212812, "nauc_map_at_100_diff1": 0.470615, "nauc_map_at_1000_max": -0.023895, "nauc_map_at_1000_std": -0.177186, "nauc_map_at_1000_diff1": 0.469654, "nauc_recall_at_1_max": -0.101127, "nauc_recall_at_1_std": -0.41842, "nauc_recall_at_1_diff1": 0.58993, "nauc_recall_at_3_max": -0.268071, "nauc_recall_at_3_std": -0.429122, "nauc_recall_at_3_diff1": 0.381309, "nauc_recall_at_5_max": -0.063575, "nauc_recall_at_5_std": -0.369415, "nauc_recall_at_5_diff1": 0.434947, "nauc_recall_at_10_max": -0.000723, "nauc_recall_at_10_std": -0.133393, "nauc_recall_at_10_diff1": 0.442956, "nauc_recall_at_20_max": -0.003093, "nauc_recall_at_20_std": -0.108312, "nauc_recall_at_20_diff1": 0.455687, "nauc_recall_at_100_max": -0.040677, "nauc_recall_at_100_std": -0.019949, "nauc_recall_at_100_diff1": 0.305985, "nauc_recall_at_1000_max": 0.110932, "nauc_recall_at_1000_std": 0.245625, "nauc_recall_at_1000_diff1": 0.231372, "nauc_precision_at_1_max": 0.096601, "nauc_precision_at_1_std": -0.343024, "nauc_precision_at_1_diff1": 0.727696, "nauc_precision_at_3_max": 0.000456, "nauc_precision_at_3_std": -0.345333, "nauc_precision_at_3_diff1": 0.538101, "nauc_precision_at_5_max": 0.105564, "nauc_precision_at_5_std": -0.250702, "nauc_precision_at_5_diff1": 0.541007, "nauc_precision_at_10_max": 0.125459, "nauc_precision_at_10_std": -0.048568, "nauc_precision_at_10_diff1": 0.465684, "nauc_precision_at_20_max": 0.118417, "nauc_precision_at_20_std": -0.053874, "nauc_precision_at_20_diff1": 0.512282, "nauc_precision_at_100_max": 0.112512, "nauc_precision_at_100_std": -0.00112, "nauc_precision_at_100_diff1": 0.450303, "nauc_precision_at_1000_max": 0.111462, "nauc_precision_at_1000_std": -0.001955, "nauc_precision_at_1000_diff1": 0.171052, "nauc_mrr_at_1_max": 0.096601, "nauc_mrr_at_1_std": -0.343024, "nauc_mrr_at_1_diff1": 0.727696, "nauc_mrr_at_3_max": 0.05387, "nauc_mrr_at_3_std": -0.34405, "nauc_mrr_at_3_diff1": 0.643431, "nauc_mrr_at_5_max": 0.061197, "nauc_mrr_at_5_std": -0.314772, "nauc_mrr_at_5_diff1": 0.615375, "nauc_mrr_at_10_max": 0.083061, "nauc_mrr_at_10_std": -0.263675, "nauc_mrr_at_10_diff1": 0.601992, "nauc_mrr_at_20_max": 0.087129, "nauc_mrr_at_20_std": -0.249663, "nauc_mrr_at_20_diff1": 0.599261, "nauc_mrr_at_100_max": 0.070299, "nauc_mrr_at_100_std": -0.238823, "nauc_mrr_at_100_diff1": 0.573847, "nauc_mrr_at_1000_max": 0.072113, "nauc_mrr_at_1000_std": -0.238592, "nauc_mrr_at_1000_diff1": 0.57583, "hit_rate_at_1": 0.02326, "hit_rate_at_3": 0.03721, "hit_rate_at_5": 0.05581, "hit_rate_at_10": 0.08837, "hit_rate_at_20": 0.10698, "hit_rate_at_100": 0.3907, "hit_rate_at_1000": 0.9814, "main_score": 0.01769, "hf_subset": "italian", "languages": [ "ita-Latn" ], "mteb_version": "2.13.1" }, { "ndcg_at_1": 0.0186, "ndcg_at_3": 0.01105, "ndcg_at_5": 0.00926, "ndcg_at_10": 0.00872, "ndcg_at_20": 0.01446, "ndcg_at_100": 0.04273, "ndcg_at_1000": 0.1731, "map_at_1": 0.00285, "map_at_3": 0.00351, "map_at_5": 0.00371, "map_at_10": 0.00396, "map_at_20": 0.0053, "map_at_100": 0.00845, "map_at_1000": 0.01367, "recall_at_1": 0.00285, "recall_at_3": 0.00417, "recall_at_5": 0.00468, "recall_at_10": 0.00711, "recall_at_20": 0.0231, "recall_at_100": 0.13447, "recall_at_1000": 0.8681, "accuracy": 0.00285, "precision_at_1": 0.0186, "precision_at_3": 0.0093, "precision_at_5": 0.00651, "precision_at_10": 0.00465, "precision_at_20": 0.00581, "precision_at_100": 0.00595, "precision_at_1000": 0.00417, "mrr_at_1": 0.018605, "mrr_at_3": 0.023256, "mrr_at_5": 0.023256, "mrr_at_10": 0.024703, "mrr_at_20": 0.02763, "mrr_at_100": 0.033492, "mrr_at_1000": 0.036318, "nauc_ndcg_at_1_max": 0.211326, "nauc_ndcg_at_1_std": -0.162846, "nauc_ndcg_at_1_diff1": 0.65962, "nauc_ndcg_at_3_max": 0.200797, "nauc_ndcg_at_3_std": -0.21618, "nauc_ndcg_at_3_diff1": 0.576707, "nauc_ndcg_at_5_max": 0.175167, "nauc_ndcg_at_5_std": -0.206167, "nauc_ndcg_at_5_diff1": 0.581699, "nauc_ndcg_at_10_max": 0.057929, "nauc_ndcg_at_10_std": -0.223603, "nauc_ndcg_at_10_diff1": 0.562691, "nauc_ndcg_at_20_max": -0.059128, "nauc_ndcg_at_20_std": -0.268393, "nauc_ndcg_at_20_diff1": 0.490116, "nauc_ndcg_at_100_max": -0.038389, "nauc_ndcg_at_100_std": 0.016178, "nauc_ndcg_at_100_diff1": 0.276634, "nauc_ndcg_at_1000_max": 0.095435, "nauc_ndcg_at_1000_std": 0.077788, "nauc_ndcg_at_1000_diff1": 0.313056, "nauc_map_at_1_max": -0.092905, "nauc_map_at_1_std": -0.306502, "nauc_map_at_1_diff1": 0.537702, "nauc_map_at_3_max": -0.114507, "nauc_map_at_3_std": -0.341695, "nauc_map_at_3_diff1": 0.431294, "nauc_map_at_5_max": -0.071524, "nauc_map_at_5_std": -0.304681, "nauc_map_at_5_diff1": 0.462953, "nauc_map_at_10_max": -0.07727, "nauc_map_at_10_std": -0.29585, "nauc_map_at_10_diff1": 0.478507, "nauc_map_at_20_max": -0.075475, "nauc_map_at_20_std": -0.281787, "nauc_map_at_20_diff1": 0.482595, "nauc_map_at_100_max": -0.074619, "nauc_map_at_100_std": -0.152126, "nauc_map_at_100_diff1": 0.396763, "nauc_map_at_1000_max": -0.024046, "nauc_map_at_1000_std": -0.07914, "nauc_map_at_1000_diff1": 0.40065, "nauc_recall_at_1_max": -0.092905, "nauc_recall_at_1_std": -0.306502, "nauc_recall_at_1_diff1": 0.537702, "nauc_recall_at_3_max": -0.129275, "nauc_recall_at_3_std": -0.365754, "nauc_recall_at_3_diff1": 0.358549, "nauc_recall_at_5_max": -0.042425, "nauc_recall_at_5_std": -0.289713, "nauc_recall_at_5_diff1": 0.429346, "nauc_recall_at_10_max": -0.090361, "nauc_recall_at_10_std": -0.252917, "nauc_recall_at_10_diff1": 0.524468, "nauc_recall_at_20_max": -0.137549, "nauc_recall_at_20_std": -0.297935, "nauc_recall_at_20_diff1": 0.480842, "nauc_recall_at_100_max": -0.082998, "nauc_recall_at_100_std": 0.09342, "nauc_recall_at_100_diff1": 0.119449, "nauc_recall_at_1000_max": 0.14349, "nauc_recall_at_1000_std": 0.303361, "nauc_recall_at_1000_diff1": 0.200582, "nauc_precision_at_1_max": 0.211326, "nauc_precision_at_1_std": -0.162846, "nauc_precision_at_1_diff1": 0.65962, "nauc_precision_at_3_max": 0.155739, "nauc_precision_at_3_std": -0.255733, "nauc_precision_at_3_diff1": 0.496931, "nauc_precision_at_5_max": 0.227438, "nauc_precision_at_5_std": -0.173026, "nauc_precision_at_5_diff1": 0.568798, "nauc_precision_at_10_max": 0.202941, "nauc_precision_at_10_std": -0.093609, "nauc_precision_at_10_diff1": 0.629684, "nauc_precision_at_20_max": -0.018204, "nauc_precision_at_20_std": -0.175021, "nauc_precision_at_20_diff1": 0.471168, "nauc_precision_at_100_max": 0.068277, "nauc_precision_at_100_std": 0.141978, "nauc_precision_at_100_diff1": 0.289391, "nauc_precision_at_1000_max": 0.127103, "nauc_precision_at_1000_std": 0.054864, "nauc_precision_at_1000_diff1": 0.127363, "nauc_mrr_at_1_max": 0.211326, "nauc_mrr_at_1_std": -0.162846, "nauc_mrr_at_1_diff1": 0.65962, "nauc_mrr_at_3_max": 0.177974, "nauc_mrr_at_3_std": -0.218578, "nauc_mrr_at_3_diff1": 0.562006, "nauc_mrr_at_5_max": 0.177974, "nauc_mrr_at_5_std": -0.218578, "nauc_mrr_at_5_diff1": 0.562006, "nauc_mrr_at_10_max": 0.177875, "nauc_mrr_at_10_std": -0.198503, "nauc_mrr_at_10_diff1": 0.57477, "nauc_mrr_at_20_max": 0.127901, "nauc_mrr_at_20_std": -0.207705, "nauc_mrr_at_20_diff1": 0.538804, "nauc_mrr_at_100_max": 0.117988, "nauc_mrr_at_100_std": -0.145184, "nauc_mrr_at_100_diff1": 0.487928, "nauc_mrr_at_1000_max": 0.1204, "nauc_mrr_at_1000_std": -0.148571, "nauc_mrr_at_1000_diff1": 0.493979, "hit_rate_at_1": 0.0186, "hit_rate_at_3": 0.02791, "hit_rate_at_5": 0.02791, "hit_rate_at_10": 0.04186, "hit_rate_at_20": 0.08837, "hit_rate_at_100": 0.35349, "hit_rate_at_1000": 0.98605, "main_score": 0.00872, "hf_subset": "portuguese", "languages": [ "por-Latn" ], "mteb_version": "2.13.1" } ] }, "evaluation_time": 733.0909236446023, "kg_co2_emissions": null, "date": 1783390461.449438, "evaluation_phases": [ { "name": "Data loading", "start": 8.647143840789795e-05, "end": 42.08094298839569, "split": "", "subset": "" }, { "name": "Dataset transform", "start": 42.08101923018694, "end": 42.081024415791035, "split": "", "subset": "" }, { "name": "Encoding queries and documents", "start": 42.09101188182831, "end": 176.09080524742603, "split": "test", "subset": "french" }, { "name": "Scoring", "start": 176.090964846313, "end": 176.30491656810045, "split": "test", "subset": "french" }, { "name": "Encoding queries and documents", "start": 176.33172827214003, "end": 289.04355903714895, "split": "test", "subset": "spanish" }, { "name": "Scoring", "start": 289.04377672076225, "end": 289.2507780343294, "split": "test", "subset": "spanish" }, { "name": "Encoding queries and documents", "start": 289.2803419902921, "end": 401.2081238180399, "split": "test", "subset": "english" }, { "name": "Scoring", "start": 401.2083375006914, "end": 401.41231498122215, "split": "test", "subset": "english" }, { "name": "Encoding queries and documents", "start": 401.44698789715767, "end": 511.14058489352465, "split": "test", "subset": "german" }, { "name": "Scoring", "start": 511.1408085152507, "end": 511.34440882503986, "split": "test", "subset": "german" }, { "name": "Encoding queries and documents", "start": 511.40281496942043, "end": 623.3928972110152, "split": "test", "subset": "italian" }, { "name": "Scoring", "start": 623.3931603431702, "end": 623.6031434312463, "split": "test", "subset": "italian" }, { "name": "Encoding queries and documents", "start": 623.6499029919505, "end": 732.859637670219, "split": "test", "subset": "portuguese" }, { "name": "Scoring", "start": 732.8598637059331, "end": 733.0643790960312, "split": "test", "subset": "portuguese" } ] }