{ "dataset_revision": "4d42cf85d8b8117935c04698b5b7fd3c70c8c9f4", "task_name": "Vidore3ComputerScienceRetrieval.v2", "mteb_version": "2.13.1", "scores": { "test": [ { "ndcg_at_1": 0.05814, "ndcg_at_3": 0.06484, "ndcg_at_5": 0.07252, "ndcg_at_10": 0.07696, "ndcg_at_20": 0.08593, "ndcg_at_100": 0.12072, "ndcg_at_1000": 0.22841, "map_at_1": 0.02598, "map_at_3": 0.03889, "map_at_5": 0.04464, "map_at_10": 0.04783, "map_at_20": 0.05036, "map_at_100": 0.05508, "map_at_1000": 0.05981, "recall_at_1": 0.02598, "recall_at_3": 0.05124, "recall_at_5": 0.07249, "recall_at_10": 0.08898, "recall_at_20": 0.11822, "recall_at_100": 0.24331, "recall_at_1000": 0.84842, "accuracy": 0.02598, "precision_at_1": 0.06977, "precision_at_3": 0.05581, "precision_at_5": 0.0493, "precision_at_10": 0.03395, "precision_at_20": 0.02256, "precision_at_100": 0.0106, "precision_at_1000": 0.00413, "mrr_at_1": 0.069767, "mrr_at_3": 0.100775, "mrr_at_5": 0.114496, "mrr_at_10": 0.119528, "mrr_at_20": 0.123897, "mrr_at_100": 0.12963, "mrr_at_1000": 0.131355, "nauc_ndcg_at_1_max": 0.261769, "nauc_ndcg_at_1_std": -0.090976, "nauc_ndcg_at_1_diff1": 0.423653, "nauc_ndcg_at_3_max": 0.156746, "nauc_ndcg_at_3_std": -0.114931, "nauc_ndcg_at_3_diff1": 0.298448, "nauc_ndcg_at_5_max": 0.122147, "nauc_ndcg_at_5_std": -0.164585, "nauc_ndcg_at_5_diff1": 0.251386, "nauc_ndcg_at_10_max": 0.09739, "nauc_ndcg_at_10_std": -0.184436, "nauc_ndcg_at_10_diff1": 0.263956, "nauc_ndcg_at_20_max": 0.090439, "nauc_ndcg_at_20_std": -0.182185, "nauc_ndcg_at_20_diff1": 0.270706, "nauc_ndcg_at_100_max": 0.100462, "nauc_ndcg_at_100_std": -0.162016, "nauc_ndcg_at_100_diff1": 0.196619, "nauc_ndcg_at_1000_max": 0.12001, "nauc_ndcg_at_1000_std": -0.110237, "nauc_ndcg_at_1000_diff1": 0.157273, "nauc_map_at_1_max": 0.187544, "nauc_map_at_1_std": -0.166477, "nauc_map_at_1_diff1": 0.41569, "nauc_map_at_3_max": 0.136441, "nauc_map_at_3_std": -0.17148, "nauc_map_at_3_diff1": 0.348075, "nauc_map_at_5_max": 0.11544, "nauc_map_at_5_std": -0.196345, "nauc_map_at_5_diff1": 0.311083, "nauc_map_at_10_max": 0.10529, "nauc_map_at_10_std": -0.19711, "nauc_map_at_10_diff1": 0.314451, "nauc_map_at_20_max": 0.109594, "nauc_map_at_20_std": -0.1926, "nauc_map_at_20_diff1": 0.316249, "nauc_map_at_100_max": 0.112137, "nauc_map_at_100_std": -0.190124, "nauc_map_at_100_diff1": 0.290768, "nauc_map_at_1000_max": 0.118047, "nauc_map_at_1000_std": -0.180382, "nauc_map_at_1000_diff1": 0.281285, "nauc_recall_at_1_max": 0.187544, "nauc_recall_at_1_std": -0.166477, "nauc_recall_at_1_diff1": 0.41569, "nauc_recall_at_3_max": 0.092246, "nauc_recall_at_3_std": -0.159993, "nauc_recall_at_3_diff1": 0.28402, "nauc_recall_at_5_max": 0.032997, "nauc_recall_at_5_std": -0.230638, "nauc_recall_at_5_diff1": 0.213841, "nauc_recall_at_10_max": 0.012099, "nauc_recall_at_10_std": -0.240374, "nauc_recall_at_10_diff1": 0.22654, "nauc_recall_at_20_max": 0.021538, "nauc_recall_at_20_std": -0.235655, "nauc_recall_at_20_diff1": 0.254222, "nauc_recall_at_100_max": 0.046288, "nauc_recall_at_100_std": -0.154205, "nauc_recall_at_100_diff1": 0.074331, "nauc_recall_at_1000_max": -0.05173, "nauc_recall_at_1000_std": -0.041885, "nauc_recall_at_1000_diff1": -0.155983, "nauc_precision_at_1_max": 0.232578, "nauc_precision_at_1_std": -0.11834, "nauc_precision_at_1_diff1": 0.371628, "nauc_precision_at_3_max": 0.154844, "nauc_precision_at_3_std": -0.11759, "nauc_precision_at_3_diff1": 0.189065, "nauc_precision_at_5_max": 0.136568, "nauc_precision_at_5_std": -0.146934, "nauc_precision_at_5_diff1": 0.13037, "nauc_precision_at_10_max": 0.129903, "nauc_precision_at_10_std": -0.129151, "nauc_precision_at_10_diff1": 0.155041, "nauc_precision_at_20_max": 0.140646, "nauc_precision_at_20_std": -0.089626, "nauc_precision_at_20_diff1": 0.11309, "nauc_precision_at_100_max": 0.169437, "nauc_precision_at_100_std": -0.05291, "nauc_precision_at_100_diff1": -0.047984, "nauc_precision_at_1000_max": 0.134192, "nauc_precision_at_1000_std": 0.097188, "nauc_precision_at_1000_diff1": -0.112416, "nauc_mrr_at_1_max": 0.232578, "nauc_mrr_at_1_std": -0.11834, "nauc_mrr_at_1_diff1": 0.371628, "nauc_mrr_at_3_max": 0.161011, "nauc_mrr_at_3_std": -0.139275, "nauc_mrr_at_3_diff1": 0.230324, "nauc_mrr_at_5_max": 0.136865, "nauc_mrr_at_5_std": -0.156738, "nauc_mrr_at_5_diff1": 0.209203, "nauc_mrr_at_10_max": 0.141517, "nauc_mrr_at_10_std": -0.154381, "nauc_mrr_at_10_diff1": 0.212207, "nauc_mrr_at_20_max": 0.138743, "nauc_mrr_at_20_std": -0.152602, "nauc_mrr_at_20_diff1": 0.210197, "nauc_mrr_at_100_max": 0.137446, "nauc_mrr_at_100_std": -0.150764, "nauc_mrr_at_100_diff1": 0.208823, "nauc_mrr_at_1000_max": 0.137953, "nauc_mrr_at_1000_std": -0.150814, "nauc_mrr_at_1000_diff1": 0.209952, "hit_rate_at_1": 0.06977, "hit_rate_at_3": 0.13953, "hit_rate_at_5": 0.2, "hit_rate_at_10": 0.24186, "hit_rate_at_20": 0.30698, "hit_rate_at_100": 0.55814, "hit_rate_at_1000": 0.96744, "main_score": 0.07696, "hf_subset": "french", "languages": [ "fra-Latn" ], "mteb_version": "2.13.1" }, { "ndcg_at_1": 0.07209, "ndcg_at_3": 0.07685, "ndcg_at_5": 0.07758, "ndcg_at_10": 0.08439, "ndcg_at_20": 0.09384, "ndcg_at_100": 0.13301, "ndcg_at_1000": 0.23892, "map_at_1": 0.02223, "map_at_3": 0.03801, "map_at_5": 0.04121, "map_at_10": 0.04652, "map_at_20": 0.04935, "map_at_100": 0.05505, "map_at_1000": 0.05984, "recall_at_1": 0.02223, "recall_at_3": 0.05331, "recall_at_5": 0.06597, "recall_at_10": 0.09539, "recall_at_20": 0.12392, "recall_at_100": 0.27147, "recall_at_1000": 0.86606, "accuracy": 0.02223, "precision_at_1": 0.07907, "precision_at_3": 0.06822, "precision_at_5": 0.05395, "precision_at_10": 0.03814, "precision_at_20": 0.02558, "precision_at_100": 0.01167, "precision_at_1000": 0.00419, "mrr_at_1": 0.07907, "mrr_at_3": 0.121705, "mrr_at_5": 0.134496, "mrr_at_10": 0.142375, "mrr_at_20": 0.14628, "mrr_at_100": 0.152415, "mrr_at_1000": 0.153995, "nauc_ndcg_at_1_max": 0.062603, "nauc_ndcg_at_1_std": -0.126848, "nauc_ndcg_at_1_diff1": 0.21529, "nauc_ndcg_at_3_max": 0.072157, "nauc_ndcg_at_3_std": -0.084465, "nauc_ndcg_at_3_diff1": 0.132797, "nauc_ndcg_at_5_max": 0.086448, "nauc_ndcg_at_5_std": -0.061309, "nauc_ndcg_at_5_diff1": 0.115542, "nauc_ndcg_at_10_max": 0.087468, "nauc_ndcg_at_10_std": -0.095542, "nauc_ndcg_at_10_diff1": 0.082803, "nauc_ndcg_at_20_max": 0.061751, "nauc_ndcg_at_20_std": -0.101258, "nauc_ndcg_at_20_diff1": 0.079007, "nauc_ndcg_at_100_max": 0.113468, "nauc_ndcg_at_100_std": -0.085132, "nauc_ndcg_at_100_diff1": 0.046304, "nauc_ndcg_at_1000_max": 0.139598, "nauc_ndcg_at_1000_std": -0.002664, "nauc_ndcg_at_1000_diff1": 0.070738, "nauc_map_at_1_max": 0.150785, "nauc_map_at_1_std": -0.118409, "nauc_map_at_1_diff1": 0.407706, "nauc_map_at_3_max": 0.060246, "nauc_map_at_3_std": -0.119815, "nauc_map_at_3_diff1": 0.202649, "nauc_map_at_5_max": 0.072292, "nauc_map_at_5_std": -0.106062, "nauc_map_at_5_diff1": 0.195982, "nauc_map_at_10_max": 0.079369, "nauc_map_at_10_std": -0.126687, "nauc_map_at_10_diff1": 0.161277, "nauc_map_at_20_max": 0.069674, "nauc_map_at_20_std": -0.130057, "nauc_map_at_20_diff1": 0.153628, "nauc_map_at_100_max": 0.093391, "nauc_map_at_100_std": -0.118016, "nauc_map_at_100_diff1": 0.138788, "nauc_map_at_1000_max": 0.100401, "nauc_map_at_1000_std": -0.106329, "nauc_map_at_1000_diff1": 0.140698, "nauc_recall_at_1_max": 0.150785, "nauc_recall_at_1_std": -0.118409, "nauc_recall_at_1_diff1": 0.407706, "nauc_recall_at_3_max": 0.034398, "nauc_recall_at_3_std": -0.072084, "nauc_recall_at_3_diff1": 0.087851, "nauc_recall_at_5_max": 0.052498, "nauc_recall_at_5_std": -0.052568, "nauc_recall_at_5_diff1": 0.074924, "nauc_recall_at_10_max": 0.044659, "nauc_recall_at_10_std": -0.140188, "nauc_recall_at_10_diff1": -0.029754, "nauc_recall_at_20_max": 0.009372, "nauc_recall_at_20_std": -0.120274, "nauc_recall_at_20_diff1": -0.025648, "nauc_recall_at_100_max": 0.090058, "nauc_recall_at_100_std": -0.107338, "nauc_recall_at_100_diff1": -0.050591, "nauc_recall_at_1000_max": 0.06312, "nauc_recall_at_1000_std": 0.132934, "nauc_recall_at_1000_diff1": 0.019508, "nauc_precision_at_1_max": 0.068429, "nauc_precision_at_1_std": -0.126847, "nauc_precision_at_1_diff1": 0.22924, "nauc_precision_at_3_max": 0.078178, "nauc_precision_at_3_std": -0.096911, "nauc_precision_at_3_diff1": 0.098512, "nauc_precision_at_5_max": 0.174553, "nauc_precision_at_5_std": -0.013593, "nauc_precision_at_5_diff1": 0.076611, "nauc_precision_at_10_max": 0.204863, "nauc_precision_at_10_std": -0.055604, "nauc_precision_at_10_diff1": 0.030571, "nauc_precision_at_20_max": 0.136724, "nauc_precision_at_20_std": -0.068679, "nauc_precision_at_20_diff1": 0.025786, "nauc_precision_at_100_max": 0.259466, "nauc_precision_at_100_std": 0.045502, "nauc_precision_at_100_diff1": -0.010815, "nauc_precision_at_1000_max": 0.181025, "nauc_precision_at_1000_std": 0.183987, "nauc_precision_at_1000_diff1": -0.009705, "nauc_mrr_at_1_max": 0.068429, "nauc_mrr_at_1_std": -0.126847, "nauc_mrr_at_1_diff1": 0.22924, "nauc_mrr_at_3_max": 0.087563, "nauc_mrr_at_3_std": -0.10115, "nauc_mrr_at_3_diff1": 0.151776, "nauc_mrr_at_5_max": 0.10994, "nauc_mrr_at_5_std": -0.080319, "nauc_mrr_at_5_diff1": 0.134554, "nauc_mrr_at_10_max": 0.105605, "nauc_mrr_at_10_std": -0.099012, "nauc_mrr_at_10_diff1": 0.120063, "nauc_mrr_at_20_max": 0.099203, "nauc_mrr_at_20_std": -0.099135, "nauc_mrr_at_20_diff1": 0.121271, "nauc_mrr_at_100_max": 0.103767, "nauc_mrr_at_100_std": -0.095905, "nauc_mrr_at_100_diff1": 0.11793, "nauc_mrr_at_1000_max": 0.103554, "nauc_mrr_at_1000_std": -0.096155, "nauc_mrr_at_1000_diff1": 0.119604, "hit_rate_at_1": 0.07907, "hit_rate_at_3": 0.17209, "hit_rate_at_5": 0.22791, "hit_rate_at_10": 0.28372, "hit_rate_at_20": 0.33953, "hit_rate_at_100": 0.59535, "hit_rate_at_1000": 0.97209, "main_score": 0.08439, "hf_subset": "spanish", "languages": [ "spa-Latn" ], "mteb_version": "2.13.1" }, { "ndcg_at_1": 0.1814, "ndcg_at_3": 0.17093, "ndcg_at_5": 0.17063, "ndcg_at_10": 0.17652, "ndcg_at_20": 0.19224, "ndcg_at_100": 0.24233, "ndcg_at_1000": 0.33211, "map_at_1": 0.0715, "map_at_3": 0.0967, "map_at_5": 0.10446, "map_at_10": 0.11226, "map_at_20": 0.11774, "map_at_100": 0.12605, "map_at_1000": 0.13147, "recall_at_1": 0.0715, "recall_at_3": 0.11589, "recall_at_5": 0.14328, "recall_at_10": 0.18355, "recall_at_20": 0.23666, "recall_at_100": 0.41816, "recall_at_1000": 0.90005, "accuracy": 0.0715, "precision_at_1": 0.20465, "precision_at_3": 0.13643, "precision_at_5": 0.10698, "precision_at_10": 0.07116, "precision_at_20": 0.04651, "precision_at_100": 0.01828, "precision_at_1000": 0.0044, "mrr_at_1": 0.204651, "mrr_at_3": 0.251163, "mrr_at_5": 0.270465, "mrr_at_10": 0.282187, "mrr_at_20": 0.287312, "mrr_at_100": 0.293637, "mrr_at_1000": 0.294333, "nauc_ndcg_at_1_max": 0.123691, "nauc_ndcg_at_1_std": -0.023563, "nauc_ndcg_at_1_diff1": 0.41026, "nauc_ndcg_at_3_max": 0.124589, "nauc_ndcg_at_3_std": -0.039545, "nauc_ndcg_at_3_diff1": 0.360921, "nauc_ndcg_at_5_max": 0.117208, "nauc_ndcg_at_5_std": -0.050991, "nauc_ndcg_at_5_diff1": 0.352626, "nauc_ndcg_at_10_max": 0.101449, "nauc_ndcg_at_10_std": -0.051832, "nauc_ndcg_at_10_diff1": 0.33349, "nauc_ndcg_at_20_max": 0.090312, "nauc_ndcg_at_20_std": -0.03646, "nauc_ndcg_at_20_diff1": 0.343919, "nauc_ndcg_at_100_max": 0.100519, "nauc_ndcg_at_100_std": -0.014019, "nauc_ndcg_at_100_diff1": 0.304392, "nauc_ndcg_at_1000_max": 0.138526, "nauc_ndcg_at_1000_std": 0.009977, "nauc_ndcg_at_1000_diff1": 0.306811, "nauc_map_at_1_max": 0.005188, "nauc_map_at_1_std": -0.120239, "nauc_map_at_1_diff1": 0.365926, "nauc_map_at_3_max": 0.024622, "nauc_map_at_3_std": -0.109665, "nauc_map_at_3_diff1": 0.342187, "nauc_map_at_5_max": 0.037988, "nauc_map_at_5_std": -0.101426, "nauc_map_at_5_diff1": 0.338842, "nauc_map_at_10_max": 0.040472, "nauc_map_at_10_std": -0.107508, "nauc_map_at_10_diff1": 0.333773, "nauc_map_at_20_max": 0.044969, "nauc_map_at_20_std": -0.098806, "nauc_map_at_20_diff1": 0.338125, "nauc_map_at_100_max": 0.047317, "nauc_map_at_100_std": -0.092585, "nauc_map_at_100_diff1": 0.324275, "nauc_map_at_1000_max": 0.054803, "nauc_map_at_1000_std": -0.086525, "nauc_map_at_1000_diff1": 0.322262, "nauc_recall_at_1_max": 0.005188, "nauc_recall_at_1_std": -0.120239, "nauc_recall_at_1_diff1": 0.365926, "nauc_recall_at_3_max": -0.00655, "nauc_recall_at_3_std": -0.120084, "nauc_recall_at_3_diff1": 0.304483, "nauc_recall_at_5_max": 0.001084, "nauc_recall_at_5_std": -0.107474, "nauc_recall_at_5_diff1": 0.276983, "nauc_recall_at_10_max": 0.035672, "nauc_recall_at_10_std": -0.082247, "nauc_recall_at_10_diff1": 0.239368, "nauc_recall_at_20_max": 0.001592, "nauc_recall_at_20_std": -0.022711, "nauc_recall_at_20_diff1": 0.245847, "nauc_recall_at_100_max": 0.016081, "nauc_recall_at_100_std": 0.065978, "nauc_recall_at_100_diff1": 0.083092, "nauc_recall_at_1000_max": -0.042742, "nauc_recall_at_1000_std": 0.27572, "nauc_recall_at_1000_diff1": -0.00451, "nauc_precision_at_1_max": 0.136042, "nauc_precision_at_1_std": -0.029926, "nauc_precision_at_1_diff1": 0.395941, "nauc_precision_at_3_max": 0.18652, "nauc_precision_at_3_std": -0.0015, "nauc_precision_at_3_diff1": 0.292491, "nauc_precision_at_5_max": 0.25002, "nauc_precision_at_5_std": 0.044419, "nauc_precision_at_5_diff1": 0.263384, "nauc_precision_at_10_max": 0.266396, "nauc_precision_at_10_std": 0.051967, "nauc_precision_at_10_diff1": 0.183066, "nauc_precision_at_20_max": 0.258806, "nauc_precision_at_20_std": 0.063927, "nauc_precision_at_20_diff1": 0.187142, "nauc_precision_at_100_max": 0.23595, "nauc_precision_at_100_std": 0.113355, "nauc_precision_at_100_diff1": 0.019111, "nauc_precision_at_1000_max": 0.28958, "nauc_precision_at_1000_std": 0.142547, "nauc_precision_at_1000_diff1": -0.007917, "nauc_mrr_at_1_max": 0.136042, "nauc_mrr_at_1_std": -0.029926, "nauc_mrr_at_1_diff1": 0.395941, "nauc_mrr_at_3_max": 0.16221, "nauc_mrr_at_3_std": -0.022444, "nauc_mrr_at_3_diff1": 0.374433, "nauc_mrr_at_5_max": 0.165357, "nauc_mrr_at_5_std": -0.019961, "nauc_mrr_at_5_diff1": 0.366593, "nauc_mrr_at_10_max": 0.173226, "nauc_mrr_at_10_std": -0.005263, "nauc_mrr_at_10_diff1": 0.356369, "nauc_mrr_at_20_max": 0.165807, "nauc_mrr_at_20_std": -0.007732, "nauc_mrr_at_20_diff1": 0.359235, "nauc_mrr_at_100_max": 0.16813, "nauc_mrr_at_100_std": -0.0076, "nauc_mrr_at_100_diff1": 0.359213, "nauc_mrr_at_1000_max": 0.167947, "nauc_mrr_at_1000_std": -0.008152, "nauc_mrr_at_1000_diff1": 0.359503, "hit_rate_at_1": 0.20465, "hit_rate_at_3": 0.30698, "hit_rate_at_5": 0.3907, "hit_rate_at_10": 0.47907, "hit_rate_at_20": 0.55814, "hit_rate_at_100": 0.8093, "hit_rate_at_1000": 0.97209, "main_score": 0.17652, "hf_subset": "english", "languages": [ "eng-Latn" ], "mteb_version": "2.13.1" }, { "ndcg_at_1": 0.03953, "ndcg_at_3": 0.04581, "ndcg_at_5": 0.04659, "ndcg_at_10": 0.05076, "ndcg_at_20": 0.0629, "ndcg_at_100": 0.0958, "ndcg_at_1000": 0.20806, "map_at_1": 0.01617, "map_at_3": 0.02444, "map_at_5": 0.02654, "map_at_10": 0.02893, "map_at_20": 0.03205, "map_at_100": 0.03636, "map_at_1000": 0.04117, "recall_at_1": 0.01617, "recall_at_3": 0.03313, "recall_at_5": 0.04105, "recall_at_10": 0.05577, "recall_at_20": 0.09134, "recall_at_100": 0.20941, "recall_at_1000": 0.84398, "accuracy": 0.01617, "precision_at_1": 0.04651, "precision_at_3": 0.04031, "precision_at_5": 0.0307, "precision_at_10": 0.02186, "precision_at_20": 0.01744, "precision_at_100": 0.00902, "precision_at_1000": 0.0041, "mrr_at_1": 0.046512, "mrr_at_3": 0.073643, "mrr_at_5": 0.079922, "mrr_at_10": 0.086355, "mrr_at_20": 0.093408, "mrr_at_100": 0.099715, "mrr_at_1000": 0.101548, "nauc_ndcg_at_1_max": 0.31328, "nauc_ndcg_at_1_std": 0.145177, "nauc_ndcg_at_1_diff1": 0.15971, "nauc_ndcg_at_3_max": 0.277554, "nauc_ndcg_at_3_std": 0.036042, "nauc_ndcg_at_3_diff1": 0.094067, "nauc_ndcg_at_5_max": 0.253959, "nauc_ndcg_at_5_std": 0.019502, "nauc_ndcg_at_5_diff1": 0.111891, "nauc_ndcg_at_10_max": 0.255504, "nauc_ndcg_at_10_std": -0.000918, "nauc_ndcg_at_10_diff1": 0.072012, "nauc_ndcg_at_20_max": 0.228637, "nauc_ndcg_at_20_std": -0.009575, "nauc_ndcg_at_20_diff1": 0.115959, "nauc_ndcg_at_100_max": 0.237865, "nauc_ndcg_at_100_std": -0.01234, "nauc_ndcg_at_100_diff1": 0.124933, "nauc_ndcg_at_1000_max": 0.290169, "nauc_ndcg_at_1000_std": 0.098689, "nauc_ndcg_at_1000_diff1": 0.121219, "nauc_map_at_1_max": 0.384157, "nauc_map_at_1_std": 0.027635, "nauc_map_at_1_diff1": 0.230917, "nauc_map_at_3_max": 0.344134, "nauc_map_at_3_std": -0.008791, "nauc_map_at_3_diff1": 0.135538, "nauc_map_at_5_max": 0.325653, "nauc_map_at_5_std": -0.010442, "nauc_map_at_5_diff1": 0.139431, "nauc_map_at_10_max": 0.319439, "nauc_map_at_10_std": -0.010205, "nauc_map_at_10_diff1": 0.1163, "nauc_map_at_20_max": 0.307992, "nauc_map_at_20_std": -0.010527, "nauc_map_at_20_diff1": 0.140252, "nauc_map_at_100_max": 0.310012, "nauc_map_at_100_std": -0.007963, "nauc_map_at_100_diff1": 0.150251, "nauc_map_at_1000_max": 0.31548, "nauc_map_at_1000_std": 0.006568, "nauc_map_at_1000_diff1": 0.153137, "nauc_recall_at_1_max": 0.384157, "nauc_recall_at_1_std": 0.027635, "nauc_recall_at_1_diff1": 0.230917, "nauc_recall_at_3_max": 0.305819, "nauc_recall_at_3_std": -0.049464, "nauc_recall_at_3_diff1": 0.062169, "nauc_recall_at_5_max": 0.234577, "nauc_recall_at_5_std": -0.062069, "nauc_recall_at_5_diff1": 0.063922, "nauc_recall_at_10_max": 0.219658, "nauc_recall_at_10_std": -0.054969, "nauc_recall_at_10_diff1": 0.004452, "nauc_recall_at_20_max": 0.172822, "nauc_recall_at_20_std": -0.03415, "nauc_recall_at_20_diff1": 0.121959, "nauc_recall_at_100_max": 0.135614, "nauc_recall_at_100_std": -0.088601, "nauc_recall_at_100_diff1": 0.080292, "nauc_recall_at_1000_max": 0.143941, "nauc_recall_at_1000_std": 0.196066, "nauc_recall_at_1000_diff1": -0.018088, "nauc_precision_at_1_max": 0.305489, "nauc_precision_at_1_std": 0.120402, "nauc_precision_at_1_diff1": 0.113337, "nauc_precision_at_3_max": 0.261131, "nauc_precision_at_3_std": 0.017811, "nauc_precision_at_3_diff1": 0.057578, "nauc_precision_at_5_max": 0.239675, "nauc_precision_at_5_std": 0.044557, "nauc_precision_at_5_diff1": 0.107593, "nauc_precision_at_10_max": 0.244166, "nauc_precision_at_10_std": 0.016494, "nauc_precision_at_10_diff1": 0.045119, "nauc_precision_at_20_max": 0.227673, "nauc_precision_at_20_std": 0.000499, "nauc_precision_at_20_diff1": 0.122701, "nauc_precision_at_100_max": 0.25251, "nauc_precision_at_100_std": 0.071222, "nauc_precision_at_100_diff1": 0.173406, "nauc_precision_at_1000_max": 0.221991, "nauc_precision_at_1000_std": 0.166349, "nauc_precision_at_1000_diff1": 0.0826, "nauc_mrr_at_1_max": 0.305489, "nauc_mrr_at_1_std": 0.120402, "nauc_mrr_at_1_diff1": 0.113337, "nauc_mrr_at_3_max": 0.233092, "nauc_mrr_at_3_std": -0.006452, "nauc_mrr_at_3_diff1": 0.056054, "nauc_mrr_at_5_max": 0.214767, "nauc_mrr_at_5_std": -0.005184, "nauc_mrr_at_5_diff1": 0.070265, "nauc_mrr_at_10_max": 0.213685, "nauc_mrr_at_10_std": -0.001043, "nauc_mrr_at_10_diff1": 0.056852, "nauc_mrr_at_20_max": 0.20794, "nauc_mrr_at_20_std": -0.002036, "nauc_mrr_at_20_diff1": 0.066064, "nauc_mrr_at_100_max": 0.21311, "nauc_mrr_at_100_std": 0.002667, "nauc_mrr_at_100_diff1": 0.070514, "nauc_mrr_at_1000_max": 0.214002, "nauc_mrr_at_1000_std": 0.004291, "nauc_mrr_at_1000_diff1": 0.070848, "hit_rate_at_1": 0.04651, "hit_rate_at_3": 0.10698, "hit_rate_at_5": 0.13488, "hit_rate_at_10": 0.1814, "hit_rate_at_20": 0.28372, "hit_rate_at_100": 0.52558, "hit_rate_at_1000": 0.96744, "main_score": 0.05076, "hf_subset": "german", "languages": [ "deu-Latn" ], "mteb_version": "2.13.1" }, { "ndcg_at_1": 0.05581, "ndcg_at_3": 0.06087, "ndcg_at_5": 0.05993, "ndcg_at_10": 0.06641, "ndcg_at_20": 0.07706, "ndcg_at_100": 0.1137, "ndcg_at_1000": 0.22162, "map_at_1": 0.01756, "map_at_3": 0.02957, "map_at_5": 0.03232, "map_at_10": 0.03615, "map_at_20": 0.03962, "map_at_100": 0.04545, "map_at_1000": 0.0502, "recall_at_1": 0.01756, "recall_at_3": 0.04271, "recall_at_5": 0.05162, "recall_at_10": 0.07349, "recall_at_20": 0.1048, "recall_at_100": 0.24338, "recall_at_1000": 0.84595, "accuracy": 0.01756, "precision_at_1": 0.06977, "precision_at_3": 0.05426, "precision_at_5": 0.04279, "precision_at_10": 0.03163, "precision_at_20": 0.02233, "precision_at_100": 0.0106, "precision_at_1000": 0.00412, "mrr_at_1": 0.069767, "mrr_at_3": 0.103876, "mrr_at_5": 0.111085, "mrr_at_10": 0.119352, "mrr_at_20": 0.123457, "mrr_at_100": 0.129019, "mrr_at_1000": 0.130889, "nauc_ndcg_at_1_max": 0.038982, "nauc_ndcg_at_1_std": 0.025088, "nauc_ndcg_at_1_diff1": 0.207781, "nauc_ndcg_at_3_max": 0.017438, "nauc_ndcg_at_3_std": -0.012398, "nauc_ndcg_at_3_diff1": 0.238735, "nauc_ndcg_at_5_max": -0.007203, "nauc_ndcg_at_5_std": -0.046878, "nauc_ndcg_at_5_diff1": 0.255248, "nauc_ndcg_at_10_max": -0.041192, "nauc_ndcg_at_10_std": -0.081993, "nauc_ndcg_at_10_diff1": 0.203646, "nauc_ndcg_at_20_max": -0.006559, "nauc_ndcg_at_20_std": -0.074983, "nauc_ndcg_at_20_diff1": 0.212562, "nauc_ndcg_at_100_max": 0.051753, "nauc_ndcg_at_100_std": -0.060703, "nauc_ndcg_at_100_diff1": 0.214043, "nauc_ndcg_at_1000_max": 0.03691, "nauc_ndcg_at_1000_std": -0.005855, "nauc_ndcg_at_1000_diff1": 0.211313, "nauc_map_at_1_max": 0.135631, "nauc_map_at_1_std": -0.111196, "nauc_map_at_1_diff1": 0.43676, "nauc_map_at_3_max": 0.033262, "nauc_map_at_3_std": -0.077546, "nauc_map_at_3_diff1": 0.334755, "nauc_map_at_5_max": 0.028071, "nauc_map_at_5_std": -0.074136, "nauc_map_at_5_diff1": 0.328253, "nauc_map_at_10_max": 0.010058, "nauc_map_at_10_std": -0.08721, "nauc_map_at_10_diff1": 0.289566, "nauc_map_at_20_max": 0.025886, "nauc_map_at_20_std": -0.08221, "nauc_map_at_20_diff1": 0.300887, "nauc_map_at_100_max": 0.041574, "nauc_map_at_100_std": -0.078025, "nauc_map_at_100_diff1": 0.288486, "nauc_map_at_1000_max": 0.044547, "nauc_map_at_1000_std": -0.071104, "nauc_map_at_1000_diff1": 0.286561, "nauc_recall_at_1_max": 0.135631, "nauc_recall_at_1_std": -0.111196, "nauc_recall_at_1_diff1": 0.43676, "nauc_recall_at_3_max": -0.027066, "nauc_recall_at_3_std": -0.076398, "nauc_recall_at_3_diff1": 0.239289, "nauc_recall_at_5_max": -0.048414, "nauc_recall_at_5_std": -0.087371, "nauc_recall_at_5_diff1": 0.220026, "nauc_recall_at_10_max": -0.103778, "nauc_recall_at_10_std": -0.140633, "nauc_recall_at_10_diff1": 0.134257, "nauc_recall_at_20_max": -0.038071, "nauc_recall_at_20_std": -0.107397, "nauc_recall_at_20_diff1": 0.166763, "nauc_recall_at_100_max": 0.058019, "nauc_recall_at_100_std": -0.077144, "nauc_recall_at_100_diff1": 0.145013, "nauc_recall_at_1000_max": -0.080637, "nauc_recall_at_1000_std": 0.068473, "nauc_recall_at_1000_diff1": 0.097533, "nauc_precision_at_1_max": -0.010046, "nauc_precision_at_1_std": -0.008349, "nauc_precision_at_1_diff1": 0.260765, "nauc_precision_at_3_max": 0.035511, "nauc_precision_at_3_std": -0.010291, "nauc_precision_at_3_diff1": 0.238259, "nauc_precision_at_5_max": 0.010311, "nauc_precision_at_5_std": -0.033295, "nauc_precision_at_5_diff1": 0.247266, "nauc_precision_at_10_max": -0.0342, "nauc_precision_at_10_std": -0.079849, "nauc_precision_at_10_diff1": 0.130847, "nauc_precision_at_20_max": 0.079323, "nauc_precision_at_20_std": -0.065629, "nauc_precision_at_20_diff1": 0.198724, "nauc_precision_at_100_max": 0.187433, "nauc_precision_at_100_std": -0.011654, "nauc_precision_at_100_diff1": 0.177159, "nauc_precision_at_1000_max": 0.095811, "nauc_precision_at_1000_std": 0.079559, "nauc_precision_at_1000_diff1": 0.11589, "nauc_mrr_at_1_max": -0.010046, "nauc_mrr_at_1_std": -0.008349, "nauc_mrr_at_1_diff1": 0.260765, "nauc_mrr_at_3_max": -0.014558, "nauc_mrr_at_3_std": -0.037472, "nauc_mrr_at_3_diff1": 0.213626, "nauc_mrr_at_5_max": -0.030564, "nauc_mrr_at_5_std": -0.054463, "nauc_mrr_at_5_diff1": 0.214273, "nauc_mrr_at_10_max": -0.043934, "nauc_mrr_at_10_std": -0.069438, "nauc_mrr_at_10_diff1": 0.200167, "nauc_mrr_at_20_max": -0.037924, "nauc_mrr_at_20_std": -0.066328, "nauc_mrr_at_20_diff1": 0.204028, "nauc_mrr_at_100_max": -0.031031, "nauc_mrr_at_100_std": -0.063732, "nauc_mrr_at_100_diff1": 0.202451, "nauc_mrr_at_1000_max": -0.031247, "nauc_mrr_at_1000_std": -0.062772, "nauc_mrr_at_1000_diff1": 0.20309, "hit_rate_at_1": 0.06977, "hit_rate_at_3": 0.14884, "hit_rate_at_5": 0.1814, "hit_rate_at_10": 0.24186, "hit_rate_at_20": 0.30233, "hit_rate_at_100": 0.53023, "hit_rate_at_1000": 0.96279, "main_score": 0.06641, "hf_subset": "italian", "languages": [ "ita-Latn" ], "mteb_version": "2.13.1" }, { "ndcg_at_1": 0.05814, "ndcg_at_3": 0.06309, "ndcg_at_5": 0.06503, "ndcg_at_10": 0.06952, "ndcg_at_20": 0.0806, "ndcg_at_100": 0.11344, "ndcg_at_1000": 0.22535, "map_at_1": 0.0225, "map_at_3": 0.0348, "map_at_5": 0.03763, "map_at_10": 0.04031, "map_at_20": 0.04406, "map_at_100": 0.04852, "map_at_1000": 0.05348, "recall_at_1": 0.0225, "recall_at_3": 0.0472, "recall_at_5": 0.0568, "recall_at_10": 0.07585, "recall_at_20": 0.10987, "recall_at_100": 0.2336, "recall_at_1000": 0.85842, "accuracy": 0.0225, "precision_at_1": 0.06512, "precision_at_3": 0.05736, "precision_at_5": 0.04744, "precision_at_10": 0.03163, "precision_at_20": 0.02302, "precision_at_100": 0.01014, "precision_at_1000": 0.00413, "mrr_at_1": 0.065116, "mrr_at_3": 0.10155, "mrr_at_5": 0.111783, "mrr_at_10": 0.118527, "mrr_at_20": 0.124705, "mrr_at_100": 0.129735, "mrr_at_1000": 0.131512, "nauc_ndcg_at_1_max": 0.011111, "nauc_ndcg_at_1_std": -0.222543, "nauc_ndcg_at_1_diff1": 0.297648, "nauc_ndcg_at_3_max": 0.008833, "nauc_ndcg_at_3_std": -0.212678, "nauc_ndcg_at_3_diff1": 0.258483, "nauc_ndcg_at_5_max": 0.006596, "nauc_ndcg_at_5_std": -0.186787, "nauc_ndcg_at_5_diff1": 0.206981, "nauc_ndcg_at_10_max": -0.022089, "nauc_ndcg_at_10_std": -0.188596, "nauc_ndcg_at_10_diff1": 0.159128, "nauc_ndcg_at_20_max": -0.026729, "nauc_ndcg_at_20_std": -0.214594, "nauc_ndcg_at_20_diff1": 0.140277, "nauc_ndcg_at_100_max": 0.011909, "nauc_ndcg_at_100_std": -0.169646, "nauc_ndcg_at_100_diff1": 0.122311, "nauc_ndcg_at_1000_max": 0.043237, "nauc_ndcg_at_1000_std": -0.084771, "nauc_ndcg_at_1000_diff1": 0.100241, "nauc_map_at_1_max": -0.016345, "nauc_map_at_1_std": -0.247094, "nauc_map_at_1_diff1": 0.32695, "nauc_map_at_3_max": -0.008788, "nauc_map_at_3_std": -0.238234, "nauc_map_at_3_diff1": 0.28188, "nauc_map_at_5_max": 0.001608, "nauc_map_at_5_std": -0.21893, "nauc_map_at_5_diff1": 0.269105, "nauc_map_at_10_max": -0.00746, "nauc_map_at_10_std": -0.214276, "nauc_map_at_10_diff1": 0.247004, "nauc_map_at_20_max": -0.007603, "nauc_map_at_20_std": -0.224921, "nauc_map_at_20_diff1": 0.237618, "nauc_map_at_100_max": 0.002909, "nauc_map_at_100_std": -0.210311, "nauc_map_at_100_diff1": 0.224315, "nauc_map_at_1000_max": 0.010391, "nauc_map_at_1000_std": -0.199295, "nauc_map_at_1000_diff1": 0.219384, "nauc_recall_at_1_max": -0.016345, "nauc_recall_at_1_std": -0.247094, "nauc_recall_at_1_diff1": 0.32695, "nauc_recall_at_3_max": -0.030216, "nauc_recall_at_3_std": -0.243679, "nauc_recall_at_3_diff1": 0.197098, "nauc_recall_at_5_max": -0.003652, "nauc_recall_at_5_std": -0.194255, "nauc_recall_at_5_diff1": 0.151298, "nauc_recall_at_10_max": -0.044754, "nauc_recall_at_10_std": -0.161979, "nauc_recall_at_10_diff1": 0.066788, "nauc_recall_at_20_max": -0.040149, "nauc_recall_at_20_std": -0.22891, "nauc_recall_at_20_diff1": 0.056903, "nauc_recall_at_100_max": 0.019227, "nauc_recall_at_100_std": -0.130652, "nauc_recall_at_100_diff1": 0.074863, "nauc_recall_at_1000_max": 0.006165, "nauc_recall_at_1000_std": 0.151496, "nauc_recall_at_1000_diff1": -0.000143, "nauc_precision_at_1_max": -0.01886, "nauc_precision_at_1_std": -0.250539, "nauc_precision_at_1_diff1": 0.293758, "nauc_precision_at_3_max": 0.013624, "nauc_precision_at_3_std": -0.224615, "nauc_precision_at_3_diff1": 0.24171, "nauc_precision_at_5_max": 0.020646, "nauc_precision_at_5_std": -0.169885, "nauc_precision_at_5_diff1": 0.16848, "nauc_precision_at_10_max": -0.009566, "nauc_precision_at_10_std": -0.174971, "nauc_precision_at_10_diff1": 0.088986, "nauc_precision_at_20_max": -0.001425, "nauc_precision_at_20_std": -0.217842, "nauc_precision_at_20_diff1": 0.059999, "nauc_precision_at_100_max": 0.135749, "nauc_precision_at_100_std": 0.012308, "nauc_precision_at_100_diff1": 0.014363, "nauc_precision_at_1000_max": 0.160659, "nauc_precision_at_1000_std": 0.184308, "nauc_precision_at_1000_diff1": -0.016864, "nauc_mrr_at_1_max": -0.01886, "nauc_mrr_at_1_std": -0.250539, "nauc_mrr_at_1_diff1": 0.293758, "nauc_mrr_at_3_max": -0.049332, "nauc_mrr_at_3_std": -0.262949, "nauc_mrr_at_3_diff1": 0.198645, "nauc_mrr_at_5_max": -0.036153, "nauc_mrr_at_5_std": -0.234156, "nauc_mrr_at_5_diff1": 0.164228, "nauc_mrr_at_10_max": -0.037545, "nauc_mrr_at_10_std": -0.228322, "nauc_mrr_at_10_diff1": 0.147325, "nauc_mrr_at_20_max": -0.036885, "nauc_mrr_at_20_std": -0.2366, "nauc_mrr_at_20_diff1": 0.140927, "nauc_mrr_at_100_max": -0.032981, "nauc_mrr_at_100_std": -0.229703, "nauc_mrr_at_100_diff1": 0.139514, "nauc_mrr_at_1000_max": -0.033109, "nauc_mrr_at_1000_std": -0.230172, "nauc_mrr_at_1000_diff1": 0.140356, "hit_rate_at_1": 0.06512, "hit_rate_at_3": 0.14884, "hit_rate_at_5": 0.19535, "hit_rate_at_10": 0.24651, "hit_rate_at_20": 0.33488, "hit_rate_at_100": 0.55349, "hit_rate_at_1000": 0.97209, "main_score": 0.06952, "hf_subset": "portuguese", "languages": [ "por-Latn" ], "mteb_version": "2.13.1" } ] }, "evaluation_time": 2385.4638245627284, "kg_co2_emissions": null, "date": 1783094078.937079, "evaluation_phases": [ { "name": "Data loading", "start": 9.61795449256897e-05, "end": 59.36650666594505, "split": "", "subset": "" }, { "name": "Dataset transform", "start": 59.36655981838703, "end": 59.3665646314621, "split": "", "subset": "" }, { "name": "Encoding queries and documents", "start": 59.3876624032855, "end": 449.1253919452429, "split": "test", "subset": "french" }, { "name": "Scoring", "start": 449.12554118037224, "end": 449.3400021120906, "split": "test", "subset": "french" }, { "name": "Encoding queries and documents", "start": 449.36792604625225, "end": 818.9332410469651, "split": "test", "subset": "spanish" }, { "name": "Scoring", "start": 818.9334439188242, "end": 819.1660908311605, "split": "test", "subset": "spanish" }, { "name": "Encoding queries and documents", "start": 819.1972664743662, "end": 1193.4237240031362, "split": "test", "subset": "english" }, { "name": "Scoring", "start": 1193.4239327684045, "end": 1193.6290734037757, "split": "test", "subset": "english" }, { "name": "Encoding queries and documents", "start": 1193.673568919301, "end": 1594.6238907277584, "split": "test", "subset": "german" }, { "name": "Scoring", "start": 1594.6241620332003, "end": 1594.8282319903374, "split": "test", "subset": "german" }, { "name": "Encoding queries and documents", "start": 1594.8646158576012, "end": 2013.8315862193704, "split": "test", "subset": "italian" }, { "name": "Scoring", "start": 2013.8320493996143, "end": 2014.0419160425663, "split": "test", "subset": "italian" }, { "name": "Encoding queries and documents", "start": 2014.0781741291285, "end": 2385.223210707307, "split": "test", "subset": "portuguese" }, { "name": "Scoring", "start": 2385.223478063941, "end": 2385.4362615197897, "split": "test", "subset": "portuguese" } ] }