{ "dataset_revision": "fb7fb69f81f7db62790f40494124b8ad22b424ab", "task_name": "Vidore3ComputerScienceRetrieval", "mteb_version": "2.18.1", "scores": { "test": [ { "ndcg_at_1": 0.61163, "ndcg_at_3": 0.56418, "ndcg_at_5": 0.57183, "ndcg_at_10": 0.60457, "ndcg_at_20": 0.64709, "ndcg_at_100": 0.70545, "ndcg_at_1000": 0.71533, "map_at_1": 0.23651, "map_at_3": 0.35836, "map_at_5": 0.41374, "map_at_10": 0.4711, "map_at_20": 0.50533, "map_at_100": 0.53047, "map_at_1000": 0.53219, "recall_at_1": 0.23651, "recall_at_3": 0.4114, "recall_at_5": 0.50902, "recall_at_10": 0.64399, "recall_at_20": 0.76734, "recall_at_100": 0.95515, "recall_at_1000": 1.0, "accuracy": 0.23651, "precision_at_1": 0.67907, "precision_at_3": 0.45891, "precision_at_5": 0.37209, "precision_at_10": 0.2614, "precision_at_20": 0.16767, "precision_at_100": 0.04535, "precision_at_1000": 0.00488, "mrr_at_1": 0.67907, "mrr_at_3": 0.754264, "mrr_at_5": 0.764961, "mrr_at_10": 0.7722, "mrr_at_20": 0.77511, "mrr_at_100": 0.775497, "mrr_at_1000": 0.775497, "nauc_ndcg_at_1_max": 0.100902, "nauc_ndcg_at_1_std": -0.046432, "nauc_ndcg_at_1_diff1": 0.323363, "nauc_ndcg_at_3_max": 0.106267, "nauc_ndcg_at_3_std": -0.137479, "nauc_ndcg_at_3_diff1": 0.273147, "nauc_ndcg_at_5_max": 0.106774, "nauc_ndcg_at_5_std": -0.185695, "nauc_ndcg_at_5_diff1": 0.299054, "nauc_ndcg_at_10_max": 0.115092, "nauc_ndcg_at_10_std": -0.178422, "nauc_ndcg_at_10_diff1": 0.282022, "nauc_ndcg_at_20_max": 0.13622, "nauc_ndcg_at_20_std": -0.162778, "nauc_ndcg_at_20_diff1": 0.292855, "nauc_ndcg_at_100_max": 0.185507, "nauc_ndcg_at_100_std": -0.111167, "nauc_ndcg_at_100_diff1": 0.291007, "nauc_ndcg_at_1000_max": 0.182884, "nauc_ndcg_at_1000_std": -0.107498, "nauc_ndcg_at_1000_diff1": 0.289866, "nauc_map_at_1_max": -0.090834, "nauc_map_at_1_std": -0.139226, "nauc_map_at_1_diff1": 0.287897, "nauc_map_at_3_max": -0.029805, "nauc_map_at_3_std": -0.141085, "nauc_map_at_3_diff1": 0.259405, "nauc_map_at_5_max": 0.01, "nauc_map_at_5_std": -0.160469, "nauc_map_at_5_diff1": 0.263913, "nauc_map_at_10_max": 0.065125, "nauc_map_at_10_std": -0.155431, "nauc_map_at_10_diff1": 0.240414, "nauc_map_at_20_max": 0.098607, "nauc_map_at_20_std": -0.149872, "nauc_map_at_20_diff1": 0.237285, "nauc_map_at_100_max": 0.121571, "nauc_map_at_100_std": -0.135578, "nauc_map_at_100_diff1": 0.234986, "nauc_map_at_1000_max": 0.12232, "nauc_map_at_1000_std": -0.13518, "nauc_map_at_1000_diff1": 0.234923, "nauc_recall_at_1_max": -0.090834, "nauc_recall_at_1_std": -0.139226, "nauc_recall_at_1_diff1": 0.287897, "nauc_recall_at_3_max": -0.060905, "nauc_recall_at_3_std": -0.174868, "nauc_recall_at_3_diff1": 0.239498, "nauc_recall_at_5_max": -0.032961, "nauc_recall_at_5_std": -0.21907, "nauc_recall_at_5_diff1": 0.241747, "nauc_recall_at_10_max": 0.01299, "nauc_recall_at_10_std": -0.215343, "nauc_recall_at_10_diff1": 0.177578, "nauc_recall_at_20_max": 0.055719, "nauc_recall_at_20_std": -0.211516, "nauc_recall_at_20_diff1": 0.204761, "nauc_recall_at_100_max": 0.196561, "nauc_recall_at_100_std": -0.043195, "nauc_recall_at_100_diff1": 0.121518, "nauc_recall_at_1000_max": NaN, "nauc_recall_at_1000_std": NaN, "nauc_recall_at_1000_diff1": NaN, "nauc_precision_at_1_max": 0.151386, "nauc_precision_at_1_std": -0.061776, "nauc_precision_at_1_diff1": 0.313572, "nauc_precision_at_3_max": 0.259983, "nauc_precision_at_3_std": -0.013225, "nauc_precision_at_3_diff1": 0.097559, "nauc_precision_at_5_max": 0.310069, "nauc_precision_at_5_std": -0.029618, "nauc_precision_at_5_diff1": 0.06468, "nauc_precision_at_10_max": 0.323377, "nauc_precision_at_10_std": 0.031629, "nauc_precision_at_10_diff1": -0.059648, "nauc_precision_at_20_max": 0.313087, "nauc_precision_at_20_std": 0.076953, "nauc_precision_at_20_diff1": -0.098353, "nauc_precision_at_100_max": 0.286371, "nauc_precision_at_100_std": 0.157598, "nauc_precision_at_100_diff1": -0.143744, "nauc_precision_at_1000_max": 0.253523, "nauc_precision_at_1000_std": 0.160582, "nauc_precision_at_1000_diff1": -0.151013, "nauc_mrr_at_1_max": 0.151386, "nauc_mrr_at_1_std": -0.061776, "nauc_mrr_at_1_diff1": 0.313572, "nauc_mrr_at_3_max": 0.210403, "nauc_mrr_at_3_std": -0.035044, "nauc_mrr_at_3_diff1": 0.314459, "nauc_mrr_at_5_max": 0.216242, "nauc_mrr_at_5_std": -0.057165, "nauc_mrr_at_5_diff1": 0.312937, "nauc_mrr_at_10_max": 0.216222, "nauc_mrr_at_10_std": -0.042994, "nauc_mrr_at_10_diff1": 0.31015, "nauc_mrr_at_20_max": 0.212047, "nauc_mrr_at_20_std": -0.040675, "nauc_mrr_at_20_diff1": 0.312575, "nauc_mrr_at_100_max": 0.210815, "nauc_mrr_at_100_std": -0.042224, "nauc_mrr_at_100_diff1": 0.311427, "nauc_mrr_at_1000_max": 0.210815, "nauc_mrr_at_1000_std": -0.042224, "nauc_mrr_at_1000_diff1": 0.311427, "hit_rate_at_1": 0.67907, "hit_rate_at_3": 0.85116, "hit_rate_at_5": 0.89767, "hit_rate_at_10": 0.94884, "hit_rate_at_20": 0.98605, "hit_rate_at_100": 1.0, "hit_rate_at_1000": 1.0, "main_score": 0.60457, "hf_subset": "french", "languages": [ "fra-Latn" ], "mteb_version": "2.18.1" }, { "ndcg_at_1": 0.55814, "ndcg_at_3": 0.56254, "ndcg_at_5": 0.56855, "ndcg_at_10": 0.60439, "ndcg_at_20": 0.64602, "ndcg_at_100": 0.70076, "ndcg_at_1000": 0.71195, "map_at_1": 0.214, "map_at_3": 0.36132, "map_at_5": 0.41444, "map_at_10": 0.47061, "map_at_20": 0.5057, "map_at_100": 0.53027, "map_at_1000": 0.5321, "recall_at_1": 0.214, "recall_at_3": 0.4334, "recall_at_5": 0.52452, "recall_at_10": 0.66067, "recall_at_20": 0.77919, "recall_at_100": 0.9488, "recall_at_1000": 1.0, "accuracy": 0.214, "precision_at_1": 0.61395, "precision_at_3": 0.47132, "precision_at_5": 0.37395, "precision_at_10": 0.26465, "precision_at_20": 0.16977, "precision_at_100": 0.04502, "precision_at_1000": 0.00488, "mrr_at_1": 0.613953, "mrr_at_3": 0.731008, "mrr_at_5": 0.739612, "mrr_at_10": 0.746639, "mrr_at_20": 0.748266, "mrr_at_100": 0.748689, "mrr_at_1000": 0.74873, "nauc_ndcg_at_1_max": 0.206859, "nauc_ndcg_at_1_std": -0.039258, "nauc_ndcg_at_1_diff1": 0.358585, "nauc_ndcg_at_3_max": 0.215057, "nauc_ndcg_at_3_std": -0.146713, "nauc_ndcg_at_3_diff1": 0.314198, "nauc_ndcg_at_5_max": 0.211067, "nauc_ndcg_at_5_std": -0.143765, "nauc_ndcg_at_5_diff1": 0.298828, "nauc_ndcg_at_10_max": 0.184218, "nauc_ndcg_at_10_std": -0.165355, "nauc_ndcg_at_10_diff1": 0.279689, "nauc_ndcg_at_20_max": 0.199156, "nauc_ndcg_at_20_std": -0.141178, "nauc_ndcg_at_20_diff1": 0.263981, "nauc_ndcg_at_100_max": 0.253204, "nauc_ndcg_at_100_std": -0.085536, "nauc_ndcg_at_100_diff1": 0.295864, "nauc_ndcg_at_1000_max": 0.250642, "nauc_ndcg_at_1000_std": -0.083563, "nauc_ndcg_at_1000_diff1": 0.29273, "nauc_map_at_1_max": 0.02801, "nauc_map_at_1_std": -0.107886, "nauc_map_at_1_diff1": 0.366595, "nauc_map_at_3_max": 0.092515, "nauc_map_at_3_std": -0.148912, "nauc_map_at_3_diff1": 0.300113, "nauc_map_at_5_max": 0.131847, "nauc_map_at_5_std": -0.138984, "nauc_map_at_5_diff1": 0.290518, "nauc_map_at_10_max": 0.1605, "nauc_map_at_10_std": -0.14666, "nauc_map_at_10_diff1": 0.272339, "nauc_map_at_20_max": 0.180419, "nauc_map_at_20_std": -0.131912, "nauc_map_at_20_diff1": 0.254759, "nauc_map_at_100_max": 0.201435, "nauc_map_at_100_std": -0.114232, "nauc_map_at_100_diff1": 0.257639, "nauc_map_at_1000_max": 0.201151, "nauc_map_at_1000_std": -0.113537, "nauc_map_at_1000_diff1": 0.256927, "nauc_recall_at_1_max": 0.02801, "nauc_recall_at_1_std": -0.107886, "nauc_recall_at_1_diff1": 0.366595, "nauc_recall_at_3_max": 0.020713, "nauc_recall_at_3_std": -0.216852, "nauc_recall_at_3_diff1": 0.209184, "nauc_recall_at_5_max": 0.062114, "nauc_recall_at_5_std": -0.194981, "nauc_recall_at_5_diff1": 0.198138, "nauc_recall_at_10_max": 0.04479, "nauc_recall_at_10_std": -0.242878, "nauc_recall_at_10_diff1": 0.137839, "nauc_recall_at_20_max": 0.032707, "nauc_recall_at_20_std": -0.238561, "nauc_recall_at_20_diff1": 0.067197, "nauc_recall_at_100_max": 0.201934, "nauc_recall_at_100_std": -0.002418, "nauc_recall_at_100_diff1": 0.185555, "nauc_recall_at_1000_max": NaN, "nauc_recall_at_1000_std": NaN, "nauc_recall_at_1000_diff1": NaN, "nauc_precision_at_1_max": 0.242071, "nauc_precision_at_1_std": -0.057611, "nauc_precision_at_1_diff1": 0.398777, "nauc_precision_at_3_max": 0.293101, "nauc_precision_at_3_std": -0.040575, "nauc_precision_at_3_diff1": 0.12917, "nauc_precision_at_5_max": 0.299367, "nauc_precision_at_5_std": 0.030944, "nauc_precision_at_5_diff1": 0.036611, "nauc_precision_at_10_max": 0.235808, "nauc_precision_at_10_std": 0.072737, "nauc_precision_at_10_diff1": -0.079848, "nauc_precision_at_20_max": 0.177566, "nauc_precision_at_20_std": 0.120752, "nauc_precision_at_20_diff1": -0.145266, "nauc_precision_at_100_max": 0.160422, "nauc_precision_at_100_std": 0.191177, "nauc_precision_at_100_diff1": -0.138595, "nauc_precision_at_1000_max": 0.122836, "nauc_precision_at_1000_std": 0.187739, "nauc_precision_at_1000_diff1": -0.154859, "nauc_mrr_at_1_max": 0.242071, "nauc_mrr_at_1_std": -0.057611, "nauc_mrr_at_1_diff1": 0.398777, "nauc_mrr_at_3_max": 0.266462, "nauc_mrr_at_3_std": -0.061405, "nauc_mrr_at_3_diff1": 0.36172, "nauc_mrr_at_5_max": 0.276597, "nauc_mrr_at_5_std": -0.048396, "nauc_mrr_at_5_diff1": 0.357637, "nauc_mrr_at_10_max": 0.270027, "nauc_mrr_at_10_std": -0.04691, "nauc_mrr_at_10_diff1": 0.357054, "nauc_mrr_at_20_max": 0.268763, "nauc_mrr_at_20_std": -0.045379, "nauc_mrr_at_20_diff1": 0.360965, "nauc_mrr_at_100_max": 0.268793, "nauc_mrr_at_100_std": -0.046564, "nauc_mrr_at_100_diff1": 0.362803, "nauc_mrr_at_1000_max": 0.268879, "nauc_mrr_at_1000_std": -0.046539, "nauc_mrr_at_1000_diff1": 0.36285, "hit_rate_at_1": 0.61395, "hit_rate_at_3": 0.87442, "hit_rate_at_5": 0.91163, "hit_rate_at_10": 0.96279, "hit_rate_at_20": 0.98605, "hit_rate_at_100": 0.99535, "hit_rate_at_1000": 1.0, "main_score": 0.60439, "hf_subset": "spanish", "languages": [ "spa-Latn" ], "mteb_version": "2.18.1" }, { "ndcg_at_1": 0.65116, "ndcg_at_3": 0.60486, "ndcg_at_5": 0.61313, "ndcg_at_10": 0.6485, "ndcg_at_20": 0.68548, "ndcg_at_100": 0.7364, "ndcg_at_1000": 0.74687, "map_at_1": 0.25149, "map_at_3": 0.38983, "map_at_5": 0.4524, "map_at_10": 0.51373, "map_at_20": 0.545, "map_at_100": 0.56834, "map_at_1000": 0.57025, "recall_at_1": 0.25149, "recall_at_3": 0.44637, "recall_at_5": 0.55285, "recall_at_10": 0.68809, "recall_at_20": 0.79395, "recall_at_100": 0.94979, "recall_at_1000": 0.99779, "accuracy": 0.25149, "precision_at_1": 0.70233, "precision_at_3": 0.48372, "precision_at_5": 0.38884, "precision_at_10": 0.27721, "precision_at_20": 0.1714, "precision_at_100": 0.04512, "precision_at_1000": 0.00487, "mrr_at_1": 0.702326, "mrr_at_3": 0.786822, "mrr_at_5": 0.794496, "mrr_at_10": 0.799247, "mrr_at_20": 0.800563, "mrr_at_100": 0.801294, "mrr_at_1000": 0.801294, "nauc_ndcg_at_1_max": 0.13533, "nauc_ndcg_at_1_std": -0.094871, "nauc_ndcg_at_1_diff1": 0.383037, "nauc_ndcg_at_3_max": 0.070219, "nauc_ndcg_at_3_std": -0.226491, "nauc_ndcg_at_3_diff1": 0.353193, "nauc_ndcg_at_5_max": 0.036221, "nauc_ndcg_at_5_std": -0.289242, "nauc_ndcg_at_5_diff1": 0.351188, "nauc_ndcg_at_10_max": 0.101776, "nauc_ndcg_at_10_std": -0.26503, "nauc_ndcg_at_10_diff1": 0.345916, "nauc_ndcg_at_20_max": 0.128296, "nauc_ndcg_at_20_std": -0.227359, "nauc_ndcg_at_20_diff1": 0.33823, "nauc_ndcg_at_100_max": 0.175995, "nauc_ndcg_at_100_std": -0.192275, "nauc_ndcg_at_100_diff1": 0.368315, "nauc_ndcg_at_1000_max": 0.162659, "nauc_ndcg_at_1000_std": -0.185772, "nauc_ndcg_at_1000_diff1": 0.362598, "nauc_map_at_1_max": -0.062799, "nauc_map_at_1_std": -0.156456, "nauc_map_at_1_diff1": 0.380538, "nauc_map_at_3_max": -0.049414, "nauc_map_at_3_std": -0.192877, "nauc_map_at_3_diff1": 0.331892, "nauc_map_at_5_max": -0.026657, "nauc_map_at_5_std": -0.221532, "nauc_map_at_5_diff1": 0.310646, "nauc_map_at_10_max": 0.059188, "nauc_map_at_10_std": -0.222122, "nauc_map_at_10_diff1": 0.308931, "nauc_map_at_20_max": 0.092422, "nauc_map_at_20_std": -0.201515, "nauc_map_at_20_diff1": 0.309492, "nauc_map_at_100_max": 0.119597, "nauc_map_at_100_std": -0.189259, "nauc_map_at_100_diff1": 0.321847, "nauc_map_at_1000_max": 0.118499, "nauc_map_at_1000_std": -0.188922, "nauc_map_at_1000_diff1": 0.321258, "nauc_recall_at_1_max": -0.062799, "nauc_recall_at_1_std": -0.156456, "nauc_recall_at_1_diff1": 0.380538, "nauc_recall_at_3_max": -0.12457, "nauc_recall_at_3_std": -0.244043, "nauc_recall_at_3_diff1": 0.251061, "nauc_recall_at_5_max": -0.122029, "nauc_recall_at_5_std": -0.287945, "nauc_recall_at_5_diff1": 0.21611, "nauc_recall_at_10_max": 0.037939, "nauc_recall_at_10_std": -0.286946, "nauc_recall_at_10_diff1": 0.205073, "nauc_recall_at_20_max": 0.109957, "nauc_recall_at_20_std": -0.24777, "nauc_recall_at_20_diff1": 0.181127, "nauc_recall_at_100_max": 0.346668, "nauc_recall_at_100_std": -0.146687, "nauc_recall_at_100_diff1": 0.263417, "nauc_recall_at_1000_max": 0.824825, "nauc_recall_at_1000_std": 0.960683, "nauc_recall_at_1000_diff1": 0.807153, "nauc_precision_at_1_max": 0.197996, "nauc_precision_at_1_std": -0.113359, "nauc_precision_at_1_diff1": 0.435944, "nauc_precision_at_3_max": 0.207815, "nauc_precision_at_3_std": -0.065378, "nauc_precision_at_3_diff1": 0.079705, "nauc_precision_at_5_max": 0.238565, "nauc_precision_at_5_std": -0.098109, "nauc_precision_at_5_diff1": 0.017975, "nauc_precision_at_10_max": 0.337722, "nauc_precision_at_10_std": 0.02245, "nauc_precision_at_10_diff1": -0.057986, "nauc_precision_at_20_max": 0.321466, "nauc_precision_at_20_std": 0.102261, "nauc_precision_at_20_diff1": -0.079767, "nauc_precision_at_100_max": 0.272855, "nauc_precision_at_100_std": 0.158477, "nauc_precision_at_100_diff1": -0.099171, "nauc_precision_at_1000_max": 0.217731, "nauc_precision_at_1000_std": 0.167989, "nauc_precision_at_1000_diff1": -0.125496, "nauc_mrr_at_1_max": 0.197996, "nauc_mrr_at_1_std": -0.113359, "nauc_mrr_at_1_diff1": 0.435944, "nauc_mrr_at_3_max": 0.196512, "nauc_mrr_at_3_std": -0.191039, "nauc_mrr_at_3_diff1": 0.428335, "nauc_mrr_at_5_max": 0.201382, "nauc_mrr_at_5_std": -0.192997, "nauc_mrr_at_5_diff1": 0.421797, "nauc_mrr_at_10_max": 0.205785, "nauc_mrr_at_10_std": -0.183434, "nauc_mrr_at_10_diff1": 0.424496, "nauc_mrr_at_20_max": 0.204162, "nauc_mrr_at_20_std": -0.18019, "nauc_mrr_at_20_diff1": 0.422754, "nauc_mrr_at_100_max": 0.202933, "nauc_mrr_at_100_std": -0.179555, "nauc_mrr_at_100_diff1": 0.424809, "nauc_mrr_at_1000_max": 0.202933, "nauc_mrr_at_1000_std": -0.179555, "nauc_mrr_at_1000_diff1": 0.424809, "hit_rate_at_1": 0.70233, "hit_rate_at_3": 0.89302, "hit_rate_at_5": 0.92558, "hit_rate_at_10": 0.96279, "hit_rate_at_20": 0.9814, "hit_rate_at_100": 1.0, "hit_rate_at_1000": 1.0, "main_score": 0.6485, "hf_subset": "english", "languages": [ "eng-Latn" ], "mteb_version": "2.18.1" }, { "ndcg_at_1": 0.56744, "ndcg_at_3": 0.5601, "ndcg_at_5": 0.57822, "ndcg_at_10": 0.60604, "ndcg_at_20": 0.64798, "ndcg_at_100": 0.70376, "ndcg_at_1000": 0.71487, "map_at_1": 0.22571, "map_at_3": 0.35874, "map_at_5": 0.41998, "map_at_10": 0.47235, "map_at_20": 0.50685, "map_at_100": 0.53176, "map_at_1000": 0.53364, "recall_at_1": 0.22571, "recall_at_3": 0.41961, "recall_at_5": 0.52912, "recall_at_10": 0.65593, "recall_at_20": 0.77207, "recall_at_100": 0.94991, "recall_at_1000": 0.99922, "accuracy": 0.22571, "precision_at_1": 0.6186, "precision_at_3": 0.45581, "precision_at_5": 0.37953, "precision_at_10": 0.26047, "precision_at_20": 0.16674, "precision_at_100": 0.04507, "precision_at_1000": 0.00487, "mrr_at_1": 0.618605, "mrr_at_3": 0.714729, "mrr_at_5": 0.730543, "mrr_at_10": 0.737693, "mrr_at_20": 0.738997, "mrr_at_100": 0.739701, "mrr_at_1000": 0.739701, "nauc_ndcg_at_1_max": 0.247767, "nauc_ndcg_at_1_std": -0.011355, "nauc_ndcg_at_1_diff1": 0.447936, "nauc_ndcg_at_3_max": 0.174155, "nauc_ndcg_at_3_std": -0.105415, "nauc_ndcg_at_3_diff1": 0.390291, "nauc_ndcg_at_5_max": 0.173127, "nauc_ndcg_at_5_std": -0.144784, "nauc_ndcg_at_5_diff1": 0.376196, "nauc_ndcg_at_10_max": 0.195959, "nauc_ndcg_at_10_std": -0.133783, "nauc_ndcg_at_10_diff1": 0.366986, "nauc_ndcg_at_20_max": 0.214237, "nauc_ndcg_at_20_std": -0.112689, "nauc_ndcg_at_20_diff1": 0.367491, "nauc_ndcg_at_100_max": 0.26859, "nauc_ndcg_at_100_std": -0.066328, "nauc_ndcg_at_100_diff1": 0.369831, "nauc_ndcg_at_1000_max": 0.26929, "nauc_ndcg_at_1000_std": -0.0598, "nauc_ndcg_at_1000_diff1": 0.373901, "nauc_map_at_1_max": 0.014461, "nauc_map_at_1_std": -0.091991, "nauc_map_at_1_diff1": 0.350651, "nauc_map_at_3_max": 0.057812, "nauc_map_at_3_std": -0.088932, "nauc_map_at_3_diff1": 0.337971, "nauc_map_at_5_max": 0.11308, "nauc_map_at_5_std": -0.10094, "nauc_map_at_5_diff1": 0.349459, "nauc_map_at_10_max": 0.175929, "nauc_map_at_10_std": -0.092202, "nauc_map_at_10_diff1": 0.335932, "nauc_map_at_20_max": 0.203487, "nauc_map_at_20_std": -0.081668, "nauc_map_at_20_diff1": 0.331189, "nauc_map_at_100_max": 0.234964, "nauc_map_at_100_std": -0.062568, "nauc_map_at_100_diff1": 0.331005, "nauc_map_at_1000_max": 0.235509, "nauc_map_at_1000_std": -0.061641, "nauc_map_at_1000_diff1": 0.330899, "nauc_recall_at_1_max": 0.014461, "nauc_recall_at_1_std": -0.091991, "nauc_recall_at_1_diff1": 0.350651, "nauc_recall_at_3_max": -0.027455, "nauc_recall_at_3_std": -0.147298, "nauc_recall_at_3_diff1": 0.258002, "nauc_recall_at_5_max": 0.028461, "nauc_recall_at_5_std": -0.178746, "nauc_recall_at_5_diff1": 0.265556, "nauc_recall_at_10_max": 0.077625, "nauc_recall_at_10_std": -0.194898, "nauc_recall_at_10_diff1": 0.208311, "nauc_recall_at_20_max": 0.090114, "nauc_recall_at_20_std": -0.183443, "nauc_recall_at_20_diff1": 0.181063, "nauc_recall_at_100_max": 0.195984, "nauc_recall_at_100_std": -0.110118, "nauc_recall_at_100_diff1": 0.11744, "nauc_recall_at_1000_max": -1.75903, "nauc_recall_at_1000_std": 0.357178, "nauc_recall_at_1000_diff1": -1.75903, "nauc_precision_at_1_max": 0.303086, "nauc_precision_at_1_std": -0.048215, "nauc_precision_at_1_diff1": 0.469823, "nauc_precision_at_3_max": 0.307575, "nauc_precision_at_3_std": 0.032433, "nauc_precision_at_3_diff1": 0.200775, "nauc_precision_at_5_max": 0.364477, "nauc_precision_at_5_std": 0.020227, "nauc_precision_at_5_diff1": 0.108819, "nauc_precision_at_10_max": 0.381513, "nauc_precision_at_10_std": 0.075095, "nauc_precision_at_10_diff1": -0.01862, "nauc_precision_at_20_max": 0.320037, "nauc_precision_at_20_std": 0.111693, "nauc_precision_at_20_diff1": -0.076521, "nauc_precision_at_100_max": 0.27318, "nauc_precision_at_100_std": 0.169236, "nauc_precision_at_100_diff1": -0.144932, "nauc_precision_at_1000_max": 0.239013, "nauc_precision_at_1000_std": 0.170686, "nauc_precision_at_1000_diff1": -0.147696, "nauc_mrr_at_1_max": 0.303086, "nauc_mrr_at_1_std": -0.048215, "nauc_mrr_at_1_diff1": 0.469823, "nauc_mrr_at_3_max": 0.306079, "nauc_mrr_at_3_std": -0.065288, "nauc_mrr_at_3_diff1": 0.43995, "nauc_mrr_at_5_max": 0.305512, "nauc_mrr_at_5_std": -0.083429, "nauc_mrr_at_5_diff1": 0.436335, "nauc_mrr_at_10_max": 0.305953, "nauc_mrr_at_10_std": -0.067982, "nauc_mrr_at_10_diff1": 0.429597, "nauc_mrr_at_20_max": 0.304731, "nauc_mrr_at_20_std": -0.065528, "nauc_mrr_at_20_diff1": 0.431641, "nauc_mrr_at_100_max": 0.304774, "nauc_mrr_at_100_std": -0.065209, "nauc_mrr_at_100_diff1": 0.433474, "nauc_mrr_at_1000_max": 0.304774, "nauc_mrr_at_1000_std": -0.065209, "nauc_mrr_at_1000_diff1": 0.433474, "hit_rate_at_1": 0.6186, "hit_rate_at_3": 0.84186, "hit_rate_at_5": 0.90698, "hit_rate_at_10": 0.95814, "hit_rate_at_20": 0.97674, "hit_rate_at_100": 1.0, "hit_rate_at_1000": 1.0, "main_score": 0.60604, "hf_subset": "german", "languages": [ "deu-Latn" ], "mteb_version": "2.18.1" }, { "ndcg_at_1": 0.53721, "ndcg_at_3": 0.55379, "ndcg_at_5": 0.56669, "ndcg_at_10": 0.60858, "ndcg_at_20": 0.6489, "ndcg_at_100": 0.70026, "ndcg_at_1000": 0.71162, "map_at_1": 0.21308, "map_at_3": 0.35895, "map_at_5": 0.41679, "map_at_10": 0.47714, "map_at_20": 0.51061, "map_at_100": 0.53442, "map_at_1000": 0.53634, "recall_at_1": 0.21308, "recall_at_3": 0.42394, "recall_at_5": 0.5291, "recall_at_10": 0.67536, "recall_at_20": 0.78827, "recall_at_100": 0.94861, "recall_at_1000": 1.0, "accuracy": 0.21308, "precision_at_1": 0.59535, "precision_at_3": 0.46202, "precision_at_5": 0.37674, "precision_at_10": 0.27116, "precision_at_20": 0.17116, "precision_at_100": 0.04502, "precision_at_1000": 0.00488, "mrr_at_1": 0.595349, "mrr_at_3": 0.710853, "mrr_at_5": 0.724109, "mrr_at_10": 0.732962, "mrr_at_20": 0.733927, "mrr_at_100": 0.734085, "mrr_at_1000": 0.734085, "nauc_ndcg_at_1_max": 0.266048, "nauc_ndcg_at_1_std": -0.037743, "nauc_ndcg_at_1_diff1": 0.457349, "nauc_ndcg_at_3_max": 0.244727, "nauc_ndcg_at_3_std": -0.115457, "nauc_ndcg_at_3_diff1": 0.334083, "nauc_ndcg_at_5_max": 0.224917, "nauc_ndcg_at_5_std": -0.140605, "nauc_ndcg_at_5_diff1": 0.332228, "nauc_ndcg_at_10_max": 0.187487, "nauc_ndcg_at_10_std": -0.153534, "nauc_ndcg_at_10_diff1": 0.307617, "nauc_ndcg_at_20_max": 0.219524, "nauc_ndcg_at_20_std": -0.134122, "nauc_ndcg_at_20_diff1": 0.313743, "nauc_ndcg_at_100_max": 0.278642, "nauc_ndcg_at_100_std": -0.077286, "nauc_ndcg_at_100_diff1": 0.340079, "nauc_ndcg_at_1000_max": 0.275391, "nauc_ndcg_at_1000_std": -0.073791, "nauc_ndcg_at_1000_diff1": 0.343633, "nauc_map_at_1_max": 0.012549, "nauc_map_at_1_std": -0.127179, "nauc_map_at_1_diff1": 0.318569, "nauc_map_at_3_max": 0.082764, "nauc_map_at_3_std": -0.130991, "nauc_map_at_3_diff1": 0.251397, "nauc_map_at_5_max": 0.123341, "nauc_map_at_5_std": -0.136063, "nauc_map_at_5_diff1": 0.270792, "nauc_map_at_10_max": 0.148185, "nauc_map_at_10_std": -0.143236, "nauc_map_at_10_diff1": 0.254096, "nauc_map_at_20_max": 0.182781, "nauc_map_at_20_std": -0.131343, "nauc_map_at_20_diff1": 0.255097, "nauc_map_at_100_max": 0.209465, "nauc_map_at_100_std": -0.111249, "nauc_map_at_100_diff1": 0.260055, "nauc_map_at_1000_max": 0.210319, "nauc_map_at_1000_std": -0.110228, "nauc_map_at_1000_diff1": 0.260545, "nauc_recall_at_1_max": 0.012549, "nauc_recall_at_1_std": -0.127179, "nauc_recall_at_1_diff1": 0.318569, "nauc_recall_at_3_max": 0.036065, "nauc_recall_at_3_std": -0.157045, "nauc_recall_at_3_diff1": 0.166736, "nauc_recall_at_5_max": 0.050415, "nauc_recall_at_5_std": -0.178794, "nauc_recall_at_5_diff1": 0.177542, "nauc_recall_at_10_max": 0.014803, "nauc_recall_at_10_std": -0.231693, "nauc_recall_at_10_diff1": 0.11872, "nauc_recall_at_20_max": 0.073691, "nauc_recall_at_20_std": -0.226704, "nauc_recall_at_20_diff1": 0.126576, "nauc_recall_at_100_max": 0.292594, "nauc_recall_at_100_std": -0.035163, "nauc_recall_at_100_diff1": 0.217841, "nauc_recall_at_1000_max": NaN, "nauc_recall_at_1000_std": NaN, "nauc_recall_at_1000_diff1": NaN, "nauc_precision_at_1_max": 0.329376, "nauc_precision_at_1_std": -0.041249, "nauc_precision_at_1_diff1": 0.456657, "nauc_precision_at_3_max": 0.353796, "nauc_precision_at_3_std": 0.02064, "nauc_precision_at_3_diff1": 0.155752, "nauc_precision_at_5_max": 0.360245, "nauc_precision_at_5_std": 0.039992, "nauc_precision_at_5_diff1": 0.116072, "nauc_precision_at_10_max": 0.281417, "nauc_precision_at_10_std": 0.060914, "nauc_precision_at_10_diff1": -0.015949, "nauc_precision_at_20_max": 0.276784, "nauc_precision_at_20_std": 0.111118, "nauc_precision_at_20_diff1": -0.043625, "nauc_precision_at_100_max": 0.256326, "nauc_precision_at_100_std": 0.197473, "nauc_precision_at_100_diff1": -0.065532, "nauc_precision_at_1000_max": 0.21521, "nauc_precision_at_1000_std": 0.199639, "nauc_precision_at_1000_diff1": -0.075342, "nauc_mrr_at_1_max": 0.329376, "nauc_mrr_at_1_std": -0.041249, "nauc_mrr_at_1_diff1": 0.456657, "nauc_mrr_at_3_max": 0.361118, "nauc_mrr_at_3_std": -0.010569, "nauc_mrr_at_3_diff1": 0.417362, "nauc_mrr_at_5_max": 0.357794, "nauc_mrr_at_5_std": -0.007794, "nauc_mrr_at_5_diff1": 0.417833, "nauc_mrr_at_10_max": 0.349427, "nauc_mrr_at_10_std": -0.012157, "nauc_mrr_at_10_diff1": 0.42351, "nauc_mrr_at_20_max": 0.348231, "nauc_mrr_at_20_std": -0.012775, "nauc_mrr_at_20_diff1": 0.425365, "nauc_mrr_at_100_max": 0.348033, "nauc_mrr_at_100_std": -0.013133, "nauc_mrr_at_100_diff1": 0.424949, "nauc_mrr_at_1000_max": 0.348033, "nauc_mrr_at_1000_std": -0.013133, "nauc_mrr_at_1000_diff1": 0.424949, "hit_rate_at_1": 0.59535, "hit_rate_at_3": 0.85116, "hit_rate_at_5": 0.91163, "hit_rate_at_10": 0.97674, "hit_rate_at_20": 0.9907, "hit_rate_at_100": 1.0, "hit_rate_at_1000": 1.0, "main_score": 0.60858, "hf_subset": "italian", "languages": [ "ita-Latn" ], "mteb_version": "2.18.1" }, { "ndcg_at_1": 0.57907, "ndcg_at_3": 0.57395, "ndcg_at_5": 0.58095, "ndcg_at_10": 0.61664, "ndcg_at_20": 0.65447, "ndcg_at_100": 0.71064, "ndcg_at_1000": 0.721, "map_at_1": 0.22606, "map_at_3": 0.36819, "map_at_5": 0.4246, "map_at_10": 0.48229, "map_at_20": 0.51488, "map_at_100": 0.54048, "map_at_1000": 0.54212, "recall_at_1": 0.22606, "recall_at_3": 0.42689, "recall_at_5": 0.52995, "recall_at_10": 0.67019, "recall_at_20": 0.77719, "recall_at_100": 0.95221, "recall_at_1000": 1.0, "accuracy": 0.22606, "precision_at_1": 0.63721, "precision_at_3": 0.47752, "precision_at_5": 0.3814, "precision_at_10": 0.26744, "precision_at_20": 0.16814, "precision_at_100": 0.0453, "precision_at_1000": 0.00488, "mrr_at_1": 0.637209, "mrr_at_3": 0.737984, "mrr_at_5": 0.751705, "mrr_at_10": 0.759319, "mrr_at_20": 0.760283, "mrr_at_100": 0.760927, "mrr_at_1000": 0.760962, "nauc_ndcg_at_1_max": 0.204852, "nauc_ndcg_at_1_std": -0.053184, "nauc_ndcg_at_1_diff1": 0.424237, "nauc_ndcg_at_3_max": 0.160601, "nauc_ndcg_at_3_std": -0.156528, "nauc_ndcg_at_3_diff1": 0.353393, "nauc_ndcg_at_5_max": 0.131624, "nauc_ndcg_at_5_std": -0.190619, "nauc_ndcg_at_5_diff1": 0.357711, "nauc_ndcg_at_10_max": 0.124065, "nauc_ndcg_at_10_std": -0.195558, "nauc_ndcg_at_10_diff1": 0.320979, "nauc_ndcg_at_20_max": 0.128315, "nauc_ndcg_at_20_std": -0.180495, "nauc_ndcg_at_20_diff1": 0.309679, "nauc_ndcg_at_100_max": 0.204123, "nauc_ndcg_at_100_std": -0.133412, "nauc_ndcg_at_100_diff1": 0.335348, "nauc_ndcg_at_1000_max": 0.196934, "nauc_ndcg_at_1000_std": -0.126713, "nauc_ndcg_at_1000_diff1": 0.332474, "nauc_map_at_1_max": -0.012271, "nauc_map_at_1_std": -0.15054, "nauc_map_at_1_diff1": 0.355455, "nauc_map_at_3_max": -0.004523, "nauc_map_at_3_std": -0.16417, "nauc_map_at_3_diff1": 0.306651, "nauc_map_at_5_max": 0.024325, "nauc_map_at_5_std": -0.185684, "nauc_map_at_5_diff1": 0.304728, "nauc_map_at_10_max": 0.068935, "nauc_map_at_10_std": -0.183352, "nauc_map_at_10_diff1": 0.270487, "nauc_map_at_20_max": 0.088579, "nauc_map_at_20_std": -0.174748, "nauc_map_at_20_diff1": 0.258867, "nauc_map_at_100_max": 0.1207, "nauc_map_at_100_std": -0.158968, "nauc_map_at_100_diff1": 0.260974, "nauc_map_at_1000_max": 0.120192, "nauc_map_at_1000_std": -0.157777, "nauc_map_at_1000_diff1": 0.260199, "nauc_recall_at_1_max": -0.012271, "nauc_recall_at_1_std": -0.15054, "nauc_recall_at_1_diff1": 0.355455, "nauc_recall_at_3_max": -0.046551, "nauc_recall_at_3_std": -0.21018, "nauc_recall_at_3_diff1": 0.264607, "nauc_recall_at_5_max": -0.036816, "nauc_recall_at_5_std": -0.236846, "nauc_recall_at_5_diff1": 0.259427, "nauc_recall_at_10_max": -0.005687, "nauc_recall_at_10_std": -0.238183, "nauc_recall_at_10_diff1": 0.160772, "nauc_recall_at_20_max": -0.025181, "nauc_recall_at_20_std": -0.241566, "nauc_recall_at_20_diff1": 0.139194, "nauc_recall_at_100_max": 0.276975, "nauc_recall_at_100_std": -0.090833, "nauc_recall_at_100_diff1": 0.225768, "nauc_recall_at_1000_max": NaN, "nauc_recall_at_1000_std": NaN, "nauc_recall_at_1000_diff1": NaN, "nauc_precision_at_1_max": 0.24975, "nauc_precision_at_1_std": -0.076333, "nauc_precision_at_1_diff1": 0.438922, "nauc_precision_at_3_max": 0.241399, "nauc_precision_at_3_std": -0.020602, "nauc_precision_at_3_diff1": 0.116575, "nauc_precision_at_5_max": 0.261874, "nauc_precision_at_5_std": -0.001178, "nauc_precision_at_5_diff1": 0.052353, "nauc_precision_at_10_max": 0.286662, "nauc_precision_at_10_std": 0.052837, "nauc_precision_at_10_diff1": -0.09112, "nauc_precision_at_20_max": 0.251673, "nauc_precision_at_20_std": 0.102291, "nauc_precision_at_20_diff1": -0.140305, "nauc_precision_at_100_max": 0.2627, "nauc_precision_at_100_std": 0.182076, "nauc_precision_at_100_diff1": -0.141425, "nauc_precision_at_1000_max": 0.224496, "nauc_precision_at_1000_std": 0.195252, "nauc_precision_at_1000_diff1": -0.154029, "nauc_mrr_at_1_max": 0.24975, "nauc_mrr_at_1_std": -0.076333, "nauc_mrr_at_1_diff1": 0.438922, "nauc_mrr_at_3_max": 0.268761, "nauc_mrr_at_3_std": -0.123734, "nauc_mrr_at_3_diff1": 0.45186, "nauc_mrr_at_5_max": 0.280846, "nauc_mrr_at_5_std": -0.103958, "nauc_mrr_at_5_diff1": 0.450773, "nauc_mrr_at_10_max": 0.274122, "nauc_mrr_at_10_std": -0.099043, "nauc_mrr_at_10_diff1": 0.448179, "nauc_mrr_at_20_max": 0.27257, "nauc_mrr_at_20_std": -0.101849, "nauc_mrr_at_20_diff1": 0.447295, "nauc_mrr_at_100_max": 0.271921, "nauc_mrr_at_100_std": -0.101878, "nauc_mrr_at_100_diff1": 0.448888, "nauc_mrr_at_1000_max": 0.271906, "nauc_mrr_at_1000_std": -0.101866, "nauc_mrr_at_1000_diff1": 0.448793, "hit_rate_at_1": 0.63721, "hit_rate_at_3": 0.85581, "hit_rate_at_5": 0.91628, "hit_rate_at_10": 0.96744, "hit_rate_at_20": 0.9814, "hit_rate_at_100": 0.99535, "hit_rate_at_1000": 1.0, "main_score": 0.61664, "hf_subset": "portuguese", "languages": [ "por-Latn" ], "mteb_version": "2.18.1" } ] }, "evaluation_time": 26986.147141884983, "kg_co2_emissions": null, "date": 1785890155.554136, "evaluation_phases": [ { "name": "Data loading", "start": 6.0224003391340375e-05, "end": 53.485152280001785, "split": "", "subset": "" }, { "name": "Dataset transform", "start": 53.48523125999782, "end": 53.48523566800577, "split": "", "subset": "" }, { "name": "Encoding queries and documents", "start": 53.5012965240021, "end": 4583.121749940998, "split": "test", "subset": "french" }, { "name": "Scoring", "start": 4583.122090735997, "end": 4583.397712883001, "split": "test", "subset": "french" }, { "name": "Encoding queries and documents", "start": 4583.436026538009, "end": 9110.785585395002, "split": "test", "subset": "spanish" }, { "name": "Scoring", "start": 9110.786026119007, "end": 9111.025358558007, "split": "test", "subset": "spanish" }, { "name": "Encoding queries and documents", "start": 9111.06655062201, "end": 13575.47299706201, "split": "test", "subset": "english" }, { "name": "Scoring", "start": 13575.473294746, "end": 13575.710957283009, "split": "test", "subset": "english" }, { "name": "Encoding queries and documents", "start": 13575.749243375001, "end": 18067.391656897, "split": "test", "subset": "german" }, { "name": "Scoring", "start": 18067.391932849, "end": 18067.630949586004, "split": "test", "subset": "german" }, { "name": "Encoding queries and documents", "start": 18067.67060097601, "end": 22524.38421975801, "split": "test", "subset": "italian" }, { "name": "Scoring", "start": 22524.384477546002, "end": 22524.623717073002, "split": "test", "subset": "italian" }, { "name": "Encoding queries and documents", "start": 22524.66358529401, "end": 26985.88079376999, "split": "test", "subset": "portuguese" }, { "name": "Scoring", "start": 26985.88104692899, "end": 26986.118263629993, "split": "test", "subset": "portuguese" } ] }