{ "dataset_revision": "87e635f6d399614218191eb8b9b85ccfd809e6bb", "task_name": "Vidore3PharmaceuticalsRetrieval.v2", "mteb_version": "2.13.1", "scores": { "test": [ { "ndcg_at_1": 0.27335, "ndcg_at_3": 0.28817, "ndcg_at_5": 0.29896, "ndcg_at_10": 0.31495, "ndcg_at_20": 0.34478, "ndcg_at_100": 0.40038, "ndcg_at_1000": 0.45656, "map_at_1": 0.13863, "map_at_3": 0.19694, "map_at_5": 0.2163, "map_at_10": 0.23431, "map_at_20": 0.24977, "map_at_100": 0.26431, "map_at_1000": 0.26863, "recall_at_1": 0.13863, "recall_at_3": 0.23537, "recall_at_5": 0.28731, "recall_at_10": 0.35624, "recall_at_20": 0.45344, "recall_at_100": 0.65059, "recall_at_1000": 0.93659, "accuracy": 0.13863, "precision_at_1": 0.29121, "precision_at_3": 0.2033, "precision_at_5": 0.16484, "precision_at_10": 0.11511, "precision_at_20": 0.07802, "precision_at_100": 0.02588, "precision_at_1000": 0.00427, "mrr_at_1": 0.291209, "mrr_at_3": 0.360348, "mrr_at_5": 0.379991, "mrr_at_10": 0.392188, "mrr_at_20": 0.398556, "mrr_at_100": 0.402588, "mrr_at_1000": 0.403192, "nauc_ndcg_at_1_max": 0.069507, "nauc_ndcg_at_1_std": 0.09083, "nauc_ndcg_at_1_diff1": 0.378552, "nauc_ndcg_at_3_max": 0.076409, "nauc_ndcg_at_3_std": 0.063318, "nauc_ndcg_at_3_diff1": 0.348933, "nauc_ndcg_at_5_max": 0.075011, "nauc_ndcg_at_5_std": 0.092205, "nauc_ndcg_at_5_diff1": 0.34869, "nauc_ndcg_at_10_max": 0.093242, "nauc_ndcg_at_10_std": 0.122587, "nauc_ndcg_at_10_diff1": 0.344644, "nauc_ndcg_at_20_max": 0.101938, "nauc_ndcg_at_20_std": 0.15089, "nauc_ndcg_at_20_diff1": 0.352542, "nauc_ndcg_at_100_max": 0.128199, "nauc_ndcg_at_100_std": 0.173526, "nauc_ndcg_at_100_diff1": 0.342179, "nauc_ndcg_at_1000_max": 0.141961, "nauc_ndcg_at_1000_std": 0.166965, "nauc_ndcg_at_1000_diff1": 0.34775, "nauc_map_at_1_max": 0.083813, "nauc_map_at_1_std": 0.086425, "nauc_map_at_1_diff1": 0.430506, "nauc_map_at_3_max": 0.0621, "nauc_map_at_3_std": 0.053149, "nauc_map_at_3_diff1": 0.372543, "nauc_map_at_5_max": 0.063748, "nauc_map_at_5_std": 0.070731, "nauc_map_at_5_diff1": 0.364883, "nauc_map_at_10_max": 0.072112, "nauc_map_at_10_std": 0.086624, "nauc_map_at_10_diff1": 0.353398, "nauc_map_at_20_max": 0.084836, "nauc_map_at_20_std": 0.101633, "nauc_map_at_20_diff1": 0.357914, "nauc_map_at_100_max": 0.093702, "nauc_map_at_100_std": 0.110324, "nauc_map_at_100_diff1": 0.350248, "nauc_map_at_1000_max": 0.095471, "nauc_map_at_1000_std": 0.11136, "nauc_map_at_1000_diff1": 0.35048, "nauc_recall_at_1_max": 0.083813, "nauc_recall_at_1_std": 0.086425, "nauc_recall_at_1_diff1": 0.430506, "nauc_recall_at_3_max": 0.037249, "nauc_recall_at_3_std": 0.038857, "nauc_recall_at_3_diff1": 0.350661, "nauc_recall_at_5_max": 0.033038, "nauc_recall_at_5_std": 0.084995, "nauc_recall_at_5_diff1": 0.319925, "nauc_recall_at_10_max": 0.069527, "nauc_recall_at_10_std": 0.120208, "nauc_recall_at_10_diff1": 0.275806, "nauc_recall_at_20_max": 0.069055, "nauc_recall_at_20_std": 0.170912, "nauc_recall_at_20_diff1": 0.276145, "nauc_recall_at_100_max": 0.109974, "nauc_recall_at_100_std": 0.239522, "nauc_recall_at_100_diff1": 0.255112, "nauc_recall_at_1000_max": 0.112515, "nauc_recall_at_1000_std": 0.364526, "nauc_recall_at_1000_diff1": 0.321362, "nauc_precision_at_1_max": 0.055937, "nauc_precision_at_1_std": 0.088895, "nauc_precision_at_1_diff1": 0.356704, "nauc_precision_at_3_max": 0.070279, "nauc_precision_at_3_std": 0.029569, "nauc_precision_at_3_diff1": 0.183447, "nauc_precision_at_5_max": 0.086923, "nauc_precision_at_5_std": 0.095023, "nauc_precision_at_5_diff1": 0.142876, "nauc_precision_at_10_max": 0.145277, "nauc_precision_at_10_std": 0.165785, "nauc_precision_at_10_diff1": 0.061614, "nauc_precision_at_20_max": 0.201669, "nauc_precision_at_20_std": 0.21165, "nauc_precision_at_20_diff1": 0.051561, "nauc_precision_at_100_max": 0.241358, "nauc_precision_at_100_std": 0.19913, "nauc_precision_at_100_diff1": -0.033201, "nauc_precision_at_1000_max": 0.214628, "nauc_precision_at_1000_std": 0.078761, "nauc_precision_at_1000_diff1": -0.089511, "nauc_mrr_at_1_max": 0.055937, "nauc_mrr_at_1_std": 0.088895, "nauc_mrr_at_1_diff1": 0.356704, "nauc_mrr_at_3_max": 0.084486, "nauc_mrr_at_3_std": 0.082876, "nauc_mrr_at_3_diff1": 0.349308, "nauc_mrr_at_5_max": 0.093072, "nauc_mrr_at_5_std": 0.107368, "nauc_mrr_at_5_diff1": 0.341206, "nauc_mrr_at_10_max": 0.101313, "nauc_mrr_at_10_std": 0.115737, "nauc_mrr_at_10_diff1": 0.336845, "nauc_mrr_at_20_max": 0.1013, "nauc_mrr_at_20_std": 0.117125, "nauc_mrr_at_20_diff1": 0.339777, "nauc_mrr_at_100_max": 0.099937, "nauc_mrr_at_100_std": 0.115028, "nauc_mrr_at_100_diff1": 0.341239, "nauc_mrr_at_1000_max": 0.099543, "nauc_mrr_at_1000_std": 0.114722, "nauc_mrr_at_1000_diff1": 0.341098, "hit_rate_at_1": 0.29121, "hit_rate_at_3": 0.4533, "hit_rate_at_5": 0.53846, "hit_rate_at_10": 0.63187, "hit_rate_at_20": 0.72253, "hit_rate_at_100": 0.85989, "hit_rate_at_1000": 0.99176, "main_score": 0.31495, "hf_subset": "french", "languages": [ "fra-Latn" ], "mteb_version": "2.13.1" }, { "ndcg_at_1": 0.19231, "ndcg_at_3": 0.19977, "ndcg_at_5": 0.21155, "ndcg_at_10": 0.22835, "ndcg_at_20": 0.25421, "ndcg_at_100": 0.31712, "ndcg_at_1000": 0.37871, "map_at_1": 0.09517, "map_at_3": 0.13475, "map_at_5": 0.1499, "map_at_10": 0.16751, "map_at_20": 0.17969, "map_at_100": 0.19285, "map_at_1000": 0.197, "recall_at_1": 0.09517, "recall_at_3": 0.16447, "recall_at_5": 0.20881, "recall_at_10": 0.27362, "recall_at_20": 0.35703, "recall_at_100": 0.5883, "recall_at_1000": 0.91261, "accuracy": 0.09517, "precision_at_1": 0.20604, "precision_at_3": 0.14377, "precision_at_5": 0.11978, "precision_at_10": 0.08901, "precision_at_20": 0.06209, "precision_at_100": 0.02258, "precision_at_1000": 0.00411, "mrr_at_1": 0.206044, "mrr_at_3": 0.260073, "mrr_at_5": 0.276969, "mrr_at_10": 0.287323, "mrr_at_20": 0.293994, "mrr_at_100": 0.30026, "mrr_at_1000": 0.300995, "nauc_ndcg_at_1_max": 0.205108, "nauc_ndcg_at_1_std": 0.108851, "nauc_ndcg_at_1_diff1": 0.331092, "nauc_ndcg_at_3_max": 0.185161, "nauc_ndcg_at_3_std": 0.138996, "nauc_ndcg_at_3_diff1": 0.314373, "nauc_ndcg_at_5_max": 0.181771, "nauc_ndcg_at_5_std": 0.125325, "nauc_ndcg_at_5_diff1": 0.32449, "nauc_ndcg_at_10_max": 0.191609, "nauc_ndcg_at_10_std": 0.1502, "nauc_ndcg_at_10_diff1": 0.313626, "nauc_ndcg_at_20_max": 0.197471, "nauc_ndcg_at_20_std": 0.188654, "nauc_ndcg_at_20_diff1": 0.330791, "nauc_ndcg_at_100_max": 0.21782, "nauc_ndcg_at_100_std": 0.22341, "nauc_ndcg_at_100_diff1": 0.318444, "nauc_ndcg_at_1000_max": 0.24121, "nauc_ndcg_at_1000_std": 0.213401, "nauc_ndcg_at_1000_diff1": 0.317111, "nauc_map_at_1_max": 0.213107, "nauc_map_at_1_std": 0.119405, "nauc_map_at_1_diff1": 0.494204, "nauc_map_at_3_max": 0.177835, "nauc_map_at_3_std": 0.125987, "nauc_map_at_3_diff1": 0.390885, "nauc_map_at_5_max": 0.171263, "nauc_map_at_5_std": 0.113304, "nauc_map_at_5_diff1": 0.375558, "nauc_map_at_10_max": 0.183635, "nauc_map_at_10_std": 0.130814, "nauc_map_at_10_diff1": 0.356447, "nauc_map_at_20_max": 0.191878, "nauc_map_at_20_std": 0.151326, "nauc_map_at_20_diff1": 0.357662, "nauc_map_at_100_max": 0.200529, "nauc_map_at_100_std": 0.165698, "nauc_map_at_100_diff1": 0.353844, "nauc_map_at_1000_max": 0.203539, "nauc_map_at_1000_std": 0.167054, "nauc_map_at_1000_diff1": 0.352733, "nauc_recall_at_1_max": 0.213107, "nauc_recall_at_1_std": 0.119405, "nauc_recall_at_1_diff1": 0.494204, "nauc_recall_at_3_max": 0.133293, "nauc_recall_at_3_std": 0.1411, "nauc_recall_at_3_diff1": 0.3084, "nauc_recall_at_5_max": 0.104851, "nauc_recall_at_5_std": 0.098027, "nauc_recall_at_5_diff1": 0.277592, "nauc_recall_at_10_max": 0.12012, "nauc_recall_at_10_std": 0.130147, "nauc_recall_at_10_diff1": 0.217915, "nauc_recall_at_20_max": 0.130102, "nauc_recall_at_20_std": 0.213966, "nauc_recall_at_20_diff1": 0.266453, "nauc_recall_at_100_max": 0.13787, "nauc_recall_at_100_std": 0.293589, "nauc_recall_at_100_diff1": 0.229936, "nauc_recall_at_1000_max": 0.283603, "nauc_recall_at_1000_std": 0.393372, "nauc_recall_at_1000_diff1": 0.206812, "nauc_precision_at_1_max": 0.195334, "nauc_precision_at_1_std": 0.098618, "nauc_precision_at_1_diff1": 0.339962, "nauc_precision_at_3_max": 0.176438, "nauc_precision_at_3_std": 0.118998, "nauc_precision_at_3_diff1": 0.192073, "nauc_precision_at_5_max": 0.190533, "nauc_precision_at_5_std": 0.098622, "nauc_precision_at_5_diff1": 0.140924, "nauc_precision_at_10_max": 0.232259, "nauc_precision_at_10_std": 0.161472, "nauc_precision_at_10_diff1": 0.09737, "nauc_precision_at_20_max": 0.249853, "nauc_precision_at_20_std": 0.214748, "nauc_precision_at_20_diff1": 0.085653, "nauc_precision_at_100_max": 0.257479, "nauc_precision_at_100_std": 0.233594, "nauc_precision_at_100_diff1": 0.001544, "nauc_precision_at_1000_max": 0.183359, "nauc_precision_at_1000_std": 0.070084, "nauc_precision_at_1000_diff1": -0.102896, "nauc_mrr_at_1_max": 0.195334, "nauc_mrr_at_1_std": 0.098618, "nauc_mrr_at_1_diff1": 0.339962, "nauc_mrr_at_3_max": 0.182733, "nauc_mrr_at_3_std": 0.12782, "nauc_mrr_at_3_diff1": 0.297432, "nauc_mrr_at_5_max": 0.193127, "nauc_mrr_at_5_std": 0.123873, "nauc_mrr_at_5_diff1": 0.290854, "nauc_mrr_at_10_max": 0.200596, "nauc_mrr_at_10_std": 0.126266, "nauc_mrr_at_10_diff1": 0.285881, "nauc_mrr_at_20_max": 0.204191, "nauc_mrr_at_20_std": 0.136808, "nauc_mrr_at_20_diff1": 0.290376, "nauc_mrr_at_100_max": 0.20358, "nauc_mrr_at_100_std": 0.137656, "nauc_mrr_at_100_diff1": 0.289109, "nauc_mrr_at_1000_max": 0.203247, "nauc_mrr_at_1000_std": 0.137012, "nauc_mrr_at_1000_diff1": 0.289517, "hit_rate_at_1": 0.20604, "hit_rate_at_3": 0.32967, "hit_rate_at_5": 0.40385, "hit_rate_at_10": 0.48077, "hit_rate_at_20": 0.57967, "hit_rate_at_100": 0.81868, "hit_rate_at_1000": 0.98626, "main_score": 0.22835, "hf_subset": "spanish", "languages": [ "spa-Latn" ], "mteb_version": "2.13.1" }, { "ndcg_at_1": 0.4272, "ndcg_at_3": 0.44285, "ndcg_at_5": 0.45807, "ndcg_at_10": 0.47838, "ndcg_at_20": 0.51142, "ndcg_at_100": 0.5683, "ndcg_at_1000": 0.60135, "map_at_1": 0.22523, "map_at_3": 0.30304, "map_at_5": 0.33768, "map_at_10": 0.36915, "map_at_20": 0.39265, "map_at_100": 0.41049, "map_at_1000": 0.41429, "recall_at_1": 0.22523, "recall_at_3": 0.35183, "recall_at_5": 0.42993, "recall_at_10": 0.53093, "recall_at_20": 0.63358, "recall_at_100": 0.83017, "recall_at_1000": 0.98024, "accuracy": 0.22523, "precision_at_1": 0.44505, "precision_at_3": 0.30586, "precision_at_5": 0.2533, "precision_at_10": 0.17775, "precision_at_20": 0.11758, "precision_at_100": 0.03511, "precision_at_1000": 0.0046, "mrr_at_1": 0.445055, "mrr_at_3": 0.534341, "mrr_at_5": 0.557555, "mrr_at_10": 0.569606, "mrr_at_20": 0.573594, "mrr_at_100": 0.575687, "mrr_at_1000": 0.575811, "nauc_ndcg_at_1_max": 0.010866, "nauc_ndcg_at_1_std": -0.143133, "nauc_ndcg_at_1_diff1": 0.363324, "nauc_ndcg_at_3_max": 0.025735, "nauc_ndcg_at_3_std": -0.139574, "nauc_ndcg_at_3_diff1": 0.302918, "nauc_ndcg_at_5_max": 0.020961, "nauc_ndcg_at_5_std": -0.133713, "nauc_ndcg_at_5_diff1": 0.304397, "nauc_ndcg_at_10_max": 0.022612, "nauc_ndcg_at_10_std": -0.119885, "nauc_ndcg_at_10_diff1": 0.297782, "nauc_ndcg_at_20_max": 0.032893, "nauc_ndcg_at_20_std": -0.10232, "nauc_ndcg_at_20_diff1": 0.313428, "nauc_ndcg_at_100_max": 0.048028, "nauc_ndcg_at_100_std": -0.082554, "nauc_ndcg_at_100_diff1": 0.323023, "nauc_ndcg_at_1000_max": 0.068947, "nauc_ndcg_at_1000_std": -0.087613, "nauc_ndcg_at_1000_diff1": 0.321092, "nauc_map_at_1_max": -0.041657, "nauc_map_at_1_std": -0.119471, "nauc_map_at_1_diff1": 0.377317, "nauc_map_at_3_max": -0.037592, "nauc_map_at_3_std": -0.141606, "nauc_map_at_3_diff1": 0.310618, "nauc_map_at_5_max": -0.025195, "nauc_map_at_5_std": -0.13558, "nauc_map_at_5_diff1": 0.302718, "nauc_map_at_10_max": -0.008172, "nauc_map_at_10_std": -0.12408, "nauc_map_at_10_diff1": 0.299585, "nauc_map_at_20_max": 0.005522, "nauc_map_at_20_std": -0.116255, "nauc_map_at_20_diff1": 0.304383, "nauc_map_at_100_max": 0.009295, "nauc_map_at_100_std": -0.111435, "nauc_map_at_100_diff1": 0.304532, "nauc_map_at_1000_max": 0.012477, "nauc_map_at_1000_std": -0.110891, "nauc_map_at_1000_diff1": 0.304616, "nauc_recall_at_1_max": -0.041657, "nauc_recall_at_1_std": -0.119471, "nauc_recall_at_1_diff1": 0.377317, "nauc_recall_at_3_max": -0.037838, "nauc_recall_at_3_std": -0.121318, "nauc_recall_at_3_diff1": 0.245937, "nauc_recall_at_5_max": -0.030554, "nauc_recall_at_5_std": -0.124398, "nauc_recall_at_5_diff1": 0.208274, "nauc_recall_at_10_max": -0.017524, "nauc_recall_at_10_std": -0.086647, "nauc_recall_at_10_diff1": 0.168019, "nauc_recall_at_20_max": -0.002774, "nauc_recall_at_20_std": -0.06547, "nauc_recall_at_20_diff1": 0.21278, "nauc_recall_at_100_max": -0.027964, "nauc_recall_at_100_std": 0.048522, "nauc_recall_at_100_diff1": 0.273132, "nauc_recall_at_1000_max": 0.108648, "nauc_recall_at_1000_std": 0.362319, "nauc_recall_at_1000_diff1": 0.395487, "nauc_precision_at_1_max": 0.014344, "nauc_precision_at_1_std": -0.150428, "nauc_precision_at_1_diff1": 0.378884, "nauc_precision_at_3_max": 0.080533, "nauc_precision_at_3_std": -0.107733, "nauc_precision_at_3_diff1": 0.113115, "nauc_precision_at_5_max": 0.119488, "nauc_precision_at_5_std": -0.030026, "nauc_precision_at_5_diff1": 0.072407, "nauc_precision_at_10_max": 0.161196, "nauc_precision_at_10_std": 0.032963, "nauc_precision_at_10_diff1": 0.018102, "nauc_precision_at_20_max": 0.190952, "nauc_precision_at_20_std": 0.065879, "nauc_precision_at_20_diff1": 0.021129, "nauc_precision_at_100_max": 0.194034, "nauc_precision_at_100_std": 0.11423, "nauc_precision_at_100_diff1": -0.045557, "nauc_precision_at_1000_max": 0.221399, "nauc_precision_at_1000_std": 0.10552, "nauc_precision_at_1000_diff1": -0.095872, "nauc_mrr_at_1_max": 0.014344, "nauc_mrr_at_1_std": -0.150428, "nauc_mrr_at_1_diff1": 0.378884, "nauc_mrr_at_3_max": 0.052037, "nauc_mrr_at_3_std": -0.152255, "nauc_mrr_at_3_diff1": 0.342059, "nauc_mrr_at_5_max": 0.05605, "nauc_mrr_at_5_std": -0.150512, "nauc_mrr_at_5_diff1": 0.34213, "nauc_mrr_at_10_max": 0.061858, "nauc_mrr_at_10_std": -0.141799, "nauc_mrr_at_10_diff1": 0.343769, "nauc_mrr_at_20_max": 0.061819, "nauc_mrr_at_20_std": -0.141423, "nauc_mrr_at_20_diff1": 0.346532, "nauc_mrr_at_100_max": 0.060221, "nauc_mrr_at_100_std": -0.141709, "nauc_mrr_at_100_diff1": 0.348311, "nauc_mrr_at_1000_max": 0.060013, "nauc_mrr_at_1000_std": -0.141915, "nauc_mrr_at_1000_diff1": 0.348284, "hit_rate_at_1": 0.44505, "hit_rate_at_3": 0.6511, "hit_rate_at_5": 0.75, "hit_rate_at_10": 0.83516, "hit_rate_at_20": 0.89286, "hit_rate_at_100": 0.97253, "hit_rate_at_1000": 0.99725, "main_score": 0.47838, "hf_subset": "english", "languages": [ "eng-Latn" ], "mteb_version": "2.13.1" }, { "ndcg_at_1": 0.16621, "ndcg_at_3": 0.17076, "ndcg_at_5": 0.17783, "ndcg_at_10": 0.19659, "ndcg_at_20": 0.22123, "ndcg_at_100": 0.27371, "ndcg_at_1000": 0.34193, "map_at_1": 0.08386, "map_at_3": 0.11616, "map_at_5": 0.12835, "map_at_10": 0.14264, "map_at_20": 0.15413, "map_at_100": 0.16553, "map_at_1000": 0.16955, "recall_at_1": 0.08386, "recall_at_3": 0.14201, "recall_at_5": 0.17801, "recall_at_10": 0.2398, "recall_at_20": 0.31524, "recall_at_100": 0.51127, "recall_at_1000": 0.87821, "accuracy": 0.08386, "precision_at_1": 0.17582, "precision_at_3": 0.12454, "precision_at_5": 0.10055, "precision_at_10": 0.07665, "precision_at_20": 0.0533, "precision_at_100": 0.0194, "precision_at_1000": 0.0039, "mrr_at_1": 0.175824, "mrr_at_3": 0.217949, "mrr_at_5": 0.233196, "mrr_at_10": 0.245871, "mrr_at_20": 0.251746, "mrr_at_100": 0.256316, "mrr_at_1000": 0.257427, "nauc_ndcg_at_1_max": 0.26384, "nauc_ndcg_at_1_std": 0.045503, "nauc_ndcg_at_1_diff1": 0.291443, "nauc_ndcg_at_3_max": 0.211654, "nauc_ndcg_at_3_std": 0.075805, "nauc_ndcg_at_3_diff1": 0.316773, "nauc_ndcg_at_5_max": 0.217508, "nauc_ndcg_at_5_std": 0.053617, "nauc_ndcg_at_5_diff1": 0.298137, "nauc_ndcg_at_10_max": 0.23139, "nauc_ndcg_at_10_std": 0.080308, "nauc_ndcg_at_10_diff1": 0.270486, "nauc_ndcg_at_20_max": 0.230851, "nauc_ndcg_at_20_std": 0.094663, "nauc_ndcg_at_20_diff1": 0.26505, "nauc_ndcg_at_100_max": 0.252756, "nauc_ndcg_at_100_std": 0.127882, "nauc_ndcg_at_100_diff1": 0.267569, "nauc_ndcg_at_1000_max": 0.285207, "nauc_ndcg_at_1000_std": 0.131085, "nauc_ndcg_at_1000_diff1": 0.28364, "nauc_map_at_1_max": 0.231846, "nauc_map_at_1_std": 0.007132, "nauc_map_at_1_diff1": 0.348037, "nauc_map_at_3_max": 0.187111, "nauc_map_at_3_std": 0.016914, "nauc_map_at_3_diff1": 0.305492, "nauc_map_at_5_max": 0.185467, "nauc_map_at_5_std": 0.016352, "nauc_map_at_5_diff1": 0.296828, "nauc_map_at_10_max": 0.20424, "nauc_map_at_10_std": 0.047358, "nauc_map_at_10_diff1": 0.286356, "nauc_map_at_20_max": 0.208846, "nauc_map_at_20_std": 0.058514, "nauc_map_at_20_diff1": 0.277583, "nauc_map_at_100_max": 0.222189, "nauc_map_at_100_std": 0.070725, "nauc_map_at_100_diff1": 0.279257, "nauc_map_at_1000_max": 0.226708, "nauc_map_at_1000_std": 0.074025, "nauc_map_at_1000_diff1": 0.280309, "nauc_recall_at_1_max": 0.231846, "nauc_recall_at_1_std": 0.007132, "nauc_recall_at_1_diff1": 0.348037, "nauc_recall_at_3_max": 0.126508, "nauc_recall_at_3_std": 0.037074, "nauc_recall_at_3_diff1": 0.265391, "nauc_recall_at_5_max": 0.115778, "nauc_recall_at_5_std": 0.020177, "nauc_recall_at_5_diff1": 0.227242, "nauc_recall_at_10_max": 0.174617, "nauc_recall_at_10_std": 0.094399, "nauc_recall_at_10_diff1": 0.186971, "nauc_recall_at_20_max": 0.157455, "nauc_recall_at_20_std": 0.103804, "nauc_recall_at_20_diff1": 0.171835, "nauc_recall_at_100_max": 0.170097, "nauc_recall_at_100_std": 0.193464, "nauc_recall_at_100_diff1": 0.167853, "nauc_recall_at_1000_max": 0.298087, "nauc_recall_at_1000_std": 0.363333, "nauc_recall_at_1000_diff1": 0.215983, "nauc_precision_at_1_max": 0.265337, "nauc_precision_at_1_std": 0.043894, "nauc_precision_at_1_diff1": 0.282012, "nauc_precision_at_3_max": 0.212969, "nauc_precision_at_3_std": 0.119528, "nauc_precision_at_3_diff1": 0.284128, "nauc_precision_at_5_max": 0.243517, "nauc_precision_at_5_std": 0.089398, "nauc_precision_at_5_diff1": 0.251669, "nauc_precision_at_10_max": 0.296192, "nauc_precision_at_10_std": 0.165662, "nauc_precision_at_10_diff1": 0.184428, "nauc_precision_at_20_max": 0.296023, "nauc_precision_at_20_std": 0.209838, "nauc_precision_at_20_diff1": 0.152918, "nauc_precision_at_100_max": 0.332408, "nauc_precision_at_100_std": 0.218391, "nauc_precision_at_100_diff1": 0.131065, "nauc_precision_at_1000_max": 0.281813, "nauc_precision_at_1000_std": 0.113623, "nauc_precision_at_1000_diff1": 0.088121, "nauc_mrr_at_1_max": 0.265337, "nauc_mrr_at_1_std": 0.043894, "nauc_mrr_at_1_diff1": 0.282012, "nauc_mrr_at_3_max": 0.248689, "nauc_mrr_at_3_std": 0.084841, "nauc_mrr_at_3_diff1": 0.30543, "nauc_mrr_at_5_max": 0.259874, "nauc_mrr_at_5_std": 0.074883, "nauc_mrr_at_5_diff1": 0.291871, "nauc_mrr_at_10_max": 0.271957, "nauc_mrr_at_10_std": 0.084891, "nauc_mrr_at_10_diff1": 0.281784, "nauc_mrr_at_20_max": 0.27146, "nauc_mrr_at_20_std": 0.087729, "nauc_mrr_at_20_diff1": 0.284125, "nauc_mrr_at_100_max": 0.270275, "nauc_mrr_at_100_std": 0.088913, "nauc_mrr_at_100_diff1": 0.283962, "nauc_mrr_at_1000_max": 0.270172, "nauc_mrr_at_1000_std": 0.088169, "nauc_mrr_at_1000_diff1": 0.284221, "hit_rate_at_1": 0.17582, "hit_rate_at_3": 0.27747, "hit_rate_at_5": 0.34341, "hit_rate_at_10": 0.43956, "hit_rate_at_20": 0.52473, "hit_rate_at_100": 0.7033, "hit_rate_at_1000": 0.97802, "main_score": 0.19659, "hf_subset": "german", "languages": [ "deu-Latn" ], "mteb_version": "2.13.1" }, { "ndcg_at_1": 0.18269, "ndcg_at_3": 0.18905, "ndcg_at_5": 0.19333, "ndcg_at_10": 0.21389, "ndcg_at_20": 0.23375, "ndcg_at_100": 0.28154, "ndcg_at_1000": 0.35369, "map_at_1": 0.1013, "map_at_3": 0.13571, "map_at_5": 0.14531, "map_at_10": 0.16131, "map_at_20": 0.17043, "map_at_100": 0.1806, "map_at_1000": 0.18477, "recall_at_1": 0.1013, "recall_at_3": 0.16382, "recall_at_5": 0.18783, "recall_at_10": 0.25743, "recall_at_20": 0.3191, "recall_at_100": 0.50199, "recall_at_1000": 0.88458, "accuracy": 0.1013, "precision_at_1": 0.19505, "precision_at_3": 0.12454, "precision_at_5": 0.0967, "precision_at_10": 0.07582, "precision_at_20": 0.05027, "precision_at_100": 0.01794, "precision_at_1000": 0.00388, "mrr_at_1": 0.195055, "mrr_at_3": 0.238095, "mrr_at_5": 0.247573, "mrr_at_10": 0.26102, "mrr_at_20": 0.265706, "mrr_at_100": 0.27029, "mrr_at_1000": 0.271547, "nauc_ndcg_at_1_max": 0.294195, "nauc_ndcg_at_1_std": 0.180817, "nauc_ndcg_at_1_diff1": 0.256738, "nauc_ndcg_at_3_max": 0.227257, "nauc_ndcg_at_3_std": 0.174456, "nauc_ndcg_at_3_diff1": 0.23933, "nauc_ndcg_at_5_max": 0.229741, "nauc_ndcg_at_5_std": 0.169398, "nauc_ndcg_at_5_diff1": 0.247654, "nauc_ndcg_at_10_max": 0.230064, "nauc_ndcg_at_10_std": 0.178449, "nauc_ndcg_at_10_diff1": 0.230568, "nauc_ndcg_at_20_max": 0.232619, "nauc_ndcg_at_20_std": 0.187656, "nauc_ndcg_at_20_diff1": 0.229335, "nauc_ndcg_at_100_max": 0.268794, "nauc_ndcg_at_100_std": 0.227782, "nauc_ndcg_at_100_diff1": 0.223196, "nauc_ndcg_at_1000_max": 0.295762, "nauc_ndcg_at_1000_std": 0.230311, "nauc_ndcg_at_1000_diff1": 0.215719, "nauc_map_at_1_max": 0.215155, "nauc_map_at_1_std": 0.157954, "nauc_map_at_1_diff1": 0.31115, "nauc_map_at_3_max": 0.190483, "nauc_map_at_3_std": 0.164971, "nauc_map_at_3_diff1": 0.260886, "nauc_map_at_5_max": 0.195499, "nauc_map_at_5_std": 0.156985, "nauc_map_at_5_diff1": 0.254932, "nauc_map_at_10_max": 0.20499, "nauc_map_at_10_std": 0.168906, "nauc_map_at_10_diff1": 0.241465, "nauc_map_at_20_max": 0.213766, "nauc_map_at_20_std": 0.177616, "nauc_map_at_20_diff1": 0.242447, "nauc_map_at_100_max": 0.228472, "nauc_map_at_100_std": 0.189679, "nauc_map_at_100_diff1": 0.241308, "nauc_map_at_1000_max": 0.232149, "nauc_map_at_1000_std": 0.19268, "nauc_map_at_1000_diff1": 0.240529, "nauc_recall_at_1_max": 0.215155, "nauc_recall_at_1_std": 0.157954, "nauc_recall_at_1_diff1": 0.31115, "nauc_recall_at_3_max": 0.150497, "nauc_recall_at_3_std": 0.135776, "nauc_recall_at_3_diff1": 0.248229, "nauc_recall_at_5_max": 0.151865, "nauc_recall_at_5_std": 0.125245, "nauc_recall_at_5_diff1": 0.247421, "nauc_recall_at_10_max": 0.166718, "nauc_recall_at_10_std": 0.134268, "nauc_recall_at_10_diff1": 0.187403, "nauc_recall_at_20_max": 0.164377, "nauc_recall_at_20_std": 0.1423, "nauc_recall_at_20_diff1": 0.180031, "nauc_recall_at_100_max": 0.217616, "nauc_recall_at_100_std": 0.243196, "nauc_recall_at_100_diff1": 0.175836, "nauc_recall_at_1000_max": 0.334184, "nauc_recall_at_1000_std": 0.295395, "nauc_recall_at_1000_diff1": 0.17964, "nauc_precision_at_1_max": 0.283184, "nauc_precision_at_1_std": 0.17813, "nauc_precision_at_1_diff1": 0.249582, "nauc_precision_at_3_max": 0.213079, "nauc_precision_at_3_std": 0.147502, "nauc_precision_at_3_diff1": 0.137222, "nauc_precision_at_5_max": 0.267295, "nauc_precision_at_5_std": 0.155795, "nauc_precision_at_5_diff1": 0.137417, "nauc_precision_at_10_max": 0.303901, "nauc_precision_at_10_std": 0.217834, "nauc_precision_at_10_diff1": 0.078229, "nauc_precision_at_20_max": 0.304761, "nauc_precision_at_20_std": 0.223839, "nauc_precision_at_20_diff1": 0.073925, "nauc_precision_at_100_max": 0.366443, "nauc_precision_at_100_std": 0.270226, "nauc_precision_at_100_diff1": 0.040979, "nauc_precision_at_1000_max": 0.238875, "nauc_precision_at_1000_std": 0.093846, "nauc_precision_at_1000_diff1": -0.089557, "nauc_mrr_at_1_max": 0.283184, "nauc_mrr_at_1_std": 0.17813, "nauc_mrr_at_1_diff1": 0.249582, "nauc_mrr_at_3_max": 0.256541, "nauc_mrr_at_3_std": 0.155356, "nauc_mrr_at_3_diff1": 0.222372, "nauc_mrr_at_5_max": 0.267641, "nauc_mrr_at_5_std": 0.168837, "nauc_mrr_at_5_diff1": 0.226188, "nauc_mrr_at_10_max": 0.28086, "nauc_mrr_at_10_std": 0.178999, "nauc_mrr_at_10_diff1": 0.217222, "nauc_mrr_at_20_max": 0.27761, "nauc_mrr_at_20_std": 0.177546, "nauc_mrr_at_20_diff1": 0.217378, "nauc_mrr_at_100_max": 0.278776, "nauc_mrr_at_100_std": 0.178757, "nauc_mrr_at_100_diff1": 0.216872, "nauc_mrr_at_1000_max": 0.278499, "nauc_mrr_at_1000_std": 0.178583, "nauc_mrr_at_1000_diff1": 0.217041, "hit_rate_at_1": 0.19505, "hit_rate_at_3": 0.2967, "hit_rate_at_5": 0.33791, "hit_rate_at_10": 0.44505, "hit_rate_at_20": 0.51099, "hit_rate_at_100": 0.70055, "hit_rate_at_1000": 0.97527, "main_score": 0.21389, "hf_subset": "italian", "languages": [ "ita-Latn" ], "mteb_version": "2.13.1" }, { "ndcg_at_1": 0.18407, "ndcg_at_3": 0.18711, "ndcg_at_5": 0.20424, "ndcg_at_10": 0.2223, "ndcg_at_20": 0.24739, "ndcg_at_100": 0.30118, "ndcg_at_1000": 0.36666, "map_at_1": 0.08998, "map_at_3": 0.12726, "map_at_5": 0.14627, "map_at_10": 0.1635, "map_at_20": 0.17526, "map_at_100": 0.18615, "map_at_1000": 0.19036, "recall_at_1": 0.08998, "recall_at_3": 0.15553, "recall_at_5": 0.21522, "recall_at_10": 0.27739, "recall_at_20": 0.34784, "recall_at_100": 0.55135, "recall_at_1000": 0.89692, "accuracy": 0.08998, "precision_at_1": 0.19505, "precision_at_3": 0.13095, "precision_at_5": 0.11209, "precision_at_10": 0.08297, "precision_at_20": 0.05962, "precision_at_100": 0.02102, "precision_at_1000": 0.00405, "mrr_at_1": 0.195055, "mrr_at_3": 0.239469, "mrr_at_5": 0.257051, "mrr_at_10": 0.268094, "mrr_at_20": 0.275904, "mrr_at_100": 0.280894, "mrr_at_1000": 0.281755, "nauc_ndcg_at_1_max": 0.324574, "nauc_ndcg_at_1_std": 0.188309, "nauc_ndcg_at_1_diff1": 0.320571, "nauc_ndcg_at_3_max": 0.290669, "nauc_ndcg_at_3_std": 0.202228, "nauc_ndcg_at_3_diff1": 0.306336, "nauc_ndcg_at_5_max": 0.25555, "nauc_ndcg_at_5_std": 0.182161, "nauc_ndcg_at_5_diff1": 0.301111, "nauc_ndcg_at_10_max": 0.246867, "nauc_ndcg_at_10_std": 0.212091, "nauc_ndcg_at_10_diff1": 0.293354, "nauc_ndcg_at_20_max": 0.242053, "nauc_ndcg_at_20_std": 0.224673, "nauc_ndcg_at_20_diff1": 0.293601, "nauc_ndcg_at_100_max": 0.278729, "nauc_ndcg_at_100_std": 0.261077, "nauc_ndcg_at_100_diff1": 0.286444, "nauc_ndcg_at_1000_max": 0.288664, "nauc_ndcg_at_1000_std": 0.238214, "nauc_ndcg_at_1000_diff1": 0.282732, "nauc_map_at_1_max": 0.26549, "nauc_map_at_1_std": 0.13971, "nauc_map_at_1_diff1": 0.406267, "nauc_map_at_3_max": 0.264307, "nauc_map_at_3_std": 0.184598, "nauc_map_at_3_diff1": 0.332172, "nauc_map_at_5_max": 0.246844, "nauc_map_at_5_std": 0.174062, "nauc_map_at_5_diff1": 0.326106, "nauc_map_at_10_max": 0.25116, "nauc_map_at_10_std": 0.193635, "nauc_map_at_10_diff1": 0.327352, "nauc_map_at_20_max": 0.253335, "nauc_map_at_20_std": 0.201637, "nauc_map_at_20_diff1": 0.330504, "nauc_map_at_100_max": 0.267721, "nauc_map_at_100_std": 0.21583, "nauc_map_at_100_diff1": 0.332957, "nauc_map_at_1000_max": 0.269979, "nauc_map_at_1000_std": 0.215885, "nauc_map_at_1000_diff1": 0.332264, "nauc_recall_at_1_max": 0.26549, "nauc_recall_at_1_std": 0.13971, "nauc_recall_at_1_diff1": 0.406267, "nauc_recall_at_3_max": 0.246584, "nauc_recall_at_3_std": 0.208815, "nauc_recall_at_3_diff1": 0.277081, "nauc_recall_at_5_max": 0.163644, "nauc_recall_at_5_std": 0.161621, "nauc_recall_at_5_diff1": 0.235231, "nauc_recall_at_10_max": 0.153869, "nauc_recall_at_10_std": 0.206877, "nauc_recall_at_10_diff1": 0.205253, "nauc_recall_at_20_max": 0.140402, "nauc_recall_at_20_std": 0.224908, "nauc_recall_at_20_diff1": 0.213733, "nauc_recall_at_100_max": 0.227806, "nauc_recall_at_100_std": 0.347956, "nauc_recall_at_100_diff1": 0.182543, "nauc_recall_at_1000_max": 0.231628, "nauc_recall_at_1000_std": 0.265375, "nauc_recall_at_1000_diff1": 0.121622, "nauc_precision_at_1_max": 0.328573, "nauc_precision_at_1_std": 0.185187, "nauc_precision_at_1_diff1": 0.310374, "nauc_precision_at_3_max": 0.288829, "nauc_precision_at_3_std": 0.202762, "nauc_precision_at_3_diff1": 0.252012, "nauc_precision_at_5_max": 0.249602, "nauc_precision_at_5_std": 0.175974, "nauc_precision_at_5_diff1": 0.237068, "nauc_precision_at_10_max": 0.247267, "nauc_precision_at_10_std": 0.227514, "nauc_precision_at_10_diff1": 0.222898, "nauc_precision_at_20_max": 0.243342, "nauc_precision_at_20_std": 0.235818, "nauc_precision_at_20_diff1": 0.193227, "nauc_precision_at_100_max": 0.275569, "nauc_precision_at_100_std": 0.219852, "nauc_precision_at_100_diff1": 0.098729, "nauc_precision_at_1000_max": 0.14855, "nauc_precision_at_1000_std": 0.038771, "nauc_precision_at_1000_diff1": -0.061819, "nauc_mrr_at_1_max": 0.328573, "nauc_mrr_at_1_std": 0.185187, "nauc_mrr_at_1_diff1": 0.310374, "nauc_mrr_at_3_max": 0.31757, "nauc_mrr_at_3_std": 0.208415, "nauc_mrr_at_3_diff1": 0.291454, "nauc_mrr_at_5_max": 0.298768, "nauc_mrr_at_5_std": 0.193984, "nauc_mrr_at_5_diff1": 0.282876, "nauc_mrr_at_10_max": 0.300238, "nauc_mrr_at_10_std": 0.200298, "nauc_mrr_at_10_diff1": 0.272793, "nauc_mrr_at_20_max": 0.301311, "nauc_mrr_at_20_std": 0.204379, "nauc_mrr_at_20_diff1": 0.270899, "nauc_mrr_at_100_max": 0.305072, "nauc_mrr_at_100_std": 0.205238, "nauc_mrr_at_100_diff1": 0.271288, "nauc_mrr_at_1000_max": 0.304863, "nauc_mrr_at_1000_std": 0.204642, "nauc_mrr_at_1000_diff1": 0.271632, "hit_rate_at_1": 0.19505, "hit_rate_at_3": 0.29945, "hit_rate_at_5": 0.37637, "hit_rate_at_10": 0.45604, "hit_rate_at_20": 0.56593, "hit_rate_at_100": 0.77747, "hit_rate_at_1000": 0.98352, "main_score": 0.2223, "hf_subset": "portuguese", "languages": [ "por-Latn" ], "mteb_version": "2.13.1" } ] }, "evaluation_time": 7339.4418316266965, "kg_co2_emissions": null, "date": 1783168261.687758, "evaluation_phases": [ { "name": "Data loading", "start": 0.00014113099314272404, "end": 68.41544015612453, "split": "", "subset": "" }, { "name": "Dataset transform", "start": 68.41549623711035, "end": 68.41550114704296, "split": "", "subset": "" }, { "name": "Encoding queries and documents", "start": 68.43211856903508, "end": 1295.4108604420908, "split": "test", "subset": "french" }, { "name": "Scoring", "start": 1295.4110958531965, "end": 1295.713697433006, "split": "test", "subset": "french" }, { "name": "Encoding queries and documents", "start": 1295.742346264189, "end": 2510.2006580079906, "split": "test", "subset": "spanish" }, { "name": "Scoring", "start": 2510.2010674600024, "end": 2510.4920232382137, "split": "test", "subset": "spanish" }, { "name": "Encoding queries and documents", "start": 2510.5290944010485, "end": 3716.6040483471006, "split": "test", "subset": "english" }, { "name": "Scoring", "start": 3716.6042829381768, "end": 3716.907512486214, "split": "test", "subset": "english" }, { "name": "Encoding queries and documents", "start": 3716.947204558179, "end": 4918.026509251213, "split": "test", "subset": "german" }, { "name": "Scoring", "start": 4918.026757822139, "end": 4918.321233573137, "split": "test", "subset": "german" }, { "name": "Encoding queries and documents", "start": 4918.355478157988, "end": 6132.721505796071, "split": "test", "subset": "italian" }, { "name": "Scoring", "start": 6132.72178249713, "end": 6133.037812174065, "split": "test", "subset": "italian" }, { "name": "Encoding queries and documents", "start": 6133.07786311605, "end": 7339.105055462103, "split": "test", "subset": "portuguese" }, { "name": "Scoring", "start": 7339.105312544154, "end": 7339.414650296094, "split": "test", "subset": "portuguese" } ] }