{ "dataset_revision": "057fe83342b446ede7928cb51bc6c7c3223da3d0", "task_name": "Vidore3TelecomRetrieval.v2", "mteb_version": "2.13.1", "scores": { "test": [ { "ndcg_at_1": 0.62069, "ndcg_at_3": 0.59488, "ndcg_at_5": 0.62234, "ndcg_at_10": 0.65489, "ndcg_at_20": 0.6859, "ndcg_at_100": 0.72872, "ndcg_at_1000": 0.73904, "map_at_1": 0.25981, "map_at_3": 0.40923, "map_at_5": 0.47233, "map_at_10": 0.52848, "map_at_20": 0.55718, "map_at_100": 0.57709, "map_at_1000": 0.57856, "recall_at_1": 0.25981, "recall_at_3": 0.46689, "recall_at_5": 0.59227, "recall_at_10": 0.71953, "recall_at_20": 0.81665, "recall_at_100": 0.9511, "recall_at_1000": 0.99707, "accuracy": 0.25981, "precision_at_1": 0.64655, "precision_at_3": 0.45307, "precision_at_5": 0.37356, "precision_at_10": 0.25575, "precision_at_20": 0.15862, "precision_at_100": 0.04126, "precision_at_1000": 0.00446, "mrr_at_1": 0.646552, "mrr_at_3": 0.734674, "mrr_at_5": 0.749186, "mrr_at_10": 0.755437, "mrr_at_20": 0.756733, "mrr_at_100": 0.757195, "mrr_at_1000": 0.757224, "nauc_ndcg_at_1_max": 0.190323, "nauc_ndcg_at_1_std": 0.045287, "nauc_ndcg_at_1_diff1": 0.384099, "nauc_ndcg_at_3_max": 0.162043, "nauc_ndcg_at_3_std": -0.019319, "nauc_ndcg_at_3_diff1": 0.305533, "nauc_ndcg_at_5_max": 0.252222, "nauc_ndcg_at_5_std": 0.019773, "nauc_ndcg_at_5_diff1": 0.329676, "nauc_ndcg_at_10_max": 0.276274, "nauc_ndcg_at_10_std": 0.026248, "nauc_ndcg_at_10_diff1": 0.335311, "nauc_ndcg_at_20_max": 0.285325, "nauc_ndcg_at_20_std": 0.049458, "nauc_ndcg_at_20_diff1": 0.323954, "nauc_ndcg_at_100_max": 0.306038, "nauc_ndcg_at_100_std": 0.102255, "nauc_ndcg_at_100_diff1": 0.33583, "nauc_ndcg_at_1000_max": 0.28255, "nauc_ndcg_at_1000_std": 0.086546, "nauc_ndcg_at_1000_diff1": 0.340931, "nauc_map_at_1_max": 0.057047, "nauc_map_at_1_std": -0.042092, "nauc_map_at_1_diff1": 0.36692, "nauc_map_at_3_max": 0.106098, "nauc_map_at_3_std": -0.078744, "nauc_map_at_3_diff1": 0.314363, "nauc_map_at_5_max": 0.188319, "nauc_map_at_5_std": -0.037814, "nauc_map_at_5_diff1": 0.309566, "nauc_map_at_10_max": 0.228109, "nauc_map_at_10_std": -0.025015, "nauc_map_at_10_diff1": 0.318794, "nauc_map_at_20_max": 0.247086, "nauc_map_at_20_std": -0.000366, "nauc_map_at_20_diff1": 0.314524, "nauc_map_at_100_max": 0.261586, "nauc_map_at_100_std": 0.027472, "nauc_map_at_100_diff1": 0.31863, "nauc_map_at_1000_max": 0.259797, "nauc_map_at_1000_std": 0.027168, "nauc_map_at_1000_diff1": 0.319479, "nauc_recall_at_1_max": 0.057047, "nauc_recall_at_1_std": -0.042092, "nauc_recall_at_1_diff1": 0.36692, "nauc_recall_at_3_max": 0.067077, "nauc_recall_at_3_std": -0.112904, "nauc_recall_at_3_diff1": 0.226914, "nauc_recall_at_5_max": 0.202038, "nauc_recall_at_5_std": -0.062219, "nauc_recall_at_5_diff1": 0.207244, "nauc_recall_at_10_max": 0.274752, "nauc_recall_at_10_std": -0.049214, "nauc_recall_at_10_diff1": 0.219929, "nauc_recall_at_20_max": 0.33473, "nauc_recall_at_20_std": 0.052754, "nauc_recall_at_20_diff1": 0.180848, "nauc_recall_at_100_max": 0.684445, "nauc_recall_at_100_std": 0.480663, "nauc_recall_at_100_diff1": 0.198775, "nauc_recall_at_1000_max": 0.801319, "nauc_recall_at_1000_std": 0.5454, "nauc_recall_at_1000_diff1": 0.506017, "nauc_precision_at_1_max": 0.237292, "nauc_precision_at_1_std": 0.074495, "nauc_precision_at_1_diff1": 0.417163, "nauc_precision_at_3_max": 0.21443, "nauc_precision_at_3_std": 0.042571, "nauc_precision_at_3_diff1": 0.095209, "nauc_precision_at_5_max": 0.308791, "nauc_precision_at_5_std": 0.153868, "nauc_precision_at_5_diff1": 0.042406, "nauc_precision_at_10_max": 0.255461, "nauc_precision_at_10_std": 0.167737, "nauc_precision_at_10_diff1": 0.006241, "nauc_precision_at_20_max": 0.189782, "nauc_precision_at_20_std": 0.20072, "nauc_precision_at_20_diff1": -0.060748, "nauc_precision_at_100_max": 0.132839, "nauc_precision_at_100_std": 0.254597, "nauc_precision_at_100_diff1": -0.091577, "nauc_precision_at_1000_max": 0.064362, "nauc_precision_at_1000_std": 0.208719, "nauc_precision_at_1000_diff1": -0.086204, "nauc_mrr_at_1_max": 0.237292, "nauc_mrr_at_1_std": 0.074495, "nauc_mrr_at_1_diff1": 0.417163, "nauc_mrr_at_3_max": 0.259535, "nauc_mrr_at_3_std": 0.090702, "nauc_mrr_at_3_diff1": 0.386148, "nauc_mrr_at_5_max": 0.269875, "nauc_mrr_at_5_std": 0.090966, "nauc_mrr_at_5_diff1": 0.403616, "nauc_mrr_at_10_max": 0.269507, "nauc_mrr_at_10_std": 0.09558, "nauc_mrr_at_10_diff1": 0.401345, "nauc_mrr_at_20_max": 0.268219, "nauc_mrr_at_20_std": 0.096852, "nauc_mrr_at_20_diff1": 0.40139, "nauc_mrr_at_100_max": 0.267011, "nauc_mrr_at_100_std": 0.095332, "nauc_mrr_at_100_diff1": 0.402195, "nauc_mrr_at_1000_max": 0.266912, "nauc_mrr_at_1000_std": 0.095209, "nauc_mrr_at_1000_diff1": 0.40216, "hit_rate_at_1": 0.64655, "hit_rate_at_3": 0.84483, "hit_rate_at_5": 0.90805, "hit_rate_at_10": 0.95402, "hit_rate_at_20": 0.97126, "hit_rate_at_100": 0.99138, "hit_rate_at_1000": 1.0, "main_score": 0.65489, "hf_subset": "english", "languages": [ "eng-Latn" ], "mteb_version": "2.13.1" }, { "ndcg_at_1": 0.58908, "ndcg_at_3": 0.56523, "ndcg_at_5": 0.58451, "ndcg_at_10": 0.62352, "ndcg_at_20": 0.65906, "ndcg_at_100": 0.70327, "ndcg_at_1000": 0.71505, "map_at_1": 0.24867, "map_at_3": 0.38675, "map_at_5": 0.44229, "map_at_10": 0.49605, "map_at_20": 0.52535, "map_at_100": 0.54562, "map_at_1000": 0.54726, "recall_at_1": 0.24867, "recall_at_3": 0.44729, "recall_at_5": 0.55816, "recall_at_10": 0.69449, "recall_at_20": 0.80158, "recall_at_100": 0.94266, "recall_at_1000": 0.99616, "accuracy": 0.24867, "precision_at_1": 0.61494, "precision_at_3": 0.43678, "precision_at_5": 0.34943, "precision_at_10": 0.24339, "precision_at_20": 0.15517, "precision_at_100": 0.04098, "precision_at_1000": 0.00445, "mrr_at_1": 0.614943, "mrr_at_3": 0.702107, "mrr_at_5": 0.7159, "mrr_at_10": 0.725121, "mrr_at_20": 0.726645, "mrr_at_100": 0.727177, "mrr_at_1000": 0.727207, "nauc_ndcg_at_1_max": 0.208954, "nauc_ndcg_at_1_std": 0.014139, "nauc_ndcg_at_1_diff1": 0.410187, "nauc_ndcg_at_3_max": 0.185414, "nauc_ndcg_at_3_std": -0.006477, "nauc_ndcg_at_3_diff1": 0.330208, "nauc_ndcg_at_5_max": 0.239456, "nauc_ndcg_at_5_std": 0.001028, "nauc_ndcg_at_5_diff1": 0.335084, "nauc_ndcg_at_10_max": 0.277696, "nauc_ndcg_at_10_std": 0.022119, "nauc_ndcg_at_10_diff1": 0.339627, "nauc_ndcg_at_20_max": 0.289755, "nauc_ndcg_at_20_std": 0.045776, "nauc_ndcg_at_20_diff1": 0.33287, "nauc_ndcg_at_100_max": 0.304849, "nauc_ndcg_at_100_std": 0.084032, "nauc_ndcg_at_100_diff1": 0.354252, "nauc_ndcg_at_1000_max": 0.285763, "nauc_ndcg_at_1000_std": 0.06688, "nauc_ndcg_at_1000_diff1": 0.352434, "nauc_map_at_1_max": 0.037026, "nauc_map_at_1_std": -0.088606, "nauc_map_at_1_diff1": 0.32753, "nauc_map_at_3_max": 0.094195, "nauc_map_at_3_std": -0.073177, "nauc_map_at_3_diff1": 0.307551, "nauc_map_at_5_max": 0.15872, "nauc_map_at_5_std": -0.047722, "nauc_map_at_5_diff1": 0.300955, "nauc_map_at_10_max": 0.214463, "nauc_map_at_10_std": -0.018231, "nauc_map_at_10_diff1": 0.310913, "nauc_map_at_20_max": 0.237755, "nauc_map_at_20_std": 0.002827, "nauc_map_at_20_diff1": 0.312953, "nauc_map_at_100_max": 0.249696, "nauc_map_at_100_std": 0.021815, "nauc_map_at_100_diff1": 0.319504, "nauc_map_at_1000_max": 0.248689, "nauc_map_at_1000_std": 0.021686, "nauc_map_at_1000_diff1": 0.319612, "nauc_recall_at_1_max": 0.037026, "nauc_recall_at_1_std": -0.088606, "nauc_recall_at_1_diff1": 0.32753, "nauc_recall_at_3_max": 0.074112, "nauc_recall_at_3_std": -0.077169, "nauc_recall_at_3_diff1": 0.242973, "nauc_recall_at_5_max": 0.172639, "nauc_recall_at_5_std": -0.068862, "nauc_recall_at_5_diff1": 0.216132, "nauc_recall_at_10_max": 0.268171, "nauc_recall_at_10_std": -0.014669, "nauc_recall_at_10_diff1": 0.213619, "nauc_recall_at_20_max": 0.338668, "nauc_recall_at_20_std": 0.071467, "nauc_recall_at_20_diff1": 0.183039, "nauc_recall_at_100_max": 0.611075, "nauc_recall_at_100_std": 0.482254, "nauc_recall_at_100_diff1": 0.335961, "nauc_recall_at_1000_max": 0.806302, "nauc_recall_at_1000_std": 0.641932, "nauc_recall_at_1000_diff1": 0.189872, "nauc_precision_at_1_max": 0.238038, "nauc_precision_at_1_std": 0.046456, "nauc_precision_at_1_diff1": 0.446654, "nauc_precision_at_3_max": 0.237012, "nauc_precision_at_3_std": 0.096543, "nauc_precision_at_3_diff1": 0.17155, "nauc_precision_at_5_max": 0.303376, "nauc_precision_at_5_std": 0.151914, "nauc_precision_at_5_diff1": 0.113921, "nauc_precision_at_10_max": 0.296991, "nauc_precision_at_10_std": 0.186622, "nauc_precision_at_10_diff1": 0.062693, "nauc_precision_at_20_max": 0.244846, "nauc_precision_at_20_std": 0.219279, "nauc_precision_at_20_diff1": 0.004084, "nauc_precision_at_100_max": 0.1613, "nauc_precision_at_100_std": 0.237182, "nauc_precision_at_100_diff1": -0.028329, "nauc_precision_at_1000_max": 0.098103, "nauc_precision_at_1000_std": 0.195796, "nauc_precision_at_1000_diff1": -0.041895, "nauc_mrr_at_1_max": 0.238038, "nauc_mrr_at_1_std": 0.046456, "nauc_mrr_at_1_diff1": 0.446654, "nauc_mrr_at_3_max": 0.275606, "nauc_mrr_at_3_std": 0.078955, "nauc_mrr_at_3_diff1": 0.44781, "nauc_mrr_at_5_max": 0.294865, "nauc_mrr_at_5_std": 0.085936, "nauc_mrr_at_5_diff1": 0.45105, "nauc_mrr_at_10_max": 0.290906, "nauc_mrr_at_10_std": 0.085501, "nauc_mrr_at_10_diff1": 0.448464, "nauc_mrr_at_20_max": 0.2891, "nauc_mrr_at_20_std": 0.087397, "nauc_mrr_at_20_diff1": 0.447477, "nauc_mrr_at_100_max": 0.288221, "nauc_mrr_at_100_std": 0.086696, "nauc_mrr_at_100_diff1": 0.448054, "nauc_mrr_at_1000_max": 0.288131, "nauc_mrr_at_1000_std": 0.086579, "nauc_mrr_at_1000_diff1": 0.448032, "hit_rate_at_1": 0.61494, "hit_rate_at_3": 0.81609, "hit_rate_at_5": 0.87644, "hit_rate_at_10": 0.9454, "hit_rate_at_20": 0.96839, "hit_rate_at_100": 0.99138, "hit_rate_at_1000": 1.0, "main_score": 0.62352, "hf_subset": "spanish", "languages": [ "spa-Latn" ], "mteb_version": "2.13.1" }, { "ndcg_at_1": 0.5977, "ndcg_at_3": 0.56669, "ndcg_at_5": 0.58557, "ndcg_at_10": 0.62821, "ndcg_at_20": 0.65948, "ndcg_at_100": 0.70619, "ndcg_at_1000": 0.71714, "map_at_1": 0.24798, "map_at_3": 0.3885, "map_at_5": 0.44282, "map_at_10": 0.49943, "map_at_20": 0.5269, "map_at_100": 0.54757, "map_at_1000": 0.54905, "recall_at_1": 0.24798, "recall_at_3": 0.44807, "recall_at_5": 0.55519, "recall_at_10": 0.70071, "recall_at_20": 0.79446, "recall_at_100": 0.94442, "recall_at_1000": 0.99433, "accuracy": 0.24798, "precision_at_1": 0.62644, "precision_at_3": 0.43295, "precision_at_5": 0.34828, "precision_at_10": 0.24655, "precision_at_20": 0.15388, "precision_at_100": 0.04109, "precision_at_1000": 0.00446, "mrr_at_1": 0.626437, "mrr_at_3": 0.70977, "mrr_at_5": 0.72342, "mrr_at_10": 0.732225, "mrr_at_20": 0.733704, "mrr_at_100": 0.734469, "mrr_at_1000": 0.734485, "nauc_ndcg_at_1_max": 0.228018, "nauc_ndcg_at_1_std": 0.025705, "nauc_ndcg_at_1_diff1": 0.442779, "nauc_ndcg_at_3_max": 0.193245, "nauc_ndcg_at_3_std": -0.018038, "nauc_ndcg_at_3_diff1": 0.366691, "nauc_ndcg_at_5_max": 0.249865, "nauc_ndcg_at_5_std": -0.023987, "nauc_ndcg_at_5_diff1": 0.383184, "nauc_ndcg_at_10_max": 0.292241, "nauc_ndcg_at_10_std": 0.011807, "nauc_ndcg_at_10_diff1": 0.370397, "nauc_ndcg_at_20_max": 0.305175, "nauc_ndcg_at_20_std": 0.032311, "nauc_ndcg_at_20_diff1": 0.356494, "nauc_ndcg_at_100_max": 0.320749, "nauc_ndcg_at_100_std": 0.069656, "nauc_ndcg_at_100_diff1": 0.37955, "nauc_ndcg_at_1000_max": 0.2997, "nauc_ndcg_at_1000_std": 0.055314, "nauc_ndcg_at_1000_diff1": 0.387788, "nauc_map_at_1_max": 0.07914, "nauc_map_at_1_std": -0.072482, "nauc_map_at_1_diff1": 0.36611, "nauc_map_at_3_max": 0.116721, "nauc_map_at_3_std": -0.077231, "nauc_map_at_3_diff1": 0.321698, "nauc_map_at_5_max": 0.185902, "nauc_map_at_5_std": -0.065449, "nauc_map_at_5_diff1": 0.332786, "nauc_map_at_10_max": 0.239909, "nauc_map_at_10_std": -0.035002, "nauc_map_at_10_diff1": 0.333622, "nauc_map_at_20_max": 0.262108, "nauc_map_at_20_std": -0.016526, "nauc_map_at_20_diff1": 0.331631, "nauc_map_at_100_max": 0.274324, "nauc_map_at_100_std": 0.002261, "nauc_map_at_100_diff1": 0.33696, "nauc_map_at_1000_max": 0.272794, "nauc_map_at_1000_std": 0.002226, "nauc_map_at_1000_diff1": 0.338139, "nauc_recall_at_1_max": 0.07914, "nauc_recall_at_1_std": -0.072482, "nauc_recall_at_1_diff1": 0.36611, "nauc_recall_at_3_max": 0.071319, "nauc_recall_at_3_std": -0.086522, "nauc_recall_at_3_diff1": 0.255647, "nauc_recall_at_5_max": 0.182657, "nauc_recall_at_5_std": -0.08268, "nauc_recall_at_5_diff1": 0.254052, "nauc_recall_at_10_max": 0.290693, "nauc_recall_at_10_std": -0.007218, "nauc_recall_at_10_diff1": 0.232243, "nauc_recall_at_20_max": 0.369019, "nauc_recall_at_20_std": 0.066747, "nauc_recall_at_20_diff1": 0.18054, "nauc_recall_at_100_max": 0.673185, "nauc_recall_at_100_std": 0.45242, "nauc_recall_at_100_diff1": 0.242438, "nauc_recall_at_1000_max": 0.869004, "nauc_recall_at_1000_std": 0.813757, "nauc_recall_at_1000_diff1": 0.237155, "nauc_precision_at_1_max": 0.262399, "nauc_precision_at_1_std": 0.04289, "nauc_precision_at_1_diff1": 0.466673, "nauc_precision_at_3_max": 0.236224, "nauc_precision_at_3_std": 0.067782, "nauc_precision_at_3_diff1": 0.191355, "nauc_precision_at_5_max": 0.291963, "nauc_precision_at_5_std": 0.100804, "nauc_precision_at_5_diff1": 0.147279, "nauc_precision_at_10_max": 0.270449, "nauc_precision_at_10_std": 0.153667, "nauc_precision_at_10_diff1": 0.058804, "nauc_precision_at_20_max": 0.21088, "nauc_precision_at_20_std": 0.174415, "nauc_precision_at_20_diff1": -0.008956, "nauc_precision_at_100_max": 0.123169, "nauc_precision_at_100_std": 0.198099, "nauc_precision_at_100_diff1": -0.046804, "nauc_precision_at_1000_max": 0.054384, "nauc_precision_at_1000_std": 0.157848, "nauc_precision_at_1000_diff1": -0.038483, "nauc_mrr_at_1_max": 0.262399, "nauc_mrr_at_1_std": 0.04289, "nauc_mrr_at_1_diff1": 0.466673, "nauc_mrr_at_3_max": 0.275689, "nauc_mrr_at_3_std": 0.063675, "nauc_mrr_at_3_diff1": 0.484226, "nauc_mrr_at_5_max": 0.29086, "nauc_mrr_at_5_std": 0.065373, "nauc_mrr_at_5_diff1": 0.482248, "nauc_mrr_at_10_max": 0.292386, "nauc_mrr_at_10_std": 0.068351, "nauc_mrr_at_10_diff1": 0.478464, "nauc_mrr_at_20_max": 0.291324, "nauc_mrr_at_20_std": 0.068977, "nauc_mrr_at_20_diff1": 0.477441, "nauc_mrr_at_100_max": 0.289978, "nauc_mrr_at_100_std": 0.068654, "nauc_mrr_at_100_diff1": 0.477715, "nauc_mrr_at_1000_max": 0.28993, "nauc_mrr_at_1000_std": 0.068593, "nauc_mrr_at_1000_diff1": 0.477707, "hit_rate_at_1": 0.62644, "hit_rate_at_3": 0.81897, "hit_rate_at_5": 0.87931, "hit_rate_at_10": 0.94253, "hit_rate_at_20": 0.96264, "hit_rate_at_100": 0.99138, "hit_rate_at_1000": 0.99713, "main_score": 0.62821, "hf_subset": "italian", "languages": [ "ita-Latn" ], "mteb_version": "2.13.1" }, { "ndcg_at_1": 0.58764, "ndcg_at_3": 0.56061, "ndcg_at_5": 0.58355, "ndcg_at_10": 0.62003, "ndcg_at_20": 0.65463, "ndcg_at_100": 0.70042, "ndcg_at_1000": 0.7111, "map_at_1": 0.24016, "map_at_3": 0.38098, "map_at_5": 0.43781, "map_at_10": 0.49135, "map_at_20": 0.52135, "map_at_100": 0.54147, "map_at_1000": 0.54289, "recall_at_1": 0.24016, "recall_at_3": 0.44562, "recall_at_5": 0.55885, "recall_at_10": 0.69136, "recall_at_20": 0.79441, "recall_at_100": 0.9447, "recall_at_1000": 0.9932, "accuracy": 0.24016, "precision_at_1": 0.61494, "precision_at_3": 0.43391, "precision_at_5": 0.35287, "precision_at_10": 0.24454, "precision_at_20": 0.15575, "precision_at_100": 0.04118, "precision_at_1000": 0.00445, "mrr_at_1": 0.614943, "mrr_at_3": 0.69636, "mrr_at_5": 0.713602, "mrr_at_10": 0.723089, "mrr_at_20": 0.724341, "mrr_at_100": 0.725097, "mrr_at_1000": 0.725111, "nauc_ndcg_at_1_max": 0.21998, "nauc_ndcg_at_1_std": 0.008848, "nauc_ndcg_at_1_diff1": 0.402816, "nauc_ndcg_at_3_max": 0.213531, "nauc_ndcg_at_3_std": -0.001066, "nauc_ndcg_at_3_diff1": 0.362575, "nauc_ndcg_at_5_max": 0.27856, "nauc_ndcg_at_5_std": 0.014482, "nauc_ndcg_at_5_diff1": 0.384816, "nauc_ndcg_at_10_max": 0.312715, "nauc_ndcg_at_10_std": 0.010417, "nauc_ndcg_at_10_diff1": 0.367597, "nauc_ndcg_at_20_max": 0.322674, "nauc_ndcg_at_20_std": 0.036225, "nauc_ndcg_at_20_diff1": 0.364437, "nauc_ndcg_at_100_max": 0.340036, "nauc_ndcg_at_100_std": 0.084674, "nauc_ndcg_at_100_diff1": 0.383364, "nauc_ndcg_at_1000_max": 0.322942, "nauc_ndcg_at_1000_std": 0.076717, "nauc_ndcg_at_1000_diff1": 0.382216, "nauc_map_at_1_max": 0.088192, "nauc_map_at_1_std": -0.082786, "nauc_map_at_1_diff1": 0.369659, "nauc_map_at_3_max": 0.148565, "nauc_map_at_3_std": -0.07844, "nauc_map_at_3_diff1": 0.342304, "nauc_map_at_5_max": 0.222447, "nauc_map_at_5_std": -0.040119, "nauc_map_at_5_diff1": 0.345894, "nauc_map_at_10_max": 0.264752, "nauc_map_at_10_std": -0.028298, "nauc_map_at_10_diff1": 0.343796, "nauc_map_at_20_max": 0.281288, "nauc_map_at_20_std": -0.011978, "nauc_map_at_20_diff1": 0.349657, "nauc_map_at_100_max": 0.295511, "nauc_map_at_100_std": 0.013137, "nauc_map_at_100_diff1": 0.35779, "nauc_map_at_1000_max": 0.2945, "nauc_map_at_1000_std": 0.013887, "nauc_map_at_1000_diff1": 0.357841, "nauc_recall_at_1_max": 0.088192, "nauc_recall_at_1_std": -0.082786, "nauc_recall_at_1_diff1": 0.369659, "nauc_recall_at_3_max": 0.105266, "nauc_recall_at_3_std": -0.090246, "nauc_recall_at_3_diff1": 0.26311, "nauc_recall_at_5_max": 0.214569, "nauc_recall_at_5_std": -0.048368, "nauc_recall_at_5_diff1": 0.268472, "nauc_recall_at_10_max": 0.292294, "nauc_recall_at_10_std": -0.050861, "nauc_recall_at_10_diff1": 0.229553, "nauc_recall_at_20_max": 0.375047, "nauc_recall_at_20_std": 0.0368, "nauc_recall_at_20_diff1": 0.217143, "nauc_recall_at_100_max": 0.618517, "nauc_recall_at_100_std": 0.381866, "nauc_recall_at_100_diff1": 0.30809, "nauc_recall_at_1000_max": 0.873964, "nauc_recall_at_1000_std": 0.842176, "nauc_recall_at_1000_diff1": 0.240502, "nauc_precision_at_1_max": 0.247994, "nauc_precision_at_1_std": 0.028202, "nauc_precision_at_1_diff1": 0.437689, "nauc_precision_at_3_max": 0.254116, "nauc_precision_at_3_std": 0.084345, "nauc_precision_at_3_diff1": 0.20244, "nauc_precision_at_5_max": 0.301784, "nauc_precision_at_5_std": 0.149545, "nauc_precision_at_5_diff1": 0.139514, "nauc_precision_at_10_max": 0.259498, "nauc_precision_at_10_std": 0.151526, "nauc_precision_at_10_diff1": 0.055082, "nauc_precision_at_20_max": 0.188444, "nauc_precision_at_20_std": 0.180671, "nauc_precision_at_20_diff1": 0.004475, "nauc_precision_at_100_max": 0.125534, "nauc_precision_at_100_std": 0.230788, "nauc_precision_at_100_diff1": -0.034095, "nauc_precision_at_1000_max": 0.064868, "nauc_precision_at_1000_std": 0.198742, "nauc_precision_at_1000_diff1": -0.048474, "nauc_mrr_at_1_max": 0.247994, "nauc_mrr_at_1_std": 0.028202, "nauc_mrr_at_1_diff1": 0.437689, "nauc_mrr_at_3_max": 0.297937, "nauc_mrr_at_3_std": 0.080257, "nauc_mrr_at_3_diff1": 0.443718, "nauc_mrr_at_5_max": 0.301649, "nauc_mrr_at_5_std": 0.076013, "nauc_mrr_at_5_diff1": 0.451764, "nauc_mrr_at_10_max": 0.299572, "nauc_mrr_at_10_std": 0.074512, "nauc_mrr_at_10_diff1": 0.447215, "nauc_mrr_at_20_max": 0.298406, "nauc_mrr_at_20_std": 0.076328, "nauc_mrr_at_20_diff1": 0.445735, "nauc_mrr_at_100_max": 0.297128, "nauc_mrr_at_100_std": 0.075904, "nauc_mrr_at_100_diff1": 0.446125, "nauc_mrr_at_1000_max": 0.297087, "nauc_mrr_at_1000_std": 0.07585, "nauc_mrr_at_1000_diff1": 0.446136, "hit_rate_at_1": 0.61494, "hit_rate_at_3": 0.8046, "hit_rate_at_5": 0.87644, "hit_rate_at_10": 0.94253, "hit_rate_at_20": 0.95977, "hit_rate_at_100": 0.99138, "hit_rate_at_1000": 0.99713, "main_score": 0.62003, "hf_subset": "portuguese", "languages": [ "por-Latn" ], "mteb_version": "2.13.1" }, { "ndcg_at_1": 0.5704, "ndcg_at_3": 0.55523, "ndcg_at_5": 0.57747, "ndcg_at_10": 0.6113, "ndcg_at_20": 0.65006, "ndcg_at_100": 0.69527, "ndcg_at_1000": 0.70648, "map_at_1": 0.23627, "map_at_3": 0.37707, "map_at_5": 0.4316, "map_at_10": 0.48203, "map_at_20": 0.51345, "map_at_100": 0.53302, "map_at_1000": 0.5346, "recall_at_1": 0.23627, "recall_at_3": 0.44468, "recall_at_5": 0.55656, "recall_at_10": 0.68103, "recall_at_20": 0.79496, "recall_at_100": 0.94362, "recall_at_1000": 0.99405, "accuracy": 0.23627, "precision_at_1": 0.59483, "precision_at_3": 0.42816, "precision_at_5": 0.34598, "precision_at_10": 0.23879, "precision_at_20": 0.15445, "precision_at_100": 0.04086, "precision_at_1000": 0.00444, "mrr_at_1": 0.594828, "mrr_at_3": 0.689655, "mrr_at_5": 0.706178, "mrr_at_10": 0.713915, "mrr_at_20": 0.715119, "mrr_at_100": 0.715893, "mrr_at_1000": 0.71592, "nauc_ndcg_at_1_max": 0.225473, "nauc_ndcg_at_1_std": 0.015575, "nauc_ndcg_at_1_diff1": 0.410241, "nauc_ndcg_at_3_max": 0.17359, "nauc_ndcg_at_3_std": -0.013734, "nauc_ndcg_at_3_diff1": 0.343264, "nauc_ndcg_at_5_max": 0.243599, "nauc_ndcg_at_5_std": -0.011339, "nauc_ndcg_at_5_diff1": 0.343873, "nauc_ndcg_at_10_max": 0.286048, "nauc_ndcg_at_10_std": 0.013484, "nauc_ndcg_at_10_diff1": 0.351983, "nauc_ndcg_at_20_max": 0.297252, "nauc_ndcg_at_20_std": 0.030231, "nauc_ndcg_at_20_diff1": 0.346519, "nauc_ndcg_at_100_max": 0.323136, "nauc_ndcg_at_100_std": 0.085198, "nauc_ndcg_at_100_diff1": 0.364129, "nauc_ndcg_at_1000_max": 0.304707, "nauc_ndcg_at_1000_std": 0.070238, "nauc_ndcg_at_1000_diff1": 0.367805, "nauc_map_at_1_max": 0.068727, "nauc_map_at_1_std": -0.066695, "nauc_map_at_1_diff1": 0.348307, "nauc_map_at_3_max": 0.105206, "nauc_map_at_3_std": -0.069614, "nauc_map_at_3_diff1": 0.321026, "nauc_map_at_5_max": 0.18372, "nauc_map_at_5_std": -0.05367, "nauc_map_at_5_diff1": 0.314117, "nauc_map_at_10_max": 0.23946, "nauc_map_at_10_std": -0.026053, "nauc_map_at_10_diff1": 0.321133, "nauc_map_at_20_max": 0.262535, "nauc_map_at_20_std": -0.007198, "nauc_map_at_20_diff1": 0.326301, "nauc_map_at_100_max": 0.281469, "nauc_map_at_100_std": 0.019646, "nauc_map_at_100_diff1": 0.332474, "nauc_map_at_1000_max": 0.28089, "nauc_map_at_1000_std": 0.019812, "nauc_map_at_1000_diff1": 0.333803, "nauc_recall_at_1_max": 0.068727, "nauc_recall_at_1_std": -0.066695, "nauc_recall_at_1_diff1": 0.348307, "nauc_recall_at_3_max": 0.049718, "nauc_recall_at_3_std": -0.085096, "nauc_recall_at_3_diff1": 0.239235, "nauc_recall_at_5_max": 0.148665, "nauc_recall_at_5_std": -0.087964, "nauc_recall_at_5_diff1": 0.19231, "nauc_recall_at_10_max": 0.258717, "nauc_recall_at_10_std": -0.034625, "nauc_recall_at_10_diff1": 0.218866, "nauc_recall_at_20_max": 0.331051, "nauc_recall_at_20_std": 0.026643, "nauc_recall_at_20_diff1": 0.211578, "nauc_recall_at_100_max": 0.637795, "nauc_recall_at_100_std": 0.472675, "nauc_recall_at_100_diff1": 0.237531, "nauc_recall_at_1000_max": 0.804246, "nauc_recall_at_1000_std": 0.795859, "nauc_recall_at_1000_diff1": 0.082623, "nauc_precision_at_1_max": 0.269186, "nauc_precision_at_1_std": 0.050581, "nauc_precision_at_1_diff1": 0.432136, "nauc_precision_at_3_max": 0.230666, "nauc_precision_at_3_std": 0.083578, "nauc_precision_at_3_diff1": 0.196559, "nauc_precision_at_5_max": 0.341152, "nauc_precision_at_5_std": 0.139488, "nauc_precision_at_5_diff1": 0.136638, "nauc_precision_at_10_max": 0.330606, "nauc_precision_at_10_std": 0.186598, "nauc_precision_at_10_diff1": 0.081131, "nauc_precision_at_20_max": 0.266462, "nauc_precision_at_20_std": 0.213674, "nauc_precision_at_20_diff1": 0.017564, "nauc_precision_at_100_max": 0.199471, "nauc_precision_at_100_std": 0.259989, "nauc_precision_at_100_diff1": -0.013042, "nauc_precision_at_1000_max": 0.13377, "nauc_precision_at_1000_std": 0.215609, "nauc_precision_at_1000_diff1": -0.010487, "nauc_mrr_at_1_max": 0.269186, "nauc_mrr_at_1_std": 0.050581, "nauc_mrr_at_1_diff1": 0.432136, "nauc_mrr_at_3_max": 0.287671, "nauc_mrr_at_3_std": 0.064933, "nauc_mrr_at_3_diff1": 0.432673, "nauc_mrr_at_5_max": 0.302447, "nauc_mrr_at_5_std": 0.075976, "nauc_mrr_at_5_diff1": 0.433092, "nauc_mrr_at_10_max": 0.30374, "nauc_mrr_at_10_std": 0.077055, "nauc_mrr_at_10_diff1": 0.436815, "nauc_mrr_at_20_max": 0.303281, "nauc_mrr_at_20_std": 0.078604, "nauc_mrr_at_20_diff1": 0.436615, "nauc_mrr_at_100_max": 0.30196, "nauc_mrr_at_100_std": 0.078331, "nauc_mrr_at_100_diff1": 0.43614, "nauc_mrr_at_1000_max": 0.301893, "nauc_mrr_at_1000_std": 0.078228, "nauc_mrr_at_1000_diff1": 0.436096, "hit_rate_at_1": 0.59483, "hit_rate_at_3": 0.81322, "hit_rate_at_5": 0.88793, "hit_rate_at_10": 0.94253, "hit_rate_at_20": 0.95977, "hit_rate_at_100": 0.99138, "hit_rate_at_1000": 1.0, "main_score": 0.6113, "hf_subset": "french", "languages": [ "fra-Latn" ], "mteb_version": "2.13.1" }, { "ndcg_at_1": 0.57902, "ndcg_at_3": 0.57371, "ndcg_at_5": 0.59988, "ndcg_at_10": 0.62636, "ndcg_at_20": 0.66324, "ndcg_at_100": 0.70667, "ndcg_at_1000": 0.71912, "map_at_1": 0.24269, "map_at_3": 0.38826, "map_at_5": 0.45059, "map_at_10": 0.49977, "map_at_20": 0.53016, "map_at_100": 0.54914, "map_at_1000": 0.55086, "recall_at_1": 0.24269, "recall_at_3": 0.45071, "recall_at_5": 0.57345, "recall_at_10": 0.69036, "recall_at_20": 0.80188, "recall_at_100": 0.94089, "recall_at_1000": 0.9972, "accuracy": 0.24269, "precision_at_1": 0.60345, "precision_at_3": 0.43774, "precision_at_5": 0.36207, "precision_at_10": 0.24253, "precision_at_20": 0.15532, "precision_at_100": 0.04066, "precision_at_1000": 0.00446, "mrr_at_1": 0.603448, "mrr_at_3": 0.703065, "mrr_at_5": 0.716427, "mrr_at_10": 0.722937, "mrr_at_20": 0.725354, "mrr_at_100": 0.726167, "mrr_at_1000": 0.726183, "nauc_ndcg_at_1_max": 0.196187, "nauc_ndcg_at_1_std": 0.049268, "nauc_ndcg_at_1_diff1": 0.439463, "nauc_ndcg_at_3_max": 0.157049, "nauc_ndcg_at_3_std": -0.017104, "nauc_ndcg_at_3_diff1": 0.38231, "nauc_ndcg_at_5_max": 0.248598, "nauc_ndcg_at_5_std": 0.022934, "nauc_ndcg_at_5_diff1": 0.401445, "nauc_ndcg_at_10_max": 0.277435, "nauc_ndcg_at_10_std": 0.0323, "nauc_ndcg_at_10_diff1": 0.391342, "nauc_ndcg_at_20_max": 0.293873, "nauc_ndcg_at_20_std": 0.055537, "nauc_ndcg_at_20_diff1": 0.391424, "nauc_ndcg_at_100_max": 0.301034, "nauc_ndcg_at_100_std": 0.102775, "nauc_ndcg_at_100_diff1": 0.400733, "nauc_ndcg_at_1000_max": 0.279009, "nauc_ndcg_at_1000_std": 0.086737, "nauc_ndcg_at_1000_diff1": 0.399697, "nauc_map_at_1_max": 0.062135, "nauc_map_at_1_std": -0.05442, "nauc_map_at_1_diff1": 0.417947, "nauc_map_at_3_max": 0.104041, "nauc_map_at_3_std": -0.08838, "nauc_map_at_3_diff1": 0.394832, "nauc_map_at_5_max": 0.18665, "nauc_map_at_5_std": -0.042864, "nauc_map_at_5_diff1": 0.387178, "nauc_map_at_10_max": 0.229677, "nauc_map_at_10_std": -0.023346, "nauc_map_at_10_diff1": 0.385723, "nauc_map_at_20_max": 0.250803, "nauc_map_at_20_std": -0.000578, "nauc_map_at_20_diff1": 0.386895, "nauc_map_at_100_max": 0.257985, "nauc_map_at_100_std": 0.021212, "nauc_map_at_100_diff1": 0.386898, "nauc_map_at_1000_max": 0.25708, "nauc_map_at_1000_std": 0.021791, "nauc_map_at_1000_diff1": 0.387538, "nauc_recall_at_1_max": 0.062135, "nauc_recall_at_1_std": -0.05442, "nauc_recall_at_1_diff1": 0.417947, "nauc_recall_at_3_max": 0.068693, "nauc_recall_at_3_std": -0.114433, "nauc_recall_at_3_diff1": 0.316518, "nauc_recall_at_5_max": 0.189926, "nauc_recall_at_5_std": -0.058983, "nauc_recall_at_5_diff1": 0.280505, "nauc_recall_at_10_max": 0.29775, "nauc_recall_at_10_std": -0.005008, "nauc_recall_at_10_diff1": 0.280784, "nauc_recall_at_20_max": 0.378452, "nauc_recall_at_20_std": 0.081787, "nauc_recall_at_20_diff1": 0.264646, "nauc_recall_at_100_max": 0.642196, "nauc_recall_at_100_std": 0.481835, "nauc_recall_at_100_diff1": 0.391813, "nauc_recall_at_1000_max": 0.734463, "nauc_recall_at_1000_std": 0.619491, "nauc_recall_at_1000_diff1": 0.311223, "nauc_precision_at_1_max": 0.227067, "nauc_precision_at_1_std": 0.077212, "nauc_precision_at_1_diff1": 0.475501, "nauc_precision_at_3_max": 0.21959, "nauc_precision_at_3_std": 0.063113, "nauc_precision_at_3_diff1": 0.175564, "nauc_precision_at_5_max": 0.318667, "nauc_precision_at_5_std": 0.182401, "nauc_precision_at_5_diff1": 0.080376, "nauc_precision_at_10_max": 0.269688, "nauc_precision_at_10_std": 0.186052, "nauc_precision_at_10_diff1": 0.003316, "nauc_precision_at_20_max": 0.203328, "nauc_precision_at_20_std": 0.207859, "nauc_precision_at_20_diff1": -0.066731, "nauc_precision_at_100_max": 0.118009, "nauc_precision_at_100_std": 0.245952, "nauc_precision_at_100_diff1": -0.118269, "nauc_precision_at_1000_max": 0.050209, "nauc_precision_at_1000_std": 0.19969, "nauc_precision_at_1000_diff1": -0.121843, "nauc_mrr_at_1_max": 0.227067, "nauc_mrr_at_1_std": 0.077212, "nauc_mrr_at_1_diff1": 0.475501, "nauc_mrr_at_3_max": 0.267284, "nauc_mrr_at_3_std": 0.104571, "nauc_mrr_at_3_diff1": 0.468689, "nauc_mrr_at_5_max": 0.272204, "nauc_mrr_at_5_std": 0.110785, "nauc_mrr_at_5_diff1": 0.46346, "nauc_mrr_at_10_max": 0.271102, "nauc_mrr_at_10_std": 0.112137, "nauc_mrr_at_10_diff1": 0.46417, "nauc_mrr_at_20_max": 0.269296, "nauc_mrr_at_20_std": 0.11276, "nauc_mrr_at_20_diff1": 0.465395, "nauc_mrr_at_100_max": 0.268056, "nauc_mrr_at_100_std": 0.111379, "nauc_mrr_at_100_diff1": 0.467188, "nauc_mrr_at_1000_max": 0.268006, "nauc_mrr_at_1000_std": 0.111315, "nauc_mrr_at_1000_diff1": 0.467176, "hit_rate_at_1": 0.60345, "hit_rate_at_3": 0.82759, "hit_rate_at_5": 0.88506, "hit_rate_at_10": 0.93391, "hit_rate_at_20": 0.96552, "hit_rate_at_100": 0.99425, "hit_rate_at_1000": 1.0, "main_score": 0.62636, "hf_subset": "german", "languages": [ "deu-Latn" ], "mteb_version": "2.13.1" } ] }, "evaluation_time": 9521.54741820041, "kg_co2_emissions": null, "date": 1782148418.602253, "evaluation_phases": [ { "name": "Data loading", "start": 6.858073174953461e-05, "end": 72.45316150318831, "split": "", "subset": "" }, { "name": "Dataset transform", "start": 72.45324011240155, "end": 72.45324445981532, "split": "", "subset": "" }, { "name": "Encoding queries and documents", "start": 72.46839837636799, "end": 1673.838474453427, "split": "test", "subset": "spanish" }, { "name": "Scoring", "start": 1673.8386953221634, "end": 1674.1942733656615, "split": "test", "subset": "spanish" }, { "name": "Encoding queries and documents", "start": 1674.2291835583746, "end": 3250.701141563244, "split": "test", "subset": "italian" }, { "name": "Scoring", "start": 3250.7013896638528, "end": 3251.03475962393, "split": "test", "subset": "italian" }, { "name": "Encoding queries and documents", "start": 3251.0721619520336, "end": 4837.300269488245, "split": "test", "subset": "portuguese" }, { "name": "Scoring", "start": 4837.300456551835, "end": 4837.634334119968, "split": "test", "subset": "portuguese" }, { "name": "Encoding queries and documents", "start": 4837.670631226152, "end": 6436.128637465648, "split": "test", "subset": "french" }, { "name": "Scoring", "start": 6436.128897177987, "end": 6436.461139812134, "split": "test", "subset": "french" }, { "name": "Encoding queries and documents", "start": 6436.502193697728, "end": 8032.22604198195, "split": "test", "subset": "german" }, { "name": "Scoring", "start": 8032.226301563904, "end": 8032.561185928993, "split": "test", "subset": "german" } ] }