{ "dataset_revision": "a54635a274ef2835721b7cbe3eb27483b9ec964b", "task_name": "Vidore3TelecomRetrieval", "mteb_version": "2.24.2", "scores": { "test": [ { "ndcg_at_1": 0.74425, "ndcg_at_3": 0.69852, "ndcg_at_5": 0.70151, "ndcg_at_10": 0.72989, "ndcg_at_20": 0.75816, "ndcg_at_100": 0.79946, "ndcg_at_1000": 0.80469, "map_at_1": 0.32636, "map_at_3": 0.50099, "map_at_5": 0.55492, "map_at_10": 0.61063, "map_at_20": 0.63814, "map_at_100": 0.6593, "map_at_1000": 0.66017, "recall_at_1": 0.32636, "recall_at_3": 0.54966, "recall_at_5": 0.64347, "recall_at_10": 0.7632, "recall_at_20": 0.85555, "recall_at_100": 0.97471, "recall_at_1000": 0.99799, "accuracy": 0.32636, "precision_at_1": 0.76724, "precision_at_3": 0.51437, "precision_at_5": 0.39655, "precision_at_10": 0.26868, "precision_at_20": 0.16408, "precision_at_100": 0.04307, "precision_at_1000": 0.00448, "mrr_at_1": 0.767241, "mrr_at_3": 0.829981, "mrr_at_5": 0.837308, "mrr_at_10": 0.841553, "mrr_at_20": 0.842718, "mrr_at_100": 0.842825, "mrr_at_1000": 0.84285, "nauc_ndcg_at_1_max": -0.102793, "nauc_ndcg_at_1_std": -0.150372, "nauc_ndcg_at_1_diff1": 0.448284, "nauc_ndcg_at_3_max": -0.187797, "nauc_ndcg_at_3_std": -0.176356, "nauc_ndcg_at_3_diff1": 0.326168, "nauc_ndcg_at_5_max": -0.145723, "nauc_ndcg_at_5_std": -0.140863, "nauc_ndcg_at_5_diff1": 0.349681, "nauc_ndcg_at_10_max": -0.083845, "nauc_ndcg_at_10_std": -0.096007, "nauc_ndcg_at_10_diff1": 0.387707, "nauc_ndcg_at_20_max": -0.085178, "nauc_ndcg_at_20_std": -0.095554, "nauc_ndcg_at_20_diff1": 0.399437, "nauc_ndcg_at_100_max": -0.076432, "nauc_ndcg_at_100_std": -0.088991, "nauc_ndcg_at_100_diff1": 0.408996, "nauc_ndcg_at_1000_max": -0.087537, "nauc_ndcg_at_1000_std": -0.102544, "nauc_ndcg_at_1000_diff1": 0.400836, "nauc_map_at_1_max": -0.060009, "nauc_map_at_1_std": -0.107536, "nauc_map_at_1_diff1": 0.486888, "nauc_map_at_3_max": -0.105871, "nauc_map_at_3_std": -0.133826, "nauc_map_at_3_diff1": 0.34929, "nauc_map_at_5_max": -0.100229, "nauc_map_at_5_std": -0.120713, "nauc_map_at_5_diff1": 0.343647, "nauc_map_at_10_max": -0.085761, "nauc_map_at_10_std": -0.105144, "nauc_map_at_10_diff1": 0.35164, "nauc_map_at_20_max": -0.083743, "nauc_map_at_20_std": -0.094869, "nauc_map_at_20_diff1": 0.355772, "nauc_map_at_100_max": -0.078954, "nauc_map_at_100_std": -0.084273, "nauc_map_at_100_diff1": 0.357977, "nauc_map_at_1000_max": -0.080033, "nauc_map_at_1000_std": -0.085481, "nauc_map_at_1000_diff1": 0.35717, "nauc_recall_at_1_max": -0.060009, "nauc_recall_at_1_std": -0.107536, "nauc_recall_at_1_diff1": 0.486888, "nauc_recall_at_3_max": -0.116678, "nauc_recall_at_3_std": -0.135802, "nauc_recall_at_3_diff1": 0.290862, "nauc_recall_at_5_max": -0.09989, "nauc_recall_at_5_std": -0.107104, "nauc_recall_at_5_diff1": 0.271748, "nauc_recall_at_10_max": 0.000388, "nauc_recall_at_10_std": -0.022602, "nauc_recall_at_10_diff1": 0.322816, "nauc_recall_at_20_max": 0.00636, "nauc_recall_at_20_std": 0.022717, "nauc_recall_at_20_diff1": 0.335134, "nauc_recall_at_100_max": 0.328822, "nauc_recall_at_100_std": 0.422693, "nauc_recall_at_100_diff1": 0.581628, "nauc_recall_at_1000_max": 0.44098, "nauc_recall_at_1000_std": 0.581575, "nauc_recall_at_1000_diff1": 0.276824, "nauc_precision_at_1_max": -0.156184, "nauc_precision_at_1_std": -0.138396, "nauc_precision_at_1_diff1": 0.473259, "nauc_precision_at_3_max": -0.123604, "nauc_precision_at_3_std": -0.060621, "nauc_precision_at_3_diff1": -0.085619, "nauc_precision_at_5_max": -0.05437, "nauc_precision_at_5_std": 0.018847, "nauc_precision_at_5_diff1": -0.131038, "nauc_precision_at_10_max": 0.037649, "nauc_precision_at_10_std": 0.094136, "nauc_precision_at_10_diff1": -0.14524, "nauc_precision_at_20_max": 0.053961, "nauc_precision_at_20_std": 0.115925, "nauc_precision_at_20_diff1": -0.159201, "nauc_precision_at_100_max": 0.051132, "nauc_precision_at_100_std": 0.106666, "nauc_precision_at_100_diff1": -0.192359, "nauc_precision_at_1000_max": 0.029731, "nauc_precision_at_1000_std": 0.079658, "nauc_precision_at_1000_diff1": -0.21226, "nauc_mrr_at_1_max": -0.156184, "nauc_mrr_at_1_std": -0.138396, "nauc_mrr_at_1_diff1": 0.473259, "nauc_mrr_at_3_max": -0.196879, "nauc_mrr_at_3_std": -0.168814, "nauc_mrr_at_3_diff1": 0.451582, "nauc_mrr_at_5_max": -0.193858, "nauc_mrr_at_5_std": -0.162207, "nauc_mrr_at_5_diff1": 0.460931, "nauc_mrr_at_10_max": -0.185211, "nauc_mrr_at_10_std": -0.161502, "nauc_mrr_at_10_diff1": 0.463587, "nauc_mrr_at_20_max": -0.183758, "nauc_mrr_at_20_std": -0.160072, "nauc_mrr_at_20_diff1": 0.463181, "nauc_mrr_at_100_max": -0.184299, "nauc_mrr_at_100_std": -0.159775, "nauc_mrr_at_100_diff1": 0.462883, "nauc_mrr_at_1000_max": -0.184462, "nauc_mrr_at_1000_std": -0.15996, "nauc_mrr_at_1000_diff1": 0.462791, "hit_rate_at_1": 0.76724, "hit_rate_at_3": 0.91092, "hit_rate_at_5": 0.94253, "hit_rate_at_10": 0.97414, "hit_rate_at_20": 0.99138, "hit_rate_at_100": 0.99425, "hit_rate_at_1000": 1.0, "main_score": 0.72989, "hf_subset": "french", "languages": [ "fra-Latn" ], "mteb_version": "2.24.2" }, { "ndcg_at_1": 0.75, "ndcg_at_3": 0.70372, "ndcg_at_5": 0.70328, "ndcg_at_10": 0.73277, "ndcg_at_20": 0.76164, "ndcg_at_100": 0.80167, "ndcg_at_1000": 0.80682, "map_at_1": 0.33237, "map_at_3": 0.50773, "map_at_5": 0.55905, "map_at_10": 0.61777, "map_at_20": 0.64562, "map_at_100": 0.66582, "map_at_1000": 0.66669, "recall_at_1": 0.33237, "recall_at_3": 0.54926, "recall_at_5": 0.63484, "recall_at_10": 0.76223, "recall_at_20": 0.8548, "recall_at_100": 0.97334, "recall_at_1000": 0.99655, "accuracy": 0.33237, "precision_at_1": 0.77586, "precision_at_3": 0.51916, "precision_at_5": 0.39655, "precision_at_10": 0.27011, "precision_at_20": 0.16552, "precision_at_100": 0.04296, "precision_at_1000": 0.00447, "mrr_at_1": 0.775862, "mrr_at_3": 0.829502, "mrr_at_5": 0.836398, "mrr_at_10": 0.841881, "mrr_at_20": 0.843068, "mrr_at_100": 0.843345, "mrr_at_1000": 0.843357, "nauc_ndcg_at_1_max": -0.088882, "nauc_ndcg_at_1_std": -0.177917, "nauc_ndcg_at_1_diff1": 0.401309, "nauc_ndcg_at_3_max": -0.151995, "nauc_ndcg_at_3_std": -0.159396, "nauc_ndcg_at_3_diff1": 0.296998, "nauc_ndcg_at_5_max": -0.113083, "nauc_ndcg_at_5_std": -0.132281, "nauc_ndcg_at_5_diff1": 0.31513, "nauc_ndcg_at_10_max": -0.096819, "nauc_ndcg_at_10_std": -0.122538, "nauc_ndcg_at_10_diff1": 0.346206, "nauc_ndcg_at_20_max": -0.083159, "nauc_ndcg_at_20_std": -0.103894, "nauc_ndcg_at_20_diff1": 0.360834, "nauc_ndcg_at_100_max": -0.067442, "nauc_ndcg_at_100_std": -0.091559, "nauc_ndcg_at_100_diff1": 0.371093, "nauc_ndcg_at_1000_max": -0.078605, "nauc_ndcg_at_1000_std": -0.104796, "nauc_ndcg_at_1000_diff1": 0.360183, "nauc_map_at_1_max": -0.056706, "nauc_map_at_1_std": -0.124844, "nauc_map_at_1_diff1": 0.450669, "nauc_map_at_3_max": -0.086259, "nauc_map_at_3_std": -0.134304, "nauc_map_at_3_diff1": 0.33814, "nauc_map_at_5_max": -0.085634, "nauc_map_at_5_std": -0.127049, "nauc_map_at_5_diff1": 0.321306, "nauc_map_at_10_max": -0.090448, "nauc_map_at_10_std": -0.129977, "nauc_map_at_10_diff1": 0.317143, "nauc_map_at_20_max": -0.080364, "nauc_map_at_20_std": -0.109024, "nauc_map_at_20_diff1": 0.321192, "nauc_map_at_100_max": -0.070234, "nauc_map_at_100_std": -0.093946, "nauc_map_at_100_diff1": 0.325332, "nauc_map_at_1000_max": -0.071537, "nauc_map_at_1000_std": -0.095323, "nauc_map_at_1000_diff1": 0.324059, "nauc_recall_at_1_max": -0.056706, "nauc_recall_at_1_std": -0.124844, "nauc_recall_at_1_diff1": 0.450669, "nauc_recall_at_3_max": -0.081929, "nauc_recall_at_3_std": -0.131015, "nauc_recall_at_3_diff1": 0.307042, "nauc_recall_at_5_max": -0.071647, "nauc_recall_at_5_std": -0.114057, "nauc_recall_at_5_diff1": 0.276066, "nauc_recall_at_10_max": -0.056115, "nauc_recall_at_10_std": -0.087721, "nauc_recall_at_10_diff1": 0.292684, "nauc_recall_at_20_max": 0.015783, "nauc_recall_at_20_std": 0.024308, "nauc_recall_at_20_diff1": 0.319784, "nauc_recall_at_100_max": 0.378965, "nauc_recall_at_100_std": 0.457292, "nauc_recall_at_100_diff1": 0.62761, "nauc_recall_at_1000_max": 0.755919, "nauc_recall_at_1000_std": 0.755919, "nauc_recall_at_1000_diff1": 0.739124, "nauc_precision_at_1_max": -0.137214, "nauc_precision_at_1_std": -0.176463, "nauc_precision_at_1_diff1": 0.419326, "nauc_precision_at_3_max": -0.077519, "nauc_precision_at_3_std": -0.017821, "nauc_precision_at_3_diff1": -0.080807, "nauc_precision_at_5_max": -0.015248, "nauc_precision_at_5_std": 0.04949, "nauc_precision_at_5_diff1": -0.133631, "nauc_precision_at_10_max": 0.027503, "nauc_precision_at_10_std": 0.093518, "nauc_precision_at_10_diff1": -0.164421, "nauc_precision_at_20_max": 0.067115, "nauc_precision_at_20_std": 0.149744, "nauc_precision_at_20_diff1": -0.17628, "nauc_precision_at_100_max": 0.08277, "nauc_precision_at_100_std": 0.155371, "nauc_precision_at_100_diff1": -0.193348, "nauc_precision_at_1000_max": 0.057928, "nauc_precision_at_1000_std": 0.125992, "nauc_precision_at_1000_diff1": -0.216967, "nauc_mrr_at_1_max": -0.137214, "nauc_mrr_at_1_std": -0.176463, "nauc_mrr_at_1_diff1": 0.419326, "nauc_mrr_at_3_max": -0.159498, "nauc_mrr_at_3_std": -0.188332, "nauc_mrr_at_3_diff1": 0.416204, "nauc_mrr_at_5_max": -0.153303, "nauc_mrr_at_5_std": -0.178736, "nauc_mrr_at_5_diff1": 0.428013, "nauc_mrr_at_10_max": -0.147898, "nauc_mrr_at_10_std": -0.174553, "nauc_mrr_at_10_diff1": 0.427978, "nauc_mrr_at_20_max": -0.149272, "nauc_mrr_at_20_std": -0.177149, "nauc_mrr_at_20_diff1": 0.424956, "nauc_mrr_at_100_max": -0.150289, "nauc_mrr_at_100_std": -0.177637, "nauc_mrr_at_100_diff1": 0.425305, "nauc_mrr_at_1000_max": -0.150364, "nauc_mrr_at_1000_std": -0.177734, "nauc_mrr_at_1000_diff1": 0.425291, "hit_rate_at_1": 0.77586, "hit_rate_at_3": 0.89655, "hit_rate_at_5": 0.92529, "hit_rate_at_10": 0.96839, "hit_rate_at_20": 0.98563, "hit_rate_at_100": 0.99425, "hit_rate_at_1000": 1.0, "main_score": 0.73277, "hf_subset": "spanish", "languages": [ "spa-Latn" ], "mteb_version": "2.24.2" }, { "ndcg_at_1": 0.75575, "ndcg_at_3": 0.70758, "ndcg_at_5": 0.71624, "ndcg_at_10": 0.74211, "ndcg_at_20": 0.77253, "ndcg_at_100": 0.81243, "ndcg_at_1000": 0.8161, "map_at_1": 0.33317, "map_at_3": 0.50755, "map_at_5": 0.56878, "map_at_10": 0.62579, "map_at_20": 0.65572, "map_at_100": 0.67647, "map_at_1000": 0.67708, "recall_at_1": 0.33317, "recall_at_3": 0.54993, "recall_at_5": 0.65333, "recall_at_10": 0.77039, "recall_at_20": 0.86713, "recall_at_100": 0.98085, "recall_at_1000": 0.99763, "accuracy": 0.33317, "precision_at_1": 0.78161, "precision_at_3": 0.51628, "precision_at_5": 0.40747, "precision_at_10": 0.27385, "precision_at_20": 0.16853, "precision_at_100": 0.04348, "precision_at_1000": 0.00447, "mrr_at_1": 0.781609, "mrr_at_3": 0.841475, "mrr_at_5": 0.847941, "mrr_at_10": 0.852481, "mrr_at_20": 0.853609, "mrr_at_100": 0.853818, "mrr_at_1000": 0.853848, "nauc_ndcg_at_1_max": -0.108872, "nauc_ndcg_at_1_std": -0.197169, "nauc_ndcg_at_1_diff1": 0.47261, "nauc_ndcg_at_3_max": -0.140417, "nauc_ndcg_at_3_std": -0.140157, "nauc_ndcg_at_3_diff1": 0.340023, "nauc_ndcg_at_5_max": -0.093832, "nauc_ndcg_at_5_std": -0.117286, "nauc_ndcg_at_5_diff1": 0.371479, "nauc_ndcg_at_10_max": -0.071976, "nauc_ndcg_at_10_std": -0.114372, "nauc_ndcg_at_10_diff1": 0.378706, "nauc_ndcg_at_20_max": -0.057218, "nauc_ndcg_at_20_std": -0.098025, "nauc_ndcg_at_20_diff1": 0.388862, "nauc_ndcg_at_100_max": -0.068429, "nauc_ndcg_at_100_std": -0.097439, "nauc_ndcg_at_100_diff1": 0.403914, "nauc_ndcg_at_1000_max": -0.076109, "nauc_ndcg_at_1000_std": -0.103979, "nauc_ndcg_at_1000_diff1": 0.396163, "nauc_map_at_1_max": -0.069578, "nauc_map_at_1_std": -0.109848, "nauc_map_at_1_diff1": 0.49012, "nauc_map_at_3_max": -0.063035, "nauc_map_at_3_std": -0.094063, "nauc_map_at_3_diff1": 0.360756, "nauc_map_at_5_max": -0.055468, "nauc_map_at_5_std": -0.095199, "nauc_map_at_5_diff1": 0.358973, "nauc_map_at_10_max": -0.055567, "nauc_map_at_10_std": -0.101769, "nauc_map_at_10_diff1": 0.349018, "nauc_map_at_20_max": -0.051412, "nauc_map_at_20_std": -0.082908, "nauc_map_at_20_diff1": 0.353842, "nauc_map_at_100_max": -0.051486, "nauc_map_at_100_std": -0.07284, "nauc_map_at_100_diff1": 0.358577, "nauc_map_at_1000_max": -0.05253, "nauc_map_at_1000_std": -0.073575, "nauc_map_at_1000_diff1": 0.357557, "nauc_recall_at_1_max": -0.069578, "nauc_recall_at_1_std": -0.109848, "nauc_recall_at_1_diff1": 0.49012, "nauc_recall_at_3_max": -0.043524, "nauc_recall_at_3_std": -0.086839, "nauc_recall_at_3_diff1": 0.298526, "nauc_recall_at_5_max": -0.021262, "nauc_recall_at_5_std": -0.085977, "nauc_recall_at_5_diff1": 0.295839, "nauc_recall_at_10_max": -0.000738, "nauc_recall_at_10_std": -0.083479, "nauc_recall_at_10_diff1": 0.294519, "nauc_recall_at_20_max": 0.05793, "nauc_recall_at_20_std": -0.008471, "nauc_recall_at_20_diff1": 0.286294, "nauc_recall_at_100_max": 0.32675, "nauc_recall_at_100_std": 0.357481, "nauc_recall_at_100_diff1": 0.537933, "nauc_recall_at_1000_max": 0.605336, "nauc_recall_at_1000_std": 0.734227, "nauc_recall_at_1000_diff1": 0.303933, "nauc_precision_at_1_max": -0.152101, "nauc_precision_at_1_std": -0.177159, "nauc_precision_at_1_diff1": 0.48863, "nauc_precision_at_3_max": -0.059705, "nauc_precision_at_3_std": -0.007435, "nauc_precision_at_3_diff1": -0.096444, "nauc_precision_at_5_max": 0.002027, "nauc_precision_at_5_std": 0.032587, "nauc_precision_at_5_diff1": -0.136257, "nauc_precision_at_10_max": 0.014208, "nauc_precision_at_10_std": 0.059007, "nauc_precision_at_10_diff1": -0.183528, "nauc_precision_at_20_max": 0.031969, "nauc_precision_at_20_std": 0.110806, "nauc_precision_at_20_diff1": -0.193019, "nauc_precision_at_100_max": 0.014618, "nauc_precision_at_100_std": 0.105018, "nauc_precision_at_100_diff1": -0.212202, "nauc_precision_at_1000_max": -0.001618, "nauc_precision_at_1000_std": 0.087612, "nauc_precision_at_1000_diff1": -0.229574, "nauc_mrr_at_1_max": -0.152101, "nauc_mrr_at_1_std": -0.177159, "nauc_mrr_at_1_diff1": 0.48863, "nauc_mrr_at_3_max": -0.165808, "nauc_mrr_at_3_std": -0.184678, "nauc_mrr_at_3_diff1": 0.452407, "nauc_mrr_at_5_max": -0.159284, "nauc_mrr_at_5_std": -0.185532, "nauc_mrr_at_5_diff1": 0.472554, "nauc_mrr_at_10_max": -0.160842, "nauc_mrr_at_10_std": -0.177647, "nauc_mrr_at_10_diff1": 0.470991, "nauc_mrr_at_20_max": -0.158324, "nauc_mrr_at_20_std": -0.177699, "nauc_mrr_at_20_diff1": 0.469826, "nauc_mrr_at_100_max": -0.159702, "nauc_mrr_at_100_std": -0.178174, "nauc_mrr_at_100_diff1": 0.469543, "nauc_mrr_at_1000_max": -0.159867, "nauc_mrr_at_1000_std": -0.178431, "nauc_mrr_at_1000_diff1": 0.469486, "hit_rate_at_1": 0.78161, "hit_rate_at_3": 0.91092, "hit_rate_at_5": 0.93966, "hit_rate_at_10": 0.97126, "hit_rate_at_20": 0.98851, "hit_rate_at_100": 0.99425, "hit_rate_at_1000": 1.0, "main_score": 0.74211, "hf_subset": "english", "languages": [ "eng-Latn" ], "mteb_version": "2.24.2" }, { "ndcg_at_1": 0.73276, "ndcg_at_3": 0.69098, "ndcg_at_5": 0.70169, "ndcg_at_10": 0.73103, "ndcg_at_20": 0.76108, "ndcg_at_100": 0.80122, "ndcg_at_1000": 0.80613, "map_at_1": 0.32952, "map_at_3": 0.49824, "map_at_5": 0.55907, "map_at_10": 0.6163, "map_at_20": 0.64529, "map_at_100": 0.66567, "map_at_1000": 0.66646, "recall_at_1": 0.32952, "recall_at_3": 0.53709, "recall_at_5": 0.63984, "recall_at_10": 0.76098, "recall_at_20": 0.85735, "recall_at_100": 0.97473, "recall_at_1000": 0.99763, "accuracy": 0.32952, "precision_at_1": 0.75575, "precision_at_3": 0.50479, "precision_at_5": 0.39885, "precision_at_10": 0.2704, "precision_at_20": 0.16652, "precision_at_100": 0.04322, "precision_at_1000": 0.00447, "mrr_at_1": 0.755747, "mrr_at_3": 0.818966, "mrr_at_5": 0.826868, "mrr_at_10": 0.83367, "mrr_at_20": 0.834745, "mrr_at_100": 0.834943, "mrr_at_1000": 0.834977, "nauc_ndcg_at_1_max": -0.134882, "nauc_ndcg_at_1_std": -0.162813, "nauc_ndcg_at_1_diff1": 0.414105, "nauc_ndcg_at_3_max": -0.182898, "nauc_ndcg_at_3_std": -0.167688, "nauc_ndcg_at_3_diff1": 0.281133, "nauc_ndcg_at_5_max": -0.158793, "nauc_ndcg_at_5_std": -0.15237, "nauc_ndcg_at_5_diff1": 0.319075, "nauc_ndcg_at_10_max": -0.118853, "nauc_ndcg_at_10_std": -0.119739, "nauc_ndcg_at_10_diff1": 0.337305, "nauc_ndcg_at_20_max": -0.111829, "nauc_ndcg_at_20_std": -0.106824, "nauc_ndcg_at_20_diff1": 0.354797, "nauc_ndcg_at_100_max": -0.101812, "nauc_ndcg_at_100_std": -0.094862, "nauc_ndcg_at_100_diff1": 0.360514, "nauc_ndcg_at_1000_max": -0.115735, "nauc_ndcg_at_1000_std": -0.107537, "nauc_ndcg_at_1000_diff1": 0.352115, "nauc_map_at_1_max": -0.090293, "nauc_map_at_1_std": -0.132946, "nauc_map_at_1_diff1": 0.450874, "nauc_map_at_3_max": -0.111025, "nauc_map_at_3_std": -0.145562, "nauc_map_at_3_diff1": 0.327746, "nauc_map_at_5_max": -0.108674, "nauc_map_at_5_std": -0.142632, "nauc_map_at_5_diff1": 0.325504, "nauc_map_at_10_max": -0.102986, "nauc_map_at_10_std": -0.129085, "nauc_map_at_10_diff1": 0.318262, "nauc_map_at_20_max": -0.099422, "nauc_map_at_20_std": -0.112593, "nauc_map_at_20_diff1": 0.323833, "nauc_map_at_100_max": -0.094176, "nauc_map_at_100_std": -0.101716, "nauc_map_at_100_diff1": 0.323027, "nauc_map_at_1000_max": -0.095574, "nauc_map_at_1000_std": -0.102934, "nauc_map_at_1000_diff1": 0.321879, "nauc_recall_at_1_max": -0.090293, "nauc_recall_at_1_std": -0.132946, "nauc_recall_at_1_diff1": 0.450874, "nauc_recall_at_3_max": -0.11942, "nauc_recall_at_3_std": -0.149416, "nauc_recall_at_3_diff1": 0.282551, "nauc_recall_at_5_max": -0.108939, "nauc_recall_at_5_std": -0.146978, "nauc_recall_at_5_diff1": 0.284356, "nauc_recall_at_10_max": -0.054119, "nauc_recall_at_10_std": -0.085949, "nauc_recall_at_10_diff1": 0.282362, "nauc_recall_at_20_max": 0.00601, "nauc_recall_at_20_std": 0.004912, "nauc_recall_at_20_diff1": 0.312112, "nauc_recall_at_100_max": 0.395189, "nauc_recall_at_100_std": 0.435001, "nauc_recall_at_100_diff1": 0.509614, "nauc_recall_at_1000_max": 0.486043, "nauc_recall_at_1000_std": 0.644973, "nauc_recall_at_1000_diff1": 0.424271, "nauc_precision_at_1_max": -0.173571, "nauc_precision_at_1_std": -0.169644, "nauc_precision_at_1_diff1": 0.441569, "nauc_precision_at_3_max": -0.091075, "nauc_precision_at_3_std": -0.046715, "nauc_precision_at_3_diff1": -0.108522, "nauc_precision_at_5_max": -0.032715, "nauc_precision_at_5_std": 0.024178, "nauc_precision_at_5_diff1": -0.129456, "nauc_precision_at_10_max": 0.04391, "nauc_precision_at_10_std": 0.101271, "nauc_precision_at_10_diff1": -0.161148, "nauc_precision_at_20_max": 0.057013, "nauc_precision_at_20_std": 0.136199, "nauc_precision_at_20_diff1": -0.173644, "nauc_precision_at_100_max": 0.058834, "nauc_precision_at_100_std": 0.134599, "nauc_precision_at_100_diff1": -0.200519, "nauc_precision_at_1000_max": 0.037439, "nauc_precision_at_1000_std": 0.111671, "nauc_precision_at_1000_diff1": -0.22124, "nauc_mrr_at_1_max": -0.173571, "nauc_mrr_at_1_std": -0.169644, "nauc_mrr_at_1_diff1": 0.441569, "nauc_mrr_at_3_max": -0.215267, "nauc_mrr_at_3_std": -0.176615, "nauc_mrr_at_3_diff1": 0.397816, "nauc_mrr_at_5_max": -0.214704, "nauc_mrr_at_5_std": -0.183627, "nauc_mrr_at_5_diff1": 0.41573, "nauc_mrr_at_10_max": -0.19988, "nauc_mrr_at_10_std": -0.176013, "nauc_mrr_at_10_diff1": 0.422645, "nauc_mrr_at_20_max": -0.196716, "nauc_mrr_at_20_std": -0.173888, "nauc_mrr_at_20_diff1": 0.422087, "nauc_mrr_at_100_max": -0.197765, "nauc_mrr_at_100_std": -0.173793, "nauc_mrr_at_100_diff1": 0.421715, "nauc_mrr_at_1000_max": -0.197995, "nauc_mrr_at_1000_std": -0.174048, "nauc_mrr_at_1000_diff1": 0.421621, "hit_rate_at_1": 0.75575, "hit_rate_at_3": 0.8908, "hit_rate_at_5": 0.92241, "hit_rate_at_10": 0.96839, "hit_rate_at_20": 0.98563, "hit_rate_at_100": 0.99138, "hit_rate_at_1000": 1.0, "main_score": 0.73103, "hf_subset": "german", "languages": [ "deu-Latn" ], "mteb_version": "2.24.2" }, { "ndcg_at_1": 0.76437, "ndcg_at_3": 0.71215, "ndcg_at_5": 0.71087, "ndcg_at_10": 0.74189, "ndcg_at_20": 0.76818, "ndcg_at_100": 0.80979, "ndcg_at_1000": 0.81459, "map_at_1": 0.34203, "map_at_3": 0.51425, "map_at_5": 0.5664, "map_at_10": 0.62659, "map_at_20": 0.65249, "map_at_100": 0.6739, "map_at_1000": 0.6747, "recall_at_1": 0.34203, "recall_at_3": 0.55356, "recall_at_5": 0.64134, "recall_at_10": 0.76709, "recall_at_20": 0.85495, "recall_at_100": 0.97528, "recall_at_1000": 0.99691, "accuracy": 0.34203, "precision_at_1": 0.79023, "precision_at_3": 0.52107, "precision_at_5": 0.39713, "precision_at_10": 0.27299, "precision_at_20": 0.16437, "precision_at_100": 0.04307, "precision_at_1000": 0.00447, "mrr_at_1": 0.79023, "mrr_at_3": 0.841954, "mrr_at_5": 0.848563, "mrr_at_10": 0.85405, "mrr_at_20": 0.85466, "mrr_at_100": 0.855005, "mrr_at_1000": 0.855021, "nauc_ndcg_at_1_max": -0.184577, "nauc_ndcg_at_1_std": -0.205842, "nauc_ndcg_at_1_diff1": 0.379149, "nauc_ndcg_at_3_max": -0.206232, "nauc_ndcg_at_3_std": -0.167253, "nauc_ndcg_at_3_diff1": 0.281648, "nauc_ndcg_at_5_max": -0.180705, "nauc_ndcg_at_5_std": -0.163053, "nauc_ndcg_at_5_diff1": 0.294598, "nauc_ndcg_at_10_max": -0.164321, "nauc_ndcg_at_10_std": -0.149672, "nauc_ndcg_at_10_diff1": 0.32367, "nauc_ndcg_at_20_max": -0.151063, "nauc_ndcg_at_20_std": -0.134784, "nauc_ndcg_at_20_diff1": 0.335016, "nauc_ndcg_at_100_max": -0.142333, "nauc_ndcg_at_100_std": -0.123245, "nauc_ndcg_at_100_diff1": 0.341864, "nauc_ndcg_at_1000_max": -0.153567, "nauc_ndcg_at_1000_std": -0.135865, "nauc_ndcg_at_1000_diff1": 0.331899, "nauc_map_at_1_max": -0.103456, "nauc_map_at_1_std": -0.1507, "nauc_map_at_1_diff1": 0.401812, "nauc_map_at_3_max": -0.127046, "nauc_map_at_3_std": -0.153907, "nauc_map_at_3_diff1": 0.309543, "nauc_map_at_5_max": -0.135294, "nauc_map_at_5_std": -0.154764, "nauc_map_at_5_diff1": 0.293814, "nauc_map_at_10_max": -0.143286, "nauc_map_at_10_std": -0.15246, "nauc_map_at_10_diff1": 0.294035, "nauc_map_at_20_max": -0.135901, "nauc_map_at_20_std": -0.134535, "nauc_map_at_20_diff1": 0.296246, "nauc_map_at_100_max": -0.130571, "nauc_map_at_100_std": -0.121097, "nauc_map_at_100_diff1": 0.298421, "nauc_map_at_1000_max": -0.131834, "nauc_map_at_1000_std": -0.122302, "nauc_map_at_1000_diff1": 0.297355, "nauc_recall_at_1_max": -0.103456, "nauc_recall_at_1_std": -0.1507, "nauc_recall_at_1_diff1": 0.401812, "nauc_recall_at_3_max": -0.116132, "nauc_recall_at_3_std": -0.13495, "nauc_recall_at_3_diff1": 0.27427, "nauc_recall_at_5_max": -0.106144, "nauc_recall_at_5_std": -0.126648, "nauc_recall_at_5_diff1": 0.258019, "nauc_recall_at_10_max": -0.096563, "nauc_recall_at_10_std": -0.102587, "nauc_recall_at_10_diff1": 0.280571, "nauc_recall_at_20_max": -0.042018, "nauc_recall_at_20_std": -0.012866, "nauc_recall_at_20_diff1": 0.292724, "nauc_recall_at_100_max": 0.309162, "nauc_recall_at_100_std": 0.414646, "nauc_recall_at_100_diff1": 0.574123, "nauc_recall_at_1000_max": 0.635987, "nauc_recall_at_1000_std": 0.635987, "nauc_recall_at_1000_diff1": 0.469639, "nauc_precision_at_1_max": -0.255716, "nauc_precision_at_1_std": -0.206995, "nauc_precision_at_1_diff1": 0.393819, "nauc_precision_at_3_max": -0.098066, "nauc_precision_at_3_std": -0.012275, "nauc_precision_at_3_diff1": -0.065356, "nauc_precision_at_5_max": -0.05379, "nauc_precision_at_5_std": 0.020962, "nauc_precision_at_5_diff1": -0.126763, "nauc_precision_at_10_max": 0.005174, "nauc_precision_at_10_std": 0.087274, "nauc_precision_at_10_diff1": -0.156872, "nauc_precision_at_20_max": 0.045891, "nauc_precision_at_20_std": 0.134715, "nauc_precision_at_20_diff1": -0.168381, "nauc_precision_at_100_max": 0.061133, "nauc_precision_at_100_std": 0.143101, "nauc_precision_at_100_diff1": -0.191538, "nauc_precision_at_1000_max": 0.040635, "nauc_precision_at_1000_std": 0.117149, "nauc_precision_at_1000_diff1": -0.212478, "nauc_mrr_at_1_max": -0.255716, "nauc_mrr_at_1_std": -0.206995, "nauc_mrr_at_1_diff1": 0.393819, "nauc_mrr_at_3_max": -0.284131, "nauc_mrr_at_3_std": -0.213194, "nauc_mrr_at_3_diff1": 0.375097, "nauc_mrr_at_5_max": -0.264599, "nauc_mrr_at_5_std": -0.209257, "nauc_mrr_at_5_diff1": 0.404064, "nauc_mrr_at_10_max": -0.265321, "nauc_mrr_at_10_std": -0.206029, "nauc_mrr_at_10_diff1": 0.398065, "nauc_mrr_at_20_max": -0.263752, "nauc_mrr_at_20_std": -0.204453, "nauc_mrr_at_20_diff1": 0.397392, "nauc_mrr_at_100_max": -0.264373, "nauc_mrr_at_100_std": -0.205102, "nauc_mrr_at_100_diff1": 0.39668, "nauc_mrr_at_1000_max": -0.264478, "nauc_mrr_at_1000_std": -0.205243, "nauc_mrr_at_1000_diff1": 0.39667, "hit_rate_at_1": 0.79023, "hit_rate_at_3": 0.90517, "hit_rate_at_5": 0.93391, "hit_rate_at_10": 0.97414, "hit_rate_at_20": 0.98276, "hit_rate_at_100": 0.99425, "hit_rate_at_1000": 1.0, "main_score": 0.74189, "hf_subset": "italian", "languages": [ "ita-Latn" ], "mteb_version": "2.24.2" }, { "ndcg_at_1": 0.74282, "ndcg_at_3": 0.70159, "ndcg_at_5": 0.70788, "ndcg_at_10": 0.73786, "ndcg_at_20": 0.76559, "ndcg_at_100": 0.80573, "ndcg_at_1000": 0.81018, "map_at_1": 0.33402, "map_at_3": 0.50433, "map_at_5": 0.56104, "map_at_10": 0.62127, "map_at_20": 0.64852, "map_at_100": 0.66889, "map_at_1000": 0.66965, "recall_at_1": 0.33402, "recall_at_3": 0.54931, "recall_at_5": 0.64439, "recall_at_10": 0.7676, "recall_at_20": 0.85915, "recall_at_100": 0.97675, "recall_at_1000": 0.99763, "accuracy": 0.33402, "precision_at_1": 0.77011, "precision_at_3": 0.51149, "precision_at_5": 0.4, "precision_at_10": 0.27471, "precision_at_20": 0.16609, "precision_at_100": 0.04319, "precision_at_1000": 0.00447, "mrr_at_1": 0.770115, "mrr_at_3": 0.831897, "mrr_at_5": 0.840086, "mrr_at_10": 0.844148, "mrr_at_20": 0.845249, "mrr_at_100": 0.845527, "mrr_at_1000": 0.845536, "nauc_ndcg_at_1_max": -0.092037, "nauc_ndcg_at_1_std": -0.147628, "nauc_ndcg_at_1_diff1": 0.402735, "nauc_ndcg_at_3_max": -0.169673, "nauc_ndcg_at_3_std": -0.170492, "nauc_ndcg_at_3_diff1": 0.278351, "nauc_ndcg_at_5_max": -0.134577, "nauc_ndcg_at_5_std": -0.156489, "nauc_ndcg_at_5_diff1": 0.302761, "nauc_ndcg_at_10_max": -0.109038, "nauc_ndcg_at_10_std": -0.132, "nauc_ndcg_at_10_diff1": 0.33392, "nauc_ndcg_at_20_max": -0.092576, "nauc_ndcg_at_20_std": -0.125863, "nauc_ndcg_at_20_diff1": 0.349173, "nauc_ndcg_at_100_max": -0.077635, "nauc_ndcg_at_100_std": -0.106007, "nauc_ndcg_at_100_diff1": 0.351264, "nauc_ndcg_at_1000_max": -0.087579, "nauc_ndcg_at_1000_std": -0.114295, "nauc_ndcg_at_1000_diff1": 0.342281, "nauc_map_at_1_max": -0.093445, "nauc_map_at_1_std": -0.129755, "nauc_map_at_1_diff1": 0.436933, "nauc_map_at_3_max": -0.126752, "nauc_map_at_3_std": -0.143703, "nauc_map_at_3_diff1": 0.306622, "nauc_map_at_5_max": -0.118495, "nauc_map_at_5_std": -0.137492, "nauc_map_at_5_diff1": 0.297203, "nauc_map_at_10_max": -0.112256, "nauc_map_at_10_std": -0.133393, "nauc_map_at_10_diff1": 0.296, "nauc_map_at_20_max": -0.096979, "nauc_map_at_20_std": -0.11965, "nauc_map_at_20_diff1": 0.301686, "nauc_map_at_100_max": -0.083486, "nauc_map_at_100_std": -0.102847, "nauc_map_at_100_diff1": 0.304111, "nauc_map_at_1000_max": -0.084717, "nauc_map_at_1000_std": -0.103829, "nauc_map_at_1000_diff1": 0.303072, "nauc_recall_at_1_max": -0.093445, "nauc_recall_at_1_std": -0.129755, "nauc_recall_at_1_diff1": 0.436933, "nauc_recall_at_3_max": -0.135993, "nauc_recall_at_3_std": -0.153379, "nauc_recall_at_3_diff1": 0.267017, "nauc_recall_at_5_max": -0.105329, "nauc_recall_at_5_std": -0.145296, "nauc_recall_at_5_diff1": 0.265856, "nauc_recall_at_10_max": -0.075815, "nauc_recall_at_10_std": -0.115499, "nauc_recall_at_10_diff1": 0.297198, "nauc_recall_at_20_max": 0.003931, "nauc_recall_at_20_std": -0.055427, "nauc_recall_at_20_diff1": 0.31664, "nauc_recall_at_100_max": 0.355794, "nauc_recall_at_100_std": 0.335729, "nauc_recall_at_100_diff1": 0.573358, "nauc_recall_at_1000_max": 0.644973, "nauc_recall_at_1000_std": 0.52568, "nauc_recall_at_1000_diff1": 0.268283, "nauc_precision_at_1_max": -0.147232, "nauc_precision_at_1_std": -0.153312, "nauc_precision_at_1_diff1": 0.413932, "nauc_precision_at_3_max": -0.075205, "nauc_precision_at_3_std": -0.037866, "nauc_precision_at_3_diff1": -0.100835, "nauc_precision_at_5_max": 0.00614, "nauc_precision_at_5_std": 0.033417, "nauc_precision_at_5_diff1": -0.142446, "nauc_precision_at_10_max": 0.075471, "nauc_precision_at_10_std": 0.092921, "nauc_precision_at_10_diff1": -0.16264, "nauc_precision_at_20_max": 0.121531, "nauc_precision_at_20_std": 0.135392, "nauc_precision_at_20_diff1": -0.159134, "nauc_precision_at_100_max": 0.119498, "nauc_precision_at_100_std": 0.148142, "nauc_precision_at_100_diff1": -0.190302, "nauc_precision_at_1000_max": 0.094379, "nauc_precision_at_1000_std": 0.124775, "nauc_precision_at_1000_diff1": -0.210931, "nauc_mrr_at_1_max": -0.147232, "nauc_mrr_at_1_std": -0.153312, "nauc_mrr_at_1_diff1": 0.413932, "nauc_mrr_at_3_max": -0.215014, "nauc_mrr_at_3_std": -0.219315, "nauc_mrr_at_3_diff1": 0.366828, "nauc_mrr_at_5_max": -0.181234, "nauc_mrr_at_5_std": -0.195353, "nauc_mrr_at_5_diff1": 0.400583, "nauc_mrr_at_10_max": -0.183632, "nauc_mrr_at_10_std": -0.193282, "nauc_mrr_at_10_diff1": 0.399578, "nauc_mrr_at_20_max": -0.182326, "nauc_mrr_at_20_std": -0.191712, "nauc_mrr_at_20_diff1": 0.397014, "nauc_mrr_at_100_max": -0.182624, "nauc_mrr_at_100_std": -0.191294, "nauc_mrr_at_100_diff1": 0.396815, "nauc_mrr_at_1000_max": -0.182678, "nauc_mrr_at_1000_std": -0.191363, "nauc_mrr_at_1000_diff1": 0.396803, "hit_rate_at_1": 0.77011, "hit_rate_at_3": 0.90805, "hit_rate_at_5": 0.94253, "hit_rate_at_10": 0.97126, "hit_rate_at_20": 0.98563, "hit_rate_at_100": 0.99425, "hit_rate_at_1000": 1.0, "main_score": 0.73786, "hf_subset": "portuguese", "languages": [ "por-Latn" ], "mteb_version": "2.24.2" } ] }, "evaluation_time": 22562.09231189266, "kg_co2_emissions": null, "date": 1791509594.372127, "evaluation_phases": [ { "name": "Data loading", "start": 0.00012433528900146484, "end": 43.22077424451709, "split": "", "subset": "" }, { "name": "Dataset transform", "start": 43.22084164619446, "end": 43.22084503248334, "split": "", "subset": "" }, { "name": "Encoding corpus", "start": 43.23248517140746, "end": 43.23249091953039, "split": "test", "subset": "french" }, { "name": "Encoding queries", "start": 43.23253624141216, "end": 3903.0552540682256, "split": "test", "subset": "french" }, { "name": "Scoring", "start": 3903.0557616986334, "end": 3903.3711392581463, "split": "test", "subset": "french" }, { "name": "Encoding corpus", "start": 3903.408052146435, "end": 3903.408066406846, "split": "test", "subset": "spanish" }, { "name": "Encoding queries", "start": 3903.4081182181835, "end": 7571.994422711432, "split": "test", "subset": "spanish" }, { "name": "Scoring", "start": 7571.994797986001, "end": 7572.271246403456, "split": "test", "subset": "spanish" }, { "name": "Encoding corpus", "start": 7574.030859082937, "end": 7574.030866235495, "split": "test", "subset": "english" }, { "name": "Encoding queries", "start": 7574.0309021137655, "end": 11098.674841441214, "split": "test", "subset": "english" }, { "name": "Scoring", "start": 11098.675216130912, "end": 11098.950714200735, "split": "test", "subset": "english" }, { "name": "Encoding corpus", "start": 11098.98907564208, "end": 11098.989082571119, "split": "test", "subset": "german" }, { "name": "Encoding queries", "start": 11098.98911749199, "end": 15021.17317847535, "split": "test", "subset": "german" }, { "name": "Scoring", "start": 15021.173599131405, "end": 15021.451539825648, "split": "test", "subset": "german" }, { "name": "Encoding corpus", "start": 15021.486063562334, "end": 15021.486070293933, "split": "test", "subset": "italian" }, { "name": "Encoding queries", "start": 15021.486106395721, "end": 18833.613203197718, "split": "test", "subset": "italian" }, { "name": "Scoring", "start": 18833.613681398332, "end": 18833.892971776426, "split": "test", "subset": "italian" }, { "name": "Encoding corpus", "start": 18833.9414957799, "end": 18833.941502530128, "split": "test", "subset": "portuguese" }, { "name": "Encoding queries", "start": 18833.94153838232, "end": 22561.769182179123, "split": "test", "subset": "portuguese" }, { "name": "Scoring", "start": 22561.76967636496, "end": 22562.057531576604, "split": "test", "subset": "portuguese" } ] }