{ "dataset_revision": "057fe83342b446ede7928cb51bc6c7c3223da3d0", "task_name": "Vidore3TelecomRetrieval.v2", "mteb_version": "2.13.1", "scores": { "test": [ { "ndcg_at_1": 0.00287, "ndcg_at_3": 0.00181, "ndcg_at_5": 0.00212, "ndcg_at_10": 0.00375, "ndcg_at_20": 0.00942, "ndcg_at_100": 0.0199, "ndcg_at_1000": 0.06091, "map_at_1": 0.00048, "map_at_3": 0.00053, "map_at_5": 0.00101, "map_at_10": 0.00156, "map_at_20": 0.00297, "map_at_100": 0.00412, "map_at_1000": 0.00533, "recall_at_1": 0.00048, "recall_at_3": 0.00064, "recall_at_5": 0.00303, "recall_at_10": 0.00735, "recall_at_20": 0.02626, "recall_at_100": 0.06834, "recall_at_1000": 0.30209, "accuracy": 0.00048, "precision_at_1": 0.00287, "precision_at_3": 0.00192, "precision_at_5": 0.0023, "precision_at_10": 0.00259, "precision_at_20": 0.00417, "precision_at_100": 0.00253, "precision_at_1000": 0.00134, "mrr_at_1": 0.002874, "mrr_at_3": 0.003831, "mrr_at_5": 0.004981, "mrr_at_10": 0.006828, "mrr_at_20": 0.010102, "mrr_at_100": 0.012242, "mrr_at_1000": 0.01348, "nauc_ndcg_at_1_max": -0.319386, "nauc_ndcg_at_1_std": -0.416095, "nauc_ndcg_at_1_diff1": 0.325795, "nauc_ndcg_at_3_max": -0.319386, "nauc_ndcg_at_3_std": -0.437647, "nauc_ndcg_at_3_diff1": 0.496976, "nauc_ndcg_at_5_max": -0.153715, "nauc_ndcg_at_5_std": -0.090761, "nauc_ndcg_at_5_diff1": 0.187335, "nauc_ndcg_at_10_max": -0.021798, "nauc_ndcg_at_10_std": 0.003132, "nauc_ndcg_at_10_diff1": 0.099427, "nauc_ndcg_at_20_max": 0.060467, "nauc_ndcg_at_20_std": -0.008817, "nauc_ndcg_at_20_diff1": 0.033713, "nauc_ndcg_at_100_max": 0.091573, "nauc_ndcg_at_100_std": 0.004819, "nauc_ndcg_at_100_diff1": 0.098732, "nauc_ndcg_at_1000_max": 0.063431, "nauc_ndcg_at_1000_std": -0.027761, "nauc_ndcg_at_1000_diff1": 0.054538, "nauc_map_at_1_max": -0.319386, "nauc_map_at_1_std": -0.416095, "nauc_map_at_1_diff1": 0.325795, "nauc_map_at_3_max": -0.319386, "nauc_map_at_3_std": -0.424584, "nauc_map_at_3_diff1": 0.393216, "nauc_map_at_5_max": -0.074617, "nauc_map_at_5_std": 0.026667, "nauc_map_at_5_diff1": 0.02545, "nauc_map_at_10_max": -0.011984, "nauc_map_at_10_std": 0.011156, "nauc_map_at_10_diff1": 0.058024, "nauc_map_at_20_max": 0.031348, "nauc_map_at_20_std": 7e-05, "nauc_map_at_20_diff1": 0.008182, "nauc_map_at_100_max": 0.051288, "nauc_map_at_100_std": -0.003656, "nauc_map_at_100_diff1": 0.058152, "nauc_map_at_1000_max": 0.043289, "nauc_map_at_1000_std": -0.006735, "nauc_map_at_1000_diff1": 0.049859, "nauc_recall_at_1_max": -0.319386, "nauc_recall_at_1_std": -0.416095, "nauc_recall_at_1_diff1": 0.325795, "nauc_recall_at_3_max": -0.319386, "nauc_recall_at_3_std": -0.437316, "nauc_recall_at_3_diff1": 0.494346, "nauc_recall_at_5_max": 0.088562, "nauc_recall_at_5_std": 0.32482, "nauc_recall_at_5_diff1": -0.198437, "nauc_recall_at_10_max": 0.097477, "nauc_recall_at_10_std": 0.118258, "nauc_recall_at_10_diff1": -0.006859, "nauc_recall_at_20_max": 0.116375, "nauc_recall_at_20_std": 0.016813, "nauc_recall_at_20_diff1": -0.036478, "nauc_recall_at_100_max": 0.117717, "nauc_recall_at_100_std": 0.011561, "nauc_recall_at_100_diff1": 0.086874, "nauc_recall_at_1000_max": 0.056501, "nauc_recall_at_1000_std": -0.00468, "nauc_recall_at_1000_diff1": 0.006595, "nauc_precision_at_1_max": -0.319386, "nauc_precision_at_1_std": -0.416095, "nauc_precision_at_1_diff1": 0.325795, "nauc_precision_at_3_max": -0.319386, "nauc_precision_at_3_std": -0.458536, "nauc_precision_at_3_diff1": 0.662898, "nauc_precision_at_5_max": -0.119213, "nauc_precision_at_5_std": 0.017905, "nauc_precision_at_5_diff1": 0.154584, "nauc_precision_at_10_max": 0.013753, "nauc_precision_at_10_std": 0.164482, "nauc_precision_at_10_diff1": -0.002373, "nauc_precision_at_20_max": 0.003509, "nauc_precision_at_20_std": 0.025194, "nauc_precision_at_20_diff1": 0.005135, "nauc_precision_at_100_max": 0.046527, "nauc_precision_at_100_std": 0.017103, "nauc_precision_at_100_diff1": 0.086337, "nauc_precision_at_1000_max": 0.079403, "nauc_precision_at_1000_std": -0.058657, "nauc_precision_at_1000_diff1": 0.071471, "nauc_mrr_at_1_max": -0.319386, "nauc_mrr_at_1_std": -0.416095, "nauc_mrr_at_1_diff1": 0.325795, "nauc_mrr_at_3_max": -0.319386, "nauc_mrr_at_3_std": -0.437316, "nauc_mrr_at_3_diff1": 0.494346, "nauc_mrr_at_5_max": -0.226998, "nauc_mrr_at_5_std": -0.222317, "nauc_mrr_at_5_diff1": 0.298637, "nauc_mrr_at_10_max": -0.133246, "nauc_mrr_at_10_std": -0.083083, "nauc_mrr_at_10_diff1": 0.17986, "nauc_mrr_at_20_max": -0.063112, "nauc_mrr_at_20_std": -0.065571, "nauc_mrr_at_20_diff1": 0.124739, "nauc_mrr_at_100_max": -0.036647, "nauc_mrr_at_100_std": -0.05721, "nauc_mrr_at_100_diff1": 0.123849, "nauc_mrr_at_1000_max": -0.041128, "nauc_mrr_at_1000_std": -0.066345, "nauc_mrr_at_1000_diff1": 0.123919, "hit_rate_at_1": 0.00287, "hit_rate_at_3": 0.00575, "hit_rate_at_5": 0.01149, "hit_rate_at_10": 0.02586, "hit_rate_at_20": 0.07471, "hit_rate_at_100": 0.18103, "hit_rate_at_1000": 0.57471, "main_score": 0.00375, "hf_subset": "french", "languages": [ "fra-Latn" ], "mteb_version": "2.13.1" }, { "ndcg_at_1": 0.0, "ndcg_at_3": 0.00088, "ndcg_at_5": 0.00157, "ndcg_at_10": 0.00428, "ndcg_at_20": 0.0071, "ndcg_at_100": 0.01682, "ndcg_at_1000": 0.05978, "map_at_1": 0.0, "map_at_3": 0.00048, "map_at_5": 0.00061, "map_at_10": 0.0017, "map_at_20": 0.00246, "map_at_100": 0.00343, "map_at_1000": 0.0048, "recall_at_1": 0.0, "recall_at_3": 0.00144, "recall_at_5": 0.00208, "recall_at_10": 0.00979, "recall_at_20": 0.01939, "recall_at_100": 0.05743, "recall_at_1000": 0.29408, "accuracy": 0.0, "precision_at_1": 0.0, "precision_at_3": 0.00096, "precision_at_5": 0.00172, "precision_at_10": 0.00287, "precision_at_20": 0.00316, "precision_at_100": 0.00236, "precision_at_1000": 0.00139, "mrr_at_1": 0.0, "mrr_at_3": 0.000958, "mrr_at_5": 0.002251, "mrr_at_10": 0.004988, "mrr_at_20": 0.006972, "mrr_at_100": 0.009463, "mrr_at_1000": 0.010976, "nauc_ndcg_at_1_max": NaN, "nauc_ndcg_at_1_std": NaN, "nauc_ndcg_at_1_diff1": NaN, "nauc_ndcg_at_3_max": -0.070865, "nauc_ndcg_at_3_std": -0.500977, "nauc_ndcg_at_3_diff1": 1.0, "nauc_ndcg_at_5_max": -0.192149, "nauc_ndcg_at_5_std": -0.361562, "nauc_ndcg_at_5_diff1": 0.593002, "nauc_ndcg_at_10_max": -0.209135, "nauc_ndcg_at_10_std": -0.184489, "nauc_ndcg_at_10_diff1": 0.007911, "nauc_ndcg_at_20_max": -0.098814, "nauc_ndcg_at_20_std": -0.042412, "nauc_ndcg_at_20_diff1": 0.084393, "nauc_ndcg_at_100_max": 0.042249, "nauc_ndcg_at_100_std": 0.020031, "nauc_ndcg_at_100_diff1": -0.001162, "nauc_ndcg_at_1000_max": -0.027169, "nauc_ndcg_at_1000_std": -0.016345, "nauc_ndcg_at_1000_diff1": -0.036908, "nauc_map_at_1_max": NaN, "nauc_map_at_1_std": NaN, "nauc_map_at_1_diff1": NaN, "nauc_map_at_3_max": -0.070865, "nauc_map_at_3_std": -0.500977, "nauc_map_at_3_diff1": 1.0, "nauc_map_at_5_max": -0.149644, "nauc_map_at_5_std": -0.422155, "nauc_map_at_5_diff1": 0.75742, "nauc_map_at_10_max": -0.217648, "nauc_map_at_10_std": -0.133521, "nauc_map_at_10_diff1": 0.064899, "nauc_map_at_20_max": -0.185792, "nauc_map_at_20_std": -0.019241, "nauc_map_at_20_diff1": 0.078106, "nauc_map_at_100_max": -0.071755, "nauc_map_at_100_std": 0.0055, "nauc_map_at_100_diff1": 0.023613, "nauc_map_at_1000_max": -0.061329, "nauc_map_at_1000_std": 0.009086, "nauc_map_at_1000_diff1": 0.015042, "nauc_recall_at_1_max": NaN, "nauc_recall_at_1_std": NaN, "nauc_recall_at_1_diff1": NaN, "nauc_recall_at_3_max": -0.070865, "nauc_recall_at_3_std": -0.500977, "nauc_recall_at_3_diff1": 1.0, "nauc_recall_at_5_max": -0.187519, "nauc_recall_at_5_std": -0.387752, "nauc_recall_at_5_diff1": 0.647278, "nauc_recall_at_10_max": -0.221827, "nauc_recall_at_10_std": -0.107757, "nauc_recall_at_10_diff1": -0.108781, "nauc_recall_at_20_max": -0.116716, "nauc_recall_at_20_std": 0.049774, "nauc_recall_at_20_diff1": 0.066498, "nauc_recall_at_100_max": 0.065751, "nauc_recall_at_100_std": 0.079277, "nauc_recall_at_100_diff1": -0.026276, "nauc_recall_at_1000_max": -0.012653, "nauc_recall_at_1000_std": 0.018772, "nauc_recall_at_1000_diff1": -0.060215, "nauc_precision_at_1_max": NaN, "nauc_precision_at_1_std": NaN, "nauc_precision_at_1_diff1": NaN, "nauc_precision_at_3_max": -0.070865, "nauc_precision_at_3_std": -0.500977, "nauc_precision_at_3_diff1": 1.0, "nauc_precision_at_5_max": -0.239366, "nauc_precision_at_5_std": -0.297076, "nauc_precision_at_5_diff1": 0.415601, "nauc_precision_at_10_max": -0.203859, "nauc_precision_at_10_std": -0.060885, "nauc_precision_at_10_diff1": -0.124661, "nauc_precision_at_20_max": -0.046257, "nauc_precision_at_20_std": 0.019392, "nauc_precision_at_20_diff1": -0.009, "nauc_precision_at_100_max": 0.043336, "nauc_precision_at_100_std": 0.006564, "nauc_precision_at_100_diff1": -0.013862, "nauc_precision_at_1000_max": -0.04153, "nauc_precision_at_1000_std": -0.07543, "nauc_precision_at_1000_diff1": -0.024089, "nauc_mrr_at_1_max": NaN, "nauc_mrr_at_1_std": NaN, "nauc_mrr_at_1_diff1": NaN, "nauc_mrr_at_3_max": -0.070865, "nauc_mrr_at_3_std": -0.500977, "nauc_mrr_at_3_diff1": 1.0, "nauc_mrr_at_5_max": -0.199929, "nauc_mrr_at_5_std": -0.333207, "nauc_mrr_at_5_diff1": 0.530858, "nauc_mrr_at_10_max": -0.20453, "nauc_mrr_at_10_std": -0.097228, "nauc_mrr_at_10_diff1": 0.043124, "nauc_mrr_at_20_max": -0.115202, "nauc_mrr_at_20_std": -0.05193, "nauc_mrr_at_20_diff1": 0.07767, "nauc_mrr_at_100_max": -0.037819, "nauc_mrr_at_100_std": -0.031861, "nauc_mrr_at_100_diff1": 0.040986, "nauc_mrr_at_1000_max": -0.045514, "nauc_mrr_at_1000_std": -0.036399, "nauc_mrr_at_1000_diff1": 0.038072, "hit_rate_at_1": 0.0, "hit_rate_at_3": 0.00287, "hit_rate_at_5": 0.00862, "hit_rate_at_10": 0.02874, "hit_rate_at_20": 0.05747, "hit_rate_at_100": 0.16667, "hit_rate_at_1000": 0.55172, "main_score": 0.00428, "hf_subset": "spanish", "languages": [ "spa-Latn" ], "mteb_version": "2.13.1" }, { "ndcg_at_1": 0.02011, "ndcg_at_3": 0.01619, "ndcg_at_5": 0.01867, "ndcg_at_10": 0.02107, "ndcg_at_20": 0.02559, "ndcg_at_100": 0.04364, "ndcg_at_1000": 0.09322, "map_at_1": 0.00613, "map_at_3": 0.00792, "map_at_5": 0.00958, "map_at_10": 0.01092, "map_at_20": 0.01207, "map_at_100": 0.01434, "map_at_1000": 0.01612, "recall_at_1": 0.00613, "recall_at_3": 0.01086, "recall_at_5": 0.01818, "recall_at_10": 0.02595, "recall_at_20": 0.0401, "recall_at_100": 0.11222, "recall_at_1000": 0.39109, "accuracy": 0.00613, "precision_at_1": 0.02299, "precision_at_3": 0.01533, "precision_at_5": 0.01494, "precision_at_10": 0.01178, "precision_at_20": 0.00848, "precision_at_100": 0.00445, "precision_at_1000": 0.00179, "mrr_at_1": 0.022989, "mrr_at_3": 0.032567, "mrr_at_5": 0.037596, "mrr_at_10": 0.041449, "mrr_at_20": 0.043647, "mrr_at_100": 0.047206, "mrr_at_1000": 0.048505, "nauc_ndcg_at_1_max": -0.168901, "nauc_ndcg_at_1_std": 0.178607, "nauc_ndcg_at_1_diff1": 0.29956, "nauc_ndcg_at_3_max": -0.119706, "nauc_ndcg_at_3_std": 0.084367, "nauc_ndcg_at_3_diff1": 0.285017, "nauc_ndcg_at_5_max": -0.11719, "nauc_ndcg_at_5_std": 0.102673, "nauc_ndcg_at_5_diff1": 0.178376, "nauc_ndcg_at_10_max": -0.068722, "nauc_ndcg_at_10_std": 0.186599, "nauc_ndcg_at_10_diff1": 0.217344, "nauc_ndcg_at_20_max": -0.048127, "nauc_ndcg_at_20_std": 0.149041, "nauc_ndcg_at_20_diff1": 0.195075, "nauc_ndcg_at_100_max": -0.089111, "nauc_ndcg_at_100_std": 0.076446, "nauc_ndcg_at_100_diff1": 0.119168, "nauc_ndcg_at_1000_max": -0.10418, "nauc_ndcg_at_1000_std": 0.076544, "nauc_ndcg_at_1000_diff1": 0.059757, "nauc_map_at_1_max": -0.075584, "nauc_map_at_1_std": 0.168568, "nauc_map_at_1_diff1": 0.231872, "nauc_map_at_3_max": -0.062576, "nauc_map_at_3_std": 0.150845, "nauc_map_at_3_diff1": 0.285877, "nauc_map_at_5_max": -0.053997, "nauc_map_at_5_std": 0.141344, "nauc_map_at_5_diff1": 0.203995, "nauc_map_at_10_max": -0.032624, "nauc_map_at_10_std": 0.180142, "nauc_map_at_10_diff1": 0.204062, "nauc_map_at_20_max": -0.017399, "nauc_map_at_20_std": 0.166155, "nauc_map_at_20_diff1": 0.19578, "nauc_map_at_100_max": -0.034742, "nauc_map_at_100_std": 0.14717, "nauc_map_at_100_diff1": 0.175724, "nauc_map_at_1000_max": -0.041989, "nauc_map_at_1000_std": 0.142007, "nauc_map_at_1000_diff1": 0.159016, "nauc_recall_at_1_max": -0.075584, "nauc_recall_at_1_std": 0.168568, "nauc_recall_at_1_diff1": 0.231872, "nauc_recall_at_3_max": -0.059648, "nauc_recall_at_3_std": 0.141558, "nauc_recall_at_3_diff1": 0.368172, "nauc_recall_at_5_max": -0.040191, "nauc_recall_at_5_std": 0.120114, "nauc_recall_at_5_diff1": 0.154906, "nauc_recall_at_10_max": -0.000701, "nauc_recall_at_10_std": 0.186107, "nauc_recall_at_10_diff1": 0.176036, "nauc_recall_at_20_max": 0.023088, "nauc_recall_at_20_std": 0.089701, "nauc_recall_at_20_diff1": 0.154852, "nauc_recall_at_100_max": -0.06556, "nauc_recall_at_100_std": 0.007086, "nauc_recall_at_100_diff1": 0.04285, "nauc_recall_at_1000_max": -0.055707, "nauc_recall_at_1000_std": 0.076809, "nauc_recall_at_1000_diff1": 0.00483, "nauc_precision_at_1_max": -0.119262, "nauc_precision_at_1_std": 0.171707, "nauc_precision_at_1_diff1": 0.278048, "nauc_precision_at_3_max": -0.086987, "nauc_precision_at_3_std": 0.030284, "nauc_precision_at_3_diff1": 0.230922, "nauc_precision_at_5_max": -0.091616, "nauc_precision_at_5_std": 0.049075, "nauc_precision_at_5_diff1": 0.05197, "nauc_precision_at_10_max": -0.012363, "nauc_precision_at_10_std": 0.160269, "nauc_precision_at_10_diff1": 0.087087, "nauc_precision_at_20_max": -0.009506, "nauc_precision_at_20_std": 0.121344, "nauc_precision_at_20_diff1": 0.060572, "nauc_precision_at_100_max": -0.127023, "nauc_precision_at_100_std": 0.004838, "nauc_precision_at_100_diff1": -0.011126, "nauc_precision_at_1000_max": -0.16108, "nauc_precision_at_1000_std": -0.057262, "nauc_precision_at_1000_diff1": -0.089936, "nauc_mrr_at_1_max": -0.119262, "nauc_mrr_at_1_std": 0.171707, "nauc_mrr_at_1_diff1": 0.278048, "nauc_mrr_at_3_max": -0.090401, "nauc_mrr_at_3_std": 0.089236, "nauc_mrr_at_3_diff1": 0.243239, "nauc_mrr_at_5_max": -0.089181, "nauc_mrr_at_5_std": 0.098399, "nauc_mrr_at_5_diff1": 0.183825, "nauc_mrr_at_10_max": -0.069744, "nauc_mrr_at_10_std": 0.113176, "nauc_mrr_at_10_diff1": 0.189092, "nauc_mrr_at_20_max": -0.076823, "nauc_mrr_at_20_std": 0.10034, "nauc_mrr_at_20_diff1": 0.182429, "nauc_mrr_at_100_max": -0.080765, "nauc_mrr_at_100_std": 0.09603, "nauc_mrr_at_100_diff1": 0.169075, "nauc_mrr_at_1000_max": -0.079776, "nauc_mrr_at_1000_std": 0.097128, "nauc_mrr_at_1000_diff1": 0.169554, "hit_rate_at_1": 0.02299, "hit_rate_at_3": 0.04598, "hit_rate_at_5": 0.06897, "hit_rate_at_10": 0.10057, "hit_rate_at_20": 0.13506, "hit_rate_at_100": 0.28736, "hit_rate_at_1000": 0.64943, "main_score": 0.02107, "hf_subset": "english", "languages": [ "eng-Latn" ], "mteb_version": "2.13.1" }, { "ndcg_at_1": 0.00144, "ndcg_at_3": 0.00244, "ndcg_at_5": 0.00236, "ndcg_at_10": 0.0041, "ndcg_at_20": 0.00637, "ndcg_at_100": 0.01953, "ndcg_at_1000": 0.05997, "map_at_1": 0.00016, "map_at_3": 0.00068, "map_at_5": 0.00072, "map_at_10": 0.00138, "map_at_20": 0.00191, "map_at_100": 0.0036, "map_at_1000": 0.00478, "recall_at_1": 0.00016, "recall_at_3": 0.00136, "recall_at_5": 0.00155, "recall_at_10": 0.00611, "recall_at_20": 0.0139, "recall_at_100": 0.06606, "recall_at_1000": 0.29151, "accuracy": 0.00016, "precision_at_1": 0.00287, "precision_at_3": 0.00287, "precision_at_5": 0.0023, "precision_at_10": 0.00287, "precision_at_20": 0.00273, "precision_at_100": 0.00276, "precision_at_1000": 0.00135, "mrr_at_1": 0.002874, "mrr_at_3": 0.005268, "mrr_at_5": 0.005843, "mrr_at_10": 0.007807, "mrr_at_20": 0.009559, "mrr_at_100": 0.012992, "mrr_at_1000": 0.014216, "nauc_ndcg_at_1_max": -0.070865, "nauc_ndcg_at_1_std": -0.319386, "nauc_ndcg_at_1_diff1": 1.0, "nauc_ndcg_at_3_max": 0.183386, "nauc_ndcg_at_3_std": 0.20873, "nauc_ndcg_at_3_diff1": 0.579098, "nauc_ndcg_at_5_max": 0.121846, "nauc_ndcg_at_5_std": 0.133994, "nauc_ndcg_at_5_diff1": 0.53328, "nauc_ndcg_at_10_max": -0.008092, "nauc_ndcg_at_10_std": 0.009684, "nauc_ndcg_at_10_diff1": 0.541519, "nauc_ndcg_at_20_max": 0.001427, "nauc_ndcg_at_20_std": -0.014735, "nauc_ndcg_at_20_diff1": 0.514495, "nauc_ndcg_at_100_max": -0.022151, "nauc_ndcg_at_100_std": -0.075784, "nauc_ndcg_at_100_diff1": 0.279902, "nauc_ndcg_at_1000_max": -0.065802, "nauc_ndcg_at_1000_std": -0.066419, "nauc_ndcg_at_1000_diff1": 0.122978, "nauc_map_at_1_max": -0.070865, "nauc_map_at_1_std": -0.319386, "nauc_map_at_1_diff1": 1.0, "nauc_map_at_3_max": 0.394861, "nauc_map_at_3_std": 0.437588, "nauc_map_at_3_diff1": 0.484432, "nauc_map_at_5_max": 0.346976, "nauc_map_at_5_std": 0.38742, "nauc_map_at_5_diff1": 0.475952, "nauc_map_at_10_max": 0.06613, "nauc_map_at_10_std": 0.103112, "nauc_map_at_10_diff1": 0.527611, "nauc_map_at_20_max": 0.018862, "nauc_map_at_20_std": 0.054338, "nauc_map_at_20_diff1": 0.510721, "nauc_map_at_100_max": -0.013497, "nauc_map_at_100_std": -0.022918, "nauc_map_at_100_diff1": 0.3773, "nauc_map_at_1000_max": -0.035118, "nauc_map_at_1000_std": -0.03149, "nauc_map_at_1000_diff1": 0.312744, "nauc_recall_at_1_max": -0.070865, "nauc_recall_at_1_std": -0.319386, "nauc_recall_at_1_diff1": 1.0, "nauc_recall_at_3_max": 0.344259, "nauc_recall_at_3_std": 0.466826, "nauc_recall_at_3_diff1": 0.405113, "nauc_recall_at_5_max": 0.239694, "nauc_recall_at_5_std": 0.347098, "nauc_recall_at_5_diff1": 0.395301, "nauc_recall_at_10_max": -0.066563, "nauc_recall_at_10_std": -0.064015, "nauc_recall_at_10_diff1": 0.485331, "nauc_recall_at_20_max": -0.07343, "nauc_recall_at_20_std": -0.049115, "nauc_recall_at_20_diff1": 0.431172, "nauc_recall_at_100_max": -0.030403, "nauc_recall_at_100_std": -0.072967, "nauc_recall_at_100_diff1": 0.226899, "nauc_recall_at_1000_max": -0.043982, "nauc_recall_at_1000_std": -0.01846, "nauc_recall_at_1000_diff1": 0.044297, "nauc_precision_at_1_max": -0.070865, "nauc_precision_at_1_std": -0.319386, "nauc_precision_at_1_diff1": 1.0, "nauc_precision_at_3_max": 0.142719, "nauc_precision_at_3_std": 0.20325, "nauc_precision_at_3_diff1": 0.55053, "nauc_precision_at_5_max": -0.018205, "nauc_precision_at_5_std": 0.027193, "nauc_precision_at_5_diff1": 0.494346, "nauc_precision_at_10_max": -0.031133, "nauc_precision_at_10_std": 0.016476, "nauc_precision_at_10_diff1": 0.464888, "nauc_precision_at_20_max": -0.051467, "nauc_precision_at_20_std": -0.043596, "nauc_precision_at_20_diff1": 0.383698, "nauc_precision_at_100_max": -0.074965, "nauc_precision_at_100_std": -0.086681, "nauc_precision_at_100_diff1": 0.180898, "nauc_precision_at_1000_max": -0.090889, "nauc_precision_at_1000_std": -0.109368, "nauc_precision_at_1000_diff1": 0.020329, "nauc_mrr_at_1_max": -0.070865, "nauc_mrr_at_1_std": -0.319386, "nauc_mrr_at_1_diff1": 1.0, "nauc_mrr_at_3_max": 0.142987, "nauc_mrr_at_3_std": 0.085632, "nauc_mrr_at_3_diff1": 0.693543, "nauc_mrr_at_5_max": 0.079646, "nauc_mrr_at_5_std": 0.027933, "nauc_mrr_at_5_diff1": 0.657371, "nauc_mrr_at_10_max": 0.05712, "nauc_mrr_at_10_std": 0.018566, "nauc_mrr_at_10_diff1": 0.629716, "nauc_mrr_at_20_max": 0.026698, "nauc_mrr_at_20_std": -0.009095, "nauc_mrr_at_20_diff1": 0.579485, "nauc_mrr_at_100_max": 0.011449, "nauc_mrr_at_100_std": -0.031411, "nauc_mrr_at_100_diff1": 0.437125, "nauc_mrr_at_1000_max": 0.007892, "nauc_mrr_at_1000_std": -0.034827, "nauc_mrr_at_1000_diff1": 0.427533, "hit_rate_at_1": 0.00287, "hit_rate_at_3": 0.00862, "hit_rate_at_5": 0.01149, "hit_rate_at_10": 0.02586, "hit_rate_at_20": 0.05172, "hit_rate_at_100": 0.18966, "hit_rate_at_1000": 0.57759, "main_score": 0.0041, "hf_subset": "german", "languages": [ "deu-Latn" ], "mteb_version": "2.13.1" }, { "ndcg_at_1": 0.00431, "ndcg_at_3": 0.00273, "ndcg_at_5": 0.00207, "ndcg_at_10": 0.00503, "ndcg_at_20": 0.00839, "ndcg_at_100": 0.02005, "ndcg_at_1000": 0.0616, "map_at_1": 0.00089, "map_at_3": 0.00094, "map_at_5": 0.00094, "map_at_10": 0.00211, "map_at_20": 0.00276, "map_at_100": 0.00419, "map_at_1000": 0.00547, "recall_at_1": 0.00089, "recall_at_3": 0.00105, "recall_at_5": 0.00105, "recall_at_10": 0.00999, "recall_at_20": 0.01922, "recall_at_100": 0.06778, "recall_at_1000": 0.30162, "accuracy": 0.00089, "precision_at_1": 0.00575, "precision_at_3": 0.00287, "precision_at_5": 0.00172, "precision_at_10": 0.00287, "precision_at_20": 0.00359, "precision_at_100": 0.00247, "precision_at_1000": 0.00133, "mrr_at_1": 0.005747, "mrr_at_3": 0.006705, "mrr_at_5": 0.006705, "mrr_at_10": 0.009358, "mrr_at_20": 0.011999, "mrr_at_100": 0.014584, "mrr_at_1000": 0.015981, "nauc_ndcg_at_1_max": -0.301019, "nauc_ndcg_at_1_std": -0.281751, "nauc_ndcg_at_1_diff1": 0.042311, "nauc_ndcg_at_3_max": -0.159729, "nauc_ndcg_at_3_std": -0.281361, "nauc_ndcg_at_3_diff1": 0.193463, "nauc_ndcg_at_5_max": -0.144089, "nauc_ndcg_at_5_std": -0.279619, "nauc_ndcg_at_5_diff1": 0.197797, "nauc_ndcg_at_10_max": -0.133055, "nauc_ndcg_at_10_std": -0.144305, "nauc_ndcg_at_10_diff1": 0.042681, "nauc_ndcg_at_20_max": -0.099628, "nauc_ndcg_at_20_std": -0.089945, "nauc_ndcg_at_20_diff1": -0.061757, "nauc_ndcg_at_100_max": -0.083286, "nauc_ndcg_at_100_std": -0.107813, "nauc_ndcg_at_100_diff1": 0.018713, "nauc_ndcg_at_1000_max": -0.08805, "nauc_ndcg_at_1000_std": -0.063163, "nauc_ndcg_at_1000_diff1": 0.016654, "nauc_map_at_1_max": -0.224111, "nauc_map_at_1_std": -0.267276, "nauc_map_at_1_diff1": 0.020547, "nauc_map_at_3_max": -0.193068, "nauc_map_at_3_std": -0.270217, "nauc_map_at_3_diff1": 0.075838, "nauc_map_at_5_max": -0.193068, "nauc_map_at_5_std": -0.270217, "nauc_map_at_5_diff1": 0.075838, "nauc_map_at_10_max": -0.172306, "nauc_map_at_10_std": -0.167373, "nauc_map_at_10_diff1": 0.005685, "nauc_map_at_20_max": -0.126611, "nauc_map_at_20_std": -0.108338, "nauc_map_at_20_diff1": -0.046971, "nauc_map_at_100_max": -0.103099, "nauc_map_at_100_std": -0.095017, "nauc_map_at_100_diff1": -0.028083, "nauc_map_at_1000_max": -0.10264, "nauc_map_at_1000_std": -0.082041, "nauc_map_at_1000_diff1": -0.024914, "nauc_recall_at_1_max": -0.224111, "nauc_recall_at_1_std": -0.267276, "nauc_recall_at_1_diff1": 0.020547, "nauc_recall_at_3_max": -0.14043, "nauc_recall_at_3_std": -0.275205, "nauc_recall_at_3_diff1": 0.169594, "nauc_recall_at_5_max": -0.14043, "nauc_recall_at_5_std": -0.275205, "nauc_recall_at_5_diff1": 0.169594, "nauc_recall_at_10_max": -0.114641, "nauc_recall_at_10_std": -0.146127, "nauc_recall_at_10_diff1": -0.015726, "nauc_recall_at_20_max": -0.06327, "nauc_recall_at_20_std": -0.048149, "nauc_recall_at_20_diff1": -0.117707, "nauc_recall_at_100_max": -0.078788, "nauc_recall_at_100_std": -0.108933, "nauc_recall_at_100_diff1": 0.033937, "nauc_recall_at_1000_max": -0.073478, "nauc_recall_at_1000_std": -0.020705, "nauc_recall_at_1000_diff1": -0.002805, "nauc_precision_at_1_max": -0.201039, "nauc_precision_at_1_std": -0.262933, "nauc_precision_at_1_diff1": 0.014017, "nauc_precision_at_3_max": -0.025428, "nauc_precision_at_3_std": -0.281751, "nauc_precision_at_3_diff1": 0.342678, "nauc_precision_at_5_max": -0.025428, "nauc_precision_at_5_std": -0.281751, "nauc_precision_at_5_diff1": 0.342678, "nauc_precision_at_10_max": -0.092406, "nauc_precision_at_10_std": -0.066939, "nauc_precision_at_10_diff1": 0.05324, "nauc_precision_at_20_max": -0.099782, "nauc_precision_at_20_std": -0.086686, "nauc_precision_at_20_diff1": -0.066138, "nauc_precision_at_100_max": -0.078368, "nauc_precision_at_100_std": -0.098722, "nauc_precision_at_100_diff1": 0.019577, "nauc_precision_at_1000_max": -0.029741, "nauc_precision_at_1000_std": -0.07202, "nauc_precision_at_1000_diff1": 0.030062, "nauc_mrr_at_1_max": -0.201039, "nauc_mrr_at_1_std": -0.262933, "nauc_mrr_at_1_diff1": 0.014017, "nauc_mrr_at_3_max": -0.125777, "nauc_mrr_at_3_std": -0.270998, "nauc_mrr_at_3_diff1": 0.154872, "nauc_mrr_at_5_max": -0.125777, "nauc_mrr_at_5_std": -0.270998, "nauc_mrr_at_5_diff1": 0.154872, "nauc_mrr_at_10_max": -0.137362, "nauc_mrr_at_10_std": -0.171465, "nauc_mrr_at_10_diff1": 0.086585, "nauc_mrr_at_20_max": -0.112976, "nauc_mrr_at_20_std": -0.145374, "nauc_mrr_at_20_diff1": 0.040925, "nauc_mrr_at_100_max": -0.100557, "nauc_mrr_at_100_std": -0.1471, "nauc_mrr_at_100_diff1": 0.048658, "nauc_mrr_at_1000_max": -0.09881, "nauc_mrr_at_1000_std": -0.140097, "nauc_mrr_at_1000_diff1": 0.049939, "hit_rate_at_1": 0.00575, "hit_rate_at_3": 0.00862, "hit_rate_at_5": 0.00862, "hit_rate_at_10": 0.02874, "hit_rate_at_20": 0.06609, "hit_rate_at_100": 0.18103, "hit_rate_at_1000": 0.57471, "main_score": 0.00503, "hf_subset": "italian", "languages": [ "ita-Latn" ], "mteb_version": "2.13.1" }, { "ndcg_at_1": 0.0, "ndcg_at_3": 0.00088, "ndcg_at_5": 0.00088, "ndcg_at_10": 0.0027, "ndcg_at_20": 0.00519, "ndcg_at_100": 0.01761, "ndcg_at_1000": 0.05888, "map_at_1": 0.0, "map_at_3": 0.00048, "map_at_5": 0.00048, "map_at_10": 0.00102, "map_at_20": 0.00154, "map_at_100": 0.00293, "map_at_1000": 0.0042, "recall_at_1": 0.0, "recall_at_3": 0.00144, "recall_at_5": 0.00144, "recall_at_10": 0.00543, "recall_at_20": 0.01371, "recall_at_100": 0.06365, "recall_at_1000": 0.29184, "accuracy": 0.0, "precision_at_1": 0.0, "precision_at_3": 0.00096, "precision_at_5": 0.00057, "precision_at_10": 0.00172, "precision_at_20": 0.0023, "precision_at_100": 0.00247, "precision_at_1000": 0.00136, "mrr_at_1": 0.0, "mrr_at_3": 0.000958, "mrr_at_5": 0.000958, "mrr_at_10": 0.002856, "mrr_at_20": 0.004716, "mrr_at_100": 0.007785, "mrr_at_1000": 0.009054, "nauc_ndcg_at_1_max": NaN, "nauc_ndcg_at_1_std": NaN, "nauc_ndcg_at_1_diff1": NaN, "nauc_ndcg_at_3_max": -0.319386, "nauc_ndcg_at_3_std": -0.576368, "nauc_ndcg_at_3_diff1": 1.0, "nauc_ndcg_at_5_max": -0.319386, "nauc_ndcg_at_5_std": -0.576368, "nauc_ndcg_at_5_diff1": 1.0, "nauc_ndcg_at_10_max": -0.329201, "nauc_ndcg_at_10_std": -0.210682, "nauc_ndcg_at_10_diff1": 0.459517, "nauc_ndcg_at_20_max": -0.223712, "nauc_ndcg_at_20_std": -0.11356, "nauc_ndcg_at_20_diff1": 0.313353, "nauc_ndcg_at_100_max": -0.101389, "nauc_ndcg_at_100_std": -0.084929, "nauc_ndcg_at_100_diff1": 0.117392, "nauc_ndcg_at_1000_max": -0.053431, "nauc_ndcg_at_1000_std": -0.044073, "nauc_ndcg_at_1000_diff1": 0.096477, "nauc_map_at_1_max": NaN, "nauc_map_at_1_std": NaN, "nauc_map_at_1_diff1": NaN, "nauc_map_at_3_max": -0.319386, "nauc_map_at_3_std": -0.576368, "nauc_map_at_3_diff1": 1.0, "nauc_map_at_5_max": -0.319386, "nauc_map_at_5_std": -0.576368, "nauc_map_at_5_diff1": 1.0, "nauc_map_at_10_max": -0.317964, "nauc_map_at_10_std": -0.275571, "nauc_map_at_10_diff1": 0.540695, "nauc_map_at_20_max": -0.217873, "nauc_map_at_20_std": -0.160375, "nauc_map_at_20_diff1": 0.408633, "nauc_map_at_100_max": -0.1409, "nauc_map_at_100_std": -0.110691, "nauc_map_at_100_diff1": 0.20583, "nauc_map_at_1000_max": -0.132604, "nauc_map_at_1000_std": -0.097549, "nauc_map_at_1000_diff1": 0.184174, "nauc_recall_at_1_max": NaN, "nauc_recall_at_1_std": NaN, "nauc_recall_at_1_diff1": NaN, "nauc_recall_at_3_max": -0.319386, "nauc_recall_at_3_std": -0.576368, "nauc_recall_at_3_diff1": 1.0, "nauc_recall_at_5_max": -0.319386, "nauc_recall_at_5_std": -0.576368, "nauc_recall_at_5_diff1": 1.0, "nauc_recall_at_10_max": -0.322067, "nauc_recall_at_10_std": -0.149503, "nauc_recall_at_10_diff1": 0.360184, "nauc_recall_at_20_max": -0.157695, "nauc_recall_at_20_std": -0.061231, "nauc_recall_at_20_diff1": 0.280583, "nauc_recall_at_100_max": -0.057315, "nauc_recall_at_100_std": -0.066995, "nauc_recall_at_100_diff1": 0.080781, "nauc_recall_at_1000_max": 0.012625, "nauc_recall_at_1000_std": 0.024122, "nauc_recall_at_1000_diff1": 0.035329, "nauc_precision_at_1_max": NaN, "nauc_precision_at_1_std": NaN, "nauc_precision_at_1_diff1": NaN, "nauc_precision_at_3_max": -0.319386, "nauc_precision_at_3_std": -0.576368, "nauc_precision_at_3_diff1": 1.0, "nauc_precision_at_5_max": -0.319386, "nauc_precision_at_5_std": -0.576368, "nauc_precision_at_5_diff1": 1.0, "nauc_precision_at_10_max": -0.348362, "nauc_precision_at_10_std": -0.095935, "nauc_precision_at_10_diff1": 0.367818, "nauc_precision_at_20_max": -0.199756, "nauc_precision_at_20_std": -0.009758, "nauc_precision_at_20_diff1": 0.132168, "nauc_precision_at_100_max": -0.139299, "nauc_precision_at_100_std": -0.07228, "nauc_precision_at_100_diff1": 0.055898, "nauc_precision_at_1000_max": -0.077312, "nauc_precision_at_1000_std": -0.074966, "nauc_precision_at_1000_diff1": 0.097714, "nauc_mrr_at_1_max": NaN, "nauc_mrr_at_1_std": NaN, "nauc_mrr_at_1_diff1": NaN, "nauc_mrr_at_3_max": -0.319386, "nauc_mrr_at_3_std": -0.576368, "nauc_mrr_at_3_diff1": 1.0, "nauc_mrr_at_5_max": -0.319386, "nauc_mrr_at_5_std": -0.576368, "nauc_mrr_at_5_diff1": 1.0, "nauc_mrr_at_10_max": -0.340046, "nauc_mrr_at_10_std": -0.196325, "nauc_mrr_at_10_diff1": 0.494725, "nauc_mrr_at_20_max": -0.247674, "nauc_mrr_at_20_std": -0.093747, "nauc_mrr_at_20_diff1": 0.28518, "nauc_mrr_at_100_max": -0.162912, "nauc_mrr_at_100_std": -0.098788, "nauc_mrr_at_100_diff1": 0.187184, "nauc_mrr_at_1000_max": -0.157375, "nauc_mrr_at_1000_std": -0.097316, "nauc_mrr_at_1000_diff1": 0.18394, "hit_rate_at_1": 0.0, "hit_rate_at_3": 0.00287, "hit_rate_at_5": 0.00287, "hit_rate_at_10": 0.01724, "hit_rate_at_20": 0.04598, "hit_rate_at_100": 0.18966, "hit_rate_at_1000": 0.55747, "main_score": 0.0027, "hf_subset": "portuguese", "languages": [ "por-Latn" ], "mteb_version": "2.13.1" } ] }, "evaluation_time": 2340.6369854547083, "kg_co2_emissions": null, "date": 1783392722.91961, "evaluation_phases": [ { "name": "Data loading", "start": 0.00011797994375228882, "end": 39.051421876996756, "split": "", "subset": "" }, { "name": "Dataset transform", "start": 39.05151053518057, "end": 39.05151665583253, "split": "", "subset": "" }, { "name": "Encoding queries and documents", "start": 39.06997933611274, "end": 430.0307632572949, "split": "test", "subset": "french" }, { "name": "Scoring", "start": 430.0309719182551, "end": 430.3493973314762, "split": "test", "subset": "french" }, { "name": "Encoding queries and documents", "start": 430.37943410873413, "end": 811.3008483052254, "split": "test", "subset": "spanish" }, { "name": "Scoring", "start": 811.3011515699327, "end": 811.623849093914, "split": "test", "subset": "spanish" }, { "name": "Encoding queries and documents", "start": 811.6622222289443, "end": 1196.311027456075, "split": "test", "subset": "english" }, { "name": "Scoring", "start": 1196.3113253451884, "end": 1196.6059698238969, "split": "test", "subset": "english" }, { "name": "Encoding queries and documents", "start": 1196.6529628001153, "end": 1576.3495617061853, "split": "test", "subset": "german" }, { "name": "Scoring", "start": 1576.3498477563262, "end": 1576.6631611399353, "split": "test", "subset": "german" }, { "name": "Encoding queries and documents", "start": 1576.697681490332, "end": 1957.7471501529217, "split": "test", "subset": "italian" }, { "name": "Scoring", "start": 1957.7473847344518, "end": 1958.0515358634293, "split": "test", "subset": "italian" }, { "name": "Encoding queries and documents", "start": 1958.2473953776062, "end": 2340.2909760810435, "split": "test", "subset": "portuguese" }, { "name": "Scoring", "start": 2340.291201785207, "end": 2340.610819991678, "split": "test", "subset": "portuguese" } ] }