{ "dataset_revision": "fa78cb14152b3dde8c5defdc4e3ddf50de69dfeb", "task_name": "Vidore3FinanceEnRetrieval", "mteb_version": "2.20.12", "scores": { "test": [ { "ndcg_at_1": 0.66828, "ndcg_at_3": 0.66792, "ndcg_at_5": 0.68, "ndcg_at_10": 0.70101, "ndcg_at_20": 0.7293, "ndcg_at_100": 0.7708, "ndcg_at_1000": 0.78481, "map_at_1": 0.33599, "map_at_3": 0.48186, "map_at_5": 0.54382, "map_at_10": 0.59481, "map_at_20": 0.62131, "map_at_100": 0.63863, "map_at_1000": 0.64104, "recall_at_1": 0.33599, "recall_at_3": 0.53311, "recall_at_5": 0.6366, "recall_at_10": 0.73754, "recall_at_20": 0.81808, "recall_at_100": 0.94463, "recall_at_1000": 0.99858, "accuracy": 0.33599, "precision_at_1": 0.70227, "precision_at_3": 0.48004, "precision_at_5": 0.38317, "precision_at_10": 0.25502, "precision_at_20": 0.15793, "precision_at_100": 0.04162, "precision_at_1000": 0.00472, "mrr_at_1": 0.702265, "mrr_at_3": 0.782632, "mrr_at_5": 0.792826, "mrr_at_10": 0.799143, "mrr_at_20": 0.800105, "mrr_at_100": 0.800803, "mrr_at_1000": 0.800815, "nauc_ndcg_at_1_max": 0.097762, "nauc_ndcg_at_1_std": -0.026954, "nauc_ndcg_at_1_diff1": 0.521207, "nauc_ndcg_at_3_max": 0.0865, "nauc_ndcg_at_3_std": -0.025211, "nauc_ndcg_at_3_diff1": 0.433226, "nauc_ndcg_at_5_max": 0.129481, "nauc_ndcg_at_5_std": -0.002465, "nauc_ndcg_at_5_diff1": 0.42821, "nauc_ndcg_at_10_max": 0.170867, "nauc_ndcg_at_10_std": 0.036528, "nauc_ndcg_at_10_diff1": 0.40568, "nauc_ndcg_at_20_max": 0.157288, "nauc_ndcg_at_20_std": 0.037247, "nauc_ndcg_at_20_diff1": 0.404419, "nauc_ndcg_at_100_max": 0.164368, "nauc_ndcg_at_100_std": 0.046917, "nauc_ndcg_at_100_diff1": 0.425842, "nauc_ndcg_at_1000_max": 0.146233, "nauc_ndcg_at_1000_std": 0.033388, "nauc_ndcg_at_1000_diff1": 0.43236, "nauc_map_at_1_max": 0.117311, "nauc_map_at_1_std": -0.048597, "nauc_map_at_1_diff1": 0.554754, "nauc_map_at_3_max": 0.102736, "nauc_map_at_3_std": -0.04791, "nauc_map_at_3_diff1": 0.457784, "nauc_map_at_5_max": 0.104708, "nauc_map_at_5_std": -0.04371, "nauc_map_at_5_diff1": 0.418392, "nauc_map_at_10_max": 0.137954, "nauc_map_at_10_std": 0.008317, "nauc_map_at_10_diff1": 0.378248, "nauc_map_at_20_max": 0.134554, "nauc_map_at_20_std": 0.014933, "nauc_map_at_20_diff1": 0.373643, "nauc_map_at_100_max": 0.140754, "nauc_map_at_100_std": 0.022931, "nauc_map_at_100_diff1": 0.373871, "nauc_map_at_1000_max": 0.139111, "nauc_map_at_1000_std": 0.021835, "nauc_map_at_1000_diff1": 0.374772, "nauc_recall_at_1_max": 0.117311, "nauc_recall_at_1_std": -0.048597, "nauc_recall_at_1_diff1": 0.554754, "nauc_recall_at_3_max": 0.110559, "nauc_recall_at_3_std": -0.03187, "nauc_recall_at_3_diff1": 0.396533, "nauc_recall_at_5_max": 0.122406, "nauc_recall_at_5_std": -0.036303, "nauc_recall_at_5_diff1": 0.327874, "nauc_recall_at_10_max": 0.209844, "nauc_recall_at_10_std": 0.056057, "nauc_recall_at_10_diff1": 0.250543, "nauc_recall_at_20_max": 0.185208, "nauc_recall_at_20_std": 0.072934, "nauc_recall_at_20_diff1": 0.227483, "nauc_recall_at_100_max": 0.340591, "nauc_recall_at_100_std": 0.235887, "nauc_recall_at_100_diff1": 0.295534, "nauc_recall_at_1000_max": 0.869234, "nauc_recall_at_1000_std": 0.925277, "nauc_recall_at_1000_diff1": 0.808752, "nauc_precision_at_1_max": 0.076028, "nauc_precision_at_1_std": -0.042927, "nauc_precision_at_1_diff1": 0.491406, "nauc_precision_at_3_max": -0.042016, "nauc_precision_at_3_std": 0.013329, "nauc_precision_at_3_diff1": -0.138521, "nauc_precision_at_5_max": 0.003917, "nauc_precision_at_5_std": 0.079993, "nauc_precision_at_5_diff1": -0.213645, "nauc_precision_at_10_max": 0.050201, "nauc_precision_at_10_std": 0.156384, "nauc_precision_at_10_diff1": -0.28985, "nauc_precision_at_20_max": -0.002262, "nauc_precision_at_20_std": 0.122368, "nauc_precision_at_20_diff1": -0.299898, "nauc_precision_at_100_max": -0.040759, "nauc_precision_at_100_std": 0.087669, "nauc_precision_at_100_diff1": -0.296172, "nauc_precision_at_1000_max": -0.093016, "nauc_precision_at_1000_std": 0.036379, "nauc_precision_at_1000_diff1": -0.281248, "nauc_mrr_at_1_max": 0.076028, "nauc_mrr_at_1_std": -0.042927, "nauc_mrr_at_1_diff1": 0.491406, "nauc_mrr_at_3_max": 0.082044, "nauc_mrr_at_3_std": -0.012933, "nauc_mrr_at_3_diff1": 0.453035, "nauc_mrr_at_5_max": 0.103527, "nauc_mrr_at_5_std": -0.002902, "nauc_mrr_at_5_diff1": 0.453985, "nauc_mrr_at_10_max": 0.105084, "nauc_mrr_at_10_std": 0.002828, "nauc_mrr_at_10_diff1": 0.456731, "nauc_mrr_at_20_max": 0.103859, "nauc_mrr_at_20_std": 0.000409, "nauc_mrr_at_20_diff1": 0.4602, "nauc_mrr_at_100_max": 0.101478, "nauc_mrr_at_100_std": -0.001563, "nauc_mrr_at_100_diff1": 0.460065, "nauc_mrr_at_1000_max": 0.101433, "nauc_mrr_at_1000_std": -0.001626, "nauc_mrr_at_1000_diff1": 0.460046, "hit_rate_at_1": 0.70227, "hit_rate_at_3": 0.87702, "hit_rate_at_5": 0.91909, "hit_rate_at_10": 0.9644, "hit_rate_at_20": 0.97735, "hit_rate_at_100": 0.99676, "hit_rate_at_1000": 1.0, "main_score": 0.70101, "hf_subset": "french", "languages": [ "fra-Latn" ], "mteb_version": "2.20.12" }, { "ndcg_at_1": 0.68447, "ndcg_at_3": 0.67875, "ndcg_at_5": 0.68809, "ndcg_at_10": 0.71222, "ndcg_at_20": 0.73814, "ndcg_at_100": 0.77858, "ndcg_at_1000": 0.79188, "map_at_1": 0.33999, "map_at_3": 0.49012, "map_at_5": 0.54921, "map_at_10": 0.60522, "map_at_20": 0.63088, "map_at_100": 0.64826, "map_at_1000": 0.65059, "recall_at_1": 0.33999, "recall_at_3": 0.53411, "recall_at_5": 0.63309, "recall_at_10": 0.74854, "recall_at_20": 0.82406, "recall_at_100": 0.94596, "recall_at_1000": 0.99919, "accuracy": 0.33999, "precision_at_1": 0.72168, "precision_at_3": 0.49191, "precision_at_5": 0.38964, "precision_at_10": 0.26052, "precision_at_20": 0.16068, "precision_at_100": 0.04217, "precision_at_1000": 0.00472, "mrr_at_1": 0.721683, "mrr_at_3": 0.799892, "mrr_at_5": 0.808792, "mrr_at_10": 0.813141, "mrr_at_20": 0.81472, "mrr_at_100": 0.81508, "mrr_at_1000": 0.815136, "nauc_ndcg_at_1_max": 0.035884, "nauc_ndcg_at_1_std": -0.041503, "nauc_ndcg_at_1_diff1": 0.486563, "nauc_ndcg_at_3_max": 0.065444, "nauc_ndcg_at_3_std": -0.056711, "nauc_ndcg_at_3_diff1": 0.406679, "nauc_ndcg_at_5_max": 0.098521, "nauc_ndcg_at_5_std": -0.05344, "nauc_ndcg_at_5_diff1": 0.383789, "nauc_ndcg_at_10_max": 0.141397, "nauc_ndcg_at_10_std": -0.009772, "nauc_ndcg_at_10_diff1": 0.401943, "nauc_ndcg_at_20_max": 0.13023, "nauc_ndcg_at_20_std": -0.0113, "nauc_ndcg_at_20_diff1": 0.404984, "nauc_ndcg_at_100_max": 0.114822, "nauc_ndcg_at_100_std": -0.016448, "nauc_ndcg_at_100_diff1": 0.410392, "nauc_ndcg_at_1000_max": 0.098416, "nauc_ndcg_at_1000_std": -0.022159, "nauc_ndcg_at_1000_diff1": 0.411929, "nauc_map_at_1_max": 0.108503, "nauc_map_at_1_std": -0.05192, "nauc_map_at_1_diff1": 0.546424, "nauc_map_at_3_max": 0.097674, "nauc_map_at_3_std": -0.06856, "nauc_map_at_3_diff1": 0.445959, "nauc_map_at_5_max": 0.099664, "nauc_map_at_5_std": -0.071727, "nauc_map_at_5_diff1": 0.390726, "nauc_map_at_10_max": 0.120312, "nauc_map_at_10_std": -0.034049, "nauc_map_at_10_diff1": 0.367963, "nauc_map_at_20_max": 0.120895, "nauc_map_at_20_std": -0.024425, "nauc_map_at_20_diff1": 0.362727, "nauc_map_at_100_max": 0.120281, "nauc_map_at_100_std": -0.020807, "nauc_map_at_100_diff1": 0.356814, "nauc_map_at_1000_max": 0.118581, "nauc_map_at_1000_std": -0.021788, "nauc_map_at_1000_diff1": 0.357369, "nauc_recall_at_1_max": 0.108503, "nauc_recall_at_1_std": -0.05192, "nauc_recall_at_1_diff1": 0.546424, "nauc_recall_at_3_max": 0.110878, "nauc_recall_at_3_std": -0.066369, "nauc_recall_at_3_diff1": 0.405858, "nauc_recall_at_5_max": 0.128928, "nauc_recall_at_5_std": -0.074108, "nauc_recall_at_5_diff1": 0.31519, "nauc_recall_at_10_max": 0.199373, "nauc_recall_at_10_std": -0.000522, "nauc_recall_at_10_diff1": 0.307121, "nauc_recall_at_20_max": 0.199298, "nauc_recall_at_20_std": 0.012148, "nauc_recall_at_20_diff1": 0.293737, "nauc_recall_at_100_max": 0.258763, "nauc_recall_at_100_std": 0.068794, "nauc_recall_at_100_diff1": 0.366152, "nauc_recall_at_1000_max": 0.869234, "nauc_recall_at_1000_std": 1.0, "nauc_recall_at_1000_diff1": 0.122718, "nauc_precision_at_1_max": -0.003861, "nauc_precision_at_1_std": -0.048925, "nauc_precision_at_1_diff1": 0.447487, "nauc_precision_at_3_max": -0.045103, "nauc_precision_at_3_std": -0.003818, "nauc_precision_at_3_diff1": -0.178115, "nauc_precision_at_5_max": -0.019408, "nauc_precision_at_5_std": 0.042533, "nauc_precision_at_5_diff1": -0.296353, "nauc_precision_at_10_max": 0.020831, "nauc_precision_at_10_std": 0.125745, "nauc_precision_at_10_diff1": -0.309103, "nauc_precision_at_20_max": -0.020747, "nauc_precision_at_20_std": 0.108325, "nauc_precision_at_20_diff1": -0.308341, "nauc_precision_at_100_max": -0.068784, "nauc_precision_at_100_std": 0.066533, "nauc_precision_at_100_diff1": -0.30616, "nauc_precision_at_1000_max": -0.109062, "nauc_precision_at_1000_std": 0.038508, "nauc_precision_at_1000_diff1": -0.295634, "nauc_mrr_at_1_max": -0.003861, "nauc_mrr_at_1_std": -0.048925, "nauc_mrr_at_1_diff1": 0.447487, "nauc_mrr_at_3_max": 0.005158, "nauc_mrr_at_3_std": -0.045321, "nauc_mrr_at_3_diff1": 0.418834, "nauc_mrr_at_5_max": 0.016539, "nauc_mrr_at_5_std": -0.031375, "nauc_mrr_at_5_diff1": 0.426423, "nauc_mrr_at_10_max": 0.017385, "nauc_mrr_at_10_std": -0.032496, "nauc_mrr_at_10_diff1": 0.426767, "nauc_mrr_at_20_max": 0.013299, "nauc_mrr_at_20_std": -0.034932, "nauc_mrr_at_20_diff1": 0.42884, "nauc_mrr_at_100_max": 0.011417, "nauc_mrr_at_100_std": -0.036559, "nauc_mrr_at_100_diff1": 0.428478, "nauc_mrr_at_1000_max": 0.01137, "nauc_mrr_at_1000_std": -0.036562, "nauc_mrr_at_1000_diff1": 0.428285, "hit_rate_at_1": 0.72168, "hit_rate_at_3": 0.8932, "hit_rate_at_5": 0.93204, "hit_rate_at_10": 0.9644, "hit_rate_at_20": 0.98382, "hit_rate_at_100": 0.99353, "hit_rate_at_1000": 1.0, "main_score": 0.71222, "hf_subset": "spanish", "languages": [ "spa-Latn" ], "mteb_version": "2.20.12" }, { "ndcg_at_1": 0.72816, "ndcg_at_3": 0.7072, "ndcg_at_5": 0.72074, "ndcg_at_10": 0.7366, "ndcg_at_20": 0.76246, "ndcg_at_100": 0.80344, "ndcg_at_1000": 0.81386, "map_at_1": 0.35492, "map_at_3": 0.51457, "map_at_5": 0.58094, "map_at_10": 0.63365, "map_at_20": 0.65874, "map_at_100": 0.67746, "map_at_1000": 0.6794, "recall_at_1": 0.35492, "recall_at_3": 0.55764, "recall_at_5": 0.66408, "recall_at_10": 0.76267, "recall_at_20": 0.83902, "recall_at_100": 0.96187, "recall_at_1000": 0.9998, "accuracy": 0.35492, "precision_at_1": 0.76375, "precision_at_3": 0.50917, "precision_at_5": 0.411, "precision_at_10": 0.26828, "precision_at_20": 0.1623, "precision_at_100": 0.04291, "precision_at_1000": 0.00472, "mrr_at_1": 0.763754, "mrr_at_3": 0.829558, "mrr_at_5": 0.840076, "mrr_at_10": 0.84371, "mrr_at_20": 0.844682, "mrr_at_100": 0.844958, "mrr_at_1000": 0.844958, "nauc_ndcg_at_1_max": 0.063146, "nauc_ndcg_at_1_std": -0.070773, "nauc_ndcg_at_1_diff1": 0.436223, "nauc_ndcg_at_3_max": 0.079472, "nauc_ndcg_at_3_std": -0.054363, "nauc_ndcg_at_3_diff1": 0.3797, "nauc_ndcg_at_5_max": 0.14717, "nauc_ndcg_at_5_std": -0.00092, "nauc_ndcg_at_5_diff1": 0.383622, "nauc_ndcg_at_10_max": 0.18843, "nauc_ndcg_at_10_std": 0.030129, "nauc_ndcg_at_10_diff1": 0.383806, "nauc_ndcg_at_20_max": 0.182275, "nauc_ndcg_at_20_std": 0.03037, "nauc_ndcg_at_20_diff1": 0.372167, "nauc_ndcg_at_100_max": 0.16578, "nauc_ndcg_at_100_std": 0.026003, "nauc_ndcg_at_100_diff1": 0.373279, "nauc_ndcg_at_1000_max": 0.147856, "nauc_ndcg_at_1000_std": 0.01037, "nauc_ndcg_at_1000_diff1": 0.381541, "nauc_map_at_1_max": 0.161597, "nauc_map_at_1_std": -0.020953, "nauc_map_at_1_diff1": 0.555743, "nauc_map_at_3_max": 0.110832, "nauc_map_at_3_std": -0.053258, "nauc_map_at_3_diff1": 0.432261, "nauc_map_at_5_max": 0.13893, "nauc_map_at_5_std": -0.026167, "nauc_map_at_5_diff1": 0.38104, "nauc_map_at_10_max": 0.163948, "nauc_map_at_10_std": 0.018589, "nauc_map_at_10_diff1": 0.345836, "nauc_map_at_20_max": 0.163886, "nauc_map_at_20_std": 0.030158, "nauc_map_at_20_diff1": 0.330497, "nauc_map_at_100_max": 0.163265, "nauc_map_at_100_std": 0.034507, "nauc_map_at_100_diff1": 0.324051, "nauc_map_at_1000_max": 0.160807, "nauc_map_at_1000_std": 0.032643, "nauc_map_at_1000_diff1": 0.324453, "nauc_recall_at_1_max": 0.161597, "nauc_recall_at_1_std": -0.020953, "nauc_recall_at_1_diff1": 0.555743, "nauc_recall_at_3_max": 0.103562, "nauc_recall_at_3_std": -0.047982, "nauc_recall_at_3_diff1": 0.397238, "nauc_recall_at_5_max": 0.187176, "nauc_recall_at_5_std": 0.014773, "nauc_recall_at_5_diff1": 0.330799, "nauc_recall_at_10_max": 0.255633, "nauc_recall_at_10_std": 0.07217, "nauc_recall_at_10_diff1": 0.285625, "nauc_recall_at_20_max": 0.246078, "nauc_recall_at_20_std": 0.079943, "nauc_recall_at_20_diff1": 0.241634, "nauc_recall_at_100_max": 0.300081, "nauc_recall_at_100_std": 0.20751, "nauc_recall_at_100_diff1": 0.191291, "nauc_recall_at_1000_max": 0.869234, "nauc_recall_at_1000_std": 1.0, "nauc_recall_at_1000_diff1": -0.170451, "nauc_precision_at_1_max": 0.024824, "nauc_precision_at_1_std": -0.08275, "nauc_precision_at_1_diff1": 0.410312, "nauc_precision_at_3_max": -0.095142, "nauc_precision_at_3_std": -0.020023, "nauc_precision_at_3_diff1": -0.246286, "nauc_precision_at_5_max": -0.036096, "nauc_precision_at_5_std": 0.068161, "nauc_precision_at_5_diff1": -0.350894, "nauc_precision_at_10_max": -0.008864, "nauc_precision_at_10_std": 0.129633, "nauc_precision_at_10_diff1": -0.379316, "nauc_precision_at_20_max": -0.0459, "nauc_precision_at_20_std": 0.112357, "nauc_precision_at_20_diff1": -0.390698, "nauc_precision_at_100_max": -0.105294, "nauc_precision_at_100_std": 0.058589, "nauc_precision_at_100_diff1": -0.367083, "nauc_precision_at_1000_max": -0.142093, "nauc_precision_at_1000_std": 0.016532, "nauc_precision_at_1000_diff1": -0.327797, "nauc_mrr_at_1_max": 0.024824, "nauc_mrr_at_1_std": -0.08275, "nauc_mrr_at_1_diff1": 0.410312, "nauc_mrr_at_3_max": -0.009121, "nauc_mrr_at_3_std": -0.090856, "nauc_mrr_at_3_diff1": 0.38014, "nauc_mrr_at_5_max": 0.033364, "nauc_mrr_at_5_std": -0.0601, "nauc_mrr_at_5_diff1": 0.387348, "nauc_mrr_at_10_max": 0.028126, "nauc_mrr_at_10_std": -0.070948, "nauc_mrr_at_10_diff1": 0.389709, "nauc_mrr_at_20_max": 0.022365, "nauc_mrr_at_20_std": -0.075018, "nauc_mrr_at_20_diff1": 0.388744, "nauc_mrr_at_100_max": 0.020602, "nauc_mrr_at_100_std": -0.076829, "nauc_mrr_at_100_diff1": 0.387828, "nauc_mrr_at_1000_max": 0.020602, "nauc_mrr_at_1000_std": -0.076829, "nauc_mrr_at_1000_diff1": 0.387828, "hit_rate_at_1": 0.76375, "hit_rate_at_3": 0.90939, "hit_rate_at_5": 0.95469, "hit_rate_at_10": 0.98058, "hit_rate_at_20": 0.99353, "hit_rate_at_100": 1.0, "hit_rate_at_1000": 1.0, "main_score": 0.7366, "hf_subset": "english", "languages": [ "eng-Latn" ], "mteb_version": "2.20.12" }, { "ndcg_at_1": 0.64078, "ndcg_at_3": 0.66093, "ndcg_at_5": 0.67438, "ndcg_at_10": 0.6955, "ndcg_at_20": 0.72294, "ndcg_at_100": 0.76492, "ndcg_at_1000": 0.77785, "map_at_1": 0.31706, "map_at_3": 0.47616, "map_at_5": 0.53863, "map_at_10": 0.59089, "map_at_20": 0.61488, "map_at_100": 0.63351, "map_at_1000": 0.63577, "recall_at_1": 0.31706, "recall_at_3": 0.53824, "recall_at_5": 0.63988, "recall_at_10": 0.74306, "recall_at_20": 0.82291, "recall_at_100": 0.95018, "recall_at_1000": 0.99899, "accuracy": 0.31706, "precision_at_1": 0.67961, "precision_at_3": 0.48544, "precision_at_5": 0.39029, "precision_at_10": 0.25793, "precision_at_20": 0.15841, "precision_at_100": 0.04191, "precision_at_1000": 0.00471, "mrr_at_1": 0.679612, "mrr_at_3": 0.777238, "mrr_at_5": 0.785976, "mrr_at_10": 0.78935, "mrr_at_20": 0.791252, "mrr_at_100": 0.791575, "mrr_at_1000": 0.791575, "nauc_ndcg_at_1_max": 0.084933, "nauc_ndcg_at_1_std": 0.019319, "nauc_ndcg_at_1_diff1": 0.508091, "nauc_ndcg_at_3_max": 0.044246, "nauc_ndcg_at_3_std": -0.051062, "nauc_ndcg_at_3_diff1": 0.411485, "nauc_ndcg_at_5_max": 0.115365, "nauc_ndcg_at_5_std": 0.004399, "nauc_ndcg_at_5_diff1": 0.434662, "nauc_ndcg_at_10_max": 0.130053, "nauc_ndcg_at_10_std": 0.027055, "nauc_ndcg_at_10_diff1": 0.40985, "nauc_ndcg_at_20_max": 0.13119, "nauc_ndcg_at_20_std": 0.038965, "nauc_ndcg_at_20_diff1": 0.412204, "nauc_ndcg_at_100_max": 0.137233, "nauc_ndcg_at_100_std": 0.043954, "nauc_ndcg_at_100_diff1": 0.428585, "nauc_ndcg_at_1000_max": 0.117662, "nauc_ndcg_at_1000_std": 0.026002, "nauc_ndcg_at_1000_diff1": 0.430759, "nauc_map_at_1_max": 0.107721, "nauc_map_at_1_std": -0.017968, "nauc_map_at_1_diff1": 0.563631, "nauc_map_at_3_max": 0.066889, "nauc_map_at_3_std": -0.063271, "nauc_map_at_3_diff1": 0.439436, "nauc_map_at_5_max": 0.08883, "nauc_map_at_5_std": -0.037397, "nauc_map_at_5_diff1": 0.412014, "nauc_map_at_10_max": 0.098331, "nauc_map_at_10_std": -0.00658, "nauc_map_at_10_diff1": 0.370973, "nauc_map_at_20_max": 0.106406, "nauc_map_at_20_std": 0.013983, "nauc_map_at_20_diff1": 0.364058, "nauc_map_at_100_max": 0.113525, "nauc_map_at_100_std": 0.020621, "nauc_map_at_100_diff1": 0.36612, "nauc_map_at_1000_max": 0.111575, "nauc_map_at_1000_std": 0.018661, "nauc_map_at_1000_diff1": 0.366249, "nauc_recall_at_1_max": 0.107721, "nauc_recall_at_1_std": -0.017968, "nauc_recall_at_1_diff1": 0.563631, "nauc_recall_at_3_max": 0.0558, "nauc_recall_at_3_std": -0.077954, "nauc_recall_at_3_diff1": 0.367419, "nauc_recall_at_5_max": 0.111676, "nauc_recall_at_5_std": -0.031424, "nauc_recall_at_5_diff1": 0.33868, "nauc_recall_at_10_max": 0.149572, "nauc_recall_at_10_std": 0.024777, "nauc_recall_at_10_diff1": 0.264748, "nauc_recall_at_20_max": 0.184211, "nauc_recall_at_20_std": 0.092737, "nauc_recall_at_20_diff1": 0.255802, "nauc_recall_at_100_max": 0.361183, "nauc_recall_at_100_std": 0.260359, "nauc_recall_at_100_diff1": 0.334674, "nauc_recall_at_1000_max": 1.0, "nauc_recall_at_1000_std": 1.0, "nauc_recall_at_1000_diff1": 0.357253, "nauc_precision_at_1_max": 0.051849, "nauc_precision_at_1_std": 0.029701, "nauc_precision_at_1_diff1": 0.462192, "nauc_precision_at_3_max": -0.079425, "nauc_precision_at_3_std": -0.034225, "nauc_precision_at_3_diff1": -0.170084, "nauc_precision_at_5_max": 0.011801, "nauc_precision_at_5_std": 0.091205, "nauc_precision_at_5_diff1": -0.210131, "nauc_precision_at_10_max": 0.027505, "nauc_precision_at_10_std": 0.143092, "nauc_precision_at_10_diff1": -0.279569, "nauc_precision_at_20_max": 0.014418, "nauc_precision_at_20_std": 0.14385, "nauc_precision_at_20_diff1": -0.307061, "nauc_precision_at_100_max": -0.008567, "nauc_precision_at_100_std": 0.105433, "nauc_precision_at_100_diff1": -0.292747, "nauc_precision_at_1000_max": -0.066324, "nauc_precision_at_1000_std": 0.038848, "nauc_precision_at_1000_diff1": -0.293931, "nauc_mrr_at_1_max": 0.051849, "nauc_mrr_at_1_std": 0.029701, "nauc_mrr_at_1_diff1": 0.462192, "nauc_mrr_at_3_max": 0.026235, "nauc_mrr_at_3_std": -0.02472, "nauc_mrr_at_3_diff1": 0.426604, "nauc_mrr_at_5_max": 0.054643, "nauc_mrr_at_5_std": 0.009479, "nauc_mrr_at_5_diff1": 0.437214, "nauc_mrr_at_10_max": 0.059362, "nauc_mrr_at_10_std": 0.01361, "nauc_mrr_at_10_diff1": 0.436215, "nauc_mrr_at_20_max": 0.054669, "nauc_mrr_at_20_std": 0.009014, "nauc_mrr_at_20_diff1": 0.436909, "nauc_mrr_at_100_max": 0.053443, "nauc_mrr_at_100_std": 0.008471, "nauc_mrr_at_100_diff1": 0.436569, "nauc_mrr_at_1000_max": 0.053443, "nauc_mrr_at_1000_std": 0.008471, "nauc_mrr_at_1000_diff1": 0.436569, "hit_rate_at_1": 0.67961, "hit_rate_at_3": 0.89968, "hit_rate_at_5": 0.93528, "hit_rate_at_10": 0.96117, "hit_rate_at_20": 0.98706, "hit_rate_at_100": 1.0, "hit_rate_at_1000": 1.0, "main_score": 0.6955, "hf_subset": "german", "languages": [ "deu-Latn" ], "mteb_version": "2.20.12" }, { "ndcg_at_1": 0.69579, "ndcg_at_3": 0.68918, "ndcg_at_5": 0.69376, "ndcg_at_10": 0.71617, "ndcg_at_20": 0.74031, "ndcg_at_100": 0.78201, "ndcg_at_1000": 0.79367, "map_at_1": 0.33387, "map_at_3": 0.49358, "map_at_5": 0.55184, "map_at_10": 0.60586, "map_at_20": 0.62978, "map_at_100": 0.64769, "map_at_1000": 0.64976, "recall_at_1": 0.33387, "recall_at_3": 0.55134, "recall_at_5": 0.64531, "recall_at_10": 0.75283, "recall_at_20": 0.83016, "recall_at_100": 0.95375, "recall_at_1000": 0.99858, "accuracy": 0.33387, "precision_at_1": 0.72816, "precision_at_3": 0.50485, "precision_at_5": 0.39547, "precision_at_10": 0.2644, "precision_at_20": 0.15987, "precision_at_100": 0.04236, "precision_at_1000": 0.00471, "mrr_at_1": 0.728155, "mrr_at_3": 0.807983, "mrr_at_5": 0.816235, "mrr_at_10": 0.820745, "mrr_at_20": 0.821381, "mrr_at_100": 0.821889, "mrr_at_1000": 0.821907, "nauc_ndcg_at_1_max": -0.005094, "nauc_ndcg_at_1_std": -0.088995, "nauc_ndcg_at_1_diff1": 0.465116, "nauc_ndcg_at_3_max": 0.039341, "nauc_ndcg_at_3_std": -0.078865, "nauc_ndcg_at_3_diff1": 0.402483, "nauc_ndcg_at_5_max": 0.060468, "nauc_ndcg_at_5_std": -0.072794, "nauc_ndcg_at_5_diff1": 0.392364, "nauc_ndcg_at_10_max": 0.096799, "nauc_ndcg_at_10_std": -0.031824, "nauc_ndcg_at_10_diff1": 0.381709, "nauc_ndcg_at_20_max": 0.094417, "nauc_ndcg_at_20_std": -0.023842, "nauc_ndcg_at_20_diff1": 0.384954, "nauc_ndcg_at_100_max": 0.086441, "nauc_ndcg_at_100_std": -0.026539, "nauc_ndcg_at_100_diff1": 0.386208, "nauc_ndcg_at_1000_max": 0.071619, "nauc_ndcg_at_1000_std": -0.039153, "nauc_ndcg_at_1000_diff1": 0.393027, "nauc_map_at_1_max": 0.079988, "nauc_map_at_1_std": -0.056254, "nauc_map_at_1_diff1": 0.566325, "nauc_map_at_3_max": 0.064333, "nauc_map_at_3_std": -0.078612, "nauc_map_at_3_diff1": 0.450405, "nauc_map_at_5_max": 0.069227, "nauc_map_at_5_std": -0.079219, "nauc_map_at_5_diff1": 0.402916, "nauc_map_at_10_max": 0.082909, "nauc_map_at_10_std": -0.04474, "nauc_map_at_10_diff1": 0.359372, "nauc_map_at_20_max": 0.080786, "nauc_map_at_20_std": -0.03742, "nauc_map_at_20_diff1": 0.351299, "nauc_map_at_100_max": 0.083075, "nauc_map_at_100_std": -0.034722, "nauc_map_at_100_diff1": 0.342917, "nauc_map_at_1000_max": 0.081337, "nauc_map_at_1000_std": -0.036353, "nauc_map_at_1000_diff1": 0.343495, "nauc_recall_at_1_max": 0.079988, "nauc_recall_at_1_std": -0.056254, "nauc_recall_at_1_diff1": 0.566325, "nauc_recall_at_3_max": 0.072696, "nauc_recall_at_3_std": -0.071098, "nauc_recall_at_3_diff1": 0.381259, "nauc_recall_at_5_max": 0.101761, "nauc_recall_at_5_std": -0.065752, "nauc_recall_at_5_diff1": 0.322312, "nauc_recall_at_10_max": 0.156241, "nauc_recall_at_10_std": 0.007863, "nauc_recall_at_10_diff1": 0.26074, "nauc_recall_at_20_max": 0.165203, "nauc_recall_at_20_std": 0.044183, "nauc_recall_at_20_diff1": 0.245826, "nauc_recall_at_100_max": 0.263952, "nauc_recall_at_100_std": 0.17863, "nauc_recall_at_100_diff1": 0.259057, "nauc_recall_at_1000_max": 1.0, "nauc_recall_at_1000_std": 1.0, "nauc_recall_at_1000_diff1": 0.122718, "nauc_precision_at_1_max": -0.043135, "nauc_precision_at_1_std": -0.108662, "nauc_precision_at_1_diff1": 0.429908, "nauc_precision_at_3_max": -0.036719, "nauc_precision_at_3_std": -0.00526, "nauc_precision_at_3_diff1": -0.201005, "nauc_precision_at_5_max": -0.026785, "nauc_precision_at_5_std": 0.031645, "nauc_precision_at_5_diff1": -0.301682, "nauc_precision_at_10_max": 0.008522, "nauc_precision_at_10_std": 0.10738, "nauc_precision_at_10_diff1": -0.361156, "nauc_precision_at_20_max": -0.00197, "nauc_precision_at_20_std": 0.112974, "nauc_precision_at_20_diff1": -0.364721, "nauc_precision_at_100_max": -0.047601, "nauc_precision_at_100_std": 0.062939, "nauc_precision_at_100_diff1": -0.358899, "nauc_precision_at_1000_max": -0.086142, "nauc_precision_at_1000_std": 0.024547, "nauc_precision_at_1000_diff1": -0.328575, "nauc_mrr_at_1_max": -0.043135, "nauc_mrr_at_1_std": -0.108662, "nauc_mrr_at_1_diff1": 0.429908, "nauc_mrr_at_3_max": -0.037643, "nauc_mrr_at_3_std": -0.114434, "nauc_mrr_at_3_diff1": 0.405466, "nauc_mrr_at_5_max": -0.038352, "nauc_mrr_at_5_std": -0.119429, "nauc_mrr_at_5_diff1": 0.409828, "nauc_mrr_at_10_max": -0.031717, "nauc_mrr_at_10_std": -0.111642, "nauc_mrr_at_10_diff1": 0.41156, "nauc_mrr_at_20_max": -0.031914, "nauc_mrr_at_20_std": -0.111791, "nauc_mrr_at_20_diff1": 0.412579, "nauc_mrr_at_100_max": -0.034315, "nauc_mrr_at_100_std": -0.113156, "nauc_mrr_at_100_diff1": 0.412443, "nauc_mrr_at_1000_max": -0.034399, "nauc_mrr_at_1000_std": -0.113265, "nauc_mrr_at_1000_diff1": 0.412392, "hit_rate_at_1": 0.72816, "hit_rate_at_3": 0.90615, "hit_rate_at_5": 0.94175, "hit_rate_at_10": 0.97411, "hit_rate_at_20": 0.98382, "hit_rate_at_100": 0.99676, "hit_rate_at_1000": 1.0, "main_score": 0.71617, "hf_subset": "italian", "languages": [ "ita-Latn" ], "mteb_version": "2.20.12" }, { "ndcg_at_1": 0.68123, "ndcg_at_3": 0.68084, "ndcg_at_5": 0.6898, "ndcg_at_10": 0.70762, "ndcg_at_20": 0.73462, "ndcg_at_100": 0.77674, "ndcg_at_1000": 0.78945, "map_at_1": 0.33798, "map_at_3": 0.48816, "map_at_5": 0.55113, "map_at_10": 0.60085, "map_at_20": 0.62583, "map_at_100": 0.64403, "map_at_1000": 0.64633, "recall_at_1": 0.33798, "recall_at_3": 0.53654, "recall_at_5": 0.64222, "recall_at_10": 0.74064, "recall_at_20": 0.82198, "recall_at_100": 0.95009, "recall_at_1000": 0.99919, "accuracy": 0.33798, "precision_at_1": 0.72492, "precision_at_3": 0.49838, "precision_at_5": 0.39417, "precision_at_10": 0.25761, "precision_at_20": 0.15858, "precision_at_100": 0.04214, "precision_at_1000": 0.00472, "mrr_at_1": 0.724919, "mrr_at_3": 0.802589, "mrr_at_5": 0.811003, "mrr_at_10": 0.814175, "mrr_at_20": 0.816055, "mrr_at_100": 0.81654, "mrr_at_1000": 0.816567, "nauc_ndcg_at_1_max": -0.009285, "nauc_ndcg_at_1_std": -0.119599, "nauc_ndcg_at_1_diff1": 0.500129, "nauc_ndcg_at_3_max": 0.00081, "nauc_ndcg_at_3_std": -0.122246, "nauc_ndcg_at_3_diff1": 0.415943, "nauc_ndcg_at_5_max": 0.052031, "nauc_ndcg_at_5_std": -0.081543, "nauc_ndcg_at_5_diff1": 0.391103, "nauc_ndcg_at_10_max": 0.110591, "nauc_ndcg_at_10_std": -0.017319, "nauc_ndcg_at_10_diff1": 0.389214, "nauc_ndcg_at_20_max": 0.108822, "nauc_ndcg_at_20_std": -0.023362, "nauc_ndcg_at_20_diff1": 0.38932, "nauc_ndcg_at_100_max": 0.089968, "nauc_ndcg_at_100_std": -0.03301, "nauc_ndcg_at_100_diff1": 0.401119, "nauc_ndcg_at_1000_max": 0.071206, "nauc_ndcg_at_1000_std": -0.045495, "nauc_ndcg_at_1000_diff1": 0.409583, "nauc_map_at_1_max": 0.11152, "nauc_map_at_1_std": -0.061291, "nauc_map_at_1_diff1": 0.559613, "nauc_map_at_3_max": 0.080508, "nauc_map_at_3_std": -0.094574, "nauc_map_at_3_diff1": 0.46041, "nauc_map_at_5_max": 0.076028, "nauc_map_at_5_std": -0.088674, "nauc_map_at_5_diff1": 0.396824, "nauc_map_at_10_max": 0.102965, "nauc_map_at_10_std": -0.036868, "nauc_map_at_10_diff1": 0.363516, "nauc_map_at_20_max": 0.103193, "nauc_map_at_20_std": -0.035367, "nauc_map_at_20_diff1": 0.357314, "nauc_map_at_100_max": 0.099944, "nauc_map_at_100_std": -0.033994, "nauc_map_at_100_diff1": 0.352145, "nauc_map_at_1000_max": 0.097895, "nauc_map_at_1000_std": -0.035627, "nauc_map_at_1000_diff1": 0.353408, "nauc_recall_at_1_max": 0.11152, "nauc_recall_at_1_std": -0.061291, "nauc_recall_at_1_diff1": 0.559613, "nauc_recall_at_3_max": 0.085084, "nauc_recall_at_3_std": -0.081089, "nauc_recall_at_3_diff1": 0.402401, "nauc_recall_at_5_max": 0.091358, "nauc_recall_at_5_std": -0.075041, "nauc_recall_at_5_diff1": 0.29356, "nauc_recall_at_10_max": 0.194552, "nauc_recall_at_10_std": 0.0534, "nauc_recall_at_10_diff1": 0.258136, "nauc_recall_at_20_max": 0.208381, "nauc_recall_at_20_std": 0.050168, "nauc_recall_at_20_diff1": 0.232227, "nauc_recall_at_100_max": 0.275727, "nauc_recall_at_100_std": 0.153105, "nauc_recall_at_100_diff1": 0.247516, "nauc_recall_at_1000_max": 1.0, "nauc_recall_at_1000_std": 1.0, "nauc_recall_at_1000_diff1": 0.122718, "nauc_precision_at_1_max": -0.037032, "nauc_precision_at_1_std": -0.113006, "nauc_precision_at_1_diff1": 0.436417, "nauc_precision_at_3_max": -0.116237, "nauc_precision_at_3_std": -0.051596, "nauc_precision_at_3_diff1": -0.184333, "nauc_precision_at_5_max": -0.059302, "nauc_precision_at_5_std": 0.048754, "nauc_precision_at_5_diff1": -0.309051, "nauc_precision_at_10_max": -0.001347, "nauc_precision_at_10_std": 0.14484, "nauc_precision_at_10_diff1": -0.340844, "nauc_precision_at_20_max": -0.02466, "nauc_precision_at_20_std": 0.119035, "nauc_precision_at_20_diff1": -0.343732, "nauc_precision_at_100_max": -0.075125, "nauc_precision_at_100_std": 0.073312, "nauc_precision_at_100_diff1": -0.320919, "nauc_precision_at_1000_max": -0.114667, "nauc_precision_at_1000_std": 0.036074, "nauc_precision_at_1000_diff1": -0.295145, "nauc_mrr_at_1_max": -0.037032, "nauc_mrr_at_1_std": -0.113006, "nauc_mrr_at_1_diff1": 0.436417, "nauc_mrr_at_3_max": -0.039409, "nauc_mrr_at_3_std": -0.097226, "nauc_mrr_at_3_diff1": 0.401073, "nauc_mrr_at_5_max": -0.033165, "nauc_mrr_at_5_std": -0.102868, "nauc_mrr_at_5_diff1": 0.39879, "nauc_mrr_at_10_max": -0.03509, "nauc_mrr_at_10_std": -0.10665, "nauc_mrr_at_10_diff1": 0.403169, "nauc_mrr_at_20_max": -0.035068, "nauc_mrr_at_20_std": -0.107561, "nauc_mrr_at_20_diff1": 0.405518, "nauc_mrr_at_100_max": -0.036518, "nauc_mrr_at_100_std": -0.108525, "nauc_mrr_at_100_diff1": 0.405954, "nauc_mrr_at_1000_max": -0.036641, "nauc_mrr_at_1000_std": -0.108684, "nauc_mrr_at_1000_diff1": 0.405858, "hit_rate_at_1": 0.72492, "hit_rate_at_3": 0.89968, "hit_rate_at_5": 0.93528, "hit_rate_at_10": 0.95793, "hit_rate_at_20": 0.98382, "hit_rate_at_100": 0.99676, "hit_rate_at_1000": 1.0, "main_score": 0.70762, "hf_subset": "portuguese", "languages": [ "por-Latn" ], "mteb_version": "2.20.12" } ] }, "evaluation_time": 1168.4148965477943, "kg_co2_emissions": null, "date": 1789123614.183192, "evaluation_phases": [ { "name": "Data loading", "start": 4.51207160949707e-05, "end": 89.39140684902668, "split": "", "subset": "" }, { "name": "Dataset transform", "start": 89.39142466336489, "end": 89.3914270401001, "split": "", "subset": "" }, { "name": "Encoding queries and documents", "start": 89.52676960080862, "end": 949.6721348762512, "split": "test", "subset": "french" }, { "name": "Scoring", "start": 949.6723747104406, "end": 960.0512126535177, "split": "test", "subset": "french" }, { "name": "Encoding queries and documents", "start": 961.5602261349559, "end": 1002.8961703702807, "split": "test", "subset": "spanish" }, { "name": "Scoring", "start": 1002.8964030742645, "end": 1003.1992956995964, "split": "test", "subset": "spanish" }, { "name": "Encoding queries and documents", "start": 1005.596444606781, "end": 1041.9101281538606, "split": "test", "subset": "english" }, { "name": "Scoring", "start": 1041.9104000404477, "end": 1042.2234000563622, "split": "test", "subset": "english" }, { "name": "Encoding queries and documents", "start": 1044.1775385290384, "end": 1084.6644755974412, "split": "test", "subset": "german" }, { "name": "Scoring", "start": 1084.6647213697433, "end": 1084.944642342627, "split": "test", "subset": "german" }, { "name": "Encoding queries and documents", "start": 1086.7466215342283, "end": 1126.677488669753, "split": "test", "subset": "italian" }, { "name": "Scoring", "start": 1126.6777641251683, "end": 1127.006143078208, "split": "test", "subset": "italian" }, { "name": "Encoding queries and documents", "start": 1129.0303030535579, "end": 1166.8445596545935, "split": "test", "subset": "portuguese" }, { "name": "Scoring", "start": 1166.844832725823, "end": 1167.1534813269973, "split": "test", "subset": "portuguese" } ] }