{ "dataset_revision": "057fe83342b446ede7928cb51bc6c7c3223da3d0", "task_name": "Vidore3TelecomRetrieval.v2", "mteb_version": "2.18.1", "scores": { "test": [ { "ndcg_at_1": 0.6954, "ndcg_at_3": 0.68851, "ndcg_at_5": 0.68766, "ndcg_at_10": 0.71441, "ndcg_at_20": 0.74874, "ndcg_at_100": 0.78664, "ndcg_at_1000": 0.79417, "map_at_1": 0.31399, "map_at_3": 0.49696, "map_at_5": 0.54421, "map_at_10": 0.60034, "map_at_20": 0.63029, "map_at_100": 0.65042, "map_at_1000": 0.65161, "recall_at_1": 0.31399, "recall_at_3": 0.54747, "recall_at_5": 0.63155, "recall_at_10": 0.74562, "recall_at_20": 0.85372, "recall_at_100": 0.96474, "recall_at_1000": 0.99726, "accuracy": 0.31399, "precision_at_1": 0.71552, "precision_at_3": 0.51628, "precision_at_5": 0.39195, "precision_at_10": 0.26494, "precision_at_20": 0.16537, "precision_at_100": 0.04236, "precision_at_1000": 0.00447, "mrr_at_1": 0.715517, "mrr_at_3": 0.799808, "mrr_at_5": 0.810441, "mrr_at_10": 0.81444, "mrr_at_20": 0.816049, "mrr_at_100": 0.816161, "mrr_at_1000": 0.816199, "nauc_ndcg_at_1_max": 0.008702, "nauc_ndcg_at_1_std": -0.117895, "nauc_ndcg_at_1_diff1": 0.486094, "nauc_ndcg_at_3_max": -0.028356, "nauc_ndcg_at_3_std": -0.175687, "nauc_ndcg_at_3_diff1": 0.340874, "nauc_ndcg_at_5_max": 0.022537, "nauc_ndcg_at_5_std": -0.162023, "nauc_ndcg_at_5_diff1": 0.365151, "nauc_ndcg_at_10_max": 0.028007, "nauc_ndcg_at_10_std": -0.17132, "nauc_ndcg_at_10_diff1": 0.357911, "nauc_ndcg_at_20_max": 0.04719, "nauc_ndcg_at_20_std": -0.165576, "nauc_ndcg_at_20_diff1": 0.370778, "nauc_ndcg_at_100_max": 0.059575, "nauc_ndcg_at_100_std": -0.103814, "nauc_ndcg_at_100_diff1": 0.379564, "nauc_ndcg_at_1000_max": 0.044733, "nauc_ndcg_at_1000_std": -0.120479, "nauc_ndcg_at_1000_diff1": 0.385994, "nauc_map_at_1_max": 0.02556, "nauc_map_at_1_std": -0.168017, "nauc_map_at_1_diff1": 0.462282, "nauc_map_at_3_max": 0.016386, "nauc_map_at_3_std": -0.201336, "nauc_map_at_3_diff1": 0.350005, "nauc_map_at_5_max": 0.034878, "nauc_map_at_5_std": -0.195974, "nauc_map_at_5_diff1": 0.354368, "nauc_map_at_10_max": 0.020465, "nauc_map_at_10_std": -0.207996, "nauc_map_at_10_diff1": 0.334328, "nauc_map_at_20_max": 0.028042, "nauc_map_at_20_std": -0.199109, "nauc_map_at_20_diff1": 0.33827, "nauc_map_at_100_max": 0.035547, "nauc_map_at_100_std": -0.173792, "nauc_map_at_100_diff1": 0.339595, "nauc_map_at_1000_max": 0.034073, "nauc_map_at_1000_std": -0.175009, "nauc_map_at_1000_diff1": 0.339965, "nauc_recall_at_1_max": 0.02556, "nauc_recall_at_1_std": -0.168017, "nauc_recall_at_1_diff1": 0.462282, "nauc_recall_at_3_max": 0.002542, "nauc_recall_at_3_std": -0.208543, "nauc_recall_at_3_diff1": 0.294027, "nauc_recall_at_5_max": 0.055721, "nauc_recall_at_5_std": -0.184006, "nauc_recall_at_5_diff1": 0.290683, "nauc_recall_at_10_max": 0.038494, "nauc_recall_at_10_std": -0.207431, "nauc_recall_at_10_diff1": 0.246732, "nauc_recall_at_20_max": 0.114112, "nauc_recall_at_20_std": -0.168551, "nauc_recall_at_20_diff1": 0.257508, "nauc_recall_at_100_max": 0.422617, "nauc_recall_at_100_std": 0.392065, "nauc_recall_at_100_diff1": 0.208285, "nauc_recall_at_1000_max": 0.687757, "nauc_recall_at_1000_std": 0.781533, "nauc_recall_at_1000_diff1": 0.502036, "nauc_precision_at_1_max": -0.015979, "nauc_precision_at_1_std": -0.139309, "nauc_precision_at_1_diff1": 0.491825, "nauc_precision_at_3_max": -0.047715, "nauc_precision_at_3_std": -0.027073, "nauc_precision_at_3_diff1": -0.060687, "nauc_precision_at_5_max": 0.001092, "nauc_precision_at_5_std": 0.046135, "nauc_precision_at_5_diff1": -0.102103, "nauc_precision_at_10_max": -0.009339, "nauc_precision_at_10_std": 0.089528, "nauc_precision_at_10_diff1": -0.164587, "nauc_precision_at_20_max": 0.006433, "nauc_precision_at_20_std": 0.134957, "nauc_precision_at_20_diff1": -0.180533, "nauc_precision_at_100_max": 0.001054, "nauc_precision_at_100_std": 0.205673, "nauc_precision_at_100_diff1": -0.190069, "nauc_precision_at_1000_max": -0.026191, "nauc_precision_at_1000_std": 0.1759, "nauc_precision_at_1000_diff1": -0.192564, "nauc_mrr_at_1_max": -0.015979, "nauc_mrr_at_1_std": -0.139309, "nauc_mrr_at_1_diff1": 0.491825, "nauc_mrr_at_3_max": 0.006711, "nauc_mrr_at_3_std": -0.099973, "nauc_mrr_at_3_diff1": 0.487895, "nauc_mrr_at_5_max": 0.016285, "nauc_mrr_at_5_std": -0.098883, "nauc_mrr_at_5_diff1": 0.493806, "nauc_mrr_at_10_max": 0.009671, "nauc_mrr_at_10_std": -0.100311, "nauc_mrr_at_10_diff1": 0.492784, "nauc_mrr_at_20_max": 0.011178, "nauc_mrr_at_20_std": -0.102223, "nauc_mrr_at_20_diff1": 0.49209, "nauc_mrr_at_100_max": 0.010638, "nauc_mrr_at_100_std": -0.10284, "nauc_mrr_at_100_diff1": 0.491878, "nauc_mrr_at_1000_max": 0.010411, "nauc_mrr_at_1000_std": -0.103093, "nauc_mrr_at_1000_diff1": 0.491892, "hit_rate_at_1": 0.71552, "hit_rate_at_3": 0.89368, "hit_rate_at_5": 0.93966, "hit_rate_at_10": 0.96839, "hit_rate_at_20": 0.98851, "hit_rate_at_100": 0.99425, "hit_rate_at_1000": 1.0, "main_score": 0.71441, "hf_subset": "french", "languages": [ "fra-Latn" ], "mteb_version": "2.18.1" }, { "ndcg_at_1": 0.71121, "ndcg_at_3": 0.68782, "ndcg_at_5": 0.68917, "ndcg_at_10": 0.71419, "ndcg_at_20": 0.74637, "ndcg_at_100": 0.78675, "ndcg_at_1000": 0.79443, "map_at_1": 0.32095, "map_at_3": 0.49707, "map_at_5": 0.54722, "map_at_10": 0.6025, "map_at_20": 0.63113, "map_at_100": 0.65162, "map_at_1000": 0.65284, "recall_at_1": 0.32095, "recall_at_3": 0.54227, "recall_at_5": 0.62988, "recall_at_10": 0.74358, "recall_at_20": 0.84768, "recall_at_100": 0.96426, "recall_at_1000": 0.99762, "accuracy": 0.32095, "precision_at_1": 0.73851, "precision_at_3": 0.51054, "precision_at_5": 0.39368, "precision_at_10": 0.26379, "precision_at_20": 0.16322, "precision_at_100": 0.04239, "precision_at_1000": 0.00447, "mrr_at_1": 0.738506, "mrr_at_3": 0.80795, "mrr_at_5": 0.816571, "mrr_at_10": 0.821235, "mrr_at_20": 0.82315, "mrr_at_100": 0.82335, "mrr_at_1000": 0.823384, "nauc_ndcg_at_1_max": 0.0521, "nauc_ndcg_at_1_std": -0.155491, "nauc_ndcg_at_1_diff1": 0.38013, "nauc_ndcg_at_3_max": 0.016615, "nauc_ndcg_at_3_std": -0.159676, "nauc_ndcg_at_3_diff1": 0.297363, "nauc_ndcg_at_5_max": 0.029879, "nauc_ndcg_at_5_std": -0.163031, "nauc_ndcg_at_5_diff1": 0.321244, "nauc_ndcg_at_10_max": 0.054467, "nauc_ndcg_at_10_std": -0.161416, "nauc_ndcg_at_10_diff1": 0.342198, "nauc_ndcg_at_20_max": 0.077738, "nauc_ndcg_at_20_std": -0.145831, "nauc_ndcg_at_20_diff1": 0.351095, "nauc_ndcg_at_100_max": 0.096879, "nauc_ndcg_at_100_std": -0.092643, "nauc_ndcg_at_100_diff1": 0.347209, "nauc_ndcg_at_1000_max": 0.080748, "nauc_ndcg_at_1000_std": -0.110741, "nauc_ndcg_at_1000_diff1": 0.345703, "nauc_map_at_1_max": 0.006666, "nauc_map_at_1_std": -0.188033, "nauc_map_at_1_diff1": 0.406801, "nauc_map_at_3_max": 0.023672, "nauc_map_at_3_std": -0.210858, "nauc_map_at_3_diff1": 0.312083, "nauc_map_at_5_max": 0.022428, "nauc_map_at_5_std": -0.209586, "nauc_map_at_5_diff1": 0.321472, "nauc_map_at_10_max": 0.025915, "nauc_map_at_10_std": -0.201649, "nauc_map_at_10_diff1": 0.311058, "nauc_map_at_20_max": 0.043866, "nauc_map_at_20_std": -0.184306, "nauc_map_at_20_diff1": 0.313681, "nauc_map_at_100_max": 0.059292, "nauc_map_at_100_std": -0.158867, "nauc_map_at_100_diff1": 0.314639, "nauc_map_at_1000_max": 0.057728, "nauc_map_at_1000_std": -0.160381, "nauc_map_at_1000_diff1": 0.314462, "nauc_recall_at_1_max": 0.006666, "nauc_recall_at_1_std": -0.188033, "nauc_recall_at_1_diff1": 0.406801, "nauc_recall_at_3_max": 0.009311, "nauc_recall_at_3_std": -0.222766, "nauc_recall_at_3_diff1": 0.277491, "nauc_recall_at_5_max": -0.001053, "nauc_recall_at_5_std": -0.230134, "nauc_recall_at_5_diff1": 0.288817, "nauc_recall_at_10_max": 0.043759, "nauc_recall_at_10_std": -0.186048, "nauc_recall_at_10_diff1": 0.289691, "nauc_recall_at_20_max": 0.147441, "nauc_recall_at_20_std": -0.115059, "nauc_recall_at_20_diff1": 0.312703, "nauc_recall_at_100_max": 0.469125, "nauc_recall_at_100_std": 0.404988, "nauc_recall_at_100_diff1": 0.277299, "nauc_recall_at_1000_max": 0.754346, "nauc_recall_at_1000_std": 0.834352, "nauc_recall_at_1000_diff1": 0.502175, "nauc_precision_at_1_max": 0.024137, "nauc_precision_at_1_std": -0.158117, "nauc_precision_at_1_diff1": 0.37122, "nauc_precision_at_3_max": 0.015451, "nauc_precision_at_3_std": 0.011087, "nauc_precision_at_3_diff1": -0.039001, "nauc_precision_at_5_max": 0.034586, "nauc_precision_at_5_std": 0.084515, "nauc_precision_at_5_diff1": -0.081265, "nauc_precision_at_10_max": 0.057457, "nauc_precision_at_10_std": 0.133196, "nauc_precision_at_10_diff1": -0.116079, "nauc_precision_at_20_max": 0.092506, "nauc_precision_at_20_std": 0.196935, "nauc_precision_at_20_diff1": -0.146754, "nauc_precision_at_100_max": 0.076472, "nauc_precision_at_100_std": 0.248355, "nauc_precision_at_100_diff1": -0.188967, "nauc_precision_at_1000_max": 0.042637, "nauc_precision_at_1000_std": 0.214567, "nauc_precision_at_1000_diff1": -0.202523, "nauc_mrr_at_1_max": 0.024137, "nauc_mrr_at_1_std": -0.158117, "nauc_mrr_at_1_diff1": 0.37122, "nauc_mrr_at_3_max": 0.051319, "nauc_mrr_at_3_std": -0.110174, "nauc_mrr_at_3_diff1": 0.393514, "nauc_mrr_at_5_max": 0.052104, "nauc_mrr_at_5_std": -0.112611, "nauc_mrr_at_5_diff1": 0.395787, "nauc_mrr_at_10_max": 0.058647, "nauc_mrr_at_10_std": -0.106153, "nauc_mrr_at_10_diff1": 0.397935, "nauc_mrr_at_20_max": 0.057762, "nauc_mrr_at_20_std": -0.106606, "nauc_mrr_at_20_diff1": 0.394474, "nauc_mrr_at_100_max": 0.056966, "nauc_mrr_at_100_std": -0.107271, "nauc_mrr_at_100_diff1": 0.393786, "nauc_mrr_at_1000_max": 0.056762, "nauc_mrr_at_1000_std": -0.107503, "nauc_mrr_at_1000_diff1": 0.393809, "hit_rate_at_1": 0.73851, "hit_rate_at_3": 0.8908, "hit_rate_at_5": 0.92816, "hit_rate_at_10": 0.95977, "hit_rate_at_20": 0.98563, "hit_rate_at_100": 0.99425, "hit_rate_at_1000": 1.0, "main_score": 0.71419, "hf_subset": "spanish", "languages": [ "spa-Latn" ], "mteb_version": "2.18.1" }, { "ndcg_at_1": 0.71695, "ndcg_at_3": 0.68888, "ndcg_at_5": 0.69355, "ndcg_at_10": 0.72445, "ndcg_at_20": 0.75686, "ndcg_at_100": 0.79443, "ndcg_at_1000": 0.80063, "map_at_1": 0.32683, "map_at_3": 0.49914, "map_at_5": 0.55354, "map_at_10": 0.61192, "map_at_20": 0.64238, "map_at_100": 0.66183, "map_at_1000": 0.66281, "recall_at_1": 0.32683, "recall_at_3": 0.54629, "recall_at_5": 0.63912, "recall_at_10": 0.75865, "recall_at_20": 0.86142, "recall_at_100": 0.97064, "recall_at_1000": 0.99866, "accuracy": 0.32683, "precision_at_1": 0.74138, "precision_at_3": 0.51245, "precision_at_5": 0.39655, "precision_at_10": 0.27126, "precision_at_20": 0.16853, "precision_at_100": 0.04299, "precision_at_1000": 0.00447, "mrr_at_1": 0.741379, "mrr_at_3": 0.812739, "mrr_at_5": 0.820211, "mrr_at_10": 0.82547, "mrr_at_20": 0.826524, "mrr_at_100": 0.826777, "mrr_at_1000": 0.826817, "nauc_ndcg_at_1_max": 0.117106, "nauc_ndcg_at_1_std": -0.020108, "nauc_ndcg_at_1_diff1": 0.482587, "nauc_ndcg_at_3_max": 0.063364, "nauc_ndcg_at_3_std": -0.120005, "nauc_ndcg_at_3_diff1": 0.376576, "nauc_ndcg_at_5_max": 0.076109, "nauc_ndcg_at_5_std": -0.12523, "nauc_ndcg_at_5_diff1": 0.355644, "nauc_ndcg_at_10_max": 0.106167, "nauc_ndcg_at_10_std": -0.137221, "nauc_ndcg_at_10_diff1": 0.375205, "nauc_ndcg_at_20_max": 0.135955, "nauc_ndcg_at_20_std": -0.127508, "nauc_ndcg_at_20_diff1": 0.395106, "nauc_ndcg_at_100_max": 0.140906, "nauc_ndcg_at_100_std": -0.067567, "nauc_ndcg_at_100_diff1": 0.38581, "nauc_ndcg_at_1000_max": 0.125942, "nauc_ndcg_at_1000_std": -0.076563, "nauc_ndcg_at_1000_diff1": 0.39104, "nauc_map_at_1_max": 0.067817, "nauc_map_at_1_std": -0.135962, "nauc_map_at_1_diff1": 0.451542, "nauc_map_at_3_max": 0.078736, "nauc_map_at_3_std": -0.175695, "nauc_map_at_3_diff1": 0.37408, "nauc_map_at_5_max": 0.077904, "nauc_map_at_5_std": -0.177288, "nauc_map_at_5_diff1": 0.357121, "nauc_map_at_10_max": 0.081833, "nauc_map_at_10_std": -0.182346, "nauc_map_at_10_diff1": 0.351131, "nauc_map_at_20_max": 0.098445, "nauc_map_at_20_std": -0.165141, "nauc_map_at_20_diff1": 0.357088, "nauc_map_at_100_max": 0.105398, "nauc_map_at_100_std": -0.138428, "nauc_map_at_100_diff1": 0.351335, "nauc_map_at_1000_max": 0.103818, "nauc_map_at_1000_std": -0.13916, "nauc_map_at_1000_diff1": 0.351713, "nauc_recall_at_1_max": 0.067817, "nauc_recall_at_1_std": -0.135962, "nauc_recall_at_1_diff1": 0.451542, "nauc_recall_at_3_max": 0.061121, "nauc_recall_at_3_std": -0.205874, "nauc_recall_at_3_diff1": 0.330258, "nauc_recall_at_5_max": 0.058984, "nauc_recall_at_5_std": -0.211158, "nauc_recall_at_5_diff1": 0.292515, "nauc_recall_at_10_max": 0.103322, "nauc_recall_at_10_std": -0.22377, "nauc_recall_at_10_diff1": 0.278746, "nauc_recall_at_20_max": 0.20088, "nauc_recall_at_20_std": -0.185126, "nauc_recall_at_20_diff1": 0.313541, "nauc_recall_at_100_max": 0.547541, "nauc_recall_at_100_std": 0.313963, "nauc_recall_at_100_diff1": 0.195212, "nauc_recall_at_1000_max": 0.626464, "nauc_recall_at_1000_std": 0.652984, "nauc_recall_at_1000_diff1": 0.449128, "nauc_precision_at_1_max": 0.120621, "nauc_precision_at_1_std": -0.012552, "nauc_precision_at_1_diff1": 0.497077, "nauc_precision_at_3_max": 0.026542, "nauc_precision_at_3_std": 0.022828, "nauc_precision_at_3_diff1": -0.013405, "nauc_precision_at_5_max": 0.003736, "nauc_precision_at_5_std": 0.069039, "nauc_precision_at_5_diff1": -0.117869, "nauc_precision_at_10_max": 0.00594, "nauc_precision_at_10_std": 0.10462, "nauc_precision_at_10_diff1": -0.156418, "nauc_precision_at_20_max": 0.019509, "nauc_precision_at_20_std": 0.167465, "nauc_precision_at_20_diff1": -0.179415, "nauc_precision_at_100_max": -0.016275, "nauc_precision_at_100_std": 0.234113, "nauc_precision_at_100_diff1": -0.227891, "nauc_precision_at_1000_max": -0.042862, "nauc_precision_at_1000_std": 0.216362, "nauc_precision_at_1000_diff1": -0.232507, "nauc_mrr_at_1_max": 0.120621, "nauc_mrr_at_1_std": -0.012552, "nauc_mrr_at_1_diff1": 0.497077, "nauc_mrr_at_3_max": 0.132823, "nauc_mrr_at_3_std": -0.001691, "nauc_mrr_at_3_diff1": 0.489758, "nauc_mrr_at_5_max": 0.132073, "nauc_mrr_at_5_std": -0.003036, "nauc_mrr_at_5_diff1": 0.486763, "nauc_mrr_at_10_max": 0.132698, "nauc_mrr_at_10_std": 0.00049, "nauc_mrr_at_10_diff1": 0.493042, "nauc_mrr_at_20_max": 0.131659, "nauc_mrr_at_20_std": -0.000581, "nauc_mrr_at_20_diff1": 0.491125, "nauc_mrr_at_100_max": 0.132275, "nauc_mrr_at_100_std": 0.000331, "nauc_mrr_at_100_diff1": 0.491296, "nauc_mrr_at_1000_max": 0.132052, "nauc_mrr_at_1000_std": 8.6e-05, "nauc_mrr_at_1000_diff1": 0.491398, "hit_rate_at_1": 0.74138, "hit_rate_at_3": 0.89943, "hit_rate_at_5": 0.93391, "hit_rate_at_10": 0.97126, "hit_rate_at_20": 0.98563, "hit_rate_at_100": 0.99425, "hit_rate_at_1000": 1.0, "main_score": 0.72445, "hf_subset": "english", "languages": [ "eng-Latn" ], "mteb_version": "2.18.1" }, { "ndcg_at_1": 0.72989, "ndcg_at_3": 0.69129, "ndcg_at_5": 0.69515, "ndcg_at_10": 0.7249, "ndcg_at_20": 0.75554, "ndcg_at_100": 0.79613, "ndcg_at_1000": 0.80264, "map_at_1": 0.33528, "map_at_3": 0.50366, "map_at_5": 0.55685, "map_at_10": 0.61313, "map_at_20": 0.64203, "map_at_100": 0.66262, "map_at_1000": 0.66373, "recall_at_1": 0.33528, "recall_at_3": 0.54313, "recall_at_5": 0.63331, "recall_at_10": 0.75382, "recall_at_20": 0.84878, "recall_at_100": 0.96835, "recall_at_1000": 0.9973, "accuracy": 0.33528, "precision_at_1": 0.75287, "precision_at_3": 0.5067, "precision_at_5": 0.39368, "precision_at_10": 0.2658, "precision_at_20": 0.16494, "precision_at_100": 0.0427, "precision_at_1000": 0.00447, "mrr_at_1": 0.752874, "mrr_at_3": 0.816571, "mrr_at_5": 0.824186, "mrr_at_10": 0.82928, "mrr_at_20": 0.831042, "mrr_at_100": 0.831274, "mrr_at_1000": 0.83132, "nauc_ndcg_at_1_max": -0.035035, "nauc_ndcg_at_1_std": -0.042368, "nauc_ndcg_at_1_diff1": 0.34224, "nauc_ndcg_at_3_max": -0.033722, "nauc_ndcg_at_3_std": -0.128481, "nauc_ndcg_at_3_diff1": 0.271782, "nauc_ndcg_at_5_max": -0.026724, "nauc_ndcg_at_5_std": -0.130196, "nauc_ndcg_at_5_diff1": 0.282359, "nauc_ndcg_at_10_max": -0.032599, "nauc_ndcg_at_10_std": -0.158939, "nauc_ndcg_at_10_diff1": 0.284329, "nauc_ndcg_at_20_max": 0.011233, "nauc_ndcg_at_20_std": -0.130031, "nauc_ndcg_at_20_diff1": 0.283885, "nauc_ndcg_at_100_max": 0.005963, "nauc_ndcg_at_100_std": -0.083967, "nauc_ndcg_at_100_diff1": 0.284896, "nauc_ndcg_at_1000_max": -0.006939, "nauc_ndcg_at_1000_std": -0.097546, "nauc_ndcg_at_1000_diff1": 0.290292, "nauc_map_at_1_max": -0.026791, "nauc_map_at_1_std": -0.148437, "nauc_map_at_1_diff1": 0.377731, "nauc_map_at_3_max": 0.000615, "nauc_map_at_3_std": -0.177639, "nauc_map_at_3_diff1": 0.283625, "nauc_map_at_5_max": -0.007533, "nauc_map_at_5_std": -0.176712, "nauc_map_at_5_diff1": 0.276198, "nauc_map_at_10_max": -0.021907, "nauc_map_at_10_std": -0.191279, "nauc_map_at_10_diff1": 0.260595, "nauc_map_at_20_max": -0.003439, "nauc_map_at_20_std": -0.170913, "nauc_map_at_20_diff1": 0.258666, "nauc_map_at_100_max": 0.000699, "nauc_map_at_100_std": -0.149899, "nauc_map_at_100_diff1": 0.261334, "nauc_map_at_1000_max": -0.000542, "nauc_map_at_1000_std": -0.151225, "nauc_map_at_1000_diff1": 0.261708, "nauc_recall_at_1_max": -0.026791, "nauc_recall_at_1_std": -0.148437, "nauc_recall_at_1_diff1": 0.377731, "nauc_recall_at_3_max": 0.001031, "nauc_recall_at_3_std": -0.192208, "nauc_recall_at_3_diff1": 0.274795, "nauc_recall_at_5_max": -0.016395, "nauc_recall_at_5_std": -0.194987, "nauc_recall_at_5_diff1": 0.263771, "nauc_recall_at_10_max": -0.016927, "nauc_recall_at_10_std": -0.208067, "nauc_recall_at_10_diff1": 0.228025, "nauc_recall_at_20_max": 0.125024, "nauc_recall_at_20_std": -0.122292, "nauc_recall_at_20_diff1": 0.230111, "nauc_recall_at_100_max": 0.403371, "nauc_recall_at_100_std": 0.391208, "nauc_recall_at_100_diff1": 0.125, "nauc_recall_at_1000_max": 0.705271, "nauc_recall_at_1000_std": 0.809269, "nauc_recall_at_1000_diff1": 0.328143, "nauc_precision_at_1_max": -0.037089, "nauc_precision_at_1_std": -0.028207, "nauc_precision_at_1_diff1": 0.342618, "nauc_precision_at_3_max": -0.000655, "nauc_precision_at_3_std": 0.007965, "nauc_precision_at_3_diff1": -0.076102, "nauc_precision_at_5_max": 0.004364, "nauc_precision_at_5_std": 0.077752, "nauc_precision_at_5_diff1": -0.115869, "nauc_precision_at_10_max": 0.000645, "nauc_precision_at_10_std": 0.098152, "nauc_precision_at_10_diff1": -0.154487, "nauc_precision_at_20_max": 0.04234, "nauc_precision_at_20_std": 0.162256, "nauc_precision_at_20_diff1": -0.170658, "nauc_precision_at_100_max": 0.015224, "nauc_precision_at_100_std": 0.209871, "nauc_precision_at_100_diff1": -0.177831, "nauc_precision_at_1000_max": -0.005061, "nauc_precision_at_1000_std": 0.185974, "nauc_precision_at_1000_diff1": -0.180713, "nauc_mrr_at_1_max": -0.037089, "nauc_mrr_at_1_std": -0.028207, "nauc_mrr_at_1_diff1": 0.342618, "nauc_mrr_at_3_max": -0.046633, "nauc_mrr_at_3_std": -0.051313, "nauc_mrr_at_3_diff1": 0.355428, "nauc_mrr_at_5_max": -0.044643, "nauc_mrr_at_5_std": -0.049567, "nauc_mrr_at_5_diff1": 0.359312, "nauc_mrr_at_10_max": -0.034602, "nauc_mrr_at_10_std": -0.034391, "nauc_mrr_at_10_diff1": 0.357237, "nauc_mrr_at_20_max": -0.036186, "nauc_mrr_at_20_std": -0.038894, "nauc_mrr_at_20_diff1": 0.355667, "nauc_mrr_at_100_max": -0.037291, "nauc_mrr_at_100_std": -0.03967, "nauc_mrr_at_100_diff1": 0.354951, "nauc_mrr_at_1000_max": -0.037598, "nauc_mrr_at_1000_std": -0.039961, "nauc_mrr_at_1000_diff1": 0.354908, "hit_rate_at_1": 0.75287, "hit_rate_at_3": 0.8908, "hit_rate_at_5": 0.92529, "hit_rate_at_10": 0.96264, "hit_rate_at_20": 0.98563, "hit_rate_at_100": 0.99425, "hit_rate_at_1000": 1.0, "main_score": 0.7249, "hf_subset": "german", "languages": [ "deu-Latn" ], "mteb_version": "2.18.1" }, { "ndcg_at_1": 0.71121, "ndcg_at_3": 0.67844, "ndcg_at_5": 0.68667, "ndcg_at_10": 0.71339, "ndcg_at_20": 0.74605, "ndcg_at_100": 0.78734, "ndcg_at_1000": 0.79394, "map_at_1": 0.31848, "map_at_3": 0.49009, "map_at_5": 0.54392, "map_at_10": 0.59917, "map_at_20": 0.6283, "map_at_100": 0.64934, "map_at_1000": 0.65044, "recall_at_1": 0.31848, "recall_at_3": 0.53475, "recall_at_5": 0.62956, "recall_at_10": 0.74531, "recall_at_20": 0.84823, "recall_at_100": 0.96959, "recall_at_1000": 0.99802, "accuracy": 0.31848, "precision_at_1": 0.72989, "precision_at_3": 0.49808, "precision_at_5": 0.39138, "precision_at_10": 0.26322, "precision_at_20": 0.16293, "precision_at_100": 0.04259, "precision_at_1000": 0.00447, "mrr_at_1": 0.729885, "mrr_at_3": 0.80364, "mrr_at_5": 0.813985, "mrr_at_10": 0.818626, "mrr_at_20": 0.820253, "mrr_at_100": 0.820497, "mrr_at_1000": 0.820538, "nauc_ndcg_at_1_max": 0.029649, "nauc_ndcg_at_1_std": -0.094552, "nauc_ndcg_at_1_diff1": 0.395262, "nauc_ndcg_at_3_max": -0.011764, "nauc_ndcg_at_3_std": -0.154735, "nauc_ndcg_at_3_diff1": 0.317246, "nauc_ndcg_at_5_max": 0.022773, "nauc_ndcg_at_5_std": -0.148105, "nauc_ndcg_at_5_diff1": 0.338726, "nauc_ndcg_at_10_max": 0.020203, "nauc_ndcg_at_10_std": -0.170174, "nauc_ndcg_at_10_diff1": 0.339141, "nauc_ndcg_at_20_max": 0.039911, "nauc_ndcg_at_20_std": -0.162156, "nauc_ndcg_at_20_diff1": 0.34066, "nauc_ndcg_at_100_max": 0.056439, "nauc_ndcg_at_100_std": -0.101514, "nauc_ndcg_at_100_diff1": 0.350605, "nauc_ndcg_at_1000_max": 0.04384, "nauc_ndcg_at_1000_std": -0.114062, "nauc_ndcg_at_1000_diff1": 0.351672, "nauc_map_at_1_max": 0.015739, "nauc_map_at_1_std": -0.157238, "nauc_map_at_1_diff1": 0.454084, "nauc_map_at_3_max": 0.015638, "nauc_map_at_3_std": -0.203632, "nauc_map_at_3_diff1": 0.345361, "nauc_map_at_5_max": 0.02331, "nauc_map_at_5_std": -0.203799, "nauc_map_at_5_diff1": 0.33934, "nauc_map_at_10_max": 0.008061, "nauc_map_at_10_std": -0.214448, "nauc_map_at_10_diff1": 0.317912, "nauc_map_at_20_max": 0.021368, "nauc_map_at_20_std": -0.20092, "nauc_map_at_20_diff1": 0.319734, "nauc_map_at_100_max": 0.031018, "nauc_map_at_100_std": -0.17549, "nauc_map_at_100_diff1": 0.321307, "nauc_map_at_1000_max": 0.029747, "nauc_map_at_1000_std": -0.176403, "nauc_map_at_1000_diff1": 0.321568, "nauc_recall_at_1_max": 0.015739, "nauc_recall_at_1_std": -0.157238, "nauc_recall_at_1_diff1": 0.454084, "nauc_recall_at_3_max": 0.004151, "nauc_recall_at_3_std": -0.214487, "nauc_recall_at_3_diff1": 0.301934, "nauc_recall_at_5_max": 0.044148, "nauc_recall_at_5_std": -0.194343, "nauc_recall_at_5_diff1": 0.292529, "nauc_recall_at_10_max": 0.027209, "nauc_recall_at_10_std": -0.214817, "nauc_recall_at_10_diff1": 0.257135, "nauc_recall_at_20_max": 0.084804, "nauc_recall_at_20_std": -0.188556, "nauc_recall_at_20_diff1": 0.239857, "nauc_recall_at_100_max": 0.423583, "nauc_recall_at_100_std": 0.36296, "nauc_recall_at_100_diff1": 0.273883, "nauc_recall_at_1000_max": 0.845396, "nauc_recall_at_1000_std": 0.901897, "nauc_recall_at_1000_diff1": 0.202829, "nauc_precision_at_1_max": 0.009343, "nauc_precision_at_1_std": -0.114097, "nauc_precision_at_1_diff1": 0.408912, "nauc_precision_at_3_max": -0.017934, "nauc_precision_at_3_std": -0.030411, "nauc_precision_at_3_diff1": -0.073772, "nauc_precision_at_5_max": 0.01482, "nauc_precision_at_5_std": 0.050568, "nauc_precision_at_5_diff1": -0.122396, "nauc_precision_at_10_max": 0.00346, "nauc_precision_at_10_std": 0.081822, "nauc_precision_at_10_diff1": -0.173607, "nauc_precision_at_20_max": 0.026402, "nauc_precision_at_20_std": 0.138818, "nauc_precision_at_20_diff1": -0.192295, "nauc_precision_at_100_max": 0.019837, "nauc_precision_at_100_std": 0.222924, "nauc_precision_at_100_diff1": -0.202606, "nauc_precision_at_1000_max": -0.006851, "nauc_precision_at_1000_std": 0.195817, "nauc_precision_at_1000_diff1": -0.206125, "nauc_mrr_at_1_max": 0.009343, "nauc_mrr_at_1_std": -0.114097, "nauc_mrr_at_1_diff1": 0.408912, "nauc_mrr_at_3_max": -0.002708, "nauc_mrr_at_3_std": -0.093047, "nauc_mrr_at_3_diff1": 0.397381, "nauc_mrr_at_5_max": 0.018791, "nauc_mrr_at_5_std": -0.076416, "nauc_mrr_at_5_diff1": 0.40259, "nauc_mrr_at_10_max": 0.015742, "nauc_mrr_at_10_std": -0.082705, "nauc_mrr_at_10_diff1": 0.408674, "nauc_mrr_at_20_max": 0.015847, "nauc_mrr_at_20_std": -0.084041, "nauc_mrr_at_20_diff1": 0.40758, "nauc_mrr_at_100_max": 0.014774, "nauc_mrr_at_100_std": -0.084725, "nauc_mrr_at_100_diff1": 0.406749, "nauc_mrr_at_1000_max": 0.01454, "nauc_mrr_at_1000_std": -0.084986, "nauc_mrr_at_1000_diff1": 0.406761, "hit_rate_at_1": 0.72989, "hit_rate_at_3": 0.88506, "hit_rate_at_5": 0.93103, "hit_rate_at_10": 0.96264, "hit_rate_at_20": 0.98563, "hit_rate_at_100": 0.99425, "hit_rate_at_1000": 1.0, "main_score": 0.71339, "hf_subset": "italian", "languages": [ "ita-Latn" ], "mteb_version": "2.18.1" }, { "ndcg_at_1": 0.70546, "ndcg_at_3": 0.68142, "ndcg_at_5": 0.68337, "ndcg_at_10": 0.71456, "ndcg_at_20": 0.74819, "ndcg_at_100": 0.78684, "ndcg_at_1000": 0.79381, "map_at_1": 0.31984, "map_at_3": 0.48932, "map_at_5": 0.53934, "map_at_10": 0.59921, "map_at_20": 0.62909, "map_at_100": 0.6488, "map_at_1000": 0.64993, "recall_at_1": 0.31984, "recall_at_3": 0.53541, "recall_at_5": 0.62301, "recall_at_10": 0.74727, "recall_at_20": 0.85399, "recall_at_100": 0.96773, "recall_at_1000": 0.99794, "accuracy": 0.31984, "precision_at_1": 0.72989, "precision_at_3": 0.50383, "precision_at_5": 0.38793, "precision_at_10": 0.26609, "precision_at_20": 0.16523, "precision_at_100": 0.04247, "precision_at_1000": 0.00447, "mrr_at_1": 0.729885, "mrr_at_3": 0.803161, "mrr_at_5": 0.812644, "mrr_at_10": 0.817775, "mrr_at_20": 0.819433, "mrr_at_100": 0.819644, "mrr_at_1000": 0.819703, "nauc_ndcg_at_1_max": -0.025631, "nauc_ndcg_at_1_std": -0.154042, "nauc_ndcg_at_1_diff1": 0.458079, "nauc_ndcg_at_3_max": 0.004216, "nauc_ndcg_at_3_std": -0.146165, "nauc_ndcg_at_3_diff1": 0.344924, "nauc_ndcg_at_5_max": 0.035434, "nauc_ndcg_at_5_std": -0.158208, "nauc_ndcg_at_5_diff1": 0.345986, "nauc_ndcg_at_10_max": 0.039956, "nauc_ndcg_at_10_std": -0.162003, "nauc_ndcg_at_10_diff1": 0.358624, "nauc_ndcg_at_20_max": 0.055759, "nauc_ndcg_at_20_std": -0.155413, "nauc_ndcg_at_20_diff1": 0.359065, "nauc_ndcg_at_100_max": 0.078647, "nauc_ndcg_at_100_std": -0.091492, "nauc_ndcg_at_100_diff1": 0.371769, "nauc_ndcg_at_1000_max": 0.062625, "nauc_ndcg_at_1000_std": -0.105199, "nauc_ndcg_at_1000_diff1": 0.372521, "nauc_map_at_1_max": -0.022506, "nauc_map_at_1_std": -0.190083, "nauc_map_at_1_diff1": 0.416033, "nauc_map_at_3_max": 0.002874, "nauc_map_at_3_std": -0.213224, "nauc_map_at_3_diff1": 0.33263, "nauc_map_at_5_max": 0.016449, "nauc_map_at_5_std": -0.21584, "nauc_map_at_5_diff1": 0.324591, "nauc_map_at_10_max": 0.015412, "nauc_map_at_10_std": -0.208116, "nauc_map_at_10_diff1": 0.323367, "nauc_map_at_20_max": 0.032735, "nauc_map_at_20_std": -0.191537, "nauc_map_at_20_diff1": 0.325053, "nauc_map_at_100_max": 0.048443, "nauc_map_at_100_std": -0.162181, "nauc_map_at_100_diff1": 0.327723, "nauc_map_at_1000_max": 0.047096, "nauc_map_at_1000_std": -0.162875, "nauc_map_at_1000_diff1": 0.327955, "nauc_recall_at_1_max": -0.022506, "nauc_recall_at_1_std": -0.190083, "nauc_recall_at_1_diff1": 0.416033, "nauc_recall_at_3_max": -0.003865, "nauc_recall_at_3_std": -0.219099, "nauc_recall_at_3_diff1": 0.290783, "nauc_recall_at_5_max": 0.028436, "nauc_recall_at_5_std": -0.231172, "nauc_recall_at_5_diff1": 0.271569, "nauc_recall_at_10_max": 0.042885, "nauc_recall_at_10_std": -0.210279, "nauc_recall_at_10_diff1": 0.275617, "nauc_recall_at_20_max": 0.106402, "nauc_recall_at_20_std": -0.180245, "nauc_recall_at_20_diff1": 0.241072, "nauc_recall_at_100_max": 0.486912, "nauc_recall_at_100_std": 0.375756, "nauc_recall_at_100_diff1": 0.27344, "nauc_recall_at_1000_max": 0.65442, "nauc_recall_at_1000_std": 0.753798, "nauc_recall_at_1000_diff1": 0.457782, "nauc_precision_at_1_max": -0.05351, "nauc_precision_at_1_std": -0.164761, "nauc_precision_at_1_diff1": 0.470019, "nauc_precision_at_3_max": 0.038358, "nauc_precision_at_3_std": 0.029644, "nauc_precision_at_3_diff1": -0.000458, "nauc_precision_at_5_max": 0.073897, "nauc_precision_at_5_std": 0.085125, "nauc_precision_at_5_diff1": -0.064346, "nauc_precision_at_10_max": 0.071687, "nauc_precision_at_10_std": 0.140208, "nauc_precision_at_10_diff1": -0.103197, "nauc_precision_at_20_max": 0.096897, "nauc_precision_at_20_std": 0.194639, "nauc_precision_at_20_diff1": -0.137466, "nauc_precision_at_100_max": 0.08629, "nauc_precision_at_100_std": 0.262062, "nauc_precision_at_100_diff1": -0.162662, "nauc_precision_at_1000_max": 0.05067, "nauc_precision_at_1000_std": 0.231635, "nauc_precision_at_1000_diff1": -0.171084, "nauc_mrr_at_1_max": -0.05351, "nauc_mrr_at_1_std": -0.164761, "nauc_mrr_at_1_diff1": 0.470019, "nauc_mrr_at_3_max": -0.008737, "nauc_mrr_at_3_std": -0.100387, "nauc_mrr_at_3_diff1": 0.474023, "nauc_mrr_at_5_max": -0.000342, "nauc_mrr_at_5_std": -0.105256, "nauc_mrr_at_5_diff1": 0.481775, "nauc_mrr_at_10_max": -0.005165, "nauc_mrr_at_10_std": -0.107629, "nauc_mrr_at_10_diff1": 0.475663, "nauc_mrr_at_20_max": -0.008623, "nauc_mrr_at_20_std": -0.10858, "nauc_mrr_at_20_diff1": 0.472906, "nauc_mrr_at_100_max": -0.009353, "nauc_mrr_at_100_std": -0.109132, "nauc_mrr_at_100_diff1": 0.472302, "nauc_mrr_at_1000_max": -0.009712, "nauc_mrr_at_1000_std": -0.109527, "nauc_mrr_at_1000_diff1": 0.472294, "hit_rate_at_1": 0.72989, "hit_rate_at_3": 0.88793, "hit_rate_at_5": 0.93103, "hit_rate_at_10": 0.96264, "hit_rate_at_20": 0.98563, "hit_rate_at_100": 0.99138, "hit_rate_at_1000": 1.0, "main_score": 0.71456, "hf_subset": "portuguese", "languages": [ "por-Latn" ], "mteb_version": "2.18.1" } ] }, "evaluation_time": 36639.80739412829, "kg_co2_emissions": null, "date": 1785365286.647369, "evaluation_phases": [ { "name": "Data loading", "start": 6.088241934776306e-05, "end": 94.9490556716919, "split": "", "subset": "" }, { "name": "Dataset transform", "start": 94.94908690080047, "end": 94.94909131154418, "split": "", "subset": "" }, { "name": "Encoding queries and documents", "start": 94.96621065586805, "end": 6487.577822797, "split": "test", "subset": "french" }, { "name": "Scoring", "start": 6487.5783656947315, "end": 6487.925114829093, "split": "test", "subset": "french" }, { "name": "Encoding queries and documents", "start": 6487.975813854486, "end": 12484.985231187195, "split": "test", "subset": "spanish" }, { "name": "Scoring", "start": 12484.985816467553, "end": 12485.318803414702, "split": "test", "subset": "spanish" }, { "name": "Encoding queries and documents", "start": 12485.358406893909, "end": 18434.54719876498, "split": "test", "subset": "english" }, { "name": "Scoring", "start": 18434.547681994736, "end": 18434.89515409991, "split": "test", "subset": "english" }, { "name": "Encoding queries and documents", "start": 18434.939401082695, "end": 24492.056753523648, "split": "test", "subset": "german" }, { "name": "Scoring", "start": 24492.05718325451, "end": 24492.392355468124, "split": "test", "subset": "german" }, { "name": "Encoding queries and documents", "start": 24492.432405613363, "end": 30824.108246445656, "split": "test", "subset": "italian" }, { "name": "Scoring", "start": 30824.108770359308, "end": 30824.449101854116, "split": "test", "subset": "italian" }, { "name": "Encoding queries and documents", "start": 30824.504249740392, "end": 36639.431239996105, "split": "test", "subset": "portuguese" }, { "name": "Scoring", "start": 36639.43169925362, "end": 36639.774854872376, "split": "test", "subset": "portuguese" } ] }