{ "dataset_revision": "057fe83342b446ede7928cb51bc6c7c3223da3d0", "task_name": "Vidore3TelecomRetrieval.v2", "mteb_version": "2.13.1", "scores": { "test": [ { "ndcg_at_1": 0.0273, "ndcg_at_3": 0.02814, "ndcg_at_5": 0.03211, "ndcg_at_10": 0.03686, "ndcg_at_20": 0.04627, "ndcg_at_100": 0.07108, "ndcg_at_1000": 0.13166, "map_at_1": 0.00744, "map_at_3": 0.01299, "map_at_5": 0.01567, "map_at_10": 0.01802, "map_at_20": 0.02079, "map_at_100": 0.0247, "map_at_1000": 0.02714, "recall_at_1": 0.00744, "recall_at_3": 0.02031, "recall_at_5": 0.03009, "recall_at_10": 0.04455, "recall_at_20": 0.07378, "recall_at_100": 0.17184, "recall_at_1000": 0.52865, "accuracy": 0.00744, "precision_at_1": 0.02874, "precision_at_3": 0.0249, "precision_at_5": 0.02414, "precision_at_10": 0.01782, "precision_at_20": 0.01422, "precision_at_100": 0.00707, "precision_at_1000": 0.00221, "mrr_at_1": 0.028736, "mrr_at_3": 0.047893, "mrr_at_5": 0.055939, "mrr_at_10": 0.0605, "mrr_at_20": 0.06485, "mrr_at_100": 0.068343, "mrr_at_1000": 0.069814, "nauc_ndcg_at_1_max": 0.026302, "nauc_ndcg_at_1_std": -0.009783, "nauc_ndcg_at_1_diff1": 0.085092, "nauc_ndcg_at_3_max": 0.026148, "nauc_ndcg_at_3_std": -0.074443, "nauc_ndcg_at_3_diff1": -0.079981, "nauc_ndcg_at_5_max": 0.011611, "nauc_ndcg_at_5_std": -0.035306, "nauc_ndcg_at_5_diff1": -0.044836, "nauc_ndcg_at_10_max": -0.001912, "nauc_ndcg_at_10_std": -0.000933, "nauc_ndcg_at_10_diff1": -0.028433, "nauc_ndcg_at_20_max": 0.008859, "nauc_ndcg_at_20_std": -0.003821, "nauc_ndcg_at_20_diff1": -0.010871, "nauc_ndcg_at_100_max": -0.002342, "nauc_ndcg_at_100_std": -0.02207, "nauc_ndcg_at_100_diff1": -0.019004, "nauc_ndcg_at_1000_max": 0.016327, "nauc_ndcg_at_1000_std": 0.004972, "nauc_ndcg_at_1000_diff1": -0.007962, "nauc_map_at_1_max": 0.065754, "nauc_map_at_1_std": -0.057796, "nauc_map_at_1_diff1": 0.188323, "nauc_map_at_3_max": 0.040503, "nauc_map_at_3_std": -0.114835, "nauc_map_at_3_diff1": -0.004392, "nauc_map_at_5_max": 0.042065, "nauc_map_at_5_std": -0.060842, "nauc_map_at_5_diff1": 0.015613, "nauc_map_at_10_max": 0.029469, "nauc_map_at_10_std": -0.026292, "nauc_map_at_10_diff1": 0.023703, "nauc_map_at_20_max": 0.030744, "nauc_map_at_20_std": -0.019398, "nauc_map_at_20_diff1": 0.034618, "nauc_map_at_100_max": 0.029597, "nauc_map_at_100_std": -0.012984, "nauc_map_at_100_diff1": 0.025277, "nauc_map_at_1000_max": 0.031123, "nauc_map_at_1000_std": -0.011091, "nauc_map_at_1000_diff1": 0.02616, "nauc_recall_at_1_max": 0.065754, "nauc_recall_at_1_std": -0.057796, "nauc_recall_at_1_diff1": 0.188323, "nauc_recall_at_3_max": 0.030221, "nauc_recall_at_3_std": -0.128127, "nauc_recall_at_3_diff1": -0.090121, "nauc_recall_at_5_max": 0.013444, "nauc_recall_at_5_std": -0.032913, "nauc_recall_at_5_diff1": -0.03568, "nauc_recall_at_10_max": -0.019349, "nauc_recall_at_10_std": 0.0472, "nauc_recall_at_10_diff1": 0.007696, "nauc_recall_at_20_max": -0.008313, "nauc_recall_at_20_std": 0.027644, "nauc_recall_at_20_diff1": 0.041857, "nauc_recall_at_100_max": -0.042921, "nauc_recall_at_100_std": -0.034484, "nauc_recall_at_100_diff1": -0.007671, "nauc_recall_at_1000_max": -0.009616, "nauc_recall_at_1000_std": 0.028459, "nauc_recall_at_1000_diff1": 0.030962, "nauc_precision_at_1_max": 0.058131, "nauc_precision_at_1_std": -0.012837, "nauc_precision_at_1_diff1": 0.060032, "nauc_precision_at_3_max": 0.076657, "nauc_precision_at_3_std": -0.054071, "nauc_precision_at_3_diff1": -0.087858, "nauc_precision_at_5_max": 0.03607, "nauc_precision_at_5_std": 0.024075, "nauc_precision_at_5_diff1": -0.03895, "nauc_precision_at_10_max": 0.006836, "nauc_precision_at_10_std": 0.071291, "nauc_precision_at_10_diff1": -0.024084, "nauc_precision_at_20_max": 0.036591, "nauc_precision_at_20_std": 0.080519, "nauc_precision_at_20_diff1": 0.020916, "nauc_precision_at_100_max": 0.025851, "nauc_precision_at_100_std": 0.046589, "nauc_precision_at_100_diff1": 0.006027, "nauc_precision_at_1000_max": 0.027248, "nauc_precision_at_1000_std": 0.081132, "nauc_precision_at_1000_diff1": 0.001453, "nauc_mrr_at_1_max": 0.058131, "nauc_mrr_at_1_std": -0.012837, "nauc_mrr_at_1_diff1": 0.060032, "nauc_mrr_at_3_max": 0.067759, "nauc_mrr_at_3_std": -0.046411, "nauc_mrr_at_3_diff1": -0.04478, "nauc_mrr_at_5_max": 0.04715, "nauc_mrr_at_5_std": -0.021104, "nauc_mrr_at_5_diff1": -0.040349, "nauc_mrr_at_10_max": 0.03561, "nauc_mrr_at_10_std": -0.018649, "nauc_mrr_at_10_diff1": -0.04218, "nauc_mrr_at_20_max": 0.035166, "nauc_mrr_at_20_std": -0.02273, "nauc_mrr_at_20_diff1": -0.041454, "nauc_mrr_at_100_max": 0.029867, "nauc_mrr_at_100_std": -0.027695, "nauc_mrr_at_100_diff1": -0.040852, "nauc_mrr_at_1000_max": 0.031054, "nauc_mrr_at_1000_std": -0.028207, "nauc_mrr_at_1000_diff1": -0.040991, "hit_rate_at_1": 0.02874, "hit_rate_at_3": 0.07471, "hit_rate_at_5": 0.1092, "hit_rate_at_10": 0.14368, "hit_rate_at_20": 0.2069, "hit_rate_at_100": 0.36494, "hit_rate_at_1000": 0.76149, "main_score": 0.03686, "hf_subset": "french", "languages": [ "fra-Latn" ], "mteb_version": "2.13.1" }, { "ndcg_at_1": 0.04167, "ndcg_at_3": 0.03297, "ndcg_at_5": 0.03315, "ndcg_at_10": 0.03773, "ndcg_at_20": 0.04525, "ndcg_at_100": 0.07013, "ndcg_at_1000": 0.13303, "map_at_1": 0.01371, "map_at_3": 0.01724, "map_at_5": 0.01881, "map_at_10": 0.02085, "map_at_20": 0.02312, "map_at_100": 0.02698, "map_at_1000": 0.02937, "recall_at_1": 0.01371, "recall_at_3": 0.0219, "recall_at_5": 0.02899, "recall_at_10": 0.04139, "recall_at_20": 0.06233, "recall_at_100": 0.15437, "recall_at_1000": 0.53162, "accuracy": 0.01371, "precision_at_1": 0.0431, "precision_at_3": 0.0249, "precision_at_5": 0.01897, "precision_at_10": 0.01609, "precision_at_20": 0.01322, "precision_at_100": 0.00693, "precision_at_1000": 0.00223, "mrr_at_1": 0.043103, "mrr_at_3": 0.055556, "mrr_at_5": 0.059866, "mrr_at_10": 0.065355, "mrr_at_20": 0.069092, "mrr_at_100": 0.072828, "mrr_at_1000": 0.074292, "nauc_ndcg_at_1_max": 0.091122, "nauc_ndcg_at_1_std": -0.089939, "nauc_ndcg_at_1_diff1": -0.051763, "nauc_ndcg_at_3_max": 0.127557, "nauc_ndcg_at_3_std": -0.054699, "nauc_ndcg_at_3_diff1": 0.014545, "nauc_ndcg_at_5_max": 0.085254, "nauc_ndcg_at_5_std": -0.060688, "nauc_ndcg_at_5_diff1": 0.010824, "nauc_ndcg_at_10_max": 0.099301, "nauc_ndcg_at_10_std": -0.04029, "nauc_ndcg_at_10_diff1": 0.05414, "nauc_ndcg_at_20_max": 0.095336, "nauc_ndcg_at_20_std": -0.028471, "nauc_ndcg_at_20_diff1": 0.061094, "nauc_ndcg_at_100_max": 0.138158, "nauc_ndcg_at_100_std": 0.029337, "nauc_ndcg_at_100_diff1": 0.076469, "nauc_ndcg_at_1000_max": 0.089994, "nauc_ndcg_at_1000_std": 0.045076, "nauc_ndcg_at_1000_diff1": 0.027401, "nauc_map_at_1_max": 0.134171, "nauc_map_at_1_std": -0.081874, "nauc_map_at_1_diff1": -0.003291, "nauc_map_at_3_max": 0.127792, "nauc_map_at_3_std": -0.063926, "nauc_map_at_3_diff1": 0.013783, "nauc_map_at_5_max": 0.093844, "nauc_map_at_5_std": -0.07814, "nauc_map_at_5_diff1": 0.013824, "nauc_map_at_10_max": 0.107671, "nauc_map_at_10_std": -0.071209, "nauc_map_at_10_diff1": 0.05135, "nauc_map_at_20_max": 0.126996, "nauc_map_at_20_std": -0.056773, "nauc_map_at_20_diff1": 0.063109, "nauc_map_at_100_max": 0.149457, "nauc_map_at_100_std": -0.015654, "nauc_map_at_100_diff1": 0.073096, "nauc_map_at_1000_max": 0.144791, "nauc_map_at_1000_std": -0.010741, "nauc_map_at_1000_diff1": 0.070299, "nauc_recall_at_1_max": 0.134171, "nauc_recall_at_1_std": -0.081874, "nauc_recall_at_1_diff1": -0.003291, "nauc_recall_at_3_max": 0.113001, "nauc_recall_at_3_std": -0.055342, "nauc_recall_at_3_diff1": 0.012527, "nauc_recall_at_5_max": 0.016138, "nauc_recall_at_5_std": -0.090428, "nauc_recall_at_5_diff1": 0.005095, "nauc_recall_at_10_max": 0.038145, "nauc_recall_at_10_std": -0.087333, "nauc_recall_at_10_diff1": 0.111599, "nauc_recall_at_20_max": 0.037943, "nauc_recall_at_20_std": -0.045587, "nauc_recall_at_20_diff1": 0.143852, "nauc_recall_at_100_max": 0.114045, "nauc_recall_at_100_std": 0.050182, "nauc_recall_at_100_diff1": 0.117681, "nauc_recall_at_1000_max": -0.015197, "nauc_recall_at_1000_std": 0.051118, "nauc_recall_at_1000_diff1": -0.038071, "nauc_precision_at_1_max": 0.110181, "nauc_precision_at_1_std": -0.083645, "nauc_precision_at_1_diff1": -0.052399, "nauc_precision_at_3_max": 0.198413, "nauc_precision_at_3_std": -0.007992, "nauc_precision_at_3_diff1": 0.041711, "nauc_precision_at_5_max": 0.148455, "nauc_precision_at_5_std": -0.002452, "nauc_precision_at_5_diff1": 0.041564, "nauc_precision_at_10_max": 0.187811, "nauc_precision_at_10_std": 0.046736, "nauc_precision_at_10_diff1": 0.127324, "nauc_precision_at_20_max": 0.175495, "nauc_precision_at_20_std": 0.056572, "nauc_precision_at_20_diff1": 0.069307, "nauc_precision_at_100_max": 0.19935, "nauc_precision_at_100_std": 0.150909, "nauc_precision_at_100_diff1": 0.099212, "nauc_precision_at_1000_max": 0.10411, "nauc_precision_at_1000_std": 0.155005, "nauc_precision_at_1000_diff1": 0.042103, "nauc_mrr_at_1_max": 0.110181, "nauc_mrr_at_1_std": -0.083645, "nauc_mrr_at_1_diff1": -0.052399, "nauc_mrr_at_3_max": 0.144939, "nauc_mrr_at_3_std": -0.040418, "nauc_mrr_at_3_diff1": 0.004326, "nauc_mrr_at_5_max": 0.133644, "nauc_mrr_at_5_std": -0.03844, "nauc_mrr_at_5_diff1": 0.00779, "nauc_mrr_at_10_max": 0.135399, "nauc_mrr_at_10_std": -0.029624, "nauc_mrr_at_10_diff1": 0.018548, "nauc_mrr_at_20_max": 0.118574, "nauc_mrr_at_20_std": -0.026612, "nauc_mrr_at_20_diff1": 0.009927, "nauc_mrr_at_100_max": 0.120346, "nauc_mrr_at_100_std": -0.024281, "nauc_mrr_at_100_diff1": 0.011541, "nauc_mrr_at_1000_max": 0.120613, "nauc_mrr_at_1000_std": -0.025273, "nauc_mrr_at_1000_diff1": 0.010918, "hit_rate_at_1": 0.0431, "hit_rate_at_3": 0.07184, "hit_rate_at_5": 0.09195, "hit_rate_at_10": 0.13218, "hit_rate_at_20": 0.18391, "hit_rate_at_100": 0.35632, "hit_rate_at_1000": 0.77586, "main_score": 0.03773, "hf_subset": "spanish", "languages": [ "spa-Latn" ], "mteb_version": "2.13.1" }, { "ndcg_at_1": 0.11494, "ndcg_at_3": 0.10842, "ndcg_at_5": 0.1111, "ndcg_at_10": 0.12127, "ndcg_at_20": 0.13851, "ndcg_at_100": 0.1845, "ndcg_at_1000": 0.25074, "map_at_1": 0.04295, "map_at_3": 0.06256, "map_at_5": 0.06902, "map_at_10": 0.07793, "map_at_20": 0.08517, "map_at_100": 0.09403, "map_at_1000": 0.09769, "recall_at_1": 0.04295, "recall_at_3": 0.07826, "recall_at_5": 0.09602, "recall_at_10": 0.13803, "recall_at_20": 0.19165, "recall_at_100": 0.36159, "recall_at_1000": 0.74043, "accuracy": 0.04295, "precision_at_1": 0.12356, "precision_at_3": 0.091, "precision_at_5": 0.07471, "precision_at_10": 0.05402, "precision_at_20": 0.03951, "precision_at_100": 0.01566, "precision_at_1000": 0.00323, "mrr_at_1": 0.123563, "mrr_at_3": 0.166188, "mrr_at_5": 0.174377, "mrr_at_10": 0.18346, "mrr_at_20": 0.190225, "mrr_at_100": 0.194914, "mrr_at_1000": 0.196032, "nauc_ndcg_at_1_max": 0.082624, "nauc_ndcg_at_1_std": -0.030223, "nauc_ndcg_at_1_diff1": 0.186893, "nauc_ndcg_at_3_max": 0.102273, "nauc_ndcg_at_3_std": 0.033606, "nauc_ndcg_at_3_diff1": 0.187432, "nauc_ndcg_at_5_max": 0.109028, "nauc_ndcg_at_5_std": 0.070795, "nauc_ndcg_at_5_diff1": 0.198788, "nauc_ndcg_at_10_max": 0.110698, "nauc_ndcg_at_10_std": 0.084464, "nauc_ndcg_at_10_diff1": 0.190789, "nauc_ndcg_at_20_max": 0.109182, "nauc_ndcg_at_20_std": 0.098918, "nauc_ndcg_at_20_diff1": 0.180853, "nauc_ndcg_at_100_max": 0.142325, "nauc_ndcg_at_100_std": 0.131065, "nauc_ndcg_at_100_diff1": 0.149391, "nauc_ndcg_at_1000_max": 0.150928, "nauc_ndcg_at_1000_std": 0.169677, "nauc_ndcg_at_1000_diff1": 0.159938, "nauc_map_at_1_max": 0.063156, "nauc_map_at_1_std": -0.049546, "nauc_map_at_1_diff1": 0.191118, "nauc_map_at_3_max": 0.054767, "nauc_map_at_3_std": -0.01396, "nauc_map_at_3_diff1": 0.18338, "nauc_map_at_5_max": 0.07111, "nauc_map_at_5_std": 0.0094, "nauc_map_at_5_diff1": 0.194392, "nauc_map_at_10_max": 0.087794, "nauc_map_at_10_std": 0.037234, "nauc_map_at_10_diff1": 0.194021, "nauc_map_at_20_max": 0.097138, "nauc_map_at_20_std": 0.058095, "nauc_map_at_20_diff1": 0.196974, "nauc_map_at_100_max": 0.111493, "nauc_map_at_100_std": 0.079532, "nauc_map_at_100_diff1": 0.184164, "nauc_map_at_1000_max": 0.11551, "nauc_map_at_1000_std": 0.087994, "nauc_map_at_1000_diff1": 0.185196, "nauc_recall_at_1_max": 0.063156, "nauc_recall_at_1_std": -0.049546, "nauc_recall_at_1_diff1": 0.191118, "nauc_recall_at_3_max": 0.063726, "nauc_recall_at_3_std": 0.004124, "nauc_recall_at_3_diff1": 0.168305, "nauc_recall_at_5_max": 0.086693, "nauc_recall_at_5_std": 0.040346, "nauc_recall_at_5_diff1": 0.179004, "nauc_recall_at_10_max": 0.110664, "nauc_recall_at_10_std": 0.085425, "nauc_recall_at_10_diff1": 0.145341, "nauc_recall_at_20_max": 0.106639, "nauc_recall_at_20_std": 0.115613, "nauc_recall_at_20_diff1": 0.13378, "nauc_recall_at_100_max": 0.170695, "nauc_recall_at_100_std": 0.143049, "nauc_recall_at_100_diff1": 0.052569, "nauc_recall_at_1000_max": 0.159145, "nauc_recall_at_1000_std": 0.28665, "nauc_recall_at_1000_diff1": -0.002754, "nauc_precision_at_1_max": 0.110108, "nauc_precision_at_1_std": -0.002091, "nauc_precision_at_1_diff1": 0.232564, "nauc_precision_at_3_max": 0.151831, "nauc_precision_at_3_std": 0.127658, "nauc_precision_at_3_diff1": 0.196792, "nauc_precision_at_5_max": 0.194059, "nauc_precision_at_5_std": 0.197266, "nauc_precision_at_5_diff1": 0.222383, "nauc_precision_at_10_max": 0.211109, "nauc_precision_at_10_std": 0.232333, "nauc_precision_at_10_diff1": 0.18479, "nauc_precision_at_20_max": 0.220182, "nauc_precision_at_20_std": 0.28473, "nauc_precision_at_20_diff1": 0.161207, "nauc_precision_at_100_max": 0.239322, "nauc_precision_at_100_std": 0.319647, "nauc_precision_at_100_diff1": 0.059893, "nauc_precision_at_1000_max": 0.174848, "nauc_precision_at_1000_std": 0.27972, "nauc_precision_at_1000_diff1": 0.075543, "nauc_mrr_at_1_max": 0.110108, "nauc_mrr_at_1_std": -0.002091, "nauc_mrr_at_1_diff1": 0.232564, "nauc_mrr_at_3_max": 0.13328, "nauc_mrr_at_3_std": 0.065126, "nauc_mrr_at_3_diff1": 0.221161, "nauc_mrr_at_5_max": 0.139899, "nauc_mrr_at_5_std": 0.071959, "nauc_mrr_at_5_diff1": 0.224607, "nauc_mrr_at_10_max": 0.144542, "nauc_mrr_at_10_std": 0.07712, "nauc_mrr_at_10_diff1": 0.213289, "nauc_mrr_at_20_max": 0.141089, "nauc_mrr_at_20_std": 0.082908, "nauc_mrr_at_20_diff1": 0.206111, "nauc_mrr_at_100_max": 0.141836, "nauc_mrr_at_100_std": 0.080688, "nauc_mrr_at_100_diff1": 0.20481, "nauc_mrr_at_1000_max": 0.141829, "nauc_mrr_at_1000_std": 0.080725, "nauc_mrr_at_1000_diff1": 0.205905, "hit_rate_at_1": 0.12356, "hit_rate_at_3": 0.22701, "hit_rate_at_5": 0.26437, "hit_rate_at_10": 0.33333, "hit_rate_at_20": 0.42529, "hit_rate_at_100": 0.62644, "hit_rate_at_1000": 0.92816, "main_score": 0.12127, "hf_subset": "english", "languages": [ "eng-Latn" ], "mteb_version": "2.13.1" }, { "ndcg_at_1": 0.01293, "ndcg_at_3": 0.01694, "ndcg_at_5": 0.01855, "ndcg_at_10": 0.02288, "ndcg_at_20": 0.02929, "ndcg_at_100": 0.05159, "ndcg_at_1000": 0.10832, "map_at_1": 0.00386, "map_at_3": 0.00708, "map_at_5": 0.0088, "map_at_10": 0.01095, "map_at_20": 0.01265, "map_at_100": 0.01615, "map_at_1000": 0.01827, "recall_at_1": 0.00386, "recall_at_3": 0.01103, "recall_at_5": 0.01689, "recall_at_10": 0.03107, "recall_at_20": 0.04994, "recall_at_100": 0.13371, "recall_at_1000": 0.46321, "accuracy": 0.00386, "precision_at_1": 0.01724, "precision_at_3": 0.01724, "precision_at_5": 0.01667, "precision_at_10": 0.01351, "precision_at_20": 0.01135, "precision_at_100": 0.00572, "precision_at_1000": 0.00198, "mrr_at_1": 0.017241, "mrr_at_3": 0.03113, "mrr_at_5": 0.03501, "mrr_at_10": 0.038761, "mrr_at_20": 0.042783, "mrr_at_100": 0.046357, "mrr_at_1000": 0.048019, "nauc_ndcg_at_1_max": -0.147617, "nauc_ndcg_at_1_std": -0.199384, "nauc_ndcg_at_1_diff1": 0.001471, "nauc_ndcg_at_3_max": -0.070001, "nauc_ndcg_at_3_std": -0.057956, "nauc_ndcg_at_3_diff1": -0.096648, "nauc_ndcg_at_5_max": -0.017816, "nauc_ndcg_at_5_std": -0.049102, "nauc_ndcg_at_5_diff1": -0.102857, "nauc_ndcg_at_10_max": -0.054328, "nauc_ndcg_at_10_std": -0.054096, "nauc_ndcg_at_10_diff1": -0.091894, "nauc_ndcg_at_20_max": 0.006748, "nauc_ndcg_at_20_std": 0.00594, "nauc_ndcg_at_20_diff1": -0.082816, "nauc_ndcg_at_100_max": 0.024795, "nauc_ndcg_at_100_std": -0.004291, "nauc_ndcg_at_100_diff1": -0.06537, "nauc_ndcg_at_1000_max": -0.017643, "nauc_ndcg_at_1000_std": -0.057551, "nauc_ndcg_at_1000_diff1": -0.046957, "nauc_map_at_1_max": -0.197599, "nauc_map_at_1_std": -0.388616, "nauc_map_at_1_diff1": -0.269581, "nauc_map_at_3_max": -0.135328, "nauc_map_at_3_std": -0.192328, "nauc_map_at_3_diff1": -0.230735, "nauc_map_at_5_max": -0.103633, "nauc_map_at_5_std": -0.168453, "nauc_map_at_5_diff1": -0.217128, "nauc_map_at_10_max": -0.105131, "nauc_map_at_10_std": -0.150608, "nauc_map_at_10_diff1": -0.168983, "nauc_map_at_20_max": -0.053597, "nauc_map_at_20_std": -0.092557, "nauc_map_at_20_diff1": -0.162523, "nauc_map_at_100_max": -0.027436, "nauc_map_at_100_std": -0.058853, "nauc_map_at_100_diff1": -0.153271, "nauc_map_at_1000_max": -0.02416, "nauc_map_at_1000_std": -0.059193, "nauc_map_at_1000_diff1": -0.143308, "nauc_recall_at_1_max": -0.197599, "nauc_recall_at_1_std": -0.388616, "nauc_recall_at_1_diff1": -0.269581, "nauc_recall_at_3_max": -0.105848, "nauc_recall_at_3_std": -0.070064, "nauc_recall_at_3_diff1": -0.225606, "nauc_recall_at_5_max": -0.026403, "nauc_recall_at_5_std": -0.004622, "nauc_recall_at_5_diff1": -0.185285, "nauc_recall_at_10_max": -0.060565, "nauc_recall_at_10_std": -0.054224, "nauc_recall_at_10_diff1": -0.097745, "nauc_recall_at_20_max": 0.061514, "nauc_recall_at_20_std": 0.059365, "nauc_recall_at_20_diff1": -0.081842, "nauc_recall_at_100_max": 0.020894, "nauc_recall_at_100_std": -0.002969, "nauc_recall_at_100_diff1": -0.053775, "nauc_recall_at_1000_max": -0.091719, "nauc_recall_at_1000_std": -0.128288, "nauc_recall_at_1000_diff1": -0.021922, "nauc_precision_at_1_max": -0.14525, "nauc_precision_at_1_std": -0.272214, "nauc_precision_at_1_diff1": -0.094581, "nauc_precision_at_3_max": -0.036933, "nauc_precision_at_3_std": -0.044957, "nauc_precision_at_3_diff1": -0.120317, "nauc_precision_at_5_max": 0.038869, "nauc_precision_at_5_std": -0.018522, "nauc_precision_at_5_diff1": -0.094549, "nauc_precision_at_10_max": 0.01629, "nauc_precision_at_10_std": -0.015867, "nauc_precision_at_10_diff1": -0.099375, "nauc_precision_at_20_max": 0.116481, "nauc_precision_at_20_std": 0.048215, "nauc_precision_at_20_diff1": -0.07045, "nauc_precision_at_100_max": 0.105277, "nauc_precision_at_100_std": 0.040918, "nauc_precision_at_100_diff1": -0.042498, "nauc_precision_at_1000_max": 0.061877, "nauc_precision_at_1000_std": -0.01672, "nauc_precision_at_1000_diff1": 0.01881, "nauc_mrr_at_1_max": -0.14525, "nauc_mrr_at_1_std": -0.272214, "nauc_mrr_at_1_diff1": -0.094581, "nauc_mrr_at_3_max": -0.051164, "nauc_mrr_at_3_std": -0.115499, "nauc_mrr_at_3_diff1": -0.094931, "nauc_mrr_at_5_max": -0.011575, "nauc_mrr_at_5_std": -0.067488, "nauc_mrr_at_5_diff1": -0.073315, "nauc_mrr_at_10_max": -0.03726, "nauc_mrr_at_10_std": -0.075421, "nauc_mrr_at_10_diff1": -0.084046, "nauc_mrr_at_20_max": -0.012927, "nauc_mrr_at_20_std": -0.063282, "nauc_mrr_at_20_diff1": -0.069871, "nauc_mrr_at_100_max": -0.006399, "nauc_mrr_at_100_std": -0.060753, "nauc_mrr_at_100_diff1": -0.066014, "nauc_mrr_at_1000_max": -0.011355, "nauc_mrr_at_1000_std": -0.064437, "nauc_mrr_at_1000_diff1": -0.067031, "hit_rate_at_1": 0.01724, "hit_rate_at_3": 0.04885, "hit_rate_at_5": 0.06609, "hit_rate_at_10": 0.0977, "hit_rate_at_20": 0.15805, "hit_rate_at_100": 0.30747, "hit_rate_at_1000": 0.73851, "main_score": 0.02288, "hf_subset": "german", "languages": [ "deu-Latn" ], "mteb_version": "2.13.1" }, { "ndcg_at_1": 0.03879, "ndcg_at_3": 0.0312, "ndcg_at_5": 0.03444, "ndcg_at_10": 0.03864, "ndcg_at_20": 0.04562, "ndcg_at_100": 0.07404, "ndcg_at_1000": 0.13249, "map_at_1": 0.01531, "map_at_3": 0.01858, "map_at_5": 0.02087, "map_at_10": 0.02284, "map_at_20": 0.02476, "map_at_100": 0.02916, "map_at_1000": 0.03154, "recall_at_1": 0.01531, "recall_at_3": 0.02248, "recall_at_5": 0.03105, "recall_at_10": 0.04312, "recall_at_20": 0.06522, "recall_at_100": 0.17529, "recall_at_1000": 0.51605, "accuracy": 0.01531, "precision_at_1": 0.04023, "precision_at_3": 0.02299, "precision_at_5": 0.02184, "precision_at_10": 0.01695, "precision_at_20": 0.01307, "precision_at_100": 0.00716, "precision_at_1000": 0.00223, "mrr_at_1": 0.04023, "mrr_at_3": 0.051724, "mrr_at_5": 0.057759, "mrr_at_10": 0.063348, "mrr_at_20": 0.066527, "mrr_at_100": 0.071069, "mrr_at_1000": 0.072462, "nauc_ndcg_at_1_max": -0.235167, "nauc_ndcg_at_1_std": -0.473807, "nauc_ndcg_at_1_diff1": 0.186897, "nauc_ndcg_at_3_max": -0.14298, "nauc_ndcg_at_3_std": -0.321891, "nauc_ndcg_at_3_diff1": 0.14704, "nauc_ndcg_at_5_max": -0.12563, "nauc_ndcg_at_5_std": -0.279823, "nauc_ndcg_at_5_diff1": 0.170573, "nauc_ndcg_at_10_max": -0.061772, "nauc_ndcg_at_10_std": -0.152404, "nauc_ndcg_at_10_diff1": 0.185622, "nauc_ndcg_at_20_max": -0.092701, "nauc_ndcg_at_20_std": -0.161043, "nauc_ndcg_at_20_diff1": 0.121853, "nauc_ndcg_at_100_max": -0.000522, "nauc_ndcg_at_100_std": -0.099324, "nauc_ndcg_at_100_diff1": 0.099199, "nauc_ndcg_at_1000_max": 0.001819, "nauc_ndcg_at_1000_std": -0.092306, "nauc_ndcg_at_1000_diff1": 0.115593, "nauc_map_at_1_max": -0.302999, "nauc_map_at_1_std": -0.493348, "nauc_map_at_1_diff1": 0.195841, "nauc_map_at_3_max": -0.235882, "nauc_map_at_3_std": -0.359915, "nauc_map_at_3_diff1": 0.174336, "nauc_map_at_5_max": -0.208351, "nauc_map_at_5_std": -0.326325, "nauc_map_at_5_diff1": 0.184889, "nauc_map_at_10_max": -0.162087, "nauc_map_at_10_std": -0.264512, "nauc_map_at_10_diff1": 0.193327, "nauc_map_at_20_max": -0.163711, "nauc_map_at_20_std": -0.255066, "nauc_map_at_20_diff1": 0.168637, "nauc_map_at_100_max": -0.123043, "nauc_map_at_100_std": -0.20829, "nauc_map_at_100_diff1": 0.149299, "nauc_map_at_1000_max": -0.11422, "nauc_map_at_1000_std": -0.201184, "nauc_map_at_1000_diff1": 0.152434, "nauc_recall_at_1_max": -0.302999, "nauc_recall_at_1_std": -0.493348, "nauc_recall_at_1_diff1": 0.195841, "nauc_recall_at_3_max": -0.190112, "nauc_recall_at_3_std": -0.261114, "nauc_recall_at_3_diff1": 0.152545, "nauc_recall_at_5_max": -0.16041, "nauc_recall_at_5_std": -0.202183, "nauc_recall_at_5_diff1": 0.192897, "nauc_recall_at_10_max": -0.001757, "nauc_recall_at_10_std": -0.010953, "nauc_recall_at_10_diff1": 0.229908, "nauc_recall_at_20_max": -0.072944, "nauc_recall_at_20_std": -0.082202, "nauc_recall_at_20_diff1": 0.094737, "nauc_recall_at_100_max": 0.087671, "nauc_recall_at_100_std": -0.018444, "nauc_recall_at_100_diff1": 0.059287, "nauc_recall_at_1000_max": 0.063203, "nauc_recall_at_1000_std": -0.032411, "nauc_recall_at_1000_diff1": 0.094224, "nauc_precision_at_1_max": -0.223236, "nauc_precision_at_1_std": -0.464259, "nauc_precision_at_1_diff1": 0.203897, "nauc_precision_at_3_max": -0.046613, "nauc_precision_at_3_std": -0.204311, "nauc_precision_at_3_diff1": 0.133261, "nauc_precision_at_5_max": 0.003381, "nauc_precision_at_5_std": -0.1433, "nauc_precision_at_5_diff1": 0.173755, "nauc_precision_at_10_max": 0.068509, "nauc_precision_at_10_std": 0.046304, "nauc_precision_at_10_diff1": 0.178957, "nauc_precision_at_20_max": 0.013719, "nauc_precision_at_20_std": 0.019256, "nauc_precision_at_20_diff1": 0.06925, "nauc_precision_at_100_max": 0.105967, "nauc_precision_at_100_std": 0.057191, "nauc_precision_at_100_diff1": 0.066623, "nauc_precision_at_1000_max": 0.06525, "nauc_precision_at_1000_std": 0.049673, "nauc_precision_at_1000_diff1": 0.131259, "nauc_mrr_at_1_max": -0.223236, "nauc_mrr_at_1_std": -0.464259, "nauc_mrr_at_1_diff1": 0.203897, "nauc_mrr_at_3_max": -0.134811, "nauc_mrr_at_3_std": -0.306228, "nauc_mrr_at_3_diff1": 0.168921, "nauc_mrr_at_5_max": -0.126523, "nauc_mrr_at_5_std": -0.268902, "nauc_mrr_at_5_diff1": 0.177028, "nauc_mrr_at_10_max": -0.101927, "nauc_mrr_at_10_std": -0.210727, "nauc_mrr_at_10_diff1": 0.176971, "nauc_mrr_at_20_max": -0.108512, "nauc_mrr_at_20_std": -0.213038, "nauc_mrr_at_20_diff1": 0.161344, "nauc_mrr_at_100_max": -0.100342, "nauc_mrr_at_100_std": -0.208962, "nauc_mrr_at_100_diff1": 0.155782, "nauc_mrr_at_1000_max": -0.100414, "nauc_mrr_at_1000_std": -0.210464, "nauc_mrr_at_1000_diff1": 0.155689, "hit_rate_at_1": 0.04023, "hit_rate_at_3": 0.06609, "hit_rate_at_5": 0.09483, "hit_rate_at_10": 0.13506, "hit_rate_at_20": 0.18391, "hit_rate_at_100": 0.37069, "hit_rate_at_1000": 0.75287, "main_score": 0.03864, "hf_subset": "italian", "languages": [ "ita-Latn" ], "mteb_version": "2.13.1" }, { "ndcg_at_1": 0.02443, "ndcg_at_3": 0.0259, "ndcg_at_5": 0.02745, "ndcg_at_10": 0.02964, "ndcg_at_20": 0.03529, "ndcg_at_100": 0.06785, "ndcg_at_1000": 0.12792, "map_at_1": 0.00839, "map_at_3": 0.0135, "map_at_5": 0.01499, "map_at_10": 0.0164, "map_at_20": 0.01786, "map_at_100": 0.02255, "map_at_1000": 0.02496, "recall_at_1": 0.00839, "recall_at_3": 0.01912, "recall_at_5": 0.02531, "recall_at_10": 0.03526, "recall_at_20": 0.05116, "recall_at_100": 0.17458, "recall_at_1000": 0.52982, "accuracy": 0.00839, "precision_at_1": 0.02586, "precision_at_3": 0.02107, "precision_at_5": 0.01782, "precision_at_10": 0.01322, "precision_at_20": 0.01006, "precision_at_100": 0.00721, "precision_at_1000": 0.00226, "mrr_at_1": 0.025862, "mrr_at_3": 0.043103, "mrr_at_5": 0.047701, "mrr_at_10": 0.052127, "mrr_at_20": 0.054936, "mrr_at_100": 0.060336, "mrr_at_1000": 0.061662, "nauc_ndcg_at_1_max": -0.084338, "nauc_ndcg_at_1_std": -0.330672, "nauc_ndcg_at_1_diff1": -0.009163, "nauc_ndcg_at_3_max": -0.038541, "nauc_ndcg_at_3_std": -0.232429, "nauc_ndcg_at_3_diff1": -0.041614, "nauc_ndcg_at_5_max": -0.046243, "nauc_ndcg_at_5_std": -0.197792, "nauc_ndcg_at_5_diff1": -0.025304, "nauc_ndcg_at_10_max": 0.005633, "nauc_ndcg_at_10_std": -0.172134, "nauc_ndcg_at_10_diff1": -0.018413, "nauc_ndcg_at_20_max": 0.039762, "nauc_ndcg_at_20_std": -0.125661, "nauc_ndcg_at_20_diff1": -0.001842, "nauc_ndcg_at_100_max": 0.076051, "nauc_ndcg_at_100_std": -0.063706, "nauc_ndcg_at_100_diff1": 0.013509, "nauc_ndcg_at_1000_max": 0.057865, "nauc_ndcg_at_1000_std": -0.052642, "nauc_ndcg_at_1000_diff1": 0.002176, "nauc_map_at_1_max": -0.105073, "nauc_map_at_1_std": -0.396129, "nauc_map_at_1_diff1": -0.051102, "nauc_map_at_3_max": -0.025727, "nauc_map_at_3_std": -0.334821, "nauc_map_at_3_diff1": -0.072517, "nauc_map_at_5_max": -0.04657, "nauc_map_at_5_std": -0.298126, "nauc_map_at_5_diff1": -0.061527, "nauc_map_at_10_max": -0.016505, "nauc_map_at_10_std": -0.261578, "nauc_map_at_10_diff1": -0.049338, "nauc_map_at_20_max": 0.000513, "nauc_map_at_20_std": -0.232315, "nauc_map_at_20_diff1": -0.032049, "nauc_map_at_100_max": 0.037235, "nauc_map_at_100_std": -0.166375, "nauc_map_at_100_diff1": -0.027142, "nauc_map_at_1000_max": 0.037793, "nauc_map_at_1000_std": -0.149479, "nauc_map_at_1000_diff1": -0.026766, "nauc_recall_at_1_max": -0.105073, "nauc_recall_at_1_std": -0.396129, "nauc_recall_at_1_diff1": -0.051102, "nauc_recall_at_3_max": 0.021339, "nauc_recall_at_3_std": -0.281401, "nauc_recall_at_3_diff1": -0.08909, "nauc_recall_at_5_max": -0.048883, "nauc_recall_at_5_std": -0.21834, "nauc_recall_at_5_diff1": -0.068478, "nauc_recall_at_10_max": 0.023528, "nauc_recall_at_10_std": -0.137969, "nauc_recall_at_10_diff1": -0.028875, "nauc_recall_at_20_max": 0.078699, "nauc_recall_at_20_std": -0.051933, "nauc_recall_at_20_diff1": 0.021549, "nauc_recall_at_100_max": 0.097623, "nauc_recall_at_100_std": -0.01634, "nauc_recall_at_100_diff1": 0.019836, "nauc_recall_at_1000_max": 0.031861, "nauc_recall_at_1000_std": -0.054704, "nauc_recall_at_1000_diff1": -0.025264, "nauc_precision_at_1_max": -0.042825, "nauc_precision_at_1_std": -0.306807, "nauc_precision_at_1_diff1": -0.036486, "nauc_precision_at_3_max": -0.006813, "nauc_precision_at_3_std": -0.141533, "nauc_precision_at_3_diff1": -0.038813, "nauc_precision_at_5_max": -0.020435, "nauc_precision_at_5_std": -0.041685, "nauc_precision_at_5_diff1": 0.020755, "nauc_precision_at_10_max": 0.062534, "nauc_precision_at_10_std": 0.008201, "nauc_precision_at_10_diff1": 0.022233, "nauc_precision_at_20_max": 0.104086, "nauc_precision_at_20_std": 0.089942, "nauc_precision_at_20_diff1": 0.037573, "nauc_precision_at_100_max": 0.127572, "nauc_precision_at_100_std": 0.143775, "nauc_precision_at_100_diff1": 0.037819, "nauc_precision_at_1000_max": 0.070604, "nauc_precision_at_1000_std": 0.137787, "nauc_precision_at_1000_diff1": 0.044037, "nauc_mrr_at_1_max": -0.042825, "nauc_mrr_at_1_std": -0.306807, "nauc_mrr_at_1_diff1": -0.036486, "nauc_mrr_at_3_max": -0.025521, "nauc_mrr_at_3_std": -0.209142, "nauc_mrr_at_3_diff1": -0.033316, "nauc_mrr_at_5_max": -0.028718, "nauc_mrr_at_5_std": -0.178415, "nauc_mrr_at_5_diff1": -0.017231, "nauc_mrr_at_10_max": -0.011034, "nauc_mrr_at_10_std": -0.167528, "nauc_mrr_at_10_diff1": -0.011571, "nauc_mrr_at_20_max": 0.000903, "nauc_mrr_at_20_std": -0.14764, "nauc_mrr_at_20_diff1": -0.011275, "nauc_mrr_at_100_max": 0.001155, "nauc_mrr_at_100_std": -0.149316, "nauc_mrr_at_100_diff1": -0.00214, "nauc_mrr_at_1000_max": 0.001671, "nauc_mrr_at_1000_std": -0.150181, "nauc_mrr_at_1000_diff1": -0.001939, "hit_rate_at_1": 0.02586, "hit_rate_at_3": 0.06322, "hit_rate_at_5": 0.08333, "hit_rate_at_10": 0.11782, "hit_rate_at_20": 0.15805, "hit_rate_at_100": 0.37356, "hit_rate_at_1000": 0.76437, "main_score": 0.02964, "hf_subset": "portuguese", "languages": [ "por-Latn" ], "mteb_version": "2.13.1" } ] }, "evaluation_time": 5521.392271775985, "kg_co2_emissions": null, "date": 1783100084.895293, "evaluation_phases": [ { "name": "Data loading", "start": 7.872004061937332e-05, "end": 71.23414466995746, "split": "", "subset": "" }, { "name": "Dataset transform", "start": 71.23420319100842, "end": 71.23420649091713, "split": "", "subset": "" }, { "name": "Encoding queries and documents", "start": 71.24894611607306, "end": 1062.7279170278925, "split": "test", "subset": "french" }, { "name": "Scoring", "start": 1062.728158120066, "end": 1063.135076483013, "split": "test", "subset": "french" }, { "name": "Encoding queries and documents", "start": 1063.1784245730378, "end": 2015.677591163898, "split": "test", "subset": "spanish" }, { "name": "Scoring", "start": 2015.6780377388932, "end": 2016.062509570038, "split": "test", "subset": "spanish" }, { "name": "Encoding queries and documents", "start": 2016.1032581289764, "end": 2914.5120519010816, "split": "test", "subset": "english" }, { "name": "Scoring", "start": 2914.5122183929197, "end": 2914.839327554917, "split": "test", "subset": "english" }, { "name": "Encoding queries and documents", "start": 2914.8758506870363, "end": 3800.825627167942, "split": "test", "subset": "german" }, { "name": "Scoring", "start": 3800.825963101117, "end": 3801.1909651220776, "split": "test", "subset": "german" }, { "name": "Encoding queries and documents", "start": 3801.237195552094, "end": 4660.53919879999, "split": "test", "subset": "italian" }, { "name": "Scoring", "start": 4660.539533372968, "end": 4660.878988221986, "split": "test", "subset": "italian" }, { "name": "Encoding queries and documents", "start": 4660.918155787047, "end": 5521.018428649055, "split": "test", "subset": "portuguese" }, { "name": "Scoring", "start": 5521.018742971122, "end": 5521.356450512074, "split": "test", "subset": "portuguese" } ] }