{ "dataset_revision": "df52bb043564336892a1eb9127559130fcb0e62c", "task_name": "LooGLE", "mteb_version": "2.3.0", "scores": { "test": [ { "ndcg_at_1": 0.33878, "ndcg_at_3": 0.31179, "ndcg_at_5": 0.34047, "ndcg_at_10": 0.39216, "ndcg_at_20": 0.43831, "ndcg_at_25": 0.45306, "ndcg_at_50": 0.49271, "ndcg_at_100": 0.52102, "ndcg_at_1000": 0.53243, "map_at_1": 0.1395, "map_at_3": 0.22777, "map_at_5": 0.2613, "map_at_10": 0.28938, "map_at_20": 0.30814, "map_at_25": 0.31275, "map_at_50": 0.323, "map_at_100": 0.32777, "map_at_1000": 0.32912, "recall_at_1": 0.1395, "recall_at_3": 0.28403, "recall_at_5": 0.36738, "recall_at_10": 0.48688, "recall_at_20": 0.61935, "recall_at_25": 0.66763, "recall_at_50": 0.81262, "recall_at_100": 0.93637, "recall_at_1000": 0.99286, "accuracy": 0.1395, "precision_at_1": 0.33878, "precision_at_3": 0.24644, "precision_at_5": 0.196, "precision_at_10": 0.13315, "precision_at_20": 0.08647, "precision_at_25": 0.07517, "precision_at_50": 0.04694, "precision_at_100": 0.02731, "precision_at_1000": 0.00291, "mrr_at_1": 0.338783, "mrr_at_3": 0.435059, "mrr_at_5": 0.453951, "mrr_at_10": 0.471803, "mrr_at_20": 0.478068, "mrr_at_25": 0.479505, "mrr_at_50": 0.481539, "mrr_at_100": 0.48205, "mrr_at_1000": 0.482071, "nauc_ndcg_at_1_max": 0.184778, "nauc_ndcg_at_1_std": -0.004438, "nauc_ndcg_at_1_diff1": 0.320612, "nauc_ndcg_at_3_max": 0.190489, "nauc_ndcg_at_3_std": 0.007739, "nauc_ndcg_at_3_diff1": 0.272442, "nauc_ndcg_at_5_max": 0.19938, "nauc_ndcg_at_5_std": 0.038086, "nauc_ndcg_at_5_diff1": 0.274229, "nauc_ndcg_at_10_max": 0.207265, "nauc_ndcg_at_10_std": 0.05087, "nauc_ndcg_at_10_diff1": 0.277556, "nauc_ndcg_at_20_max": 0.226584, "nauc_ndcg_at_20_std": 0.054758, "nauc_ndcg_at_20_diff1": 0.288298, "nauc_ndcg_at_25_max": 0.229287, "nauc_ndcg_at_25_std": 0.053595, "nauc_ndcg_at_25_diff1": 0.286828, "nauc_ndcg_at_50_max": 0.240949, "nauc_ndcg_at_50_std": 0.05129, "nauc_ndcg_at_50_diff1": 0.278335, "nauc_ndcg_at_100_max": 0.231668, "nauc_ndcg_at_100_std": 0.039778, "nauc_ndcg_at_100_diff1": 0.27109, "nauc_ndcg_at_1000_max": 0.222401, "nauc_ndcg_at_1000_std": 0.028671, "nauc_ndcg_at_1000_diff1": 0.274113, "nauc_map_at_1_max": 0.152845, "nauc_map_at_1_std": 0.025203, "nauc_map_at_1_diff1": 0.320343, "nauc_map_at_3_max": 0.18089, "nauc_map_at_3_std": 0.023765, "nauc_map_at_3_diff1": 0.297856, "nauc_map_at_5_max": 0.190063, "nauc_map_at_5_std": 0.040302, "nauc_map_at_5_diff1": 0.289227, "nauc_map_at_10_max": 0.194491, "nauc_map_at_10_std": 0.046747, "nauc_map_at_10_diff1": 0.289003, "nauc_map_at_20_max": 0.204821, "nauc_map_at_20_std": 0.046864, "nauc_map_at_20_diff1": 0.292269, "nauc_map_at_25_max": 0.205761, "nauc_map_at_25_std": 0.046442, "nauc_map_at_25_diff1": 0.291633, "nauc_map_at_50_max": 0.208232, "nauc_map_at_50_std": 0.044968, "nauc_map_at_50_diff1": 0.2903, "nauc_map_at_100_max": 0.207518, "nauc_map_at_100_std": 0.043095, "nauc_map_at_100_diff1": 0.288828, "nauc_map_at_1000_max": 0.20669, "nauc_map_at_1000_std": 0.041876, "nauc_map_at_1000_diff1": 0.289082, "nauc_recall_at_1_max": 0.152845, "nauc_recall_at_1_std": 0.025203, "nauc_recall_at_1_diff1": 0.320343, "nauc_recall_at_3_max": 0.177432, "nauc_recall_at_3_std": 0.032257, "nauc_recall_at_3_diff1": 0.260617, "nauc_recall_at_5_max": 0.191484, "nauc_recall_at_5_std": 0.072705, "nauc_recall_at_5_diff1": 0.233143, "nauc_recall_at_10_max": 0.190982, "nauc_recall_at_10_std": 0.099774, "nauc_recall_at_10_diff1": 0.233417, "nauc_recall_at_20_max": 0.234057, "nauc_recall_at_20_std": 0.122855, "nauc_recall_at_20_diff1": 0.259081, "nauc_recall_at_25_max": 0.241495, "nauc_recall_at_25_std": 0.125827, "nauc_recall_at_25_diff1": 0.249263, "nauc_recall_at_50_max": 0.331637, "nauc_recall_at_50_std": 0.17244, "nauc_recall_at_50_diff1": 0.211313, "nauc_recall_at_100_max": 0.370684, "nauc_recall_at_100_std": 0.235911, "nauc_recall_at_100_diff1": 0.104865, "nauc_recall_at_1000_max": 0.340877, "nauc_recall_at_1000_std": 0.294775, "nauc_recall_at_1000_diff1": -0.214671, "nauc_precision_at_1_max": 0.184778, "nauc_precision_at_1_std": -0.004438, "nauc_precision_at_1_diff1": 0.320612, "nauc_precision_at_3_max": 0.19259, "nauc_precision_at_3_std": -0.017222, "nauc_precision_at_3_diff1": 0.200608, "nauc_precision_at_5_max": 0.194274, "nauc_precision_at_5_std": 0.011999, "nauc_precision_at_5_diff1": 0.162042, "nauc_precision_at_10_max": 0.19557, "nauc_precision_at_10_std": 0.016749, "nauc_precision_at_10_diff1": 0.12982, "nauc_precision_at_20_max": 0.204079, "nauc_precision_at_20_std": 0.010014, "nauc_precision_at_20_diff1": 0.096304, "nauc_precision_at_25_max": 0.193051, "nauc_precision_at_25_std": -0.000871, "nauc_precision_at_25_diff1": 0.074305, "nauc_precision_at_50_max": 0.161816, "nauc_precision_at_50_std": -0.039353, "nauc_precision_at_50_diff1": -0.020863, "nauc_precision_at_100_max": 0.088807, "nauc_precision_at_100_std": -0.094765, "nauc_precision_at_100_diff1": -0.109367, "nauc_precision_at_1000_max": 0.039091, "nauc_precision_at_1000_std": -0.135139, "nauc_precision_at_1000_diff1": -0.133363, "nauc_mrr_at_1_max": 0.184778, "nauc_mrr_at_1_std": -0.004438, "nauc_mrr_at_1_diff1": 0.320612, "nauc_mrr_at_3_max": 0.200076, "nauc_mrr_at_3_std": -0.00223, "nauc_mrr_at_3_diff1": 0.280649, "nauc_mrr_at_5_max": 0.200782, "nauc_mrr_at_5_std": 0.000462, "nauc_mrr_at_5_diff1": 0.274191, "nauc_mrr_at_10_max": 0.204151, "nauc_mrr_at_10_std": 0.002912, "nauc_mrr_at_10_diff1": 0.279238, "nauc_mrr_at_20_max": 0.204071, "nauc_mrr_at_20_std": 0.002495, "nauc_mrr_at_20_diff1": 0.282692, "nauc_mrr_at_25_max": 0.204343, "nauc_mrr_at_25_std": 0.002515, "nauc_mrr_at_25_diff1": 0.282479, "nauc_mrr_at_50_max": 0.203892, "nauc_mrr_at_50_std": 0.002784, "nauc_mrr_at_50_diff1": 0.282203, "nauc_mrr_at_100_max": 0.203221, "nauc_mrr_at_100_std": 0.002606, "nauc_mrr_at_100_diff1": 0.282371, "nauc_mrr_at_1000_max": 0.203178, "nauc_mrr_at_1000_std": 0.002611, "nauc_mrr_at_1000_diff1": 0.28235, "hit_rate_at_1": 0.33878, "hit_rate_at_3": 0.55677, "hit_rate_at_5": 0.63942, "hit_rate_at_10": 0.7693, "hit_rate_at_20": 0.86013, "hit_rate_at_25": 0.89282, "hit_rate_at_50": 0.96094, "hit_rate_at_100": 0.99546, "hit_rate_at_1000": 0.99818, "R_cap_at_1": 0.338783, "R_cap_at_3": 0.311989, "R_cap_at_5": 0.369709, "R_cap_at_10": 0.487175, "R_cap_at_20": 0.619392, "R_cap_at_25": 0.66763, "R_cap_at_50": 0.812625, "R_cap_at_100": 0.936367, "R_cap_at_1000": 0.992861, "main_score": 0.39216, "hf_subset": "LongDepQA", "languages": [ "eng-Latn" ], "mteb_version": "2.3.0" }, { "ndcg_at_1": 0.50692, "ndcg_at_3": 0.62216, "ndcg_at_5": 0.65273, "ndcg_at_10": 0.67896, "ndcg_at_20": 0.69491, "ndcg_at_25": 0.69772, "ndcg_at_50": 0.70562, "ndcg_at_100": 0.70947, "ndcg_at_1000": 0.70976, "map_at_1": 0.50692, "map_at_3": 0.59457, "map_at_5": 0.61151, "map_at_10": 0.62236, "map_at_20": 0.62667, "map_at_25": 0.62725, "map_at_50": 0.62845, "map_at_100": 0.62879, "map_at_1000": 0.62881, "recall_at_1": 0.50692, "recall_at_3": 0.70169, "recall_at_5": 0.77601, "recall_at_10": 0.857, "recall_at_20": 0.92055, "recall_at_25": 0.93337, "recall_at_50": 0.97386, "recall_at_100": 0.99744, "recall_at_1000": 0.99949, "accuracy": 0.50692, "precision_at_1": 0.50692, "precision_at_3": 0.2339, "precision_at_5": 0.1552, "precision_at_10": 0.0857, "precision_at_20": 0.04603, "precision_at_25": 0.03733, "precision_at_50": 0.01948, "precision_at_100": 0.00997, "precision_at_1000": 0.001, "mrr_at_1": 0.50692, "mrr_at_3": 0.594567, "mrr_at_5": 0.611507, "mrr_at_10": 0.622361, "mrr_at_20": 0.626674, "mrr_at_25": 0.627246, "mrr_at_50": 0.628451, "mrr_at_100": 0.628795, "mrr_at_1000": 0.628811, "nauc_ndcg_at_1_max": 0.118635, "nauc_ndcg_at_1_std": 0.056463, "nauc_ndcg_at_1_diff1": 0.617058, "nauc_ndcg_at_3_max": 0.135977, "nauc_ndcg_at_3_std": 0.088879, "nauc_ndcg_at_3_diff1": 0.589626, "nauc_ndcg_at_5_max": 0.145529, "nauc_ndcg_at_5_std": 0.104697, "nauc_ndcg_at_5_diff1": 0.588282, "nauc_ndcg_at_10_max": 0.147093, "nauc_ndcg_at_10_std": 0.111845, "nauc_ndcg_at_10_diff1": 0.58866, "nauc_ndcg_at_20_max": 0.138735, "nauc_ndcg_at_20_std": 0.105841, "nauc_ndcg_at_20_diff1": 0.587023, "nauc_ndcg_at_25_max": 0.138494, "nauc_ndcg_at_25_std": 0.105175, "nauc_ndcg_at_25_diff1": 0.587355, "nauc_ndcg_at_50_max": 0.14128, "nauc_ndcg_at_50_std": 0.103409, "nauc_ndcg_at_50_diff1": 0.592286, "nauc_ndcg_at_100_max": 0.138486, "nauc_ndcg_at_100_std": 0.095394, "nauc_ndcg_at_100_diff1": 0.593854, "nauc_ndcg_at_1000_max": 0.137449, "nauc_ndcg_at_1000_std": 0.094355, "nauc_ndcg_at_1000_diff1": 0.593806, "nauc_map_at_1_max": 0.118635, "nauc_map_at_1_std": 0.056463, "nauc_map_at_1_diff1": 0.617058, "nauc_map_at_3_max": 0.131055, "nauc_map_at_3_std": 0.079346, "nauc_map_at_3_diff1": 0.596803, "nauc_map_at_5_max": 0.1356, "nauc_map_at_5_std": 0.08723, "nauc_map_at_5_diff1": 0.596645, "nauc_map_at_10_max": 0.136336, "nauc_map_at_10_std": 0.089357, "nauc_map_at_10_diff1": 0.596896, "nauc_map_at_20_max": 0.13415, "nauc_map_at_20_std": 0.087475, "nauc_map_at_20_diff1": 0.596706, "nauc_map_at_25_max": 0.134097, "nauc_map_at_25_std": 0.087315, "nauc_map_at_25_diff1": 0.596794, "nauc_map_at_50_max": 0.134401, "nauc_map_at_50_std": 0.086962, "nauc_map_at_50_diff1": 0.597352, "nauc_map_at_100_max": 0.134233, "nauc_map_at_100_std": 0.086369, "nauc_map_at_100_diff1": 0.597444, "nauc_map_at_1000_max": 0.134188, "nauc_map_at_1000_std": 0.086323, "nauc_map_at_1000_diff1": 0.597442, "nauc_recall_at_1_max": 0.118635, "nauc_recall_at_1_std": 0.056463, "nauc_recall_at_1_diff1": 0.617058, "nauc_recall_at_3_max": 0.153057, "nauc_recall_at_3_std": 0.122118, "nauc_recall_at_3_diff1": 0.565011, "nauc_recall_at_5_max": 0.189444, "nauc_recall_at_5_std": 0.181191, "nauc_recall_at_5_diff1": 0.552763, "nauc_recall_at_10_max": 0.214245, "nauc_recall_at_10_std": 0.258035, "nauc_recall_at_10_diff1": 0.539036, "nauc_recall_at_20_max": 0.167492, "nauc_recall_at_20_std": 0.298636, "nauc_recall_at_20_diff1": 0.481693, "nauc_recall_at_25_max": 0.16927, "nauc_recall_at_25_std": 0.325505, "nauc_recall_at_25_diff1": 0.466235, "nauc_recall_at_50_max": 0.343567, "nauc_recall_at_50_std": 0.597047, "nauc_recall_at_50_diff1": 0.510111, "nauc_recall_at_100_max": 0.631683, "nauc_recall_at_100_std": 0.884701, "nauc_recall_at_100_diff1": 0.64944, "nauc_recall_at_1000_max": -0.563834, "nauc_recall_at_1000_std": 0.869288, "nauc_recall_at_1000_diff1": 0.722249, "nauc_precision_at_1_max": 0.118635, "nauc_precision_at_1_std": 0.056463, "nauc_precision_at_1_diff1": 0.617058, "nauc_precision_at_3_max": 0.153057, "nauc_precision_at_3_std": 0.122118, "nauc_precision_at_3_diff1": 0.565011, "nauc_precision_at_5_max": 0.189444, "nauc_precision_at_5_std": 0.181191, "nauc_precision_at_5_diff1": 0.552763, "nauc_precision_at_10_max": 0.214245, "nauc_precision_at_10_std": 0.258035, "nauc_precision_at_10_diff1": 0.539036, "nauc_precision_at_20_max": 0.167492, "nauc_precision_at_20_std": 0.298636, "nauc_precision_at_20_diff1": 0.481693, "nauc_precision_at_25_max": 0.16927, "nauc_precision_at_25_std": 0.325505, "nauc_precision_at_25_diff1": 0.466235, "nauc_precision_at_50_max": 0.343567, "nauc_precision_at_50_std": 0.597047, "nauc_precision_at_50_diff1": 0.510111, "nauc_precision_at_100_max": 0.631683, "nauc_precision_at_100_std": 0.884701, "nauc_precision_at_100_diff1": 0.64944, "nauc_precision_at_1000_max": -0.563834, "nauc_precision_at_1000_std": 0.869288, "nauc_precision_at_1000_diff1": 0.722249, "nauc_mrr_at_1_max": 0.118635, "nauc_mrr_at_1_std": 0.056463, "nauc_mrr_at_1_diff1": 0.617058, "nauc_mrr_at_3_max": 0.131055, "nauc_mrr_at_3_std": 0.079346, "nauc_mrr_at_3_diff1": 0.596803, "nauc_mrr_at_5_max": 0.1356, "nauc_mrr_at_5_std": 0.08723, "nauc_mrr_at_5_diff1": 0.596645, "nauc_mrr_at_10_max": 0.136336, "nauc_mrr_at_10_std": 0.089357, "nauc_mrr_at_10_diff1": 0.596896, "nauc_mrr_at_20_max": 0.13415, "nauc_mrr_at_20_std": 0.087475, "nauc_mrr_at_20_diff1": 0.596706, "nauc_mrr_at_25_max": 0.134097, "nauc_mrr_at_25_std": 0.087315, "nauc_mrr_at_25_diff1": 0.596794, "nauc_mrr_at_50_max": 0.134401, "nauc_mrr_at_50_std": 0.086962, "nauc_mrr_at_50_diff1": 0.597352, "nauc_mrr_at_100_max": 0.134233, "nauc_mrr_at_100_std": 0.086369, "nauc_mrr_at_100_diff1": 0.597444, "nauc_mrr_at_1000_max": 0.134188, "nauc_mrr_at_1000_std": 0.086323, "nauc_mrr_at_1000_diff1": 0.597442, "hit_rate_at_1": 0.50692, "hit_rate_at_3": 0.70169, "hit_rate_at_5": 0.77601, "hit_rate_at_10": 0.857, "hit_rate_at_20": 0.92055, "hit_rate_at_25": 0.93337, "hit_rate_at_50": 0.97386, "hit_rate_at_100": 0.99744, "hit_rate_at_1000": 0.99949, "R_cap_at_1": 0.50692, "R_cap_at_3": 0.701691, "R_cap_at_5": 0.776012, "R_cap_at_10": 0.856996, "R_cap_at_20": 0.920554, "R_cap_at_25": 0.933368, "R_cap_at_50": 0.97386, "R_cap_at_100": 0.997437, "R_cap_at_1000": 0.999487, "main_score": 0.67896, "hf_subset": "ShortDepQA", "languages": [ "eng-Latn" ], "mteb_version": "2.3.0" } ] }, "evaluation_time": 79.65111524425447, "kg_co2_emissions": null, "date": 1784277034.956629, "evaluation_phases": [ { "name": "Data loading", "start": 6.985850632190704e-06, "end": 18.191319367848337, "split": "", "subset": "" }, { "name": "Dataset transform", "start": 18.191336754709482, "end": 18.19134236406535, "split": "", "subset": "" }, { "name": "Encoding queries and documents", "start": 18.21831317897886, "end": 38.283391615375876, "split": "test", "subset": "ShortDepQA" }, { "name": "Scoring", "start": 38.28344385419041, "end": 38.6209986275062, "split": "test", "subset": "ShortDepQA" } ] }