{ "dataset_revision": "df52bb043564336892a1eb9127559130fcb0e62c", "task_name": "LooGLE", "mteb_version": "2.3.0", "scores": { "test": [ { "ndcg_at_1": 0.2198, "ndcg_at_3": 0.20234, "ndcg_at_5": 0.22138, "ndcg_at_10": 0.26005, "ndcg_at_20": 0.29493, "ndcg_at_25": 0.30609, "ndcg_at_50": 0.33868, "ndcg_at_100": 0.36156, "ndcg_at_1000": 0.39257, "map_at_1": 0.09161, "map_at_3": 0.1447, "map_at_5": 0.16321, "map_at_10": 0.18279, "map_at_20": 0.19523, "map_at_25": 0.19852, "map_at_50": 0.20611, "map_at_100": 0.20953, "map_at_1000": 0.21144, "recall_at_1": 0.09161, "recall_at_3": 0.18794, "recall_at_5": 0.2427, "recall_at_10": 0.33355, "recall_at_20": 0.43475, "recall_at_25": 0.47105, "recall_at_50": 0.59134, "recall_at_100": 0.69088, "recall_at_1000": 0.86737, "accuracy": 0.09161, "precision_at_1": 0.2198, "precision_at_3": 0.15562, "precision_at_5": 0.12371, "precision_at_10": 0.08701, "precision_at_20": 0.05827, "precision_at_25": 0.05119, "precision_at_50": 0.03319, "precision_at_100": 0.01975, "precision_at_1000": 0.00251, "mrr_at_1": 0.2198, "mrr_at_3": 0.290948, "mrr_at_5": 0.309158, "mrr_at_10": 0.323019, "mrr_at_20": 0.330608, "mrr_at_25": 0.3318, "mrr_at_50": 0.334213, "mrr_at_100": 0.335104, "mrr_at_1000": 0.335579, "nauc_ndcg_at_1_max": 0.293994, "nauc_ndcg_at_1_std": 0.198704, "nauc_ndcg_at_1_diff1": 0.287755, "nauc_ndcg_at_3_max": 0.292832, "nauc_ndcg_at_3_std": 0.197915, "nauc_ndcg_at_3_diff1": 0.267584, "nauc_ndcg_at_5_max": 0.296368, "nauc_ndcg_at_5_std": 0.214639, "nauc_ndcg_at_5_diff1": 0.26723, "nauc_ndcg_at_10_max": 0.3215, "nauc_ndcg_at_10_std": 0.253859, "nauc_ndcg_at_10_diff1": 0.247831, "nauc_ndcg_at_20_max": 0.346717, "nauc_ndcg_at_20_std": 0.282228, "nauc_ndcg_at_20_diff1": 0.248515, "nauc_ndcg_at_25_max": 0.365807, "nauc_ndcg_at_25_std": 0.302419, "nauc_ndcg_at_25_diff1": 0.246306, "nauc_ndcg_at_50_max": 0.400809, "nauc_ndcg_at_50_std": 0.33734, "nauc_ndcg_at_50_diff1": 0.24272, "nauc_ndcg_at_100_max": 0.42466, "nauc_ndcg_at_100_std": 0.365071, "nauc_ndcg_at_100_diff1": 0.239299, "nauc_ndcg_at_1000_max": 0.411606, "nauc_ndcg_at_1000_std": 0.350016, "nauc_ndcg_at_1000_diff1": 0.240646, "nauc_map_at_1_max": 0.230071, "nauc_map_at_1_std": 0.162941, "nauc_map_at_1_diff1": 0.336525, "nauc_map_at_3_max": 0.270061, "nauc_map_at_3_std": 0.184748, "nauc_map_at_3_diff1": 0.303089, "nauc_map_at_5_max": 0.277898, "nauc_map_at_5_std": 0.197937, "nauc_map_at_5_diff1": 0.291177, "nauc_map_at_10_max": 0.294479, "nauc_map_at_10_std": 0.221206, "nauc_map_at_10_diff1": 0.279556, "nauc_map_at_20_max": 0.307968, "nauc_map_at_20_std": 0.23609, "nauc_map_at_20_diff1": 0.276212, "nauc_map_at_25_max": 0.315332, "nauc_map_at_25_std": 0.244059, "nauc_map_at_25_diff1": 0.275214, "nauc_map_at_50_max": 0.327336, "nauc_map_at_50_std": 0.255926, "nauc_map_at_50_diff1": 0.273423, "nauc_map_at_100_max": 0.332404, "nauc_map_at_100_std": 0.261559, "nauc_map_at_100_diff1": 0.272946, "nauc_map_at_1000_max": 0.332312, "nauc_map_at_1000_std": 0.261423, "nauc_map_at_1000_diff1": 0.272696, "nauc_recall_at_1_max": 0.230071, "nauc_recall_at_1_std": 0.162941, "nauc_recall_at_1_diff1": 0.336525, "nauc_recall_at_3_max": 0.252819, "nauc_recall_at_3_std": 0.181366, "nauc_recall_at_3_diff1": 0.256777, "nauc_recall_at_5_max": 0.261754, "nauc_recall_at_5_std": 0.205968, "nauc_recall_at_5_diff1": 0.227553, "nauc_recall_at_10_max": 0.297778, "nauc_recall_at_10_std": 0.274079, "nauc_recall_at_10_diff1": 0.185681, "nauc_recall_at_20_max": 0.33851, "nauc_recall_at_20_std": 0.326127, "nauc_recall_at_20_diff1": 0.188946, "nauc_recall_at_25_max": 0.384109, "nauc_recall_at_25_std": 0.376512, "nauc_recall_at_25_diff1": 0.183636, "nauc_recall_at_50_max": 0.484804, "nauc_recall_at_50_std": 0.485918, "nauc_recall_at_50_diff1": 0.172424, "nauc_recall_at_100_max": 0.601555, "nauc_recall_at_100_std": 0.629098, "nauc_recall_at_100_diff1": 0.157317, "nauc_recall_at_1000_max": 0.700131, "nauc_recall_at_1000_std": 0.782987, "nauc_recall_at_1000_diff1": 0.151021, "nauc_precision_at_1_max": 0.293994, "nauc_precision_at_1_std": 0.198704, "nauc_precision_at_1_diff1": 0.287755, "nauc_precision_at_3_max": 0.325622, "nauc_precision_at_3_std": 0.206101, "nauc_precision_at_3_diff1": 0.201216, "nauc_precision_at_5_max": 0.336844, "nauc_precision_at_5_std": 0.234583, "nauc_precision_at_5_diff1": 0.174508, "nauc_precision_at_10_max": 0.361522, "nauc_precision_at_10_std": 0.29547, "nauc_precision_at_10_diff1": 0.119447, "nauc_precision_at_20_max": 0.395742, "nauc_precision_at_20_std": 0.33872, "nauc_precision_at_20_diff1": 0.101637, "nauc_precision_at_25_max": 0.432105, "nauc_precision_at_25_std": 0.378856, "nauc_precision_at_25_diff1": 0.087872, "nauc_precision_at_50_max": 0.463331, "nauc_precision_at_50_std": 0.413568, "nauc_precision_at_50_diff1": 0.052191, "nauc_precision_at_100_max": 0.478665, "nauc_precision_at_100_std": 0.447109, "nauc_precision_at_100_diff1": 0.014508, "nauc_precision_at_1000_max": 0.324796, "nauc_precision_at_1000_std": 0.294385, "nauc_precision_at_1000_diff1": -0.039805, "nauc_mrr_at_1_max": 0.293994, "nauc_mrr_at_1_std": 0.198704, "nauc_mrr_at_1_diff1": 0.287755, "nauc_mrr_at_3_max": 0.304292, "nauc_mrr_at_3_std": 0.208255, "nauc_mrr_at_3_diff1": 0.253801, "nauc_mrr_at_5_max": 0.314416, "nauc_mrr_at_5_std": 0.219088, "nauc_mrr_at_5_diff1": 0.251542, "nauc_mrr_at_10_max": 0.323169, "nauc_mrr_at_10_std": 0.234132, "nauc_mrr_at_10_diff1": 0.243965, "nauc_mrr_at_20_max": 0.325081, "nauc_mrr_at_20_std": 0.235883, "nauc_mrr_at_20_diff1": 0.247006, "nauc_mrr_at_25_max": 0.326171, "nauc_mrr_at_25_std": 0.236756, "nauc_mrr_at_25_diff1": 0.247068, "nauc_mrr_at_50_max": 0.326775, "nauc_mrr_at_50_std": 0.23681, "nauc_mrr_at_50_diff1": 0.247842, "nauc_mrr_at_100_max": 0.326629, "nauc_mrr_at_100_std": 0.236682, "nauc_mrr_at_100_diff1": 0.247796, "nauc_mrr_at_1000_max": 0.326193, "nauc_mrr_at_1000_std": 0.236283, "nauc_mrr_at_1000_diff1": 0.247985, "hit_rate_at_1": 0.2198, "hit_rate_at_3": 0.38601, "hit_rate_at_5": 0.46594, "hit_rate_at_10": 0.56948, "hit_rate_at_20": 0.68029, "hit_rate_at_25": 0.70754, "hit_rate_at_50": 0.78837, "hit_rate_at_100": 0.84923, "hit_rate_at_1000": 0.94732, "R_cap_at_1": 0.2198, "R_cap_at_3": 0.204662, "R_cap_at_5": 0.243854, "R_cap_at_10": 0.333746, "R_cap_at_20": 0.434778, "R_cap_at_25": 0.471054, "R_cap_at_50": 0.591336, "R_cap_at_100": 0.690885, "R_cap_at_1000": 0.867368, "main_score": 0.26005, "hf_subset": "LongDepQA", "languages": [ "eng-Latn" ], "mteb_version": "2.3.0" }, { "ndcg_at_1": 0.34085, "ndcg_at_3": 0.44125, "ndcg_at_5": 0.46757, "ndcg_at_10": 0.49323, "ndcg_at_20": 0.51343, "ndcg_at_25": 0.51746, "ndcg_at_50": 0.52943, "ndcg_at_100": 0.53773, "ndcg_at_1000": 0.55022, "map_at_1": 0.34085, "map_at_3": 0.41645, "map_at_5": 0.43101, "map_at_10": 0.44149, "map_at_20": 0.44716, "map_at_25": 0.44797, "map_at_50": 0.44974, "map_at_100": 0.45047, "map_at_1000": 0.45093, "recall_at_1": 0.34085, "recall_at_3": 0.51307, "recall_at_5": 0.57714, "recall_at_10": 0.6571, "recall_at_20": 0.73603, "recall_at_25": 0.75448, "recall_at_50": 0.8165, "recall_at_100": 0.86776, "recall_at_1000": 0.96668, "accuracy": 0.34085, "precision_at_1": 0.34085, "precision_at_3": 0.17102, "precision_at_5": 0.11543, "precision_at_10": 0.06571, "precision_at_20": 0.0368, "precision_at_25": 0.03018, "precision_at_50": 0.01633, "precision_at_100": 0.00868, "precision_at_1000": 0.00097, "mrr_at_1": 0.340851, "mrr_at_3": 0.416453, "mrr_at_5": 0.43101, "mrr_at_10": 0.441493, "mrr_at_20": 0.447163, "mrr_at_25": 0.447967, "mrr_at_50": 0.449742, "mrr_at_100": 0.450469, "mrr_at_1000": 0.450931, "nauc_ndcg_at_1_max": 0.276848, "nauc_ndcg_at_1_std": 0.060641, "nauc_ndcg_at_1_diff1": 0.550622, "nauc_ndcg_at_3_max": 0.308872, "nauc_ndcg_at_3_std": 0.095279, "nauc_ndcg_at_3_diff1": 0.502904, "nauc_ndcg_at_5_max": 0.319959, "nauc_ndcg_at_5_std": 0.110644, "nauc_ndcg_at_5_diff1": 0.496255, "nauc_ndcg_at_10_max": 0.339452, "nauc_ndcg_at_10_std": 0.129383, "nauc_ndcg_at_10_diff1": 0.49186, "nauc_ndcg_at_20_max": 0.352886, "nauc_ndcg_at_20_std": 0.138939, "nauc_ndcg_at_20_diff1": 0.490858, "nauc_ndcg_at_25_max": 0.35796, "nauc_ndcg_at_25_std": 0.144474, "nauc_ndcg_at_25_diff1": 0.49267, "nauc_ndcg_at_50_max": 0.361299, "nauc_ndcg_at_50_std": 0.152226, "nauc_ndcg_at_50_diff1": 0.4927, "nauc_ndcg_at_100_max": 0.362146, "nauc_ndcg_at_100_std": 0.153232, "nauc_ndcg_at_100_diff1": 0.496702, "nauc_ndcg_at_1000_max": 0.347108, "nauc_ndcg_at_1000_std": 0.138772, "nauc_ndcg_at_1000_diff1": 0.500546, "nauc_map_at_1_max": 0.276848, "nauc_map_at_1_std": 0.060641, "nauc_map_at_1_diff1": 0.550622, "nauc_map_at_3_max": 0.300576, "nauc_map_at_3_std": 0.086035, "nauc_map_at_3_diff1": 0.514287, "nauc_map_at_5_max": 0.306627, "nauc_map_at_5_std": 0.0943, "nauc_map_at_5_diff1": 0.511019, "nauc_map_at_10_max": 0.314068, "nauc_map_at_10_std": 0.101584, "nauc_map_at_10_diff1": 0.509505, "nauc_map_at_20_max": 0.317469, "nauc_map_at_20_std": 0.103908, "nauc_map_at_20_diff1": 0.509312, "nauc_map_at_25_max": 0.318361, "nauc_map_at_25_std": 0.104888, "nauc_map_at_25_diff1": 0.509681, "nauc_map_at_50_max": 0.318663, "nauc_map_at_50_std": 0.105808, "nauc_map_at_50_diff1": 0.509794, "nauc_map_at_100_max": 0.318687, "nauc_map_at_100_std": 0.105818, "nauc_map_at_100_diff1": 0.510149, "nauc_map_at_1000_max": 0.318222, "nauc_map_at_1000_std": 0.105427, "nauc_map_at_1000_diff1": 0.510282, "nauc_recall_at_1_max": 0.276848, "nauc_recall_at_1_std": 0.060641, "nauc_recall_at_1_diff1": 0.550622, "nauc_recall_at_3_max": 0.333402, "nauc_recall_at_3_std": 0.122719, "nauc_recall_at_3_diff1": 0.469625, "nauc_recall_at_5_max": 0.362987, "nauc_recall_at_5_std": 0.163938, "nauc_recall_at_5_diff1": 0.449396, "nauc_recall_at_10_max": 0.439491, "nauc_recall_at_10_std": 0.238012, "nauc_recall_at_10_diff1": 0.426978, "nauc_recall_at_20_max": 0.53027, "nauc_recall_at_20_std": 0.310504, "nauc_recall_at_20_diff1": 0.409045, "nauc_recall_at_25_max": 0.573833, "nauc_recall_at_25_std": 0.356669, "nauc_recall_at_25_diff1": 0.415851, "nauc_recall_at_50_max": 0.662936, "nauc_recall_at_50_std": 0.484258, "nauc_recall_at_50_diff1": 0.394081, "nauc_recall_at_100_max": 0.777805, "nauc_recall_at_100_std": 0.613017, "nauc_recall_at_100_diff1": 0.416125, "nauc_recall_at_1000_max": 0.914072, "nauc_recall_at_1000_std": 0.88118, "nauc_recall_at_1000_diff1": 0.447562, "nauc_precision_at_1_max": 0.276848, "nauc_precision_at_1_std": 0.060641, "nauc_precision_at_1_diff1": 0.550622, "nauc_precision_at_3_max": 0.333402, "nauc_precision_at_3_std": 0.122719, "nauc_precision_at_3_diff1": 0.469625, "nauc_precision_at_5_max": 0.362987, "nauc_precision_at_5_std": 0.163938, "nauc_precision_at_5_diff1": 0.449396, "nauc_precision_at_10_max": 0.439491, "nauc_precision_at_10_std": 0.238012, "nauc_precision_at_10_diff1": 0.426978, "nauc_precision_at_20_max": 0.53027, "nauc_precision_at_20_std": 0.310504, "nauc_precision_at_20_diff1": 0.409045, "nauc_precision_at_25_max": 0.573833, "nauc_precision_at_25_std": 0.356669, "nauc_precision_at_25_diff1": 0.415851, "nauc_precision_at_50_max": 0.662936, "nauc_precision_at_50_std": 0.484258, "nauc_precision_at_50_diff1": 0.394081, "nauc_precision_at_100_max": 0.777805, "nauc_precision_at_100_std": 0.613017, "nauc_precision_at_100_diff1": 0.416125, "nauc_precision_at_1000_max": 0.914072, "nauc_precision_at_1000_std": 0.88118, "nauc_precision_at_1000_diff1": 0.447562, "nauc_mrr_at_1_max": 0.276848, "nauc_mrr_at_1_std": 0.060641, "nauc_mrr_at_1_diff1": 0.550622, "nauc_mrr_at_3_max": 0.300576, "nauc_mrr_at_3_std": 0.086035, "nauc_mrr_at_3_diff1": 0.514287, "nauc_mrr_at_5_max": 0.306627, "nauc_mrr_at_5_std": 0.0943, "nauc_mrr_at_5_diff1": 0.511019, "nauc_mrr_at_10_max": 0.314068, "nauc_mrr_at_10_std": 0.101584, "nauc_mrr_at_10_diff1": 0.509505, "nauc_mrr_at_20_max": 0.317469, "nauc_mrr_at_20_std": 0.103908, "nauc_mrr_at_20_diff1": 0.509312, "nauc_mrr_at_25_max": 0.318361, "nauc_mrr_at_25_std": 0.104888, "nauc_mrr_at_25_diff1": 0.509681, "nauc_mrr_at_50_max": 0.318663, "nauc_mrr_at_50_std": 0.105808, "nauc_mrr_at_50_diff1": 0.509794, "nauc_mrr_at_100_max": 0.318687, "nauc_mrr_at_100_std": 0.105818, "nauc_mrr_at_100_diff1": 0.510149, "nauc_mrr_at_1000_max": 0.318222, "nauc_mrr_at_1000_std": 0.105427, "nauc_mrr_at_1000_diff1": 0.510282, "hit_rate_at_1": 0.34085, "hit_rate_at_3": 0.51307, "hit_rate_at_5": 0.57714, "hit_rate_at_10": 0.6571, "hit_rate_at_20": 0.73603, "hit_rate_at_25": 0.75448, "hit_rate_at_50": 0.8165, "hit_rate_at_100": 0.86776, "hit_rate_at_1000": 0.96668, "R_cap_at_1": 0.340851, "R_cap_at_3": 0.51307, "R_cap_at_5": 0.57714, "R_cap_at_10": 0.657099, "R_cap_at_20": 0.736033, "R_cap_at_25": 0.754485, "R_cap_at_50": 0.816504, "R_cap_at_100": 0.86776, "R_cap_at_1000": 0.966684, "main_score": 0.49323, "hf_subset": "ShortDepQA", "languages": [ "eng-Latn" ], "mteb_version": "2.3.0" } ] }, "evaluation_time": 1770.679881611839, "kg_co2_emissions": null, "date": 1787326819.06416, "evaluation_phases": [ { "name": "Data loading", "start": 7.893890142440796e-06, "end": 796.914414877072, "split": "", "subset": "" }, { "name": "Dataset transform", "start": 796.9144305735826, "end": 796.9144371980801, "split": "", "subset": "" }, { "name": "Encoding queries and documents", "start": 796.9454040564597, "end": 808.9217484612018, "split": "test", "subset": "ShortDepQA" }, { "name": "Scoring", "start": 808.9218670791015, "end": 811.0813780091703, "split": "test", "subset": "ShortDepQA" } ] }