{ "dataset_revision": "837028901907a7d419b4ab906f28e011ce1cc824", "task_name": "MultiLongDocRetrieval", "mteb_version": "2.20.12", "scores": { "dev": [ { "ndcg_at_1": 0.36, "ndcg_at_3": 0.4694, "ndcg_at_5": 0.50254, "ndcg_at_10": 0.52577, "ndcg_at_20": 0.54071, "ndcg_at_100": 0.55956, "ndcg_at_1000": 0.57151, "map_at_1": 0.36, "map_at_3": 0.44167, "map_at_5": 0.46017, "map_at_10": 0.47009, "map_at_20": 0.47406, "map_at_100": 0.47677, "map_at_1000": 0.4772, "recall_at_1": 0.36, "recall_at_3": 0.55, "recall_at_5": 0.63, "recall_at_10": 0.7, "recall_at_20": 0.76, "recall_at_100": 0.86, "recall_at_1000": 0.955, "accuracy": 0.36, "precision_at_1": 0.36, "precision_at_3": 0.18333, "precision_at_5": 0.126, "precision_at_10": 0.07, "precision_at_20": 0.038, "precision_at_100": 0.0086, "precision_at_1000": 0.00096, "mrr_at_1": 0.36, "mrr_at_3": 0.441667, "mrr_at_5": 0.460167, "mrr_at_10": 0.470091, "mrr_at_20": 0.474063, "mrr_at_100": 0.476773, "mrr_at_1000": 0.477201, "nauc_ndcg_at_1_max": 0.334127, "nauc_ndcg_at_1_std": 0.180214, "nauc_ndcg_at_1_diff1": 0.680293, "nauc_ndcg_at_3_max": 0.400223, "nauc_ndcg_at_3_std": 0.259066, "nauc_ndcg_at_3_diff1": 0.602387, "nauc_ndcg_at_5_max": 0.395436, "nauc_ndcg_at_5_std": 0.290534, "nauc_ndcg_at_5_diff1": 0.576038, "nauc_ndcg_at_10_max": 0.411642, "nauc_ndcg_at_10_std": 0.310842, "nauc_ndcg_at_10_diff1": 0.583826, "nauc_ndcg_at_20_max": 0.403499, "nauc_ndcg_at_20_std": 0.308964, "nauc_ndcg_at_20_diff1": 0.586088, "nauc_ndcg_at_100_max": 0.391718, "nauc_ndcg_at_100_std": 0.307106, "nauc_ndcg_at_100_diff1": 0.586696, "nauc_ndcg_at_1000_max": 0.396075, "nauc_ndcg_at_1000_std": 0.301062, "nauc_ndcg_at_1000_diff1": 0.592985, "nauc_map_at_1_max": 0.334127, "nauc_map_at_1_std": 0.180214, "nauc_map_at_1_diff1": 0.680293, "nauc_map_at_3_max": 0.386175, "nauc_map_at_3_std": 0.239382, "nauc_map_at_3_diff1": 0.622892, "nauc_map_at_5_max": 0.382862, "nauc_map_at_5_std": 0.25577, "nauc_map_at_5_diff1": 0.609434, "nauc_map_at_10_max": 0.389499, "nauc_map_at_10_std": 0.263681, "nauc_map_at_10_diff1": 0.613024, "nauc_map_at_20_max": 0.387189, "nauc_map_at_20_std": 0.262561, "nauc_map_at_20_diff1": 0.613824, "nauc_map_at_100_max": 0.385722, "nauc_map_at_100_std": 0.262402, "nauc_map_at_100_diff1": 0.614053, "nauc_map_at_1000_max": 0.385852, "nauc_map_at_1000_std": 0.26225, "nauc_map_at_1000_diff1": 0.614318, "nauc_recall_at_1_max": 0.334127, "nauc_recall_at_1_std": 0.180214, "nauc_recall_at_1_diff1": 0.680293, "nauc_recall_at_3_max": 0.441649, "nauc_recall_at_3_std": 0.318578, "nauc_recall_at_3_diff1": 0.539942, "nauc_recall_at_5_max": 0.435536, "nauc_recall_at_5_std": 0.410505, "nauc_recall_at_5_diff1": 0.460694, "nauc_recall_at_10_max": 0.499207, "nauc_recall_at_10_std": 0.501342, "nauc_recall_at_10_diff1": 0.472951, "nauc_recall_at_20_max": 0.4719, "nauc_recall_at_20_std": 0.531929, "nauc_recall_at_20_diff1": 0.463615, "nauc_recall_at_100_max": 0.372211, "nauc_recall_at_100_std": 0.630975, "nauc_recall_at_100_diff1": 0.396789, "nauc_recall_at_1000_max": 0.541861, "nauc_recall_at_1000_std": 0.941903, "nauc_recall_at_1000_diff1": 0.312325, "nauc_precision_at_1_max": 0.334127, "nauc_precision_at_1_std": 0.180214, "nauc_precision_at_1_diff1": 0.680293, "nauc_precision_at_3_max": 0.441649, "nauc_precision_at_3_std": 0.318578, "nauc_precision_at_3_diff1": 0.539942, "nauc_precision_at_5_max": 0.435536, "nauc_precision_at_5_std": 0.410505, "nauc_precision_at_5_diff1": 0.460694, "nauc_precision_at_10_max": 0.499207, "nauc_precision_at_10_std": 0.501342, "nauc_precision_at_10_diff1": 0.472951, "nauc_precision_at_20_max": 0.4719, "nauc_precision_at_20_std": 0.531929, "nauc_precision_at_20_diff1": 0.463615, "nauc_precision_at_100_max": 0.372211, "nauc_precision_at_100_std": 0.630975, "nauc_precision_at_100_diff1": 0.396789, "nauc_precision_at_1000_max": 0.541861, "nauc_precision_at_1000_std": 0.941903, "nauc_precision_at_1000_diff1": 0.312325, "nauc_mrr_at_1_max": 0.334127, "nauc_mrr_at_1_std": 0.180214, "nauc_mrr_at_1_diff1": 0.680293, "nauc_mrr_at_3_max": 0.386175, "nauc_mrr_at_3_std": 0.239382, "nauc_mrr_at_3_diff1": 0.622892, "nauc_mrr_at_5_max": 0.382862, "nauc_mrr_at_5_std": 0.25577, "nauc_mrr_at_5_diff1": 0.609434, "nauc_mrr_at_10_max": 0.389499, "nauc_mrr_at_10_std": 0.263681, "nauc_mrr_at_10_diff1": 0.613024, "nauc_mrr_at_20_max": 0.387189, "nauc_mrr_at_20_std": 0.262561, "nauc_mrr_at_20_diff1": 0.613824, "nauc_mrr_at_100_max": 0.385722, "nauc_mrr_at_100_std": 0.262402, "nauc_mrr_at_100_diff1": 0.614053, "nauc_mrr_at_1000_max": 0.385852, "nauc_mrr_at_1000_std": 0.26225, "nauc_mrr_at_1000_diff1": 0.614318, "hit_rate_at_1": 0.36, "hit_rate_at_3": 0.55, "hit_rate_at_5": 0.63, "hit_rate_at_10": 0.7, "hit_rate_at_20": 0.76, "hit_rate_at_100": 0.86, "hit_rate_at_1000": 0.955, "main_score": 0.52577, "hf_subset": "it", "languages": [ "ita-Latn" ], "mteb_version": "2.20.12" } ], "test": [ { "ndcg_at_1": 0.51, "ndcg_at_3": 0.55917, "ndcg_at_5": 0.58413, "ndcg_at_10": 0.60915, "ndcg_at_20": 0.62184, "ndcg_at_100": 0.64017, "ndcg_at_1000": 0.65084, "map_at_1": 0.51, "map_at_3": 0.54833, "map_at_5": 0.56233, "map_at_10": 0.57312, "map_at_20": 0.57663, "map_at_100": 0.57904, "map_at_1000": 0.57933, "recall_at_1": 0.51, "recall_at_3": 0.59, "recall_at_5": 0.65, "recall_at_10": 0.725, "recall_at_20": 0.775, "recall_at_100": 0.875, "recall_at_1000": 0.965, "accuracy": 0.51, "precision_at_1": 0.51, "precision_at_3": 0.19667, "precision_at_5": 0.13, "precision_at_10": 0.0725, "precision_at_20": 0.03875, "precision_at_100": 0.00875, "precision_at_1000": 0.00097, "mrr_at_1": 0.51, "mrr_at_3": 0.548333, "mrr_at_5": 0.562333, "mrr_at_10": 0.573115, "mrr_at_20": 0.576634, "mrr_at_100": 0.579045, "mrr_at_1000": 0.579331, "nauc_ndcg_at_1_max": 0.543153, "nauc_ndcg_at_1_std": 0.073408, "nauc_ndcg_at_1_diff1": 0.778879, "nauc_ndcg_at_3_max": 0.546225, "nauc_ndcg_at_3_std": 0.106998, "nauc_ndcg_at_3_diff1": 0.755442, "nauc_ndcg_at_5_max": 0.548214, "nauc_ndcg_at_5_std": 0.136257, "nauc_ndcg_at_5_diff1": 0.737861, "nauc_ndcg_at_10_max": 0.537932, "nauc_ndcg_at_10_std": 0.134591, "nauc_ndcg_at_10_diff1": 0.714021, "nauc_ndcg_at_20_max": 0.535813, "nauc_ndcg_at_20_std": 0.139124, "nauc_ndcg_at_20_diff1": 0.707058, "nauc_ndcg_at_100_max": 0.527536, "nauc_ndcg_at_100_std": 0.145614, "nauc_ndcg_at_100_diff1": 0.71724, "nauc_ndcg_at_1000_max": 0.533571, "nauc_ndcg_at_1000_std": 0.145679, "nauc_ndcg_at_1000_diff1": 0.720133, "nauc_map_at_1_max": 0.543153, "nauc_map_at_1_std": 0.073408, "nauc_map_at_1_diff1": 0.778879, "nauc_map_at_3_max": 0.546252, "nauc_map_at_3_std": 0.101171, "nauc_map_at_3_diff1": 0.761761, "nauc_map_at_5_max": 0.547603, "nauc_map_at_5_std": 0.117301, "nauc_map_at_5_diff1": 0.75211, "nauc_map_at_10_max": 0.543752, "nauc_map_at_10_std": 0.116289, "nauc_map_at_10_diff1": 0.742876, "nauc_map_at_20_max": 0.543246, "nauc_map_at_20_std": 0.117335, "nauc_map_at_20_diff1": 0.741036, "nauc_map_at_100_max": 0.542544, "nauc_map_at_100_std": 0.118388, "nauc_map_at_100_diff1": 0.74255, "nauc_map_at_1000_max": 0.542668, "nauc_map_at_1000_std": 0.118411, "nauc_map_at_1000_diff1": 0.742628, "nauc_recall_at_1_max": 0.543153, "nauc_recall_at_1_std": 0.073408, "nauc_recall_at_1_diff1": 0.778879, "nauc_recall_at_3_max": 0.545798, "nauc_recall_at_3_std": 0.123476, "nauc_recall_at_3_diff1": 0.736166, "nauc_recall_at_5_max": 0.54976, "nauc_recall_at_5_std": 0.201212, "nauc_recall_at_5_diff1": 0.688534, "nauc_recall_at_10_max": 0.509057, "nauc_recall_at_10_std": 0.206855, "nauc_recall_at_10_diff1": 0.588779, "nauc_recall_at_20_max": 0.49216, "nauc_recall_at_20_std": 0.24489, "nauc_recall_at_20_diff1": 0.53106, "nauc_recall_at_100_max": 0.353423, "nauc_recall_at_100_std": 0.383931, "nauc_recall_at_100_diff1": 0.533941, "nauc_recall_at_1000_max": 0.335534, "nauc_recall_at_1000_std": 0.943978, "nauc_recall_at_1000_diff1": 0.270908, "nauc_precision_at_1_max": 0.543153, "nauc_precision_at_1_std": 0.073408, "nauc_precision_at_1_diff1": 0.778879, "nauc_precision_at_3_max": 0.545798, "nauc_precision_at_3_std": 0.123476, "nauc_precision_at_3_diff1": 0.736166, "nauc_precision_at_5_max": 0.54976, "nauc_precision_at_5_std": 0.201212, "nauc_precision_at_5_diff1": 0.688534, "nauc_precision_at_10_max": 0.509057, "nauc_precision_at_10_std": 0.206855, "nauc_precision_at_10_diff1": 0.588779, "nauc_precision_at_20_max": 0.49216, "nauc_precision_at_20_std": 0.24489, "nauc_precision_at_20_diff1": 0.53106, "nauc_precision_at_100_max": 0.353423, "nauc_precision_at_100_std": 0.383931, "nauc_precision_at_100_diff1": 0.533941, "nauc_precision_at_1000_max": 0.335534, "nauc_precision_at_1000_std": 0.943978, "nauc_precision_at_1000_diff1": 0.270908, "nauc_mrr_at_1_max": 0.543153, "nauc_mrr_at_1_std": 0.073408, "nauc_mrr_at_1_diff1": 0.778879, "nauc_mrr_at_3_max": 0.546252, "nauc_mrr_at_3_std": 0.101171, "nauc_mrr_at_3_diff1": 0.761761, "nauc_mrr_at_5_max": 0.547603, "nauc_mrr_at_5_std": 0.117301, "nauc_mrr_at_5_diff1": 0.75211, "nauc_mrr_at_10_max": 0.543752, "nauc_mrr_at_10_std": 0.116289, "nauc_mrr_at_10_diff1": 0.742876, "nauc_mrr_at_20_max": 0.543246, "nauc_mrr_at_20_std": 0.117335, "nauc_mrr_at_20_diff1": 0.741036, "nauc_mrr_at_100_max": 0.542544, "nauc_mrr_at_100_std": 0.118388, "nauc_mrr_at_100_diff1": 0.74255, "nauc_mrr_at_1000_max": 0.542668, "nauc_mrr_at_1000_std": 0.118411, "nauc_mrr_at_1000_diff1": 0.742628, "hit_rate_at_1": 0.51, "hit_rate_at_3": 0.59, "hit_rate_at_5": 0.65, "hit_rate_at_10": 0.725, "hit_rate_at_20": 0.775, "hit_rate_at_100": 0.875, "hit_rate_at_1000": 0.965, "main_score": 0.60915, "hf_subset": "it", "languages": [ "ita-Latn" ], "mteb_version": "2.20.12" } ] }, "evaluation_time": 10776.093715541065, "kg_co2_emissions": null, "date": 1789404450.285954, "evaluation_phases": [ { "name": "Data loading", "start": 1.3614073395729065e-05, "end": 17.641393775120378, "split": "", "subset": "" }, { "name": "Dataset transform", "start": 17.64142170920968, "end": 17.641435120254755, "split": "", "subset": "" }, { "name": "Encoding queries and documents", "start": 17.64734373614192, "end": 4941.672157976776, "split": "test", "subset": "it" }, { "name": "Scoring", "start": 4941.672245135531, "end": 4941.987188030034, "split": "test", "subset": "it" } ] }