{ "dataset_revision": "837028901907a7d419b4ab906f28e011ce1cc824", "task_name": "MultiLongDocRetrieval", "mteb_version": "2.2.0", "scores": { "dev": [ { "ndcg_at_1": 0.38, "ndcg_at_3": 0.42524, "ndcg_at_5": 0.44203, "ndcg_at_10": 0.45999, "ndcg_at_20": 0.46916, "ndcg_at_100": 0.49717, "ndcg_at_1000": 0.51635, "map_at_1": 0.38, "map_at_3": 0.41333, "map_at_5": 0.42283, "map_at_10": 0.43037, "map_at_20": 0.43305, "map_at_100": 0.43701, "map_at_1000": 0.43765, "recall_at_1": 0.38, "recall_at_3": 0.46, "recall_at_5": 0.5, "recall_at_10": 0.555, "recall_at_20": 0.59, "recall_at_100": 0.74, "recall_at_1000": 0.895, "accuracy": 0.38, "precision_at_1": 0.38, "precision_at_3": 0.15333, "precision_at_5": 0.1, "precision_at_10": 0.0555, "precision_at_20": 0.0295, "precision_at_100": 0.0074, "precision_at_1000": 0.0009, "mrr_at_1": 0.38, "mrr_at_3": 0.413333, "mrr_at_5": 0.422833, "mrr_at_10": 0.430365, "mrr_at_20": 0.433051, "mrr_at_100": 0.437013, "mrr_at_1000": 0.43765, "nauc_ndcg_at_1_max": 0.668189, "nauc_ndcg_at_1_std": 0.104053, "nauc_ndcg_at_1_diff1": 0.759364, "nauc_ndcg_at_3_max": 0.685856, "nauc_ndcg_at_3_std": 0.138507, "nauc_ndcg_at_3_diff1": 0.753269, "nauc_ndcg_at_5_max": 0.695046, "nauc_ndcg_at_5_std": 0.177203, "nauc_ndcg_at_5_diff1": 0.731012, "nauc_ndcg_at_10_max": 0.710384, "nauc_ndcg_at_10_std": 0.206042, "nauc_ndcg_at_10_diff1": 0.737836, "nauc_ndcg_at_20_max": 0.705644, "nauc_ndcg_at_20_std": 0.211938, "nauc_ndcg_at_20_diff1": 0.730019, "nauc_ndcg_at_100_max": 0.709725, "nauc_ndcg_at_100_std": 0.235313, "nauc_ndcg_at_100_diff1": 0.732116, "nauc_ndcg_at_1000_max": 0.709176, "nauc_ndcg_at_1000_std": 0.228651, "nauc_ndcg_at_1000_diff1": 0.736339, "nauc_map_at_1_max": 0.668189, "nauc_map_at_1_std": 0.104053, "nauc_map_at_1_diff1": 0.759364, "nauc_map_at_3_max": 0.683002, "nauc_map_at_3_std": 0.132235, "nauc_map_at_3_diff1": 0.75673, "nauc_map_at_5_max": 0.688243, "nauc_map_at_5_std": 0.154629, "nauc_map_at_5_diff1": 0.743879, "nauc_map_at_10_max": 0.693571, "nauc_map_at_10_std": 0.165554, "nauc_map_at_10_diff1": 0.74634, "nauc_map_at_20_max": 0.691981, "nauc_map_at_20_std": 0.166835, "nauc_map_at_20_diff1": 0.744143, "nauc_map_at_100_max": 0.69263, "nauc_map_at_100_std": 0.1698, "nauc_map_at_100_diff1": 0.744471, "nauc_map_at_1000_max": 0.692661, "nauc_map_at_1000_std": 0.169511, "nauc_map_at_1000_diff1": 0.744628, "nauc_recall_at_1_max": 0.668189, "nauc_recall_at_1_std": 0.104053, "nauc_recall_at_1_diff1": 0.759364, "nauc_recall_at_3_max": 0.693563, "nauc_recall_at_3_std": 0.155889, "nauc_recall_at_3_diff1": 0.742379, "nauc_recall_at_5_max": 0.71557, "nauc_recall_at_5_std": 0.247029, "nauc_recall_at_5_diff1": 0.689813, "nauc_recall_at_10_max": 0.771648, "nauc_recall_at_10_std": 0.348941, "nauc_recall_at_10_diff1": 0.712035, "nauc_recall_at_20_max": 0.755888, "nauc_recall_at_20_std": 0.381337, "nauc_recall_at_20_diff1": 0.67846, "nauc_recall_at_100_max": 0.802978, "nauc_recall_at_100_std": 0.620291, "nauc_recall_at_100_diff1": 0.676564, "nauc_recall_at_1000_max": 0.890944, "nauc_recall_at_1000_std": 0.920697, "nauc_recall_at_1000_diff1": 0.711243, "nauc_precision_at_1_max": 0.668189, "nauc_precision_at_1_std": 0.104053, "nauc_precision_at_1_diff1": 0.759364, "nauc_precision_at_3_max": 0.693563, "nauc_precision_at_3_std": 0.155889, "nauc_precision_at_3_diff1": 0.742379, "nauc_precision_at_5_max": 0.71557, "nauc_precision_at_5_std": 0.247029, "nauc_precision_at_5_diff1": 0.689813, "nauc_precision_at_10_max": 0.771648, "nauc_precision_at_10_std": 0.348941, "nauc_precision_at_10_diff1": 0.712035, "nauc_precision_at_20_max": 0.755888, "nauc_precision_at_20_std": 0.381337, "nauc_precision_at_20_diff1": 0.67846, "nauc_precision_at_100_max": 0.802978, "nauc_precision_at_100_std": 0.620291, "nauc_precision_at_100_diff1": 0.676564, "nauc_precision_at_1000_max": 0.890944, "nauc_precision_at_1000_std": 0.920697, "nauc_precision_at_1000_diff1": 0.711243, "nauc_mrr_at_1_max": 0.668189, "nauc_mrr_at_1_std": 0.104053, "nauc_mrr_at_1_diff1": 0.759364, "nauc_mrr_at_3_max": 0.683002, "nauc_mrr_at_3_std": 0.132235, "nauc_mrr_at_3_diff1": 0.75673, "nauc_mrr_at_5_max": 0.688243, "nauc_mrr_at_5_std": 0.154629, "nauc_mrr_at_5_diff1": 0.743879, "nauc_mrr_at_10_max": 0.693571, "nauc_mrr_at_10_std": 0.165554, "nauc_mrr_at_10_diff1": 0.74634, "nauc_mrr_at_20_max": 0.691981, "nauc_mrr_at_20_std": 0.166835, "nauc_mrr_at_20_diff1": 0.744143, "nauc_mrr_at_100_max": 0.69263, "nauc_mrr_at_100_std": 0.1698, "nauc_mrr_at_100_diff1": 0.744471, "nauc_mrr_at_1000_max": 0.692661, "nauc_mrr_at_1000_std": 0.169511, "nauc_mrr_at_1000_diff1": 0.744628, "cv_recall_at_1": 0.38, "cv_recall_at_3": 0.46, "cv_recall_at_5": 0.5, "cv_recall_at_10": 0.555, "cv_recall_at_20": 0.59, "cv_recall_at_100": 0.74, "cv_recall_at_1000": 0.895, "main_score": 0.45999, "hf_subset": "ja", "languages": [ "jpn-Jpan" ] } ], "test": [ { "ndcg_at_1": 0.4, "ndcg_at_3": 0.44708, "ndcg_at_5": 0.45741, "ndcg_at_10": 0.46526, "ndcg_at_20": 0.47263, "ndcg_at_100": 0.49625, "ndcg_at_1000": 0.51997, "map_at_1": 0.4, "map_at_3": 0.43417, "map_at_5": 0.43992, "map_at_10": 0.44301, "map_at_20": 0.44492, "map_at_100": 0.44833, "map_at_1000": 0.44908, "recall_at_1": 0.4, "recall_at_3": 0.485, "recall_at_5": 0.51, "recall_at_10": 0.535, "recall_at_20": 0.565, "recall_at_100": 0.69, "recall_at_1000": 0.885, "accuracy": 0.4, "precision_at_1": 0.4, "precision_at_3": 0.16167, "precision_at_5": 0.102, "precision_at_10": 0.0535, "precision_at_20": 0.02825, "precision_at_100": 0.0069, "precision_at_1000": 0.00089, "mrr_at_1": 0.4, "mrr_at_3": 0.434167, "mrr_at_5": 0.439917, "mrr_at_10": 0.443006, "mrr_at_20": 0.444921, "mrr_at_100": 0.448324, "mrr_at_1000": 0.44908, "nauc_ndcg_at_1_max": 0.501545, "nauc_ndcg_at_1_std": 0.05587, "nauc_ndcg_at_1_diff1": 0.612706, "nauc_ndcg_at_3_max": 0.539123, "nauc_ndcg_at_3_std": 0.116116, "nauc_ndcg_at_3_diff1": 0.604881, "nauc_ndcg_at_5_max": 0.538448, "nauc_ndcg_at_5_std": 0.106565, "nauc_ndcg_at_5_diff1": 0.608402, "nauc_ndcg_at_10_max": 0.54358, "nauc_ndcg_at_10_std": 0.115379, "nauc_ndcg_at_10_diff1": 0.604305, "nauc_ndcg_at_20_max": 0.540648, "nauc_ndcg_at_20_std": 0.141352, "nauc_ndcg_at_20_diff1": 0.592216, "nauc_ndcg_at_100_max": 0.545236, "nauc_ndcg_at_100_std": 0.151976, "nauc_ndcg_at_100_diff1": 0.593342, "nauc_ndcg_at_1000_max": 0.546723, "nauc_ndcg_at_1000_std": 0.155639, "nauc_ndcg_at_1000_diff1": 0.589493, "nauc_map_at_1_max": 0.501545, "nauc_map_at_1_std": 0.05587, "nauc_map_at_1_diff1": 0.612706, "nauc_map_at_3_max": 0.532035, "nauc_map_at_3_std": 0.101254, "nauc_map_at_3_diff1": 0.608433, "nauc_map_at_5_max": 0.531035, "nauc_map_at_5_std": 0.095467, "nauc_map_at_5_diff1": 0.610308, "nauc_map_at_10_max": 0.533422, "nauc_map_at_10_std": 0.098828, "nauc_map_at_10_diff1": 0.608884, "nauc_map_at_20_max": 0.5328, "nauc_map_at_20_std": 0.105453, "nauc_map_at_20_diff1": 0.605703, "nauc_map_at_100_max": 0.53343, "nauc_map_at_100_std": 0.106567, "nauc_map_at_100_diff1": 0.605715, "nauc_map_at_1000_max": 0.533446, "nauc_map_at_1000_std": 0.106805, "nauc_map_at_1000_diff1": 0.605507, "nauc_recall_at_1_max": 0.501545, "nauc_recall_at_1_std": 0.05587, "nauc_recall_at_1_diff1": 0.612706, "nauc_recall_at_3_max": 0.559067, "nauc_recall_at_3_std": 0.159833, "nauc_recall_at_3_diff1": 0.593761, "nauc_recall_at_5_max": 0.560283, "nauc_recall_at_5_std": 0.138834, "nauc_recall_at_5_diff1": 0.602655, "nauc_recall_at_10_max": 0.57553, "nauc_recall_at_10_std": 0.169121, "nauc_recall_at_10_diff1": 0.587925, "nauc_recall_at_20_max": 0.562567, "nauc_recall_at_20_std": 0.283303, "nauc_recall_at_20_diff1": 0.535821, "nauc_recall_at_100_max": 0.596795, "nauc_recall_at_100_std": 0.391643, "nauc_recall_at_100_diff1": 0.532383, "nauc_recall_at_1000_max": 0.700553, "nauc_recall_at_1000_std": 0.748524, "nauc_recall_at_1000_diff1": 0.378805, "nauc_precision_at_1_max": 0.501545, "nauc_precision_at_1_std": 0.05587, "nauc_precision_at_1_diff1": 0.612706, "nauc_precision_at_3_max": 0.559067, "nauc_precision_at_3_std": 0.159833, "nauc_precision_at_3_diff1": 0.593761, "nauc_precision_at_5_max": 0.560283, "nauc_precision_at_5_std": 0.138834, "nauc_precision_at_5_diff1": 0.602655, "nauc_precision_at_10_max": 0.57553, "nauc_precision_at_10_std": 0.169121, "nauc_precision_at_10_diff1": 0.587925, "nauc_precision_at_20_max": 0.562567, "nauc_precision_at_20_std": 0.283303, "nauc_precision_at_20_diff1": 0.535821, "nauc_precision_at_100_max": 0.596795, "nauc_precision_at_100_std": 0.391643, "nauc_precision_at_100_diff1": 0.532383, "nauc_precision_at_1000_max": 0.700553, "nauc_precision_at_1000_std": 0.748524, "nauc_precision_at_1000_diff1": 0.378805, "nauc_mrr_at_1_max": 0.501545, "nauc_mrr_at_1_std": 0.05587, "nauc_mrr_at_1_diff1": 0.612706, "nauc_mrr_at_3_max": 0.532035, "nauc_mrr_at_3_std": 0.101254, "nauc_mrr_at_3_diff1": 0.608433, "nauc_mrr_at_5_max": 0.531035, "nauc_mrr_at_5_std": 0.095467, "nauc_mrr_at_5_diff1": 0.610308, "nauc_mrr_at_10_max": 0.533422, "nauc_mrr_at_10_std": 0.098828, "nauc_mrr_at_10_diff1": 0.608884, "nauc_mrr_at_20_max": 0.5328, "nauc_mrr_at_20_std": 0.105453, "nauc_mrr_at_20_diff1": 0.605703, "nauc_mrr_at_100_max": 0.533433, "nauc_mrr_at_100_std": 0.106565, "nauc_mrr_at_100_diff1": 0.605715, "nauc_mrr_at_1000_max": 0.533449, "nauc_mrr_at_1000_std": 0.106804, "nauc_mrr_at_1000_diff1": 0.605507, "cv_recall_at_1": 0.4, "cv_recall_at_3": 0.485, "cv_recall_at_5": 0.51, "cv_recall_at_10": 0.535, "cv_recall_at_20": 0.565, "cv_recall_at_100": 0.69, "cv_recall_at_1000": 0.885, "main_score": 0.46526, "hf_subset": "ja", "languages": [ "jpn-Jpan" ] } ] }, "evaluation_time": 1527.9800336360931, "kg_co2_emissions": null }