{ "dataset_revision": "837028901907a7d419b4ab906f28e011ce1cc824", "task_name": "MultiLongDocRetrieval", "mteb_version": "2.2.0", "scores": { "dev": [ { "ndcg_at_1": 0.25, "ndcg_at_3": 0.30917, "ndcg_at_5": 0.31993, "ndcg_at_10": 0.33603, "ndcg_at_20": 0.34858, "ndcg_at_100": 0.36996, "ndcg_at_1000": 0.39744, "map_at_1": 0.25, "map_at_3": 0.295, "map_at_5": 0.30125, "map_at_10": 0.30785, "map_at_20": 0.31125, "map_at_100": 0.31423, "map_at_1000": 0.3152, "recall_at_1": 0.25, "recall_at_3": 0.35, "recall_at_5": 0.375, "recall_at_10": 0.425, "recall_at_20": 0.475, "recall_at_100": 0.59, "recall_at_1000": 0.81, "accuracy": 0.25, "precision_at_1": 0.25, "precision_at_3": 0.11667, "precision_at_5": 0.075, "precision_at_10": 0.0425, "precision_at_20": 0.02375, "precision_at_100": 0.0059, "precision_at_1000": 0.00081, "mrr_at_1": 0.25, "mrr_at_3": 0.295, "mrr_at_5": 0.30125, "mrr_at_10": 0.307845, "mrr_at_20": 0.311245, "mrr_at_100": 0.314229, "mrr_at_1000": 0.3152, "nauc_ndcg_at_1_max": 0.325031, "nauc_ndcg_at_1_std": 0.057352, "nauc_ndcg_at_1_diff1": 0.645649, "nauc_ndcg_at_3_max": 0.407544, "nauc_ndcg_at_3_std": 0.086958, "nauc_ndcg_at_3_diff1": 0.597999, "nauc_ndcg_at_5_max": 0.398615, "nauc_ndcg_at_5_std": 0.097201, "nauc_ndcg_at_5_diff1": 0.585676, "nauc_ndcg_at_10_max": 0.406299, "nauc_ndcg_at_10_std": 0.126632, "nauc_ndcg_at_10_diff1": 0.586997, "nauc_ndcg_at_20_max": 0.393561, "nauc_ndcg_at_20_std": 0.133813, "nauc_ndcg_at_20_diff1": 0.56741, "nauc_ndcg_at_100_max": 0.391722, "nauc_ndcg_at_100_std": 0.169683, "nauc_ndcg_at_100_diff1": 0.560779, "nauc_ndcg_at_1000_max": 0.381553, "nauc_ndcg_at_1000_std": 0.165784, "nauc_ndcg_at_1000_diff1": 0.564071, "nauc_map_at_1_max": 0.325031, "nauc_map_at_1_std": 0.057352, "nauc_map_at_1_diff1": 0.645649, "nauc_map_at_3_max": 0.391215, "nauc_map_at_3_std": 0.083526, "nauc_map_at_3_diff1": 0.60964, "nauc_map_at_5_max": 0.385995, "nauc_map_at_5_std": 0.089584, "nauc_map_at_5_diff1": 0.602327, "nauc_map_at_10_max": 0.388514, "nauc_map_at_10_std": 0.101472, "nauc_map_at_10_diff1": 0.6033, "nauc_map_at_20_max": 0.384588, "nauc_map_at_20_std": 0.10263, "nauc_map_at_20_diff1": 0.597606, "nauc_map_at_100_max": 0.384151, "nauc_map_at_100_std": 0.108231, "nauc_map_at_100_diff1": 0.597106, "nauc_map_at_1000_max": 0.383577, "nauc_map_at_1000_std": 0.107901, "nauc_map_at_1000_diff1": 0.597159, "nauc_recall_at_1_max": 0.325031, "nauc_recall_at_1_std": 0.057352, "nauc_recall_at_1_diff1": 0.645649, "nauc_recall_at_3_max": 0.451117, "nauc_recall_at_3_std": 0.095023, "nauc_recall_at_3_diff1": 0.565937, "nauc_recall_at_5_max": 0.430829, "nauc_recall_at_5_std": 0.117603, "nauc_recall_at_5_diff1": 0.539039, "nauc_recall_at_10_max": 0.456886, "nauc_recall_at_10_std": 0.206466, "nauc_recall_at_10_diff1": 0.54079, "nauc_recall_at_20_max": 0.411736, "nauc_recall_at_20_std": 0.241026, "nauc_recall_at_20_diff1": 0.467252, "nauc_recall_at_100_max": 0.406254, "nauc_recall_at_100_std": 0.446124, "nauc_recall_at_100_diff1": 0.414318, "nauc_recall_at_1000_max": 0.294659, "nauc_recall_at_1000_std": 0.614005, "nauc_recall_at_1000_diff1": 0.356946, "nauc_precision_at_1_max": 0.325031, "nauc_precision_at_1_std": 0.057352, "nauc_precision_at_1_diff1": 0.645649, "nauc_precision_at_3_max": 0.451117, "nauc_precision_at_3_std": 0.095023, "nauc_precision_at_3_diff1": 0.565937, "nauc_precision_at_5_max": 0.430829, "nauc_precision_at_5_std": 0.117603, "nauc_precision_at_5_diff1": 0.539039, "nauc_precision_at_10_max": 0.456886, "nauc_precision_at_10_std": 0.206466, "nauc_precision_at_10_diff1": 0.54079, "nauc_precision_at_20_max": 0.411736, "nauc_precision_at_20_std": 0.241026, "nauc_precision_at_20_diff1": 0.467252, "nauc_precision_at_100_max": 0.406254, "nauc_precision_at_100_std": 0.446124, "nauc_precision_at_100_diff1": 0.414318, "nauc_precision_at_1000_max": 0.294659, "nauc_precision_at_1000_std": 0.614005, "nauc_precision_at_1000_diff1": 0.356946, "nauc_mrr_at_1_max": 0.325031, "nauc_mrr_at_1_std": 0.057352, "nauc_mrr_at_1_diff1": 0.645649, "nauc_mrr_at_3_max": 0.391215, "nauc_mrr_at_3_std": 0.083526, "nauc_mrr_at_3_diff1": 0.60964, "nauc_mrr_at_5_max": 0.385995, "nauc_mrr_at_5_std": 0.089584, "nauc_mrr_at_5_diff1": 0.602327, "nauc_mrr_at_10_max": 0.388514, "nauc_mrr_at_10_std": 0.101472, "nauc_mrr_at_10_diff1": 0.6033, "nauc_mrr_at_20_max": 0.384588, "nauc_mrr_at_20_std": 0.10263, "nauc_mrr_at_20_diff1": 0.597606, "nauc_mrr_at_100_max": 0.384151, "nauc_mrr_at_100_std": 0.108231, "nauc_mrr_at_100_diff1": 0.597106, "nauc_mrr_at_1000_max": 0.383577, "nauc_mrr_at_1000_std": 0.107901, "nauc_mrr_at_1000_diff1": 0.597159, "cv_recall_at_1": 0.25, "cv_recall_at_3": 0.35, "cv_recall_at_5": 0.375, "cv_recall_at_10": 0.425, "cv_recall_at_20": 0.475, "cv_recall_at_100": 0.59, "cv_recall_at_1000": 0.81, "main_score": 0.33603, "hf_subset": "ja", "languages": [ "jpn-Jpan" ] } ], "test": [ { "ndcg_at_1": 0.235, "ndcg_at_3": 0.28351, "ndcg_at_5": 0.30826, "ndcg_at_10": 0.32938, "ndcg_at_20": 0.34109, "ndcg_at_100": 0.36737, "ndcg_at_1000": 0.38521, "map_at_1": 0.235, "map_at_3": 0.2725, "map_at_5": 0.28625, "map_at_10": 0.29503, "map_at_20": 0.29844, "map_at_100": 0.30186, "map_at_1000": 0.30244, "recall_at_1": 0.235, "recall_at_3": 0.315, "recall_at_5": 0.375, "recall_at_10": 0.44, "recall_at_20": 0.485, "recall_at_100": 0.63, "recall_at_1000": 0.775, "accuracy": 0.235, "precision_at_1": 0.235, "precision_at_3": 0.105, "precision_at_5": 0.075, "precision_at_10": 0.044, "precision_at_20": 0.02425, "precision_at_100": 0.0063, "precision_at_1000": 0.00078, "mrr_at_1": 0.235, "mrr_at_3": 0.2725, "mrr_at_5": 0.28625, "mrr_at_10": 0.295032, "mrr_at_20": 0.298442, "mrr_at_100": 0.30186, "mrr_at_1000": 0.302439, "nauc_ndcg_at_1_max": 0.350452, "nauc_ndcg_at_1_std": -0.01092, "nauc_ndcg_at_1_diff1": 0.472391, "nauc_ndcg_at_3_max": 0.33448, "nauc_ndcg_at_3_std": 0.023342, "nauc_ndcg_at_3_diff1": 0.403545, "nauc_ndcg_at_5_max": 0.313164, "nauc_ndcg_at_5_std": 0.012375, "nauc_ndcg_at_5_diff1": 0.384017, "nauc_ndcg_at_10_max": 0.306965, "nauc_ndcg_at_10_std": 0.028889, "nauc_ndcg_at_10_diff1": 0.386772, "nauc_ndcg_at_20_max": 0.305766, "nauc_ndcg_at_20_std": 0.05305, "nauc_ndcg_at_20_diff1": 0.373251, "nauc_ndcg_at_100_max": 0.310474, "nauc_ndcg_at_100_std": 0.069266, "nauc_ndcg_at_100_diff1": 0.378967, "nauc_ndcg_at_1000_max": 0.310489, "nauc_ndcg_at_1000_std": 0.086259, "nauc_ndcg_at_1000_diff1": 0.376125, "nauc_map_at_1_max": 0.350452, "nauc_map_at_1_std": -0.01092, "nauc_map_at_1_diff1": 0.472391, "nauc_map_at_3_max": 0.3374, "nauc_map_at_3_std": 0.01517, "nauc_map_at_3_diff1": 0.417154, "nauc_map_at_5_max": 0.324621, "nauc_map_at_5_std": 0.008484, "nauc_map_at_5_diff1": 0.40547, "nauc_map_at_10_max": 0.322662, "nauc_map_at_10_std": 0.017172, "nauc_map_at_10_diff1": 0.40644, "nauc_map_at_20_max": 0.322584, "nauc_map_at_20_std": 0.024119, "nauc_map_at_20_diff1": 0.402468, "nauc_map_at_100_max": 0.322705, "nauc_map_at_100_std": 0.026276, "nauc_map_at_100_diff1": 0.402311, "nauc_map_at_1000_max": 0.3227, "nauc_map_at_1000_std": 0.026936, "nauc_map_at_1000_diff1": 0.402243, "nauc_recall_at_1_max": 0.350452, "nauc_recall_at_1_std": -0.01092, "nauc_recall_at_1_diff1": 0.472391, "nauc_recall_at_3_max": 0.326815, "nauc_recall_at_3_std": 0.045721, "nauc_recall_at_3_diff1": 0.367319, "nauc_recall_at_5_max": 0.280873, "nauc_recall_at_5_std": 0.022016, "nauc_recall_at_5_diff1": 0.32568, "nauc_recall_at_10_max": 0.259664, "nauc_recall_at_10_std": 0.061968, "nauc_recall_at_10_diff1": 0.334414, "nauc_recall_at_20_max": 0.252656, "nauc_recall_at_20_std": 0.15245, "nauc_recall_at_20_diff1": 0.28395, "nauc_recall_at_100_max": 0.27853, "nauc_recall_at_100_std": 0.262014, "nauc_recall_at_100_diff1": 0.317116, "nauc_recall_at_1000_max": 0.263349, "nauc_recall_at_1000_std": 0.543597, "nauc_recall_at_1000_diff1": 0.251574, "nauc_precision_at_1_max": 0.350452, "nauc_precision_at_1_std": -0.01092, "nauc_precision_at_1_diff1": 0.472391, "nauc_precision_at_3_max": 0.326815, "nauc_precision_at_3_std": 0.045721, "nauc_precision_at_3_diff1": 0.367319, "nauc_precision_at_5_max": 0.280873, "nauc_precision_at_5_std": 0.022016, "nauc_precision_at_5_diff1": 0.32568, "nauc_precision_at_10_max": 0.259664, "nauc_precision_at_10_std": 0.061968, "nauc_precision_at_10_diff1": 0.334414, "nauc_precision_at_20_max": 0.252656, "nauc_precision_at_20_std": 0.15245, "nauc_precision_at_20_diff1": 0.28395, "nauc_precision_at_100_max": 0.27853, "nauc_precision_at_100_std": 0.262014, "nauc_precision_at_100_diff1": 0.317116, "nauc_precision_at_1000_max": 0.263349, "nauc_precision_at_1000_std": 0.543597, "nauc_precision_at_1000_diff1": 0.251574, "nauc_mrr_at_1_max": 0.350452, "nauc_mrr_at_1_std": -0.01092, "nauc_mrr_at_1_diff1": 0.472391, "nauc_mrr_at_3_max": 0.3374, "nauc_mrr_at_3_std": 0.01517, "nauc_mrr_at_3_diff1": 0.417154, "nauc_mrr_at_5_max": 0.324621, "nauc_mrr_at_5_std": 0.008484, "nauc_mrr_at_5_diff1": 0.40547, "nauc_mrr_at_10_max": 0.322662, "nauc_mrr_at_10_std": 0.017172, "nauc_mrr_at_10_diff1": 0.40644, "nauc_mrr_at_20_max": 0.322584, "nauc_mrr_at_20_std": 0.024119, "nauc_mrr_at_20_diff1": 0.402468, "nauc_mrr_at_100_max": 0.322705, "nauc_mrr_at_100_std": 0.026276, "nauc_mrr_at_100_diff1": 0.402311, "nauc_mrr_at_1000_max": 0.3227, "nauc_mrr_at_1000_std": 0.026936, "nauc_mrr_at_1000_diff1": 0.402243, "cv_recall_at_1": 0.235, "cv_recall_at_3": 0.315, "cv_recall_at_5": 0.375, "cv_recall_at_10": 0.44, "cv_recall_at_20": 0.485, "cv_recall_at_100": 0.63, "cv_recall_at_1000": 0.775, "main_score": 0.32938, "hf_subset": "ja", "languages": [ "jpn-Jpan" ] } ] }, "evaluation_time": 860.5383894443512, "kg_co2_emissions": null }