{ "dataset_revision": "27a168819829fe9bcd655c2df245fb19452e8e06", "task_name": "FiQA2018", "mteb_version": "1.18.0", "scores": { "train": [ { "ndcg_at_1": 0.40509, "ndcg_at_3": 0.38231, "ndcg_at_5": 0.39696, "ndcg_at_10": 0.42289, "ndcg_at_20": 0.44938, "ndcg_at_100": 0.48915, "ndcg_at_1000": 0.51675, "map_at_1": 0.21715, "map_at_3": 0.30368, "map_at_5": 0.32705, "map_at_10": 0.34524, "map_at_20": 0.35567, "map_at_100": 0.36381, "map_at_1000": 0.36545, "recall_at_1": 0.21715, "recall_at_3": 0.35434, "recall_at_5": 0.41732, "recall_at_10": 0.49564, "recall_at_20": 0.5795, "recall_at_100": 0.74293, "recall_at_1000": 0.90918, "precision_at_1": 0.40509, "precision_at_3": 0.24721, "precision_at_5": 0.18153, "precision_at_10": 0.1126, "precision_at_20": 0.0676, "precision_at_100": 0.01813, "precision_at_1000": 0.0023, "mrr_at_1": 0.405091, "mrr_at_3": 0.470545, "mrr_at_5": 0.484891, "mrr_at_10": 0.495365, "mrr_at_20": 0.500485, "mrr_at_100": 0.503547, "mrr_at_1000": 0.503921, "nauc_ndcg_at_1_max": 0.299308, "nauc_ndcg_at_1_std": 0.014225, "nauc_ndcg_at_1_diff1": 0.478661, "nauc_ndcg_at_3_max": 0.230444, "nauc_ndcg_at_3_std": 0.010601, "nauc_ndcg_at_3_diff1": 0.386935, "nauc_ndcg_at_5_max": 0.209495, "nauc_ndcg_at_5_std": 0.01365, "nauc_ndcg_at_5_diff1": 0.383215, "nauc_ndcg_at_10_max": 0.21043, "nauc_ndcg_at_10_std": 0.026182, "nauc_ndcg_at_10_diff1": 0.381094, "nauc_ndcg_at_20_max": 0.216555, "nauc_ndcg_at_20_std": 0.043566, "nauc_ndcg_at_20_diff1": 0.37689, "nauc_ndcg_at_100_max": 0.237338, "nauc_ndcg_at_100_std": 0.064348, "nauc_ndcg_at_100_diff1": 0.379047, "nauc_ndcg_at_1000_max": 0.250338, "nauc_ndcg_at_1000_std": 0.066263, "nauc_ndcg_at_1000_diff1": 0.383614, "nauc_map_at_1_max": 0.107289, "nauc_map_at_1_std": -0.037893, "nauc_map_at_1_diff1": 0.433775, "nauc_map_at_3_max": 0.148897, "nauc_map_at_3_std": -0.023186, "nauc_map_at_3_diff1": 0.393041, "nauc_map_at_5_max": 0.161848, "nauc_map_at_5_std": -0.012102, "nauc_map_at_5_diff1": 0.38881, "nauc_map_at_10_max": 0.177402, "nauc_map_at_10_std": -9.8e-05, "nauc_map_at_10_diff1": 0.385427, "nauc_map_at_20_max": 0.183634, "nauc_map_at_20_std": 0.008639, "nauc_map_at_20_diff1": 0.383634, "nauc_map_at_100_max": 0.189904, "nauc_map_at_100_std": 0.01446, "nauc_map_at_100_diff1": 0.383911, "nauc_map_at_1000_max": 0.191156, "nauc_map_at_1000_std": 0.015094, "nauc_map_at_1000_diff1": 0.38403, "nauc_recall_at_1_max": 0.107289, "nauc_recall_at_1_std": -0.037893, "nauc_recall_at_1_diff1": 0.433775, "nauc_recall_at_3_max": 0.118296, "nauc_recall_at_3_std": -0.015112, "nauc_recall_at_3_diff1": 0.329438, "nauc_recall_at_5_max": 0.117683, "nauc_recall_at_5_std": 0.004071, "nauc_recall_at_5_diff1": 0.307182, "nauc_recall_at_10_max": 0.131036, "nauc_recall_at_10_std": 0.039062, "nauc_recall_at_10_diff1": 0.284758, "nauc_recall_at_20_max": 0.12794, "nauc_recall_at_20_std": 0.088797, "nauc_recall_at_20_diff1": 0.254203, "nauc_recall_at_100_max": 0.162795, "nauc_recall_at_100_std": 0.192875, "nauc_recall_at_100_diff1": 0.222544, "nauc_recall_at_1000_max": 0.228036, "nauc_recall_at_1000_std": 0.360733, "nauc_recall_at_1000_diff1": 0.179056, "nauc_precision_at_1_max": 0.299308, "nauc_precision_at_1_std": 0.014225, "nauc_precision_at_1_diff1": 0.478661, "nauc_precision_at_3_max": 0.319897, "nauc_precision_at_3_std": 0.059097, "nauc_precision_at_3_diff1": 0.268846, "nauc_precision_at_5_max": 0.329048, "nauc_precision_at_5_std": 0.088568, "nauc_precision_at_5_diff1": 0.218426, "nauc_precision_at_10_max": 0.353257, "nauc_precision_at_10_std": 0.134001, "nauc_precision_at_10_diff1": 0.161402, "nauc_precision_at_20_max": 0.357531, "nauc_precision_at_20_std": 0.181612, "nauc_precision_at_20_diff1": 0.11135, "nauc_precision_at_100_max": 0.369801, "nauc_precision_at_100_std": 0.227908, "nauc_precision_at_100_diff1": 0.035241, "nauc_precision_at_1000_max": 0.352814, "nauc_precision_at_1000_std": 0.20275, "nauc_precision_at_1000_diff1": -0.025772, "nauc_mrr_at_1_max": 0.299308, "nauc_mrr_at_1_std": 0.014225, "nauc_mrr_at_1_diff1": 0.478661, "nauc_mrr_at_3_max": 0.30434, "nauc_mrr_at_3_std": 0.034239, "nauc_mrr_at_3_diff1": 0.439404, "nauc_mrr_at_5_max": 0.303932, "nauc_mrr_at_5_std": 0.038398, "nauc_mrr_at_5_diff1": 0.436734, "nauc_mrr_at_10_max": 0.305182, "nauc_mrr_at_10_std": 0.042364, "nauc_mrr_at_10_diff1": 0.437323, "nauc_mrr_at_20_max": 0.30535, "nauc_mrr_at_20_std": 0.044179, "nauc_mrr_at_20_diff1": 0.437147, "nauc_mrr_at_100_max": 0.305545, "nauc_mrr_at_100_std": 0.044366, "nauc_mrr_at_100_diff1": 0.437705, "nauc_mrr_at_1000_max": 0.305509, "nauc_mrr_at_1000_std": 0.044228, "nauc_mrr_at_1000_diff1": 0.43782, "main_score": 0.42289, "hf_subset": "default", "languages": [ "eng-Latn" ] } ], "dev": [ { "ndcg_at_1": 0.404, "ndcg_at_3": 0.37228, "ndcg_at_5": 0.38452, "ndcg_at_10": 0.41554, "ndcg_at_20": 0.43877, "ndcg_at_100": 0.48086, "ndcg_at_1000": 0.50668, "map_at_1": 0.21871, "map_at_3": 0.29404, "map_at_5": 0.31818, "map_at_10": 0.33818, "map_at_20": 0.34752, "map_at_100": 0.3561, "map_at_1000": 0.35765, "recall_at_1": 0.21871, "recall_at_3": 0.33521, "recall_at_5": 0.39572, "recall_at_10": 0.49294, "recall_at_20": 0.5635, "recall_at_100": 0.74111, "recall_at_1000": 0.89911, "precision_at_1": 0.404, "precision_at_3": 0.23667, "precision_at_5": 0.1752, "precision_at_10": 0.111, "precision_at_20": 0.0662, "precision_at_100": 0.01784, "precision_at_1000": 0.00223, "mrr_at_1": 0.404, "mrr_at_3": 0.462333, "mrr_at_5": 0.474533, "mrr_at_10": 0.486514, "mrr_at_20": 0.49067, "mrr_at_100": 0.494358, "mrr_at_1000": 0.494739, "nauc_ndcg_at_1_max": 0.370489, "nauc_ndcg_at_1_std": -0.02754, "nauc_ndcg_at_1_diff1": 0.584002, "nauc_ndcg_at_3_max": 0.285487, "nauc_ndcg_at_3_std": 0.02577, "nauc_ndcg_at_3_diff1": 0.461269, "nauc_ndcg_at_5_max": 0.281296, "nauc_ndcg_at_5_std": 0.026407, "nauc_ndcg_at_5_diff1": 0.466762, "nauc_ndcg_at_10_max": 0.28783, "nauc_ndcg_at_10_std": 0.045295, "nauc_ndcg_at_10_diff1": 0.481403, "nauc_ndcg_at_20_max": 0.294024, "nauc_ndcg_at_20_std": 0.065667, "nauc_ndcg_at_20_diff1": 0.476839, "nauc_ndcg_at_100_max": 0.315252, "nauc_ndcg_at_100_std": 0.089696, "nauc_ndcg_at_100_diff1": 0.476942, "nauc_ndcg_at_1000_max": 0.32422, "nauc_ndcg_at_1000_std": 0.088752, "nauc_ndcg_at_1000_diff1": 0.478164, "nauc_map_at_1_max": 0.195139, "nauc_map_at_1_std": -0.056594, "nauc_map_at_1_diff1": 0.566961, "nauc_map_at_3_max": 0.216465, "nauc_map_at_3_std": -0.015091, "nauc_map_at_3_diff1": 0.495368, "nauc_map_at_5_max": 0.239991, "nauc_map_at_5_std": -0.001088, "nauc_map_at_5_diff1": 0.491975, "nauc_map_at_10_max": 0.248449, "nauc_map_at_10_std": 0.01089, "nauc_map_at_10_diff1": 0.489918, "nauc_map_at_20_max": 0.254131, "nauc_map_at_20_std": 0.02065, "nauc_map_at_20_diff1": 0.485334, "nauc_map_at_100_max": 0.260505, "nauc_map_at_100_std": 0.02941, "nauc_map_at_100_diff1": 0.48484, "nauc_map_at_1000_max": 0.261412, "nauc_map_at_1000_std": 0.02988, "nauc_map_at_1000_diff1": 0.484852, "nauc_recall_at_1_max": 0.195139, "nauc_recall_at_1_std": -0.056594, "nauc_recall_at_1_diff1": 0.566961, "nauc_recall_at_3_max": 0.188341, "nauc_recall_at_3_std": 0.019383, "nauc_recall_at_3_diff1": 0.414257, "nauc_recall_at_5_max": 0.205971, "nauc_recall_at_5_std": 0.031313, "nauc_recall_at_5_diff1": 0.398325, "nauc_recall_at_10_max": 0.205952, "nauc_recall_at_10_std": 0.08124, "nauc_recall_at_10_diff1": 0.372466, "nauc_recall_at_20_max": 0.203847, "nauc_recall_at_20_std": 0.136991, "nauc_recall_at_20_diff1": 0.345022, "nauc_recall_at_100_max": 0.242286, "nauc_recall_at_100_std": 0.253237, "nauc_recall_at_100_diff1": 0.296553, "nauc_recall_at_1000_max": 0.295753, "nauc_recall_at_1000_std": 0.391815, "nauc_recall_at_1000_diff1": 0.195791, "nauc_precision_at_1_max": 0.370489, "nauc_precision_at_1_std": -0.02754, "nauc_precision_at_1_diff1": 0.584002, "nauc_precision_at_3_max": 0.336248, "nauc_precision_at_3_std": 0.07754, "nauc_precision_at_3_diff1": 0.271578, "nauc_precision_at_5_max": 0.33827, "nauc_precision_at_5_std": 0.1069, "nauc_precision_at_5_diff1": 0.206071, "nauc_precision_at_10_max": 0.347905, "nauc_precision_at_10_std": 0.153578, "nauc_precision_at_10_diff1": 0.155311, "nauc_precision_at_20_max": 0.334774, "nauc_precision_at_20_std": 0.20846, "nauc_precision_at_20_diff1": 0.082395, "nauc_precision_at_100_max": 0.331013, "nauc_precision_at_100_std": 0.256811, "nauc_precision_at_100_diff1": -0.0077, "nauc_precision_at_1000_max": 0.29575, "nauc_precision_at_1000_std": 0.217755, "nauc_precision_at_1000_diff1": -0.07456, "nauc_mrr_at_1_max": 0.370489, "nauc_mrr_at_1_std": -0.02754, "nauc_mrr_at_1_diff1": 0.584002, "nauc_mrr_at_3_max": 0.385141, "nauc_mrr_at_3_std": 0.037229, "nauc_mrr_at_3_diff1": 0.53852, "nauc_mrr_at_5_max": 0.383611, "nauc_mrr_at_5_std": 0.034342, "nauc_mrr_at_5_diff1": 0.53707, "nauc_mrr_at_10_max": 0.386252, "nauc_mrr_at_10_std": 0.04018, "nauc_mrr_at_10_diff1": 0.539062, "nauc_mrr_at_20_max": 0.385617, "nauc_mrr_at_20_std": 0.043231, "nauc_mrr_at_20_diff1": 0.539046, "nauc_mrr_at_100_max": 0.385723, "nauc_mrr_at_100_std": 0.04242, "nauc_mrr_at_100_diff1": 0.539386, "nauc_mrr_at_1000_max": 0.385635, "nauc_mrr_at_1000_std": 0.042249, "nauc_mrr_at_1000_diff1": 0.539429, "main_score": 0.41554, "hf_subset": "default", "languages": [ "eng-Latn" ] } ], "test": [ { "ndcg_at_1": 0.41204, "ndcg_at_3": 0.37083, "ndcg_at_5": 0.38708, "ndcg_at_10": 0.40803, "ndcg_at_20": 0.43282, "ndcg_at_100": 0.47319, "ndcg_at_1000": 0.50254, "map_at_1": 0.20514, "map_at_3": 0.28573, "map_at_5": 0.31182, "map_at_10": 0.32944, "map_at_20": 0.33928, "map_at_100": 0.34722, "map_at_1000": 0.34893, "recall_at_1": 0.20514, "recall_at_3": 0.332, "recall_at_5": 0.40114, "recall_at_10": 0.47063, "recall_at_20": 0.54854, "recall_at_100": 0.71556, "recall_at_1000": 0.89127, "precision_at_1": 0.41204, "precision_at_3": 0.24486, "precision_at_5": 0.1858, "precision_at_10": 0.11404, "precision_at_20": 0.06705, "precision_at_100": 0.01798, "precision_at_1000": 0.00233, "mrr_at_1": 0.412037, "mrr_at_3": 0.470679, "mrr_at_5": 0.487114, "mrr_at_10": 0.494691, "mrr_at_20": 0.499662, "mrr_at_100": 0.502872, "mrr_at_1000": 0.503253, "nauc_ndcg_at_1_max": 0.324811, "nauc_ndcg_at_1_std": -0.037527, "nauc_ndcg_at_1_diff1": 0.505968, "nauc_ndcg_at_3_max": 0.27816, "nauc_ndcg_at_3_std": -0.012802, "nauc_ndcg_at_3_diff1": 0.390389, "nauc_ndcg_at_5_max": 0.258236, "nauc_ndcg_at_5_std": 0.000467, "nauc_ndcg_at_5_diff1": 0.374911, "nauc_ndcg_at_10_max": 0.257867, "nauc_ndcg_at_10_std": 0.004502, "nauc_ndcg_at_10_diff1": 0.371201, "nauc_ndcg_at_20_max": 0.256777, "nauc_ndcg_at_20_std": 0.022038, "nauc_ndcg_at_20_diff1": 0.361087, "nauc_ndcg_at_100_max": 0.275013, "nauc_ndcg_at_100_std": 0.041088, "nauc_ndcg_at_100_diff1": 0.370393, "nauc_ndcg_at_1000_max": 0.290227, "nauc_ndcg_at_1000_std": 0.040088, "nauc_ndcg_at_1000_diff1": 0.378752, "nauc_map_at_1_max": 0.159645, "nauc_map_at_1_std": -0.045162, "nauc_map_at_1_diff1": 0.423076, "nauc_map_at_3_max": 0.198603, "nauc_map_at_3_std": -0.031293, "nauc_map_at_3_diff1": 0.386553, "nauc_map_at_5_max": 0.216618, "nauc_map_at_5_std": -0.015847, "nauc_map_at_5_diff1": 0.373889, "nauc_map_at_10_max": 0.233833, "nauc_map_at_10_std": -0.010351, "nauc_map_at_10_diff1": 0.370254, "nauc_map_at_20_max": 0.236645, "nauc_map_at_20_std": -0.002733, "nauc_map_at_20_diff1": 0.365583, "nauc_map_at_100_max": 0.241434, "nauc_map_at_100_std": 0.002271, "nauc_map_at_100_diff1": 0.366755, "nauc_map_at_1000_max": 0.242791, "nauc_map_at_1000_std": 0.002398, "nauc_map_at_1000_diff1": 0.367466, "nauc_recall_at_1_max": 0.159645, "nauc_recall_at_1_std": -0.045162, "nauc_recall_at_1_diff1": 0.423076, "nauc_recall_at_3_max": 0.169919, "nauc_recall_at_3_std": -0.010851, "nauc_recall_at_3_diff1": 0.321538, "nauc_recall_at_5_max": 0.169755, "nauc_recall_at_5_std": 0.019908, "nauc_recall_at_5_diff1": 0.281454, "nauc_recall_at_10_max": 0.184857, "nauc_recall_at_10_std": 0.029915, "nauc_recall_at_10_diff1": 0.262053, "nauc_recall_at_20_max": 0.168192, "nauc_recall_at_20_std": 0.080712, "nauc_recall_at_20_diff1": 0.214772, "nauc_recall_at_100_max": 0.178273, "nauc_recall_at_100_std": 0.174001, "nauc_recall_at_100_diff1": 0.203979, "nauc_recall_at_1000_max": 0.235242, "nauc_recall_at_1000_std": 0.302267, "nauc_recall_at_1000_diff1": 0.159491, "nauc_precision_at_1_max": 0.324811, "nauc_precision_at_1_std": -0.037527, "nauc_precision_at_1_diff1": 0.505968, "nauc_precision_at_3_max": 0.341315, "nauc_precision_at_3_std": 0.015867, "nauc_precision_at_3_diff1": 0.290965, "nauc_precision_at_5_max": 0.341449, "nauc_precision_at_5_std": 0.057736, "nauc_precision_at_5_diff1": 0.210366, "nauc_precision_at_10_max": 0.360407, "nauc_precision_at_10_std": 0.058185, "nauc_precision_at_10_diff1": 0.170419, "nauc_precision_at_20_max": 0.346395, "nauc_precision_at_20_std": 0.099196, "nauc_precision_at_20_diff1": 0.123634, "nauc_precision_at_100_max": 0.36949, "nauc_precision_at_100_std": 0.134324, "nauc_precision_at_100_diff1": 0.084464, "nauc_precision_at_1000_max": 0.359465, "nauc_precision_at_1000_std": 0.100095, "nauc_precision_at_1000_diff1": 0.061776, "nauc_mrr_at_1_max": 0.324811, "nauc_mrr_at_1_std": -0.037527, "nauc_mrr_at_1_diff1": 0.505968, "nauc_mrr_at_3_max": 0.32738, "nauc_mrr_at_3_std": -0.012255, "nauc_mrr_at_3_diff1": 0.470211, "nauc_mrr_at_5_max": 0.32466, "nauc_mrr_at_5_std": -0.003476, "nauc_mrr_at_5_diff1": 0.4597, "nauc_mrr_at_10_max": 0.324345, "nauc_mrr_at_10_std": -0.005828, "nauc_mrr_at_10_diff1": 0.457507, "nauc_mrr_at_20_max": 0.323861, "nauc_mrr_at_20_std": -0.003281, "nauc_mrr_at_20_diff1": 0.45629, "nauc_mrr_at_100_max": 0.32485, "nauc_mrr_at_100_std": -0.001627, "nauc_mrr_at_100_diff1": 0.458821, "nauc_mrr_at_1000_max": 0.324741, "nauc_mrr_at_1000_std": -0.001668, "nauc_mrr_at_1000_diff1": 0.458727, "main_score": 0.40803, "hf_subset": "default", "languages": [ "eng-Latn" ] } ] }, "evaluation_time": 765.0487155914307, "kg_co2_emissions": 0.0727347137302762 }