{ "dataset_revision": "d56462d0e63a25450459c4f213e49ffdb866f7f9", "task_name": "SciFact", "mteb_version": "2.21.0", "scores": { "test": [ { "ndcg_at_1": 0.64667, "ndcg_at_3": 0.70137, "ndcg_at_5": 0.71913, "ndcg_at_10": 0.74177, "ndcg_at_20": 0.75245, "ndcg_at_100": 0.7653, "ndcg_at_1000": 0.77104, "map_at_1": 0.61967, "map_at_3": 0.67808, "map_at_5": 0.69122, "map_at_10": 0.70272, "map_at_20": 0.70607, "map_at_100": 0.70806, "map_at_1000": 0.70834, "recall_at_1": 0.61967, "recall_at_3": 0.73961, "recall_at_5": 0.78433, "recall_at_10": 0.84717, "recall_at_20": 0.88867, "recall_at_100": 0.95433, "recall_at_1000": 0.99667, "accuracy": 0.61967, "precision_at_1": 0.64667, "precision_at_3": 0.26778, "precision_at_5": 0.17333, "precision_at_10": 0.096, "precision_at_20": 0.0505, "precision_at_100": 0.0108, "precision_at_1000": 0.00113, "mrr_at_1": 0.646667, "mrr_at_3": 0.698889, "mrr_at_5": 0.707389, "mrr_at_10": 0.714907, "mrr_at_20": 0.717259, "mrr_at_100": 0.719159, "mrr_at_1000": 0.719392, "nauc_ndcg_at_1_max": 0.493499, "nauc_ndcg_at_1_std": 0.014818, "nauc_ndcg_at_1_diff1": 0.722744, "nauc_ndcg_at_3_max": 0.540198, "nauc_ndcg_at_3_std": -0.008901, "nauc_ndcg_at_3_diff1": 0.696842, "nauc_ndcg_at_5_max": 0.542535, "nauc_ndcg_at_5_std": -0.000686, "nauc_ndcg_at_5_diff1": 0.694858, "nauc_ndcg_at_10_max": 0.559916, "nauc_ndcg_at_10_std": 0.027287, "nauc_ndcg_at_10_diff1": 0.695902, "nauc_ndcg_at_20_max": 0.553846, "nauc_ndcg_at_20_std": 0.029235, "nauc_ndcg_at_20_diff1": 0.692424, "nauc_ndcg_at_100_max": 0.553384, "nauc_ndcg_at_100_std": 0.02388, "nauc_ndcg_at_100_diff1": 0.697481, "nauc_ndcg_at_1000_max": 0.548779, "nauc_ndcg_at_1000_std": 0.024198, "nauc_ndcg_at_1000_diff1": 0.697463, "nauc_map_at_1_max": 0.467699, "nauc_map_at_1_std": -0.0201, "nauc_map_at_1_diff1": 0.707789, "nauc_map_at_3_max": 0.518725, "nauc_map_at_3_std": -0.025468, "nauc_map_at_3_diff1": 0.706364, "nauc_map_at_5_max": 0.526116, "nauc_map_at_5_std": -0.010988, "nauc_map_at_5_diff1": 0.705872, "nauc_map_at_10_max": 0.53784, "nauc_map_at_10_std": 0.006379, "nauc_map_at_10_diff1": 0.70669, "nauc_map_at_20_max": 0.536917, "nauc_map_at_20_std": 0.008565, "nauc_map_at_20_diff1": 0.705282, "nauc_map_at_100_max": 0.536824, "nauc_map_at_100_std": 0.00703, "nauc_map_at_100_diff1": 0.706149, "nauc_map_at_1000_max": 0.536581, "nauc_map_at_1000_std": 0.00709, "nauc_map_at_1000_diff1": 0.706067, "nauc_recall_at_1_max": 0.467699, "nauc_recall_at_1_std": -0.0201, "nauc_recall_at_1_diff1": 0.707789, "nauc_recall_at_3_max": 0.560749, "nauc_recall_at_3_std": -0.045184, "nauc_recall_at_3_diff1": 0.668799, "nauc_recall_at_5_max": 0.566639, "nauc_recall_at_5_std": -0.021801, "nauc_recall_at_5_diff1": 0.654343, "nauc_recall_at_10_max": 0.633435, "nauc_recall_at_10_std": 0.063239, "nauc_recall_at_10_diff1": 0.645584, "nauc_recall_at_20_max": 0.605473, "nauc_recall_at_20_std": 0.069984, "nauc_recall_at_20_diff1": 0.610176, "nauc_recall_at_100_max": 0.662196, "nauc_recall_at_100_std": 0.055344, "nauc_recall_at_100_diff1": 0.602561, "nauc_recall_at_1000_max": 0.358077, "nauc_recall_at_1000_std": 0.554155, "nauc_recall_at_1000_diff1": -0.171335, "nauc_precision_at_1_max": 0.493499, "nauc_precision_at_1_std": 0.014818, "nauc_precision_at_1_diff1": 0.722744, "nauc_precision_at_3_max": 0.519042, "nauc_precision_at_3_std": 0.074632, "nauc_precision_at_3_diff1": 0.492556, "nauc_precision_at_5_max": 0.462784, "nauc_precision_at_5_std": 0.158612, "nauc_precision_at_5_diff1": 0.355639, "nauc_precision_at_10_max": 0.371617, "nauc_precision_at_10_std": 0.272168, "nauc_precision_at_10_diff1": 0.159625, "nauc_precision_at_20_max": 0.296338, "nauc_precision_at_20_std": 0.291817, "nauc_precision_at_20_diff1": 0.054821, "nauc_precision_at_100_max": 0.201062, "nauc_precision_at_100_std": 0.348451, "nauc_precision_at_100_diff1": -0.103514, "nauc_precision_at_1000_max": 0.084688, "nauc_precision_at_1000_std": 0.403573, "nauc_precision_at_1000_diff1": -0.256626, "nauc_mrr_at_1_max": 0.493499, "nauc_mrr_at_1_std": 0.014818, "nauc_mrr_at_1_diff1": 0.722744, "nauc_mrr_at_3_max": 0.543121, "nauc_mrr_at_3_std": 0.027818, "nauc_mrr_at_3_diff1": 0.704416, "nauc_mrr_at_5_max": 0.537843, "nauc_mrr_at_5_std": 0.028009, "nauc_mrr_at_5_diff1": 0.699419, "nauc_mrr_at_10_max": 0.539942, "nauc_mrr_at_10_std": 0.031725, "nauc_mrr_at_10_diff1": 0.700903, "nauc_mrr_at_20_max": 0.537549, "nauc_mrr_at_20_std": 0.031351, "nauc_mrr_at_20_diff1": 0.70059, "nauc_mrr_at_100_max": 0.537627, "nauc_mrr_at_100_std": 0.029642, "nauc_mrr_at_100_diff1": 0.701607, "nauc_mrr_at_1000_max": 0.537467, "nauc_mrr_at_1000_std": 0.02961, "nauc_mrr_at_1000_diff1": 0.701614, "hit_rate_at_1": 0.64667, "hit_rate_at_3": 0.76667, "hit_rate_at_5": 0.80333, "hit_rate_at_10": 0.85667, "hit_rate_at_20": 0.89333, "hit_rate_at_100": 0.95667, "hit_rate_at_1000": 0.99667, "main_score": 0.74177, "hf_subset": "default", "languages": [ "eng-Latn" ], "mteb_version": "2.21.0" } ] }, "evaluation_time": 2229.605887900001, "kg_co2_emissions": null, "date": 1790673370.071462, "evaluation_phases": [ { "name": "Data loading", "start": 7.300001016119495e-06, "end": 6.1587712000000465, "split": "", "subset": "" }, { "name": "Dataset transform", "start": 6.158780600002501, "end": 6.158782100003009, "split": "", "subset": "" }, { "name": "Encoding queries and documents", "start": 6.170020500001556, "end": 2229.2355786000007, "split": "test", "subset": "default" }, { "name": "Scoring", "start": 2229.235625000001, "end": 2229.5702040000033, "split": "test", "subset": "default" } ] }