{ "dataset_revision": "432cf48c879c2de792720d5c0af6900312ebe7b6", "task_name": "LMEB_SciFact", "mteb_version": "2.3.0", "scores": { "test": [ { "ndcg_at_1": 0.60106, "ndcg_at_3": 0.65478, "ndcg_at_5": 0.70061, "ndcg_at_10": 0.76132, "ndcg_at_20": 0.77411, "ndcg_at_25": 0.77624, "ndcg_at_50": 0.77919, "ndcg_at_100": 0.77919, "ndcg_at_1000": 0.77919, "map_at_1": 0.41995, "map_at_3": 0.57415, "map_at_5": 0.62044, "map_at_10": 0.66525, "map_at_20": 0.67306, "map_at_25": 0.67405, "map_at_50": 0.67533, "map_at_100": 0.67533, "map_at_1000": 0.67533, "recall_at_1": 0.41995, "recall_at_3": 0.68485, "recall_at_5": 0.79908, "recall_at_10": 0.95007, "recall_at_20": 0.9843, "recall_at_25": 0.99072, "recall_at_50": 1.0, "recall_at_100": 1.0, "recall_at_1000": 1.0, "accuracy": 0.41995, "precision_at_1": 0.60106, "precision_at_3": 0.37589, "precision_at_5": 0.27766, "precision_at_10": 0.175, "precision_at_20": 0.09388, "precision_at_25": 0.07617, "precision_at_50": 0.03894, "precision_at_100": 0.01947, "precision_at_1000": 0.00195, "mrr_at_1": 0.601064, "mrr_at_3": 0.714539, "mrr_at_5": 0.728103, "mrr_at_10": 0.738804, "mrr_at_20": 0.73962, "mrr_at_25": 0.73962, "mrr_at_50": 0.73962, "mrr_at_100": 0.73962, "mrr_at_1000": 0.73962, "nauc_ndcg_at_1_max": 0.040103, "nauc_ndcg_at_1_std": -0.078843, "nauc_ndcg_at_1_diff1": 0.533599, "nauc_ndcg_at_3_max": -0.06914, "nauc_ndcg_at_3_std": -0.24251, "nauc_ndcg_at_3_diff1": 0.546756, "nauc_ndcg_at_5_max": 0.071084, "nauc_ndcg_at_5_std": -0.218623, "nauc_ndcg_at_5_diff1": 0.544432, "nauc_ndcg_at_10_max": 0.037408, "nauc_ndcg_at_10_std": -0.239661, "nauc_ndcg_at_10_diff1": 0.541319, "nauc_ndcg_at_20_max": 0.033559, "nauc_ndcg_at_20_std": -0.199298, "nauc_ndcg_at_20_diff1": 0.523323, "nauc_ndcg_at_25_max": 0.035072, "nauc_ndcg_at_25_std": -0.188946, "nauc_ndcg_at_25_diff1": 0.532263, "nauc_ndcg_at_50_max": 0.034015, "nauc_ndcg_at_50_std": -0.161069, "nauc_ndcg_at_50_diff1": 0.529693, "nauc_ndcg_at_100_max": 0.034015, "nauc_ndcg_at_100_std": -0.161069, "nauc_ndcg_at_100_diff1": 0.529693, "nauc_ndcg_at_1000_max": 0.034015, "nauc_ndcg_at_1000_std": -0.161069, "nauc_ndcg_at_1000_diff1": 0.529693, "nauc_map_at_1_max": -0.008756, "nauc_map_at_1_std": -0.230516, "nauc_map_at_1_diff1": 0.565373, "nauc_map_at_3_max": -0.043778, "nauc_map_at_3_std": -0.28327, "nauc_map_at_3_diff1": 0.58454, "nauc_map_at_5_max": 0.032714, "nauc_map_at_5_std": -0.252447, "nauc_map_at_5_diff1": 0.565469, "nauc_map_at_10_max": 0.015573, "nauc_map_at_10_std": -0.242512, "nauc_map_at_10_diff1": 0.550188, "nauc_map_at_20_max": 0.013565, "nauc_map_at_20_std": -0.222642, "nauc_map_at_20_diff1": 0.541866, "nauc_map_at_25_max": 0.014002, "nauc_map_at_25_std": -0.218695, "nauc_map_at_25_diff1": 0.543752, "nauc_map_at_50_max": 0.013724, "nauc_map_at_50_std": -0.210341, "nauc_map_at_50_diff1": 0.543353, "nauc_map_at_100_max": 0.013724, "nauc_map_at_100_std": -0.210341, "nauc_map_at_100_diff1": 0.543353, "nauc_map_at_1000_max": 0.013724, "nauc_map_at_1000_std": -0.210341, "nauc_map_at_1000_diff1": 0.543353, "nauc_recall_at_1_max": -0.008756, "nauc_recall_at_1_std": -0.230516, "nauc_recall_at_1_diff1": 0.565373, "nauc_recall_at_3_max": -0.106381, "nauc_recall_at_3_std": -0.392158, "nauc_recall_at_3_diff1": 0.532943, "nauc_recall_at_5_max": 0.177939, "nauc_recall_at_5_std": -0.335313, "nauc_recall_at_5_diff1": 0.466447, "nauc_recall_at_10_max": 0.138787, "nauc_recall_at_10_std": -0.836067, "nauc_recall_at_10_diff1": 0.470406, "nauc_recall_at_20_max": 0.059701, "nauc_recall_at_20_std": -1.344174, "nauc_recall_at_20_diff1": 0.118831, "nauc_recall_at_25_max": 0.124348, "nauc_recall_at_25_std": -1.703044, "nauc_recall_at_25_diff1": 0.530212, "nauc_recall_at_50_max": NaN, "nauc_recall_at_50_std": NaN, "nauc_recall_at_50_diff1": NaN, "nauc_recall_at_100_max": NaN, "nauc_recall_at_100_std": NaN, "nauc_recall_at_100_diff1": NaN, "nauc_recall_at_1000_max": NaN, "nauc_recall_at_1000_std": NaN, "nauc_recall_at_1000_diff1": NaN, "nauc_precision_at_1_max": 0.040103, "nauc_precision_at_1_std": -0.078843, "nauc_precision_at_1_diff1": 0.533599, "nauc_precision_at_3_max": 0.000532, "nauc_precision_at_3_std": 0.049007, "nauc_precision_at_3_diff1": 0.063245, "nauc_precision_at_5_max": 0.171468, "nauc_precision_at_5_std": 0.208148, "nauc_precision_at_5_diff1": -0.138999, "nauc_precision_at_10_max": 0.052449, "nauc_precision_at_10_std": 0.26082, "nauc_precision_at_10_diff1": -0.280933, "nauc_precision_at_20_max": 0.044219, "nauc_precision_at_20_std": 0.362828, "nauc_precision_at_20_diff1": -0.330939, "nauc_precision_at_25_max": 0.04815, "nauc_precision_at_25_std": 0.38671, "nauc_precision_at_25_diff1": -0.324502, "nauc_precision_at_50_max": 0.045046, "nauc_precision_at_50_std": 0.432342, "nauc_precision_at_50_diff1": -0.325392, "nauc_precision_at_100_max": 0.045046, "nauc_precision_at_100_std": 0.432342, "nauc_precision_at_100_diff1": -0.325392, "nauc_precision_at_1000_max": 0.045046, "nauc_precision_at_1000_std": 0.432342, "nauc_precision_at_1000_diff1": -0.325392, "nauc_mrr_at_1_max": 0.040103, "nauc_mrr_at_1_std": -0.078843, "nauc_mrr_at_1_diff1": 0.533599, "nauc_mrr_at_3_max": 0.028377, "nauc_mrr_at_3_std": -0.119783, "nauc_mrr_at_3_diff1": 0.529598, "nauc_mrr_at_5_max": 0.063353, "nauc_mrr_at_5_std": -0.096948, "nauc_mrr_at_5_diff1": 0.531355, "nauc_mrr_at_10_max": 0.052707, "nauc_mrr_at_10_std": -0.103033, "nauc_mrr_at_10_diff1": 0.533765, "nauc_mrr_at_20_max": 0.049521, "nauc_mrr_at_20_std": -0.102996, "nauc_mrr_at_20_diff1": 0.532614, "nauc_mrr_at_25_max": 0.049521, "nauc_mrr_at_25_std": -0.102996, "nauc_mrr_at_25_diff1": 0.532614, "nauc_mrr_at_50_max": 0.049521, "nauc_mrr_at_50_std": -0.102996, "nauc_mrr_at_50_diff1": 0.532614, "nauc_mrr_at_100_max": 0.049521, "nauc_mrr_at_100_std": -0.102996, "nauc_mrr_at_100_diff1": 0.532614, "nauc_mrr_at_1000_max": 0.049521, "nauc_mrr_at_1000_std": -0.102996, "nauc_mrr_at_1000_diff1": 0.532614, "hit_rate_at_1": 0.60106, "hit_rate_at_3": 0.8617, "hit_rate_at_5": 0.92021, "hit_rate_at_10": 0.98936, "hit_rate_at_20": 1.0, "hit_rate_at_25": 1.0, "hit_rate_at_50": 1.0, "hit_rate_at_100": 1.0, "hit_rate_at_1000": 1.0, "R_cap_at_1": 0.601064, "R_cap_at_3": 0.708333, "R_cap_at_5": 0.804433, "R_cap_at_10": 0.950602, "R_cap_at_20": 0.984296, "R_cap_at_25": 0.990723, "R_cap_at_50": 1.0, "R_cap_at_100": 1.0, "R_cap_at_1000": 1.0, "main_score": 0.76132, "hf_subset": "LMEB_SciFact", "languages": [ "eng-Latn" ], "mteb_version": "2.3.0" } ] }, "evaluation_time": 106.55629836022854, "kg_co2_emissions": null, "date": 1786007391.260928, "evaluation_phases": [ { "name": "Data loading", "start": 7.432885468006134e-06, "end": 104.9937266279012, "split": "", "subset": "" }, { "name": "Dataset transform", "start": 104.99373992998153, "end": 104.99374502617866, "split": "", "subset": "" }, { "name": "Encoding queries and documents", "start": 105.0396294305101, "end": 106.24566016159952, "split": "test", "subset": "LMEB_SciFact" }, { "name": "Scoring", "start": 106.24569680262357, "end": 106.44648023508489, "split": "test", "subset": "LMEB_SciFact" } ] }