{ "dataset_revision": "bf110dabd08865a8aff2d40d03696984e76085e0", "task_name": "CovidQA", "mteb_version": "2.3.0", "scores": { "test": [ { "ndcg_at_1": 0.58506, "ndcg_at_3": 0.69801, "ndcg_at_5": 0.72686, "ndcg_at_10": 0.74834, "ndcg_at_20": 0.76198, "ndcg_at_25": 0.76416, "ndcg_at_50": 0.76852, "ndcg_at_100": 0.76852, "ndcg_at_1000": 0.76852, "map_at_1": 0.58506, "map_at_3": 0.67072, "map_at_5": 0.68669, "map_at_10": 0.69552, "map_at_20": 0.69925, "map_at_25": 0.6997, "map_at_50": 0.70035, "map_at_100": 0.70035, "map_at_1000": 0.70035, "recall_at_1": 0.58506, "recall_at_3": 0.77678, "recall_at_5": 0.84698, "recall_at_10": 0.91359, "recall_at_20": 0.9676, "recall_at_25": 0.9775, "recall_at_50": 1.0, "recall_at_100": 1.0, "recall_at_1000": 1.0, "accuracy": 0.58506, "precision_at_1": 0.58506, "precision_at_3": 0.25893, "precision_at_5": 0.1694, "precision_at_10": 0.09136, "precision_at_20": 0.04838, "precision_at_25": 0.0391, "precision_at_50": 0.02, "precision_at_100": 0.01, "precision_at_1000": 0.001, "mrr_at_1": 0.585059, "mrr_at_3": 0.670717, "mrr_at_5": 0.686694, "mrr_at_10": 0.695524, "mrr_at_20": 0.699255, "mrr_at_25": 0.699701, "mrr_at_50": 0.70035, "mrr_at_100": 0.70035, "mrr_at_1000": 0.70035, "nauc_ndcg_at_1_max": 0.261889, "nauc_ndcg_at_1_std": 0.166772, "nauc_ndcg_at_1_diff1": 0.649703, "nauc_ndcg_at_3_max": 0.267907, "nauc_ndcg_at_3_std": 0.204136, "nauc_ndcg_at_3_diff1": 0.594328, "nauc_ndcg_at_5_max": 0.266269, "nauc_ndcg_at_5_std": 0.219844, "nauc_ndcg_at_5_diff1": 0.591882, "nauc_ndcg_at_10_max": 0.271952, "nauc_ndcg_at_10_std": 0.227032, "nauc_ndcg_at_10_diff1": 0.591658, "nauc_ndcg_at_20_max": 0.275705, "nauc_ndcg_at_20_std": 0.214188, "nauc_ndcg_at_20_diff1": 0.602601, "nauc_ndcg_at_25_max": 0.274475, "nauc_ndcg_at_25_std": 0.217899, "nauc_ndcg_at_25_diff1": 0.601689, "nauc_ndcg_at_50_max": 0.269841, "nauc_ndcg_at_50_std": 0.206463, "nauc_ndcg_at_50_diff1": 0.605619, "nauc_ndcg_at_100_max": 0.269841, "nauc_ndcg_at_100_std": 0.206463, "nauc_ndcg_at_100_diff1": 0.605619, "nauc_ndcg_at_1000_max": 0.269841, "nauc_ndcg_at_1000_std": 0.206463, "nauc_ndcg_at_1000_diff1": 0.605619, "nauc_map_at_1_max": 0.261889, "nauc_map_at_1_std": 0.166772, "nauc_map_at_1_diff1": 0.649703, "nauc_map_at_3_max": 0.266077, "nauc_map_at_3_std": 0.192937, "nauc_map_at_3_diff1": 0.608217, "nauc_map_at_5_max": 0.265248, "nauc_map_at_5_std": 0.20048, "nauc_map_at_5_diff1": 0.60765, "nauc_map_at_10_max": 0.267657, "nauc_map_at_10_std": 0.203072, "nauc_map_at_10_diff1": 0.608019, "nauc_map_at_20_max": 0.268667, "nauc_map_at_20_std": 0.200105, "nauc_map_at_20_diff1": 0.610596, "nauc_map_at_25_max": 0.268487, "nauc_map_at_25_std": 0.200745, "nauc_map_at_25_diff1": 0.610455, "nauc_map_at_50_max": 0.267912, "nauc_map_at_50_std": 0.199321, "nauc_map_at_50_diff1": 0.610875, "nauc_map_at_100_max": 0.267912, "nauc_map_at_100_std": 0.199321, "nauc_map_at_100_diff1": 0.610875, "nauc_map_at_1000_max": 0.267912, "nauc_map_at_1000_std": 0.199321, "nauc_map_at_1000_diff1": 0.610875, "nauc_recall_at_1_max": 0.261889, "nauc_recall_at_1_std": 0.166772, "nauc_recall_at_1_diff1": 0.649703, "nauc_recall_at_3_max": 0.274945, "nauc_recall_at_3_std": 0.247108, "nauc_recall_at_3_diff1": 0.542451, "nauc_recall_at_5_max": 0.270937, "nauc_recall_at_5_std": 0.321374, "nauc_recall_at_5_diff1": 0.513152, "nauc_recall_at_10_max": 0.309907, "nauc_recall_at_10_std": 0.434506, "nauc_recall_at_10_diff1": 0.458091, "nauc_recall_at_20_max": 0.448709, "nauc_recall_at_20_std": 0.460999, "nauc_recall_at_20_diff1": 0.501578, "nauc_recall_at_25_max": 0.473292, "nauc_recall_at_25_std": 0.707805, "nauc_recall_at_25_diff1": 0.422274, "nauc_recall_at_50_max": NaN, "nauc_recall_at_50_std": NaN, "nauc_recall_at_50_diff1": NaN, "nauc_recall_at_100_max": NaN, "nauc_recall_at_100_std": NaN, "nauc_recall_at_100_diff1": NaN, "nauc_recall_at_1000_max": NaN, "nauc_recall_at_1000_std": NaN, "nauc_recall_at_1000_diff1": NaN, "nauc_precision_at_1_max": 0.261889, "nauc_precision_at_1_std": 0.166772, "nauc_precision_at_1_diff1": 0.649703, "nauc_precision_at_3_max": 0.274945, "nauc_precision_at_3_std": 0.247108, "nauc_precision_at_3_diff1": 0.542451, "nauc_precision_at_5_max": 0.270937, "nauc_precision_at_5_std": 0.321374, "nauc_precision_at_5_diff1": 0.513152, "nauc_precision_at_10_max": 0.309907, "nauc_precision_at_10_std": 0.434506, "nauc_precision_at_10_diff1": 0.458091, "nauc_precision_at_20_max": 0.448709, "nauc_precision_at_20_std": 0.460999, "nauc_precision_at_20_diff1": 0.501578, "nauc_precision_at_25_max": 0.473292, "nauc_precision_at_25_std": 0.707805, "nauc_precision_at_25_diff1": 0.422274, "nauc_precision_at_50_max": 1.0, "nauc_precision_at_50_std": 1.0, "nauc_precision_at_50_diff1": 1.0, "nauc_precision_at_100_max": 1.0, "nauc_precision_at_100_std": 1.0, "nauc_precision_at_100_diff1": 1.0, "nauc_precision_at_1000_max": 1.0, "nauc_precision_at_1000_std": 1.0, "nauc_precision_at_1000_diff1": 1.0, "nauc_mrr_at_1_max": 0.261889, "nauc_mrr_at_1_std": 0.166772, "nauc_mrr_at_1_diff1": 0.649703, "nauc_mrr_at_3_max": 0.266077, "nauc_mrr_at_3_std": 0.192937, "nauc_mrr_at_3_diff1": 0.608217, "nauc_mrr_at_5_max": 0.265248, "nauc_mrr_at_5_std": 0.20048, "nauc_mrr_at_5_diff1": 0.60765, "nauc_mrr_at_10_max": 0.267657, "nauc_mrr_at_10_std": 0.203072, "nauc_mrr_at_10_diff1": 0.608019, "nauc_mrr_at_20_max": 0.268667, "nauc_mrr_at_20_std": 0.200105, "nauc_mrr_at_20_diff1": 0.610596, "nauc_mrr_at_25_max": 0.268487, "nauc_mrr_at_25_std": 0.200745, "nauc_mrr_at_25_diff1": 0.610455, "nauc_mrr_at_50_max": 0.267912, "nauc_mrr_at_50_std": 0.199321, "nauc_mrr_at_50_diff1": 0.610875, "nauc_mrr_at_100_max": 0.267912, "nauc_mrr_at_100_std": 0.199321, "nauc_mrr_at_100_diff1": 0.610875, "nauc_mrr_at_1000_max": 0.267912, "nauc_mrr_at_1000_std": 0.199321, "nauc_mrr_at_1000_diff1": 0.610875, "hit_rate_at_1": 0.58506, "hit_rate_at_3": 0.77678, "hit_rate_at_5": 0.84698, "hit_rate_at_10": 0.91359, "hit_rate_at_20": 0.9676, "hit_rate_at_25": 0.9775, "hit_rate_at_50": 1.0, "hit_rate_at_100": 1.0, "hit_rate_at_1000": 1.0, "R_cap_at_1": 0.585059, "R_cap_at_3": 0.776778, "R_cap_at_5": 0.846985, "R_cap_at_10": 0.913591, "R_cap_at_20": 0.967597, "R_cap_at_25": 0.977498, "R_cap_at_50": 1.0, "R_cap_at_100": 1.0, "R_cap_at_1000": 1.0, "main_score": 0.74834, "hf_subset": "Covid-QA", "languages": [ "eng-Latn" ], "mteb_version": "2.3.0" } ] }, "evaluation_time": 21.50273930374533, "kg_co2_emissions": null, "date": 1784004283.206075, "evaluation_phases": [ { "name": "Data loading", "start": 5.578622221946716e-06, "end": 19.205509307794273, "split": "", "subset": "" }, { "name": "Dataset transform", "start": 19.20553160086274, "end": 19.205534449778497, "split": "", "subset": "" }, { "name": "Encoding queries and documents", "start": 19.24271593708545, "end": 21.163038545288146, "split": "test", "subset": "Covid-QA" }, { "name": "Scoring", "start": 21.163073337636888, "end": 21.385610576719046, "split": "test", "subset": "Covid-QA" } ] }