{ "dataset_revision": "6f868e5b4c46a9f04c89f7179ddc7c8f162c1068", "task_name": "ToolBench", "mteb_version": "2.3.0", "scores": { "test": [ { "ndcg_at_1": 0.52909, "ndcg_at_3": 0.46629, "ndcg_at_5": 0.49875, "ndcg_at_10": 0.53889, "ndcg_at_20": 0.56196, "ndcg_at_25": 0.56714, "ndcg_at_50": 0.58233, "ndcg_at_100": 0.59441, "ndcg_at_1000": 0.61315, "map_at_1": 0.24752, "map_at_3": 0.39649, "map_at_5": 0.42688, "map_at_10": 0.45084, "map_at_20": 0.46001, "map_at_25": 0.46156, "map_at_50": 0.46494, "map_at_100": 0.46661, "map_at_1000": 0.4677, "recall_at_1": 0.24752, "recall_at_3": 0.45003, "recall_at_5": 0.52074, "recall_at_10": 0.61568, "recall_at_20": 0.68582, "recall_at_25": 0.70353, "recall_at_50": 0.76332, "recall_at_100": 0.82035, "recall_at_1000": 0.9332, "accuracy": 0.24752, "precision_at_1": 0.52909, "precision_at_3": 0.33242, "precision_at_5": 0.23491, "precision_at_10": 0.14082, "precision_at_20": 0.07909, "precision_at_25": 0.06516, "precision_at_50": 0.03558, "precision_at_100": 0.01922, "precision_at_1000": 0.00221, "mrr_at_1": 0.529091, "mrr_at_3": 0.602576, "mrr_at_5": 0.613258, "mrr_at_10": 0.622862, "mrr_at_20": 0.626087, "mrr_at_25": 0.626473, "mrr_at_50": 0.627427, "mrr_at_100": 0.627801, "mrr_at_1000": 0.62803, "nauc_ndcg_at_1_max": 0.400717, "nauc_ndcg_at_1_std": -0.048354, "nauc_ndcg_at_1_diff1": 0.597711, "nauc_ndcg_at_3_max": 0.408601, "nauc_ndcg_at_3_std": -0.056331, "nauc_ndcg_at_3_diff1": 0.457681, "nauc_ndcg_at_5_max": 0.398831, "nauc_ndcg_at_5_std": -0.067031, "nauc_ndcg_at_5_diff1": 0.436766, "nauc_ndcg_at_10_max": 0.404091, "nauc_ndcg_at_10_std": -0.061544, "nauc_ndcg_at_10_diff1": 0.441996, "nauc_ndcg_at_20_max": 0.409593, "nauc_ndcg_at_20_std": -0.057632, "nauc_ndcg_at_20_diff1": 0.446603, "nauc_ndcg_at_25_max": 0.412926, "nauc_ndcg_at_25_std": -0.05105, "nauc_ndcg_at_25_diff1": 0.447026, "nauc_ndcg_at_50_max": 0.422074, "nauc_ndcg_at_50_std": -0.035631, "nauc_ndcg_at_50_diff1": 0.449385, "nauc_ndcg_at_100_max": 0.424135, "nauc_ndcg_at_100_std": -0.027307, "nauc_ndcg_at_100_diff1": 0.452201, "nauc_ndcg_at_1000_max": 0.426013, "nauc_ndcg_at_1000_std": -0.020872, "nauc_ndcg_at_1000_diff1": 0.458076, "nauc_map_at_1_max": 0.357896, "nauc_map_at_1_std": -0.076707, "nauc_map_at_1_diff1": 0.556208, "nauc_map_at_3_max": 0.398972, "nauc_map_at_3_std": -0.059684, "nauc_map_at_3_diff1": 0.44653, "nauc_map_at_5_max": 0.39138, "nauc_map_at_5_std": -0.065331, "nauc_map_at_5_diff1": 0.425393, "nauc_map_at_10_max": 0.396554, "nauc_map_at_10_std": -0.064722, "nauc_map_at_10_diff1": 0.429177, "nauc_map_at_20_max": 0.400038, "nauc_map_at_20_std": -0.062078, "nauc_map_at_20_diff1": 0.432323, "nauc_map_at_25_max": 0.400918, "nauc_map_at_25_std": -0.060288, "nauc_map_at_25_diff1": 0.43263, "nauc_map_at_50_max": 0.40289, "nauc_map_at_50_std": -0.056839, "nauc_map_at_50_diff1": 0.433178, "nauc_map_at_100_max": 0.403142, "nauc_map_at_100_std": -0.055836, "nauc_map_at_100_diff1": 0.4336, "nauc_map_at_1000_max": 0.403184, "nauc_map_at_1000_std": -0.055255, "nauc_map_at_1000_diff1": 0.433891, "nauc_recall_at_1_max": 0.357896, "nauc_recall_at_1_std": -0.076707, "nauc_recall_at_1_diff1": 0.556208, "nauc_recall_at_3_max": 0.389788, "nauc_recall_at_3_std": -0.074233, "nauc_recall_at_3_diff1": 0.38756, "nauc_recall_at_5_max": 0.36311, "nauc_recall_at_5_std": -0.082716, "nauc_recall_at_5_diff1": 0.329571, "nauc_recall_at_10_max": 0.36066, "nauc_recall_at_10_std": -0.07375, "nauc_recall_at_10_diff1": 0.322012, "nauc_recall_at_20_max": 0.369727, "nauc_recall_at_20_std": -0.068144, "nauc_recall_at_20_diff1": 0.320111, "nauc_recall_at_25_max": 0.380004, "nauc_recall_at_25_std": -0.043649, "nauc_recall_at_25_diff1": 0.313959, "nauc_recall_at_50_max": 0.420034, "nauc_recall_at_50_std": 0.027367, "nauc_recall_at_50_diff1": 0.308357, "nauc_recall_at_100_max": 0.435214, "nauc_recall_at_100_std": 0.100515, "nauc_recall_at_100_diff1": 0.295551, "nauc_recall_at_1000_max": 0.560678, "nauc_recall_at_1000_std": 0.372747, "nauc_recall_at_1000_diff1": 0.277396, "nauc_precision_at_1_max": 0.400717, "nauc_precision_at_1_std": -0.048354, "nauc_precision_at_1_diff1": 0.597711, "nauc_precision_at_3_max": 0.392698, "nauc_precision_at_3_std": -0.022882, "nauc_precision_at_3_diff1": 0.363024, "nauc_precision_at_5_max": 0.337793, "nauc_precision_at_5_std": -0.026336, "nauc_precision_at_5_diff1": 0.277337, "nauc_precision_at_10_max": 0.299303, "nauc_precision_at_10_std": 0.000472, "nauc_precision_at_10_diff1": 0.23522, "nauc_precision_at_20_max": 0.27494, "nauc_precision_at_20_std": 0.028953, "nauc_precision_at_20_diff1": 0.201002, "nauc_precision_at_25_max": 0.270203, "nauc_precision_at_25_std": 0.054807, "nauc_precision_at_25_diff1": 0.187862, "nauc_precision_at_50_max": 0.271849, "nauc_precision_at_50_std": 0.133581, "nauc_precision_at_50_diff1": 0.149214, "nauc_precision_at_100_max": 0.222099, "nauc_precision_at_100_std": 0.180853, "nauc_precision_at_100_diff1": 0.100356, "nauc_precision_at_1000_max": 0.108857, "nauc_precision_at_1000_std": 0.25685, "nauc_precision_at_1000_diff1": -0.01555, "nauc_mrr_at_1_max": 0.400717, "nauc_mrr_at_1_std": -0.048354, "nauc_mrr_at_1_diff1": 0.597711, "nauc_mrr_at_3_max": 0.422736, "nauc_mrr_at_3_std": -0.049639, "nauc_mrr_at_3_diff1": 0.572353, "nauc_mrr_at_5_max": 0.422552, "nauc_mrr_at_5_std": -0.049589, "nauc_mrr_at_5_diff1": 0.570782, "nauc_mrr_at_10_max": 0.421548, "nauc_mrr_at_10_std": -0.043502, "nauc_mrr_at_10_diff1": 0.573063, "nauc_mrr_at_20_max": 0.421086, "nauc_mrr_at_20_std": -0.044312, "nauc_mrr_at_20_diff1": 0.572139, "nauc_mrr_at_25_max": 0.421554, "nauc_mrr_at_25_std": -0.043706, "nauc_mrr_at_25_diff1": 0.572164, "nauc_mrr_at_50_max": 0.421881, "nauc_mrr_at_50_std": -0.04337, "nauc_mrr_at_50_diff1": 0.572336, "nauc_mrr_at_100_max": 0.422061, "nauc_mrr_at_100_std": -0.043187, "nauc_mrr_at_100_diff1": 0.572521, "nauc_mrr_at_1000_max": 0.422077, "nauc_mrr_at_1000_std": -0.04312, "nauc_mrr_at_1000_diff1": 0.572557, "hit_rate_at_1": 0.52909, "hit_rate_at_3": 0.69364, "hit_rate_at_5": 0.74091, "hit_rate_at_10": 0.81273, "hit_rate_at_20": 0.85818, "hit_rate_at_25": 0.86727, "hit_rate_at_50": 0.9, "hit_rate_at_100": 0.92636, "hit_rate_at_1000": 0.97273, "R_cap_at_1": 0.529091, "R_cap_at_3": 0.458333, "R_cap_at_5": 0.521015, "R_cap_at_10": 0.615682, "R_cap_at_20": 0.685818, "R_cap_at_25": 0.70353, "R_cap_at_50": 0.763318, "R_cap_at_100": 0.820348, "R_cap_at_1000": 0.933197, "main_score": 0.53889, "hf_subset": "ToolBench", "languages": [ "eng-Latn" ], "mteb_version": "2.3.0" } ] }, "evaluation_time": 779.3782968465239, "kg_co2_emissions": null, "date": 1785862902.600119, "evaluation_phases": [ { "name": "Data loading", "start": 8.33161175251007e-06, "end": 759.4875403642654, "split": "", "subset": "" }, { "name": "Dataset transform", "start": 759.4875550456345, "end": 759.4875594712794, "split": "", "subset": "" }, { "name": "Encoding queries and documents", "start": 759.5095468554646, "end": 778.1335495784879, "split": "test", "subset": "ToolBench" }, { "name": "Scoring", "start": 778.1335930712521, "end": 779.0797492396086, "split": "test", "subset": "ToolBench" } ] }