{ "dataset_revision": "6f868e5b4c46a9f04c89f7179ddc7c8f162c1068", "task_name": "ToolBench", "mteb_version": "2.3.0", "scores": { "test": [ { "ndcg_at_1": 0.57091, "ndcg_at_3": 0.50892, "ndcg_at_5": 0.55095, "ndcg_at_10": 0.59222, "ndcg_at_20": 0.61752, "ndcg_at_25": 0.6238, "ndcg_at_50": 0.64109, "ndcg_at_100": 0.65166, "ndcg_at_1000": 0.66342, "map_at_1": 0.26462, "map_at_3": 0.43334, "map_at_5": 0.47315, "map_at_10": 0.49889, "map_at_20": 0.50971, "map_at_25": 0.51166, "map_at_50": 0.51572, "map_at_100": 0.51731, "map_at_1000": 0.51809, "recall_at_1": 0.26462, "recall_at_3": 0.49502, "recall_at_5": 0.58423, "recall_at_10": 0.68133, "recall_at_20": 0.75667, "recall_at_25": 0.779, "recall_at_50": 0.84747, "recall_at_100": 0.89611, "recall_at_1000": 0.96494, "accuracy": 0.26462, "precision_at_1": 0.57091, "precision_at_3": 0.36515, "precision_at_5": 0.26364, "precision_at_10": 0.15645, "precision_at_20": 0.08791, "precision_at_25": 0.07244, "precision_at_50": 0.03964, "precision_at_100": 0.0211, "precision_at_1000": 0.00229, "mrr_at_1": 0.570909, "mrr_at_3": 0.651061, "mrr_at_5": 0.66197, "mrr_at_10": 0.668934, "mrr_at_20": 0.67215, "mrr_at_25": 0.672543, "mrr_at_50": 0.673304, "mrr_at_100": 0.673529, "mrr_at_1000": 0.673715, "nauc_ndcg_at_1_max": 0.354629, "nauc_ndcg_at_1_std": -0.238548, "nauc_ndcg_at_1_diff1": 0.534089, "nauc_ndcg_at_3_max": 0.361448, "nauc_ndcg_at_3_std": -0.264809, "nauc_ndcg_at_3_diff1": 0.382577, "nauc_ndcg_at_5_max": 0.37907, "nauc_ndcg_at_5_std": -0.266296, "nauc_ndcg_at_5_diff1": 0.381046, "nauc_ndcg_at_10_max": 0.392531, "nauc_ndcg_at_10_std": -0.253397, "nauc_ndcg_at_10_diff1": 0.386736, "nauc_ndcg_at_20_max": 0.39686, "nauc_ndcg_at_20_std": -0.243712, "nauc_ndcg_at_20_diff1": 0.397606, "nauc_ndcg_at_25_max": 0.397596, "nauc_ndcg_at_25_std": -0.237203, "nauc_ndcg_at_25_diff1": 0.399293, "nauc_ndcg_at_50_max": 0.396677, "nauc_ndcg_at_50_std": -0.220242, "nauc_ndcg_at_50_diff1": 0.403587, "nauc_ndcg_at_100_max": 0.400598, "nauc_ndcg_at_100_std": -0.21587, "nauc_ndcg_at_100_diff1": 0.411488, "nauc_ndcg_at_1000_max": 0.396619, "nauc_ndcg_at_1000_std": -0.217728, "nauc_ndcg_at_1000_diff1": 0.410059, "nauc_map_at_1_max": 0.324798, "nauc_map_at_1_std": -0.221588, "nauc_map_at_1_diff1": 0.535352, "nauc_map_at_3_max": 0.350427, "nauc_map_at_3_std": -0.256996, "nauc_map_at_3_diff1": 0.376411, "nauc_map_at_5_max": 0.364638, "nauc_map_at_5_std": -0.267346, "nauc_map_at_5_diff1": 0.370844, "nauc_map_at_10_max": 0.372475, "nauc_map_at_10_std": -0.262989, "nauc_map_at_10_diff1": 0.373335, "nauc_map_at_20_max": 0.374468, "nauc_map_at_20_std": -0.26033, "nauc_map_at_20_diff1": 0.377368, "nauc_map_at_25_max": 0.374889, "nauc_map_at_25_std": -0.25827, "nauc_map_at_25_diff1": 0.377504, "nauc_map_at_50_max": 0.374315, "nauc_map_at_50_std": -0.255175, "nauc_map_at_50_diff1": 0.378051, "nauc_map_at_100_max": 0.374789, "nauc_map_at_100_std": -0.254635, "nauc_map_at_100_diff1": 0.379092, "nauc_map_at_1000_max": 0.374596, "nauc_map_at_1000_std": -0.254476, "nauc_map_at_1000_diff1": 0.378949, "nauc_recall_at_1_max": 0.324798, "nauc_recall_at_1_std": -0.221588, "nauc_recall_at_1_diff1": 0.535352, "nauc_recall_at_3_max": 0.353425, "nauc_recall_at_3_std": -0.267367, "nauc_recall_at_3_diff1": 0.320118, "nauc_recall_at_5_max": 0.373465, "nauc_recall_at_5_std": -0.265615, "nauc_recall_at_5_diff1": 0.294323, "nauc_recall_at_10_max": 0.395933, "nauc_recall_at_10_std": -0.224577, "nauc_recall_at_10_diff1": 0.283702, "nauc_recall_at_20_max": 0.413548, "nauc_recall_at_20_std": -0.181951, "nauc_recall_at_20_diff1": 0.305521, "nauc_recall_at_25_max": 0.418951, "nauc_recall_at_25_std": -0.150211, "nauc_recall_at_25_diff1": 0.309905, "nauc_recall_at_50_max": 0.438142, "nauc_recall_at_50_std": -0.018998, "nauc_recall_at_50_diff1": 0.32863, "nauc_recall_at_100_max": 0.511724, "nauc_recall_at_100_std": 0.090925, "nauc_recall_at_100_diff1": 0.402572, "nauc_recall_at_1000_max": 0.653441, "nauc_recall_at_1000_std": 0.522971, "nauc_recall_at_1000_diff1": 0.423356, "nauc_precision_at_1_max": 0.354629, "nauc_precision_at_1_std": -0.238548, "nauc_precision_at_1_diff1": 0.534089, "nauc_precision_at_3_max": 0.339678, "nauc_precision_at_3_std": -0.253847, "nauc_precision_at_3_diff1": 0.253064, "nauc_precision_at_5_max": 0.321809, "nauc_precision_at_5_std": -0.225901, "nauc_precision_at_5_diff1": 0.194428, "nauc_precision_at_10_max": 0.279792, "nauc_precision_at_10_std": -0.146347, "nauc_precision_at_10_diff1": 0.135832, "nauc_precision_at_20_max": 0.230132, "nauc_precision_at_20_std": -0.081062, "nauc_precision_at_20_diff1": 0.101314, "nauc_precision_at_25_max": 0.218566, "nauc_precision_at_25_std": -0.048924, "nauc_precision_at_25_diff1": 0.086977, "nauc_precision_at_50_max": 0.153773, "nauc_precision_at_50_std": 0.049456, "nauc_precision_at_50_diff1": 0.031023, "nauc_precision_at_100_max": 0.126565, "nauc_precision_at_100_std": 0.105935, "nauc_precision_at_100_diff1": 0.006916, "nauc_precision_at_1000_max": -0.00324, "nauc_precision_at_1000_std": 0.145353, "nauc_precision_at_1000_diff1": -0.115699, "nauc_mrr_at_1_max": 0.354629, "nauc_mrr_at_1_std": -0.238548, "nauc_mrr_at_1_diff1": 0.534089, "nauc_mrr_at_3_max": 0.381143, "nauc_mrr_at_3_std": -0.251113, "nauc_mrr_at_3_diff1": 0.506485, "nauc_mrr_at_5_max": 0.382554, "nauc_mrr_at_5_std": -0.241981, "nauc_mrr_at_5_diff1": 0.504721, "nauc_mrr_at_10_max": 0.382646, "nauc_mrr_at_10_std": -0.238627, "nauc_mrr_at_10_diff1": 0.507873, "nauc_mrr_at_20_max": 0.383237, "nauc_mrr_at_20_std": -0.236612, "nauc_mrr_at_20_diff1": 0.509467, "nauc_mrr_at_25_max": 0.383104, "nauc_mrr_at_25_std": -0.236286, "nauc_mrr_at_25_diff1": 0.5099, "nauc_mrr_at_50_max": 0.383366, "nauc_mrr_at_50_std": -0.234714, "nauc_mrr_at_50_diff1": 0.50997, "nauc_mrr_at_100_max": 0.383381, "nauc_mrr_at_100_std": -0.234918, "nauc_mrr_at_100_diff1": 0.510143, "nauc_mrr_at_1000_max": 0.383321, "nauc_mrr_at_1000_std": -0.235062, "nauc_mrr_at_1000_diff1": 0.510126, "hit_rate_at_1": 0.57091, "hit_rate_at_3": 0.75182, "hit_rate_at_5": 0.8, "hit_rate_at_10": 0.85182, "hit_rate_at_20": 0.89636, "hit_rate_at_25": 0.90545, "hit_rate_at_50": 0.93182, "hit_rate_at_100": 0.94727, "hit_rate_at_1000": 0.98455, "R_cap_at_1": 0.570909, "R_cap_at_3": 0.503939, "R_cap_at_5": 0.584591, "R_cap_at_10": 0.681333, "R_cap_at_20": 0.756667, "R_cap_at_25": 0.779, "R_cap_at_50": 0.84747, "R_cap_at_100": 0.896106, "R_cap_at_1000": 0.964939, "main_score": 0.59222, "hf_subset": "ToolBench", "languages": [ "eng-Latn" ], "mteb_version": "2.3.0" } ] }, "evaluation_time": 76.94922871515155, "kg_co2_emissions": null, "date": 1785767512.702282, "evaluation_phases": [ { "name": "Data loading", "start": 5.852431058883667e-06, "end": 24.3057064935565, "split": "", "subset": "" }, { "name": "Dataset transform", "start": 24.305720143020153, "end": 24.305722873657942, "split": "", "subset": "" }, { "name": "Encoding queries and documents", "start": 24.325102135539055, "end": 75.48949540033937, "split": "test", "subset": "ToolBench" }, { "name": "Scoring", "start": 75.4896153844893, "end": 76.66493117064238, "split": "test", "subset": "ToolBench" } ] }