{ "dataset_revision": "d213819e87aab9010628da8b73ab4eb337c89340", "task_name": "CodeFeedbackST", "mteb_version": "2.15.4", "scores": { "test": [ { "ndcg_at_1": 0.72162, "ndcg_at_3": 0.82894, "ndcg_at_5": 0.84405, "ndcg_at_10": 0.85207, "ndcg_at_20": 0.85553, "ndcg_at_100": 0.85854, "ndcg_at_1000": 0.85931, "map_at_1": 0.72162, "map_at_3": 0.80372, "map_at_5": 0.81217, "map_at_10": 0.81556, "map_at_20": 0.81653, "map_at_100": 0.81697, "map_at_1000": 0.817, "recall_at_1": 0.72162, "recall_at_3": 0.90146, "recall_at_5": 0.93787, "recall_at_10": 0.96224, "recall_at_20": 0.97576, "recall_at_100": 0.99173, "recall_at_1000": 0.9976, "accuracy": 0.72162, "precision_at_1": 0.72162, "precision_at_3": 0.30049, "precision_at_5": 0.18757, "precision_at_10": 0.09622, "precision_at_20": 0.04879, "precision_at_100": 0.00992, "precision_at_1000": 0.001, "mrr_at_1": 0.728934, "mrr_at_3": 0.8087, "mrr_at_5": 0.816673, "mrr_at_10": 0.820009, "mrr_at_20": 0.820976, "mrr_at_100": 0.821412, "mrr_at_1000": 0.821444, "nauc_ndcg_at_1_max": 0.213446, "nauc_ndcg_at_1_std": -0.331527, "nauc_ndcg_at_1_diff1": 0.808372, "nauc_ndcg_at_3_max": 0.217756, "nauc_ndcg_at_3_std": -0.383391, "nauc_ndcg_at_3_diff1": 0.758249, "nauc_ndcg_at_5_max": 0.216471, "nauc_ndcg_at_5_std": -0.384606, "nauc_ndcg_at_5_diff1": 0.762249, "nauc_ndcg_at_10_max": 0.213656, "nauc_ndcg_at_10_std": -0.378738, "nauc_ndcg_at_10_diff1": 0.767071, "nauc_ndcg_at_20_max": 0.214478, "nauc_ndcg_at_20_std": -0.371859, "nauc_ndcg_at_20_diff1": 0.769607, "nauc_ndcg_at_100_max": 0.217292, "nauc_ndcg_at_100_std": -0.366077, "nauc_ndcg_at_100_diff1": 0.772645, "nauc_ndcg_at_1000_max": 0.216795, "nauc_ndcg_at_1000_std": -0.366298, "nauc_ndcg_at_1000_diff1": 0.773324, "nauc_map_at_1_max": 0.213446, "nauc_map_at_1_std": -0.331527, "nauc_map_at_1_diff1": 0.808372, "nauc_map_at_3_max": 0.216975, "nauc_map_at_3_std": -0.367249, "nauc_map_at_3_diff1": 0.77308, "nauc_map_at_5_max": 0.216275, "nauc_map_at_5_std": -0.367208, "nauc_map_at_5_diff1": 0.775646, "nauc_map_at_10_max": 0.215201, "nauc_map_at_10_std": -0.364973, "nauc_map_at_10_diff1": 0.777541, "nauc_map_at_20_max": 0.215421, "nauc_map_at_20_std": -0.363391, "nauc_map_at_20_diff1": 0.778167, "nauc_map_at_100_max": 0.215581, "nauc_map_at_100_std": -0.362818, "nauc_map_at_100_diff1": 0.778517, "nauc_map_at_1000_max": 0.215585, "nauc_map_at_1000_std": -0.362806, "nauc_map_at_1000_diff1": 0.778537, "nauc_recall_at_1_max": 0.213446, "nauc_recall_at_1_std": -0.331527, "nauc_recall_at_1_diff1": 0.808372, "nauc_recall_at_3_max": 0.22135, "nauc_recall_at_3_std": -0.466745, "nauc_recall_at_3_diff1": 0.682278, "nauc_recall_at_5_max": 0.217006, "nauc_recall_at_5_std": -0.520102, "nauc_recall_at_5_diff1": 0.660427, "nauc_recall_at_10_max": 0.189345, "nauc_recall_at_10_std": -0.54118, "nauc_recall_at_10_diff1": 0.647454, "nauc_recall_at_20_max": 0.191768, "nauc_recall_at_20_std": -0.485124, "nauc_recall_at_20_diff1": 0.634283, "nauc_recall_at_100_max": 0.422555, "nauc_recall_at_100_std": -0.204327, "nauc_recall_at_100_diff1": 0.625842, "nauc_recall_at_1000_max": 0.651086, "nauc_recall_at_1000_std": 0.042189, "nauc_recall_at_1000_diff1": 0.549822, "nauc_precision_at_1_max": 0.213446, "nauc_precision_at_1_std": -0.331527, "nauc_precision_at_1_diff1": 0.808372, "nauc_precision_at_3_max": 0.22135, "nauc_precision_at_3_std": -0.466745, "nauc_precision_at_3_diff1": 0.682278, "nauc_precision_at_5_max": 0.217006, "nauc_precision_at_5_std": -0.520102, "nauc_precision_at_5_diff1": 0.660427, "nauc_precision_at_10_max": 0.189345, "nauc_precision_at_10_std": -0.54118, "nauc_precision_at_10_diff1": 0.647454, "nauc_precision_at_20_max": 0.191768, "nauc_precision_at_20_std": -0.485124, "nauc_precision_at_20_diff1": 0.634283, "nauc_precision_at_100_max": 0.422555, "nauc_precision_at_100_std": -0.204327, "nauc_precision_at_100_diff1": 0.625842, "nauc_precision_at_1000_max": 0.651086, "nauc_precision_at_1000_std": 0.042189, "nauc_precision_at_1000_diff1": 0.549822, "nauc_mrr_at_1_max": 0.21218, "nauc_mrr_at_1_std": -0.334799, "nauc_mrr_at_1_diff1": 0.795665, "nauc_mrr_at_3_max": 0.217541, "nauc_mrr_at_3_std": -0.371046, "nauc_mrr_at_3_diff1": 0.763174, "nauc_mrr_at_5_max": 0.216351, "nauc_mrr_at_5_std": -0.370878, "nauc_mrr_at_5_diff1": 0.765578, "nauc_mrr_at_10_max": 0.215285, "nauc_mrr_at_10_std": -0.368676, "nauc_mrr_at_10_diff1": 0.767373, "nauc_mrr_at_20_max": 0.215508, "nauc_mrr_at_20_std": -0.367079, "nauc_mrr_at_20_diff1": 0.76796, "nauc_mrr_at_100_max": 0.215669, "nauc_mrr_at_100_std": -0.366505, "nauc_mrr_at_100_diff1": 0.768289, "nauc_mrr_at_1000_max": 0.215674, "nauc_mrr_at_1000_std": -0.366494, "nauc_mrr_at_1000_diff1": 0.768308, "hit_rate_at_1": 0.72162, "hit_rate_at_3": 0.90146, "hit_rate_at_5": 0.93787, "hit_rate_at_10": 0.96224, "hit_rate_at_20": 0.97576, "hit_rate_at_100": 0.99173, "hit_rate_at_1000": 0.9976, "main_score": 0.85207, "hf_subset": "default", "languages": [ "eng-Latn" ], "mteb_version": "2.15.4" } ] }, "evaluation_time": 1897.036269961999, "kg_co2_emissions": null, "date": 1781269004.458324, "evaluation_phases": [ { "name": "Data loading", "start": 0.00040796800021780655, "end": 9.976467409000179, "split": "", "subset": "" }, { "name": "Dataset transform", "start": 9.976478638000117, "end": 9.976480416999948, "split": "", "subset": "" }, { "name": "Encoding queries and documents", "start": 10.135290240000359, "end": 1875.8459057750006, "split": "test", "subset": "default" }, { "name": "Scoring", "start": 1875.8468834220002, "end": 1896.4595569410003, "split": "test", "subset": "default" } ] }