{ "dataset_revision": "1064f7bba54d5400d4836f5831fe4c2332a566a6", "task_name": "CodeRAGProgrammingSolutions", "mteb_version": "2.22.1", "scores": { "train": [ { "ndcg_at_1": 0.66401, "ndcg_at_3": 0.74149, "ndcg_at_5": 0.75397, "ndcg_at_10": 0.76152, "ndcg_at_20": 0.7633, "ndcg_at_100": 0.76823, "ndcg_at_1000": 0.78126, "map_at_1": 0.66401, "map_at_3": 0.7234, "map_at_5": 0.73036, "map_at_10": 0.73354, "map_at_20": 0.73402, "map_at_100": 0.73465, "map_at_1000": 0.73498, "recall_at_1": 0.66401, "recall_at_3": 0.79344, "recall_at_5": 0.82358, "recall_at_10": 0.84663, "recall_at_20": 0.85372, "recall_at_100": 0.88121, "recall_at_1000": 0.99291, "accuracy": 0.66401, "precision_at_1": 0.66401, "precision_at_3": 0.26448, "precision_at_5": 0.16472, "precision_at_10": 0.08466, "precision_at_20": 0.04269, "precision_at_100": 0.00881, "precision_at_1000": 0.00099, "mrr_at_1": 0.664007, "mrr_at_3": 0.723404, "mrr_at_5": 0.730363, "mrr_at_10": 0.733543, "mrr_at_20": 0.734018, "mrr_at_100": 0.73465, "mrr_at_1000": 0.734981, "nauc_ndcg_at_1_max": 0.603525, "nauc_ndcg_at_1_std": 0.225973, "nauc_ndcg_at_1_diff1": 0.844972, "nauc_ndcg_at_3_max": 0.747941, "nauc_ndcg_at_3_std": 0.415243, "nauc_ndcg_at_3_diff1": 0.851439, "nauc_ndcg_at_5_max": 0.757841, "nauc_ndcg_at_5_std": 0.450268, "nauc_ndcg_at_5_diff1": 0.861637, "nauc_ndcg_at_10_max": 0.762367, "nauc_ndcg_at_10_std": 0.477515, "nauc_ndcg_at_10_diff1": 0.869146, "nauc_ndcg_at_20_max": 0.761304, "nauc_ndcg_at_20_std": 0.479422, "nauc_ndcg_at_20_diff1": 0.868283, "nauc_ndcg_at_100_max": 0.75589, "nauc_ndcg_at_100_std": 0.472817, "nauc_ndcg_at_100_diff1": 0.867127, "nauc_ndcg_at_1000_max": 0.74049, "nauc_ndcg_at_1000_std": 0.440416, "nauc_ndcg_at_1000_diff1": 0.862403, "nauc_map_at_1_max": 0.603525, "nauc_map_at_1_std": 0.225973, "nauc_map_at_1_diff1": 0.844972, "nauc_map_at_3_max": 0.708537, "nauc_map_at_3_std": 0.362727, "nauc_map_at_3_diff1": 0.848222, "nauc_map_at_5_max": 0.712732, "nauc_map_at_5_std": 0.379693, "nauc_map_at_5_diff1": 0.853397, "nauc_map_at_10_max": 0.713759, "nauc_map_at_10_std": 0.38895, "nauc_map_at_10_diff1": 0.856196, "nauc_map_at_20_max": 0.713402, "nauc_map_at_20_std": 0.389198, "nauc_map_at_20_diff1": 0.855979, "nauc_map_at_100_max": 0.712735, "nauc_map_at_100_std": 0.388355, "nauc_map_at_100_diff1": 0.855905, "nauc_map_at_1000_max": 0.712324, "nauc_map_at_1000_std": 0.387545, "nauc_map_at_1000_diff1": 0.85576, "nauc_recall_at_1_max": 0.603525, "nauc_recall_at_1_std": 0.225973, "nauc_recall_at_1_diff1": 0.844972, "nauc_recall_at_3_max": 0.889207, "nauc_recall_at_3_std": 0.603981, "nauc_recall_at_3_diff1": 0.863744, "nauc_recall_at_5_max": 0.941461, "nauc_recall_at_5_std": 0.74185, "nauc_recall_at_5_diff1": 0.897791, "nauc_recall_at_10_max": 0.987773, "nauc_recall_at_10_std": 0.903766, "nauc_recall_at_10_diff1": 0.935399, "nauc_recall_at_20_max": 0.9915, "nauc_recall_at_20_std": 0.934543, "nauc_recall_at_20_diff1": 0.933002, "nauc_recall_at_100_max": 0.984062, "nauc_recall_at_100_std": 0.959301, "nauc_recall_at_100_diff1": 0.932441, "nauc_recall_at_1000_max": 0.983657, "nauc_recall_at_1000_std": 0.965266, "nauc_recall_at_1000_diff1": 0.899894, "nauc_precision_at_1_max": 0.603525, "nauc_precision_at_1_std": 0.225973, "nauc_precision_at_1_diff1": 0.844972, "nauc_precision_at_3_max": 0.889207, "nauc_precision_at_3_std": 0.603981, "nauc_precision_at_3_diff1": 0.863744, "nauc_precision_at_5_max": 0.941461, "nauc_precision_at_5_std": 0.74185, "nauc_precision_at_5_diff1": 0.897791, "nauc_precision_at_10_max": 0.987773, "nauc_precision_at_10_std": 0.903766, "nauc_precision_at_10_diff1": 0.935399, "nauc_precision_at_20_max": 0.9915, "nauc_precision_at_20_std": 0.934543, "nauc_precision_at_20_diff1": 0.933002, "nauc_precision_at_100_max": 0.984062, "nauc_precision_at_100_std": 0.959301, "nauc_precision_at_100_diff1": 0.932441, "nauc_precision_at_1000_max": 0.983657, "nauc_precision_at_1000_std": 0.965266, "nauc_precision_at_1000_diff1": 0.899894, "nauc_mrr_at_1_max": 0.603525, "nauc_mrr_at_1_std": 0.225973, "nauc_mrr_at_1_diff1": 0.844972, "nauc_mrr_at_3_max": 0.708537, "nauc_mrr_at_3_std": 0.362727, "nauc_mrr_at_3_diff1": 0.848222, "nauc_mrr_at_5_max": 0.712732, "nauc_mrr_at_5_std": 0.379693, "nauc_mrr_at_5_diff1": 0.853397, "nauc_mrr_at_10_max": 0.713759, "nauc_mrr_at_10_std": 0.38895, "nauc_mrr_at_10_diff1": 0.856196, "nauc_mrr_at_20_max": 0.713402, "nauc_mrr_at_20_std": 0.389198, "nauc_mrr_at_20_diff1": 0.855979, "nauc_mrr_at_100_max": 0.712735, "nauc_mrr_at_100_std": 0.388355, "nauc_mrr_at_100_diff1": 0.855905, "nauc_mrr_at_1000_max": 0.712324, "nauc_mrr_at_1000_std": 0.387545, "nauc_mrr_at_1000_diff1": 0.85576, "hit_rate_at_1": 0.66401, "hit_rate_at_3": 0.79344, "hit_rate_at_5": 0.82358, "hit_rate_at_10": 0.84663, "hit_rate_at_20": 0.85372, "hit_rate_at_100": 0.88121, "hit_rate_at_1000": 0.99291, "main_score": 0.76152, "hf_subset": "default", "languages": [ "python-Code" ], "mteb_version": "2.22.1" } ] }, "evaluation_time": 55.59019768820144, "kg_co2_emissions": null, "date": 1790991221.437673, "evaluation_phases": [ { "name": "Data loading", "start": 0.0, "end": 0.8200893481262028, "split": "", "subset": "" }, { "name": "Encoding queries and documents", "start": 0.8322653020732105, "end": 54.84390661818907, "split": "train", "subset": "default" }, { "name": "Scoring", "start": 54.84406331204809, "end": 55.576768273022026, "split": "train", "subset": "default" } ] }