{ "dataset_revision": "7a20df1abe4dafc46f93f9a7965bf9c6968bdf04", "task_name": "FreshStackRetrieval", "mteb_version": "2.20.1", "scores": { "test": [ { "ndcg_at_1": 0.21726, "ndcg_at_3": 0.18827, "ndcg_at_5": 0.17918, "ndcg_at_10": 0.17733, "ndcg_at_20": 0.19932, "ndcg_at_100": 0.27162, "ndcg_at_1000": 0.36534, "map_at_1": 0.03292, "map_at_3": 0.06075, "map_at_5": 0.07659, "map_at_10": 0.09442, "map_at_20": 0.10912, "map_at_100": 0.12705, "map_at_1000": 0.13469, "recall_at_1": 0.03292, "recall_at_3": 0.07667, "recall_at_5": 0.11073, "recall_at_10": 0.16493, "recall_at_20": 0.2424, "recall_at_100": 0.44855, "recall_at_1000": 0.84678, "accuracy": 0.03292, "precision_at_1": 0.21726, "precision_at_3": 0.17312, "precision_at_5": 0.15298, "precision_at_10": 0.1186, "precision_at_20": 0.08802, "precision_at_100": 0.03533, "precision_at_1000": 0.00684, "mrr_at_1": 0.217262, "mrr_at_3": 0.277034, "mrr_at_5": 0.294519, "mrr_at_10": 0.307791, "mrr_at_20": 0.315134, "mrr_at_100": 0.320563, "mrr_at_1000": 0.321177, "nauc_ndcg_at_1_max": 0.105389, "nauc_ndcg_at_1_std": 0.067928, "nauc_ndcg_at_1_diff1": 0.177666, "nauc_ndcg_at_3_max": 0.124435, "nauc_ndcg_at_3_std": 0.089831, "nauc_ndcg_at_3_diff1": 0.15109, "nauc_ndcg_at_5_max": 0.121631, "nauc_ndcg_at_5_std": 0.088766, "nauc_ndcg_at_5_diff1": 0.149478, "nauc_ndcg_at_10_max": 0.114027, "nauc_ndcg_at_10_std": 0.081811, "nauc_ndcg_at_10_diff1": 0.164882, "nauc_ndcg_at_20_max": 0.115, "nauc_ndcg_at_20_std": 0.075917, "nauc_ndcg_at_20_diff1": 0.17284, "nauc_ndcg_at_100_max": 0.120664, "nauc_ndcg_at_100_std": 0.117823, "nauc_ndcg_at_100_diff1": 0.166713, "nauc_ndcg_at_1000_max": 0.126414, "nauc_ndcg_at_1000_std": 0.152856, "nauc_ndcg_at_1000_diff1": 0.166006, "nauc_map_at_1_max": 0.063341, "nauc_map_at_1_std": -0.006194, "nauc_map_at_1_diff1": 0.227458, "nauc_map_at_3_max": 0.076564, "nauc_map_at_3_std": -0.017795, "nauc_map_at_3_diff1": 0.21675, "nauc_map_at_5_max": 0.102834, "nauc_map_at_5_std": 0.01008, "nauc_map_at_5_diff1": 0.212843, "nauc_map_at_10_max": 0.115708, "nauc_map_at_10_std": 0.032693, "nauc_map_at_10_diff1": 0.212387, "nauc_map_at_20_max": 0.12151, "nauc_map_at_20_std": 0.052731, "nauc_map_at_20_diff1": 0.199653, "nauc_map_at_100_max": 0.125835, "nauc_map_at_100_std": 0.078233, "nauc_map_at_100_diff1": 0.188774, "nauc_map_at_1000_max": 0.125361, "nauc_map_at_1000_std": 0.087925, "nauc_map_at_1000_diff1": 0.187343, "nauc_recall_at_1_max": 0.063341, "nauc_recall_at_1_std": -0.006194, "nauc_recall_at_1_diff1": 0.227458, "nauc_recall_at_3_max": 0.070856, "nauc_recall_at_3_std": -0.038002, "nauc_recall_at_3_diff1": 0.181471, "nauc_recall_at_5_max": 0.089655, "nauc_recall_at_5_std": -0.006341, "nauc_recall_at_5_diff1": 0.164422, "nauc_recall_at_10_max": 0.085995, "nauc_recall_at_10_std": 0.016348, "nauc_recall_at_10_diff1": 0.151, "nauc_recall_at_20_max": 0.080553, "nauc_recall_at_20_std": 0.030805, "nauc_recall_at_20_diff1": 0.131938, "nauc_recall_at_100_max": 0.085154, "nauc_recall_at_100_std": 0.098876, "nauc_recall_at_100_diff1": 0.12342, "nauc_recall_at_1000_max": 0.074477, "nauc_recall_at_1000_std": 0.12637, "nauc_recall_at_1000_diff1": 0.123213, "nauc_precision_at_1_max": 0.105389, "nauc_precision_at_1_std": 0.067928, "nauc_precision_at_1_diff1": 0.177666, "nauc_precision_at_3_max": 0.137099, "nauc_precision_at_3_std": 0.102707, "nauc_precision_at_3_diff1": 0.138585, "nauc_precision_at_5_max": 0.131516, "nauc_precision_at_5_std": 0.113844, "nauc_precision_at_5_diff1": 0.12669, "nauc_precision_at_10_max": 0.131493, "nauc_precision_at_10_std": 0.159113, "nauc_precision_at_10_diff1": 0.116595, "nauc_precision_at_20_max": 0.13495, "nauc_precision_at_20_std": 0.201017, "nauc_precision_at_20_diff1": 0.086985, "nauc_precision_at_100_max": 0.1134, "nauc_precision_at_100_std": 0.25293, "nauc_precision_at_100_diff1": 0.048895, "nauc_precision_at_1000_max": 0.09637, "nauc_precision_at_1000_std": 0.272186, "nauc_precision_at_1000_diff1": 0.012819, "nauc_mrr_at_1_max": 0.105389, "nauc_mrr_at_1_std": 0.067928, "nauc_mrr_at_1_diff1": 0.177666, "nauc_mrr_at_3_max": 0.128312, "nauc_mrr_at_3_std": 0.08808, "nauc_mrr_at_3_diff1": 0.169829, "nauc_mrr_at_5_max": 0.11665, "nauc_mrr_at_5_std": 0.08448, "nauc_mrr_at_5_diff1": 0.158354, "nauc_mrr_at_10_max": 0.114973, "nauc_mrr_at_10_std": 0.088563, "nauc_mrr_at_10_diff1": 0.15835, "nauc_mrr_at_20_max": 0.113965, "nauc_mrr_at_20_std": 0.089994, "nauc_mrr_at_20_diff1": 0.161727, "nauc_mrr_at_100_max": 0.114912, "nauc_mrr_at_100_std": 0.090978, "nauc_mrr_at_100_diff1": 0.162265, "nauc_mrr_at_1000_max": 0.115095, "nauc_mrr_at_1000_std": 0.09056, "nauc_mrr_at_1000_diff1": 0.162321, "hit_rate_at_1": 0.21726, "hit_rate_at_3": 0.3497, "hit_rate_at_5": 0.42708, "hit_rate_at_10": 0.52679, "hit_rate_at_20": 0.63244, "hit_rate_at_100": 0.8497, "hit_rate_at_1000": 0.98661, "main_score": 0.17733, "hf_subset": "default", "languages": [ "eng-Latn", "python-Code", "javascript-Code", "go-Code" ], "mteb_version": "2.20.1" } ] }, "evaluation_time": 13.371997589012608, "kg_co2_emissions": null, "date": 1787844733.581576, "evaluation_phases": [ { "name": "Data loading", "start": 0.0, "end": 2.8750287940492854, "split": "", "subset": "" }, { "name": "Encoding queries and documents", "start": 3.2469759120140225, "end": 12.803156504058279, "split": "test", "subset": "default" }, { "name": "Scoring", "start": 12.804211989976466, "end": 13.351939299027435, "split": "test", "subset": "default" } ] }