{ "dataset_revision": "68e8f0731a656fa4bd5b7c81936d95ad48a39bfe", "task_name": "CodeSearchNetRetrieval", "mteb_version": "2.15.4", "scores": { "test": [ { "ndcg_at_1": 0.859, "ndcg_at_3": 0.91451, "ndcg_at_5": 0.92144, "ndcg_at_10": 0.92667, "ndcg_at_20": 0.92819, "ndcg_at_100": 0.92891, "ndcg_at_1000": 0.92941, "map_at_1": 0.859, "map_at_3": 0.90117, "map_at_5": 0.90497, "map_at_10": 0.90716, "map_at_20": 0.90758, "map_at_100": 0.90767, "map_at_1000": 0.90769, "recall_at_1": 0.859, "recall_at_3": 0.953, "recall_at_5": 0.97, "recall_at_10": 0.986, "recall_at_20": 0.992, "recall_at_100": 0.996, "recall_at_1000": 1.0, "accuracy": 0.859, "precision_at_1": 0.859, "precision_at_3": 0.31767, "precision_at_5": 0.194, "precision_at_10": 0.0986, "precision_at_20": 0.0496, "precision_at_100": 0.00996, "precision_at_1000": 0.001, "mrr_at_1": 0.859, "mrr_at_3": 0.901167, "mrr_at_5": 0.904967, "mrr_at_10": 0.907158, "mrr_at_20": 0.907577, "mrr_at_100": 0.907665, "mrr_at_1000": 0.907685, "nauc_ndcg_at_1_max": 0.678983, "nauc_ndcg_at_1_std": 0.256548, "nauc_ndcg_at_1_diff1": 0.875505, "nauc_ndcg_at_3_max": 0.726737, "nauc_ndcg_at_3_std": 0.301291, "nauc_ndcg_at_3_diff1": 0.855975, "nauc_ndcg_at_5_max": 0.718683, "nauc_ndcg_at_5_std": 0.30188, "nauc_ndcg_at_5_diff1": 0.856813, "nauc_ndcg_at_10_max": 0.721593, "nauc_ndcg_at_10_std": 0.310177, "nauc_ndcg_at_10_diff1": 0.857422, "nauc_ndcg_at_20_max": 0.716955, "nauc_ndcg_at_20_std": 0.301776, "nauc_ndcg_at_20_diff1": 0.860812, "nauc_ndcg_at_100_max": 0.714358, "nauc_ndcg_at_100_std": 0.296814, "nauc_ndcg_at_100_diff1": 0.860452, "nauc_ndcg_at_1000_max": 0.713019, "nauc_ndcg_at_1000_std": 0.293516, "nauc_ndcg_at_1000_diff1": 0.861477, "nauc_map_at_1_max": 0.678983, "nauc_map_at_1_std": 0.256548, "nauc_map_at_1_diff1": 0.875505, "nauc_map_at_3_max": 0.712477, "nauc_map_at_3_std": 0.286082, "nauc_map_at_3_diff1": 0.862017, "nauc_map_at_5_max": 0.708151, "nauc_map_at_5_std": 0.285078, "nauc_map_at_5_diff1": 0.862399, "nauc_map_at_10_max": 0.709458, "nauc_map_at_10_std": 0.287895, "nauc_map_at_10_diff1": 0.862772, "nauc_map_at_20_max": 0.708383, "nauc_map_at_20_std": 0.285906, "nauc_map_at_20_diff1": 0.863605, "nauc_map_at_100_max": 0.708129, "nauc_map_at_100_std": 0.285418, "nauc_map_at_100_diff1": 0.863578, "nauc_map_at_1000_max": 0.708088, "nauc_map_at_1000_std": 0.285353, "nauc_map_at_1000_diff1": 0.863599, "nauc_recall_at_1_max": 0.678983, "nauc_recall_at_1_std": 0.256548, "nauc_recall_at_1_diff1": 0.875505, "nauc_recall_at_3_max": 0.809246, "nauc_recall_at_3_std": 0.391074, "nauc_recall_at_3_diff1": 0.820818, "nauc_recall_at_5_max": 0.806505, "nauc_recall_at_5_std": 0.453875, "nauc_recall_at_5_diff1": 0.80929, "nauc_recall_at_10_max": 0.932306, "nauc_recall_at_10_std": 0.740496, "nauc_recall_at_10_diff1": 0.763239, "nauc_recall_at_20_max": 0.932598, "nauc_recall_at_20_std": 0.780696, "nauc_recall_at_20_diff1": 0.802754, "nauc_recall_at_100_max": 0.897876, "nauc_recall_at_100_std": 0.780696, "nauc_recall_at_100_diff1": 0.707633, "nauc_recall_at_1000_max": NaN, "nauc_recall_at_1000_std": NaN, "nauc_recall_at_1000_diff1": NaN, "nauc_precision_at_1_max": 0.678983, "nauc_precision_at_1_std": 0.256548, "nauc_precision_at_1_diff1": 0.875505, "nauc_precision_at_3_max": 0.809246, "nauc_precision_at_3_std": 0.391074, "nauc_precision_at_3_diff1": 0.820818, "nauc_precision_at_5_max": 0.806505, "nauc_precision_at_5_std": 0.453875, "nauc_precision_at_5_diff1": 0.80929, "nauc_precision_at_10_max": 0.932306, "nauc_precision_at_10_std": 0.740496, "nauc_precision_at_10_diff1": 0.763239, "nauc_precision_at_20_max": 0.932598, "nauc_precision_at_20_std": 0.780696, "nauc_precision_at_20_diff1": 0.802754, "nauc_precision_at_100_max": 0.897876, "nauc_precision_at_100_std": 0.780696, "nauc_precision_at_100_diff1": 0.707633, "nauc_precision_at_1000_max": NaN, "nauc_precision_at_1000_std": NaN, "nauc_precision_at_1000_diff1": NaN, "nauc_mrr_at_1_max": 0.678983, "nauc_mrr_at_1_std": 0.256548, "nauc_mrr_at_1_diff1": 0.875505, "nauc_mrr_at_3_max": 0.712477, "nauc_mrr_at_3_std": 0.286082, "nauc_mrr_at_3_diff1": 0.862017, "nauc_mrr_at_5_max": 0.708151, "nauc_mrr_at_5_std": 0.285078, "nauc_mrr_at_5_diff1": 0.862399, "nauc_mrr_at_10_max": 0.709458, "nauc_mrr_at_10_std": 0.287895, "nauc_mrr_at_10_diff1": 0.862772, "nauc_mrr_at_20_max": 0.708383, "nauc_mrr_at_20_std": 0.285906, "nauc_mrr_at_20_diff1": 0.863605, "nauc_mrr_at_100_max": 0.708129, "nauc_mrr_at_100_std": 0.285418, "nauc_mrr_at_100_diff1": 0.863578, "nauc_mrr_at_1000_max": 0.708088, "nauc_mrr_at_1000_std": 0.285353, "nauc_mrr_at_1000_diff1": 0.863599, "hit_rate_at_1": 0.859, "hit_rate_at_3": 0.953, "hit_rate_at_5": 0.97, "hit_rate_at_10": 0.986, "hit_rate_at_20": 0.992, "hit_rate_at_100": 0.996, "hit_rate_at_1000": 1.0, "main_score": 0.92667, "hf_subset": "python", "languages": [ "python-Code" ], "mteb_version": "2.15.4" }, { "ndcg_at_1": 0.724, "ndcg_at_3": 0.79352, "ndcg_at_5": 0.80217, "ndcg_at_10": 0.81066, "ndcg_at_20": 0.81617, "ndcg_at_100": 0.82104, "ndcg_at_1000": 0.82875, "map_at_1": 0.724, "map_at_3": 0.77733, "map_at_5": 0.78213, "map_at_10": 0.78568, "map_at_20": 0.78717, "map_at_100": 0.7878, "map_at_1000": 0.78803, "recall_at_1": 0.724, "recall_at_3": 0.84, "recall_at_5": 0.861, "recall_at_10": 0.887, "recall_at_20": 0.909, "recall_at_100": 0.936, "recall_at_1000": 1.0, "accuracy": 0.724, "precision_at_1": 0.724, "precision_at_3": 0.28, "precision_at_5": 0.1722, "precision_at_10": 0.0887, "precision_at_20": 0.04545, "precision_at_100": 0.00936, "precision_at_1000": 0.001, "mrr_at_1": 0.724, "mrr_at_3": 0.777333, "mrr_at_5": 0.782133, "mrr_at_10": 0.785684, "mrr_at_20": 0.787168, "mrr_at_100": 0.787803, "mrr_at_1000": 0.788028, "nauc_ndcg_at_1_max": 0.713282, "nauc_ndcg_at_1_std": 0.212587, "nauc_ndcg_at_1_diff1": 0.829315, "nauc_ndcg_at_3_max": 0.764948, "nauc_ndcg_at_3_std": 0.32562, "nauc_ndcg_at_3_diff1": 0.801725, "nauc_ndcg_at_5_max": 0.767847, "nauc_ndcg_at_5_std": 0.335904, "nauc_ndcg_at_5_diff1": 0.801995, "nauc_ndcg_at_10_max": 0.775709, "nauc_ndcg_at_10_std": 0.369606, "nauc_ndcg_at_10_diff1": 0.795534, "nauc_ndcg_at_20_max": 0.775701, "nauc_ndcg_at_20_std": 0.385536, "nauc_ndcg_at_20_diff1": 0.798022, "nauc_ndcg_at_100_max": 0.77231, "nauc_ndcg_at_100_std": 0.375347, "nauc_ndcg_at_100_diff1": 0.798817, "nauc_ndcg_at_1000_max": 0.7645, "nauc_ndcg_at_1000_std": 0.348209, "nauc_ndcg_at_1000_diff1": 0.802213, "nauc_map_at_1_max": 0.713282, "nauc_map_at_1_std": 0.212587, "nauc_map_at_1_diff1": 0.829315, "nauc_map_at_3_max": 0.749774, "nauc_map_at_3_std": 0.29473, "nauc_map_at_3_diff1": 0.809125, "nauc_map_at_5_max": 0.750656, "nauc_map_at_5_std": 0.29898, "nauc_map_at_5_diff1": 0.809125, "nauc_map_at_10_max": 0.753502, "nauc_map_at_10_std": 0.310868, "nauc_map_at_10_diff1": 0.806799, "nauc_map_at_20_max": 0.753464, "nauc_map_at_20_std": 0.314618, "nauc_map_at_20_diff1": 0.80749, "nauc_map_at_100_max": 0.753069, "nauc_map_at_100_std": 0.313407, "nauc_map_at_100_diff1": 0.807562, "nauc_map_at_1000_max": 0.752839, "nauc_map_at_1000_std": 0.312689, "nauc_map_at_1000_diff1": 0.807648, "nauc_recall_at_1_max": 0.713282, "nauc_recall_at_1_std": 0.212587, "nauc_recall_at_1_diff1": 0.829315, "nauc_recall_at_3_max": 0.822907, "nauc_recall_at_3_std": 0.442366, "nauc_recall_at_3_diff1": 0.773834, "nauc_recall_at_5_max": 0.841997, "nauc_recall_at_5_std": 0.494087, "nauc_recall_at_5_diff1": 0.772716, "nauc_recall_at_10_max": 0.893023, "nauc_recall_at_10_std": 0.688577, "nauc_recall_at_10_diff1": 0.736101, "nauc_recall_at_20_max": 0.917275, "nauc_recall_at_20_std": 0.871036, "nauc_recall_at_20_diff1": 0.74083, "nauc_recall_at_100_max": 0.926434, "nauc_recall_at_100_std": 0.924683, "nauc_recall_at_100_diff1": 0.729816, "nauc_recall_at_1000_max": NaN, "nauc_recall_at_1000_std": NaN, "nauc_recall_at_1000_diff1": NaN, "nauc_precision_at_1_max": 0.713282, "nauc_precision_at_1_std": 0.212587, "nauc_precision_at_1_diff1": 0.829315, "nauc_precision_at_3_max": 0.822907, "nauc_precision_at_3_std": 0.442366, "nauc_precision_at_3_diff1": 0.773834, "nauc_precision_at_5_max": 0.841997, "nauc_precision_at_5_std": 0.494087, "nauc_precision_at_5_diff1": 0.772716, "nauc_precision_at_10_max": 0.893023, "nauc_precision_at_10_std": 0.688577, "nauc_precision_at_10_diff1": 0.736101, "nauc_precision_at_20_max": 0.917275, "nauc_precision_at_20_std": 0.871036, "nauc_precision_at_20_diff1": 0.74083, "nauc_precision_at_100_max": 0.926434, "nauc_precision_at_100_std": 0.924683, "nauc_precision_at_100_diff1": 0.729816, "nauc_precision_at_1000_max": NaN, "nauc_precision_at_1000_std": NaN, "nauc_precision_at_1000_diff1": NaN, "nauc_mrr_at_1_max": 0.713282, "nauc_mrr_at_1_std": 0.212587, "nauc_mrr_at_1_diff1": 0.829315, "nauc_mrr_at_3_max": 0.749774, "nauc_mrr_at_3_std": 0.29473, "nauc_mrr_at_3_diff1": 0.809125, "nauc_mrr_at_5_max": 0.750656, "nauc_mrr_at_5_std": 0.29898, "nauc_mrr_at_5_diff1": 0.809125, "nauc_mrr_at_10_max": 0.753502, "nauc_mrr_at_10_std": 0.310868, "nauc_mrr_at_10_diff1": 0.806799, "nauc_mrr_at_20_max": 0.753464, "nauc_mrr_at_20_std": 0.314618, "nauc_mrr_at_20_diff1": 0.80749, "nauc_mrr_at_100_max": 0.753069, "nauc_mrr_at_100_std": 0.313407, "nauc_mrr_at_100_diff1": 0.807562, "nauc_mrr_at_1000_max": 0.752839, "nauc_mrr_at_1000_std": 0.312689, "nauc_mrr_at_1000_diff1": 0.807648, "hit_rate_at_1": 0.724, "hit_rate_at_3": 0.84, "hit_rate_at_5": 0.861, "hit_rate_at_10": 0.887, "hit_rate_at_20": 0.909, "hit_rate_at_100": 0.936, "hit_rate_at_1000": 1.0, "main_score": 0.81066, "hf_subset": "javascript", "languages": [ "javascript-Code" ], "mteb_version": "2.15.4" }, { "ndcg_at_1": 0.92, "ndcg_at_3": 0.95518, "ndcg_at_5": 0.95978, "ndcg_at_10": 0.96075, "ndcg_at_20": 0.96122, "ndcg_at_100": 0.96193, "ndcg_at_1000": 0.96226, "map_at_1": 0.92, "map_at_3": 0.9475, "map_at_5": 0.9501, "map_at_10": 0.95049, "map_at_20": 0.95061, "map_at_100": 0.95069, "map_at_1000": 0.9507, "recall_at_1": 0.92, "recall_at_3": 0.977, "recall_at_5": 0.988, "recall_at_10": 0.991, "recall_at_20": 0.993, "recall_at_100": 0.997, "recall_at_1000": 1.0, "accuracy": 0.92, "precision_at_1": 0.92, "precision_at_3": 0.32567, "precision_at_5": 0.1976, "precision_at_10": 0.0991, "precision_at_20": 0.04965, "precision_at_100": 0.00997, "precision_at_1000": 0.001, "mrr_at_1": 0.92, "mrr_at_3": 0.9475, "mrr_at_5": 0.9501, "mrr_at_10": 0.950493, "mrr_at_20": 0.950608, "mrr_at_100": 0.950692, "mrr_at_1000": 0.950697, "nauc_ndcg_at_1_max": 0.532405, "nauc_ndcg_at_1_std": -0.029517, "nauc_ndcg_at_1_diff1": 0.934582, "nauc_ndcg_at_3_max": 0.68466, "nauc_ndcg_at_3_std": 0.132987, "nauc_ndcg_at_3_diff1": 0.945104, "nauc_ndcg_at_5_max": 0.652616, "nauc_ndcg_at_5_std": 0.11985, "nauc_ndcg_at_5_diff1": 0.942732, "nauc_ndcg_at_10_max": 0.644083, "nauc_ndcg_at_10_std": 0.10933, "nauc_ndcg_at_10_diff1": 0.941325, "nauc_ndcg_at_20_max": 0.639713, "nauc_ndcg_at_20_std": 0.098395, "nauc_ndcg_at_20_diff1": 0.941429, "nauc_ndcg_at_100_max": 0.633004, "nauc_ndcg_at_100_std": 0.084103, "nauc_ndcg_at_100_diff1": 0.941008, "nauc_ndcg_at_1000_max": 0.629852, "nauc_ndcg_at_1000_std": 0.082522, "nauc_ndcg_at_1000_diff1": 0.94088, "nauc_map_at_1_max": 0.532405, "nauc_map_at_1_std": -0.029517, "nauc_map_at_1_diff1": 0.934582, "nauc_map_at_3_max": 0.638135, "nauc_map_at_3_std": 0.083161, "nauc_map_at_3_diff1": 0.941858, "nauc_map_at_5_max": 0.621049, "nauc_map_at_5_std": 0.073439, "nauc_map_at_5_diff1": 0.940532, "nauc_map_at_10_max": 0.618042, "nauc_map_at_10_std": 0.069634, "nauc_map_at_10_diff1": 0.94006, "nauc_map_at_20_max": 0.617151, "nauc_map_at_20_std": 0.067465, "nauc_map_at_20_diff1": 0.940085, "nauc_map_at_100_max": 0.616496, "nauc_map_at_100_std": 0.066052, "nauc_map_at_100_diff1": 0.940061, "nauc_map_at_1000_max": 0.616457, "nauc_map_at_1000_std": 0.06604, "nauc_map_at_1000_diff1": 0.940059, "nauc_recall_at_1_max": 0.532405, "nauc_recall_at_1_std": -0.029517, "nauc_recall_at_1_diff1": 0.934582, "nauc_recall_at_3_max": 0.982239, "nauc_recall_at_3_std": 0.451874, "nauc_recall_at_3_diff1": 0.965899, "nauc_recall_at_5_max": 1.0, "nauc_recall_at_5_std": 0.658847, "nauc_recall_at_5_diff1": 0.96732, "nauc_recall_at_10_max": 1.0, "nauc_recall_at_10_std": 0.696857, "nauc_recall_at_10_diff1": 0.956427, "nauc_recall_at_20_max": 1.0, "nauc_recall_at_20_std": 0.610244, "nauc_recall_at_20_diff1": 0.962652, "nauc_recall_at_100_max": 1.0, "nauc_recall_at_100_std": 0.282758, "nauc_recall_at_100_diff1": 0.956427, "nauc_recall_at_1000_max": NaN, "nauc_recall_at_1000_std": NaN, "nauc_recall_at_1000_diff1": NaN, "nauc_precision_at_1_max": 0.532405, "nauc_precision_at_1_std": -0.029517, "nauc_precision_at_1_diff1": 0.934582, "nauc_precision_at_3_max": 0.982239, "nauc_precision_at_3_std": 0.451874, "nauc_precision_at_3_diff1": 0.965899, "nauc_precision_at_5_max": 1.0, "nauc_precision_at_5_std": 0.658847, "nauc_precision_at_5_diff1": 0.96732, "nauc_precision_at_10_max": 1.0, "nauc_precision_at_10_std": 0.696857, "nauc_precision_at_10_diff1": 0.956427, "nauc_precision_at_20_max": 1.0, "nauc_precision_at_20_std": 0.610244, "nauc_precision_at_20_diff1": 0.962652, "nauc_precision_at_100_max": 1.0, "nauc_precision_at_100_std": 0.282758, "nauc_precision_at_100_diff1": 0.956427, "nauc_precision_at_1000_max": NaN, "nauc_precision_at_1000_std": NaN, "nauc_precision_at_1000_diff1": NaN, "nauc_mrr_at_1_max": 0.532405, "nauc_mrr_at_1_std": -0.029517, "nauc_mrr_at_1_diff1": 0.934582, "nauc_mrr_at_3_max": 0.638135, "nauc_mrr_at_3_std": 0.083161, "nauc_mrr_at_3_diff1": 0.941858, "nauc_mrr_at_5_max": 0.621049, "nauc_mrr_at_5_std": 0.073439, "nauc_mrr_at_5_diff1": 0.940532, "nauc_mrr_at_10_max": 0.618042, "nauc_mrr_at_10_std": 0.069634, "nauc_mrr_at_10_diff1": 0.94006, "nauc_mrr_at_20_max": 0.617151, "nauc_mrr_at_20_std": 0.067465, "nauc_mrr_at_20_diff1": 0.940085, "nauc_mrr_at_100_max": 0.616496, "nauc_mrr_at_100_std": 0.066052, "nauc_mrr_at_100_diff1": 0.940061, "nauc_mrr_at_1000_max": 0.616457, "nauc_mrr_at_1000_std": 0.06604, "nauc_mrr_at_1000_diff1": 0.940059, "hit_rate_at_1": 0.92, "hit_rate_at_3": 0.977, "hit_rate_at_5": 0.988, "hit_rate_at_10": 0.991, "hit_rate_at_20": 0.993, "hit_rate_at_100": 0.997, "hit_rate_at_1000": 1.0, "main_score": 0.96075, "hf_subset": "go", "languages": [ "go-Code" ], "mteb_version": "2.15.4" }, { "ndcg_at_1": 0.771, "ndcg_at_3": 0.84528, "ndcg_at_5": 0.85273, "ndcg_at_10": 0.85985, "ndcg_at_20": 0.86191, "ndcg_at_100": 0.86586, "ndcg_at_1000": 0.87005, "map_at_1": 0.771, "map_at_3": 0.82767, "map_at_5": 0.83182, "map_at_10": 0.83475, "map_at_20": 0.83534, "map_at_100": 0.83591, "map_at_1000": 0.83604, "recall_at_1": 0.771, "recall_at_3": 0.896, "recall_at_5": 0.914, "recall_at_10": 0.936, "recall_at_20": 0.944, "recall_at_100": 0.965, "recall_at_1000": 1.0, "accuracy": 0.771, "precision_at_1": 0.771, "precision_at_3": 0.29867, "precision_at_5": 0.1828, "precision_at_10": 0.0936, "precision_at_20": 0.0472, "precision_at_100": 0.00965, "precision_at_1000": 0.001, "mrr_at_1": 0.771, "mrr_at_3": 0.827667, "mrr_at_5": 0.831817, "mrr_at_10": 0.834754, "mrr_at_20": 0.835344, "mrr_at_100": 0.835911, "mrr_at_1000": 0.836036, "nauc_ndcg_at_1_max": 0.66721, "nauc_ndcg_at_1_std": 0.143201, "nauc_ndcg_at_1_diff1": 0.879514, "nauc_ndcg_at_3_max": 0.750716, "nauc_ndcg_at_3_std": 0.280357, "nauc_ndcg_at_3_diff1": 0.864923, "nauc_ndcg_at_5_max": 0.754428, "nauc_ndcg_at_5_std": 0.290831, "nauc_ndcg_at_5_diff1": 0.863893, "nauc_ndcg_at_10_max": 0.750334, "nauc_ndcg_at_10_std": 0.316136, "nauc_ndcg_at_10_diff1": 0.865209, "nauc_ndcg_at_20_max": 0.752241, "nauc_ndcg_at_20_std": 0.319887, "nauc_ndcg_at_20_diff1": 0.867596, "nauc_ndcg_at_100_max": 0.747756, "nauc_ndcg_at_100_std": 0.311178, "nauc_ndcg_at_100_diff1": 0.867778, "nauc_ndcg_at_1000_max": 0.740481, "nauc_ndcg_at_1000_std": 0.288436, "nauc_ndcg_at_1000_diff1": 0.868417, "nauc_map_at_1_max": 0.66721, "nauc_map_at_1_std": 0.143201, "nauc_map_at_1_diff1": 0.879514, "nauc_map_at_3_max": 0.728263, "nauc_map_at_3_std": 0.243298, "nauc_map_at_3_diff1": 0.869089, "nauc_map_at_5_max": 0.72967, "nauc_map_at_5_std": 0.247304, "nauc_map_at_5_diff1": 0.868663, "nauc_map_at_10_max": 0.727775, "nauc_map_at_10_std": 0.255899, "nauc_map_at_10_diff1": 0.869163, "nauc_map_at_20_max": 0.728243, "nauc_map_at_20_std": 0.256831, "nauc_map_at_20_diff1": 0.869826, "nauc_map_at_100_max": 0.72767, "nauc_map_at_100_std": 0.255843, "nauc_map_at_100_diff1": 0.869891, "nauc_map_at_1000_max": 0.727482, "nauc_map_at_1000_std": 0.255266, "nauc_map_at_1000_diff1": 0.869912, "nauc_recall_at_1_max": 0.66721, "nauc_recall_at_1_std": 0.143201, "nauc_recall_at_1_diff1": 0.879514, "nauc_recall_at_3_max": 0.849072, "nauc_recall_at_3_std": 0.442824, "nauc_recall_at_3_diff1": 0.846495, "nauc_recall_at_5_max": 0.882396, "nauc_recall_at_5_std": 0.518305, "nauc_recall_at_5_diff1": 0.838914, "nauc_recall_at_10_max": 0.900035, "nauc_recall_at_10_std": 0.751262, "nauc_recall_at_10_diff1": 0.83949, "nauc_recall_at_20_max": 0.9366, "nauc_recall_at_20_std": 0.841795, "nauc_recall_at_20_diff1": 0.855734, "nauc_recall_at_100_max": 0.958917, "nauc_recall_at_100_std": 0.971855, "nauc_recall_at_100_diff1": 0.850207, "nauc_recall_at_1000_max": NaN, "nauc_recall_at_1000_std": NaN, "nauc_recall_at_1000_diff1": NaN, "nauc_precision_at_1_max": 0.66721, "nauc_precision_at_1_std": 0.143201, "nauc_precision_at_1_diff1": 0.879514, "nauc_precision_at_3_max": 0.849072, "nauc_precision_at_3_std": 0.442824, "nauc_precision_at_3_diff1": 0.846495, "nauc_precision_at_5_max": 0.882396, "nauc_precision_at_5_std": 0.518305, "nauc_precision_at_5_diff1": 0.838914, "nauc_precision_at_10_max": 0.900035, "nauc_precision_at_10_std": 0.751262, "nauc_precision_at_10_diff1": 0.83949, "nauc_precision_at_20_max": 0.9366, "nauc_precision_at_20_std": 0.841795, "nauc_precision_at_20_diff1": 0.855734, "nauc_precision_at_100_max": 0.958917, "nauc_precision_at_100_std": 0.971855, "nauc_precision_at_100_diff1": 0.850207, "nauc_precision_at_1000_max": NaN, "nauc_precision_at_1000_std": NaN, "nauc_precision_at_1000_diff1": NaN, "nauc_mrr_at_1_max": 0.66721, "nauc_mrr_at_1_std": 0.143201, "nauc_mrr_at_1_diff1": 0.879514, "nauc_mrr_at_3_max": 0.728263, "nauc_mrr_at_3_std": 0.243298, "nauc_mrr_at_3_diff1": 0.869089, "nauc_mrr_at_5_max": 0.72967, "nauc_mrr_at_5_std": 0.247304, "nauc_mrr_at_5_diff1": 0.868663, "nauc_mrr_at_10_max": 0.727775, "nauc_mrr_at_10_std": 0.255899, "nauc_mrr_at_10_diff1": 0.869163, "nauc_mrr_at_20_max": 0.728243, "nauc_mrr_at_20_std": 0.256831, "nauc_mrr_at_20_diff1": 0.869826, "nauc_mrr_at_100_max": 0.72767, "nauc_mrr_at_100_std": 0.255843, "nauc_mrr_at_100_diff1": 0.869891, "nauc_mrr_at_1000_max": 0.727482, "nauc_mrr_at_1000_std": 0.255266, "nauc_mrr_at_1000_diff1": 0.869912, "hit_rate_at_1": 0.771, "hit_rate_at_3": 0.896, "hit_rate_at_5": 0.914, "hit_rate_at_10": 0.936, "hit_rate_at_20": 0.944, "hit_rate_at_100": 0.965, "hit_rate_at_1000": 1.0, "main_score": 0.85985, "hf_subset": "ruby", "languages": [ "ruby-Code" ], "mteb_version": "2.15.4" }, { "ndcg_at_1": 0.845, "ndcg_at_3": 0.90417, "ndcg_at_5": 0.90825, "ndcg_at_10": 0.91197, "ndcg_at_20": 0.91296, "ndcg_at_100": 0.91473, "ndcg_at_1000": 0.91723, "map_at_1": 0.845, "map_at_3": 0.89, "map_at_5": 0.89225, "map_at_10": 0.89387, "map_at_20": 0.89414, "map_at_100": 0.89435, "map_at_1000": 0.89443, "recall_at_1": 0.845, "recall_at_3": 0.945, "recall_at_5": 0.955, "recall_at_10": 0.966, "recall_at_20": 0.97, "recall_at_100": 0.98, "recall_at_1000": 1.0, "accuracy": 0.845, "precision_at_1": 0.845, "precision_at_3": 0.315, "precision_at_5": 0.191, "precision_at_10": 0.0966, "precision_at_20": 0.0485, "precision_at_100": 0.0098, "precision_at_1000": 0.001, "mrr_at_1": 0.845, "mrr_at_3": 0.89, "mrr_at_5": 0.89225, "mrr_at_10": 0.893873, "mrr_at_20": 0.894139, "mrr_at_100": 0.894347, "mrr_at_1000": 0.894435, "nauc_ndcg_at_1_max": 0.567929, "nauc_ndcg_at_1_std": 0.250062, "nauc_ndcg_at_1_diff1": 0.834099, "nauc_ndcg_at_3_max": 0.711168, "nauc_ndcg_at_3_std": 0.430187, "nauc_ndcg_at_3_diff1": 0.8217, "nauc_ndcg_at_5_max": 0.705776, "nauc_ndcg_at_5_std": 0.427818, "nauc_ndcg_at_5_diff1": 0.817757, "nauc_ndcg_at_10_max": 0.696065, "nauc_ndcg_at_10_std": 0.425653, "nauc_ndcg_at_10_diff1": 0.820937, "nauc_ndcg_at_20_max": 0.69283, "nauc_ndcg_at_20_std": 0.420606, "nauc_ndcg_at_20_diff1": 0.821238, "nauc_ndcg_at_100_max": 0.686546, "nauc_ndcg_at_100_std": 0.410177, "nauc_ndcg_at_100_diff1": 0.822439, "nauc_ndcg_at_1000_max": 0.676953, "nauc_ndcg_at_1000_std": 0.393093, "nauc_ndcg_at_1000_diff1": 0.823495, "nauc_map_at_1_max": 0.567929, "nauc_map_at_1_std": 0.250062, "nauc_map_at_1_diff1": 0.834099, "nauc_map_at_3_max": 0.668373, "nauc_map_at_3_std": 0.372013, "nauc_map_at_3_diff1": 0.826255, "nauc_map_at_5_max": 0.664754, "nauc_map_at_5_std": 0.369297, "nauc_map_at_5_diff1": 0.824478, "nauc_map_at_10_max": 0.66053, "nauc_map_at_10_std": 0.367413, "nauc_map_at_10_diff1": 0.825739, "nauc_map_at_20_max": 0.659712, "nauc_map_at_20_std": 0.366082, "nauc_map_at_20_diff1": 0.825799, "nauc_map_at_100_max": 0.659048, "nauc_map_at_100_std": 0.365002, "nauc_map_at_100_diff1": 0.825901, "nauc_map_at_1000_max": 0.65876, "nauc_map_at_1000_std": 0.364485, "nauc_map_at_1000_diff1": 0.825926, "nauc_recall_at_1_max": 0.567929, "nauc_recall_at_1_std": 0.250062, "nauc_recall_at_1_diff1": 0.834099, "nauc_recall_at_3_max": 0.945938, "nauc_recall_at_3_std": 0.753238, "nauc_recall_at_3_diff1": 0.795951, "nauc_recall_at_5_max": 0.973493, "nauc_recall_at_5_std": 0.816859, "nauc_recall_at_5_diff1": 0.771304, "nauc_recall_at_10_max": 0.988466, "nauc_recall_at_10_std": 0.929093, "nauc_recall_at_10_diff1": 0.780098, "nauc_recall_at_20_max": 0.991285, "nauc_recall_at_20_std": 0.941036, "nauc_recall_at_20_diff1": 0.778556, "nauc_recall_at_100_max": 0.993464, "nauc_recall_at_100_std": 0.955415, "nauc_recall_at_100_diff1": 0.787348, "nauc_recall_at_1000_max": NaN, "nauc_recall_at_1000_std": NaN, "nauc_recall_at_1000_diff1": NaN, "nauc_precision_at_1_max": 0.567929, "nauc_precision_at_1_std": 0.250062, "nauc_precision_at_1_diff1": 0.834099, "nauc_precision_at_3_max": 0.945938, "nauc_precision_at_3_std": 0.753238, "nauc_precision_at_3_diff1": 0.795951, "nauc_precision_at_5_max": 0.973493, "nauc_precision_at_5_std": 0.816859, "nauc_precision_at_5_diff1": 0.771304, "nauc_precision_at_10_max": 0.988466, "nauc_precision_at_10_std": 0.929093, "nauc_precision_at_10_diff1": 0.780098, "nauc_precision_at_20_max": 0.991285, "nauc_precision_at_20_std": 0.941036, "nauc_precision_at_20_diff1": 0.778556, "nauc_precision_at_100_max": 0.993464, "nauc_precision_at_100_std": 0.955415, "nauc_precision_at_100_diff1": 0.787348, "nauc_precision_at_1000_max": NaN, "nauc_precision_at_1000_std": NaN, "nauc_precision_at_1000_diff1": NaN, "nauc_mrr_at_1_max": 0.567929, "nauc_mrr_at_1_std": 0.250062, "nauc_mrr_at_1_diff1": 0.834099, "nauc_mrr_at_3_max": 0.668373, "nauc_mrr_at_3_std": 0.372013, "nauc_mrr_at_3_diff1": 0.826255, "nauc_mrr_at_5_max": 0.664754, "nauc_mrr_at_5_std": 0.369297, "nauc_mrr_at_5_diff1": 0.824478, "nauc_mrr_at_10_max": 0.66053, "nauc_mrr_at_10_std": 0.367413, "nauc_mrr_at_10_diff1": 0.825739, "nauc_mrr_at_20_max": 0.659712, "nauc_mrr_at_20_std": 0.366082, "nauc_mrr_at_20_diff1": 0.825799, "nauc_mrr_at_100_max": 0.659048, "nauc_mrr_at_100_std": 0.365002, "nauc_mrr_at_100_diff1": 0.825901, "nauc_mrr_at_1000_max": 0.65876, "nauc_mrr_at_1000_std": 0.364485, "nauc_mrr_at_1000_diff1": 0.825926, "hit_rate_at_1": 0.845, "hit_rate_at_3": 0.945, "hit_rate_at_5": 0.955, "hit_rate_at_10": 0.966, "hit_rate_at_20": 0.97, "hit_rate_at_100": 0.98, "hit_rate_at_1000": 1.0, "main_score": 0.91197, "hf_subset": "java", "languages": [ "java-Code" ], "mteb_version": "2.15.4" }, { "ndcg_at_1": 0.802, "ndcg_at_3": 0.86963, "ndcg_at_5": 0.87785, "ndcg_at_10": 0.88442, "ndcg_at_20": 0.8865, "ndcg_at_100": 0.88843, "ndcg_at_1000": 0.89156, "map_at_1": 0.802, "map_at_3": 0.85383, "map_at_5": 0.85838, "map_at_10": 0.86116, "map_at_20": 0.86176, "map_at_100": 0.86205, "map_at_1000": 0.86214, "recall_at_1": 0.802, "recall_at_3": 0.915, "recall_at_5": 0.935, "recall_at_10": 0.955, "recall_at_20": 0.963, "recall_at_100": 0.973, "recall_at_1000": 1.0, "accuracy": 0.802, "precision_at_1": 0.802, "precision_at_3": 0.305, "precision_at_5": 0.187, "precision_at_10": 0.0955, "precision_at_20": 0.04815, "precision_at_100": 0.00973, "precision_at_1000": 0.001, "mrr_at_1": 0.802, "mrr_at_3": 0.853833, "mrr_at_5": 0.858383, "mrr_at_10": 0.861162, "mrr_at_20": 0.861761, "mrr_at_100": 0.862053, "mrr_at_1000": 0.862135, "nauc_ndcg_at_1_max": 0.547747, "nauc_ndcg_at_1_std": -0.006567, "nauc_ndcg_at_1_diff1": 0.879932, "nauc_ndcg_at_3_max": 0.648919, "nauc_ndcg_at_3_std": 0.103591, "nauc_ndcg_at_3_diff1": 0.854583, "nauc_ndcg_at_5_max": 0.661014, "nauc_ndcg_at_5_std": 0.121037, "nauc_ndcg_at_5_diff1": 0.865021, "nauc_ndcg_at_10_max": 0.661526, "nauc_ndcg_at_10_std": 0.135218, "nauc_ndcg_at_10_diff1": 0.874675, "nauc_ndcg_at_20_max": 0.657412, "nauc_ndcg_at_20_std": 0.147632, "nauc_ndcg_at_20_diff1": 0.874543, "nauc_ndcg_at_100_max": 0.652239, "nauc_ndcg_at_100_std": 0.142139, "nauc_ndcg_at_100_diff1": 0.875536, "nauc_ndcg_at_1000_max": 0.641724, "nauc_ndcg_at_1000_std": 0.116199, "nauc_ndcg_at_1000_diff1": 0.872741, "nauc_map_at_1_max": 0.547747, "nauc_map_at_1_std": -0.006567, "nauc_map_at_1_diff1": 0.879932, "nauc_map_at_3_max": 0.620567, "nauc_map_at_3_std": 0.070234, "nauc_map_at_3_diff1": 0.86233, "nauc_map_at_5_max": 0.625492, "nauc_map_at_5_std": 0.07742, "nauc_map_at_5_diff1": 0.867571, "nauc_map_at_10_max": 0.625319, "nauc_map_at_10_std": 0.081733, "nauc_map_at_10_diff1": 0.870997, "nauc_map_at_20_max": 0.624192, "nauc_map_at_20_std": 0.084155, "nauc_map_at_20_diff1": 0.870996, "nauc_map_at_100_max": 0.623478, "nauc_map_at_100_std": 0.08342, "nauc_map_at_100_diff1": 0.871072, "nauc_map_at_1000_max": 0.623241, "nauc_map_at_1000_std": 0.082842, "nauc_map_at_1000_diff1": 0.871017, "nauc_recall_at_1_max": 0.547747, "nauc_recall_at_1_std": -0.006567, "nauc_recall_at_1_diff1": 0.879932, "nauc_recall_at_3_max": 0.780436, "nauc_recall_at_3_std": 0.260252, "nauc_recall_at_3_diff1": 0.818147, "nauc_recall_at_5_max": 0.873404, "nauc_recall_at_5_std": 0.385671, "nauc_recall_at_5_diff1": 0.85307, "nauc_recall_at_10_max": 0.96566, "nauc_recall_at_10_std": 0.606256, "nauc_recall_at_10_diff1": 0.920635, "nauc_recall_at_20_max": 0.984417, "nauc_recall_at_20_std": 0.855616, "nauc_recall_at_20_diff1": 0.92789, "nauc_recall_at_100_max": 1.0, "nauc_recall_at_100_std": 1.0, "nauc_recall_at_100_diff1": 0.970346, "nauc_recall_at_1000_max": NaN, "nauc_recall_at_1000_std": NaN, "nauc_recall_at_1000_diff1": NaN, "nauc_precision_at_1_max": 0.547747, "nauc_precision_at_1_std": -0.006567, "nauc_precision_at_1_diff1": 0.879932, "nauc_precision_at_3_max": 0.780436, "nauc_precision_at_3_std": 0.260252, "nauc_precision_at_3_diff1": 0.818147, "nauc_precision_at_5_max": 0.873404, "nauc_precision_at_5_std": 0.385671, "nauc_precision_at_5_diff1": 0.85307, "nauc_precision_at_10_max": 0.96566, "nauc_precision_at_10_std": 0.606256, "nauc_precision_at_10_diff1": 0.920635, "nauc_precision_at_20_max": 0.984417, "nauc_precision_at_20_std": 0.855616, "nauc_precision_at_20_diff1": 0.92789, "nauc_precision_at_100_max": 1.0, "nauc_precision_at_100_std": 1.0, "nauc_precision_at_100_diff1": 0.970346, "nauc_precision_at_1000_max": NaN, "nauc_precision_at_1000_std": NaN, "nauc_precision_at_1000_diff1": NaN, "nauc_mrr_at_1_max": 0.547747, "nauc_mrr_at_1_std": -0.006567, "nauc_mrr_at_1_diff1": 0.879932, "nauc_mrr_at_3_max": 0.620567, "nauc_mrr_at_3_std": 0.070234, "nauc_mrr_at_3_diff1": 0.86233, "nauc_mrr_at_5_max": 0.625492, "nauc_mrr_at_5_std": 0.07742, "nauc_mrr_at_5_diff1": 0.867571, "nauc_mrr_at_10_max": 0.625319, "nauc_mrr_at_10_std": 0.081733, "nauc_mrr_at_10_diff1": 0.870997, "nauc_mrr_at_20_max": 0.624192, "nauc_mrr_at_20_std": 0.084155, "nauc_mrr_at_20_diff1": 0.870996, "nauc_mrr_at_100_max": 0.623478, "nauc_mrr_at_100_std": 0.08342, "nauc_mrr_at_100_diff1": 0.871072, "nauc_mrr_at_1000_max": 0.623241, "nauc_mrr_at_1000_std": 0.082842, "nauc_mrr_at_1000_diff1": 0.871017, "hit_rate_at_1": 0.802, "hit_rate_at_3": 0.915, "hit_rate_at_5": 0.935, "hit_rate_at_10": 0.955, "hit_rate_at_20": 0.963, "hit_rate_at_100": 0.973, "hit_rate_at_1000": 1.0, "main_score": 0.88442, "hf_subset": "php", "languages": [ "php-Code" ], "mteb_version": "2.15.4" } ] }, "evaluation_time": 66.81729948800057, "kg_co2_emissions": null, "date": 1781271763.498216, "evaluation_phases": [ { "name": "Data loading", "start": 0.0004377999994176207, "end": 39.05039095800021, "split": "", "subset": "" }, { "name": "Dataset transform", "start": 39.050400871999955, "end": 39.050402662000124, "split": "", "subset": "" }, { "name": "Encoding queries and documents", "start": 39.056779489999826, "end": 44.18611791799958, "split": "test", "subset": "python" }, { "name": "Scoring", "start": 44.187008315999265, "end": 44.67660215400065, "split": "test", "subset": "python" }, { "name": "Encoding queries and documents", "start": 44.69616846799909, "end": 49.66828635300044, "split": "test", "subset": "javascript" }, { "name": "Scoring", "start": 49.66913890700016, "end": 50.17069534999973, "split": "test", "subset": "javascript" }, { "name": "Encoding queries and documents", "start": 50.190499349000675, "end": 53.45359480300067, "split": "test", "subset": "go" }, { "name": "Scoring", "start": 53.45450437600084, "end": 53.95956708400081, "split": "test", "subset": "go" }, { "name": "Encoding queries and documents", "start": 53.97823919899929, "end": 57.586306093999156, "split": "test", "subset": "ruby" }, { "name": "Scoring", "start": 57.58717690999947, "end": 58.08464151199951, "split": "test", "subset": "ruby" }, { "name": "Encoding queries and documents", "start": 58.104137799000455, "end": 62.3329270840004, "split": "test", "subset": "java" }, { "name": "Scoring", "start": 62.333796246999555, "end": 62.837159008000526, "split": "test", "subset": "java" }, { "name": "Encoding queries and documents", "start": 62.857382608999615, "end": 66.31973180900059, "split": "test", "subset": "php" }, { "name": "Scoring", "start": 66.32060472000012, "end": 66.80355941299968, "split": "test", "subset": "php" } ] }