{ "dataset_revision": "4adc7bc41202b5c13543c9c886a25f340634dab3", "task_name": "COIRCodeSearchNetRetrieval", "mteb_version": "2.21.8", "scores": { "test": [ { "ndcg_at_1": 0.83376, "ndcg_at_3": 0.88828, "ndcg_at_5": 0.89634, "ndcg_at_10": 0.90219, "ndcg_at_20": 0.90509, "ndcg_at_100": 0.90796, "ndcg_at_1000": 0.90901, "map_at_1": 0.83376, "map_at_3": 0.87554, "map_at_5": 0.88003, "map_at_10": 0.88247, "map_at_20": 0.88329, "map_at_100": 0.8837, "map_at_1000": 0.88374, "recall_at_1": 0.83376, "recall_at_3": 0.92486, "recall_at_5": 0.94436, "recall_at_10": 0.96233, "recall_at_20": 0.97366, "recall_at_100": 0.98894, "recall_at_1000": 0.99725, "accuracy": 0.83376, "precision_at_1": 0.83376, "precision_at_3": 0.30829, "precision_at_5": 0.18887, "precision_at_10": 0.09623, "precision_at_20": 0.04868, "precision_at_100": 0.00989, "precision_at_1000": 0.001, "mrr_at_1": 0.833758, "mrr_at_3": 0.875542, "mrr_at_5": 0.88003, "mrr_at_10": 0.882473, "mrr_at_20": 0.883289, "mrr_at_100": 0.883699, "mrr_at_1000": 0.883738, "nauc_ndcg_at_1_max": 0.693602, "nauc_ndcg_at_1_std": -0.10124, "nauc_ndcg_at_1_diff1": 0.867725, "nauc_ndcg_at_3_max": 0.732405, "nauc_ndcg_at_3_std": -0.078134, "nauc_ndcg_at_3_diff1": 0.847807, "nauc_ndcg_at_5_max": 0.737589, "nauc_ndcg_at_5_std": -0.069248, "nauc_ndcg_at_5_diff1": 0.851914, "nauc_ndcg_at_10_max": 0.735205, "nauc_ndcg_at_10_std": -0.066369, "nauc_ndcg_at_10_diff1": 0.853967, "nauc_ndcg_at_20_max": 0.73354, "nauc_ndcg_at_20_std": -0.064124, "nauc_ndcg_at_20_diff1": 0.854464, "nauc_ndcg_at_100_max": 0.730036, "nauc_ndcg_at_100_std": -0.066804, "nauc_ndcg_at_100_diff1": 0.854944, "nauc_ndcg_at_1000_max": 0.727716, "nauc_ndcg_at_1000_std": -0.071737, "nauc_ndcg_at_1000_diff1": 0.855042, "nauc_map_at_1_max": 0.693602, "nauc_map_at_1_std": -0.10124, "nauc_map_at_1_diff1": 0.867725, "nauc_map_at_3_max": 0.720921, "nauc_map_at_3_std": -0.085536, "nauc_map_at_3_diff1": 0.852951, "nauc_map_at_5_max": 0.723007, "nauc_map_at_5_std": -0.081653, "nauc_map_at_5_diff1": 0.855134, "nauc_map_at_10_max": 0.7219, "nauc_map_at_10_std": -0.080875, "nauc_map_at_10_diff1": 0.855939, "nauc_map_at_20_max": 0.721385, "nauc_map_at_20_std": -0.080604, "nauc_map_at_20_diff1": 0.856083, "nauc_map_at_100_max": 0.720938, "nauc_map_at_100_std": -0.080969, "nauc_map_at_100_diff1": 0.85613, "nauc_map_at_1000_max": 0.720872, "nauc_map_at_1000_std": -0.081081, "nauc_map_at_1000_diff1": 0.856134, "nauc_recall_at_1_max": 0.693602, "nauc_recall_at_1_std": -0.10124, "nauc_recall_at_1_diff1": 0.867725, "nauc_recall_at_3_max": 0.784859, "nauc_recall_at_3_std": -0.043926, "nauc_recall_at_3_diff1": 0.824836, "nauc_recall_at_5_max": 0.825704, "nauc_recall_at_5_std": 0.008001, "nauc_recall_at_5_diff1": 0.834769, "nauc_recall_at_10_max": 0.84873, "nauc_recall_at_10_std": 0.066398, "nauc_recall_at_10_diff1": 0.842119, "nauc_recall_at_20_max": 0.876112, "nauc_recall_at_20_std": 0.158439, "nauc_recall_at_20_diff1": 0.8433, "nauc_recall_at_100_max": 0.923571, "nauc_recall_at_100_std": 0.350368, "nauc_recall_at_100_diff1": 0.850042, "nauc_recall_at_1000_max": 0.909714, "nauc_recall_at_1000_std": 0.283774, "nauc_recall_at_1000_diff1": 0.857892, "nauc_precision_at_1_max": 0.693602, "nauc_precision_at_1_std": -0.10124, "nauc_precision_at_1_diff1": 0.867725, "nauc_precision_at_3_max": 0.784859, "nauc_precision_at_3_std": -0.043926, "nauc_precision_at_3_diff1": 0.824836, "nauc_precision_at_5_max": 0.825704, "nauc_precision_at_5_std": 0.008001, "nauc_precision_at_5_diff1": 0.834769, "nauc_precision_at_10_max": 0.84873, "nauc_precision_at_10_std": 0.066398, "nauc_precision_at_10_diff1": 0.842119, "nauc_precision_at_20_max": 0.876112, "nauc_precision_at_20_std": 0.158439, "nauc_precision_at_20_diff1": 0.8433, "nauc_precision_at_100_max": 0.923571, "nauc_precision_at_100_std": 0.350368, "nauc_precision_at_100_diff1": 0.850042, "nauc_precision_at_1000_max": 0.909714, "nauc_precision_at_1000_std": 0.283774, "nauc_precision_at_1000_diff1": 0.857892, "nauc_mrr_at_1_max": 0.693602, "nauc_mrr_at_1_std": -0.10124, "nauc_mrr_at_1_diff1": 0.867725, "nauc_mrr_at_3_max": 0.720921, "nauc_mrr_at_3_std": -0.085536, "nauc_mrr_at_3_diff1": 0.852951, "nauc_mrr_at_5_max": 0.723007, "nauc_mrr_at_5_std": -0.081653, "nauc_mrr_at_5_diff1": 0.855134, "nauc_mrr_at_10_max": 0.7219, "nauc_mrr_at_10_std": -0.080875, "nauc_mrr_at_10_diff1": 0.855939, "nauc_mrr_at_20_max": 0.721385, "nauc_mrr_at_20_std": -0.080604, "nauc_mrr_at_20_diff1": 0.856083, "nauc_mrr_at_100_max": 0.720938, "nauc_mrr_at_100_std": -0.080969, "nauc_mrr_at_100_diff1": 0.85613, "nauc_mrr_at_1000_max": 0.720872, "nauc_mrr_at_1000_std": -0.081081, "nauc_mrr_at_1000_diff1": 0.856134, "hit_rate_at_1": 0.83376, "hit_rate_at_3": 0.92486, "hit_rate_at_5": 0.94436, "hit_rate_at_10": 0.96233, "hit_rate_at_20": 0.97366, "hit_rate_at_100": 0.98894, "hit_rate_at_1000": 0.99725, "main_score": 0.90219, "hf_subset": "python", "languages": [ "python-Code" ], "mteb_version": "2.21.8" }, { "ndcg_at_1": 0.56913, "ndcg_at_3": 0.64913, "ndcg_at_5": 0.66503, "ndcg_at_10": 0.68068, "ndcg_at_20": 0.68906, "ndcg_at_100": 0.70178, "ndcg_at_1000": 0.70928, "map_at_1": 0.56913, "map_at_3": 0.63025, "map_at_5": 0.63908, "map_at_10": 0.64555, "map_at_20": 0.64789, "map_at_100": 0.64963, "map_at_1000": 0.64989, "recall_at_1": 0.56913, "recall_at_3": 0.70343, "recall_at_5": 0.74202, "recall_at_10": 0.79034, "recall_at_20": 0.82315, "recall_at_100": 0.89213, "recall_at_1000": 0.95229, "accuracy": 0.56913, "precision_at_1": 0.56913, "precision_at_3": 0.23448, "precision_at_5": 0.1484, "precision_at_10": 0.07903, "precision_at_20": 0.04116, "precision_at_100": 0.00892, "precision_at_1000": 0.00095, "mrr_at_1": 0.569128, "mrr_at_3": 0.630254, "mrr_at_5": 0.639081, "mrr_at_10": 0.645554, "mrr_at_20": 0.647891, "mrr_at_100": 0.64963, "mrr_at_1000": 0.649893, "nauc_ndcg_at_1_max": 0.615355, "nauc_ndcg_at_1_std": 0.007054, "nauc_ndcg_at_1_diff1": 0.767504, "nauc_ndcg_at_3_max": 0.651737, "nauc_ndcg_at_3_std": 0.042779, "nauc_ndcg_at_3_diff1": 0.723821, "nauc_ndcg_at_5_max": 0.652495, "nauc_ndcg_at_5_std": 0.053476, "nauc_ndcg_at_5_diff1": 0.71555, "nauc_ndcg_at_10_max": 0.652791, "nauc_ndcg_at_10_std": 0.062626, "nauc_ndcg_at_10_diff1": 0.713313, "nauc_ndcg_at_20_max": 0.650969, "nauc_ndcg_at_20_std": 0.062228, "nauc_ndcg_at_20_diff1": 0.712995, "nauc_ndcg_at_100_max": 0.649124, "nauc_ndcg_at_100_std": 0.063367, "nauc_ndcg_at_100_diff1": 0.714861, "nauc_ndcg_at_1000_max": 0.648438, "nauc_ndcg_at_1000_std": 0.059849, "nauc_ndcg_at_1000_diff1": 0.718188, "nauc_map_at_1_max": 0.615355, "nauc_map_at_1_std": 0.007054, "nauc_map_at_1_diff1": 0.767504, "nauc_map_at_3_max": 0.643249, "nauc_map_at_3_std": 0.033011, "nauc_map_at_3_diff1": 0.73533, "nauc_map_at_5_max": 0.643464, "nauc_map_at_5_std": 0.038317, "nauc_map_at_5_diff1": 0.731167, "nauc_map_at_10_max": 0.643516, "nauc_map_at_10_std": 0.04148, "nauc_map_at_10_diff1": 0.730644, "nauc_map_at_20_max": 0.643057, "nauc_map_at_20_std": 0.041324, "nauc_map_at_20_diff1": 0.730662, "nauc_map_at_100_max": 0.642769, "nauc_map_at_100_std": 0.041291, "nauc_map_at_100_diff1": 0.731019, "nauc_map_at_1000_max": 0.642749, "nauc_map_at_1000_std": 0.041208, "nauc_map_at_1000_diff1": 0.731112, "nauc_recall_at_1_max": 0.615355, "nauc_recall_at_1_std": 0.007054, "nauc_recall_at_1_diff1": 0.767504, "nauc_recall_at_3_max": 0.679274, "nauc_recall_at_3_std": 0.075247, "nauc_recall_at_3_diff1": 0.685769, "nauc_recall_at_5_max": 0.684807, "nauc_recall_at_5_std": 0.110786, "nauc_recall_at_5_diff1": 0.65748, "nauc_recall_at_10_max": 0.691181, "nauc_recall_at_10_std": 0.159877, "nauc_recall_at_10_diff1": 0.637138, "nauc_recall_at_20_max": 0.685657, "nauc_recall_at_20_std": 0.172139, "nauc_recall_at_20_diff1": 0.623092, "nauc_recall_at_100_max": 0.682867, "nauc_recall_at_100_std": 0.249413, "nauc_recall_at_100_diff1": 0.594437, "nauc_recall_at_1000_max": 0.694665, "nauc_recall_at_1000_std": 0.324984, "nauc_recall_at_1000_diff1": 0.584658, "nauc_precision_at_1_max": 0.615355, "nauc_precision_at_1_std": 0.007054, "nauc_precision_at_1_diff1": 0.767504, "nauc_precision_at_3_max": 0.679274, "nauc_precision_at_3_std": 0.075247, "nauc_precision_at_3_diff1": 0.685769, "nauc_precision_at_5_max": 0.684807, "nauc_precision_at_5_std": 0.110786, "nauc_precision_at_5_diff1": 0.65748, "nauc_precision_at_10_max": 0.691181, "nauc_precision_at_10_std": 0.159877, "nauc_precision_at_10_diff1": 0.637138, "nauc_precision_at_20_max": 0.685657, "nauc_precision_at_20_std": 0.172139, "nauc_precision_at_20_diff1": 0.623092, "nauc_precision_at_100_max": 0.682867, "nauc_precision_at_100_std": 0.249413, "nauc_precision_at_100_diff1": 0.594437, "nauc_precision_at_1000_max": 0.694665, "nauc_precision_at_1000_std": 0.324984, "nauc_precision_at_1000_diff1": 0.584658, "nauc_mrr_at_1_max": 0.615355, "nauc_mrr_at_1_std": 0.007054, "nauc_mrr_at_1_diff1": 0.767504, "nauc_mrr_at_3_max": 0.643249, "nauc_mrr_at_3_std": 0.033011, "nauc_mrr_at_3_diff1": 0.73533, "nauc_mrr_at_5_max": 0.643464, "nauc_mrr_at_5_std": 0.038317, "nauc_mrr_at_5_diff1": 0.731167, "nauc_mrr_at_10_max": 0.643516, "nauc_mrr_at_10_std": 0.04148, "nauc_mrr_at_10_diff1": 0.730644, "nauc_mrr_at_20_max": 0.643057, "nauc_mrr_at_20_std": 0.041324, "nauc_mrr_at_20_diff1": 0.730662, "nauc_mrr_at_100_max": 0.642769, "nauc_mrr_at_100_std": 0.041291, "nauc_mrr_at_100_diff1": 0.731019, "nauc_mrr_at_1000_max": 0.642749, "nauc_mrr_at_1000_std": 0.041208, "nauc_mrr_at_1000_diff1": 0.731112, "hit_rate_at_1": 0.56913, "hit_rate_at_3": 0.70343, "hit_rate_at_5": 0.74202, "hit_rate_at_10": 0.79034, "hit_rate_at_20": 0.82315, "hit_rate_at_100": 0.89213, "hit_rate_at_1000": 0.95229, "main_score": 0.68068, "hf_subset": "javascript", "languages": [ "javascript-Code" ], "mteb_version": "2.21.8" }, { "ndcg_at_1": 0.76213, "ndcg_at_3": 0.82332, "ndcg_at_5": 0.83545, "ndcg_at_10": 0.84478, "ndcg_at_20": 0.84963, "ndcg_at_100": 0.85578, "ndcg_at_1000": 0.85786, "map_at_1": 0.76213, "map_at_3": 0.80875, "map_at_5": 0.81552, "map_at_10": 0.81941, "map_at_20": 0.82075, "map_at_100": 0.82159, "map_at_1000": 0.82167, "recall_at_1": 0.76213, "recall_at_3": 0.8653, "recall_at_5": 0.89461, "recall_at_10": 0.92317, "recall_at_20": 0.94238, "recall_at_100": 0.9755, "recall_at_1000": 0.992, "accuracy": 0.76213, "precision_at_1": 0.76213, "precision_at_3": 0.28843, "precision_at_5": 0.17892, "precision_at_10": 0.09232, "precision_at_20": 0.04712, "precision_at_100": 0.00975, "precision_at_1000": 0.00099, "mrr_at_1": 0.762128, "mrr_at_3": 0.80875, "mrr_at_5": 0.815515, "mrr_at_10": 0.819414, "mrr_at_20": 0.820747, "mrr_at_100": 0.821595, "mrr_at_1000": 0.821672, "nauc_ndcg_at_1_max": 0.437535, "nauc_ndcg_at_1_std": -0.369709, "nauc_ndcg_at_1_diff1": 0.85239, "nauc_ndcg_at_3_max": 0.496726, "nauc_ndcg_at_3_std": -0.389689, "nauc_ndcg_at_3_diff1": 0.825336, "nauc_ndcg_at_5_max": 0.502522, "nauc_ndcg_at_5_std": -0.383692, "nauc_ndcg_at_5_diff1": 0.820877, "nauc_ndcg_at_10_max": 0.50214, "nauc_ndcg_at_10_std": -0.370105, "nauc_ndcg_at_10_diff1": 0.819507, "nauc_ndcg_at_20_max": 0.50463, "nauc_ndcg_at_20_std": -0.364119, "nauc_ndcg_at_20_diff1": 0.822097, "nauc_ndcg_at_100_max": 0.499905, "nauc_ndcg_at_100_std": -0.357869, "nauc_ndcg_at_100_diff1": 0.824869, "nauc_ndcg_at_1000_max": 0.495701, "nauc_ndcg_at_1000_std": -0.36124, "nauc_ndcg_at_1000_diff1": 0.82595, "nauc_map_at_1_max": 0.437535, "nauc_map_at_1_std": -0.369709, "nauc_map_at_1_diff1": 0.85239, "nauc_map_at_3_max": 0.479991, "nauc_map_at_3_std": -0.385019, "nauc_map_at_3_diff1": 0.832579, "nauc_map_at_5_max": 0.482374, "nauc_map_at_5_std": -0.381783, "nauc_map_at_5_diff1": 0.830603, "nauc_map_at_10_max": 0.481866, "nauc_map_at_10_std": -0.376535, "nauc_map_at_10_diff1": 0.830251, "nauc_map_at_20_max": 0.48232, "nauc_map_at_20_std": -0.375195, "nauc_map_at_20_diff1": 0.830972, "nauc_map_at_100_max": 0.481786, "nauc_map_at_100_std": -0.374377, "nauc_map_at_100_diff1": 0.831339, "nauc_map_at_1000_max": 0.481652, "nauc_map_at_1000_std": -0.374454, "nauc_map_at_1000_diff1": 0.831385, "nauc_recall_at_1_max": 0.437535, "nauc_recall_at_1_std": -0.369709, "nauc_recall_at_1_diff1": 0.85239, "nauc_recall_at_3_max": 0.561897, "nauc_recall_at_3_std": -0.407307, "nauc_recall_at_3_diff1": 0.797364, "nauc_recall_at_5_max": 0.598083, "nauc_recall_at_5_std": -0.390642, "nauc_recall_at_5_diff1": 0.774574, "nauc_recall_at_10_max": 0.629498, "nauc_recall_at_10_std": -0.317729, "nauc_recall_at_10_diff1": 0.750716, "nauc_recall_at_20_max": 0.695016, "nauc_recall_at_20_std": -0.24269, "nauc_recall_at_20_diff1": 0.751987, "nauc_recall_at_100_max": 0.80215, "nauc_recall_at_100_std": 0.086893, "nauc_recall_at_100_diff1": 0.738375, "nauc_recall_at_1000_max": 0.876763, "nauc_recall_at_1000_std": 0.52038, "nauc_recall_at_1000_diff1": 0.689573, "nauc_precision_at_1_max": 0.437535, "nauc_precision_at_1_std": -0.369709, "nauc_precision_at_1_diff1": 0.85239, "nauc_precision_at_3_max": 0.561897, "nauc_precision_at_3_std": -0.407307, "nauc_precision_at_3_diff1": 0.797364, "nauc_precision_at_5_max": 0.598083, "nauc_precision_at_5_std": -0.390642, "nauc_precision_at_5_diff1": 0.774574, "nauc_precision_at_10_max": 0.629498, "nauc_precision_at_10_std": -0.317729, "nauc_precision_at_10_diff1": 0.750716, "nauc_precision_at_20_max": 0.695016, "nauc_precision_at_20_std": -0.24269, "nauc_precision_at_20_diff1": 0.751987, "nauc_precision_at_100_max": 0.80215, "nauc_precision_at_100_std": 0.086893, "nauc_precision_at_100_diff1": 0.738375, "nauc_precision_at_1000_max": 0.876763, "nauc_precision_at_1000_std": 0.52038, "nauc_precision_at_1000_diff1": 0.689573, "nauc_mrr_at_1_max": 0.437535, "nauc_mrr_at_1_std": -0.369709, "nauc_mrr_at_1_diff1": 0.85239, "nauc_mrr_at_3_max": 0.479991, "nauc_mrr_at_3_std": -0.385019, "nauc_mrr_at_3_diff1": 0.832579, "nauc_mrr_at_5_max": 0.482374, "nauc_mrr_at_5_std": -0.381783, "nauc_mrr_at_5_diff1": 0.830603, "nauc_mrr_at_10_max": 0.481866, "nauc_mrr_at_10_std": -0.376535, "nauc_mrr_at_10_diff1": 0.830251, "nauc_mrr_at_20_max": 0.48232, "nauc_mrr_at_20_std": -0.375195, "nauc_mrr_at_20_diff1": 0.830972, "nauc_mrr_at_100_max": 0.481786, "nauc_mrr_at_100_std": -0.374377, "nauc_mrr_at_100_diff1": 0.831339, "nauc_mrr_at_1000_max": 0.481652, "nauc_mrr_at_1000_std": -0.374454, "nauc_mrr_at_1000_diff1": 0.831385, "hit_rate_at_1": 0.76213, "hit_rate_at_3": 0.8653, "hit_rate_at_5": 0.89461, "hit_rate_at_10": 0.92317, "hit_rate_at_20": 0.94238, "hit_rate_at_100": 0.9755, "hit_rate_at_1000": 0.992, "main_score": 0.84478, "hf_subset": "go", "languages": [ "go-Code" ], "mteb_version": "2.21.8" }, { "ndcg_at_1": 0.58842, "ndcg_at_3": 0.67828, "ndcg_at_5": 0.69206, "ndcg_at_10": 0.70932, "ndcg_at_20": 0.71984, "ndcg_at_100": 0.73106, "ndcg_at_1000": 0.73725, "map_at_1": 0.58842, "map_at_3": 0.65689, "map_at_5": 0.66458, "map_at_10": 0.6719, "map_at_20": 0.67483, "map_at_100": 0.67646, "map_at_1000": 0.6767, "recall_at_1": 0.58842, "recall_at_3": 0.73989, "recall_at_5": 0.7732, "recall_at_10": 0.82554, "recall_at_20": 0.86677, "recall_at_100": 0.92625, "recall_at_1000": 0.97462, "accuracy": 0.58842, "precision_at_1": 0.58842, "precision_at_3": 0.24663, "precision_at_5": 0.15464, "precision_at_10": 0.08255, "precision_at_20": 0.04334, "precision_at_100": 0.00926, "precision_at_1000": 0.00097, "mrr_at_1": 0.588422, "mrr_at_3": 0.656886, "mrr_at_5": 0.664578, "mrr_at_10": 0.671899, "mrr_at_20": 0.674833, "mrr_at_100": 0.676458, "mrr_at_1000": 0.676697, "nauc_ndcg_at_1_max": 0.545293, "nauc_ndcg_at_1_std": -0.116357, "nauc_ndcg_at_1_diff1": 0.695805, "nauc_ndcg_at_3_max": 0.582541, "nauc_ndcg_at_3_std": -0.084986, "nauc_ndcg_at_3_diff1": 0.644093, "nauc_ndcg_at_5_max": 0.576906, "nauc_ndcg_at_5_std": -0.086724, "nauc_ndcg_at_5_diff1": 0.642583, "nauc_ndcg_at_10_max": 0.579456, "nauc_ndcg_at_10_std": -0.071428, "nauc_ndcg_at_10_diff1": 0.639732, "nauc_ndcg_at_20_max": 0.580926, "nauc_ndcg_at_20_std": -0.066297, "nauc_ndcg_at_20_diff1": 0.642589, "nauc_ndcg_at_100_max": 0.578553, "nauc_ndcg_at_100_std": -0.064448, "nauc_ndcg_at_100_diff1": 0.642987, "nauc_ndcg_at_1000_max": 0.57526, "nauc_ndcg_at_1000_std": -0.071659, "nauc_ndcg_at_1000_diff1": 0.645282, "nauc_map_at_1_max": 0.545293, "nauc_map_at_1_std": -0.116357, "nauc_map_at_1_diff1": 0.695805, "nauc_map_at_3_max": 0.57283, "nauc_map_at_3_std": -0.09275, "nauc_map_at_3_diff1": 0.656861, "nauc_map_at_5_max": 0.569887, "nauc_map_at_5_std": -0.09397, "nauc_map_at_5_diff1": 0.656465, "nauc_map_at_10_max": 0.57052, "nauc_map_at_10_std": -0.088555, "nauc_map_at_10_diff1": 0.655639, "nauc_map_at_20_max": 0.570792, "nauc_map_at_20_std": -0.087403, "nauc_map_at_20_diff1": 0.656427, "nauc_map_at_100_max": 0.570336, "nauc_map_at_100_std": -0.08699, "nauc_map_at_100_diff1": 0.656415, "nauc_map_at_1000_max": 0.570223, "nauc_map_at_1000_std": -0.087198, "nauc_map_at_1000_diff1": 0.656488, "nauc_recall_at_1_max": 0.545293, "nauc_recall_at_1_std": -0.116357, "nauc_recall_at_1_diff1": 0.695805, "nauc_recall_at_3_max": 0.616294, "nauc_recall_at_3_std": -0.058218, "nauc_recall_at_3_diff1": 0.600085, "nauc_recall_at_5_max": 0.602168, "nauc_recall_at_5_std": -0.05948, "nauc_recall_at_5_diff1": 0.58967, "nauc_recall_at_10_max": 0.621559, "nauc_recall_at_10_std": 0.018521, "nauc_recall_at_10_diff1": 0.563723, "nauc_recall_at_20_max": 0.643811, "nauc_recall_at_20_std": 0.079383, "nauc_recall_at_20_diff1": 0.563976, "nauc_recall_at_100_max": 0.65801, "nauc_recall_at_100_std": 0.203002, "nauc_recall_at_100_diff1": 0.518247, "nauc_recall_at_1000_max": 0.589401, "nauc_recall_at_1000_std": 0.20783, "nauc_recall_at_1000_diff1": 0.441231, "nauc_precision_at_1_max": 0.545293, "nauc_precision_at_1_std": -0.116357, "nauc_precision_at_1_diff1": 0.695805, "nauc_precision_at_3_max": 0.616294, "nauc_precision_at_3_std": -0.058218, "nauc_precision_at_3_diff1": 0.600085, "nauc_precision_at_5_max": 0.602168, "nauc_precision_at_5_std": -0.05948, "nauc_precision_at_5_diff1": 0.58967, "nauc_precision_at_10_max": 0.621559, "nauc_precision_at_10_std": 0.018521, "nauc_precision_at_10_diff1": 0.563723, "nauc_precision_at_20_max": 0.643811, "nauc_precision_at_20_std": 0.079383, "nauc_precision_at_20_diff1": 0.563976, "nauc_precision_at_100_max": 0.65801, "nauc_precision_at_100_std": 0.203002, "nauc_precision_at_100_diff1": 0.518247, "nauc_precision_at_1000_max": 0.589401, "nauc_precision_at_1000_std": 0.20783, "nauc_precision_at_1000_diff1": 0.441231, "nauc_mrr_at_1_max": 0.545293, "nauc_mrr_at_1_std": -0.116357, "nauc_mrr_at_1_diff1": 0.695805, "nauc_mrr_at_3_max": 0.57283, "nauc_mrr_at_3_std": -0.09275, "nauc_mrr_at_3_diff1": 0.656861, "nauc_mrr_at_5_max": 0.569887, "nauc_mrr_at_5_std": -0.09397, "nauc_mrr_at_5_diff1": 0.656465, "nauc_mrr_at_10_max": 0.57052, "nauc_mrr_at_10_std": -0.088555, "nauc_mrr_at_10_diff1": 0.655639, "nauc_mrr_at_20_max": 0.570792, "nauc_mrr_at_20_std": -0.087403, "nauc_mrr_at_20_diff1": 0.656427, "nauc_mrr_at_100_max": 0.570336, "nauc_mrr_at_100_std": -0.08699, "nauc_mrr_at_100_diff1": 0.656415, "nauc_mrr_at_1000_max": 0.570223, "nauc_mrr_at_1000_std": -0.087198, "nauc_mrr_at_1000_diff1": 0.656488, "hit_rate_at_1": 0.58842, "hit_rate_at_3": 0.73989, "hit_rate_at_5": 0.7732, "hit_rate_at_10": 0.82554, "hit_rate_at_20": 0.86677, "hit_rate_at_100": 0.92625, "hit_rate_at_1000": 0.97462, "main_score": 0.70932, "hf_subset": "ruby", "languages": [ "ruby-Code" ], "mteb_version": "2.21.8" }, { "ndcg_at_1": 0.67905, "ndcg_at_3": 0.74876, "ndcg_at_5": 0.76111, "ndcg_at_10": 0.77189, "ndcg_at_20": 0.77914, "ndcg_at_100": 0.78738, "ndcg_at_1000": 0.79182, "map_at_1": 0.67905, "map_at_3": 0.73228, "map_at_5": 0.73915, "map_at_10": 0.74362, "map_at_20": 0.74562, "map_at_100": 0.74676, "map_at_1000": 0.74692, "recall_at_1": 0.67905, "recall_at_3": 0.79617, "recall_at_5": 0.82611, "recall_at_10": 0.85933, "recall_at_20": 0.88791, "recall_at_100": 0.93227, "recall_at_1000": 0.96769, "accuracy": 0.67905, "precision_at_1": 0.67905, "precision_at_3": 0.26539, "precision_at_5": 0.16522, "precision_at_10": 0.08593, "precision_at_20": 0.0444, "precision_at_100": 0.00932, "precision_at_1000": 0.00097, "mrr_at_1": 0.679051, "mrr_at_3": 0.732284, "mrr_at_5": 0.739153, "mrr_at_10": 0.743617, "mrr_at_20": 0.745622, "mrr_at_100": 0.746762, "mrr_at_1000": 0.74692, "nauc_ndcg_at_1_max": 0.538823, "nauc_ndcg_at_1_std": -0.098331, "nauc_ndcg_at_1_diff1": 0.787173, "nauc_ndcg_at_3_max": 0.595642, "nauc_ndcg_at_3_std": -0.064754, "nauc_ndcg_at_3_diff1": 0.747737, "nauc_ndcg_at_5_max": 0.601337, "nauc_ndcg_at_5_std": -0.055388, "nauc_ndcg_at_5_diff1": 0.747151, "nauc_ndcg_at_10_max": 0.600428, "nauc_ndcg_at_10_std": -0.048009, "nauc_ndcg_at_10_diff1": 0.745531, "nauc_ndcg_at_20_max": 0.599002, "nauc_ndcg_at_20_std": -0.042598, "nauc_ndcg_at_20_diff1": 0.745651, "nauc_ndcg_at_100_max": 0.597499, "nauc_ndcg_at_100_std": -0.038669, "nauc_ndcg_at_100_diff1": 0.747409, "nauc_ndcg_at_1000_max": 0.5948, "nauc_ndcg_at_1000_std": -0.042803, "nauc_ndcg_at_1000_diff1": 0.749067, "nauc_map_at_1_max": 0.538823, "nauc_map_at_1_std": -0.098331, "nauc_map_at_1_diff1": 0.787173, "nauc_map_at_3_max": 0.581097, "nauc_map_at_3_std": -0.074271, "nauc_map_at_3_diff1": 0.758137, "nauc_map_at_5_max": 0.583676, "nauc_map_at_5_std": -0.069736, "nauc_map_at_5_diff1": 0.758093, "nauc_map_at_10_max": 0.583025, "nauc_map_at_10_std": -0.067171, "nauc_map_at_10_diff1": 0.757692, "nauc_map_at_20_max": 0.582506, "nauc_map_at_20_std": -0.066045, "nauc_map_at_20_diff1": 0.75777, "nauc_map_at_100_max": 0.582222, "nauc_map_at_100_std": -0.065626, "nauc_map_at_100_diff1": 0.757947, "nauc_map_at_1000_max": 0.58214, "nauc_map_at_1000_std": -0.065737, "nauc_map_at_1000_diff1": 0.758009, "nauc_recall_at_1_max": 0.538823, "nauc_recall_at_1_std": -0.098331, "nauc_recall_at_1_diff1": 0.787173, "nauc_recall_at_3_max": 0.646769, "nauc_recall_at_3_std": -0.030789, "nauc_recall_at_3_diff1": 0.711011, "nauc_recall_at_5_max": 0.67248, "nauc_recall_at_5_std": 0.004152, "nauc_recall_at_5_diff1": 0.70352, "nauc_recall_at_10_max": 0.683511, "nauc_recall_at_10_std": 0.050546, "nauc_recall_at_10_diff1": 0.686606, "nauc_recall_at_20_max": 0.692745, "nauc_recall_at_20_std": 0.111305, "nauc_recall_at_20_diff1": 0.674901, "nauc_recall_at_100_max": 0.730696, "nauc_recall_at_100_std": 0.264723, "nauc_recall_at_100_diff1": 0.660656, "nauc_recall_at_1000_max": 0.750907, "nauc_recall_at_1000_std": 0.396931, "nauc_recall_at_1000_diff1": 0.638691, "nauc_precision_at_1_max": 0.538823, "nauc_precision_at_1_std": -0.098331, "nauc_precision_at_1_diff1": 0.787173, "nauc_precision_at_3_max": 0.646769, "nauc_precision_at_3_std": -0.030789, "nauc_precision_at_3_diff1": 0.711011, "nauc_precision_at_5_max": 0.67248, "nauc_precision_at_5_std": 0.004152, "nauc_precision_at_5_diff1": 0.70352, "nauc_precision_at_10_max": 0.683511, "nauc_precision_at_10_std": 0.050546, "nauc_precision_at_10_diff1": 0.686606, "nauc_precision_at_20_max": 0.692745, "nauc_precision_at_20_std": 0.111305, "nauc_precision_at_20_diff1": 0.674901, "nauc_precision_at_100_max": 0.730696, "nauc_precision_at_100_std": 0.264723, "nauc_precision_at_100_diff1": 0.660656, "nauc_precision_at_1000_max": 0.750907, "nauc_precision_at_1000_std": 0.396931, "nauc_precision_at_1000_diff1": 0.638691, "nauc_mrr_at_1_max": 0.538823, "nauc_mrr_at_1_std": -0.098331, "nauc_mrr_at_1_diff1": 0.787173, "nauc_mrr_at_3_max": 0.581097, "nauc_mrr_at_3_std": -0.074271, "nauc_mrr_at_3_diff1": 0.758137, "nauc_mrr_at_5_max": 0.583676, "nauc_mrr_at_5_std": -0.069736, "nauc_mrr_at_5_diff1": 0.758093, "nauc_mrr_at_10_max": 0.583025, "nauc_mrr_at_10_std": -0.067171, "nauc_mrr_at_10_diff1": 0.757692, "nauc_mrr_at_20_max": 0.582506, "nauc_mrr_at_20_std": -0.066045, "nauc_mrr_at_20_diff1": 0.75777, "nauc_mrr_at_100_max": 0.582222, "nauc_mrr_at_100_std": -0.065626, "nauc_mrr_at_100_diff1": 0.757947, "nauc_mrr_at_1000_max": 0.58214, "nauc_mrr_at_1000_std": -0.065737, "nauc_mrr_at_1000_diff1": 0.758009, "hit_rate_at_1": 0.67905, "hit_rate_at_3": 0.79617, "hit_rate_at_5": 0.82611, "hit_rate_at_10": 0.85933, "hit_rate_at_20": 0.88791, "hit_rate_at_100": 0.93227, "hit_rate_at_1000": 0.96769, "main_score": 0.77189, "hf_subset": "java", "languages": [ "java-Code" ], "mteb_version": "2.21.8" }, { "ndcg_at_1": 0.69573, "ndcg_at_3": 0.77165, "ndcg_at_5": 0.78481, "ndcg_at_10": 0.79672, "ndcg_at_20": 0.80382, "ndcg_at_100": 0.81159, "ndcg_at_1000": 0.8151, "map_at_1": 0.69573, "map_at_3": 0.75371, "map_at_5": 0.76103, "map_at_10": 0.766, "map_at_20": 0.76796, "map_at_100": 0.76906, "map_at_1000": 0.76919, "recall_at_1": 0.69573, "recall_at_3": 0.82325, "recall_at_5": 0.85514, "recall_at_10": 0.89168, "recall_at_20": 0.91972, "recall_at_100": 0.96125, "recall_at_1000": 0.98887, "accuracy": 0.69573, "precision_at_1": 0.69573, "precision_at_3": 0.27442, "precision_at_5": 0.17103, "precision_at_10": 0.08917, "precision_at_20": 0.04599, "precision_at_100": 0.00961, "precision_at_1000": 0.00099, "mrr_at_1": 0.695733, "mrr_at_3": 0.753711, "mrr_at_5": 0.761032, "mrr_at_10": 0.766, "mrr_at_20": 0.767959, "mrr_at_100": 0.769058, "mrr_at_1000": 0.769191, "nauc_ndcg_at_1_max": 0.443415, "nauc_ndcg_at_1_std": -0.208362, "nauc_ndcg_at_1_diff1": 0.813545, "nauc_ndcg_at_3_max": 0.50322, "nauc_ndcg_at_3_std": -0.183234, "nauc_ndcg_at_3_diff1": 0.774983, "nauc_ndcg_at_5_max": 0.507173, "nauc_ndcg_at_5_std": -0.172175, "nauc_ndcg_at_5_diff1": 0.771822, "nauc_ndcg_at_10_max": 0.506663, "nauc_ndcg_at_10_std": -0.162126, "nauc_ndcg_at_10_diff1": 0.770434, "nauc_ndcg_at_20_max": 0.506805, "nauc_ndcg_at_20_std": -0.157606, "nauc_ndcg_at_20_diff1": 0.771935, "nauc_ndcg_at_100_max": 0.504259, "nauc_ndcg_at_100_std": -0.156694, "nauc_ndcg_at_100_diff1": 0.774853, "nauc_ndcg_at_1000_max": 0.500128, "nauc_ndcg_at_1000_std": -0.163613, "nauc_ndcg_at_1000_diff1": 0.777234, "nauc_map_at_1_max": 0.443415, "nauc_map_at_1_std": -0.208362, "nauc_map_at_1_diff1": 0.813545, "nauc_map_at_3_max": 0.48684, "nauc_map_at_3_std": -0.190885, "nauc_map_at_3_diff1": 0.785091, "nauc_map_at_5_max": 0.48841, "nauc_map_at_5_std": -0.185398, "nauc_map_at_5_diff1": 0.78378, "nauc_map_at_10_max": 0.487947, "nauc_map_at_10_std": -0.181811, "nauc_map_at_10_diff1": 0.783558, "nauc_map_at_20_max": 0.487812, "nauc_map_at_20_std": -0.180845, "nauc_map_at_20_diff1": 0.783979, "nauc_map_at_100_max": 0.487459, "nauc_map_at_100_std": -0.180788, "nauc_map_at_100_diff1": 0.784371, "nauc_map_at_1000_max": 0.487338, "nauc_map_at_1000_std": -0.180974, "nauc_map_at_1000_diff1": 0.784449, "nauc_recall_at_1_max": 0.443415, "nauc_recall_at_1_std": -0.208362, "nauc_recall_at_1_diff1": 0.813545, "nauc_recall_at_3_max": 0.564807, "nauc_recall_at_3_std": -0.154028, "nauc_recall_at_3_diff1": 0.73723, "nauc_recall_at_5_max": 0.590128, "nauc_recall_at_5_std": -0.111446, "nauc_recall_at_5_diff1": 0.719071, "nauc_recall_at_10_max": 0.610789, "nauc_recall_at_10_std": -0.042314, "nauc_recall_at_10_diff1": 0.695742, "nauc_recall_at_20_max": 0.646944, "nauc_recall_at_20_std": 0.0358, "nauc_recall_at_20_diff1": 0.684671, "nauc_recall_at_100_max": 0.73413, "nauc_recall_at_100_std": 0.258918, "nauc_recall_at_100_diff1": 0.660265, "nauc_recall_at_1000_max": 0.805193, "nauc_recall_at_1000_std": 0.439616, "nauc_recall_at_1000_diff1": 0.657119, "nauc_precision_at_1_max": 0.443415, "nauc_precision_at_1_std": -0.208362, "nauc_precision_at_1_diff1": 0.813545, "nauc_precision_at_3_max": 0.564807, "nauc_precision_at_3_std": -0.154028, "nauc_precision_at_3_diff1": 0.73723, "nauc_precision_at_5_max": 0.590128, "nauc_precision_at_5_std": -0.111446, "nauc_precision_at_5_diff1": 0.719071, "nauc_precision_at_10_max": 0.610789, "nauc_precision_at_10_std": -0.042314, "nauc_precision_at_10_diff1": 0.695742, "nauc_precision_at_20_max": 0.646944, "nauc_precision_at_20_std": 0.0358, "nauc_precision_at_20_diff1": 0.684671, "nauc_precision_at_100_max": 0.73413, "nauc_precision_at_100_std": 0.258918, "nauc_precision_at_100_diff1": 0.660265, "nauc_precision_at_1000_max": 0.805193, "nauc_precision_at_1000_std": 0.439616, "nauc_precision_at_1000_diff1": 0.657119, "nauc_mrr_at_1_max": 0.443415, "nauc_mrr_at_1_std": -0.208362, "nauc_mrr_at_1_diff1": 0.813545, "nauc_mrr_at_3_max": 0.48684, "nauc_mrr_at_3_std": -0.190885, "nauc_mrr_at_3_diff1": 0.785091, "nauc_mrr_at_5_max": 0.48841, "nauc_mrr_at_5_std": -0.185398, "nauc_mrr_at_5_diff1": 0.78378, "nauc_mrr_at_10_max": 0.487947, "nauc_mrr_at_10_std": -0.181811, "nauc_mrr_at_10_diff1": 0.783558, "nauc_mrr_at_20_max": 0.487812, "nauc_mrr_at_20_std": -0.180845, "nauc_mrr_at_20_diff1": 0.783979, "nauc_mrr_at_100_max": 0.487459, "nauc_mrr_at_100_std": -0.180788, "nauc_mrr_at_100_diff1": 0.784371, "nauc_mrr_at_1000_max": 0.487338, "nauc_mrr_at_1000_std": -0.180974, "nauc_mrr_at_1000_diff1": 0.784449, "hit_rate_at_1": 0.69573, "hit_rate_at_3": 0.82325, "hit_rate_at_5": 0.85514, "hit_rate_at_10": 0.89168, "hit_rate_at_20": 0.91972, "hit_rate_at_100": 0.96125, "hit_rate_at_1000": 0.98887, "main_score": 0.79672, "hf_subset": "php", "languages": [ "php-Code" ], "mteb_version": "2.21.8" } ] }, "evaluation_time": 944.5102071349975, "kg_co2_emissions": null, "date": 1790689608.379744, "evaluation_phases": [ { "name": "Data loading", "start": 0.0, "end": 41.770428008865565, "split": "", "subset": "" }, { "name": "Encoding queries and documents", "start": 43.11954454286024, "end": 255.30785734788515, "split": "test", "subset": "python" }, { "name": "Scoring", "start": 273.2398676488083, "end": 288.7789994578343, "split": "test", "subset": "python" }, { "name": "Encoding queries and documents", "start": 289.32690720795654, "end": 321.82946128584445, "split": "test", "subset": "javascript" }, { "name": "Scoring", "start": 350.4805404699873, "end": 353.5408122409135, "split": "test", "subset": "javascript" }, { "name": "Encoding queries and documents", "start": 353.69289420498535, "end": 419.18045164993964, "split": "test", "subset": "go" }, { "name": "Scoring", "start": 458.1036285588052, "end": 466.0194466719404, "split": "test", "subset": "go" }, { "name": "Encoding queries and documents", "start": 466.3127582699526, "end": 481.0792731998954, "split": "test", "subset": "ruby" }, { "name": "Scoring", "start": 525.5462771288585, "end": 526.5616111569107, "split": "test", "subset": "ruby" }, { "name": "Encoding queries and documents", "start": 526.6697382018901, "end": 624.6409210248385, "split": "test", "subset": "java" }, { "name": "Scoring", "start": 682.0917881049681, "end": 692.4047938720323, "split": "test", "subset": "java" }, { "name": "Encoding queries and documents", "start": 692.788185208803, "end": 845.5702604108956, "split": "test", "subset": "php" }, { "name": "Scoring", "start": 927.1499607488513, "end": 943.7577207079157, "split": "test", "subset": "php" } ] }