{ "dataset_revision": "d10af93643bb45ce0a9d736f2a482515f3d9e8b1", "task_name": "XGlueWPRReranking", "mteb_version": "2.18.12", "scores": { "validation": [ { "ndcg_at_1": 0.64103, "ndcg_at_3": 0.67776, "ndcg_at_5": 0.71252, "ndcg_at_10": 0.76633, "ndcg_at_20": 0.80984, "ndcg_at_100": 0.82848, "ndcg_at_1000": 0.82848, "map_at_1": 0.17478, "map_at_3": 0.37136, "map_at_5": 0.48239, "map_at_10": 0.61269, "map_at_20": 0.68759, "map_at_100": 0.71316, "map_at_1000": 0.71316, "recall_at_1": 0.17478, "recall_at_3": 0.45143, "recall_at_5": 0.62676, "recall_at_10": 0.82939, "recall_at_20": 0.95326, "recall_at_100": 1.0, "recall_at_1000": 1.0, "accuracy": 0.17478, "precision_at_1": 0.64103, "precision_at_3": 0.60399, "precision_at_5": 0.55071, "precision_at_10": 0.44074, "precision_at_20": 0.29722, "precision_at_100": 0.06875, "precision_at_1000": 0.00687, "mrr_at_1": 0.641026, "mrr_at_3": 0.77113, "mrr_at_5": 0.783167, "mrr_at_10": 0.784727, "mrr_at_20": 0.785289, "mrr_at_100": 0.785289, "mrr_at_1000": 0.785289, "nauc_ndcg_at_1_max": 0.046959, "nauc_ndcg_at_1_std": 0.044431, "nauc_ndcg_at_1_diff1": -0.067217, "nauc_ndcg_at_3_max": 0.057551, "nauc_ndcg_at_3_std": 0.010808, "nauc_ndcg_at_3_diff1": 0.042121, "nauc_ndcg_at_5_max": -0.014029, "nauc_ndcg_at_5_std": -0.072045, "nauc_ndcg_at_5_diff1": 0.088777, "nauc_ndcg_at_10_max": -0.018075, "nauc_ndcg_at_10_std": -0.069342, "nauc_ndcg_at_10_diff1": 0.087059, "nauc_ndcg_at_20_max": 0.060398, "nauc_ndcg_at_20_std": 0.005394, "nauc_ndcg_at_20_diff1": 0.034659, "nauc_ndcg_at_100_max": 0.08895, "nauc_ndcg_at_100_std": 0.023652, "nauc_ndcg_at_100_diff1": -0.011575, "nauc_ndcg_at_1000_max": 0.08895, "nauc_ndcg_at_1000_std": 0.023652, "nauc_ndcg_at_1000_diff1": -0.011575, "nauc_map_at_1_max": -0.180264, "nauc_map_at_1_std": -0.144163, "nauc_map_at_1_diff1": 0.189677, "nauc_map_at_3_max": -0.175613, "nauc_map_at_3_std": -0.149649, "nauc_map_at_3_diff1": 0.22114, "nauc_map_at_5_max": -0.182078, "nauc_map_at_5_std": -0.16186, "nauc_map_at_5_diff1": 0.208263, "nauc_map_at_10_max": -0.103936, "nauc_map_at_10_std": -0.103948, "nauc_map_at_10_diff1": 0.153806, "nauc_map_at_20_max": -0.000935, "nauc_map_at_20_std": -0.031072, "nauc_map_at_20_diff1": 0.085404, "nauc_map_at_100_max": 0.043562, "nauc_map_at_100_std": -0.016727, "nauc_map_at_100_diff1": 0.04494, "nauc_map_at_1000_max": 0.043562, "nauc_map_at_1000_std": -0.016727, "nauc_map_at_1000_diff1": 0.04494, "nauc_recall_at_1_max": -0.180264, "nauc_recall_at_1_std": -0.144163, "nauc_recall_at_1_diff1": 0.189677, "nauc_recall_at_3_max": -0.217448, "nauc_recall_at_3_std": -0.183191, "nauc_recall_at_3_diff1": 0.261132, "nauc_recall_at_5_max": -0.276825, "nauc_recall_at_5_std": -0.245503, "nauc_recall_at_5_diff1": 0.271295, "nauc_recall_at_10_max": -0.28347, "nauc_recall_at_10_std": -0.247884, "nauc_recall_at_10_diff1": 0.301034, "nauc_recall_at_20_max": -0.217216, "nauc_recall_at_20_std": -0.146832, "nauc_recall_at_20_diff1": 0.331802, "nauc_recall_at_100_max": NaN, "nauc_recall_at_100_std": NaN, "nauc_recall_at_100_diff1": NaN, "nauc_recall_at_1000_max": NaN, "nauc_recall_at_1000_std": NaN, "nauc_recall_at_1000_diff1": NaN, "nauc_precision_at_1_max": 0.046959, "nauc_precision_at_1_std": 0.044431, "nauc_precision_at_1_diff1": -0.067217, "nauc_precision_at_3_max": 0.174612, "nauc_precision_at_3_std": 0.107069, "nauc_precision_at_3_diff1": -0.106385, "nauc_precision_at_5_max": 0.19326, "nauc_precision_at_5_std": 0.1123, "nauc_precision_at_5_diff1": -0.159996, "nauc_precision_at_10_max": 0.278023, "nauc_precision_at_10_std": 0.183218, "nauc_precision_at_10_diff1": -0.216528, "nauc_precision_at_20_max": 0.285355, "nauc_precision_at_20_std": 0.187603, "nauc_precision_at_20_diff1": -0.230023, "nauc_precision_at_100_max": 0.265722, "nauc_precision_at_100_std": 0.159473, "nauc_precision_at_100_diff1": -0.229598, "nauc_precision_at_1000_max": 0.265722, "nauc_precision_at_1000_std": 0.159473, "nauc_precision_at_1000_diff1": -0.229598, "nauc_mrr_at_1_max": 0.046959, "nauc_mrr_at_1_std": 0.044431, "nauc_mrr_at_1_diff1": -0.067217, "nauc_mrr_at_3_max": 0.071338, "nauc_mrr_at_3_std": 0.03568, "nauc_mrr_at_3_diff1": -0.048341, "nauc_mrr_at_5_max": 0.069382, "nauc_mrr_at_5_std": 0.031356, "nauc_mrr_at_5_diff1": -0.056293, "nauc_mrr_at_10_max": 0.068202, "nauc_mrr_at_10_std": 0.034863, "nauc_mrr_at_10_diff1": -0.051618, "nauc_mrr_at_20_max": 0.067788, "nauc_mrr_at_20_std": 0.035603, "nauc_mrr_at_20_diff1": -0.052702, "nauc_mrr_at_100_max": 0.067788, "nauc_mrr_at_100_std": 0.035603, "nauc_mrr_at_100_diff1": -0.052702, "nauc_mrr_at_1000_max": 0.067788, "nauc_mrr_at_1000_std": 0.035603, "nauc_mrr_at_1000_diff1": -0.052702, "hit_rate_at_1": 0.64103, "hit_rate_at_3": 0.9302, "hit_rate_at_5": 0.98291, "hit_rate_at_10": 0.99288, "hit_rate_at_20": 1.0, "hit_rate_at_100": 1.0, "hit_rate_at_1000": 1.0, "main_score": 0.71316, "hf_subset": "pt", "languages": [ "por-Latn" ], "mteb_version": "2.18.12" } ], "test": [ { "ndcg_at_1": 0.61448, "ndcg_at_3": 0.66409, "ndcg_at_5": 0.70084, "ndcg_at_10": 0.75938, "ndcg_at_20": 0.80394, "ndcg_at_100": 0.82195, "ndcg_at_1000": 0.82195, "map_at_1": 0.16791, "map_at_3": 0.35846, "map_at_5": 0.47016, "map_at_10": 0.6106, "map_at_20": 0.68533, "map_at_100": 0.70712, "map_at_1000": 0.70712, "recall_at_1": 0.16791, "recall_at_3": 0.43594, "recall_at_5": 0.60898, "recall_at_10": 0.82993, "recall_at_20": 0.95756, "recall_at_100": 1.0, "recall_at_1000": 1.0, "accuracy": 0.16791, "precision_at_1": 0.61448, "precision_at_3": 0.59527, "precision_at_5": 0.55362, "precision_at_10": 0.45081, "precision_at_20": 0.29889, "precision_at_100": 0.06761, "precision_at_1000": 0.00676, "mrr_at_1": 0.614476, "mrr_at_3": 0.748892, "mrr_at_5": 0.763516, "mrr_at_10": 0.766993, "mrr_at_20": 0.76725, "mrr_at_100": 0.76725, "mrr_at_1000": 0.76725, "nauc_ndcg_at_1_max": 0.140405, "nauc_ndcg_at_1_std": 0.04418, "nauc_ndcg_at_1_diff1": 0.008676, "nauc_ndcg_at_3_max": 0.106972, "nauc_ndcg_at_3_std": -0.014236, "nauc_ndcg_at_3_diff1": 0.067007, "nauc_ndcg_at_5_max": 0.069114, "nauc_ndcg_at_5_std": -0.040812, "nauc_ndcg_at_5_diff1": 0.098024, "nauc_ndcg_at_10_max": 0.04343, "nauc_ndcg_at_10_std": -0.040673, "nauc_ndcg_at_10_diff1": 0.098486, "nauc_ndcg_at_20_max": 0.087336, "nauc_ndcg_at_20_std": -0.008664, "nauc_ndcg_at_20_diff1": 0.063503, "nauc_ndcg_at_100_max": 0.130098, "nauc_ndcg_at_100_std": 0.019969, "nauc_ndcg_at_100_diff1": 0.023228, "nauc_ndcg_at_1000_max": 0.130098, "nauc_ndcg_at_1000_std": 0.019969, "nauc_ndcg_at_1000_diff1": 0.023228, "nauc_map_at_1_max": -0.125479, "nauc_map_at_1_std": -0.118555, "nauc_map_at_1_diff1": 0.100769, "nauc_map_at_3_max": -0.117789, "nauc_map_at_3_std": -0.122594, "nauc_map_at_3_diff1": 0.12104, "nauc_map_at_5_max": -0.110819, "nauc_map_at_5_std": -0.101121, "nauc_map_at_5_diff1": 0.126664, "nauc_map_at_10_max": -0.0497, "nauc_map_at_10_std": -0.066668, "nauc_map_at_10_diff1": 0.127225, "nauc_map_at_20_max": 0.034537, "nauc_map_at_20_std": -0.027992, "nauc_map_at_20_diff1": 0.08515, "nauc_map_at_100_max": 0.088268, "nauc_map_at_100_std": 0.000847, "nauc_map_at_100_diff1": 0.054796, "nauc_map_at_1000_max": 0.088268, "nauc_map_at_1000_std": 0.000847, "nauc_map_at_1000_diff1": 0.054796, "nauc_recall_at_1_max": -0.125479, "nauc_recall_at_1_std": -0.118555, "nauc_recall_at_1_diff1": 0.100769, "nauc_recall_at_3_max": -0.156625, "nauc_recall_at_3_std": -0.178219, "nauc_recall_at_3_diff1": 0.146223, "nauc_recall_at_5_max": -0.197709, "nauc_recall_at_5_std": -0.167865, "nauc_recall_at_5_diff1": 0.186327, "nauc_recall_at_10_max": -0.231952, "nauc_recall_at_10_std": -0.176092, "nauc_recall_at_10_diff1": 0.267801, "nauc_recall_at_20_max": -0.286506, "nauc_recall_at_20_std": -0.245736, "nauc_recall_at_20_diff1": 0.342537, "nauc_recall_at_100_max": NaN, "nauc_recall_at_100_std": NaN, "nauc_recall_at_100_diff1": NaN, "nauc_recall_at_1000_max": NaN, "nauc_recall_at_1000_std": NaN, "nauc_recall_at_1000_diff1": NaN, "nauc_precision_at_1_max": 0.140405, "nauc_precision_at_1_std": 0.04418, "nauc_precision_at_1_diff1": 0.008676, "nauc_precision_at_3_max": 0.188844, "nauc_precision_at_3_std": 0.078273, "nauc_precision_at_3_diff1": -0.003924, "nauc_precision_at_5_max": 0.211741, "nauc_precision_at_5_std": 0.120134, "nauc_precision_at_5_diff1": -0.037109, "nauc_precision_at_10_max": 0.241792, "nauc_precision_at_10_std": 0.136501, "nauc_precision_at_10_diff1": -0.076012, "nauc_precision_at_20_max": 0.261826, "nauc_precision_at_20_std": 0.14393, "nauc_precision_at_20_diff1": -0.11576, "nauc_precision_at_100_max": 0.273511, "nauc_precision_at_100_std": 0.157132, "nauc_precision_at_100_diff1": -0.147421, "nauc_precision_at_1000_max": 0.273511, "nauc_precision_at_1000_std": 0.157132, "nauc_precision_at_1000_diff1": -0.147421, "nauc_mrr_at_1_max": 0.140405, "nauc_mrr_at_1_std": 0.04418, "nauc_mrr_at_1_diff1": 0.008676, "nauc_mrr_at_3_max": 0.14365, "nauc_mrr_at_3_std": -0.000439, "nauc_mrr_at_3_diff1": 0.03946, "nauc_mrr_at_5_max": 0.146883, "nauc_mrr_at_5_std": 0.014494, "nauc_mrr_at_5_diff1": 0.03229, "nauc_mrr_at_10_max": 0.14445, "nauc_mrr_at_10_std": 0.016487, "nauc_mrr_at_10_diff1": 0.030366, "nauc_mrr_at_20_max": 0.144873, "nauc_mrr_at_20_std": 0.016784, "nauc_mrr_at_20_diff1": 0.030657, "nauc_mrr_at_100_max": 0.144873, "nauc_mrr_at_100_std": 0.016784, "nauc_mrr_at_100_diff1": 0.030657, "nauc_mrr_at_1000_max": 0.144873, "nauc_mrr_at_1000_std": 0.016784, "nauc_mrr_at_1000_diff1": 0.030657, "hit_rate_at_1": 0.61448, "hit_rate_at_3": 0.9099, "hit_rate_at_5": 0.97194, "hit_rate_at_10": 0.99705, "hit_rate_at_20": 1.0, "hit_rate_at_100": 1.0, "hit_rate_at_1000": 1.0, "main_score": 0.70712, "hf_subset": "pt", "languages": [ "por-Latn" ], "mteb_version": "2.18.12" } ] }, "evaluation_time": 32.10045784525573, "kg_co2_emissions": null, "date": 1785870185.236382, "evaluation_phases": [ { "name": "Data loading", "start": 3.7224963307380676e-05, "end": 30.139617369975895, "split": "", "subset": "" }, { "name": "Dataset transform", "start": 30.13962749717757, "end": 30.139630849007517, "split": "", "subset": "" }, { "name": "Encoding queries and documents", "start": 30.14772836305201, "end": 31.044299425091594, "split": "validation", "subset": "pt" }, { "name": "Scoring", "start": 31.044526406098157, "end": 31.12916013225913, "split": "validation", "subset": "pt" }, { "name": "Encoding queries and documents", "start": 31.143778371158987, "end": 32.00991778029129, "split": "test", "subset": "pt" }, { "name": "Scoring", "start": 32.010096851270646, "end": 32.092387080192566, "split": "test", "subset": "pt" } ] }