{ "dataset_revision": "803771c366038ed587b21e3d8fe25f8f73134fad", "task_name": "WikipediaRerankingMultilingual", "mteb_version": "2.20.11", "scores": { "test": [ { "ndcg_at_1": 0.80333, "ndcg_at_3": 0.8822, "ndcg_at_5": 0.89583, "ndcg_at_10": 0.90596, "ndcg_at_20": 0.90596, "ndcg_at_100": 0.90596, "ndcg_at_1000": 0.90596, "map_at_1": 0.80333, "map_at_3": 0.86311, "map_at_5": 0.87078, "map_at_10": 0.87521, "map_at_20": 0.87521, "map_at_100": 0.87521, "map_at_1000": 0.87521, "recall_at_1": 0.80333, "recall_at_3": 0.93733, "recall_at_5": 0.97, "recall_at_10": 1.0, "recall_at_20": 1.0, "recall_at_100": 1.0, "recall_at_1000": 1.0, "accuracy": 0.80333, "precision_at_1": 0.80333, "precision_at_3": 0.31244, "precision_at_5": 0.194, "precision_at_10": 0.1, "precision_at_20": 0.05, "precision_at_100": 0.01, "precision_at_1000": 0.001, "mrr_at_1": 0.803333, "mrr_at_3": 0.863111, "mrr_at_5": 0.870778, "mrr_at_10": 0.875206, "mrr_at_20": 0.875206, "mrr_at_100": 0.875206, "mrr_at_1000": 0.875206, "nauc_ndcg_at_1_max": 0.44343, "nauc_ndcg_at_1_std": 0.266737, "nauc_ndcg_at_1_diff1": 0.788361, "nauc_ndcg_at_3_max": 0.486091, "nauc_ndcg_at_3_std": 0.339866, "nauc_ndcg_at_3_diff1": 0.767308, "nauc_ndcg_at_5_max": 0.483613, "nauc_ndcg_at_5_std": 0.332973, "nauc_ndcg_at_5_diff1": 0.763466, "nauc_ndcg_at_10_max": 0.465181, "nauc_ndcg_at_10_std": 0.306259, "nauc_ndcg_at_10_diff1": 0.773204, "nauc_ndcg_at_20_max": 0.465181, "nauc_ndcg_at_20_std": 0.306259, "nauc_ndcg_at_20_diff1": 0.773204, "nauc_ndcg_at_100_max": 0.465181, "nauc_ndcg_at_100_std": 0.306259, "nauc_ndcg_at_100_diff1": 0.773204, "nauc_ndcg_at_1000_max": 0.465181, "nauc_ndcg_at_1000_std": 0.306259, "nauc_ndcg_at_1000_diff1": 0.773204, "nauc_map_at_1_max": 0.44343, "nauc_map_at_1_std": 0.266737, "nauc_map_at_1_diff1": 0.788361, "nauc_map_at_3_max": 0.471778, "nauc_map_at_3_std": 0.316663, "nauc_map_at_3_diff1": 0.772348, "nauc_map_at_5_max": 0.469398, "nauc_map_at_5_std": 0.311757, "nauc_map_at_5_diff1": 0.770881, "nauc_map_at_10_max": 0.463068, "nauc_map_at_10_std": 0.302661, "nauc_map_at_10_diff1": 0.774383, "nauc_map_at_20_max": 0.463068, "nauc_map_at_20_std": 0.302661, "nauc_map_at_20_diff1": 0.774383, "nauc_map_at_100_max": 0.463068, "nauc_map_at_100_std": 0.302661, "nauc_map_at_100_diff1": 0.774383, "nauc_map_at_1000_max": 0.463068, "nauc_map_at_1000_std": 0.302661, "nauc_map_at_1000_diff1": 0.774383, "nauc_recall_at_1_max": 0.44343, "nauc_recall_at_1_std": 0.266737, "nauc_recall_at_1_diff1": 0.788361, "nauc_recall_at_3_max": 0.57189, "nauc_recall_at_3_std": 0.477795, "nauc_recall_at_3_diff1": 0.73886, "nauc_recall_at_5_max": 0.651717, "nauc_recall_at_5_std": 0.578649, "nauc_recall_at_5_diff1": 0.677913, "nauc_recall_at_10_max": NaN, "nauc_recall_at_10_std": NaN, "nauc_recall_at_10_diff1": NaN, "nauc_recall_at_20_max": NaN, "nauc_recall_at_20_std": NaN, "nauc_recall_at_20_diff1": NaN, "nauc_recall_at_100_max": NaN, "nauc_recall_at_100_std": NaN, "nauc_recall_at_100_diff1": NaN, "nauc_recall_at_1000_max": NaN, "nauc_recall_at_1000_std": NaN, "nauc_recall_at_1000_diff1": NaN, "nauc_precision_at_1_max": 0.44343, "nauc_precision_at_1_std": 0.266737, "nauc_precision_at_1_diff1": 0.788361, "nauc_precision_at_3_max": 0.57189, "nauc_precision_at_3_std": 0.477795, "nauc_precision_at_3_diff1": 0.73886, "nauc_precision_at_5_max": 0.651717, "nauc_precision_at_5_std": 0.578649, "nauc_precision_at_5_diff1": 0.677913, "nauc_precision_at_10_max": NaN, "nauc_precision_at_10_std": NaN, "nauc_precision_at_10_diff1": NaN, "nauc_precision_at_20_max": NaN, "nauc_precision_at_20_std": NaN, "nauc_precision_at_20_diff1": NaN, "nauc_precision_at_100_max": 1.0, "nauc_precision_at_100_std": 1.0, "nauc_precision_at_100_diff1": 1.0, "nauc_precision_at_1000_max": 1.0, "nauc_precision_at_1000_std": 1.0, "nauc_precision_at_1000_diff1": 1.0, "nauc_mrr_at_1_max": 0.44343, "nauc_mrr_at_1_std": 0.266737, "nauc_mrr_at_1_diff1": 0.788361, "nauc_mrr_at_3_max": 0.471778, "nauc_mrr_at_3_std": 0.316663, "nauc_mrr_at_3_diff1": 0.772348, "nauc_mrr_at_5_max": 0.469398, "nauc_mrr_at_5_std": 0.311757, "nauc_mrr_at_5_diff1": 0.770881, "nauc_mrr_at_10_max": 0.463068, "nauc_mrr_at_10_std": 0.302661, "nauc_mrr_at_10_diff1": 0.774383, "nauc_mrr_at_20_max": 0.463068, "nauc_mrr_at_20_std": 0.302661, "nauc_mrr_at_20_diff1": 0.774383, "nauc_mrr_at_100_max": 0.463068, "nauc_mrr_at_100_std": 0.302661, "nauc_mrr_at_100_diff1": 0.774383, "nauc_mrr_at_1000_max": 0.463068, "nauc_mrr_at_1000_std": 0.302661, "nauc_mrr_at_1000_diff1": 0.774383, "hit_rate_at_1": 0.80333, "hit_rate_at_3": 0.93733, "hit_rate_at_5": 0.97, "hit_rate_at_10": 1.0, "hit_rate_at_20": 1.0, "hit_rate_at_100": 1.0, "hit_rate_at_1000": 1.0, "main_score": 0.87521, "hf_subset": "bn", "languages": [ "ben-Beng" ], "mteb_version": "2.20.11" }, { "ndcg_at_1": 0.81533, "ndcg_at_3": 0.88987, "ndcg_at_5": 0.90571, "ndcg_at_10": 0.91296, "ndcg_at_20": 0.91296, "ndcg_at_100": 0.91296, "ndcg_at_1000": 0.91296, "map_at_1": 0.81533, "map_at_3": 0.87222, "map_at_5": 0.88112, "map_at_10": 0.88432, "map_at_20": 0.88432, "map_at_100": 0.88432, "map_at_1000": 0.88432, "recall_at_1": 0.81533, "recall_at_3": 0.94067, "recall_at_5": 0.97867, "recall_at_10": 1.0, "recall_at_20": 1.0, "recall_at_100": 1.0, "recall_at_1000": 1.0, "accuracy": 0.81533, "precision_at_1": 0.81533, "precision_at_3": 0.31356, "precision_at_5": 0.19573, "precision_at_10": 0.1, "precision_at_20": 0.05, "precision_at_100": 0.01, "precision_at_1000": 0.001, "mrr_at_1": 0.815333, "mrr_at_3": 0.872222, "mrr_at_5": 0.881122, "mrr_at_10": 0.884317, "mrr_at_20": 0.884317, "mrr_at_100": 0.884317, "mrr_at_1000": 0.884317, "nauc_ndcg_at_1_max": 0.420525, "nauc_ndcg_at_1_std": 0.256156, "nauc_ndcg_at_1_diff1": 0.788825, "nauc_ndcg_at_3_max": 0.435469, "nauc_ndcg_at_3_std": 0.300062, "nauc_ndcg_at_3_diff1": 0.75534, "nauc_ndcg_at_5_max": 0.440908, "nauc_ndcg_at_5_std": 0.297354, "nauc_ndcg_at_5_diff1": 0.760189, "nauc_ndcg_at_10_max": 0.427244, "nauc_ndcg_at_10_std": 0.273707, "nauc_ndcg_at_10_diff1": 0.767079, "nauc_ndcg_at_20_max": 0.427244, "nauc_ndcg_at_20_std": 0.273707, "nauc_ndcg_at_20_diff1": 0.767079, "nauc_ndcg_at_100_max": 0.427244, "nauc_ndcg_at_100_std": 0.273707, "nauc_ndcg_at_100_diff1": 0.767079, "nauc_ndcg_at_1000_max": 0.427244, "nauc_ndcg_at_1000_std": 0.273707, "nauc_ndcg_at_1000_diff1": 0.767079, "nauc_map_at_1_max": 0.420525, "nauc_map_at_1_std": 0.256156, "nauc_map_at_1_diff1": 0.788825, "nauc_map_at_3_max": 0.429526, "nauc_map_at_3_std": 0.282359, "nauc_map_at_3_diff1": 0.764166, "nauc_map_at_5_max": 0.431222, "nauc_map_at_5_std": 0.279838, "nauc_map_at_5_diff1": 0.766685, "nauc_map_at_10_max": 0.426366, "nauc_map_at_10_std": 0.271544, "nauc_map_at_10_diff1": 0.76882, "nauc_map_at_20_max": 0.426366, "nauc_map_at_20_std": 0.271544, "nauc_map_at_20_diff1": 0.76882, "nauc_map_at_100_max": 0.426366, "nauc_map_at_100_std": 0.271544, "nauc_map_at_100_diff1": 0.76882, "nauc_map_at_1000_max": 0.426366, "nauc_map_at_1000_std": 0.271544, "nauc_map_at_1000_diff1": 0.76882, "nauc_recall_at_1_max": 0.420525, "nauc_recall_at_1_std": 0.256156, "nauc_recall_at_1_diff1": 0.788825, "nauc_recall_at_3_max": 0.471506, "nauc_recall_at_3_std": 0.407589, "nauc_recall_at_3_diff1": 0.705111, "nauc_recall_at_5_max": 0.596785, "nauc_recall_at_5_std": 0.569882, "nauc_recall_at_5_diff1": 0.672371, "nauc_recall_at_10_max": NaN, "nauc_recall_at_10_std": NaN, "nauc_recall_at_10_diff1": NaN, "nauc_recall_at_20_max": NaN, "nauc_recall_at_20_std": NaN, "nauc_recall_at_20_diff1": NaN, "nauc_recall_at_100_max": NaN, "nauc_recall_at_100_std": NaN, "nauc_recall_at_100_diff1": NaN, "nauc_recall_at_1000_max": NaN, "nauc_recall_at_1000_std": NaN, "nauc_recall_at_1000_diff1": NaN, "nauc_precision_at_1_max": 0.420525, "nauc_precision_at_1_std": 0.256156, "nauc_precision_at_1_diff1": 0.788825, "nauc_precision_at_3_max": 0.471506, "nauc_precision_at_3_std": 0.407589, "nauc_precision_at_3_diff1": 0.705111, "nauc_precision_at_5_max": 0.596785, "nauc_precision_at_5_std": 0.569882, "nauc_precision_at_5_diff1": 0.672371, "nauc_precision_at_10_max": NaN, "nauc_precision_at_10_std": NaN, "nauc_precision_at_10_diff1": NaN, "nauc_precision_at_20_max": NaN, "nauc_precision_at_20_std": NaN, "nauc_precision_at_20_diff1": NaN, "nauc_precision_at_100_max": 1.0, "nauc_precision_at_100_std": 1.0, "nauc_precision_at_100_diff1": 1.0, "nauc_precision_at_1000_max": 1.0, "nauc_precision_at_1000_std": 1.0, "nauc_precision_at_1000_diff1": 1.0, "nauc_mrr_at_1_max": 0.420525, "nauc_mrr_at_1_std": 0.256156, "nauc_mrr_at_1_diff1": 0.788825, "nauc_mrr_at_3_max": 0.429526, "nauc_mrr_at_3_std": 0.282359, "nauc_mrr_at_3_diff1": 0.764166, "nauc_mrr_at_5_max": 0.431222, "nauc_mrr_at_5_std": 0.279838, "nauc_mrr_at_5_diff1": 0.766685, "nauc_mrr_at_10_max": 0.426366, "nauc_mrr_at_10_std": 0.271544, "nauc_mrr_at_10_diff1": 0.76882, "nauc_mrr_at_20_max": 0.426366, "nauc_mrr_at_20_std": 0.271544, "nauc_mrr_at_20_diff1": 0.76882, "nauc_mrr_at_100_max": 0.426366, "nauc_mrr_at_100_std": 0.271544, "nauc_mrr_at_100_diff1": 0.76882, "nauc_mrr_at_1000_max": 0.426366, "nauc_mrr_at_1000_std": 0.271544, "nauc_mrr_at_1000_diff1": 0.76882, "hit_rate_at_1": 0.81533, "hit_rate_at_3": 0.94067, "hit_rate_at_5": 0.97867, "hit_rate_at_10": 1.0, "hit_rate_at_20": 1.0, "hit_rate_at_100": 1.0, "hit_rate_at_1000": 1.0, "main_score": 0.88432, "hf_subset": "hi", "languages": [ "hin-Deva" ], "mteb_version": "2.20.11" } ] }, "evaluation_time": 2846.0344490539283, "kg_co2_emissions": null, "date": 1789032923.144652, "evaluation_phases": [ { "name": "Data loading", "start": 4.5869965106248856e-06, "end": 29.8960622579325, "split": "", "subset": "" }, { "name": "Dataset transform", "start": 29.896074696909636, "end": 29.89608157798648, "split": "", "subset": "" }, { "name": "Encoding queries and documents", "start": 29.910682881018147, "end": 1493.8077672240324, "split": "test", "subset": "bn" }, { "name": "Scoring", "start": 1493.8078251320403, "end": 1493.9132697570603, "split": "test", "subset": "bn" }, { "name": "Encoding queries and documents", "start": 1494.6710836230777, "end": 2845.9181345799007, "split": "test", "subset": "hi" }, { "name": "Scoring", "start": 2845.9181951039936, "end": 2846.0129961520433, "split": "test", "subset": "hi" } ] }