{ "dataset_revision": "6268b37d6f975f2a134791ba2f250a91d0bdfb4f", "task_name": "WikipediaRerankingMultilingual", "mteb_version": "1.39.1", "scores": { "test": [ { "map": 0.909341, "mrr": 0.909341, "recall_at_10": 1.0, "nAUC_map_max": 0.551323, "nAUC_map_std": 0.395947, "nAUC_map_diff1": 0.833995, "nAUC_mrr_max": 0.551323, "nAUC_mrr_std": 0.395947, "nAUC_mrr_diff1": 0.833995, "nAUC_recall_at_10_max": NaN, "nAUC_recall_at_10_std": NaN, "nAUC_recall_at_10_diff1": NaN, "main_score": 0.909341, "hf_subset": "bg", "languages": [ "bul-Cyrl" ] }, { "map": 0.873308, "mrr": 0.873308, "recall_at_10": 1.0, "nAUC_map_max": 0.431304, "nAUC_map_std": 0.255294, "nAUC_map_diff1": 0.751903, "nAUC_mrr_max": 0.431304, "nAUC_mrr_std": 0.255294, "nAUC_mrr_diff1": 0.751903, "nAUC_recall_at_10_max": NaN, "nAUC_recall_at_10_std": NaN, "nAUC_recall_at_10_diff1": NaN, "main_score": 0.873308, "hf_subset": "bn", "languages": [ "ben-Beng" ] }, { "map": 0.930733, "mrr": 0.930733, "recall_at_10": 1.0, "nAUC_map_max": 0.589054, "nAUC_map_std": 0.414675, "nAUC_map_diff1": 0.860305, "nAUC_mrr_max": 0.589054, "nAUC_mrr_std": 0.414675, "nAUC_mrr_diff1": 0.860305, "nAUC_recall_at_10_max": NaN, "nAUC_recall_at_10_std": NaN, "nAUC_recall_at_10_diff1": NaN, "main_score": 0.930733, "hf_subset": "cs", "languages": [ "ces-Latn" ] }, { "map": 0.932137, "mrr": 0.932582, "recall_at_10": 1.0, "nAUC_map_max": 0.50516, "nAUC_map_std": 0.372346, "nAUC_map_diff1": 0.879844, "nAUC_mrr_max": 0.519729, "nAUC_mrr_std": 0.386119, "nAUC_mrr_diff1": 0.878474, "nAUC_recall_at_10_max": NaN, "nAUC_recall_at_10_std": NaN, "nAUC_recall_at_10_diff1": NaN, "main_score": 0.932137, "hf_subset": "da", "languages": [ "dan-Latn" ] }, { "map": 0.921576, "mrr": 0.921576, "recall_at_10": 1.0, "nAUC_map_max": 0.513301, "nAUC_map_std": 0.299996, "nAUC_map_diff1": 0.828022, "nAUC_mrr_max": 0.513301, "nAUC_mrr_std": 0.299996, "nAUC_mrr_diff1": 0.828022, "nAUC_recall_at_10_max": NaN, "nAUC_recall_at_10_std": NaN, "nAUC_recall_at_10_diff1": NaN, "main_score": 0.921576, "hf_subset": "de", "languages": [ "deu-Latn" ] }, { "map": 0.943099, "mrr": 0.943099, "recall_at_10": 1.0, "nAUC_map_max": 0.494561, "nAUC_map_std": 0.513904, "nAUC_map_diff1": 0.853053, "nAUC_mrr_max": 0.494561, "nAUC_mrr_std": 0.513904, "nAUC_mrr_diff1": 0.853053, "nAUC_recall_at_10_max": NaN, "nAUC_recall_at_10_std": NaN, "nAUC_recall_at_10_diff1": NaN, "main_score": 0.943099, "hf_subset": "en", "languages": [ "eng-Latn" ] }, { "map": 0.913209, "mrr": 0.913209, "recall_at_10": 1.0, "nAUC_map_max": 0.494739, "nAUC_map_std": 0.340003, "nAUC_map_diff1": 0.84242, "nAUC_mrr_max": 0.494739, "nAUC_mrr_std": 0.340003, "nAUC_mrr_diff1": 0.84242, "nAUC_recall_at_10_max": NaN, "nAUC_recall_at_10_std": NaN, "nAUC_recall_at_10_diff1": NaN, "main_score": 0.913209, "hf_subset": "fa", "languages": [ "fas-Arab" ] }, { "map": 0.933293, "mrr": 0.933293, "recall_at_10": 1.0, "nAUC_map_max": 0.392125, "nAUC_map_std": 0.287024, "nAUC_map_diff1": 0.810495, "nAUC_mrr_max": 0.392125, "nAUC_mrr_std": 0.287024, "nAUC_mrr_diff1": 0.810495, "nAUC_recall_at_10_max": NaN, "nAUC_recall_at_10_std": NaN, "nAUC_recall_at_10_diff1": NaN, "main_score": 0.933293, "hf_subset": "fi", "languages": [ "fin-Latn" ] }, { "map": 0.896431, "mrr": 0.897441, "recall_at_10": 1.0, "nAUC_map_max": 0.518695, "nAUC_map_std": 0.273856, "nAUC_map_diff1": 0.841112, "nAUC_mrr_max": 0.524812, "nAUC_mrr_std": 0.282575, "nAUC_mrr_diff1": 0.838403, "nAUC_recall_at_10_max": NaN, "nAUC_recall_at_10_std": NaN, "nAUC_recall_at_10_diff1": NaN, "main_score": 0.896431, "hf_subset": "hi", "languages": [ "hin-Deva" ] }, { "map": 0.923519, "mrr": 0.923519, "recall_at_10": 1.0, "nAUC_map_max": 0.592505, "nAUC_map_std": 0.379554, "nAUC_map_diff1": 0.853647, "nAUC_mrr_max": 0.592505, "nAUC_mrr_std": 0.379554, "nAUC_mrr_diff1": 0.853647, "nAUC_recall_at_10_max": NaN, "nAUC_recall_at_10_std": NaN, "nAUC_recall_at_10_diff1": NaN, "main_score": 0.923519, "hf_subset": "it", "languages": [ "ita-Latn" ] }, { "map": 0.916758, "mrr": 0.917091, "recall_at_10": 1.0, "nAUC_map_max": 0.415238, "nAUC_map_std": 0.283426, "nAUC_map_diff1": 0.779189, "nAUC_mrr_max": 0.41525, "nAUC_mrr_std": 0.289186, "nAUC_mrr_diff1": 0.77786, "nAUC_recall_at_10_max": NaN, "nAUC_recall_at_10_std": NaN, "nAUC_recall_at_10_diff1": NaN, "main_score": 0.916758, "hf_subset": "nl", "languages": [ "nld-Latn" ] }, { "map": 0.911957, "mrr": 0.911957, "recall_at_10": 1.0, "nAUC_map_max": 0.502948, "nAUC_map_std": 0.318297, "nAUC_map_diff1": 0.820296, "nAUC_mrr_max": 0.502948, "nAUC_mrr_std": 0.318297, "nAUC_mrr_diff1": 0.820296, "nAUC_recall_at_10_max": NaN, "nAUC_recall_at_10_std": NaN, "nAUC_recall_at_10_diff1": NaN, "main_score": 0.911957, "hf_subset": "pt", "languages": [ "por-Latn" ] }, { "map": 0.918883, "mrr": 0.919217, "recall_at_10": 1.0, "nAUC_map_max": 0.463619, "nAUC_map_std": 0.317754, "nAUC_map_diff1": 0.825299, "nAUC_mrr_max": 0.461649, "nAUC_mrr_std": 0.318403, "nAUC_mrr_diff1": 0.824105, "nAUC_recall_at_10_max": NaN, "nAUC_recall_at_10_std": NaN, "nAUC_recall_at_10_diff1": NaN, "main_score": 0.918883, "hf_subset": "ro", "languages": [ "ron-Latn" ] }, { "map": 0.91705, "mrr": 0.917383, "recall_at_10": 1.0, "nAUC_map_max": 0.506926, "nAUC_map_std": 0.354902, "nAUC_map_diff1": 0.863848, "nAUC_mrr_max": 0.515957, "nAUC_mrr_std": 0.363386, "nAUC_mrr_diff1": 0.862816, "nAUC_recall_at_10_max": NaN, "nAUC_recall_at_10_std": NaN, "nAUC_recall_at_10_diff1": NaN, "main_score": 0.91705, "hf_subset": "sr", "languages": [ "srp-Cyrl" ] }, { "map": 0.899752, "mrr": 0.899752, "recall_at_10": 1.0, "nAUC_map_max": 0.493498, "nAUC_map_std": 0.248363, "nAUC_map_diff1": 0.832936, "nAUC_mrr_max": 0.493498, "nAUC_mrr_std": 0.248363, "nAUC_mrr_diff1": 0.832936, "nAUC_recall_at_10_max": NaN, "nAUC_recall_at_10_std": NaN, "nAUC_recall_at_10_diff1": NaN, "main_score": 0.899752, "hf_subset": "no", "languages": [ "nor-Latn" ] }, { "map": 0.928002, "mrr": 0.928002, "recall_at_10": 1.0, "nAUC_map_max": 0.427988, "nAUC_map_std": 0.269653, "nAUC_map_diff1": 0.869196, "nAUC_mrr_max": 0.427988, "nAUC_mrr_std": 0.269653, "nAUC_mrr_diff1": 0.869196, "nAUC_recall_at_10_max": NaN, "nAUC_recall_at_10_std": NaN, "nAUC_recall_at_10_diff1": NaN, "main_score": 0.928002, "hf_subset": "sv", "languages": [ "swe-Latn" ] } ] }, "evaluation_time": 5554.873353242874, "kg_co2_emissions": null }