{ "dataset_revision": "58d73d2cddefab7df4bc0b814aa5c53c5fd4928e", "task_name": "MInDS14", "mteb_version": "1.38.33", "scores": { "train": [ { "accuracy": 0.478094, "f1": 0.47197, "f1_weighted": 0.479532, "scores_per_experiment": [ { "accuracy": 0.445378, "f1": 0.453552, "f1_weighted": 0.442107 }, { "accuracy": 0.478992, "f1": 0.464282, "f1_weighted": 0.483878 }, { "accuracy": 0.40678, "f1": 0.395435, "f1_weighted": 0.404207 }, { "accuracy": 0.5, "f1": 0.494244, "f1_weighted": 0.510097 }, { "accuracy": 0.559322, "f1": 0.552335, "f1_weighted": 0.557372 } ], "main_score": 0.478094, "hf_subset": "en-GB", "languages": [ "eng-Latn" ] }, { "accuracy": 0.183679, "f1": 0.164768, "f1_weighted": 0.183351, "scores_per_experiment": [ { "accuracy": 0.185185, "f1": 0.16394, "f1_weighted": 0.174722 }, { "accuracy": 0.12963, "f1": 0.122866, "f1_weighted": 0.132464 }, { "accuracy": 0.203704, "f1": 0.154068, "f1_weighted": 0.212658 }, { "accuracy": 0.212963, "f1": 0.206442, "f1_weighted": 0.217001 }, { "accuracy": 0.186916, "f1": 0.176525, "f1_weighted": 0.179913 } ], "main_score": 0.183679, "hf_subset": "fr-FR", "languages": [ "fra-Latn" ] }, { "accuracy": 0.22409, "f1": 0.212964, "f1_weighted": 0.223077, "scores_per_experiment": [ { "accuracy": 0.257143, "f1": 0.24757, "f1_weighted": 0.252465 }, { "accuracy": 0.258993, "f1": 0.245224, "f1_weighted": 0.254322 }, { "accuracy": 0.230216, "f1": 0.216596, "f1_weighted": 0.223925 }, { "accuracy": 0.208633, "f1": 0.203293, "f1_weighted": 0.215659 }, { "accuracy": 0.165468, "f1": 0.152138, "f1_weighted": 0.169014 } ], "main_score": 0.22409, "hf_subset": "it-IT", "languages": [ "ita-Latn" ] }, { "accuracy": 0.170713, "f1": 0.155515, "f1_weighted": 0.169267, "scores_per_experiment": [ { "accuracy": 0.204082, "f1": 0.167191, "f1_weighted": 0.205706 }, { "accuracy": 0.134021, "f1": 0.128294, "f1_weighted": 0.133101 }, { "accuracy": 0.195876, "f1": 0.176885, "f1_weighted": 0.192026 }, { "accuracy": 0.185567, "f1": 0.188361, "f1_weighted": 0.179443 }, { "accuracy": 0.134021, "f1": 0.116845, "f1_weighted": 0.136058 } ], "main_score": 0.170713, "hf_subset": "es-ES", "languages": [ "spa-Latn" ] }, { "accuracy": 0.221818, "f1": 0.202844, "f1_weighted": 0.22366, "scores_per_experiment": [ { "accuracy": 0.256198, "f1": 0.253144, "f1_weighted": 0.258455 }, { "accuracy": 0.239669, "f1": 0.231798, "f1_weighted": 0.243414 }, { "accuracy": 0.214876, "f1": 0.183244, "f1_weighted": 0.228145 }, { "accuracy": 0.198347, "f1": 0.157034, "f1_weighted": 0.191489 }, { "accuracy": 0.2, "f1": 0.188998, "f1_weighted": 0.196798 } ], "main_score": 0.221818, "hf_subset": "pt-PT", "languages": [ "por-Latn" ] }, { "accuracy": 0.171758, "f1": 0.165658, "f1_weighted": 0.175077, "scores_per_experiment": [ { "accuracy": 0.227642, "f1": 0.218826, "f1_weighted": 0.218521 }, { "accuracy": 0.229508, "f1": 0.221085, "f1_weighted": 0.236254 }, { "accuracy": 0.147541, "f1": 0.138816, "f1_weighted": 0.151269 }, { "accuracy": 0.122951, "f1": 0.119463, "f1_weighted": 0.126868 }, { "accuracy": 0.131148, "f1": 0.130099, "f1_weighted": 0.142471 } ], "main_score": 0.171758, "hf_subset": "de-DE", "languages": [ "deu-Latn" ] }, { "accuracy": 0.229325, "f1": 0.22106, "f1_weighted": 0.224682, "scores_per_experiment": [ { "accuracy": 0.198473, "f1": 0.197702, "f1_weighted": 0.193526 }, { "accuracy": 0.229008, "f1": 0.223141, "f1_weighted": 0.220325 }, { "accuracy": 0.236641, "f1": 0.237006, "f1_weighted": 0.233673 }, { "accuracy": 0.274809, "f1": 0.242188, "f1_weighted": 0.272092 }, { "accuracy": 0.207692, "f1": 0.205261, "f1_weighted": 0.203792 } ], "main_score": 0.229325, "hf_subset": "nl-NL", "languages": [ "nld-Latn" ] }, { "accuracy": 0.202233, "f1": 0.185625, "f1_weighted": 0.203972, "scores_per_experiment": [ { "accuracy": 0.25, "f1": 0.218341, "f1_weighted": 0.235984 }, { "accuracy": 0.240741, "f1": 0.213757, "f1_weighted": 0.255069 }, { "accuracy": 0.166667, "f1": 0.146407, "f1_weighted": 0.164582 }, { "accuracy": 0.148148, "f1": 0.146339, "f1_weighted": 0.153361 }, { "accuracy": 0.205607, "f1": 0.203282, "f1_weighted": 0.210863 } ], "main_score": 0.202233, "hf_subset": "ru-RU", "languages": [ "rus-Cyrl" ] }, { "accuracy": 0.225996, "f1": 0.21797, "f1_weighted": 0.225972, "scores_per_experiment": [ { "accuracy": 0.168142, "f1": 0.170491, "f1_weighted": 0.173864 }, { "accuracy": 0.274336, "f1": 0.266294, "f1_weighted": 0.270529 }, { "accuracy": 0.196429, "f1": 0.169399, "f1_weighted": 0.186708 }, { "accuracy": 0.223214, "f1": 0.219763, "f1_weighted": 0.232458 }, { "accuracy": 0.267857, "f1": 0.263901, "f1_weighted": 0.266304 } ], "main_score": 0.225996, "hf_subset": "pl-PL", "languages": [ "pol-Latn" ] }, { "accuracy": 0.261297, "f1": 0.249122, "f1_weighted": 0.266137, "scores_per_experiment": [ { "accuracy": 0.330435, "f1": 0.331922, "f1_weighted": 0.340256 }, { "accuracy": 0.252174, "f1": 0.238032, "f1_weighted": 0.258946 }, { "accuracy": 0.208696, "f1": 0.185796, "f1_weighted": 0.216811 }, { "accuracy": 0.269565, "f1": 0.250763, "f1_weighted": 0.27609 }, { "accuracy": 0.245614, "f1": 0.239097, "f1_weighted": 0.238584 } ], "main_score": 0.261297, "hf_subset": "cs-CZ", "languages": [ "ces-Latn" ] }, { "accuracy": 0.143641, "f1": 0.137948, "f1_weighted": 0.141816, "scores_per_experiment": [ { "accuracy": 0.092437, "f1": 0.087521, "f1_weighted": 0.095476 }, { "accuracy": 0.159664, "f1": 0.160911, "f1_weighted": 0.165481 }, { "accuracy": 0.118644, "f1": 0.108857, "f1_weighted": 0.117581 }, { "accuracy": 0.177966, "f1": 0.170947, "f1_weighted": 0.163058 }, { "accuracy": 0.169492, "f1": 0.161506, "f1_weighted": 0.167485 } ], "main_score": 0.143641, "hf_subset": "ko-KR", "languages": [ "kor-Hang" ] }, { "accuracy": 0.135347, "f1": 0.128186, "f1_weighted": 0.136806, "scores_per_experiment": [ { "accuracy": 0.158416, "f1": 0.144369, "f1_weighted": 0.164758 }, { "accuracy": 0.168317, "f1": 0.160585, "f1_weighted": 0.160548 }, { "accuracy": 0.14, "f1": 0.119104, "f1_weighted": 0.147108 }, { "accuracy": 0.09, "f1": 0.095402, "f1_weighted": 0.09225 }, { "accuracy": 0.12, "f1": 0.12147, "f1_weighted": 0.119367 } ], "main_score": 0.135347, "hf_subset": "zh-CN", "languages": [ "zho-Hans" ] } ] }, "evaluation_time": 2457.885462284088, "kg_co2_emissions": null }