{ "dataset_revision": "58d73d2cddefab7df4bc0b814aa5c53c5fd4928e", "task_name": "MInDS14", "mteb_version": "1.38.33", "scores": { "train": [ { "accuracy": 0.319228, "f1": 0.309539, "f1_weighted": 0.316452, "scores_per_experiment": [ { "accuracy": 0.252101, "f1": 0.246006, "f1_weighted": 0.245003 }, { "accuracy": 0.403361, "f1": 0.376582, "f1_weighted": 0.399896 }, { "accuracy": 0.322034, "f1": 0.322491, "f1_weighted": 0.331958 }, { "accuracy": 0.322034, "f1": 0.312745, "f1_weighted": 0.321531 }, { "accuracy": 0.29661, "f1": 0.28987, "f1_weighted": 0.283873 } ], "main_score": 0.319228, "hf_subset": "en-GB", "languages": [ "eng-Latn" ] }, { "accuracy": 0.345119, "f1": 0.319707, "f1_weighted": 0.341634, "scores_per_experiment": [ { "accuracy": 0.388889, "f1": 0.378683, "f1_weighted": 0.410134 }, { "accuracy": 0.342593, "f1": 0.315674, "f1_weighted": 0.331644 }, { "accuracy": 0.333333, "f1": 0.300644, "f1_weighted": 0.340744 }, { "accuracy": 0.296296, "f1": 0.288369, "f1_weighted": 0.293904 }, { "accuracy": 0.364486, "f1": 0.315169, "f1_weighted": 0.331742 } ], "main_score": 0.345119, "hf_subset": "fr-FR", "languages": [ "fra-Latn" ] }, { "accuracy": 0.327677, "f1": 0.328371, "f1_weighted": 0.329243, "scores_per_experiment": [ { "accuracy": 0.264286, "f1": 0.24681, "f1_weighted": 0.256745 }, { "accuracy": 0.359712, "f1": 0.361921, "f1_weighted": 0.357276 }, { "accuracy": 0.374101, "f1": 0.381264, "f1_weighted": 0.381265 }, { "accuracy": 0.28777, "f1": 0.302377, "f1_weighted": 0.293533 }, { "accuracy": 0.352518, "f1": 0.349482, "f1_weighted": 0.357396 } ], "main_score": 0.327677, "hf_subset": "it-IT", "languages": [ "ita-Latn" ] }, { "accuracy": 0.308626, "f1": 0.290517, "f1_weighted": 0.3119, "scores_per_experiment": [ { "accuracy": 0.316327, "f1": 0.296074, "f1_weighted": 0.326356 }, { "accuracy": 0.360825, "f1": 0.347492, "f1_weighted": 0.376259 }, { "accuracy": 0.298969, "f1": 0.261349, "f1_weighted": 0.288687 }, { "accuracy": 0.309278, "f1": 0.29248, "f1_weighted": 0.298003 }, { "accuracy": 0.257732, "f1": 0.255189, "f1_weighted": 0.270196 } ], "main_score": 0.308626, "hf_subset": "es-ES", "languages": [ "spa-Latn" ] }, { "accuracy": 0.331088, "f1": 0.29433, "f1_weighted": 0.323605, "scores_per_experiment": [ { "accuracy": 0.338843, "f1": 0.329787, "f1_weighted": 0.331826 }, { "accuracy": 0.347107, "f1": 0.316002, "f1_weighted": 0.334672 }, { "accuracy": 0.305785, "f1": 0.229736, "f1_weighted": 0.295666 }, { "accuracy": 0.355372, "f1": 0.327151, "f1_weighted": 0.340364 }, { "accuracy": 0.308333, "f1": 0.268973, "f1_weighted": 0.315498 } ], "main_score": 0.331088, "hf_subset": "pt-PT", "languages": [ "por-Latn" ] }, { "accuracy": 0.312488, "f1": 0.297952, "f1_weighted": 0.31558, "scores_per_experiment": [ { "accuracy": 0.382114, "f1": 0.3801, "f1_weighted": 0.39338 }, { "accuracy": 0.327869, "f1": 0.32167, "f1_weighted": 0.330474 }, { "accuracy": 0.286885, "f1": 0.274385, "f1_weighted": 0.304081 }, { "accuracy": 0.262295, "f1": 0.24121, "f1_weighted": 0.262364 }, { "accuracy": 0.303279, "f1": 0.272395, "f1_weighted": 0.2876 } ], "main_score": 0.312488, "hf_subset": "de-DE", "languages": [ "deu-Latn" ] }, { "accuracy": 0.380705, "f1": 0.366839, "f1_weighted": 0.383396, "scores_per_experiment": [ { "accuracy": 0.343511, "f1": 0.323756, "f1_weighted": 0.353258 }, { "accuracy": 0.40458, "f1": 0.409632, "f1_weighted": 0.402343 }, { "accuracy": 0.366412, "f1": 0.359084, "f1_weighted": 0.35878 }, { "accuracy": 0.427481, "f1": 0.391673, "f1_weighted": 0.445753 }, { "accuracy": 0.361538, "f1": 0.350049, "f1_weighted": 0.356846 } ], "main_score": 0.380705, "hf_subset": "nl-NL", "languages": [ "nld-Latn" ] }, { "accuracy": 0.311734, "f1": 0.289581, "f1_weighted": 0.30812, "scores_per_experiment": [ { "accuracy": 0.342593, "f1": 0.309589, "f1_weighted": 0.338394 }, { "accuracy": 0.240741, "f1": 0.228552, "f1_weighted": 0.241004 }, { "accuracy": 0.314815, "f1": 0.278834, "f1_weighted": 0.314098 }, { "accuracy": 0.324074, "f1": 0.317061, "f1_weighted": 0.329518 }, { "accuracy": 0.336449, "f1": 0.313871, "f1_weighted": 0.317587 } ], "main_score": 0.311734, "hf_subset": "ru-RU", "languages": [ "rus-Cyrl" ] }, { "accuracy": 0.24921, "f1": 0.236958, "f1_weighted": 0.238908, "scores_per_experiment": [ { "accuracy": 0.238938, "f1": 0.229974, "f1_weighted": 0.21622 }, { "accuracy": 0.20354, "f1": 0.190776, "f1_weighted": 0.191754 }, { "accuracy": 0.258929, "f1": 0.251939, "f1_weighted": 0.260663 }, { "accuracy": 0.267857, "f1": 0.245189, "f1_weighted": 0.261221 }, { "accuracy": 0.276786, "f1": 0.266912, "f1_weighted": 0.264683 } ], "main_score": 0.24921, "hf_subset": "pl-PL", "languages": [ "pol-Latn" ] }, { "accuracy": 0.343249, "f1": 0.333816, "f1_weighted": 0.340843, "scores_per_experiment": [ { "accuracy": 0.295652, "f1": 0.262043, "f1_weighted": 0.274013 }, { "accuracy": 0.356522, "f1": 0.344895, "f1_weighted": 0.369382 }, { "accuracy": 0.426087, "f1": 0.430474, "f1_weighted": 0.415614 }, { "accuracy": 0.269565, "f1": 0.276542, "f1_weighted": 0.288008 }, { "accuracy": 0.368421, "f1": 0.355125, "f1_weighted": 0.357197 } ], "main_score": 0.343249, "hf_subset": "cs-CZ", "languages": [ "ces-Latn" ] }, { "accuracy": 0.402136, "f1": 0.389365, "f1_weighted": 0.40184, "scores_per_experiment": [ { "accuracy": 0.352941, "f1": 0.336225, "f1_weighted": 0.3568 }, { "accuracy": 0.386555, "f1": 0.377181, "f1_weighted": 0.383084 }, { "accuracy": 0.347458, "f1": 0.336599, "f1_weighted": 0.352503 }, { "accuracy": 0.449153, "f1": 0.450138, "f1_weighted": 0.454575 }, { "accuracy": 0.474576, "f1": 0.446682, "f1_weighted": 0.462237 } ], "main_score": 0.402136, "hf_subset": "ko-KR", "languages": [ "kor-Hang" ] }, { "accuracy": 0.314475, "f1": 0.29837, "f1_weighted": 0.310599, "scores_per_experiment": [ { "accuracy": 0.405941, "f1": 0.37868, "f1_weighted": 0.395704 }, { "accuracy": 0.356436, "f1": 0.348083, "f1_weighted": 0.360404 }, { "accuracy": 0.26, "f1": 0.224664, "f1_weighted": 0.265127 }, { "accuracy": 0.3, "f1": 0.295384, "f1_weighted": 0.290839 }, { "accuracy": 0.25, "f1": 0.245039, "f1_weighted": 0.240919 } ], "main_score": 0.314475, "hf_subset": "zh-CN", "languages": [ "zho-Hans" ] } ] }, "evaluation_time": 4598.499708414078, "kg_co2_emissions": null }