{ "dataset_revision": "18193f187b92da67168c655c9973a165ed9593dd", "evaluation_time": 144.59256649017334, "kg_co2_emissions": 0.009906069318430912, "mteb_version": "1.18.0", "scores": { "test": [ { "accuracy": 0.32074468085106383, "f1": 0.315045168802926, "f1_weighted": 0.3177010450815628, "hf_subset": "amh", "languages": [ "amh-Ethi" ], "main_score": 0.32074468085106383, "scores_per_experiment": [ { "accuracy": 0.30851063829787234, "f1": 0.29530105630663334, "f1_weighted": 0.30021030074241284 }, { "accuracy": 0.30319148936170215, "f1": 0.2938458511779765, "f1_weighted": 0.2964467783288158 }, { "accuracy": 0.3670212765957447, "f1": 0.3595824899789658, "f1_weighted": 0.3619880473284784 }, { "accuracy": 0.28191489361702127, "f1": 0.2719616301579112, "f1_weighted": 0.2759484613945511 }, { "accuracy": 0.375, "f1": 0.37355324561814235, "f1_weighted": 0.37380237582176956 }, { "accuracy": 0.3484042553191489, "f1": 0.34836604147228317, "f1_weighted": 0.34869463626072567 }, { "accuracy": 0.324468085106383, "f1": 0.3214865588062636, "f1_weighted": 0.32342418158581976 }, { "accuracy": 0.3324468085106383, "f1": 0.32920168153971446, "f1_weighted": 0.3326879627086948 }, { "accuracy": 0.2978723404255319, "f1": 0.28950641087438544, "f1_weighted": 0.295035597736349 }, { "accuracy": 0.26861702127659576, "f1": 0.26764672209698387, "f1_weighted": 0.26877210890801173 } ] }, { "accuracy": 0.8133966244725739, "f1": 0.8054373590724719, "f1_weighted": 0.812832994671197, "hf_subset": "eng", "languages": [ "eng-Latn" ], "main_score": 0.8133966244725739, "scores_per_experiment": [ { "accuracy": 0.8343881856540084, "f1": 0.8294856031987542, "f1_weighted": 0.8336832092603166 }, { "accuracy": 0.8090717299578059, "f1": 0.8021620117887043, "f1_weighted": 0.8076738139268238 }, { "accuracy": 0.819620253164557, "f1": 0.8103263818667403, "f1_weighted": 0.8198446549215389 }, { "accuracy": 0.8132911392405063, "f1": 0.8041352214579979, "f1_weighted": 0.8121117783368051 }, { "accuracy": 0.8227848101265823, "f1": 0.8160064071358347, "f1_weighted": 0.8223558112382418 }, { "accuracy": 0.8069620253164557, "f1": 0.7959390729507761, "f1_weighted": 0.8040087654818168 }, { "accuracy": 0.8037974683544303, "f1": 0.7968589536434679, "f1_weighted": 0.8060262553980407 }, { "accuracy": 0.7710970464135021, "f1": 0.7616913625797408, "f1_weighted": 0.7699737965911323 }, { "accuracy": 0.8143459915611815, "f1": 0.8063696395834808, "f1_weighted": 0.8144813012931457 }, { "accuracy": 0.8386075949367089, "f1": 0.8313989365192227, "f1_weighted": 0.8381705602641093 } ] }, { "accuracy": 0.7670616113744076, "f1": 0.7247451672671642, "f1_weighted": 0.7744176904456347, "hf_subset": "fra", "languages": [ "fra-Latn" ], "main_score": 0.7670616113744076, "scores_per_experiment": [ { "accuracy": 0.8246445497630331, "f1": 0.7839168735834058, "f1_weighted": 0.8243400006397245 }, { "accuracy": 0.7725118483412322, "f1": 0.7236313141015812, "f1_weighted": 0.7812749051942349 }, { "accuracy": 0.7843601895734598, "f1": 0.7394469460974641, "f1_weighted": 0.7918116722282251 }, { "accuracy": 0.7109004739336493, "f1": 0.6701617596490943, "f1_weighted": 0.7212170784295602 }, { "accuracy": 0.7962085308056872, "f1": 0.7555821275043414, "f1_weighted": 0.8028224259530111 }, { "accuracy": 0.7203791469194313, "f1": 0.6813916714114996, "f1_weighted": 0.7353228088060175 }, { "accuracy": 0.7843601895734598, "f1": 0.7394683891301375, "f1_weighted": 0.7946029143582615 }, { "accuracy": 0.7559241706161137, "f1": 0.714779560351759, "f1_weighted": 0.7576649473035224 }, { "accuracy": 0.8104265402843602, "f1": 0.7652862746049951, "f1_weighted": 0.8127538396998697 }, { "accuracy": 0.7109004739336493, "f1": 0.6737867562373643, "f1_weighted": 0.7223663118439199 } ] }, { "accuracy": 0.6496075353218211, "f1": 0.6374566509759267, "f1_weighted": 0.6529811992512473, "hf_subset": "hau", "languages": [ "hau-Latn" ], "main_score": 0.6496075353218211, "scores_per_experiment": [ { "accuracy": 0.6232339089481946, "f1": 0.6143521255752171, "f1_weighted": 0.6289384325167346 }, { "accuracy": 0.6357927786499215, "f1": 0.6196695556839715, "f1_weighted": 0.6350406545619905 }, { "accuracy": 0.6577708006279435, "f1": 0.6434407224144233, "f1_weighted": 0.6582684070368021 }, { "accuracy": 0.6452119309262166, "f1": 0.6389118338031272, "f1_weighted": 0.6524098470230469 }, { "accuracy": 0.6483516483516484, "f1": 0.6359609592759174, "f1_weighted": 0.6530935737462162 }, { "accuracy": 0.6656200941915228, "f1": 0.6514353697347933, "f1_weighted": 0.6689717883473721 }, { "accuracy": 0.6216640502354788, "f1": 0.6150360476599527, "f1_weighted": 0.6267063068679616 }, { "accuracy": 0.7127158555729984, "f1": 0.7010360395069838, "f1_weighted": 0.7145201492169718 }, { "accuracy": 0.6357927786499215, "f1": 0.6195292239584864, "f1_weighted": 0.6376246060245342 }, { "accuracy": 0.6499215070643642, "f1": 0.6351946321463942, "f1_weighted": 0.6542382271708436 } ] }, { "accuracy": 0.6233333333333333, "f1": 0.6180601642700726, "f1_weighted": 0.6255772962275972, "hf_subset": "ibo", "languages": [ "ibo-Latn" ], "main_score": 0.6233333333333333, "scores_per_experiment": [ { "accuracy": 0.6333333333333333, "f1": 0.6265642447120053, "f1_weighted": 0.6418245885127648 }, { "accuracy": 0.6333333333333333, "f1": 0.6196039763974887, "f1_weighted": 0.6381033781554836 }, { "accuracy": 0.6358974358974359, "f1": 0.6296740112756645, "f1_weighted": 0.6392133763739544 }, { "accuracy": 0.6256410256410256, "f1": 0.636622509083191, "f1_weighted": 0.6322008874006426 }, { "accuracy": 0.5769230769230769, "f1": 0.5792010708143894, "f1_weighted": 0.570686328540051 }, { "accuracy": 0.6615384615384615, "f1": 0.6446269583023447, "f1_weighted": 0.6644198261007398 }, { "accuracy": 0.6128205128205129, "f1": 0.6090583799342338, "f1_weighted": 0.6026653973926909 }, { "accuracy": 0.6, "f1": 0.5862356706337156, "f1_weighted": 0.595260859150383 }, { "accuracy": 0.6358974358974359, "f1": 0.6283568582860797, "f1_weighted": 0.6442402883298894 }, { "accuracy": 0.617948717948718, "f1": 0.6206579632616128, "f1_weighted": 0.6271580323193723 } ] }, { "accuracy": 0.7725714285714286, "f1": 0.765922112317743, "f1_weighted": 0.7870414357439683, "hf_subset": "lin", "languages": [ "lin-Latn" ], "main_score": 0.7725714285714286, "scores_per_experiment": [ { "accuracy": 0.7485714285714286, "f1": 0.7405978004529168, "f1_weighted": 0.7612714065156917 }, { "accuracy": 0.7257142857142858, "f1": 0.7384814455241107, "f1_weighted": 0.7420334921211601 }, { "accuracy": 0.76, "f1": 0.7604840000914934, "f1_weighted": 0.7874012541131187 }, { "accuracy": 0.8228571428571428, "f1": 0.8044201126544637, "f1_weighted": 0.8262304293224698 }, { "accuracy": 0.7542857142857143, "f1": 0.7307583474816072, "f1_weighted": 0.774346188050772 }, { "accuracy": 0.7314285714285714, "f1": 0.7375980660463419, "f1_weighted": 0.7521635780749083 }, { "accuracy": 0.7942857142857143, "f1": 0.7750245143102287, "f1_weighted": 0.7992992759523372 }, { "accuracy": 0.7942857142857143, "f1": 0.7846196638003848, "f1_weighted": 0.8126861201939339 }, { "accuracy": 0.76, "f1": 0.7633519388954171, "f1_weighted": 0.7788130490739187 }, { "accuracy": 0.8342857142857143, "f1": 0.823885233920466, "f1_weighted": 0.8361695640213731 } ] }, { "accuracy": 0.5950672645739912, "f1": 0.5670063558877705, "f1_weighted": 0.6099150891816448, "hf_subset": "lug", "languages": [ "lug-Latn" ], "main_score": 0.5950672645739912, "scores_per_experiment": [ { "accuracy": 0.6457399103139013, "f1": 0.5973743706326826, "f1_weighted": 0.6483006219003375 }, { "accuracy": 0.5067264573991032, "f1": 0.4733606364834195, "f1_weighted": 0.5326134742229781 }, { "accuracy": 0.6636771300448431, "f1": 0.6334013801648425, "f1_weighted": 0.6735544499121642 }, { "accuracy": 0.6502242152466368, "f1": 0.6180988974108735, "f1_weighted": 0.6665878250648372 }, { "accuracy": 0.5650224215246636, "f1": 0.5538273511993722, "f1_weighted": 0.5892041245243621 }, { "accuracy": 0.5201793721973094, "f1": 0.5214786719871466, "f1_weighted": 0.5361638673791901 }, { "accuracy": 0.5246636771300448, "f1": 0.5027759424111611, "f1_weighted": 0.5456744088074199 }, { "accuracy": 0.57847533632287, "f1": 0.5248804199954596, "f1_weighted": 0.5849833766532792 }, { "accuracy": 0.6681614349775785, "f1": 0.6396208265180279, "f1_weighted": 0.6810802751497514 }, { "accuracy": 0.6278026905829597, "f1": 0.6052450620747198, "f1_weighted": 0.6409884682021281 } ] }, { "accuracy": 0.5904615384615384, "f1": 0.530274800831309, "f1_weighted": 0.6076830441558899, "hf_subset": "orm", "languages": [ "orm-Ethi" ], "main_score": 0.5904615384615384, "scores_per_experiment": [ { "accuracy": 0.5876923076923077, "f1": 0.5235728065823826, "f1_weighted": 0.6177342262073399 }, { "accuracy": 0.6215384615384615, "f1": 0.5642891541219157, "f1_weighted": 0.6208996680703744 }, { "accuracy": 0.6153846153846154, "f1": 0.5568306931950988, "f1_weighted": 0.6430617626323768 }, { "accuracy": 0.6307692307692307, "f1": 0.5700930569856401, "f1_weighted": 0.6434218288105756 }, { "accuracy": 0.5692307692307692, "f1": 0.5311382113821138, "f1_weighted": 0.5858073417097808 }, { "accuracy": 0.5446153846153846, "f1": 0.46620919948436573, "f1_weighted": 0.5625520025956278 }, { "accuracy": 0.5692307692307692, "f1": 0.5121521128029367, "f1_weighted": 0.5865466115635578 }, { "accuracy": 0.6, "f1": 0.5353576438621402, "f1_weighted": 0.6020403691249492 }, { "accuracy": 0.5230769230769231, "f1": 0.46409075248247744, "f1_weighted": 0.5605806953543027 }, { "accuracy": 0.6430769230769231, "f1": 0.5790143774140175, "f1_weighted": 0.6541859354900139 } ] }, { "accuracy": 0.9347540983606557, "f1": 0.9131301814752588, "f1_weighted": 0.9360320620303918, "hf_subset": "pcm", "languages": [ "pcm-Latn" ], "main_score": 0.9347540983606557, "scores_per_experiment": [ { "accuracy": 0.9377049180327869, "f1": 0.916989528316523, "f1_weighted": 0.9385935610347541 }, { "accuracy": 0.9278688524590164, "f1": 0.9053823862694215, "f1_weighted": 0.9291629872489969 }, { "accuracy": 0.9311475409836065, "f1": 0.9119922510371546, "f1_weighted": 0.933402517933558 }, { "accuracy": 0.9573770491803278, "f1": 0.939672345428486, "f1_weighted": 0.9577617949973566 }, { "accuracy": 0.9442622950819672, "f1": 0.9334472428104912, "f1_weighted": 0.9447272821392315 }, { "accuracy": 0.940983606557377, "f1": 0.9178194998839203, "f1_weighted": 0.9412436757255449 }, { "accuracy": 0.9442622950819672, "f1": 0.921780504076185, "f1_weighted": 0.9457268521140154 }, { "accuracy": 0.9114754098360656, "f1": 0.885643742095939, "f1_weighted": 0.9142087585816748 }, { "accuracy": 0.9344262295081968, "f1": 0.9153932083337283, "f1_weighted": 0.9353755494648623 }, { "accuracy": 0.9180327868852459, "f1": 0.8831811065007399, "f1_weighted": 0.9201176410639255 } ] }, { "accuracy": 0.6381987577639752, "f1": 0.5518804025778669, "f1_weighted": 0.6696603062037989, "hf_subset": "run", "languages": [ "run-Latn" ], "main_score": 0.6381987577639752, "scores_per_experiment": [ { "accuracy": 0.6086956521739131, "f1": 0.5347403501909213, "f1_weighted": 0.6245891380922323 }, { "accuracy": 0.6459627329192547, "f1": 0.5654953026103584, "f1_weighted": 0.6873473490882888 }, { "accuracy": 0.6055900621118012, "f1": 0.49407334613368775, "f1_weighted": 0.6360917710282715 }, { "accuracy": 0.6428571428571429, "f1": 0.5579392563672472, "f1_weighted": 0.681385796121103 }, { "accuracy": 0.6739130434782609, "f1": 0.6004113439940778, "f1_weighted": 0.7017282568892973 }, { "accuracy": 0.6366459627329193, "f1": 0.5472367962030424, "f1_weighted": 0.6743577384160473 }, { "accuracy": 0.6677018633540373, "f1": 0.5710924468361539, "f1_weighted": 0.6817298184283701 }, { "accuracy": 0.6242236024844721, "f1": 0.5383249382532023, "f1_weighted": 0.6620342889557893 }, { "accuracy": 0.6055900621118012, "f1": 0.5242534363536184, "f1_weighted": 0.6501704670954229 }, { "accuracy": 0.6708074534161491, "f1": 0.5852368088363586, "f1_weighted": 0.6971684379231664 } ] }, { "accuracy": 0.7666666666666666, "f1": 0.7733061690600607, "f1_weighted": 0.7663937636270853, "hf_subset": "sna", "languages": [ "sna-Latn" ], "main_score": 0.7666666666666666, "scores_per_experiment": [ { "accuracy": 0.8048780487804879, "f1": 0.811695795077926, "f1_weighted": 0.805917076670646 }, { "accuracy": 0.7615176151761518, "f1": 0.7696184738955822, "f1_weighted": 0.7625161894189223 }, { "accuracy": 0.7696476964769647, "f1": 0.7781077883377214, "f1_weighted": 0.770561693323599 }, { "accuracy": 0.7479674796747967, "f1": 0.754836261472169, "f1_weighted": 0.7486935780744944 }, { "accuracy": 0.7994579945799458, "f1": 0.8017975032077085, "f1_weighted": 0.7946287340366983 }, { "accuracy": 0.7831978319783198, "f1": 0.7896912830943867, "f1_weighted": 0.7856064584726422 }, { "accuracy": 0.7831978319783198, "f1": 0.7910200482586935, "f1_weighted": 0.7852386342955698 }, { "accuracy": 0.7669376693766937, "f1": 0.774219269036129, "f1_weighted": 0.769258555017462 }, { "accuracy": 0.7452574525745257, "f1": 0.7477900109075439, "f1_weighted": 0.7386075768249968 }, { "accuracy": 0.7046070460704607, "f1": 0.7142852573127462, "f1_weighted": 0.7029091401358228 } ] }, { "accuracy": 0.5227891156462585, "f1": 0.48457425976820867, "f1_weighted": 0.5469729179849301, "hf_subset": "som", "languages": [ "som-Latn" ], "main_score": 0.5227891156462585, "scores_per_experiment": [ { "accuracy": 0.4965986394557823, "f1": 0.46006763447809945, "f1_weighted": 0.5095892229222386 }, { "accuracy": 0.5034013605442177, "f1": 0.4610932921029979, "f1_weighted": 0.5375980217822689 }, { "accuracy": 0.5408163265306123, "f1": 0.5071473422067999, "f1_weighted": 0.5657156977016996 }, { "accuracy": 0.5578231292517006, "f1": 0.5067663361318239, "f1_weighted": 0.570909290344836 }, { "accuracy": 0.5306122448979592, "f1": 0.47228693559700863, "f1_weighted": 0.5514195661652803 }, { "accuracy": 0.5, "f1": 0.45872271959410676, "f1_weighted": 0.5287200995714697 }, { "accuracy": 0.5340136054421769, "f1": 0.5034767815335339, "f1_weighted": 0.5687785802464863 }, { "accuracy": 0.5272108843537415, "f1": 0.46899489937535127, "f1_weighted": 0.5411109094371178 }, { "accuracy": 0.5034013605442177, "f1": 0.5003553024812658, "f1_weighted": 0.5356167242466169 }, { "accuracy": 0.5340136054421769, "f1": 0.5068313541810986, "f1_weighted": 0.5602710674312863 } ] }, { "accuracy": 0.5745798319327731, "f1": 0.5291516261500997, "f1_weighted": 0.5866768768593404, "hf_subset": "swa", "languages": [ "swa-Latn" ], "main_score": 0.5745798319327731, "scores_per_experiment": [ { "accuracy": 0.4894957983193277, "f1": 0.4365836587307526, "f1_weighted": 0.5131698463412003 }, { "accuracy": 0.5777310924369747, "f1": 0.5400719602177294, "f1_weighted": 0.5836042851070148 }, { "accuracy": 0.6176470588235294, "f1": 0.5683939449877332, "f1_weighted": 0.6239743838922724 }, { "accuracy": 0.5777310924369747, "f1": 0.521861382336811, "f1_weighted": 0.6004110674194412 }, { "accuracy": 0.592436974789916, "f1": 0.5394816543284977, "f1_weighted": 0.6079190060641968 }, { "accuracy": 0.5651260504201681, "f1": 0.5162893727074737, "f1_weighted": 0.5639579453171799 }, { "accuracy": 0.5378151260504201, "f1": 0.49852617384239, "f1_weighted": 0.5528845297338059 }, { "accuracy": 0.6008403361344538, "f1": 0.5573050269600983, "f1_weighted": 0.6071456500977044 }, { "accuracy": 0.6092436974789915, "f1": 0.568616494727016, "f1_weighted": 0.6276665175102243 }, { "accuracy": 0.5777310924369747, "f1": 0.5443865926624943, "f1_weighted": 0.5860355371103638 } ] }, { "accuracy": 0.2455882352941176, "f1": 0.21841174807153302, "f1_weighted": 0.26213051022189293, "hf_subset": "tir", "languages": [ "tir-Ethi" ], "main_score": 0.2455882352941176, "scores_per_experiment": [ { "accuracy": 0.20588235294117646, "f1": 0.18419363014719006, "f1_weighted": 0.22200307846883566 }, { "accuracy": 0.2647058823529412, "f1": 0.23894223701201855, "f1_weighted": 0.2768956087216862 }, { "accuracy": 0.27941176470588236, "f1": 0.2608036283703345, "f1_weighted": 0.29657828771300837 }, { "accuracy": 0.20955882352941177, "f1": 0.18659192855485832, "f1_weighted": 0.22155499460019537 }, { "accuracy": 0.2610294117647059, "f1": 0.21382246183409845, "f1_weighted": 0.2800700323099241 }, { "accuracy": 0.24632352941176472, "f1": 0.20951208098831842, "f1_weighted": 0.2654738474458854 }, { "accuracy": 0.2647058823529412, "f1": 0.23192341425206905, "f1_weighted": 0.2708470883562504 }, { "accuracy": 0.2426470588235294, "f1": 0.21883206233051433, "f1_weighted": 0.2677728549905749 }, { "accuracy": 0.25735294117647056, "f1": 0.24012994763122641, "f1_weighted": 0.27246076973870953 }, { "accuracy": 0.22426470588235295, "f1": 0.19936608959470206, "f1_weighted": 0.24764853987385968 } ] }, { "accuracy": 0.6562289562289562, "f1": 0.5466851175873437, "f1_weighted": 0.6862345268336224, "hf_subset": "xho", "languages": [ "xho-Latn" ], "main_score": 0.6562289562289562, "scores_per_experiment": [ { "accuracy": 0.6801346801346801, "f1": 0.544490472770607, "f1_weighted": 0.6869431097168334 }, { "accuracy": 0.6599326599326599, "f1": 0.5547537354516073, "f1_weighted": 0.6831915634143795 }, { "accuracy": 0.6195286195286195, "f1": 0.5179961697087931, "f1_weighted": 0.6603619669364392 }, { "accuracy": 0.6565656565656566, "f1": 0.5484337651643628, "f1_weighted": 0.6884382995167468 }, { "accuracy": 0.5959595959595959, "f1": 0.5064851394234617, "f1_weighted": 0.6332131292743349 }, { "accuracy": 0.7037037037037037, "f1": 0.5812038336108657, "f1_weighted": 0.738955264025597 }, { "accuracy": 0.5993265993265994, "f1": 0.49976515536453314, "f1_weighted": 0.6417971733264832 }, { "accuracy": 0.6296296296296297, "f1": 0.5238632736315043, "f1_weighted": 0.665062760806328 }, { "accuracy": 0.7037037037037037, "f1": 0.5803358212238926, "f1_weighted": 0.729830133433881 }, { "accuracy": 0.7138047138047138, "f1": 0.6095238095238095, "f1_weighted": 0.7345518678852012 } ] }, { "accuracy": 0.65669099756691, "f1": 0.6615072385794145, "f1_weighted": 0.6631645561040478, "hf_subset": "yor", "languages": [ "yor-Latn" ], "main_score": 0.65669099756691, "scores_per_experiment": [ { "accuracy": 0.5766423357664233, "f1": 0.596690290003765, "f1_weighted": 0.5965586416938822 }, { "accuracy": 0.6472019464720195, "f1": 0.6545086105253389, "f1_weighted": 0.6570701686109558 }, { "accuracy": 0.6058394160583942, "f1": 0.6042459477321863, "f1_weighted": 0.60522064352507 }, { "accuracy": 0.6788321167883211, "f1": 0.6850950841143075, "f1_weighted": 0.6900463081545787 }, { "accuracy": 0.6253041362530414, "f1": 0.6340112166710957, "f1_weighted": 0.6285562112710541 }, { "accuracy": 0.6277372262773723, "f1": 0.6348808227465215, "f1_weighted": 0.6405554893189851 }, { "accuracy": 0.6909975669099757, "f1": 0.6948843595338193, "f1_weighted": 0.6945493353857042 }, { "accuracy": 0.732360097323601, "f1": 0.7222879222769754, "f1_weighted": 0.7298346442567359 }, { "accuracy": 0.683698296836983, "f1": 0.688446986245744, "f1_weighted": 0.6905402960285804 }, { "accuracy": 0.6982968369829684, "f1": 0.700021145944393, "f1_weighted": 0.6987138227949309 } ] } ] }, "task_name": "MasakhaNEWSClassification" }