{ "dataset_revision": "8f95949846bb9e33c6aaf730ccfdb8fe6bcfb7a9", "evaluation_time": 42.798163175582886, "kg_co2_emissions": 0.0017985532987692498, "mteb_version": "1.18.0", "scores": { "test": [ { "accuracy": 0.6934999999999999, "ap": 0.4299299918244065, "ap_weighted": 0.4299299918244065, "f1": 0.6505502584564751, "f1_weighted": 0.6943029525540237, "hf_subset": "eng", "languages": [ "eng-Latn" ], "main_score": 0.6934999999999999, "scores_per_experiment": [ { "accuracy": 0.686, "ap": 0.4241783439490446, "ap_weighted": 0.4241783439490446, "f1": 0.651654530044242, "f1_weighted": 0.6923440676988343 }, { "accuracy": 0.734, "ap": 0.4697307469600463, "ap_weighted": 0.4697307469600463, "f1": 0.6953956649993587, "f1_weighted": 0.7357350262921636 }, { "accuracy": 0.759, "ap": 0.48183856056330454, "ap_weighted": 0.48183856056330454, "f1": 0.6981459145114159, "f1_weighted": 0.7485639987024032 }, { "accuracy": 0.658, "ap": 0.40344585987261145, "ap_weighted": 0.40344585987261145, "f1": 0.6275337506697858, "f1_weighted": 0.667161179868526 }, { "accuracy": 0.692, "ap": 0.4438332312591866, "ap_weighted": 0.4438332312591866, "f1": 0.6677812533707259, "f1_weighted": 0.7011493042821701 }, { "accuracy": 0.611, "ap": 0.39162665648893175, "ap_weighted": 0.39162665648893175, "f1": 0.5984188646093462, "f1_weighted": 0.6248605728880083 }, { "accuracy": 0.703, "ap": 0.4342416459644732, "ap_weighted": 0.4342416459644732, "f1": 0.6628735208150061, "f1_weighted": 0.7061403331536082 }, { "accuracy": 0.71, "ap": 0.3977713170330799, "ap_weighted": 0.3977713170330799, "f1": 0.6133333333333333, "f1_weighted": 0.6852533333333334 }, { "accuracy": 0.633, "ap": 0.38724316223304606, "ap_weighted": 0.38724316223304606, "f1": 0.6061720459415869, "f1_weighted": 0.6444095896570263 }, { "accuracy": 0.749, "ap": 0.4653903939203412, "ap_weighted": 0.4653903939203412, "f1": 0.6841937062699502, "f1_weighted": 0.7374121196641632 } ] }, { "accuracy": 0.5540999999999999, "ap": 0.3199216019490791, "ap_weighted": 0.3199216019490791, "f1": 0.5171254529973521, "f1_weighted": 0.5653756209003936, "hf_subset": "hin", "languages": [ "hin-Deva" ], "main_score": 0.5540999999999999, "scores_per_experiment": [ { "accuracy": 0.5, "ap": 0.3116110908958591, "ap_weighted": 0.3116110908958591, "f1": 0.4881747903563941, "f1_weighted": 0.5189825733752621 }, { "accuracy": 0.524, "ap": 0.322428621221929, "ap_weighted": 0.322428621221929, "f1": 0.5105116592832081, "f1_weighted": 0.5426886648485671 }, { "accuracy": 0.563, "ap": 0.31275103741467797, "ap_weighted": 0.31275103741467797, "f1": 0.5175274440767937, "f1_weighted": 0.5761825976454897 }, { "accuracy": 0.482, "ap": 0.32583067015006983, "ap_weighted": 0.32583067015006983, "f1": 0.48099520672021034, "f1_weighted": 0.49003834623831743 }, { "accuracy": 0.515, "ap": 0.30836507126257673, "ap_weighted": 0.30836507126257673, "f1": 0.4937237401811112, "f1_weighted": 0.5348232469532086 }, { "accuracy": 0.608, "ap": 0.33964616840113526, "ap_weighted": 0.33964616840113526, "f1": 0.5632798573975044, "f1_weighted": 0.6186210338680926 }, { "accuracy": 0.59, "ap": 0.32613019585740455, "ap_weighted": 0.32613019585740455, "f1": 0.5425720393254165, "f1_weighted": 0.600899593446954 }, { "accuracy": 0.534, "ap": 0.30731727874774234, "ap_weighted": 0.30731727874774234, "f1": 0.5007692011021645, "f1_weighted": 0.5517746133639586 }, { "accuracy": 0.657, "ap": 0.33516331650961284, "ap_weighted": 0.33516331650961284, "f1": 0.5592160471060541, "f1_weighted": 0.6414293068640214 }, { "accuracy": 0.568, "ap": 0.3099725690297832, "ap_weighted": 0.3099725690297832, "f1": 0.5144845444246642, "f1_weighted": 0.5783162324000648 } ] }, { "accuracy": 0.5594, "ap": 0.323269618880416, "ap_weighted": 0.323269618880416, "f1": 0.5264064871528981, "f1_weighted": 0.5741070958778735, "hf_subset": "ita", "languages": [ "ita-Latn" ], "main_score": 0.5594, "scores_per_experiment": [ { "accuracy": 0.507, "ap": 0.32240994854202404, "ap_weighted": 0.32240994854202404, "f1": 0.5001576581828653, "f1_weighted": 0.5235502797799071 }, { "accuracy": 0.572, "ap": 0.33153982300884954, "ap_weighted": 0.33153982300884954, "f1": 0.5439511729405346, "f1_weighted": 0.5891912165848336 }, { "accuracy": 0.559, "ap": 0.2985217391304348, "ap_weighted": 0.2985217391304348, "f1": 0.4954089075774478, "f1_weighted": 0.5670608427014503 }, { "accuracy": 0.555, "ap": 0.3270779401530967, "ap_weighted": 0.3270779401530967, "f1": 0.5321497247042819, "f1_weighted": 0.5735076890404233 }, { "accuracy": 0.585, "ap": 0.3339765807962529, "ap_weighted": 0.3339765807962529, "f1": 0.5515796821294239, "f1_weighted": 0.6005471808408908 }, { "accuracy": 0.548, "ap": 0.322278940027894, "ap_weighted": 0.322278940027894, "f1": 0.5245688512657828, "f1_weighted": 0.5667871372733813 }, { "accuracy": 0.492, "ap": 0.2946124661246612, "ap_weighted": 0.2946124661246612, "f1": 0.4690280286306776, "f1_weighted": 0.5132048966486052 }, { "accuracy": 0.592, "ap": 0.31642105263157894, "ap_weighted": 0.31642105263157894, "f1": 0.5320220775466945, "f1_weighted": 0.5990365160420079 }, { "accuracy": 0.659, "ap": 0.3673733733733734, "ap_weighted": 0.3673733733733734, "f1": 0.6059220326564669, "f1_weighted": 0.6637726782624975 }, { "accuracy": 0.525, "ap": 0.3184843250159949, "ap_weighted": 0.3184843250159949, "f1": 0.5092767358948055, "f1_weighted": 0.5444125216047374 } ] }, { "accuracy": 0.5631999999999999, "ap": 0.3208745483881388, "ap_weighted": 0.3208745483881388, "f1": 0.5260567214819153, "f1_weighted": 0.5757190379123963, "hf_subset": "pol", "languages": [ "pol-Latn" ], "main_score": 0.5631999999999999, "scores_per_experiment": [ { "accuracy": 0.602, "ap": 0.3206575218807646, "ap_weighted": 0.3206575218807646, "f1": 0.5427389705882353, "f1_weighted": 0.6092430147058824 }, { "accuracy": 0.618, "ap": 0.332858748244108, "ap_weighted": 0.332858748244108, "f1": 0.5618442000559738, "f1_weighted": 0.6252155497134783 }, { "accuracy": 0.474, "ap": 0.3168451213216314, "ap_weighted": 0.3168451213216314, "f1": 0.47257383966244726, "f1_weighted": 0.48365400843881856 }, { "accuracy": 0.565, "ap": 0.3280968463592408, "ap_weighted": 0.3280968463592408, "f1": 0.5391124046571897, "f1_weighted": 0.5832414701360731 }, { "accuracy": 0.553, "ap": 0.306658702679641, "ap_weighted": 0.306658702679641, "f1": 0.5110217021039194, "f1_weighted": 0.5689030411824743 }, { "accuracy": 0.597, "ap": 0.3430208285119185, "ap_weighted": 0.3430208285119185, "f1": 0.5660652237348325, "f1_weighted": 0.6128729001810036 }, { "accuracy": 0.536, "ap": 0.3356863125059079, "ap_weighted": 0.3356863125059079, "f1": 0.5275160787093043, "f1_weighted": 0.5530944682722974 }, { "accuracy": 0.508, "ap": 0.30168313522546564, "ap_weighted": 0.30168313522546564, "f1": 0.4866358369922703, "f1_weighted": 0.5289452578507153 }, { "accuracy": 0.575, "ap": 0.3106261690696286, "ap_weighted": 0.3106261690696286, "f1": 0.5227073387396555, "f1_weighted": 0.5865328225740337 }, { "accuracy": 0.604, "ap": 0.3126120980830811, "ap_weighted": 0.3126120980830811, "f1": 0.530351619575324, "f1_weighted": 0.6054878460691854 } ] }, { "accuracy": 0.5338, "ap": 0.3382936640109857, "ap_weighted": 0.3382936640109857, "f1": 0.5086127257867952, "f1_weighted": 0.5443561127856591, "hf_subset": "cmn", "languages": [ "cmn-Hans" ], "main_score": 0.5338, "scores_per_experiment": [ { "accuracy": 0.527, "ap": 0.34362085506985673, "ap_weighted": 0.34362085506985673, "f1": 0.5140898533126642, "f1_weighted": 0.5414942137655366 }, { "accuracy": 0.443, "ap": 0.31289929493713897, "ap_weighted": 0.31289929493713897, "f1": 0.43770940781816103, "f1_weighted": 0.4565810046729679 }, { "accuracy": 0.559, "ap": 0.3336515265453452, "ap_weighted": 0.3336515265453452, "f1": 0.5131112193086179, "f1_weighted": 0.5648295193712178 }, { "accuracy": 0.529, "ap": 0.3444028269402104, "ap_weighted": 0.3444028269402104, "f1": 0.5158181491095063, "f1_weighted": 0.5434600909768446 }, { "accuracy": 0.539, "ap": 0.336698081734779, "ap_weighted": 0.336698081734779, "f1": 0.5125360707446567, "f1_weighted": 0.551834437793364 }, { "accuracy": 0.559, "ap": 0.34115008932085866, "ap_weighted": 0.34115008932085866, "f1": 0.524599547884468, "f1_weighted": 0.5688469695646691 }, { "accuracy": 0.577, "ap": 0.35789045128468944, "ap_weighted": 0.35789045128468944, "f1": 0.549988031596585, "f1_weighted": 0.5881355461581425 }, { "accuracy": 0.53, "ap": 0.34398323818556553, "ap_weighted": 0.34398323818556553, "f1": 0.5160127690248172, "f1_weighted": 0.5444808979507775 }, { "accuracy": 0.481, "ap": 0.32788422892092617, "ap_weighted": 0.32788422892092617, "f1": 0.4742871729108993, "f1_weighted": 0.4948414930244289 }, { "accuracy": 0.594, "ap": 0.34075604717048674, "ap_weighted": 0.34075604717048674, "f1": 0.5279750361575772, "f1_weighted": 0.5890569545786422 } ] }, { "accuracy": 0.5409, "ap": 0.3165741071275047, "ap_weighted": 0.3165741071275047, "f1": 0.5131497077487225, "f1_weighted": 0.557702517022247, "hf_subset": "fra", "languages": [ "fra-Latn" ], "main_score": 0.5409, "scores_per_experiment": [ { "accuracy": 0.556, "ap": 0.3236903640008975, "ap_weighted": 0.3236903640008975, "f1": 0.5288624787775892, "f1_weighted": 0.5738655348047539 }, { "accuracy": 0.586, "ap": 0.3272147693342865, "ap_weighted": 0.3272147693342865, "f1": 0.5438438756082081, "f1_weighted": 0.5990351174106198 }, { "accuracy": 0.476, "ap": 0.30689496810292793, "ap_weighted": 0.30689496810292793, "f1": 0.4695819414920539, "f1_weighted": 0.4928036440935318 }, { "accuracy": 0.55, "ap": 0.3194085288280495, "ap_weighted": 0.3194085288280495, "f1": 0.5220068066230736, "f1_weighted": 0.5680451990363657 }, { "accuracy": 0.546, "ap": 0.321297429620563, "ap_weighted": 0.321297429620563, "f1": 0.5220167104642545, "f1_weighted": 0.5646298766929452 }, { "accuracy": 0.552, "ap": 0.31211278320089014, "ap_weighted": 0.31211278320089014, "f1": 0.5144768271060651, "f1_weighted": 0.5681970530477416 }, { "accuracy": 0.496, "ap": 0.304031007751938, "ap_weighted": 0.304031007751938, "f1": 0.48026448762740737, "f1_weighted": 0.5162572113302342 }, { "accuracy": 0.577, "ap": 0.3240000818290428, "ap_weighted": 0.3240000818290428, "f1": 0.5372755704473331, "f1_weighted": 0.5912357170751878 }, { "accuracy": 0.554, "ap": 0.3078568918048151, "ap_weighted": 0.3078568918048151, "f1": 0.5092407977147787, "f1_weighted": 0.5682280907926531 }, { "accuracy": 0.516, "ap": 0.31923424680163726, "ap_weighted": 0.31923424680163726, "f1": 0.5039275816264616, "f1_weighted": 0.5347277259384379 } ] }, { "accuracy": 0.5401, "ap": 0.32879121420811674, "ap_weighted": 0.32879121420811674, "f1": 0.5208653163630474, "f1_weighted": 0.5553259760410051, "hf_subset": "deu", "languages": [ "deu-Latn" ], "main_score": 0.5401, "scores_per_experiment": [ { "accuracy": 0.529, "ap": 0.3521814872594904, "ap_weighted": 0.3521814872594904, "f1": 0.527354721786539, "f1_weighted": 0.5385091503523767 }, { "accuracy": 0.565, "ap": 0.3239387755102041, "ap_weighted": 0.3239387755102041, "f1": 0.5337215771144119, "f1_weighted": 0.5820280217253588 }, { "accuracy": 0.579, "ap": 0.34453222453222454, "ap_weighted": 0.34453222453222454, "f1": 0.5577912249393144, "f1_weighted": 0.5965287136346306 }, { "accuracy": 0.528, "ap": 0.2935892255892256, "ap_weighted": 0.2935892255892256, "f1": 0.4799379451378605, "f1_weighted": 0.5431774910090967 }, { "accuracy": 0.474, "ap": 0.3002002355712603, "ap_weighted": 0.3002002355712603, "f1": 0.4643824512954613, "f1_weighted": 0.49309155190602455 }, { "accuracy": 0.487, "ap": 0.3198528624933119, "ap_weighted": 0.3198528624933119, "f1": 0.4839402819315723, "f1_weighted": 0.4998349212480798 }, { "accuracy": 0.601, "ap": 0.3385344970906068, "ap_weighted": 0.3385344970906068, "f1": 0.5618268853798434, "f1_weighted": 0.6142323898884141 }, { "accuracy": 0.565, "ap": 0.33068807994289795, "ap_weighted": 0.33068807994289795, "f1": 0.5400286133924634, "f1_weighted": 0.5828979466242858 }, { "accuracy": 0.546, "ap": 0.34067388688327316, "ap_weighted": 0.34067388688327316, "f1": 0.5361117582386143, "f1_weighted": 0.5632028315574792 }, { "accuracy": 0.527, "ap": 0.3437208672086721, "ap_weighted": 0.3437208672086721, "f1": 0.523557704414394, "f1_weighted": 0.5397567424643046 } ] }, { "accuracy": 0.5650999999999999, "ap": 0.3156502391744335, "ap_weighted": 0.3156502391744335, "f1": 0.5240904976445389, "f1_weighted": 0.5789296493140856, "hf_subset": "spa", "languages": [ "spa-Latn" ], "main_score": 0.5650999999999999, "scores_per_experiment": [ { "accuracy": 0.562, "ap": 0.31093354223789005, "ap_weighted": 0.31093354223789005, "f1": 0.5217818539141827, "f1_weighted": 0.5783646249590566 }, { "accuracy": 0.612, "ap": 0.31702044352044356, "ap_weighted": 0.31702044352044356, "f1": 0.541553236539625, "f1_weighted": 0.6148753781004234 }, { "accuracy": 0.522, "ap": 0.3235223787617405, "ap_weighted": 0.3235223787617405, "f1": 0.5124439004487964, "f1_weighted": 0.5402931048551612 }, { "accuracy": 0.525, "ap": 0.30026190896779126, "ap_weighted": 0.30026190896779126, "f1": 0.4946674113673236, "f1_weighted": 0.545180456927046 }, { "accuracy": 0.568, "ap": 0.31466581799190496, "ap_weighted": 0.31466581799190496, "f1": 0.5283327874222077, "f1_weighted": 0.5841404520144121 }, { "accuracy": 0.583, "ap": 0.3137939279762336, "ap_weighted": 0.3137939279762336, "f1": 0.5316916711869089, "f1_weighted": 0.5949357743764592 }, { "accuracy": 0.511, "ap": 0.30549354383396937, "ap_weighted": 0.30549354383396937, "f1": 0.49328052673955547, "f1_weighted": 0.5319411956714344 }, { "accuracy": 0.615, "ap": 0.35729852301280873, "ap_weighted": 0.35729852301280873, "f1": 0.5863910849552711, "f1_weighted": 0.6307729759752306 }, { "accuracy": 0.582, "ap": 0.2945917252869308, "ap_weighted": 0.2945917252869308, "f1": 0.4965408259621129, "f1_weighted": 0.5811702992811857 }, { "accuracy": 0.571, "ap": 0.31892058015462266, "ap_weighted": 0.31892058015462266, "f1": 0.5342216779094044, "f1_weighted": 0.5876222309804471 } ] }, { "accuracy": 0.5347999999999999, "ap": 0.31876659162465765, "ap_weighted": 0.31876659162465765, "f1": 0.5116589300616652, "f1_weighted": 0.551502931862943, "hf_subset": "ara", "languages": [ "ara-Arab" ], "main_score": 0.5347999999999999, "scores_per_experiment": [ { "accuracy": 0.549, "ap": 0.3325791632059801, "ap_weighted": 0.3325791632059801, "f1": 0.5326575001217577, "f1_weighted": 0.5674399330176958 }, { "accuracy": 0.58, "ap": 0.3304597519694007, "ap_weighted": 0.3304597519694007, "f1": 0.5453679488733785, "f1_weighted": 0.5953083704255356 }, { "accuracy": 0.537, "ap": 0.3272091877649215, "ap_weighted": 0.3272091877649215, "f1": 0.5220255380825894, "f1_weighted": 0.5556969270268234 }, { "accuracy": 0.532, "ap": 0.32377554026565314, "ap_weighted": 0.32377554026565314, "f1": 0.5166867014487007, "f1_weighted": 0.5509265487712689 }, { "accuracy": 0.476, "ap": 0.3031553723432109, "ap_weighted": 0.3031553723432109, "f1": 0.4672717808575705, "f1_weighted": 0.4944110872535624 }, { "accuracy": 0.561, "ap": 0.31393222918678293, "ap_weighted": 0.31393222918678293, "f1": 0.519772991551724, "f1_weighted": 0.5757741839149112 }, { "accuracy": 0.503, "ap": 0.3101292705905687, "ap_weighted": 0.3101292705905687, "f1": 0.4894347930156323, "f1_weighted": 0.5225572002535361 }, { "accuracy": 0.542, "ap": 0.31597141278531604, "ap_weighted": 0.31597141278531604, "f1": 0.5149866991985632, "f1_weighted": 0.5605430285162405 }, { "accuracy": 0.48, "ap": 0.3116627879337317, "ap_weighted": 0.3116627879337317, "f1": 0.4749574917507744, "f1_weighted": 0.4954362497425272 }, { "accuracy": 0.588, "ap": 0.3187912002010106, "ap_weighted": 0.3187912002010106, "f1": 0.5334278557159617, "f1_weighted": 0.5969357897073279 } ] }, { "accuracy": 0.5655, "ap": 0.3288508880435578, "ap_weighted": 0.3288508880435578, "f1": 0.534689189288164, "f1_weighted": 0.5803088425253049, "hf_subset": "por", "languages": [ "por-Latn" ], "main_score": 0.5655, "scores_per_experiment": [ { "accuracy": 0.57, "ap": 0.34076427124952113, "ap_weighted": 0.34076427124952113, "f1": 0.5493626100917623, "f1_weighted": 0.587551425062146 }, { "accuracy": 0.541, "ap": 0.32356785728678317, "ap_weighted": 0.32356785728678317, "f1": 0.5208643231817114, "f1_weighted": 0.5597605574258201 }, { "accuracy": 0.516, "ap": 0.31947412312975226, "ap_weighted": 0.31947412312975226, "f1": 0.5036080719070886, "f1_weighted": 0.5346663220640059 }, { "accuracy": 0.555, "ap": 0.3358121434660022, "ap_weighted": 0.3358121434660022, "f1": 0.5377828742842127, "f1_weighted": 0.5731092047684189 }, { "accuracy": 0.581, "ap": 0.3179765956295, "ap_weighted": 0.3179765956295, "f1": 0.5294455880750956, "f1_weighted": 0.5911239781725643 }, { "accuracy": 0.599, "ap": 0.34543891785261377, "ap_weighted": 0.34543891785261377, "f1": 0.5661933739012847, "f1_weighted": 0.6134349154834348 }, { "accuracy": 0.583, "ap": 0.31631582874731834, "ap_weighted": 0.31631582874731834, "f1": 0.5273989602777126, "f1_weighted": 0.5915914143011114 }, { "accuracy": 0.572, "ap": 0.33101866890413834, "ap_weighted": 0.33101866890413834, "f1": 0.5419834257927539, "f1_weighted": 0.5884153140195877 }, { "accuracy": 0.614, "ap": 0.33523266852964817, "ap_weighted": 0.33523266852964817, "f1": 0.5594090145372197, "f1_weighted": 0.6208238731828476 }, { "accuracy": 0.524, "ap": 0.32290780564030064, "ap_weighted": 0.32290780564030064, "f1": 0.5108436508327989, "f1_weighted": 0.5426114207731136 } ] }, { "accuracy": 0.5526000000000001, "ap": 0.3211878792405975, "ap_weighted": 0.3211878792405975, "f1": 0.5231911038367375, "f1_weighted": 0.5672965060662272, "hf_subset": "nld", "languages": [ "nld-Latn" ], "main_score": 0.5526000000000001, "scores_per_experiment": [ { "accuracy": 0.617, "ap": 0.3283790222835013, "ap_weighted": 0.3283790222835013, "f1": 0.5558694232912279, "f1_weighted": 0.6224374906506455 }, { "accuracy": 0.479, "ap": 0.3153169678469246, "ap_weighted": 0.3153169678469246, "f1": 0.4763603324358089, "f1_weighted": 0.49138041266022014 }, { "accuracy": 0.609, "ap": 0.3343319358727531, "ap_weighted": 0.3343319358727531, "f1": 0.5615407301748567, "f1_weighted": 0.6198189824829354 }, { "accuracy": 0.556, "ap": 0.32597343400447426, "ap_weighted": 0.32597343400447426, "f1": 0.5329835618628487, "f1_weighted": 0.5748693321664935 }, { "accuracy": 0.533, "ap": 0.3151166022642533, "ap_weighted": 0.3151166022642533, "f1": 0.5120937030834215, "f1_weighted": 0.5528963308819611 }, { "accuracy": 0.496, "ap": 0.3190153340148451, "ap_weighted": 0.3190153340148451, "f1": 0.49111264585075065, "f1_weighted": 0.5112605139762277 }, { "accuracy": 0.55, "ap": 0.31024956500124284, "ap_weighted": 0.31024956500124284, "f1": 0.5146154675870995, "f1_weighted": 0.5675612123826987 }, { "accuracy": 0.528, "ap": 0.31716310285625093, "ap_weighted": 0.31716310285625093, "f1": 0.5110855144581957, "f1_weighted": 0.5478245045597491 }, { "accuracy": 0.543, "ap": 0.3148993717891845, "ap_weighted": 0.3148993717891845, "f1": 0.5167656926904731, "f1_weighted": 0.5622535045061334 }, { "accuracy": 0.615, "ap": 0.33143345647254513, "ap_weighted": 0.33143345647254513, "f1": 0.5594839669326925, "f1_weighted": 0.6226627763952058 } ] } ] }, "task_name": "MultiHateClassification" }