{ "dataset_revision": "8f95949846bb9e33c6aaf730ccfdb8fe6bcfb7a9", "evaluation_time": 198.53276324272156, "kg_co2_emissions": 0.016675235359517888, "mteb_version": "1.18.0", "scores": { "test": [ { "accuracy": 0.7253000000000001, "ap": 0.4519149056395788, "ap_weighted": 0.4519149056395788, "f1": 0.6821461361440649, "f1_weighted": 0.7275405145028998, "hf_subset": "pol", "languages": [ "pol-Latn" ], "main_score": 0.7253000000000001, "scores_per_experiment": [ { "accuracy": 0.806, "ap": 0.561381271971876, "ap_weighted": 0.561381271971876, "f1": 0.7759929056722291, "f1_weighted": 0.8091154906679107 }, { "accuracy": 0.751, "ap": 0.4403278658921961, "ap_weighted": 0.4403278658921961, "f1": 0.6768579693443331, "f1_weighted": 0.7393911225486952 }, { "accuracy": 0.64, "ap": 0.40937918187153904, "ap_weighted": 0.40937918187153904, "f1": 0.6284921735684565, "f1_weighted": 0.6549078660590449 }, { "accuracy": 0.652, "ap": 0.3643676784624771, "ap_weighted": 0.3643676784624771, "f1": 0.6034188034188034, "f1_weighted": 0.6594953846153846 }, { "accuracy": 0.742, "ap": 0.4551612459129238, "ap_weighted": 0.4551612459129238, "f1": 0.695718834768251, "f1_weighted": 0.7436613751621654 }, { "accuracy": 0.725, "ap": 0.4553862847680953, "ap_weighted": 0.4553862847680953, "f1": 0.6934085058514381, "f1_weighted": 0.7331685171786002 }, { "accuracy": 0.731, "ap": 0.44315095444186575, "ap_weighted": 0.44315095444186575, "f1": 0.6853208615891413, "f1_weighted": 0.7337575332898943 }, { "accuracy": 0.774, "ap": 0.466005132254244, "ap_weighted": 0.466005132254244, "f1": 0.6847872174242932, "f1_weighted": 0.7525352703577247 }, { "accuracy": 0.785, "ap": 0.5231276622270243, "ap_weighted": 0.5231276622270243, "f1": 0.7489370696168313, "f1_weighted": 0.7873788212653806 }, { "accuracy": 0.647, "ap": 0.40086177859354644, "ap_weighted": 0.40086177859354644, "f1": 0.6285270201868729, "f1_weighted": 0.6619937638841972 } ] }, { "accuracy": 0.7455, "ap": 0.4889072164752504, "ap_weighted": 0.4889072164752504, "f1": 0.7157568672858378, "f1_weighted": 0.7519238037197954, "hf_subset": "ita", "languages": [ "ita-Latn" ], "main_score": 0.7455, "scores_per_experiment": [ { "accuracy": 0.763, "ap": 0.4968660801564027, "ap_weighted": 0.4968660801564027, "f1": 0.7279361373130422, "f1_weighted": 0.7670045079948893 }, { "accuracy": 0.726, "ap": 0.4532142857142857, "ap_weighted": 0.4532142857142857, "f1": 0.69112978861554, "f1_weighted": 0.7326419450256113 }, { "accuracy": 0.648, "ap": 0.38469841269841265, "ap_weighted": 0.38469841269841265, "f1": 0.6180555555555556, "f1_weighted": 0.6608333333333333 }, { "accuracy": 0.749, "ap": 0.5210104384133613, "ap_weighted": 0.5210104384133613, "f1": 0.73611141775455, "f1_weighted": 0.7594391684250477 }, { "accuracy": 0.822, "ap": 0.581400462962963, "ap_weighted": 0.581400462962963, "f1": 0.7856082943092251, "f1_weighted": 0.8209400474070647 }, { "accuracy": 0.767, "ap": 0.4986065239551478, "ap_weighted": 0.4986065239551478, "f1": 0.7293438854369586, "f1_weighted": 0.7697257777297645 }, { "accuracy": 0.74, "ap": 0.46907865168539326, "ap_weighted": 0.46907865168539326, "f1": 0.7051030778164924, "f1_weighted": 0.7456808943089431 }, { "accuracy": 0.774, "ap": 0.509579268292683, "ap_weighted": 0.509579268292683, "f1": 0.7377021782325305, "f1_weighted": 0.7767320941115299 }, { "accuracy": 0.729, "ap": 0.4929816772374912, "ap_weighted": 0.4929816772374912, "f1": 0.7142769784210588, "f1_weighted": 0.7402206287804266 }, { "accuracy": 0.737, "ap": 0.48163636363636364, "ap_weighted": 0.48163636363636364, "f1": 0.7123013594034246, "f1_weighted": 0.7460196400813432 } ] }, { "accuracy": 0.758, "ap": 0.5415052931260519, "ap_weighted": 0.5415052931260519, "f1": 0.7397717460081179, "f1_weighted": 0.7627566382555203, "hf_subset": "cmn", "languages": [ "cmn-Hans" ], "main_score": 0.758, "scores_per_experiment": [ { "accuracy": 0.738, "ap": 0.512926322212295, "ap_weighted": 0.512926322212295, "f1": 0.7202337223010029, "f1_weighted": 0.7446271035861339 }, { "accuracy": 0.746, "ap": 0.5373342973702859, "ap_weighted": 0.5373342973702859, "f1": 0.7360666026576224, "f1_weighted": 0.753782868020832 }, { "accuracy": 0.809, "ap": 0.6005386755127076, "ap_weighted": 0.6005386755127076, "f1": 0.787361715534507, "f1_weighted": 0.8108314535441012 }, { "accuracy": 0.8, "ap": 0.5765738580059457, "ap_weighted": 0.5765738580059457, "f1": 0.7649834783385272, "f1_weighted": 0.7963713449055468 }, { "accuracy": 0.743, "ap": 0.5263951749915052, "ap_weighted": 0.5263951749915052, "f1": 0.7295508333938424, "f1_weighted": 0.7504181501908403 }, { "accuracy": 0.727, "ap": 0.4939525993883792, "ap_weighted": 0.4939525993883792, "f1": 0.7050150680032113, "f1_weighted": 0.732878754709764 }, { "accuracy": 0.817, "ap": 0.6117494153624752, "ap_weighted": 0.6117494153624752, "f1": 0.7941764301645127, "f1_weighted": 0.8178910102338177 }, { "accuracy": 0.75, "ap": 0.5193856404851446, "ap_weighted": 0.5193856404851446, "f1": 0.7270444371656295, "f1_weighted": 0.7544327983404302 }, { "accuracy": 0.732, "ap": 0.5179561606835609, "ap_weighted": 0.5179561606835609, "f1": 0.7205993352766276, "f1_weighted": 0.7401272065354734 }, { "accuracy": 0.718, "ap": 0.518240787248219, "ap_weighted": 0.518240787248219, "f1": 0.7126858372456965, "f1_weighted": 0.7262056924882629 } ] }, { "accuracy": 0.8848, "ap": 0.7266796442376191, "ap_weighted": 0.7266796442376191, "f1": 0.8675998697973905, "f1_weighted": 0.8852555256904345, "hf_subset": "eng", "languages": [ "eng-Latn" ], "main_score": 0.8848, "scores_per_experiment": [ { "accuracy": 0.869, "ap": 0.6935144000440536, "ap_weighted": 0.6935144000440536, "f1": 0.8519906132672903, "f1_weighted": 0.8706557810093789 }, { "accuracy": 0.889, "ap": 0.7343403864838605, "ap_weighted": 0.7343403864838605, "f1": 0.8689676420904264, "f1_weighted": 0.8880265606130897 }, { "accuracy": 0.875, "ap": 0.7064882904264851, "ap_weighted": 0.7064882904264851, "f1": 0.8477598161425748, "f1_weighted": 0.8717157225136438 }, { "accuracy": 0.908, "ap": 0.7756165772712036, "ap_weighted": 0.7756165772712036, "f1": 0.892281248975501, "f1_weighted": 0.9075885143710863 }, { "accuracy": 0.898, "ap": 0.7515099139947462, "ap_weighted": 0.7515099139947462, "f1": 0.8853644063224901, "f1_weighted": 0.8995223606840372 }, { "accuracy": 0.901, "ap": 0.7587085637841184, "ap_weighted": 0.7587085637841184, "f1": 0.8855902675454259, "f1_weighted": 0.9012099418590542 }, { "accuracy": 0.845, "ap": 0.6552995575436379, "ap_weighted": 0.6552995575436379, "f1": 0.8304437669487863, "f1_weighted": 0.8489247181264364 }, { "accuracy": 0.885, "ap": 0.7250886498903624, "ap_weighted": 0.7250886498903624, "f1": 0.8654717438020493, "f1_weighted": 0.8845387026094973 }, { "accuracy": 0.889, "ap": 0.7329166553733568, "ap_weighted": 0.7329166553733568, "f1": 0.872786805585003, "f1_weighted": 0.8896812266560924 }, { "accuracy": 0.889, "ap": 0.7333134475643671, "ap_weighted": 0.7333134475643671, "f1": 0.8753423872943571, "f1_weighted": 0.8906917284620284 } ] }, { "accuracy": 0.784, "ap": 0.5339023689134468, "ap_weighted": 0.5339023689134468, "f1": 0.7460078263411711, "f1_weighted": 0.7839150182923657, "hf_subset": "deu", "languages": [ "deu-Latn" ], "main_score": 0.784, "scores_per_experiment": [ { "accuracy": 0.777, "ap": 0.5110862250262881, "ap_weighted": 0.5110862250262881, "f1": 0.7386650353739728, "f1_weighted": 0.77870155195468 }, { "accuracy": 0.76, "ap": 0.4933875968992248, "ap_weighted": 0.4933875968992248, "f1": 0.7251690210520529, "f1_weighted": 0.7643049524542405 }, { "accuracy": 0.744, "ap": 0.5114326241134751, "ap_weighted": 0.5114326241134751, "f1": 0.7297011931158273, "f1_weighted": 0.7545686833491712 }, { "accuracy": 0.784, "ap": 0.5275, "ap_weighted": 0.5275, "f1": 0.7510097936147845, "f1_weighted": 0.7872627676644718 }, { "accuracy": 0.801, "ap": 0.5264273504273504, "ap_weighted": 0.5264273504273504, "f1": 0.7328769421792678, "f1_weighted": 0.7868357998590557 }, { "accuracy": 0.805, "ap": 0.5383333333333333, "ap_weighted": 0.5383333333333333, "f1": 0.7481840193704601, "f1_weighted": 0.7960290556900727 }, { "accuracy": 0.76, "ap": 0.5203100775193799, "ap_weighted": 0.5203100775193799, "f1": 0.7411282493797864, "f1_weighted": 0.7690863984467694 }, { "accuracy": 0.85, "ap": 0.6382355072463768, "ap_weighted": 0.6382355072463768, "f1": 0.7955690267995377, "f1_weighted": 0.8377635796681131 }, { "accuracy": 0.743, "ap": 0.5084802867383513, "ap_weighted": 0.5084802867383513, "f1": 0.7279775607948982, "f1_weighted": 0.7535476700801778 }, { "accuracy": 0.816, "ap": 0.5638306878306878, "ap_weighted": 0.5638306878306878, "f1": 0.7697974217311234, "f1_weighted": 0.811049723756906 } ] }, { "accuracy": 0.7112, "ap": 0.46303116791235455, "ap_weighted": 0.46303116791235455, "f1": 0.6852384668019443, "f1_weighted": 0.7188942613917929, "hf_subset": "fra", "languages": [ "fra-Latn" ], "main_score": 0.7112, "scores_per_experiment": [ { "accuracy": 0.613, "ap": 0.39660671120160995, "ap_weighted": 0.39660671120160995, "f1": 0.6051499868893471, "f1_weighted": 0.6273081799250907 }, { "accuracy": 0.735, "ap": 0.4628613097584323, "ap_weighted": 0.4628613097584323, "f1": 0.6991969125117058, "f1_weighted": 0.7405001844547234 }, { "accuracy": 0.832, "ap": 0.6040793504599629, "ap_weighted": 0.6040793504599629, "f1": 0.8007552325471072, "f1_weighted": 0.8321578018558228 }, { "accuracy": 0.742, "ap": 0.4561580649759328, "ap_weighted": 0.4561580649759328, "f1": 0.6940169642687717, "f1_weighted": 0.7422423385642991 }, { "accuracy": 0.754, "ap": 0.4861894650681304, "ap_weighted": 0.4861894650681304, "f1": 0.71875557344138, "f1_weighted": 0.7583806631880771 }, { "accuracy": 0.631, "ap": 0.41083660525520993, "ap_weighted": 0.41083660525520993, "f1": 0.6226218271853418, "f1_weighted": 0.6450011075895966 }, { "accuracy": 0.665, "ap": 0.43918682087277994, "ap_weighted": 0.43918682087277994, "f1": 0.6553891479470966, "f1_weighted": 0.6782940528396448 }, { "accuracy": 0.681, "ap": 0.4143139370415446, "ap_weighted": 0.4143139370415446, "f1": 0.6505928420822038, "f1_weighted": 0.6916167364933321 }, { "accuracy": 0.79, "ap": 0.5310659076003398, "ap_weighted": 0.5310659076003398, "f1": 0.7523292841136926, "f1_weighted": 0.7907727326335653 }, { "accuracy": 0.669, "ap": 0.42901350688960294, "ap_weighted": 0.42901350688960294, "f1": 0.6535768970327972, "f1_weighted": 0.682668816373777 } ] }, { "accuracy": 0.5769, "ap": 0.3484477593578816, "ap_weighted": 0.3484477593578816, "f1": 0.5532729582772637, "f1_weighted": 0.5892296252345928, "hf_subset": "hin", "languages": [ "hin-Deva" ], "main_score": 0.5769, "scores_per_experiment": [ { "accuracy": 0.525, "ap": 0.3232847433619522, "ap_weighted": 0.3232847433619522, "f1": 0.5117062013312431, "f1_weighted": 0.5436113181362597 }, { "accuracy": 0.519, "ap": 0.3450747249309068, "ap_weighted": 0.3450747249309068, "f1": 0.5170833036321159, "f1_weighted": 0.5291311093731019 }, { "accuracy": 0.552, "ap": 0.35438528936528524, "ap_weighted": 0.35438528936528524, "f1": 0.5458894889634928, "f1_weighted": 0.5667495093984658 }, { "accuracy": 0.508, "ap": 0.3338756675923948, "ap_weighted": 0.3338756675923948, "f1": 0.5049883491160219, "f1_weighted": 0.5202782689885261 }, { "accuracy": 0.613, "ap": 0.3696681225803707, "ap_weighted": 0.3696681225803707, "f1": 0.5911852353431456, "f1_weighted": 0.628581974754896 }, { "accuracy": 0.563, "ap": 0.341411513999082, "ap_weighted": 0.341411513999082, "f1": 0.5460923956453954, "f1_weighted": 0.5807836460310091 }, { "accuracy": 0.601, "ap": 0.35213211995615024, "ap_weighted": 0.35213211995615024, "f1": 0.5727827262886382, "f1_weighted": 0.6162614826687909 }, { "accuracy": 0.605, "ap": 0.3530054601348974, "ap_weighted": 0.3530054601348974, "f1": 0.575166034793364, "f1_weighted": 0.6197481111021484 }, { "accuracy": 0.631, "ap": 0.37263770938839114, "ap_weighted": 0.37263770938839114, "f1": 0.6012841029054394, "f1_weighted": 0.6443884811085383 }, { "accuracy": 0.652, "ap": 0.3390022422693852, "ap_weighted": 0.3390022422693852, "f1": 0.5665517447537814, "f1_weighted": 0.6427623507841925 } ] }, { "accuracy": 0.7316, "ap": 0.46191974311572465, "ap_weighted": 0.46191974311572465, "f1": 0.6946447309716207, "f1_weighted": 0.7374201257878317, "hf_subset": "spa", "languages": [ "spa-Latn" ], "main_score": 0.7316, "scores_per_experiment": [ { "accuracy": 0.624, "ap": 0.3478820193250245, "ap_weighted": 0.3478820193250245, "f1": 0.5816998117649153, "f1_weighted": 0.6359717513872881 }, { "accuracy": 0.726, "ap": 0.4644429795649308, "ap_weighted": 0.4644429795649308, "f1": 0.7000757472930195, "f1_weighted": 0.7360522612537272 }, { "accuracy": 0.687, "ap": 0.4241838341838342, "ap_weighted": 0.4241838341838342, "f1": 0.6613928329952671, "f1_weighted": 0.6993845571331981 }, { "accuracy": 0.743, "ap": 0.4723449443994403, "ap_weighted": 0.4723449443994403, "f1": 0.7100733164792297, "f1_weighted": 0.7499370757566015 }, { "accuracy": 0.727, "ap": 0.4326369052758829, "ap_weighted": 0.4326369052758829, "f1": 0.677731227844022, "f1_weighted": 0.7291421205285208 }, { "accuracy": 0.698, "ap": 0.4219571488587032, "ap_weighted": 0.4219571488587032, "f1": 0.664024848811182, "f1_weighted": 0.7076156088270239 }, { "accuracy": 0.756, "ap": 0.49479374110953056, "ap_weighted": 0.49479374110953056, "f1": 0.7273816121131947, "f1_weighted": 0.7634195820447273 }, { "accuracy": 0.797, "ap": 0.5489302503636782, "ap_weighted": 0.5489302503636782, "f1": 0.7684760133166211, "f1_weighted": 0.8016321004015744 }, { "accuracy": 0.738, "ap": 0.45176521306582657, "ap_weighted": 0.45176521306582657, "f1": 0.6943237554776716, "f1_weighted": 0.7414663686128831 }, { "accuracy": 0.82, "ap": 0.560260395010395, "ap_weighted": 0.560260395010395, "f1": 0.7612681436210849, "f1_weighted": 0.8095798319327732 } ] }, { "accuracy": 0.7439, "ap": 0.4955654606380365, "ap_weighted": 0.4955654606380365, "f1": 0.7177671366655252, "f1_weighted": 0.7502744965977499, "hf_subset": "por", "languages": [ "por-Latn" ], "main_score": 0.7439, "scores_per_experiment": [ { "accuracy": 0.717, "ap": 0.4542002682538352, "ap_weighted": 0.4542002682538352, "f1": 0.6883915387842425, "f1_weighted": 0.7257808148285989 }, { "accuracy": 0.73, "ap": 0.4709353133988896, "ap_weighted": 0.4709353133988896, "f1": 0.7029036219349557, "f1_weighted": 0.7384339719805105 }, { "accuracy": 0.674, "ap": 0.4652194564969171, "ap_weighted": 0.4652194564969171, "f1": 0.6693966793634567, "f1_weighted": 0.6848451113301611 }, { "accuracy": 0.778, "ap": 0.5459390228664251, "ap_weighted": 0.5459390228664251, "f1": 0.7599802794607989, "f1_weighted": 0.7860233792181843 }, { "accuracy": 0.745, "ap": 0.5012095895274703, "ap_weighted": 0.5012095895274703, "f1": 0.7251085821100666, "f1_weighted": 0.7543911155093738 }, { "accuracy": 0.804, "ap": 0.556586771947699, "ap_weighted": 0.556586771947699, "f1": 0.7696838087717566, "f1_weighted": 0.804889020498141 }, { "accuracy": 0.772, "ap": 0.4826230924272963, "ap_weighted": 0.4826230924272963, "f1": 0.7080576099649669, "f1_weighted": 0.7621627092253795 }, { "accuracy": 0.769, "ap": 0.5175122063810521, "ap_weighted": 0.5175122063810521, "f1": 0.7417190775681342, "f1_weighted": 0.7749598322851153 }, { "accuracy": 0.737, "ap": 0.4944908771455602, "ap_weighted": 0.4944908771455602, "f1": 0.7184006441451425, "f1_weighted": 0.7470595737892031 }, { "accuracy": 0.713, "ap": 0.46693800793522033, "ap_weighted": 0.46693800793522033, "f1": 0.6940295245517318, "f1_weighted": 0.724199437312833 } ] }, { "accuracy": 0.5173, "ap": 0.31080199138280784, "ap_weighted": 0.31080199138280784, "f1": 0.49253257426733194, "f1_weighted": 0.5326676608131118, "hf_subset": "ara", "languages": [ "ara-Arab" ], "main_score": 0.5173, "scores_per_experiment": [ { "accuracy": 0.482, "ap": 0.3168377889533542, "ap_weighted": 0.3168377889533542, "f1": 0.4786634460547504, "f1_weighted": 0.4952628019323671 }, { "accuracy": 0.593, "ap": 0.3152861169837914, "ap_weighted": 0.3152861169837914, "f1": 0.5288473893862167, "f1_weighted": 0.5980418040861781 }, { "accuracy": 0.536, "ap": 0.3045668458767419, "ap_weighted": 0.3045668458767419, "f1": 0.49892874421176303, "f1_weighted": 0.5531727140783745 }, { "accuracy": 0.511, "ap": 0.33197307221542227, "ap_weighted": 0.33197307221542227, "f1": 0.5069167778372704, "f1_weighted": 0.5247752659775602 }, { "accuracy": 0.467, "ap": 0.29924421498605114, "ap_weighted": 0.29924421498605114, "f1": 0.4585396825396826, "f1_weighted": 0.48547733333333337 }, { "accuracy": 0.481, "ap": 0.28974585401200387, "ap_weighted": 0.28974585401200387, "f1": 0.45633443707568677, "f1_weighted": 0.5024231415070325 }, { "accuracy": 0.525, "ap": 0.3163907186020352, "ap_weighted": 0.3163907186020352, "f1": 0.5070154298981131, "f1_weighted": 0.5444911309481183 }, { "accuracy": 0.511, "ap": 0.3142020481043214, "ap_weighted": 0.3142020481043214, "f1": 0.49731438410732187, "f1_weighted": 0.5303257485029941 }, { "accuracy": 0.519, "ap": 0.3251670621737067, "ap_weighted": 0.3251670621737067, "f1": 0.5095235036133128, "f1_weighted": 0.536657644346417 }, { "accuracy": 0.548, "ap": 0.29460619192065096, "ap_weighted": 0.29460619192065096, "f1": 0.4832419479492022, "f1_weighted": 0.5560490234187432 } ] }, { "accuracy": 0.7701, "ap": 0.5180926579362777, "ap_weighted": 0.5180926579362777, "f1": 0.7424382930560531, "f1_weighted": 0.7760920467236796, "hf_subset": "nld", "languages": [ "nld-Latn" ], "main_score": 0.7701, "scores_per_experiment": [ { "accuracy": 0.761, "ap": 0.515828625690052, "ap_weighted": 0.515828625690052, "f1": 0.7398710239177165, "f1_weighted": 0.7698222742238306 }, { "accuracy": 0.761, "ap": 0.5119690115171099, "ap_weighted": 0.5119690115171099, "f1": 0.7378785269869959, "f1_weighted": 0.7693299582908802 }, { "accuracy": 0.789, "ap": 0.5352374625785117, "ap_weighted": 0.5352374625785117, "f1": 0.7577828057934679, "f1_weighted": 0.7929130187724065 }, { "accuracy": 0.779, "ap": 0.4983564531823058, "ap_weighted": 0.4983564531823058, "f1": 0.7280036824116842, "f1_weighted": 0.7755845422400436 }, { "accuracy": 0.753, "ap": 0.49942137943731274, "ap_weighted": 0.49942137943731274, "f1": 0.7283920479349548, "f1_weighted": 0.7614206613378726 }, { "accuracy": 0.798, "ap": 0.5506877836831284, "ap_weighted": 0.5506877836831284, "f1": 0.7686839260521445, "f1_weighted": 0.801952729071621 }, { "accuracy": 0.757, "ap": 0.48326517121147994, "ap_weighted": 0.48326517121147994, "f1": 0.7186975813780114, "f1_weighted": 0.760633020736503 }, { "accuracy": 0.753, "ap": 0.5052882862680399, "ap_weighted": 0.5052882862680399, "f1": 0.7314957642641516, "f1_weighted": 0.7621943905443027 }, { "accuracy": 0.781, "ap": 0.5580242868213654, "ap_weighted": 0.5580242868213654, "f1": 0.7672567434436012, "f1_weighted": 0.790105614426256 }, { "accuracy": 0.769, "ap": 0.5228481189734707, "ap_weighted": 0.5228481189734707, "f1": 0.746320828377804, "f1_weighted": 0.7769642575930789 } ] } ] }, "task_name": "MultiHateClassification" }