{ "dataset_revision": "d80d48c1eb48d3562165c59d59d0034df9fff0bf", "evaluation_time": 87.15009808540344, "kg_co2_emissions": 0.012946139037923216, "mteb_version": "1.12.75", "scores": { "test": [ { "accuracy": 0.8703374373005015, "f1": 0.8679311485415827, "f1_weighted": 0.8711585143399239, "hf_subset": "en", "languages": [ "eng-Latn" ], "main_score": 0.8703374373005015, "scores_per_experiment": [ { "accuracy": 0.8846329229366166, "f1": 0.8802880173866291, "f1_weighted": 0.8838197497038114 }, { "accuracy": 0.8752849977200182, "f1": 0.8727652170865816, "f1_weighted": 0.8748086649482343 }, { "accuracy": 0.8627450980392157, "f1": 0.8609089563883005, "f1_weighted": 0.8629105959684115 }, { "accuracy": 0.8855449156406748, "f1": 0.8820930210216577, "f1_weighted": 0.886338366662533 }, { "accuracy": 0.8618331053351573, "f1": 0.8596302785138655, "f1_weighted": 0.8635527601089641 }, { "accuracy": 0.8563611491108071, "f1": 0.855499264602919, "f1_weighted": 0.8566540856240256 }, { "accuracy": 0.8663930688554492, "f1": 0.8625555614360626, "f1_weighted": 0.8657795413451896 }, { "accuracy": 0.8757409940720474, "f1": 0.8707609196384435, "f1_weighted": 0.8781332533251927 }, { "accuracy": 0.8620611035111719, "f1": 0.8665985744416692, "f1_weighted": 0.8656167730959052 }, { "accuracy": 0.8727770177838577, "f1": 0.8682116748996986, "f1_weighted": 0.8739713526169718 } ] }, { "accuracy": 0.7921386306001691, "f1": 0.7818896954320964, "f1_weighted": 0.7944276154936083, "hf_subset": "de", "languages": [ "deu-Latn" ], "main_score": 0.7921386306001691, "scores_per_experiment": [ { "accuracy": 0.7855734009580163, "f1": 0.7762739260884003, "f1_weighted": 0.783925356154485 }, { "accuracy": 0.7731755424063116, "f1": 0.762497925621729, "f1_weighted": 0.777005404502143 }, { "accuracy": 0.7923358692589462, "f1": 0.77379620099731, "f1_weighted": 0.7930308337651722 }, { "accuracy": 0.813468582699352, "f1": 0.8027935244815457, "f1_weighted": 0.8154858465116918 }, { "accuracy": 0.7827557058326289, "f1": 0.7703691251975093, "f1_weighted": 0.7850500235931168 }, { "accuracy": 0.7940264863341786, "f1": 0.779180293509305, "f1_weighted": 0.7946433365284697 }, { "accuracy": 0.813468582699352, "f1": 0.7975973162885068, "f1_weighted": 0.8134966558367017 }, { "accuracy": 0.7917723302338687, "f1": 0.7912232948379966, "f1_weighted": 0.7984504184334207 }, { "accuracy": 0.7847280924204001, "f1": 0.7819974013632394, "f1_weighted": 0.7884661864806406 }, { "accuracy": 0.7900817131586363, "f1": 0.7831679459354224, "f1_weighted": 0.7947220931302427 } ] }, { "accuracy": 0.8305870580386925, "f1": 0.8273055351599465, "f1_weighted": 0.8312346479533238, "hf_subset": "es", "languages": [ "spa-Latn" ], "main_score": 0.8305870580386925, "scores_per_experiment": [ { "accuracy": 0.8385590393595731, "f1": 0.8374800514510562, "f1_weighted": 0.8383648614098377 }, { "accuracy": 0.8198799199466311, "f1": 0.8187733737286155, "f1_weighted": 0.8197870937016141 }, { "accuracy": 0.8202134756504337, "f1": 0.8177838457042416, "f1_weighted": 0.8197794447246641 }, { "accuracy": 0.8455637091394262, "f1": 0.8431365802858608, "f1_weighted": 0.8454173347431949 }, { "accuracy": 0.8408939292861908, "f1": 0.833343828771089, "f1_weighted": 0.8417453777792677 }, { "accuracy": 0.8308872581721147, "f1": 0.8262355560211637, "f1_weighted": 0.8314834360143549 }, { "accuracy": 0.8172114743162108, "f1": 0.8162584394297029, "f1_weighted": 0.8184114247667019 }, { "accuracy": 0.8182121414276184, "f1": 0.8187695287967284, "f1_weighted": 0.8203993342289941 }, { "accuracy": 0.8392261507671781, "f1": 0.8368707301819243, "f1_weighted": 0.8402358378394055 }, { "accuracy": 0.8352234823215477, "f1": 0.8244034172290832, "f1_weighted": 0.8367223343252026 } ] }, { "accuracy": 0.7863764484810523, "f1": 0.7898770057097882, "f1_weighted": 0.7857879423415014, "hf_subset": "fr", "languages": [ "fra-Latn" ], "main_score": 0.7863764484810523, "scores_per_experiment": [ { "accuracy": 0.7895396179141873, "f1": 0.7962651412503446, "f1_weighted": 0.7880000762454575 }, { "accuracy": 0.7882868775446289, "f1": 0.7919156592026745, "f1_weighted": 0.7861799577331968 }, { "accuracy": 0.7854682117131224, "f1": 0.7920745084248112, "f1_weighted": 0.7857499668615223 }, { "accuracy": 0.7735671782023176, "f1": 0.7807176586383245, "f1_weighted": 0.772959020391754 }, { "accuracy": 0.7785781396805512, "f1": 0.7810131726071551, "f1_weighted": 0.7793012323512559 }, { "accuracy": 0.7954901346695897, "f1": 0.7978129101928565, "f1_weighted": 0.7957638972995347 }, { "accuracy": 0.7782649545881616, "f1": 0.7762185320966211, "f1_weighted": 0.774285662809868 }, { "accuracy": 0.7961165048543689, "f1": 0.8041347235671562, "f1_weighted": 0.7966554363584503 }, { "accuracy": 0.7854682117131224, "f1": 0.7885115642581773, "f1_weighted": 0.7850531980922384 }, { "accuracy": 0.7929846539304729, "f1": 0.7901061868597616, "f1_weighted": 0.7939309752717362 } ] }, { "accuracy": 0.8136249551810686, "f1": 0.8028945150124537, "f1_weighted": 0.8169817171256362, "hf_subset": "hi", "languages": [ "hin-Deva" ], "main_score": 0.8136249551810686, "scores_per_experiment": [ { "accuracy": 0.8139117963427752, "f1": 0.806637549395083, "f1_weighted": 0.816352197270922 }, { "accuracy": 0.8164216565077088, "f1": 0.8019887665998553, "f1_weighted": 0.81497652766431 }, { "accuracy": 0.8214413768375762, "f1": 0.8108918852756827, "f1_weighted": 0.8247403085057199 }, { "accuracy": 0.8103262818214414, "f1": 0.8021441199592101, "f1_weighted": 0.8142042270677015 }, { "accuracy": 0.8038723556830405, "f1": 0.7923782580928399, "f1_weighted": 0.8100703902091152 }, { "accuracy": 0.8235926855503765, "f1": 0.8098721799946016, "f1_weighted": 0.8235422432225745 }, { "accuracy": 0.8096091789171747, "f1": 0.8018294550173988, "f1_weighted": 0.8110168502439102 }, { "accuracy": 0.8207242739333094, "f1": 0.8093316683286549, "f1_weighted": 0.8253282229911694 }, { "accuracy": 0.8035138042309071, "f1": 0.7936677192225005, "f1_weighted": 0.8137121446012187 }, { "accuracy": 0.812836141986375, "f1": 0.8002035482387101, "f1_weighted": 0.8158740594797205 } ] }, { "accuracy": 0.7996745027124774, "f1": 0.7972732812795513, "f1_weighted": 0.8002422623671632, "hf_subset": "th", "languages": [ "tha-Thai" ], "main_score": 0.7996745027124774, "scores_per_experiment": [ { "accuracy": 0.8213381555153707, "f1": 0.8199286780693413, "f1_weighted": 0.8226071952171516 }, { "accuracy": 0.799638336347197, "f1": 0.8009053471513535, "f1_weighted": 0.7998069362646336 }, { "accuracy": 0.8018083182640144, "f1": 0.7997781121486792, "f1_weighted": 0.8010841187793712 }, { "accuracy": 0.7978300180831827, "f1": 0.7952895375591602, "f1_weighted": 0.7966077410117441 }, { "accuracy": 0.8065099457504521, "f1": 0.8018586302313704, "f1_weighted": 0.8059292443863632 }, { "accuracy": 0.7822784810126582, "f1": 0.784405986822494, "f1_weighted": 0.7839507114602381 }, { "accuracy": 0.799638336347197, "f1": 0.7909747682974072, "f1_weighted": 0.7994139003776578 }, { "accuracy": 0.7989150090415913, "f1": 0.7940347901967691, "f1_weighted": 0.7999242629026668 }, { "accuracy": 0.783001808318264, "f1": 0.7833695876786855, "f1_weighted": 0.7864012450075403 }, { "accuracy": 0.8057866184448463, "f1": 0.802187374640253, "f1_weighted": 0.8066972682642668 } ] } ], "validation": [ { "accuracy": 0.8777628635346757, "f1": 0.8796201952954297, "f1_weighted": 0.8779580308649889, "hf_subset": "en", "languages": [ "eng-Latn" ], "main_score": 0.8777628635346757, "scores_per_experiment": [ { "accuracy": 0.8926174496644296, "f1": 0.892789815373061, "f1_weighted": 0.8917911868071896 }, { "accuracy": 0.8818791946308725, "f1": 0.8833332423724454, "f1_weighted": 0.8810880428158848 }, { "accuracy": 0.8657718120805369, "f1": 0.867325700717434, "f1_weighted": 0.8651842143579975 }, { "accuracy": 0.8935123042505593, "f1": 0.8966084613404874, "f1_weighted": 0.8940743986694614 }, { "accuracy": 0.8680089485458613, "f1": 0.8721646053497459, "f1_weighted": 0.8697193912244653 }, { "accuracy": 0.8590604026845637, "f1": 0.8619776279848952, "f1_weighted": 0.8583533417963461 }, { "accuracy": 0.8733780760626398, "f1": 0.8720916896635675, "f1_weighted": 0.8721180656358145 }, { "accuracy": 0.8863534675615212, "f1": 0.8860671587678994, "f1_weighted": 0.8874784015212873 }, { "accuracy": 0.8697986577181208, "f1": 0.874435731247247, "f1_weighted": 0.8718229705099304 }, { "accuracy": 0.887248322147651, "f1": 0.8894079201375145, "f1_weighted": 0.8879502953115121 } ] }, { "accuracy": 0.790633608815427, "f1": 0.783540337932569, "f1_weighted": 0.793138784552942, "hf_subset": "de", "languages": [ "deu-Latn" ], "main_score": 0.790633608815427, "scores_per_experiment": [ { "accuracy": 0.78732782369146, "f1": 0.780281888630677, "f1_weighted": 0.7850171738921852 }, { "accuracy": 0.7823691460055097, "f1": 0.7759896927850558, "f1_weighted": 0.7861414001731801 }, { "accuracy": 0.790633608815427, "f1": 0.7787872876071101, "f1_weighted": 0.7925729878897965 }, { "accuracy": 0.8038567493112948, "f1": 0.7964731859440639, "f1_weighted": 0.8059211558067628 }, { "accuracy": 0.7652892561983471, "f1": 0.7544318179713877, "f1_weighted": 0.7692744985067117 }, { "accuracy": 0.7895316804407714, "f1": 0.7746028713450499, "f1_weighted": 0.7884024877054149 }, { "accuracy": 0.8165289256198347, "f1": 0.8075411335721566, "f1_weighted": 0.8160721737666106 }, { "accuracy": 0.8005509641873279, "f1": 0.7983208668248952, "f1_weighted": 0.8063694143881648 }, { "accuracy": 0.7779614325068871, "f1": 0.7812839782105432, "f1_weighted": 0.78480671117757 }, { "accuracy": 0.7922865013774105, "f1": 0.7876906564347518, "f1_weighted": 0.7968098422230232 } ] }, { "accuracy": 0.8333988212180747, "f1": 0.8351855646709113, "f1_weighted": 0.8332721205296085, "hf_subset": "es", "languages": [ "spa-Latn" ], "main_score": 0.8333988212180747, "scores_per_experiment": [ { "accuracy": 0.8375900458415193, "f1": 0.8394774471669144, "f1_weighted": 0.8367812065041427 }, { "accuracy": 0.8205631958087753, "f1": 0.8265786352017105, "f1_weighted": 0.819348256581481 }, { "accuracy": 0.8212180746561886, "f1": 0.8242520205041267, "f1_weighted": 0.8203983749800486 }, { "accuracy": 0.8461034708578913, "f1": 0.845404927758242, "f1_weighted": 0.8455996445528338 }, { "accuracy": 0.8506876227897839, "f1": 0.8489156120617999, "f1_weighted": 0.8505400929712902 }, { "accuracy": 0.8343156516044532, "f1": 0.8366768961436787, "f1_weighted": 0.835107280859138 }, { "accuracy": 0.8199083169613621, "f1": 0.8254155412957934, "f1_weighted": 0.8196070021121802 }, { "accuracy": 0.8297314996725605, "f1": 0.8317531993154212, "f1_weighted": 0.8310937501480972 }, { "accuracy": 0.8408644400785854, "f1": 0.8441252655197357, "f1_weighted": 0.8413863802088503 }, { "accuracy": 0.8330058939096268, "f1": 0.8292561017416912, "f1_weighted": 0.8328592163780225 } ] }, { "accuracy": 0.7974635383639822, "f1": 0.8012605106992015, "f1_weighted": 0.7954032688914869, "hf_subset": "fr", "languages": [ "fra-Latn" ], "main_score": 0.7974635383639822, "scores_per_experiment": [ { "accuracy": 0.8034242232086239, "f1": 0.8102560549876938, "f1_weighted": 0.8014024439479344 }, { "accuracy": 0.7913760304375397, "f1": 0.7964261994150427, "f1_weighted": 0.7880336232117687 }, { "accuracy": 0.8040583386176284, "f1": 0.8139898894488987, "f1_weighted": 0.8026666214004045 }, { "accuracy": 0.7901077996195307, "f1": 0.7971306691882635, "f1_weighted": 0.7892716053012713 }, { "accuracy": 0.777425491439442, "f1": 0.7813698931906732, "f1_weighted": 0.7763795167204857 }, { "accuracy": 0.8091312618896639, "f1": 0.8131773944988923, "f1_weighted": 0.8077696332299826 }, { "accuracy": 0.7793278376664553, "f1": 0.7739784151897915, "f1_weighted": 0.77350281775987 }, { "accuracy": 0.813570069752695, "f1": 0.8186073471722899, "f1_weighted": 0.8122975241873636 }, { "accuracy": 0.8021559923906151, "f1": 0.807777056916946, "f1_weighted": 0.799836460318834 }, { "accuracy": 0.8040583386176284, "f1": 0.7998921869835243, "f1_weighted": 0.8028724428369546 } ] }, { "accuracy": 0.7968190854870775, "f1": 0.7955002646490942, "f1_weighted": 0.7978040460459005, "hf_subset": "hi", "languages": [ "hin-Deva" ], "main_score": 0.7968190854870775, "scores_per_experiment": [ { "accuracy": 0.8096421471172962, "f1": 0.8083432803664963, "f1_weighted": 0.8090701270586176 }, { "accuracy": 0.7818091451292246, "f1": 0.7732783516616809, "f1_weighted": 0.7794630636963291 }, { "accuracy": 0.8066600397614314, "f1": 0.8055044495412246, "f1_weighted": 0.8069927610083054 }, { "accuracy": 0.794234592445328, "f1": 0.7944888485450806, "f1_weighted": 0.7931088924225738 }, { "accuracy": 0.7862823061630219, "f1": 0.784984401516798, "f1_weighted": 0.7898267290112224 }, { "accuracy": 0.8021868787276342, "f1": 0.797075928446452, "f1_weighted": 0.7996725348127107 }, { "accuracy": 0.7882703777335984, "f1": 0.7897536436403019, "f1_weighted": 0.7884708875236003 }, { "accuracy": 0.8141153081510935, "f1": 0.8143533338945294, "f1_weighted": 0.8177741714527961 }, { "accuracy": 0.786779324055666, "f1": 0.7885238628843116, "f1_weighted": 0.7941226787319912 }, { "accuracy": 0.7982107355864811, "f1": 0.7986965459940667, "f1_weighted": 0.7995386147408584 } ] }, { "accuracy": 0.806463195691203, "f1": 0.8056651726675795, "f1_weighted": 0.8066290211477194, "hf_subset": "th", "languages": [ "tha-Thai" ], "main_score": 0.806463195691203, "scores_per_experiment": [ { "accuracy": 0.827049670855775, "f1": 0.8285589584333781, "f1_weighted": 0.8283949280884569 }, { "accuracy": 0.8204667863554758, "f1": 0.8218240696911233, "f1_weighted": 0.8206139324817736 }, { "accuracy": 0.7851585876720527, "f1": 0.7852951341485324, "f1_weighted": 0.7840300223034863 }, { "accuracy": 0.7995212447636146, "f1": 0.7945322913900106, "f1_weighted": 0.7973139865442107 }, { "accuracy": 0.7983243566726511, "f1": 0.7930989319272634, "f1_weighted": 0.7961559697204058 }, { "accuracy": 0.8031119090365051, "f1": 0.8052594645683165, "f1_weighted": 0.8044977153859033 }, { "accuracy": 0.8037103530819868, "f1": 0.8002869768937539, "f1_weighted": 0.8041470004930663 }, { "accuracy": 0.8096947935368043, "f1": 0.8086911097958788, "f1_weighted": 0.8101076897980692 }, { "accuracy": 0.8084979054458408, "f1": 0.8111365775006486, "f1_weighted": 0.8108310838495307 }, { "accuracy": 0.8090963494913226, "f1": 0.80796821232689, "f1_weighted": 0.810197882812291 } ] } ] }, "task_name": "MTOPDomainClassification" }