{ "dataset_revision": "1399c76144fd37290681b995c656ef9b2e06e26d", "evaluation_time": 327.2148103713989, "kg_co2_emissions": 0.11900839634169257, "mteb_version": "1.12.75", "scores": { "test": [ { "accuracy": 0.5991199999999999, "f1": 0.5759888214463873, "f1_weighted": 0.5759888214463873, "hf_subset": "en", "languages": [ "eng-Latn" ], "main_score": 0.5991199999999999, "scores_per_experiment": [ { "accuracy": 0.5882, "f1": 0.5663137207592932, "f1_weighted": 0.5663137207592933 }, { "accuracy": 0.5948, "f1": 0.5718212675347834, "f1_weighted": 0.5718212675347834 }, { "accuracy": 0.6088, "f1": 0.5961498346334754, "f1_weighted": 0.5961498346334754 }, { "accuracy": 0.613, "f1": 0.5990646264538344, "f1_weighted": 0.5990646264538344 }, { "accuracy": 0.5946, "f1": 0.5601430328125957, "f1_weighted": 0.5601430328125957 }, { "accuracy": 0.6126, "f1": 0.5961293864232428, "f1_weighted": 0.5961293864232428 }, { "accuracy": 0.6226, "f1": 0.6093784399786709, "f1_weighted": 0.6093784399786709 }, { "accuracy": 0.6074, "f1": 0.5975268613654351, "f1_weighted": 0.5975268613654351 }, { "accuracy": 0.5958, "f1": 0.5687907433567946, "f1_weighted": 0.5687907433567946 }, { "accuracy": 0.5534, "f1": 0.4945703011457473, "f1_weighted": 0.49457030114574735 } ] }, { "accuracy": 0.58212, "f1": 0.5527574711402388, "f1_weighted": 0.5527574711402388, "hf_subset": "de", "languages": [ "deu-Latn" ], "main_score": 0.58212, "scores_per_experiment": [ { "accuracy": 0.6082, "f1": 0.5906790886149192, "f1_weighted": 0.5906790886149191 }, { "accuracy": 0.5846, "f1": 0.5461622974847402, "f1_weighted": 0.5461622974847402 }, { "accuracy": 0.5194, "f1": 0.46770630312950534, "f1_weighted": 0.46770630312950523 }, { "accuracy": 0.5742, "f1": 0.5398947987690128, "f1_weighted": 0.5398947987690129 }, { "accuracy": 0.5766, "f1": 0.5446624837588325, "f1_weighted": 0.5446624837588326 }, { "accuracy": 0.624, "f1": 0.6132922138404118, "f1_weighted": 0.6132922138404117 }, { "accuracy": 0.582, "f1": 0.577815377806285, "f1_weighted": 0.577815377806285 }, { "accuracy": 0.5946, "f1": 0.5711780379836758, "f1_weighted": 0.5711780379836758 }, { "accuracy": 0.565, "f1": 0.49902538871732327, "f1_weighted": 0.4990253887173233 }, { "accuracy": 0.5926, "f1": 0.5771587212976821, "f1_weighted": 0.5771587212976821 } ] }, { "accuracy": 0.55738, "f1": 0.5268417200556671, "f1_weighted": 0.5268417200556671, "hf_subset": "es", "languages": [ "spa-Latn" ], "main_score": 0.55738, "scores_per_experiment": [ { "accuracy": 0.5536, "f1": 0.5250542840853866, "f1_weighted": 0.5250542840853867 }, { "accuracy": 0.5504, "f1": 0.5168425364265652, "f1_weighted": 0.5168425364265651 }, { "accuracy": 0.5592, "f1": 0.5282381408828358, "f1_weighted": 0.5282381408828359 }, { "accuracy": 0.5756, "f1": 0.5556918645473083, "f1_weighted": 0.5556918645473085 }, { "accuracy": 0.5364, "f1": 0.48768765057128877, "f1_weighted": 0.48768765057128877 }, { "accuracy": 0.525, "f1": 0.44814941652223605, "f1_weighted": 0.44814941652223605 }, { "accuracy": 0.5672, "f1": 0.5469446735982382, "f1_weighted": 0.5469446735982382 }, { "accuracy": 0.5486, "f1": 0.5377268819761696, "f1_weighted": 0.5377268819761696 }, { "accuracy": 0.5918, "f1": 0.5769893215412036, "f1_weighted": 0.5769893215412036 }, { "accuracy": 0.566, "f1": 0.5450924304054391, "f1_weighted": 0.5450924304054392 } ] }, { "accuracy": 0.54434, "f1": 0.5131726956537492, "f1_weighted": 0.5131726956537491, "hf_subset": "fr", "languages": [ "fra-Latn" ], "main_score": 0.54434, "scores_per_experiment": [ { "accuracy": 0.5334, "f1": 0.5140068061678076, "f1_weighted": 0.5140068061678076 }, { "accuracy": 0.5564, "f1": 0.5332158548770539, "f1_weighted": 0.5332158548770539 }, { "accuracy": 0.5526, "f1": 0.5342899282170979, "f1_weighted": 0.534289928217098 }, { "accuracy": 0.5342, "f1": 0.49469055606304996, "f1_weighted": 0.49469055606304985 }, { "accuracy": 0.5516, "f1": 0.5243650188045604, "f1_weighted": 0.5243650188045604 }, { "accuracy": 0.5812, "f1": 0.5693503820593087, "f1_weighted": 0.5693503820593087 }, { "accuracy": 0.5738, "f1": 0.5622332786882012, "f1_weighted": 0.562233278688201 }, { "accuracy": 0.541, "f1": 0.5002028382621229, "f1_weighted": 0.5002028382621228 }, { "accuracy": 0.514, "f1": 0.45409938287034846, "f1_weighted": 0.4540993828703484 }, { "accuracy": 0.5052, "f1": 0.4452729105279408, "f1_weighted": 0.4452729105279408 } ] }, { "accuracy": 0.5451, "f1": 0.5190612819920248, "f1_weighted": 0.5190612819920247, "hf_subset": "ja", "languages": [ "jpn-Jpan" ], "main_score": 0.5451, "scores_per_experiment": [ { "accuracy": 0.5416, "f1": 0.5062126385771998, "f1_weighted": 0.5062126385771997 }, { "accuracy": 0.561, "f1": 0.5453392116359035, "f1_weighted": 0.5453392116359034 }, { "accuracy": 0.5384, "f1": 0.5133772936766057, "f1_weighted": 0.5133772936766058 }, { "accuracy": 0.5226, "f1": 0.48457507669956784, "f1_weighted": 0.48457507669956784 }, { "accuracy": 0.5502, "f1": 0.5295630299065739, "f1_weighted": 0.5295630299065739 }, { "accuracy": 0.5328, "f1": 0.49968483538345154, "f1_weighted": 0.49968483538345154 }, { "accuracy": 0.557, "f1": 0.5430678629902338, "f1_weighted": 0.5430678629902337 }, { "accuracy": 0.5536, "f1": 0.5373685717509764, "f1_weighted": 0.5373685717509763 }, { "accuracy": 0.5654, "f1": 0.5432524279998562, "f1_weighted": 0.5432524279998561 }, { "accuracy": 0.5284, "f1": 0.4881718712998799, "f1_weighted": 0.4881718712998799 } ] }, { "accuracy": 0.5101, "f1": 0.4727595347715847, "f1_weighted": 0.4727595347715847, "hf_subset": "zh", "languages": [ "cmn-Hans" ], "main_score": 0.5101, "scores_per_experiment": [ { "accuracy": 0.5098, "f1": 0.46974536426392766, "f1_weighted": 0.4697453642639276 }, { "accuracy": 0.5016, "f1": 0.4540476144899938, "f1_weighted": 0.4540476144899938 }, { "accuracy": 0.5038, "f1": 0.4554387154407783, "f1_weighted": 0.4554387154407783 }, { "accuracy": 0.5414, "f1": 0.5162835543981426, "f1_weighted": 0.5162835543981426 }, { "accuracy": 0.4952, "f1": 0.4493267839370455, "f1_weighted": 0.44932678393704545 }, { "accuracy": 0.5182, "f1": 0.4820588354760602, "f1_weighted": 0.48205883547606027 }, { "accuracy": 0.4932, "f1": 0.46571468900591045, "f1_weighted": 0.46571468900591045 }, { "accuracy": 0.5138, "f1": 0.47742935293350053, "f1_weighted": 0.4774293529335006 }, { "accuracy": 0.5186, "f1": 0.4819276545564657, "f1_weighted": 0.4819276545564657 }, { "accuracy": 0.5054, "f1": 0.47562278321402224, "f1_weighted": 0.4756227832140223 } ] } ], "validation": [ { "accuracy": 0.6016600000000001, "f1": 0.5785348254244708, "f1_weighted": 0.5785348254244707, "hf_subset": "en", "languages": [ "eng-Latn" ], "main_score": 0.6016600000000001, "scores_per_experiment": [ { "accuracy": 0.595, "f1": 0.5731282884494437, "f1_weighted": 0.5731282884494437 }, { "accuracy": 0.6034, "f1": 0.5779973237439916, "f1_weighted": 0.5779973237439916 }, { "accuracy": 0.6104, "f1": 0.5994095723324714, "f1_weighted": 0.5994095723324713 }, { "accuracy": 0.613, "f1": 0.598495850170021, "f1_weighted": 0.5984958501700212 }, { "accuracy": 0.5948, "f1": 0.5560174863637093, "f1_weighted": 0.5560174863637092 }, { "accuracy": 0.6102, "f1": 0.594232222571101, "f1_weighted": 0.594232222571101 }, { "accuracy": 0.6186, "f1": 0.6051199743744864, "f1_weighted": 0.6051199743744863 }, { "accuracy": 0.6162, "f1": 0.6078965303927228, "f1_weighted": 0.6078965303927228 }, { "accuracy": 0.6024, "f1": 0.5785051673835954, "f1_weighted": 0.5785051673835953 }, { "accuracy": 0.5526, "f1": 0.49454583846316513, "f1_weighted": 0.4945458384631651 } ] }, { "accuracy": 0.5795399999999999, "f1": 0.5500041795140346, "f1_weighted": 0.5500041795140345, "hf_subset": "de", "languages": [ "deu-Latn" ], "main_score": 0.5795399999999999, "scores_per_experiment": [ { "accuracy": 0.6076, "f1": 0.5919567890688248, "f1_weighted": 0.5919567890688247 }, { "accuracy": 0.5818, "f1": 0.5424660847991294, "f1_weighted": 0.5424660847991294 }, { "accuracy": 0.516, "f1": 0.4574055515979225, "f1_weighted": 0.45740555159792246 }, { "accuracy": 0.58, "f1": 0.5464428351147, "f1_weighted": 0.5464428351147 }, { "accuracy": 0.5782, "f1": 0.5445932365305048, "f1_weighted": 0.5445932365305047 }, { "accuracy": 0.617, "f1": 0.6063827777991572, "f1_weighted": 0.6063827777991572 }, { "accuracy": 0.5732, "f1": 0.5704516024856434, "f1_weighted": 0.5704516024856433 }, { "accuracy": 0.5936, "f1": 0.5702701940366038, "f1_weighted": 0.5702701940366038 }, { "accuracy": 0.5598, "f1": 0.4960005098330778, "f1_weighted": 0.4960005098330778 }, { "accuracy": 0.5882, "f1": 0.5740722138747821, "f1_weighted": 0.5740722138747822 } ] }, { "accuracy": 0.55504, "f1": 0.5253540978900322, "f1_weighted": 0.5253540978900322, "hf_subset": "es", "languages": [ "spa-Latn" ], "main_score": 0.55504, "scores_per_experiment": [ { "accuracy": 0.5572, "f1": 0.5275845916106057, "f1_weighted": 0.5275845916106057 }, { "accuracy": 0.5472, "f1": 0.5148743768300605, "f1_weighted": 0.5148743768300604 }, { "accuracy": 0.5622, "f1": 0.5325112345960857, "f1_weighted": 0.5325112345960857 }, { "accuracy": 0.5634, "f1": 0.5421398871874568, "f1_weighted": 0.5421398871874568 }, { "accuracy": 0.5354, "f1": 0.4882555990825385, "f1_weighted": 0.4882555990825385 }, { "accuracy": 0.5282, "f1": 0.45607864454613234, "f1_weighted": 0.4560786445461323 }, { "accuracy": 0.5734, "f1": 0.5552840959792934, "f1_weighted": 0.5552840959792934 }, { "accuracy": 0.5436, "f1": 0.5341059670855957, "f1_weighted": 0.5341059670855957 }, { "accuracy": 0.576, "f1": 0.5592670740904186, "f1_weighted": 0.5592670740904185 }, { "accuracy": 0.5638, "f1": 0.5434395078921346, "f1_weighted": 0.5434395078921347 } ] }, { "accuracy": 0.54594, "f1": 0.5145467287513261, "f1_weighted": 0.5145467287513261, "hf_subset": "fr", "languages": [ "fra-Latn" ], "main_score": 0.54594, "scores_per_experiment": [ { "accuracy": 0.5466, "f1": 0.5258424837285257, "f1_weighted": 0.5258424837285256 }, { "accuracy": 0.5548, "f1": 0.5298131842307164, "f1_weighted": 0.5298131842307163 }, { "accuracy": 0.5502, "f1": 0.5346695957868492, "f1_weighted": 0.5346695957868491 }, { "accuracy": 0.5446, "f1": 0.5099370235221213, "f1_weighted": 0.5099370235221214 }, { "accuracy": 0.554, "f1": 0.5271032309116828, "f1_weighted": 0.5271032309116829 }, { "accuracy": 0.5832, "f1": 0.572151251860519, "f1_weighted": 0.572151251860519 }, { "accuracy": 0.586, "f1": 0.574512758299534, "f1_weighted": 0.574512758299534 }, { "accuracy": 0.5326, "f1": 0.488845990859158, "f1_weighted": 0.488845990859158 }, { "accuracy": 0.513, "f1": 0.45123769170443345, "f1_weighted": 0.45123769170443356 }, { "accuracy": 0.4944, "f1": 0.4313540766097222, "f1_weighted": 0.43135407660972214 } ] }, { "accuracy": 0.54356, "f1": 0.5155107142775774, "f1_weighted": 0.5155107142775774, "hf_subset": "ja", "languages": [ "jpn-Jpan" ], "main_score": 0.54356, "scores_per_experiment": [ { "accuracy": 0.5336, "f1": 0.4976429227972024, "f1_weighted": 0.4976429227972024 }, { "accuracy": 0.56, "f1": 0.5422626791579175, "f1_weighted": 0.5422626791579174 }, { "accuracy": 0.5478, "f1": 0.5228534312453037, "f1_weighted": 0.5228534312453037 }, { "accuracy": 0.5262, "f1": 0.4855961269408575, "f1_weighted": 0.4855961269408575 }, { "accuracy": 0.5522, "f1": 0.52778864607525, "f1_weighted": 0.5277886460752499 }, { "accuracy": 0.526, "f1": 0.48978492112673216, "f1_weighted": 0.4897849211267321 }, { "accuracy": 0.5528, "f1": 0.5368564338280788, "f1_weighted": 0.5368564338280788 }, { "accuracy": 0.561, "f1": 0.5438077340969476, "f1_weighted": 0.5438077340969477 }, { "accuracy": 0.552, "f1": 0.5261991839306279, "f1_weighted": 0.5261991839306279 }, { "accuracy": 0.524, "f1": 0.48231506357685683, "f1_weighted": 0.48231506357685683 } ] }, { "accuracy": 0.4964799999999999, "f1": 0.45888956677557935, "f1_weighted": 0.4588895667755793, "hf_subset": "zh", "languages": [ "cmn-Hans" ], "main_score": 0.4964799999999999, "scores_per_experiment": [ { "accuracy": 0.4968, "f1": 0.4563998811067155, "f1_weighted": 0.4563998811067155 }, { "accuracy": 0.4942, "f1": 0.4479812945040633, "f1_weighted": 0.4479812945040633 }, { "accuracy": 0.494, "f1": 0.44572886388396055, "f1_weighted": 0.44572886388396055 }, { "accuracy": 0.5134, "f1": 0.4845632988462688, "f1_weighted": 0.48456329884626875 }, { "accuracy": 0.4808, "f1": 0.43399307727070013, "f1_weighted": 0.4339930772707001 }, { "accuracy": 0.5082, "f1": 0.47034653768040774, "f1_weighted": 0.4703465376804077 }, { "accuracy": 0.4764, "f1": 0.45010788512352773, "f1_weighted": 0.45010788512352773 }, { "accuracy": 0.5024, "f1": 0.46961240372106305, "f1_weighted": 0.46961240372106294 }, { "accuracy": 0.511, "f1": 0.4745373902407515, "f1_weighted": 0.47453739024075137 }, { "accuracy": 0.4876, "f1": 0.45562503537833504, "f1_weighted": 0.4556250353783351 } ] } ] }, "task_name": "AmazonReviewsClassification" }