{ "dataset_revision": "1399c76144fd37290681b995c656ef9b2e06e26d", "task_name": "AmazonReviewsClassification", "mteb_version": "1.18.0", "scores": { "validation": [ { "accuracy": 0.4594, "f1": 0.439086, "f1_weighted": 0.439086, "scores_per_experiment": [ { "accuracy": 0.458, "f1": 0.435927, "f1_weighted": 0.435927 }, { "accuracy": 0.475, "f1": 0.448457, "f1_weighted": 0.448457 }, { "accuracy": 0.4542, "f1": 0.439919, "f1_weighted": 0.439919 }, { "accuracy": 0.467, "f1": 0.459656, "f1_weighted": 0.459656 }, { "accuracy": 0.4728, "f1": 0.437898, "f1_weighted": 0.437898 }, { "accuracy": 0.4434, "f1": 0.424482, "f1_weighted": 0.424482 }, { "accuracy": 0.4336, "f1": 0.430335, "f1_weighted": 0.430335 }, { "accuracy": 0.4738, "f1": 0.467008, "f1_weighted": 0.467008 }, { "accuracy": 0.4866, "f1": 0.469128, "f1_weighted": 0.469128 }, { "accuracy": 0.4296, "f1": 0.378053, "f1_weighted": 0.378053 } ], "main_score": 0.4594, "hf_subset": "en", "languages": [ "eng-Latn" ] }, { "accuracy": 0.2927, "f1": 0.284503, "f1_weighted": 0.284503, "scores_per_experiment": [ { "accuracy": 0.3268, "f1": 0.326321, "f1_weighted": 0.326321 }, { "accuracy": 0.2824, "f1": 0.269124, "f1_weighted": 0.269124 }, { "accuracy": 0.291, "f1": 0.2903, "f1_weighted": 0.2903 }, { "accuracy": 0.2968, "f1": 0.287837, "f1_weighted": 0.287837 }, { "accuracy": 0.2882, "f1": 0.280221, "f1_weighted": 0.280221 }, { "accuracy": 0.2876, "f1": 0.272231, "f1_weighted": 0.272231 }, { "accuracy": 0.2508, "f1": 0.249956, "f1_weighted": 0.249956 }, { "accuracy": 0.3276, "f1": 0.31557, "f1_weighted": 0.31557 }, { "accuracy": 0.2912, "f1": 0.273897, "f1_weighted": 0.273897 }, { "accuracy": 0.2846, "f1": 0.279569, "f1_weighted": 0.279569 } ], "main_score": 0.2927, "hf_subset": "de", "languages": [ "deu-Latn" ] }, { "accuracy": 0.34658, "f1": 0.333549, "f1_weighted": 0.333549, "scores_per_experiment": [ { "accuracy": 0.3622, "f1": 0.340254, "f1_weighted": 0.340254 }, { "accuracy": 0.3386, "f1": 0.334594, "f1_weighted": 0.334594 }, { "accuracy": 0.3554, "f1": 0.330247, "f1_weighted": 0.330247 }, { "accuracy": 0.3602, "f1": 0.356903, "f1_weighted": 0.356903 }, { "accuracy": 0.3312, "f1": 0.321582, "f1_weighted": 0.321582 }, { "accuracy": 0.3148, "f1": 0.298695, "f1_weighted": 0.298695 }, { "accuracy": 0.3538, "f1": 0.334416, "f1_weighted": 0.334416 }, { "accuracy": 0.375, "f1": 0.365639, "f1_weighted": 0.365639 }, { "accuracy": 0.3506, "f1": 0.335725, "f1_weighted": 0.335725 }, { "accuracy": 0.324, "f1": 0.317438, "f1_weighted": 0.317438 } ], "main_score": 0.34658, "hf_subset": "es", "languages": [ "spa-Latn" ] }, { "accuracy": 0.30034, "f1": 0.294157, "f1_weighted": 0.294157, "scores_per_experiment": [ { "accuracy": 0.3078, "f1": 0.297087, "f1_weighted": 0.297087 }, { "accuracy": 0.34, "f1": 0.330016, "f1_weighted": 0.330016 }, { "accuracy": 0.3208, "f1": 0.31798, "f1_weighted": 0.31798 }, { "accuracy": 0.2676, "f1": 0.262294, "f1_weighted": 0.262294 }, { "accuracy": 0.3086, "f1": 0.305312, "f1_weighted": 0.305312 }, { "accuracy": 0.2862, "f1": 0.280449, "f1_weighted": 0.280449 }, { "accuracy": 0.3, "f1": 0.292444, "f1_weighted": 0.292444 }, { "accuracy": 0.2824, "f1": 0.281056, "f1_weighted": 0.281056 }, { "accuracy": 0.2864, "f1": 0.274, "f1_weighted": 0.274 }, { "accuracy": 0.3036, "f1": 0.300928, "f1_weighted": 0.300928 } ], "main_score": 0.30034, "hf_subset": "fr", "languages": [ "fra-Latn" ] }, { "accuracy": 0.23066, "f1": 0.226906, "f1_weighted": 0.226906, "scores_per_experiment": [ { "accuracy": 0.235, "f1": 0.235394, "f1_weighted": 0.235394 }, { "accuracy": 0.2452, "f1": 0.237204, "f1_weighted": 0.237204 }, { "accuracy": 0.2382, "f1": 0.235643, "f1_weighted": 0.235643 }, { "accuracy": 0.2328, "f1": 0.228876, "f1_weighted": 0.228876 }, { "accuracy": 0.2502, "f1": 0.248522, "f1_weighted": 0.248522 }, { "accuracy": 0.239, "f1": 0.231856, "f1_weighted": 0.231856 }, { "accuracy": 0.2056, "f1": 0.203279, "f1_weighted": 0.203279 }, { "accuracy": 0.2392, "f1": 0.236049, "f1_weighted": 0.236049 }, { "accuracy": 0.2138, "f1": 0.209814, "f1_weighted": 0.209814 }, { "accuracy": 0.2076, "f1": 0.20242, "f1_weighted": 0.20242 } ], "main_score": 0.23066, "hf_subset": "ja", "languages": [ "jpn-Jpan" ] }, { "accuracy": 0.2294, "f1": 0.223487, "f1_weighted": 0.223487, "scores_per_experiment": [ { "accuracy": 0.229, "f1": 0.222622, "f1_weighted": 0.222622 }, { "accuracy": 0.2112, "f1": 0.194734, "f1_weighted": 0.194734 }, { "accuracy": 0.2134, "f1": 0.20764, "f1_weighted": 0.20764 }, { "accuracy": 0.2328, "f1": 0.228063, "f1_weighted": 0.228063 }, { "accuracy": 0.2218, "f1": 0.21937, "f1_weighted": 0.21937 }, { "accuracy": 0.2344, "f1": 0.231737, "f1_weighted": 0.231737 }, { "accuracy": 0.243, "f1": 0.231466, "f1_weighted": 0.231466 }, { "accuracy": 0.2366, "f1": 0.232575, "f1_weighted": 0.232575 }, { "accuracy": 0.2436, "f1": 0.239456, "f1_weighted": 0.239456 }, { "accuracy": 0.2282, "f1": 0.227208, "f1_weighted": 0.227208 } ], "main_score": 0.2294, "hf_subset": "zh", "languages": [ "cmn-Hans" ] } ], "test": [ { "accuracy": 0.46432, "f1": 0.444239, "f1_weighted": 0.444239, "scores_per_experiment": [ { "accuracy": 0.4616, "f1": 0.435465, "f1_weighted": 0.435465 }, { "accuracy": 0.4788, "f1": 0.453073, "f1_weighted": 0.453073 }, { "accuracy": 0.4604, "f1": 0.446256, "f1_weighted": 0.446256 }, { "accuracy": 0.4718, "f1": 0.465595, "f1_weighted": 0.465595 }, { "accuracy": 0.4766, "f1": 0.444965, "f1_weighted": 0.444965 }, { "accuracy": 0.4554, "f1": 0.434524, "f1_weighted": 0.434524 }, { "accuracy": 0.4378, "f1": 0.433378, "f1_weighted": 0.433378 }, { "accuracy": 0.4818, "f1": 0.47403, "f1_weighted": 0.47403 }, { "accuracy": 0.4818, "f1": 0.467013, "f1_weighted": 0.467013 }, { "accuracy": 0.4372, "f1": 0.388094, "f1_weighted": 0.388094 } ], "main_score": 0.46432, "hf_subset": "en", "languages": [ "eng-Latn" ] }, { "accuracy": 0.29228, "f1": 0.284251, "f1_weighted": 0.284251, "scores_per_experiment": [ { "accuracy": 0.3232, "f1": 0.321592, "f1_weighted": 0.321592 }, { "accuracy": 0.2782, "f1": 0.262374, "f1_weighted": 0.262374 }, { "accuracy": 0.2976, "f1": 0.297421, "f1_weighted": 0.297421 }, { "accuracy": 0.2992, "f1": 0.291315, "f1_weighted": 0.291315 }, { "accuracy": 0.2886, "f1": 0.277899, "f1_weighted": 0.277899 }, { "accuracy": 0.2966, "f1": 0.283185, "f1_weighted": 0.283185 }, { "accuracy": 0.2576, "f1": 0.257278, "f1_weighted": 0.257278 }, { "accuracy": 0.3108, "f1": 0.299254, "f1_weighted": 0.299254 }, { "accuracy": 0.285, "f1": 0.270262, "f1_weighted": 0.270262 }, { "accuracy": 0.286, "f1": 0.281934, "f1_weighted": 0.281934 } ], "main_score": 0.29228, "hf_subset": "de", "languages": [ "deu-Latn" ] }, { "accuracy": 0.34698, "f1": 0.333734, "f1_weighted": 0.333734, "scores_per_experiment": [ { "accuracy": 0.3652, "f1": 0.347976, "f1_weighted": 0.347976 }, { "accuracy": 0.3434, "f1": 0.339372, "f1_weighted": 0.339372 }, { "accuracy": 0.3554, "f1": 0.329538, "f1_weighted": 0.329538 }, { "accuracy": 0.356, "f1": 0.353108, "f1_weighted": 0.353108 }, { "accuracy": 0.3308, "f1": 0.318717, "f1_weighted": 0.318717 }, { "accuracy": 0.323, "f1": 0.308314, "f1_weighted": 0.308314 }, { "accuracy": 0.3528, "f1": 0.334582, "f1_weighted": 0.334582 }, { "accuracy": 0.3616, "f1": 0.35012, "f1_weighted": 0.35012 }, { "accuracy": 0.3522, "f1": 0.334697, "f1_weighted": 0.334697 }, { "accuracy": 0.3294, "f1": 0.320917, "f1_weighted": 0.320917 } ], "main_score": 0.34698, "hf_subset": "es", "languages": [ "spa-Latn" ] }, { "accuracy": 0.30108, "f1": 0.294827, "f1_weighted": 0.294827, "scores_per_experiment": [ { "accuracy": 0.2922, "f1": 0.280828, "f1_weighted": 0.280828 }, { "accuracy": 0.3392, "f1": 0.330732, "f1_weighted": 0.330732 }, { "accuracy": 0.3176, "f1": 0.31661, "f1_weighted": 0.31661 }, { "accuracy": 0.263, "f1": 0.258001, "f1_weighted": 0.258001 }, { "accuracy": 0.307, "f1": 0.304644, "f1_weighted": 0.304644 }, { "accuracy": 0.3076, "f1": 0.301912, "f1_weighted": 0.301912 }, { "accuracy": 0.2942, "f1": 0.285751, "f1_weighted": 0.285751 }, { "accuracy": 0.2936, "f1": 0.290707, "f1_weighted": 0.290707 }, { "accuracy": 0.2976, "f1": 0.282322, "f1_weighted": 0.282322 }, { "accuracy": 0.2988, "f1": 0.29676, "f1_weighted": 0.29676 } ], "main_score": 0.30108, "hf_subset": "fr", "languages": [ "fra-Latn" ] }, { "accuracy": 0.22896, "f1": 0.22519, "f1_weighted": 0.22519, "scores_per_experiment": [ { "accuracy": 0.2266, "f1": 0.225423, "f1_weighted": 0.225423 }, { "accuracy": 0.2386, "f1": 0.231646, "f1_weighted": 0.231646 }, { "accuracy": 0.2348, "f1": 0.233432, "f1_weighted": 0.233432 }, { "accuracy": 0.231, "f1": 0.226663, "f1_weighted": 0.226663 }, { "accuracy": 0.2496, "f1": 0.247722, "f1_weighted": 0.247722 }, { "accuracy": 0.2252, "f1": 0.217304, "f1_weighted": 0.217304 }, { "accuracy": 0.2136, "f1": 0.21003, "f1_weighted": 0.21003 }, { "accuracy": 0.2396, "f1": 0.237231, "f1_weighted": 0.237231 }, { "accuracy": 0.2174, "f1": 0.214822, "f1_weighted": 0.214822 }, { "accuracy": 0.2132, "f1": 0.207628, "f1_weighted": 0.207628 } ], "main_score": 0.22896, "hf_subset": "ja", "languages": [ "jpn-Jpan" ] }, { "accuracy": 0.22954, "f1": 0.223768, "f1_weighted": 0.223768, "scores_per_experiment": [ { "accuracy": 0.2288, "f1": 0.223384, "f1_weighted": 0.223384 }, { "accuracy": 0.2236, "f1": 0.207672, "f1_weighted": 0.207672 }, { "accuracy": 0.208, "f1": 0.203599, "f1_weighted": 0.203599 }, { "accuracy": 0.2246, "f1": 0.220005, "f1_weighted": 0.220005 }, { "accuracy": 0.2186, "f1": 0.215891, "f1_weighted": 0.215891 }, { "accuracy": 0.2454, "f1": 0.24187, "f1_weighted": 0.24187 }, { "accuracy": 0.2342, "f1": 0.223205, "f1_weighted": 0.223205 }, { "accuracy": 0.2452, "f1": 0.240488, "f1_weighted": 0.240488 }, { "accuracy": 0.2432, "f1": 0.238533, "f1_weighted": 0.238533 }, { "accuracy": 0.2238, "f1": 0.223034, "f1_weighted": 0.223034 } ], "main_score": 0.22954, "hf_subset": "zh", "languages": [ "cmn-Hans" ] } ] }, "evaluation_time": 59.36974000930786, "kg_co2_emissions": 0.0022568867973774345 }