{ "dataset_revision": "1399c76144fd37290681b995c656ef9b2e06e26d", "task_name": "AmazonReviewsClassification", "mteb_version": "1.18.0", "scores": { "validation": [ { "accuracy": 0.28288, "f1": 0.282123, "f1_weighted": 0.282123, "scores_per_experiment": [ { "accuracy": 0.2986, "f1": 0.296198, "f1_weighted": 0.296198 }, { "accuracy": 0.3046, "f1": 0.300511, "f1_weighted": 0.300511 }, { "accuracy": 0.288, "f1": 0.290131, "f1_weighted": 0.290131 }, { "accuracy": 0.2812, "f1": 0.283488, "f1_weighted": 0.283488 }, { "accuracy": 0.3314, "f1": 0.327545, "f1_weighted": 0.327545 }, { "accuracy": 0.2618, "f1": 0.263025, "f1_weighted": 0.263025 }, { "accuracy": 0.2554, "f1": 0.254393, "f1_weighted": 0.254393 }, { "accuracy": 0.266, "f1": 0.265458, "f1_weighted": 0.265458 }, { "accuracy": 0.2858, "f1": 0.286947, "f1_weighted": 0.286947 }, { "accuracy": 0.256, "f1": 0.253537, "f1_weighted": 0.253537 } ], "main_score": 0.28288, "hf_subset": "en", "languages": [ "eng-Latn" ] }, { "accuracy": 0.24982, "f1": 0.24795, "f1_weighted": 0.24795, "scores_per_experiment": [ { "accuracy": 0.2754, "f1": 0.274393, "f1_weighted": 0.274393 }, { "accuracy": 0.2596, "f1": 0.258947, "f1_weighted": 0.258947 }, { "accuracy": 0.251, "f1": 0.25128, "f1_weighted": 0.25128 }, { "accuracy": 0.2248, "f1": 0.22282, "f1_weighted": 0.22282 }, { "accuracy": 0.2468, "f1": 0.247612, "f1_weighted": 0.247612 }, { "accuracy": 0.2466, "f1": 0.246212, "f1_weighted": 0.246212 }, { "accuracy": 0.2358, "f1": 0.231127, "f1_weighted": 0.231127 }, { "accuracy": 0.2508, "f1": 0.250896, "f1_weighted": 0.250896 }, { "accuracy": 0.2558, "f1": 0.250883, "f1_weighted": 0.250883 }, { "accuracy": 0.2516, "f1": 0.245329, "f1_weighted": 0.245329 } ], "main_score": 0.24982, "hf_subset": "de", "languages": [ "deu-Latn" ] }, { "accuracy": 0.25764, "f1": 0.255081, "f1_weighted": 0.255081, "scores_per_experiment": [ { "accuracy": 0.2596, "f1": 0.259961, "f1_weighted": 0.259961 }, { "accuracy": 0.2452, "f1": 0.244196, "f1_weighted": 0.244196 }, { "accuracy": 0.2684, "f1": 0.268444, "f1_weighted": 0.268444 }, { "accuracy": 0.262, "f1": 0.259217, "f1_weighted": 0.259217 }, { "accuracy": 0.231, "f1": 0.22756, "f1_weighted": 0.22756 }, { "accuracy": 0.254, "f1": 0.243686, "f1_weighted": 0.243686 }, { "accuracy": 0.2578, "f1": 0.257668, "f1_weighted": 0.257668 }, { "accuracy": 0.2718, "f1": 0.27253, "f1_weighted": 0.27253 }, { "accuracy": 0.272, "f1": 0.265534, "f1_weighted": 0.265534 }, { "accuracy": 0.2546, "f1": 0.252015, "f1_weighted": 0.252015 } ], "main_score": 0.25764, "hf_subset": "es", "languages": [ "spa-Latn" ] }, { "accuracy": 0.25516, "f1": 0.251837, "f1_weighted": 0.251837, "scores_per_experiment": [ { "accuracy": 0.2844, "f1": 0.280521, "f1_weighted": 0.280521 }, { "accuracy": 0.2804, "f1": 0.276507, "f1_weighted": 0.276507 }, { "accuracy": 0.261, "f1": 0.259339, "f1_weighted": 0.259339 }, { "accuracy": 0.2218, "f1": 0.217126, "f1_weighted": 0.217126 }, { "accuracy": 0.2722, "f1": 0.270934, "f1_weighted": 0.270934 }, { "accuracy": 0.2466, "f1": 0.244865, "f1_weighted": 0.244865 }, { "accuracy": 0.2528, "f1": 0.24036, "f1_weighted": 0.24036 }, { "accuracy": 0.2402, "f1": 0.239885, "f1_weighted": 0.239885 }, { "accuracy": 0.2264, "f1": 0.226363, "f1_weighted": 0.226363 }, { "accuracy": 0.2658, "f1": 0.262475, "f1_weighted": 0.262475 } ], "main_score": 0.25516, "hf_subset": "fr", "languages": [ "fra-Latn" ] }, { "accuracy": 0.21902, "f1": 0.212956, "f1_weighted": 0.212956, "scores_per_experiment": [ { "accuracy": 0.2322, "f1": 0.224415, "f1_weighted": 0.224415 }, { "accuracy": 0.2148, "f1": 0.208686, "f1_weighted": 0.208686 }, { "accuracy": 0.2118, "f1": 0.207413, "f1_weighted": 0.207413 }, { "accuracy": 0.2208, "f1": 0.219627, "f1_weighted": 0.219627 }, { "accuracy": 0.2282, "f1": 0.221477, "f1_weighted": 0.221477 }, { "accuracy": 0.215, "f1": 0.209763, "f1_weighted": 0.209763 }, { "accuracy": 0.2288, "f1": 0.22295, "f1_weighted": 0.22295 }, { "accuracy": 0.2338, "f1": 0.230273, "f1_weighted": 0.230273 }, { "accuracy": 0.2054, "f1": 0.194416, "f1_weighted": 0.194416 }, { "accuracy": 0.1994, "f1": 0.190536, "f1_weighted": 0.190536 } ], "main_score": 0.21902, "hf_subset": "ja", "languages": [ "jpn-Jpan" ] }, { "accuracy": 0.22754, "f1": 0.2215, "f1_weighted": 0.2215, "scores_per_experiment": [ { "accuracy": 0.2214, "f1": 0.219016, "f1_weighted": 0.219016 }, { "accuracy": 0.2132, "f1": 0.204882, "f1_weighted": 0.204882 }, { "accuracy": 0.2234, "f1": 0.223505, "f1_weighted": 0.223505 }, { "accuracy": 0.2344, "f1": 0.22832, "f1_weighted": 0.22832 }, { "accuracy": 0.2224, "f1": 0.218044, "f1_weighted": 0.218044 }, { "accuracy": 0.225, "f1": 0.222337, "f1_weighted": 0.222337 }, { "accuracy": 0.2482, "f1": 0.232792, "f1_weighted": 0.232792 }, { "accuracy": 0.221, "f1": 0.215014, "f1_weighted": 0.215014 }, { "accuracy": 0.2314, "f1": 0.21945, "f1_weighted": 0.21945 }, { "accuracy": 0.235, "f1": 0.231641, "f1_weighted": 0.231641 } ], "main_score": 0.22754, "hf_subset": "zh", "languages": [ "cmn-Hans" ] } ], "test": [ { "accuracy": 0.28552, "f1": 0.283953, "f1_weighted": 0.283953, "scores_per_experiment": [ { "accuracy": 0.3106, "f1": 0.308056, "f1_weighted": 0.308056 }, { "accuracy": 0.3, "f1": 0.294814, "f1_weighted": 0.294814 }, { "accuracy": 0.2866, "f1": 0.287754, "f1_weighted": 0.287754 }, { "accuracy": 0.282, "f1": 0.283153, "f1_weighted": 0.283153 }, { "accuracy": 0.3328, "f1": 0.328063, "f1_weighted": 0.328063 }, { "accuracy": 0.2674, "f1": 0.266356, "f1_weighted": 0.266356 }, { "accuracy": 0.2552, "f1": 0.253868, "f1_weighted": 0.253868 }, { "accuracy": 0.2722, "f1": 0.272378, "f1_weighted": 0.272378 }, { "accuracy": 0.295, "f1": 0.29465, "f1_weighted": 0.29465 }, { "accuracy": 0.2534, "f1": 0.250442, "f1_weighted": 0.250442 } ], "main_score": 0.28552, "hf_subset": "en", "languages": [ "eng-Latn" ] }, { "accuracy": 0.24556, "f1": 0.243085, "f1_weighted": 0.243085, "scores_per_experiment": [ { "accuracy": 0.2664, "f1": 0.262508, "f1_weighted": 0.262508 }, { "accuracy": 0.258, "f1": 0.256823, "f1_weighted": 0.256823 }, { "accuracy": 0.2382, "f1": 0.237895, "f1_weighted": 0.237895 }, { "accuracy": 0.227, "f1": 0.225605, "f1_weighted": 0.225605 }, { "accuracy": 0.24, "f1": 0.240337, "f1_weighted": 0.240337 }, { "accuracy": 0.2372, "f1": 0.235932, "f1_weighted": 0.235932 }, { "accuracy": 0.2274, "f1": 0.223735, "f1_weighted": 0.223735 }, { "accuracy": 0.2508, "f1": 0.251068, "f1_weighted": 0.251068 }, { "accuracy": 0.2522, "f1": 0.245546, "f1_weighted": 0.245546 }, { "accuracy": 0.2584, "f1": 0.251398, "f1_weighted": 0.251398 } ], "main_score": 0.24556, "hf_subset": "de", "languages": [ "deu-Latn" ] }, { "accuracy": 0.25924, "f1": 0.256871, "f1_weighted": 0.256871, "scores_per_experiment": [ { "accuracy": 0.2606, "f1": 0.261502, "f1_weighted": 0.261502 }, { "accuracy": 0.2438, "f1": 0.242761, "f1_weighted": 0.242761 }, { "accuracy": 0.2654, "f1": 0.265773, "f1_weighted": 0.265773 }, { "accuracy": 0.2652, "f1": 0.263472, "f1_weighted": 0.263472 }, { "accuracy": 0.2292, "f1": 0.225109, "f1_weighted": 0.225109 }, { "accuracy": 0.254, "f1": 0.24476, "f1_weighted": 0.24476 }, { "accuracy": 0.2754, "f1": 0.275726, "f1_weighted": 0.275726 }, { "accuracy": 0.2748, "f1": 0.275475, "f1_weighted": 0.275475 }, { "accuracy": 0.2722, "f1": 0.264202, "f1_weighted": 0.264202 }, { "accuracy": 0.2518, "f1": 0.249934, "f1_weighted": 0.249934 } ], "main_score": 0.25924, "hf_subset": "es", "languages": [ "spa-Latn" ] }, { "accuracy": 0.25834, "f1": 0.254768, "f1_weighted": 0.254768, "scores_per_experiment": [ { "accuracy": 0.2826, "f1": 0.278518, "f1_weighted": 0.278518 }, { "accuracy": 0.2882, "f1": 0.284125, "f1_weighted": 0.284125 }, { "accuracy": 0.26, "f1": 0.256182, "f1_weighted": 0.256182 }, { "accuracy": 0.2226, "f1": 0.218793, "f1_weighted": 0.218793 }, { "accuracy": 0.2788, "f1": 0.276983, "f1_weighted": 0.276983 }, { "accuracy": 0.2526, "f1": 0.250364, "f1_weighted": 0.250364 }, { "accuracy": 0.25, "f1": 0.237624, "f1_weighted": 0.237624 }, { "accuracy": 0.2586, "f1": 0.258495, "f1_weighted": 0.258495 }, { "accuracy": 0.2284, "f1": 0.228804, "f1_weighted": 0.228804 }, { "accuracy": 0.2616, "f1": 0.257796, "f1_weighted": 0.257796 } ], "main_score": 0.25834, "hf_subset": "fr", "languages": [ "fra-Latn" ] }, { "accuracy": 0.21844, "f1": 0.212045, "f1_weighted": 0.212045, "scores_per_experiment": [ { "accuracy": 0.2204, "f1": 0.212935, "f1_weighted": 0.212935 }, { "accuracy": 0.218, "f1": 0.211674, "f1_weighted": 0.211674 }, { "accuracy": 0.2256, "f1": 0.220435, "f1_weighted": 0.220435 }, { "accuracy": 0.2124, "f1": 0.211124, "f1_weighted": 0.211124 }, { "accuracy": 0.2246, "f1": 0.216061, "f1_weighted": 0.216061 }, { "accuracy": 0.2066, "f1": 0.200365, "f1_weighted": 0.200365 }, { "accuracy": 0.2172, "f1": 0.214707, "f1_weighted": 0.214707 }, { "accuracy": 0.2384, "f1": 0.233426, "f1_weighted": 0.233426 }, { "accuracy": 0.2104, "f1": 0.197808, "f1_weighted": 0.197808 }, { "accuracy": 0.2108, "f1": 0.201916, "f1_weighted": 0.201916 } ], "main_score": 0.21844, "hf_subset": "ja", "languages": [ "jpn-Jpan" ] }, { "accuracy": 0.22808, "f1": 0.221118, "f1_weighted": 0.221118, "scores_per_experiment": [ { "accuracy": 0.2176, "f1": 0.212364, "f1_weighted": 0.212364 }, { "accuracy": 0.2252, "f1": 0.217753, "f1_weighted": 0.217753 }, { "accuracy": 0.2374, "f1": 0.236701, "f1_weighted": 0.236701 }, { "accuracy": 0.2278, "f1": 0.220983, "f1_weighted": 0.220983 }, { "accuracy": 0.2126, "f1": 0.206382, "f1_weighted": 0.206382 }, { "accuracy": 0.2268, "f1": 0.225584, "f1_weighted": 0.225584 }, { "accuracy": 0.2486, "f1": 0.231943, "f1_weighted": 0.231943 }, { "accuracy": 0.2156, "f1": 0.20764, "f1_weighted": 0.20764 }, { "accuracy": 0.239, "f1": 0.226866, "f1_weighted": 0.226866 }, { "accuracy": 0.2302, "f1": 0.224961, "f1_weighted": 0.224961 } ], "main_score": 0.22808, "hf_subset": "zh", "languages": [ "cmn-Hans" ] } ] }, "evaluation_time": 88.52297878265381, "kg_co2_emissions": 0.004523072112462735 }