{ "dataset_revision": "e8379541af4e31359cca9fbcf4b00f2671dba205", "task_name": "AmazonCounterfactualClassification", "mteb_version": "1.18.0", "scores": { "validation": [ { "accuracy": 0.751502, "f1": 0.62035, "f1_weighted": 0.798378, "ap": 0.233164, "ap_weighted": 0.233164, "scores_per_experiment": [ { "accuracy": 0.807808, "f1": 0.661327, "f1_weighted": 0.839244, "ap": 0.257286, "ap_weighted": 0.257286 }, { "accuracy": 0.753754, "f1": 0.623225, "f1_weighted": 0.800371, "ap": 0.23613, "ap_weighted": 0.23613 }, { "accuracy": 0.758258, "f1": 0.625014, "f1_weighted": 0.803568, "ap": 0.234977, "ap_weighted": 0.234977 }, { "accuracy": 0.695195, "f1": 0.564343, "f1_weighted": 0.755065, "ap": 0.180172, "ap_weighted": 0.180172 }, { "accuracy": 0.746246, "f1": 0.622802, "f1_weighted": 0.79517, "ap": 0.24383, "ap_weighted": 0.24383 }, { "accuracy": 0.728228, "f1": 0.605949, "f1_weighted": 0.781293, "ap": 0.227702, "ap_weighted": 0.227702 }, { "accuracy": 0.744745, "f1": 0.61563, "f1_weighted": 0.793581, "ap": 0.229986, "ap_weighted": 0.229986 }, { "accuracy": 0.744745, "f1": 0.607311, "f1_weighted": 0.792881, "ap": 0.213394, "ap_weighted": 0.213394 }, { "accuracy": 0.785285, "f1": 0.65322, "f1_weighted": 0.824166, "ap": 0.265484, "ap_weighted": 0.265484 }, { "accuracy": 0.750751, "f1": 0.624674, "f1_weighted": 0.798438, "ap": 0.24268, "ap_weighted": 0.24268 } ], "main_score": 0.751502, "hf_subset": "en-ext", "languages": [ "eng-Latn" ] }, { "accuracy": 0.725075, "f1": 0.650973, "f1_weighted": 0.75559, "ap": 0.318647, "ap_weighted": 0.318647, "scores_per_experiment": [ { "accuracy": 0.671642, "f1": 0.595943, "f1_weighted": 0.710274, "ap": 0.262718, "ap_weighted": 0.262718 }, { "accuracy": 0.737313, "f1": 0.672532, "f1_weighted": 0.767748, "ap": 0.350276, "ap_weighted": 0.350276 }, { "accuracy": 0.689552, "f1": 0.629362, "f1_weighted": 0.727004, "ap": 0.309598, "ap_weighted": 0.309598 }, { "accuracy": 0.713433, "f1": 0.630107, "f1_weighted": 0.744877, "ap": 0.287513, "ap_weighted": 0.287513 }, { "accuracy": 0.713433, "f1": 0.642762, "f1_weighted": 0.746634, "ap": 0.31157, "ap_weighted": 0.31157 }, { "accuracy": 0.767164, "f1": 0.682488, "f1_weighted": 0.789679, "ap": 0.340202, "ap_weighted": 0.340202 }, { "accuracy": 0.770149, "f1": 0.687867, "f1_weighted": 0.792633, "ap": 0.348389, "ap_weighted": 0.348389 }, { "accuracy": 0.752239, "f1": 0.660704, "f1_weighted": 0.775912, "ap": 0.31132, "ap_weighted": 0.31132 }, { "accuracy": 0.695522, "f1": 0.644418, "f1_weighted": 0.732543, "ap": 0.337799, "ap_weighted": 0.337799 }, { "accuracy": 0.740299, "f1": 0.66355, "f1_weighted": 0.7686, "ap": 0.327084, "ap_weighted": 0.327084 } ], "main_score": 0.725075, "hf_subset": "en", "languages": [ "eng-Latn" ] }, { "accuracy": 0.743991, "f1": 0.724227, "f1_weighted": 0.75308, "ap": 0.835725, "ap_weighted": 0.835725, "scores_per_experiment": [ { "accuracy": 0.733906, "f1": 0.716967, "f1_weighted": 0.744306, "ap": 0.835412, "ap_weighted": 0.835412 }, { "accuracy": 0.766094, "f1": 0.747959, "f1_weighted": 0.774654, "ap": 0.852103, "ap_weighted": 0.852103 }, { "accuracy": 0.781116, "f1": 0.755055, "f1_weighted": 0.786602, "ap": 0.842795, "ap_weighted": 0.842795 }, { "accuracy": 0.751073, "f1": 0.736384, "f1_weighted": 0.760954, "ap": 0.852346, "ap_weighted": 0.852346 }, { "accuracy": 0.736052, "f1": 0.716969, "f1_weighted": 0.745987, "ap": 0.831679, "ap_weighted": 0.831679 }, { "accuracy": 0.740343, "f1": 0.71951, "f1_weighted": 0.749694, "ap": 0.830436, "ap_weighted": 0.830436 }, { "accuracy": 0.723176, "f1": 0.695278, "f1_weighted": 0.731684, "ap": 0.807801, "ap_weighted": 0.807801 }, { "accuracy": 0.744635, "f1": 0.727455, "f1_weighted": 0.754474, "ap": 0.841479, "ap_weighted": 0.841479 }, { "accuracy": 0.761803, "f1": 0.736268, "f1_weighted": 0.76867, "ap": 0.833349, "ap_weighted": 0.833349 }, { "accuracy": 0.701717, "f1": 0.690425, "f1_weighted": 0.71377, "ap": 0.829856, "ap_weighted": 0.829856 } ], "main_score": 0.743991, "hf_subset": "de", "languages": [ "deu-Latn" ] }, { "accuracy": 0.645923, "f1": 0.51818, "f1_weighted": 0.71689, "ap": 0.147127, "ap_weighted": 0.147127, "scores_per_experiment": [ { "accuracy": 0.590129, "f1": 0.481882, "f1_weighted": 0.671949, "ap": 0.133113, "ap_weighted": 0.133113 }, { "accuracy": 0.66309, "f1": 0.528459, "f1_weighted": 0.730676, "ap": 0.148751, "ap_weighted": 0.148751 }, { "accuracy": 0.603004, "f1": 0.485089, "f1_weighted": 0.682848, "ap": 0.129286, "ap_weighted": 0.129286 }, { "accuracy": 0.690987, "f1": 0.554399, "f1_weighted": 0.752399, "ap": 0.167456, "ap_weighted": 0.167456 }, { "accuracy": 0.699571, "f1": 0.566776, "f1_weighted": 0.759277, "ap": 0.180877, "ap_weighted": 0.180877 }, { "accuracy": 0.693133, "f1": 0.555969, "f1_weighted": 0.754036, "ap": 0.168309, "ap_weighted": 0.168309 }, { "accuracy": 0.630901, "f1": 0.506453, "f1_weighted": 0.705357, "ap": 0.139524, "ap_weighted": 0.139524 }, { "accuracy": 0.61588, "f1": 0.481859, "f1_weighted": 0.693352, "ap": 0.11931, "ap_weighted": 0.11931 }, { "accuracy": 0.609442, "f1": 0.486523, "f1_weighted": 0.688153, "ap": 0.127395, "ap_weighted": 0.127395 }, { "accuracy": 0.66309, "f1": 0.534389, "f1_weighted": 0.730855, "ap": 0.157248, "ap_weighted": 0.157248 } ], "main_score": 0.645923, "hf_subset": "ja", "languages": [ "jpn-Jpan" ] } ], "test": [ { "accuracy": 0.770465, "f1": 0.641933, "f1_weighted": 0.811542, "ap": 0.256833, "ap_weighted": 0.256833, "scores_per_experiment": [ { "accuracy": 0.833583, "f1": 0.70009, "f1_weighted": 0.858482, "ap": 0.311613, "ap_weighted": 0.311613 }, { "accuracy": 0.781109, "f1": 0.657212, "f1_weighted": 0.820349, "ap": 0.277849, "ap_weighted": 0.277849 }, { "accuracy": 0.781109, "f1": 0.65024, "f1_weighted": 0.819601, "ap": 0.261874, "ap_weighted": 0.261874 }, { "accuracy": 0.715142, "f1": 0.580371, "f1_weighted": 0.768623, "ap": 0.190602, "ap_weighted": 0.190602 }, { "accuracy": 0.766117, "f1": 0.644618, "f1_weighted": 0.809109, "ap": 0.267303, "ap_weighted": 0.267303 }, { "accuracy": 0.724138, "f1": 0.606217, "f1_weighted": 0.776799, "ap": 0.231167, "ap_weighted": 0.231167 }, { "accuracy": 0.755622, "f1": 0.623626, "f1_weighted": 0.800067, "ap": 0.233538, "ap_weighted": 0.233538 }, { "accuracy": 0.775862, "f1": 0.644452, "f1_weighted": 0.81556, "ap": 0.255252, "ap_weighted": 0.255252 }, { "accuracy": 0.784858, "f1": 0.653684, "f1_weighted": 0.822404, "ap": 0.265114, "ap_weighted": 0.265114 }, { "accuracy": 0.787106, "f1": 0.658821, "f1_weighted": 0.824432, "ap": 0.274021, "ap_weighted": 0.274021 } ], "main_score": 0.770465, "hf_subset": "en-ext", "languages": [ "eng-Latn" ] }, { "accuracy": 0.731194, "f1": 0.671949, "f1_weighted": 0.756408, "ap": 0.36058, "ap_weighted": 0.36058, "scores_per_experiment": [ { "accuracy": 0.710448, "f1": 0.652296, "f1_weighted": 0.738887, "ap": 0.338626, "ap_weighted": 0.338626 }, { "accuracy": 0.764179, "f1": 0.713409, "f1_weighted": 0.786864, "ap": 0.416698, "ap_weighted": 0.416698 }, { "accuracy": 0.677612, "f1": 0.625466, "f1_weighted": 0.710568, "ap": 0.318093, "ap_weighted": 0.318093 }, { "accuracy": 0.728358, "f1": 0.661368, "f1_weighted": 0.753086, "ap": 0.339595, "ap_weighted": 0.339595 }, { "accuracy": 0.731343, "f1": 0.667277, "f1_weighted": 0.756185, "ap": 0.34849, "ap_weighted": 0.34849 }, { "accuracy": 0.749254, "f1": 0.682066, "f1_weighted": 0.771068, "ap": 0.361675, "ap_weighted": 0.361675 }, { "accuracy": 0.81194, "f1": 0.743548, "f1_weighted": 0.824196, "ap": 0.435413, "ap_weighted": 0.435413 }, { "accuracy": 0.747761, "f1": 0.683979, "f1_weighted": 0.770435, "ap": 0.367008, "ap_weighted": 0.367008 }, { "accuracy": 0.673134, "f1": 0.626385, "f1_weighted": 0.706865, "ap": 0.325215, "ap_weighted": 0.325215 }, { "accuracy": 0.71791, "f1": 0.6637, "f1_weighted": 0.745922, "ap": 0.354984, "ap_weighted": 0.354984 } ], "main_score": 0.731194, "hf_subset": "en", "languages": [ "eng-Latn" ] }, { "accuracy": 0.731906, "f1": 0.713332, "f1_weighted": 0.741623, "ap": 0.829569, "ap_weighted": 0.829569, "scores_per_experiment": [ { "accuracy": 0.730193, "f1": 0.711792, "f1_weighted": 0.740329, "ap": 0.827813, "ap_weighted": 0.827813 }, { "accuracy": 0.744111, "f1": 0.726817, "f1_weighted": 0.753752, "ap": 0.839024, "ap_weighted": 0.839024 }, { "accuracy": 0.739829, "f1": 0.715912, "f1_weighted": 0.748213, "ap": 0.822276, "ap_weighted": 0.822276 }, { "accuracy": 0.750535, "f1": 0.736301, "f1_weighted": 0.760309, "ap": 0.851452, "ap_weighted": 0.851452 }, { "accuracy": 0.734475, "f1": 0.715704, "f1_weighted": 0.74433, "ap": 0.829601, "ap_weighted": 0.829601 }, { "accuracy": 0.77409, "f1": 0.757103, "f1_weighted": 0.782274, "ap": 0.858175, "ap_weighted": 0.858175 }, { "accuracy": 0.705567, "f1": 0.676721, "f1_weighted": 0.714563, "ap": 0.794859, "ap_weighted": 0.794859 }, { "accuracy": 0.719486, "f1": 0.704249, "f1_weighted": 0.730555, "ap": 0.828666, "ap_weighted": 0.828666 }, { "accuracy": 0.740899, "f1": 0.718401, "f1_weighted": 0.749592, "ap": 0.825603, "ap_weighted": 0.825603 }, { "accuracy": 0.679872, "f1": 0.670317, "f1_weighted": 0.69231, "ap": 0.818226, "ap_weighted": 0.818226 } ], "main_score": 0.731906, "hf_subset": "de", "languages": [ "deu-Latn" ] }, { "accuracy": 0.643148, "f1": 0.523682, "f1_weighted": 0.71265, "ap": 0.157708, "ap_weighted": 0.157708, "scores_per_experiment": [ { "accuracy": 0.573876, "f1": 0.482766, "f1_weighted": 0.655224, "ap": 0.146772, "ap_weighted": 0.146772 }, { "accuracy": 0.647752, "f1": 0.539403, "f1_weighted": 0.716875, "ap": 0.17769, "ap_weighted": 0.17769 }, { "accuracy": 0.610278, "f1": 0.495165, "f1_weighted": 0.686676, "ap": 0.13855, "ap_weighted": 0.13855 }, { "accuracy": 0.675589, "f1": 0.558496, "f1_weighted": 0.739126, "ap": 0.186612, "ap_weighted": 0.186612 }, { "accuracy": 0.705567, "f1": 0.572568, "f1_weighted": 0.761984, "ap": 0.185355, "ap_weighted": 0.185355 }, { "accuracy": 0.691649, "f1": 0.560654, "f1_weighted": 0.751238, "ap": 0.176828, "ap_weighted": 0.176828 }, { "accuracy": 0.647752, "f1": 0.527627, "f1_weighted": 0.716868, "ap": 0.158462, "ap_weighted": 0.158462 }, { "accuracy": 0.61242, "f1": 0.492513, "f1_weighted": 0.688484, "ap": 0.134043, "ap_weighted": 0.134043 }, { "accuracy": 0.608137, "f1": 0.484081, "f1_weighted": 0.685063, "ap": 0.126819, "ap_weighted": 0.126819 }, { "accuracy": 0.658458, "f1": 0.523551, "f1_weighted": 0.724962, "ap": 0.14595, "ap_weighted": 0.14595 } ], "main_score": 0.643148, "hf_subset": "ja", "languages": [ "jpn-Jpan" ] } ] }, "evaluation_time": 19.77475619316101, "kg_co2_emissions": 0.0006082274759405994 }