{ "dataset_revision": "e8379541af4e31359cca9fbcf4b00f2671dba205", "task_name": "AmazonCounterfactualClassification", "mteb_version": "1.18.0", "scores": { "validation": [ { "accuracy": 0.953153, "f1": 0.877689, "f1_weighted": 0.954432, "ap": 0.630213, "ap_weighted": 0.630213, "scores_per_experiment": [ { "accuracy": 0.954955, "f1": 0.881761, "f1_weighted": 0.956072, "ap": 0.640596, "ap_weighted": 0.640596 }, { "accuracy": 0.953453, "f1": 0.878572, "f1_weighted": 0.954742, "ap": 0.632385, "ap_weighted": 0.632385 }, { "accuracy": 0.954955, "f1": 0.881761, "f1_weighted": 0.956072, "ap": 0.640596, "ap_weighted": 0.640596 }, { "accuracy": 0.947447, "f1": 0.866184, "f1_weighted": 0.949483, "ap": 0.601591, "ap_weighted": 0.601591 }, { "accuracy": 0.954955, "f1": 0.880273, "f1_weighted": 0.955807, "ap": 0.636501, "ap_weighted": 0.636501 }, { "accuracy": 0.951952, "f1": 0.875421, "f1_weighted": 0.953418, "ap": 0.624386, "ap_weighted": 0.624386 }, { "accuracy": 0.953453, "f1": 0.878572, "f1_weighted": 0.954742, "ap": 0.632385, "ap_weighted": 0.632385 }, { "accuracy": 0.954955, "f1": 0.881761, "f1_weighted": 0.956072, "ap": 0.640596, "ap_weighted": 0.640596 }, { "accuracy": 0.954955, "f1": 0.880273, "f1_weighted": 0.955807, "ap": 0.636501, "ap_weighted": 0.636501 }, { "accuracy": 0.95045, "f1": 0.872306, "f1_weighted": 0.9521, "ap": 0.616593, "ap_weighted": 0.616593 } ], "main_score": 0.953153, "hf_subset": "en-ext", "languages": [ "eng-Latn" ] }, { "accuracy": 0.930746, "f1": 0.887511, "f1_weighted": 0.933099, "ap": 0.692072, "ap_weighted": 0.692072, "scores_per_experiment": [ { "accuracy": 0.931343, "f1": 0.88827, "f1_weighted": 0.933621, "ap": 0.693573, "ap_weighted": 0.693573 }, { "accuracy": 0.934328, "f1": 0.892478, "f1_weighted": 0.936331, "ap": 0.703509, "ap_weighted": 0.703509 }, { "accuracy": 0.934328, "f1": 0.892478, "f1_weighted": 0.936331, "ap": 0.703509, "ap_weighted": 0.703509 }, { "accuracy": 0.931343, "f1": 0.88827, "f1_weighted": 0.933621, "ap": 0.693573, "ap_weighted": 0.693573 }, { "accuracy": 0.925373, "f1": 0.879995, "f1_weighted": 0.928237, "ap": 0.67454, "ap_weighted": 0.67454 }, { "accuracy": 0.934328, "f1": 0.892478, "f1_weighted": 0.936331, "ap": 0.703509, "ap_weighted": 0.703509 }, { "accuracy": 0.934328, "f1": 0.892478, "f1_weighted": 0.936331, "ap": 0.703509, "ap_weighted": 0.703509 }, { "accuracy": 0.934328, "f1": 0.892478, "f1_weighted": 0.936331, "ap": 0.703509, "ap_weighted": 0.703509 }, { "accuracy": 0.931343, "f1": 0.88827, "f1_weighted": 0.933621, "ap": 0.693573, "ap_weighted": 0.693573 }, { "accuracy": 0.916418, "f1": 0.867917, "f1_weighted": 0.920241, "ap": 0.64792, "ap_weighted": 0.64792 } ], "main_score": 0.930746, "hf_subset": "en", "languages": [ "eng-Latn" ] }, { "accuracy": 0.807082, "f1": 0.797166, "f1_weighted": 0.8148, "ap": 0.906948, "ap_weighted": 0.906948, "scores_per_experiment": [ { "accuracy": 0.787554, "f1": 0.779131, "f1_weighted": 0.796161, "ap": 0.901513, "ap_weighted": 0.901513 }, { "accuracy": 0.824034, "f1": 0.811978, "f1_weighted": 0.830777, "ap": 0.907251, "ap_weighted": 0.907251 }, { "accuracy": 0.806867, "f1": 0.797544, "f1_weighted": 0.814698, "ap": 0.909887, "ap_weighted": 0.909887 }, { "accuracy": 0.793991, "f1": 0.785253, "f1_weighted": 0.802358, "ap": 0.904304, "ap_weighted": 0.904304 }, { "accuracy": 0.800429, "f1": 0.79139, "f1_weighted": 0.808536, "ap": 0.907096, "ap_weighted": 0.907096 }, { "accuracy": 0.802575, "f1": 0.79344, "f1_weighted": 0.810592, "ap": 0.908026, "ap_weighted": 0.908026 }, { "accuracy": 0.824034, "f1": 0.811537, "f1_weighted": 0.8307, "ap": 0.9053, "ap_weighted": 0.9053 }, { "accuracy": 0.824034, "f1": 0.813247, "f1_weighted": 0.830969, "ap": 0.913232, "ap_weighted": 0.913232 }, { "accuracy": 0.804721, "f1": 0.795096, "f1_weighted": 0.812631, "ap": 0.906899, "ap_weighted": 0.906899 }, { "accuracy": 0.802575, "f1": 0.793045, "f1_weighted": 0.810581, "ap": 0.905969, "ap_weighted": 0.905969 } ], "main_score": 0.807082, "hf_subset": "de", "languages": [ "deu-Latn" ] }, { "accuracy": 0.769099, "f1": 0.600737, "f1_weighted": 0.808363, "ap": 0.183424, "ap_weighted": 0.183424, "scores_per_experiment": [ { "accuracy": 0.770386, "f1": 0.606005, "f1_weighted": 0.810253, "ap": 0.190116, "ap_weighted": 0.190116 }, { "accuracy": 0.757511, "f1": 0.591631, "f1_weighted": 0.800517, "ap": 0.177191, "ap_weighted": 0.177191 }, { "accuracy": 0.733906, "f1": 0.573109, "f1_weighted": 0.783382, "ap": 0.165817, "ap_weighted": 0.165817 }, { "accuracy": 0.751073, "f1": 0.586475, "f1_weighted": 0.795862, "ap": 0.173892, "ap_weighted": 0.173892 }, { "accuracy": 0.7897, "f1": 0.614811, "f1_weighted": 0.823118, "ap": 0.19028, "ap_weighted": 0.19028 }, { "accuracy": 0.798283, "f1": 0.622483, "f1_weighted": 0.829242, "ap": 0.196143, "ap_weighted": 0.196143 }, { "accuracy": 0.819742, "f1": 0.638413, "f1_weighted": 0.84392, "ap": 0.206203, "ap_weighted": 0.206203 }, { "accuracy": 0.684549, "f1": 0.543548, "f1_weighted": 0.747156, "ap": 0.155858, "ap_weighted": 0.155858 }, { "accuracy": 0.796137, "f1": 0.612058, "f1_weighted": 0.82653, "ap": 0.182306, "ap_weighted": 0.182306 }, { "accuracy": 0.7897, "f1": 0.618836, "f1_weighted": 0.823653, "ap": 0.196437, "ap_weighted": 0.196437 } ], "main_score": 0.769099, "hf_subset": "ja", "languages": [ "jpn-Jpan" ] } ], "test": [ { "accuracy": 0.957871, "f1": 0.893942, "f1_weighted": 0.959124, "ap": 0.675322, "ap_weighted": 0.675322, "scores_per_experiment": [ { "accuracy": 0.958021, "f1": 0.894226, "f1_weighted": 0.959253, "ap": 0.675952, "ap_weighted": 0.675952 }, { "accuracy": 0.957271, "f1": 0.892652, "f1_weighted": 0.958582, "ap": 0.671812, "ap_weighted": 0.671812 }, { "accuracy": 0.95952, "f1": 0.89679, "f1_weighted": 0.960485, "ap": 0.682611, "ap_weighted": 0.682611 }, { "accuracy": 0.955022, "f1": 0.88926, "f1_weighted": 0.956814, "ap": 0.663741, "ap_weighted": 0.663741 }, { "accuracy": 0.96027, "f1": 0.899002, "f1_weighted": 0.961272, "ap": 0.688692, "ap_weighted": 0.688692 }, { "accuracy": 0.957271, "f1": 0.892652, "f1_weighted": 0.958582, "ap": 0.671812, "ap_weighted": 0.671812 }, { "accuracy": 0.957271, "f1": 0.892652, "f1_weighted": 0.958582, "ap": 0.671812, "ap_weighted": 0.671812 }, { "accuracy": 0.958771, "f1": 0.895809, "f1_weighted": 0.959924, "ap": 0.680145, "ap_weighted": 0.680145 }, { "accuracy": 0.96027, "f1": 0.898398, "f1_weighted": 0.961161, "ap": 0.686951, "ap_weighted": 0.686951 }, { "accuracy": 0.955022, "f1": 0.887983, "f1_weighted": 0.956582, "ap": 0.659695, "ap_weighted": 0.659695 } ], "main_score": 0.957871, "hf_subset": "en-ext", "languages": [ "eng-Latn" ] }, { "accuracy": 0.93791, "f1": 0.907772, "f1_weighted": 0.939875, "ap": 0.749295, "ap_weighted": 0.749295, "scores_per_experiment": [ { "accuracy": 0.935821, "f1": 0.905149, "f1_weighted": 0.937995, "ap": 0.74288, "ap_weighted": 0.74288 }, { "accuracy": 0.938806, "f1": 0.908639, "f1_weighted": 0.940608, "ap": 0.751214, "ap_weighted": 0.751214 }, { "accuracy": 0.941791, "f1": 0.913096, "f1_weighted": 0.943505, "ap": 0.761866, "ap_weighted": 0.761866 }, { "accuracy": 0.940299, "f1": 0.911095, "f1_weighted": 0.942124, "ap": 0.757028, "ap_weighted": 0.757028 }, { "accuracy": 0.934328, "f1": 0.903186, "f1_weighted": 0.936623, "ap": 0.738283, "ap_weighted": 0.738283 }, { "accuracy": 0.943284, "f1": 0.914666, "f1_weighted": 0.944759, "ap": 0.765872, "ap_weighted": 0.765872 }, { "accuracy": 0.940299, "f1": 0.910638, "f1_weighted": 0.941989, "ap": 0.756036, "ap_weighted": 0.756036 }, { "accuracy": 0.940299, "f1": 0.910174, "f1_weighted": 0.941851, "ap": 0.75508, "ap_weighted": 0.75508 }, { "accuracy": 0.935821, "f1": 0.905149, "f1_weighted": 0.937995, "ap": 0.74288, "ap_weighted": 0.74288 }, { "accuracy": 0.928358, "f1": 0.89593, "f1_weighted": 0.931306, "ap": 0.721806, "ap_weighted": 0.721806 } ], "main_score": 0.93791, "hf_subset": "en", "languages": [ "eng-Latn" ] }, { "accuracy": 0.759957, "f1": 0.752829, "f1_weighted": 0.769118, "ap": 0.886768, "ap_weighted": 0.886768, "scores_per_experiment": [ { "accuracy": 0.746253, "f1": 0.740673, "f1_weighted": 0.755579, "ap": 0.884908, "ap_weighted": 0.884908 }, { "accuracy": 0.780514, "f1": 0.769855, "f1_weighted": 0.789263, "ap": 0.883957, "ap_weighted": 0.883957 }, { "accuracy": 0.754818, "f1": 0.74859, "f1_weighted": 0.764096, "ap": 0.88761, "ap_weighted": 0.88761 }, { "accuracy": 0.749465, "f1": 0.743706, "f1_weighted": 0.75876, "ap": 0.886311, "ap_weighted": 0.886311 }, { "accuracy": 0.753747, "f1": 0.747579, "f1_weighted": 0.763041, "ap": 0.887143, "ap_weighted": 0.887143 }, { "accuracy": 0.754818, "f1": 0.74859, "f1_weighted": 0.764096, "ap": 0.88761, "ap_weighted": 0.88761 }, { "accuracy": 0.781585, "f1": 0.771523, "f1_weighted": 0.790311, "ap": 0.887294, "ap_weighted": 0.887294 }, { "accuracy": 0.769807, "f1": 0.761432, "f1_weighted": 0.778948, "ap": 0.887058, "ap_weighted": 0.887058 }, { "accuracy": 0.754818, "f1": 0.74859, "f1_weighted": 0.764096, "ap": 0.88761, "ap_weighted": 0.88761 }, { "accuracy": 0.753747, "f1": 0.747751, "f1_weighted": 0.762991, "ap": 0.888181, "ap_weighted": 0.888181 } ], "main_score": 0.759957, "hf_subset": "de", "languages": [ "deu-Latn" ] }, { "accuracy": 0.758565, "f1": 0.61441, "f1_weighted": 0.80115, "ap": 0.214293, "ap_weighted": 0.214293, "scores_per_experiment": [ { "accuracy": 0.762313, "f1": 0.625901, "f1_weighted": 0.805364, "ap": 0.231736, "ap_weighted": 0.231736 }, { "accuracy": 0.744111, "f1": 0.598071, "f1_weighted": 0.790543, "ap": 0.19712, "ap_weighted": 0.19712 }, { "accuracy": 0.724839, "f1": 0.582783, "f1_weighted": 0.776188, "ap": 0.186983, "ap_weighted": 0.186983 }, { "accuracy": 0.749465, "f1": 0.604065, "f1_weighted": 0.794677, "ap": 0.202963, "ap_weighted": 0.202963 }, { "accuracy": 0.785867, "f1": 0.630187, "f1_weighted": 0.820805, "ap": 0.218169, "ap_weighted": 0.218169 }, { "accuracy": 0.779443, "f1": 0.624472, "f1_weighted": 0.81612, "ap": 0.213533, "ap_weighted": 0.213533 }, { "accuracy": 0.796574, "f1": 0.64352, "f1_weighted": 0.829085, "ap": 0.232894, "ap_weighted": 0.232894 }, { "accuracy": 0.669165, "f1": 0.56138, "f1_weighted": 0.734115, "ap": 0.19831, "ap_weighted": 0.19831 }, { "accuracy": 0.782655, "f1": 0.621911, "f1_weighted": 0.81776, "ap": 0.206733, "ap_weighted": 0.206733 }, { "accuracy": 0.791221, "f1": 0.651809, "f1_weighted": 0.82684, "ap": 0.254493, "ap_weighted": 0.254493 } ], "main_score": 0.758565, "hf_subset": "ja", "languages": [ "jpn-Jpan" ] } ] }, "evaluation_time": 93.66645741462708, "kg_co2_emissions": 0.003095242328378673 }