{ "dataset_revision": "a450ba4b1b6d2216c3674d3e576b2e85ce729add", "task_name": "NusaX-senti", "mteb_version": "1.18.0", "scores": { "test": [ { "accuracy": 0.56575, "f1": 0.546036, "f1_weighted": 0.555892, "scores_per_experiment": [ { "accuracy": 0.6225, "f1": 0.603291, "f1_weighted": 0.607076 }, { "accuracy": 0.5425, "f1": 0.518068, "f1_weighted": 0.534565 }, { "accuracy": 0.55, "f1": 0.537932, "f1_weighted": 0.540935 }, { "accuracy": 0.5375, "f1": 0.494886, "f1_weighted": 0.503054 }, { "accuracy": 0.6, "f1": 0.586066, "f1_weighted": 0.597254 }, { "accuracy": 0.54, "f1": 0.519943, "f1_weighted": 0.539765 }, { "accuracy": 0.5275, "f1": 0.493122, "f1_weighted": 0.504713 }, { "accuracy": 0.6375, "f1": 0.6299, "f1_weighted": 0.637896 }, { "accuracy": 0.5825, "f1": 0.564262, "f1_weighted": 0.57327 }, { "accuracy": 0.5175, "f1": 0.51289, "f1_weighted": 0.520389 } ], "main_score": 0.56575, "hf_subset": "ace", "languages": [ "ace-Latn" ] }, { "accuracy": 0.549, "f1": 0.531942, "f1_weighted": 0.541208, "scores_per_experiment": [ { "accuracy": 0.5875, "f1": 0.57599, "f1_weighted": 0.577741 }, { "accuracy": 0.545, "f1": 0.516055, "f1_weighted": 0.529799 }, { "accuracy": 0.515, "f1": 0.50206, "f1_weighted": 0.517312 }, { "accuracy": 0.5525, "f1": 0.529687, "f1_weighted": 0.536965 }, { "accuracy": 0.5625, "f1": 0.548648, "f1_weighted": 0.54732 }, { "accuracy": 0.535, "f1": 0.517684, "f1_weighted": 0.533656 }, { "accuracy": 0.505, "f1": 0.463186, "f1_weighted": 0.480418 }, { "accuracy": 0.535, "f1": 0.534062, "f1_weighted": 0.543045 }, { "accuracy": 0.57, "f1": 0.56109, "f1_weighted": 0.566828 }, { "accuracy": 0.5825, "f1": 0.570963, "f1_weighted": 0.578996 } ], "main_score": 0.549, "hf_subset": "ban", "languages": [ "ban-Latn" ] }, { "accuracy": 0.5445, "f1": 0.519777, "f1_weighted": 0.526024, "scores_per_experiment": [ { "accuracy": 0.54, "f1": 0.517942, "f1_weighted": 0.520549 }, { "accuracy": 0.5375, "f1": 0.50436, "f1_weighted": 0.511662 }, { "accuracy": 0.5375, "f1": 0.519395, "f1_weighted": 0.516541 }, { "accuracy": 0.535, "f1": 0.491263, "f1_weighted": 0.496991 }, { "accuracy": 0.5125, "f1": 0.479786, "f1_weighted": 0.47883 }, { "accuracy": 0.5025, "f1": 0.476778, "f1_weighted": 0.490384 }, { "accuracy": 0.5225, "f1": 0.495502, "f1_weighted": 0.513664 }, { "accuracy": 0.625, "f1": 0.614005, "f1_weighted": 0.625567 }, { "accuracy": 0.6175, "f1": 0.601524, "f1_weighted": 0.603996 }, { "accuracy": 0.515, "f1": 0.497213, "f1_weighted": 0.502059 } ], "main_score": 0.5445, "hf_subset": "bjn", "languages": [ "bjn-Latn" ] }, { "accuracy": 0.57225, "f1": 0.55017, "f1_weighted": 0.557707, "scores_per_experiment": [ { "accuracy": 0.61, "f1": 0.595695, "f1_weighted": 0.600507 }, { "accuracy": 0.585, "f1": 0.549679, "f1_weighted": 0.567112 }, { "accuracy": 0.58, "f1": 0.568477, "f1_weighted": 0.569702 }, { "accuracy": 0.6025, "f1": 0.578026, "f1_weighted": 0.581741 }, { "accuracy": 0.525, "f1": 0.484977, "f1_weighted": 0.480526 }, { "accuracy": 0.5425, "f1": 0.509822, "f1_weighted": 0.527795 }, { "accuracy": 0.59, "f1": 0.566614, "f1_weighted": 0.579549 }, { "accuracy": 0.5925, "f1": 0.585922, "f1_weighted": 0.596018 }, { "accuracy": 0.54, "f1": 0.520244, "f1_weighted": 0.532093 }, { "accuracy": 0.555, "f1": 0.542246, "f1_weighted": 0.542024 } ], "main_score": 0.57225, "hf_subset": "bug", "languages": [ "bug-Latn" ] }, { "accuracy": 0.807, "f1": 0.78768, "f1_weighted": 0.803162, "scores_per_experiment": [ { "accuracy": 0.88, "f1": 0.86652, "f1_weighted": 0.877688 }, { "accuracy": 0.76, "f1": 0.727736, "f1_weighted": 0.748068 }, { "accuracy": 0.83, "f1": 0.819838, "f1_weighted": 0.830129 }, { "accuracy": 0.7775, "f1": 0.746879, "f1_weighted": 0.766474 }, { "accuracy": 0.755, "f1": 0.741475, "f1_weighted": 0.755228 }, { "accuracy": 0.805, "f1": 0.774207, "f1_weighted": 0.797895 }, { "accuracy": 0.79, "f1": 0.773423, "f1_weighted": 0.787137 }, { "accuracy": 0.8125, "f1": 0.800445, "f1_weighted": 0.81292 }, { "accuracy": 0.8325, "f1": 0.811886, "f1_weighted": 0.827762 }, { "accuracy": 0.8275, "f1": 0.814389, "f1_weighted": 0.828321 } ], "main_score": 0.807, "hf_subset": "eng", "languages": [ "eng-Latn" ] }, { "accuracy": 0.56575, "f1": 0.545972, "f1_weighted": 0.55411, "scores_per_experiment": [ { "accuracy": 0.5975, "f1": 0.576271, "f1_weighted": 0.578551 }, { "accuracy": 0.5625, "f1": 0.525321, "f1_weighted": 0.544731 }, { "accuracy": 0.51, "f1": 0.505625, "f1_weighted": 0.511597 }, { "accuracy": 0.5725, "f1": 0.536889, "f1_weighted": 0.539394 }, { "accuracy": 0.5625, "f1": 0.545584, "f1_weighted": 0.555004 }, { "accuracy": 0.535, "f1": 0.510048, "f1_weighted": 0.522056 }, { "accuracy": 0.5825, "f1": 0.555943, "f1_weighted": 0.568183 }, { "accuracy": 0.615, "f1": 0.608467, "f1_weighted": 0.616004 }, { "accuracy": 0.565, "f1": 0.548899, "f1_weighted": 0.550919 }, { "accuracy": 0.555, "f1": 0.54667, "f1_weighted": 0.554658 } ], "main_score": 0.56575, "hf_subset": "ind", "languages": [ "ind-Latn" ] }, { "accuracy": 0.562, "f1": 0.539035, "f1_weighted": 0.545385, "scores_per_experiment": [ { "accuracy": 0.565, "f1": 0.544467, "f1_weighted": 0.543275 }, { "accuracy": 0.5475, "f1": 0.50779, "f1_weighted": 0.52081 }, { "accuracy": 0.5375, "f1": 0.523501, "f1_weighted": 0.527823 }, { "accuracy": 0.585, "f1": 0.545077, "f1_weighted": 0.557763 }, { "accuracy": 0.58, "f1": 0.556356, "f1_weighted": 0.555907 }, { "accuracy": 0.5375, "f1": 0.516784, "f1_weighted": 0.527891 }, { "accuracy": 0.545, "f1": 0.515383, "f1_weighted": 0.529887 }, { "accuracy": 0.5825, "f1": 0.576455, "f1_weighted": 0.579707 }, { "accuracy": 0.6075, "f1": 0.58768, "f1_weighted": 0.595588 }, { "accuracy": 0.5325, "f1": 0.516855, "f1_weighted": 0.515198 } ], "main_score": 0.562, "hf_subset": "jav", "languages": [ "jav-Latn" ] }, { "accuracy": 0.546, "f1": 0.52814, "f1_weighted": 0.536231, "scores_per_experiment": [ { "accuracy": 0.53, "f1": 0.514671, "f1_weighted": 0.521888 }, { "accuracy": 0.565, "f1": 0.549829, "f1_weighted": 0.55934 }, { "accuracy": 0.535, "f1": 0.511874, "f1_weighted": 0.510309 }, { "accuracy": 0.525, "f1": 0.490567, "f1_weighted": 0.495142 }, { "accuracy": 0.5725, "f1": 0.54781, "f1_weighted": 0.553316 }, { "accuracy": 0.4825, "f1": 0.466982, "f1_weighted": 0.483675 }, { "accuracy": 0.515, "f1": 0.492174, "f1_weighted": 0.505126 }, { "accuracy": 0.6075, "f1": 0.599741, "f1_weighted": 0.608416 }, { "accuracy": 0.5525, "f1": 0.535817, "f1_weighted": 0.543495 }, { "accuracy": 0.575, "f1": 0.571934, "f1_weighted": 0.581603 } ], "main_score": 0.546, "hf_subset": "mad", "languages": [ "mad-Latn" ] }, { "accuracy": 0.5645, "f1": 0.539645, "f1_weighted": 0.545956, "scores_per_experiment": [ { "accuracy": 0.5875, "f1": 0.548856, "f1_weighted": 0.54012 }, { "accuracy": 0.5725, "f1": 0.546859, "f1_weighted": 0.555894 }, { "accuracy": 0.52, "f1": 0.512705, "f1_weighted": 0.506392 }, { "accuracy": 0.5325, "f1": 0.485608, "f1_weighted": 0.494259 }, { "accuracy": 0.5425, "f1": 0.516481, "f1_weighted": 0.525083 }, { "accuracy": 0.58, "f1": 0.558858, "f1_weighted": 0.570172 }, { "accuracy": 0.5175, "f1": 0.471153, "f1_weighted": 0.489658 }, { "accuracy": 0.6175, "f1": 0.603237, "f1_weighted": 0.616623 }, { "accuracy": 0.61, "f1": 0.596977, "f1_weighted": 0.603923 }, { "accuracy": 0.565, "f1": 0.555715, "f1_weighted": 0.557435 } ], "main_score": 0.5645, "hf_subset": "min", "languages": [ "min-Latn" ] }, { "accuracy": 0.55825, "f1": 0.53861, "f1_weighted": 0.54465, "scores_per_experiment": [ { "accuracy": 0.585, "f1": 0.571151, "f1_weighted": 0.570792 }, { "accuracy": 0.555, "f1": 0.528525, "f1_weighted": 0.544747 }, { "accuracy": 0.47, "f1": 0.468533, "f1_weighted": 0.467267 }, { "accuracy": 0.5625, "f1": 0.538493, "f1_weighted": 0.548243 }, { "accuracy": 0.595, "f1": 0.568526, "f1_weighted": 0.570335 }, { "accuracy": 0.5225, "f1": 0.508616, "f1_weighted": 0.50941 }, { "accuracy": 0.53, "f1": 0.498711, "f1_weighted": 0.51376 }, { "accuracy": 0.5325, "f1": 0.524451, "f1_weighted": 0.534194 }, { "accuracy": 0.625, "f1": 0.604994, "f1_weighted": 0.609689 }, { "accuracy": 0.605, "f1": 0.574101, "f1_weighted": 0.578064 } ], "main_score": 0.55825, "hf_subset": "nij", "languages": [ "nij-Latn" ] }, { "accuracy": 0.5645, "f1": 0.544829, "f1_weighted": 0.55205, "scores_per_experiment": [ { "accuracy": 0.5875, "f1": 0.558335, "f1_weighted": 0.560566 }, { "accuracy": 0.57, "f1": 0.541358, "f1_weighted": 0.562015 }, { "accuracy": 0.5325, "f1": 0.525813, "f1_weighted": 0.526328 }, { "accuracy": 0.57, "f1": 0.53253, "f1_weighted": 0.543818 }, { "accuracy": 0.5825, "f1": 0.561485, "f1_weighted": 0.565588 }, { "accuracy": 0.5275, "f1": 0.509797, "f1_weighted": 0.521699 }, { "accuracy": 0.56, "f1": 0.530047, "f1_weighted": 0.541937 }, { "accuracy": 0.58, "f1": 0.573427, "f1_weighted": 0.577358 }, { "accuracy": 0.5775, "f1": 0.558972, "f1_weighted": 0.557854 }, { "accuracy": 0.5575, "f1": 0.556522, "f1_weighted": 0.563342 } ], "main_score": 0.5645, "hf_subset": "sun", "languages": [ "sun-Latn" ] }, { "accuracy": 0.58125, "f1": 0.558616, "f1_weighted": 0.566141, "scores_per_experiment": [ { "accuracy": 0.5925, "f1": 0.580565, "f1_weighted": 0.585826 }, { "accuracy": 0.565, "f1": 0.541322, "f1_weighted": 0.556918 }, { "accuracy": 0.5625, "f1": 0.531374, "f1_weighted": 0.531396 }, { "accuracy": 0.575, "f1": 0.550017, "f1_weighted": 0.55368 }, { "accuracy": 0.5875, "f1": 0.556267, "f1_weighted": 0.552101 }, { "accuracy": 0.61, "f1": 0.588268, "f1_weighted": 0.607254 }, { "accuracy": 0.5475, "f1": 0.505958, "f1_weighted": 0.521744 }, { "accuracy": 0.6175, "f1": 0.611018, "f1_weighted": 0.621303 }, { "accuracy": 0.5975, "f1": 0.581056, "f1_weighted": 0.586365 }, { "accuracy": 0.5575, "f1": 0.540316, "f1_weighted": 0.544824 } ], "main_score": 0.58125, "hf_subset": "bbc", "languages": [ "bbc-Latn" ] } ] }, "evaluation_time": 21.547587156295776, "kg_co2_emissions": 0.0008236360570719399 }