name: belebele dataset: facebook/belebele trust_remote_code: True num_fewshot: 5 fewshot_split: test languages: - afr_Latn - amh_Ethi - ary_Arab - arz_Arab - bam_Latn - eng_Latn - fuv_Latn - fra_Latn - gaz_Latn - hau_Latn - ibo_Latn - kea_Latn - kin_Latn - lin_Latn - lug_Latn - luo_Latn - nya_Latn - plt_Latn - por_Latn - sna_Latn - som_Latn - sot_Latn - ssw_Latn - swh_Latn - tir_Ethi - tso_Latn - tsn_Latn - wol_Latn - xho_Latn - yor_Latn - zul_Latn test_split: test target: correct_answer_num prompts: - "P: {{flores_passage}}\nQ: {{question}}\nA: {{mc_answer1}}\nB: {{mc_answer2}}\nC: {{mc_answer3}}\nD: {{mc_answer4}}\nPlease choose the correct answer from the options above:" - "Passage: {{flores_passage}}\nQuestion: {{question}}\n1: {{mc_answer1}}\n2: {{mc_answer2}}\n3: {{mc_answer3}}\n4: {{mc_answer4}}\nPlease select the correct answer from the given choices:" - "Context: {{flores_passage}}\nQuery: {{question}}\nOption A: {{mc_answer1}}\nOption B: {{mc_answer2}}\nOption C: {{mc_answer3}}\nOption D: {{mc_answer4}}\nPlease indicate the correct option from the list above:" - "{{flores_passage}}\nBased on the above passage, answer the following question:\n{{question}}\nChoices:\nA) {{mc_answer1}}\nB) {{mc_answer2}}\nC) {{mc_answer3}}\nD) {{mc_answer4}}\nPlease provide the correct answer from the choices given:" - "Read the passage: {{flores_passage}}\nThen answer the question: {{question}}\nOptions:\nA. {{mc_answer1}}\nB. {{mc_answer2}}\nC. {{mc_answer3}}\nD. {{mc_answer4}}\nPlease choose the correct option from the above list:" choices: - A - B - C - D verbalizer: "A": ['a:', 'a', 'a.', '1', '1:', 'a)', '(a)', 'option a', 'option a:', 'option_a:'] "B": ['b:', 'b', 'b.', '2', '2:', 'b)', '(b)', 'option b', 'option b:', 'option_b:'] "C": ['c:', 'c', 'c.', '3', '3:', 'c)', '(c)', 'option c', 'option c:', 'option_c:'] "D": ['d:', 'd', 'd.', '4', '4:', 'd)', '(d)', 'option d', 'option d:', 'option_d:'] map_response: A: '1' B: '2' C: '3' D: '4' metrics: - f1 - acc