{ "results": { "eus_proficiency": { "acc,none": 0.22963822789707874, "acc_stderr,none": 0.005850705022634062, "alias": "eus_proficiency" } }, "configs": { "eus_proficiency": { "task": "eus_proficiency", "dataset_path": "HiTZ/EusProficiency", "dataset_name": "default", "test_split": "test", "fewshot_split": "test", "doc_to_text": "Galdera: {{question}}\nA: {{candidates[0]}}\nB: {{candidates[1]}}\nC: {{candidates[2]}}\nD: {{candidates[3]}}\nErantzuna:", "doc_to_target": "answer", "doc_to_choice": [ "A", "B", "C", "D" ], "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "num_fewshot": 5, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 0.0 } } }, "versions": { "eus_proficiency": "Yaml" }, "n-shot": { "eus_proficiency": 5 }, "config": { "model": "hf", "model_args": "pretrained=facebook/xglm-7.5B,dtype=float16", "batch_size": "auto", "batch_sizes": [ 32 ], "device": "cuda", "use_cache": null, "limit": null, "bootstrap_iters": 100000, "gen_kwargs": null }, "git_hash": "75a76af" }