{ "results": { "eus_proficiency": { "acc,none": 0.4819113948539369, "acc_stderr,none": 0.006950633119867768, "alias": "eus_proficiency" } }, "configs": { "eus_proficiency": { "task": "eus_proficiency", "dataset_path": "HiTZ/EusProficiency", "dataset_name": "default", "test_split": "test", "fewshot_split": "test", "doc_to_text": "Galdera: {{question}}\nA: {{candidates[0]}}\nB: {{candidates[1]}}\nC: {{candidates[2]}}\nD: {{candidates[3]}}\nErantzuna:", "doc_to_target": "answer", "doc_to_choice": [ "A", "B", "C", "D" ], "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "num_fewshot": 5, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 0.0 } } }, "versions": { "eus_proficiency": "Yaml" }, "n-shot": { "eus_proficiency": 5 }, "config": { "model": "hf", "model_args": "pretrained=/gaueko1/hizkuntza-ereduak/llama-2-eus/latxa-70b-v1,tokenizer=/gaueko1/hizkuntza-ereduak/llama-2-eus/latxa-7b-v1,parallelize=True", "batch_size": 1, "batch_sizes": [], "device": "cuda", "use_cache": null, "limit": null, "bootstrap_iters": 100000, "gen_kwargs": null }, "git_hash": "e57707a" }