{ "cells": [ { "cell_type": "code", "execution_count": 1, "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "zJvyUmZdktu0", "outputId": "25275dd8-4787-4b77-f7bf-4f2bbe66f0b8" }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "/workspace/axolotl\n" ] } ], "source": [ "%cd /workspace/axolotl" ] }, { "cell_type": "code", "execution_count": 2, "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "mC48y25Lkqa5", "outputId": "2757a3c8-3790-4fd3-be39-03be8f533b35" }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Setting ds_accelerator to cuda (auto detect)\n", "accelerate configuration saved at /root/.cache/huggingface/accelerate/default_config.yaml\n" ] } ], "source": [ "!accelerate config --config_file configs/accelerate/default_config.yaml default" ] }, { "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ "Open files to modify:\n", "\n", "/workspace/axolotl/examples/falcon/qlora.yml" ] }, { "cell_type": "code", "execution_count": 18, "metadata": { "scrolled": true }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "base_model: tiiuae/falcon-rw-1b\n", "base_model_config: tiiuae/falcon-rw-1b\n", "trust_remote_code: true\n", "model_type: AutoModelForCausalLM\n", "tokenizer_type: AutoTokenizer\n", "load_in_8bit: false\n", "load_in_4bit: true\n", "gptq: false\n", "strict: false\n", "push_dataset_to_hub:\n", "datasets:\n", " # - path: AtlasUnified/Code-Instruct-Sets\n", " # data_files:\n", " # - unmasked-set-1.jsonl\n", " # - unmasked-set-2.jsonl\n", " # - unmasked-set-3.jsonl\n", " # - unmasked-set-4.jsonl\n", " # type: alpaca_code_instruct\n", " # - path: winglian/pygmalion-cleaned\n", " # data_files:\n", " # - v13_no_ai.cleaned.jsonl\n", " # type: pygmalion\n", " # shards: 4\n", " # - path: winglian/evals\n", " # data_files:\n", " # - hf/ARC-Challenge.jsonl\n", " # - hf/ARC-Easy.jsonl\n", " # - hf/riddle_sense.jsonl\n", " # type: explainchoice:chat\n", " # - path: winglian/evals\n", " # data_files:\n", " # - hf/gsm8k.jsonl\n", " # - custom/logic_inference_oa.jsonl\n", " # type: alpaca_chat.load_qa\n", " # - path: winglian/evals\n", " # data_files:\n", " # - custom/in_context_qa.jsonl\n", " # type: context_qa\n", " # - path: winglian/evals\n", " # data_files:\n", " # - custom/in_context_qa.jsonl\n", " # type: context_qa.load_404\n", " # - path: winglian/evals\n", " # data_files:\n", " # - custom/jokes_explained_500up.jsonl\n", " # type: sharegpt_jokes\n", " # - path: winglian/evals\n", " # data_files:\n", " # - custom/classify-self-chat.sharegpt.jsonl\n", " # - custom/coding-self-chat.sharegpt.jsonl\n", " # - custom/prose-gpt4.sharegpt.jsonl\n", " # - custom/prose-rewrite-gpt4.sharegpt.jsonl\n", " # type: sharegpt_simple\n", " # - path: winglian/evals\n", " # data_files:\n", " # - custom/guanaco-cleaned.en.jsonl\n", " # type: sharegpt_simple.load_guanaco\n", " # - path: winglian/evals\n", " # data_files:\n", " # - openai/tldr.jsonl\n", " # type: summarizetldr:chat\n", " # - path: winglian/evals\n", " # data_files:\n", " # - hellaswag/hellaswag.jsonl\n", " # type: explainchoice:chat\n", " # shards: 60\n", " # - path: metaeval/ScienceQA_text_only\n", " # type: concisechoice:chat\n", " # shards: 13\n", " # - path: teknium/GPTeacher-General-Instruct\n", " # data_files: \n", " # - gpt4-instruct-similarity-0.6-dataset.json\n", " # type: gpteacher:chat\n", " - path: QingyiSi/Alpaca-CoT\n", " data_files:\n", " # - chain-of-thought/formatted_cot_data/aqua_train.jsonl\n", " # - Chain-of-Thought/formatted_cot_data/creak_train.json\n", " # - Chain-of-Thought/formatted_cot_data/ecqa_train.json\n", " # - Chain-of-Thought/formatted_cot_data/esnli_train.json\n", " - Chain-of-Thought/formatted_cot_data/gsm8k_train.json\n", " # - Chain-of-Thought/formatted_cot_data/qasc_train.json\n", " # - Chain-of-Thought/formatted_cot_data/qed_train.json\n", " # - Chain-of-Thought/formatted_cot_data/sensemaking_train.json\n", " # - Chain-of-Thought/formatted_cot_data/strategyqa_train.json\n", " # - GPTeacher/Roleplay/formatted_roleplay-similarity_0.6-instruct-dataset.json\n", " type: \"alpaca:chat\"\n", "dataset_prepared_path: last_run_prepared\n", "val_set_size: 0.01\n", "adapter: qlora\n", "lora_model_dir:\n", "sequence_len: 2048\n", "max_packed_sequence_len: 2048\n", "lora_r: 64\n", "lora_alpha: 16\n", "lora_dropout: 0.05\n", "lora_target_modules:\n", "lora_target_linear: true\n", "lora_fan_in_fan_out:\n", "wandb_project: falcon-qlora\n", "wandb_watch:\n", "wandb_run_id:\n", "wandb_log_model: checkpoint\n", "output_dir: ./qlora-out\n", "batch_size: 24\n", "micro_batch_size: 12\n", "num_epochs: 3\n", "optimizer: paged_adamw_32bit\n", "torchdistx_path:\n", "lr_scheduler: cosine\n", "learning_rate: 0.0002\n", "train_on_inputs: false\n", "group_by_length: false\n", "bf16: true\n", "fp16: false\n", "tf32: true\n", "gradient_checkpointing: true\n", "# stop training after this many evaluation losses have increased in a row\n", "# https://huggingface.co/transformers/v4.2.2/_modules/transformers/trainer_callback.html#EarlyStoppingCallback\n", "early_stopping_patience: 3\n", "resume_from_checkpoint:\n", "auto_resume_from_checkpoints: true\n", "local_rank:\n", "logging_steps: 1\n", "xformers_attention: false\n", "flash_attention:\n", "gptq_groupsize:\n", "gptq_model_v1:\n", "warmup_steps: 10\n", "eval_steps: 5\n", "save_steps: 10\n", "debug:\n", "deepspeed:\n", "weight_decay: 0.000001\n", "fsdp:\n", "fsdp_config:\n", "special_tokens:\n", " pad_token: \"<|endoftext|>\"\n", " bos_token: \">>ABSTRACT<<\"\n", " eos_token: \"<|endoftext|>\"\n" ] } ], "source": [ "!cat examples/falcon/qlora.yml" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "jICMPJuomFsx", "scrolled": true }, "outputs": [], "source": [ "!accelerate launch scripts/finetune.py examples/falcon/qlora.yml" ] }, { "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ "## Below are ad hoc cells handling issues during training" ] }, { "cell_type": "code", "execution_count": 17, "metadata": {}, "outputs": [], "source": [ "!ps aux|grep python|grep finetune|awk '{print $2}'|xargs kill" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "!apt install zip\n", "!zip -r last_run_prepared.zip -xi last_run_prepared" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "!nvitop -m full" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "scrolled": true }, "outputs": [], "source": [ "!apt install lsof" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "scrolled": true }, "outputs": [], "source": [ "!lsof /dev/nvidia*" ] } ], "metadata": { "accelerator": "GPU", "colab": { "authorship_tag": "ABX9TyP0jHt4WNuLaC5ecmX0YtWl", "gpuType": "T4", "include_colab_link": true, "provenance": [], "toc_visible": true }, "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.9.16" } }, "nbformat": 4, "nbformat_minor": 4 }