{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "zJvyUmZdktu0",
    "outputId": "25275dd8-4787-4b77-f7bf-4f2bbe66f0b8"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/workspace/axolotl\n"
     ]
    }
   ],
   "source": [
    "%cd /workspace/axolotl"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "mC48y25Lkqa5",
    "outputId": "2757a3c8-3790-4fd3-be39-03be8f533b35"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Setting ds_accelerator to cuda (auto detect)\n",
      "accelerate configuration saved at /root/.cache/huggingface/accelerate/default_config.yaml\n"
     ]
    }
   ],
   "source": [
    "!accelerate config --config_file configs/accelerate/default_config.yaml default"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Open files to modify:\n",
    "\n",
    "/workspace/axolotl/examples/falcon/qlora.yml"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "base_model: tiiuae/falcon-rw-1b\n",
      "base_model_config: tiiuae/falcon-rw-1b\n",
      "trust_remote_code: true\n",
      "model_type: AutoModelForCausalLM\n",
      "tokenizer_type: AutoTokenizer\n",
      "load_in_8bit: false\n",
      "load_in_4bit: true\n",
      "gptq: false\n",
      "strict: false\n",
      "push_dataset_to_hub:\n",
      "datasets:\n",
      "  # - path: AtlasUnified/Code-Instruct-Sets\n",
      "  #   data_files:\n",
      "  #     - unmasked-set-1.jsonl\n",
      "  #     - unmasked-set-2.jsonl\n",
      "  #     - unmasked-set-3.jsonl\n",
      "  #     - unmasked-set-4.jsonl\n",
      "  #   type: alpaca_code_instruct\n",
      "  # - path: winglian/pygmalion-cleaned\n",
      "  #   data_files:\n",
      "  #     - v13_no_ai.cleaned.jsonl\n",
      "  #   type: pygmalion\n",
      "  #   shards: 4\n",
      "  # - path: winglian/evals\n",
      "  #   data_files:\n",
      "  #     - hf/ARC-Challenge.jsonl\n",
      "  #     - hf/ARC-Easy.jsonl\n",
      "  #     - hf/riddle_sense.jsonl\n",
      "  #   type: explainchoice:chat\n",
      "  # - path: winglian/evals\n",
      "  #   data_files:\n",
      "  #     - hf/gsm8k.jsonl\n",
      "  #     - custom/logic_inference_oa.jsonl\n",
      "  #   type: alpaca_chat.load_qa\n",
      "  # - path: winglian/evals\n",
      "  #   data_files:\n",
      "  #     - custom/in_context_qa.jsonl\n",
      "  #   type: context_qa\n",
      "  # - path: winglian/evals\n",
      "  #   data_files:\n",
      "  #     - custom/in_context_qa.jsonl\n",
      "  #   type: context_qa.load_404\n",
      "  # - path: winglian/evals\n",
      "  #   data_files:\n",
      "  #     - custom/jokes_explained_500up.jsonl\n",
      "  #   type: sharegpt_jokes\n",
      "  # - path: winglian/evals\n",
      "  #   data_files:\n",
      "  #     - custom/classify-self-chat.sharegpt.jsonl\n",
      "  #     - custom/coding-self-chat.sharegpt.jsonl\n",
      "  #     - custom/prose-gpt4.sharegpt.jsonl\n",
      "  #     - custom/prose-rewrite-gpt4.sharegpt.jsonl\n",
      "  #   type: sharegpt_simple\n",
      "  # - path: winglian/evals\n",
      "  #   data_files:\n",
      "  #     - custom/guanaco-cleaned.en.jsonl\n",
      "  #   type: sharegpt_simple.load_guanaco\n",
      "  # - path: winglian/evals\n",
      "  #   data_files:\n",
      "  #     - openai/tldr.jsonl\n",
      "  #   type: summarizetldr:chat\n",
      "  # - path: winglian/evals\n",
      "  #   data_files:\n",
      "  #     - hellaswag/hellaswag.jsonl\n",
      "  #   type: explainchoice:chat\n",
      "  #   shards: 60\n",
      "  # - path: metaeval/ScienceQA_text_only\n",
      "  #   type: concisechoice:chat\n",
      "  #   shards: 13\n",
      "  # - path: teknium/GPTeacher-General-Instruct\n",
      "  #   data_files: \n",
      "  #     - gpt4-instruct-similarity-0.6-dataset.json\n",
      "  #   type: gpteacher:chat\n",
      "  - path: QingyiSi/Alpaca-CoT\n",
      "    data_files:\n",
      "      # - chain-of-thought/formatted_cot_data/aqua_train.jsonl\n",
      "      # - Chain-of-Thought/formatted_cot_data/creak_train.json\n",
      "      # - Chain-of-Thought/formatted_cot_data/ecqa_train.json\n",
      "      # - Chain-of-Thought/formatted_cot_data/esnli_train.json\n",
      "      - Chain-of-Thought/formatted_cot_data/gsm8k_train.json\n",
      "      # - Chain-of-Thought/formatted_cot_data/qasc_train.json\n",
      "      # - Chain-of-Thought/formatted_cot_data/qed_train.json\n",
      "      # - Chain-of-Thought/formatted_cot_data/sensemaking_train.json\n",
      "      # - Chain-of-Thought/formatted_cot_data/strategyqa_train.json\n",
      "      # - GPTeacher/Roleplay/formatted_roleplay-similarity_0.6-instruct-dataset.json\n",
      "    type: \"alpaca:chat\"\n",
      "dataset_prepared_path: last_run_prepared\n",
      "val_set_size: 0.01\n",
      "adapter: qlora\n",
      "lora_model_dir:\n",
      "sequence_len: 2048\n",
      "max_packed_sequence_len: 2048\n",
      "lora_r: 64\n",
      "lora_alpha: 16\n",
      "lora_dropout: 0.05\n",
      "lora_target_modules:\n",
      "lora_target_linear: true\n",
      "lora_fan_in_fan_out:\n",
      "wandb_project: falcon-qlora\n",
      "wandb_watch:\n",
      "wandb_run_id:\n",
      "wandb_log_model: checkpoint\n",
      "output_dir: ./qlora-out\n",
      "batch_size: 24\n",
      "micro_batch_size: 12\n",
      "num_epochs: 3\n",
      "optimizer: paged_adamw_32bit\n",
      "torchdistx_path:\n",
      "lr_scheduler: cosine\n",
      "learning_rate: 0.0002\n",
      "train_on_inputs: false\n",
      "group_by_length: false\n",
      "bf16: true\n",
      "fp16: false\n",
      "tf32: true\n",
      "gradient_checkpointing: true\n",
      "# stop training after this many evaluation losses have increased in a row\n",
      "# https://huggingface.co/transformers/v4.2.2/_modules/transformers/trainer_callback.html#EarlyStoppingCallback\n",
      "early_stopping_patience: 3\n",
      "resume_from_checkpoint:\n",
      "auto_resume_from_checkpoints: true\n",
      "local_rank:\n",
      "logging_steps: 1\n",
      "xformers_attention: false\n",
      "flash_attention:\n",
      "gptq_groupsize:\n",
      "gptq_model_v1:\n",
      "warmup_steps: 10\n",
      "eval_steps: 5\n",
      "save_steps: 10\n",
      "debug:\n",
      "deepspeed:\n",
      "weight_decay: 0.000001\n",
      "fsdp:\n",
      "fsdp_config:\n",
      "special_tokens:\n",
      "  pad_token: \"<|endoftext|>\"\n",
      "  bos_token: \">>ABSTRACT<<\"\n",
      "  eos_token: \"<|endoftext|>\"\n"
     ]
    }
   ],
   "source": [
    "!cat examples/falcon/qlora.yml"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "jICMPJuomFsx",
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "!accelerate launch scripts/finetune.py examples/falcon/qlora.yml"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Below are ad hoc cells handling issues during training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "!ps aux|grep python|grep finetune|awk '{print $2}'|xargs kill"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!apt install zip\n",
    "!zip -r last_run_prepared.zip -xi last_run_prepared"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!nvitop -m full"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "!apt install lsof"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "!lsof /dev/nvidia*"
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "authorship_tag": "ABX9TyP0jHt4WNuLaC5ecmX0YtWl",
   "gpuType": "T4",
   "include_colab_link": true,
   "provenance": [],
   "toc_visible": true
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.16"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}