{
"cells": [
{
"attachments": {},
"cell_type": "markdown",
"metadata": {},
"source": [
"# [Finetuning falcon-40b with Axolotl+QLoRA](https://wandb.ai/utensil/falcon-qlora/reports/Finetuning-falcon-7b-with-Axolotl-QLoRA--Vmlldzo0NTIzNTMw)"
]
},
{
"attachments": {},
"cell_type": "markdown",
"metadata": {},
"source": [
"\n",
"\n",
"\n",
"\n",
"\n",
"\n",
"\n",
""
]
},
{
"cell_type": "code",
"execution_count": 1,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "zJvyUmZdktu0",
"outputId": "25275dd8-4787-4b77-f7bf-4f2bbe66f0b8"
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"/workspace/axolotl\n"
]
}
],
"source": [
"%cd /workspace/axolotl"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "mC48y25Lkqa5",
"outputId": "2757a3c8-3790-4fd3-be39-03be8f533b35"
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Setting ds_accelerator to cuda (auto detect)\n",
"accelerate configuration saved at /root/.cache/huggingface/accelerate/default_config.yaml\n"
]
}
],
"source": [
"!accelerate config --config_file configs/accelerate/default_config.yaml default"
]
},
{
"attachments": {},
"cell_type": "markdown",
"metadata": {},
"source": [
"## #1 40b 4*2"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {
"scrolled": true
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"base_model: tiiuae/falcon-40b\n",
"base_model_config: tiiuae/falcon-40b\n",
"trust_remote_code: true\n",
"model_type: AutoModelForCausalLM\n",
"tokenizer_type: AutoTokenizer\n",
"load_in_8bit: false\n",
"load_in_4bit: true\n",
"gptq: false\n",
"strict: false\n",
"push_dataset_to_hub:\n",
"datasets:\n",
" - path: QingyiSi/Alpaca-CoT\n",
" data_files:\n",
" - Chain-of-Thought/formatted_cot_data/gsm8k_train.json\n",
" type: \"alpaca:chat\"\n",
"dataset_prepared_path: last_run_prepared\n",
"val_set_size: 0.01\n",
"adapter: qlora\n",
"lora_model_dir:\n",
"sequence_len: 2048\n",
"max_packed_sequence_len:\n",
"lora_r: 64\n",
"lora_alpha: 16\n",
"lora_dropout: 0.05\n",
"lora_target_modules:\n",
"lora_target_linear: true\n",
"lora_fan_in_fan_out:\n",
"wandb_project: falcon-qlora\n",
"wandb_watch:\n",
"wandb_run_id:\n",
"wandb_log_model:\n",
"output_dir: ./qlora-out\n",
"micro_batch_size: 4\n",
"gradient_accumulation_steps: 2\n",
"num_epochs: 3\n",
"optimizer: paged_adamw_32bit\n",
"torchdistx_path:\n",
"lr_scheduler: cosine\n",
"learning_rate: 0.0002\n",
"train_on_inputs: false\n",
"group_by_length: false\n",
"bf16: true\n",
"fp16: false\n",
"tf32: true\n",
"gradient_checkpointing: true\n",
"# stop training after this many evaluation losses have increased in a row\n",
"# https://huggingface.co/transformers/v4.2.2/_modules/transformers/trainer_callback.html#EarlyStoppingCallback\n",
"early_stopping_patience: 3\n",
"resume_from_checkpoint:\n",
"auto_resume_from_checkpoints: true\n",
"local_rank:\n",
"logging_steps: 1\n",
"xformers_attention: true\n",
"flash_attention:\n",
"gptq_groupsize:\n",
"gptq_model_v1:\n",
"warmup_steps: 10\n",
"eval_steps: 5\n",
"save_steps: 10\n",
"debug:\n",
"deepspeed:\n",
"weight_decay: 0.000001\n",
"fsdp:\n",
"fsdp_config:\n",
"special_tokens:\n",
" pad_token: \"<|endoftext|>\"\n",
" bos_token: \">>ABSTRACT<<\"\n",
" eos_token: \"<|endoftext|>\""
]
}
],
"source": [
"!cat examples/falcon/qlora.yml"
]
},
{
"cell_type": "code",
"execution_count": 12,
"metadata": {
"id": "jICMPJuomFsx",
"scrolled": true
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Setting ds_accelerator to cuda (auto detect)\n",
"\n",
"===================================BUG REPORT===================================\n",
"Welcome to bitsandbytes. For bug reports, please run\n",
"\n",
"python -m bitsandbytes\n",
"\n",
" and submit this information together with your error trace to: https://github.com/TimDettmers/bitsandbytes/issues\n",
"================================================================================\n",
"bin /root/miniconda3/envs/py3.9/lib/python3.9/site-packages/bitsandbytes-0.39.0-py3.9.egg/bitsandbytes/libbitsandbytes_cuda118.so\n",
"/root/miniconda3/envs/py3.9/lib/python3.9/site-packages/bitsandbytes-0.39.0-py3.9.egg/bitsandbytes/cuda_setup/main.py:149: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('/usr/local/nvidia/lib64'), PosixPath('/usr/local/nvidia/lib')}\n",
" warn(msg)\n",
"/root/miniconda3/envs/py3.9/lib/python3.9/site-packages/bitsandbytes-0.39.0-py3.9.egg/bitsandbytes/cuda_setup/main.py:149: UserWarning: /usr/local/nvidia/lib:/usr/local/nvidia/lib64 did not contain ['libcudart.so', 'libcudart.so.11.0', 'libcudart.so.12.0'] as expected! Searching further paths...\n",
" warn(msg)\n",
"/root/miniconda3/envs/py3.9/lib/python3.9/site-packages/bitsandbytes-0.39.0-py3.9.egg/bitsandbytes/cuda_setup/main.py:149: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('//matplotlib_inline.backend_inline'), PosixPath('module')}\n",
" warn(msg)\n",
"/root/miniconda3/envs/py3.9/lib/python3.9/site-packages/bitsandbytes-0.39.0-py3.9.egg/bitsandbytes/cuda_setup/main.py:149: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIK1tFOFrWbmoa2ckCJYhzgBHKTSMeR/AeuScCCzugqlI utensilcandel@gmail.com')}\n",
" warn(msg)\n",
"CUDA_SETUP: WARNING! libcudart.so not found in any environmental path. Searching in backup paths...\n",
"/root/miniconda3/envs/py3.9/lib/python3.9/site-packages/bitsandbytes-0.39.0-py3.9.egg/bitsandbytes/cuda_setup/main.py:149: UserWarning: Found duplicate ['libcudart.so', 'libcudart.so.11.0', 'libcudart.so.12.0'] files: {PosixPath('/usr/local/cuda/lib64/libcudart.so'), PosixPath('/usr/local/cuda/lib64/libcudart.so.11.0')}.. We'll flip a coin and try one of these, in order to fail forward.\n",
"Either way, this might cause trouble in the future:\n",
"If you get `CUDA error: invalid device function` errors, the above might be the cause and the solution is to make sure only one ['libcudart.so', 'libcudart.so.11.0', 'libcudart.so.12.0'] in the paths that we search based on your env.\n",
" warn(msg)\n",
"CUDA SETUP: CUDA runtime path found: /usr/local/cuda/lib64/libcudart.so\n",
"CUDA SETUP: Highest compute capability among GPUs detected: 8.6\n",
"CUDA SETUP: Detected CUDA version 118\n",
"CUDA SETUP: Loading binary /root/miniconda3/envs/py3.9/lib/python3.9/site-packages/bitsandbytes-0.39.0-py3.9.egg/bitsandbytes/libbitsandbytes_cuda118.so...\n",
"Setting ds_accelerator to cuda (auto detect)\n",
"WARNING:root:`trust_remote_code` is set to true. Please make sure that you reviewed the remote code/model.\n",
"INFO:root:loading tokenizer... tiiuae/falcon-40b\n",
"Downloading (…)okenizer_config.json: 100%|█████| 175/175 [00:00<00:00, 23.9kB/s]\n",
"Downloading (…)/main/tokenizer.json: 100%|█| 2.73M/2.73M [00:00<00:00, 66.6MB/s]\n",
"Downloading (…)cial_tokens_map.json: 100%|██████| 281/281 [00:00<00:00, 382kB/s]\n",
"Using bos_token, but it is not set yet.\n",
"Using pad_token, but it is not set yet.\n",
"Using unk_token, but it is not set yet.\n",
"INFO:root:Unable to find prepared dataset in last_run_prepared/31a4e867d804a957707db033c9abcd13\n",
"INFO:root:Loading raw datasets...\n",
"Downloading readme: 100%|██████████████████| 8.26k/8.26k [00:00<00:00, 16.3MB/s]\n",
"Downloading and preparing dataset json/QingyiSi--Alpaca-CoT to /root/.cache/huggingface/datasets/QingyiSi___json/QingyiSi--Alpaca-CoT-2953efcfeb19f105/0.0.0/e347ab1c932092252e717ff3f949105a4dd28b27e842dd53157d2f72e276c2e4...\n",
"Downloading data files: 0%| | 0/1 [00:00\n",
" fire.Fire(train)\n",
" File \"/root/miniconda3/envs/py3.9/lib/python3.9/site-packages/fire/core.py\", line 141, in Fire\n",
" component_trace = _Fire(component, args, parsed_flag_args, context, name)\n",
" File \"/root/miniconda3/envs/py3.9/lib/python3.9/site-packages/fire/core.py\", line 475, in _Fire\n",
" component, remaining_args = _CallAndUpdateTrace(\n",
" File \"/root/miniconda3/envs/py3.9/lib/python3.9/site-packages/fire/core.py\", line 691, in _CallAndUpdateTrace\n",
" component = fn(*varargs, **kwargs)\n",
" File \"/workspace/axolotl/scripts/finetune.py\", line 281, in train\n",
" trainer.train(resume_from_checkpoint=resume_from_checkpoint)\n",
" File \"/root/miniconda3/envs/py3.9/lib/python3.9/site-packages/transformers/trainer.py\", line 1696, in train\n",
" return inner_training_loop(\n",
" File \"/root/miniconda3/envs/py3.9/lib/python3.9/site-packages/transformers/trainer.py\", line 2052, in _inner_training_loop\n",
" self._maybe_log_save_evaluate(tr_loss, model, trial, epoch, ignore_keys_for_eval)\n",
" File \"/root/miniconda3/envs/py3.9/lib/python3.9/site-packages/transformers/trainer.py\", line 2360, in _maybe_log_save_evaluate\n",
" self._save_checkpoint(model, trial, metrics=metrics)\n",
" File \"/root/miniconda3/envs/py3.9/lib/python3.9/site-packages/transformers/trainer.py\", line 2417, in _save_checkpoint\n",
" self.save_model(output_dir, _internal_call=True)\n",
" File \"/root/miniconda3/envs/py3.9/lib/python3.9/site-packages/transformers/trainer.py\", line 2918, in save_model\n",
" self._save(output_dir)\n",
" File \"/root/miniconda3/envs/py3.9/lib/python3.9/site-packages/transformers/trainer.py\", line 2982, in _save\n",
" torch.save(self.args, os.path.join(output_dir, TRAINING_ARGS_NAME))\n",
" File \"/root/miniconda3/envs/py3.9/lib/python3.9/site-packages/torch/serialization.py\", line 440, in save\n",
" with _open_zipfile_writer(f) as opened_zipfile:\n",
" File \"/root/miniconda3/envs/py3.9/lib/python3.9/site-packages/torch/serialization.py\", line 315, in _open_zipfile_writer\n",
" return container(name_or_buffer)\n",
" File \"/root/miniconda3/envs/py3.9/lib/python3.9/site-packages/torch/serialization.py\", line 288, in __init__\n",
" super().__init__(torch._C.PyTorchFileWriter(str(name)))\n",
"RuntimeError: Parent directory ./qlora-out/checkpoint-170 does not exist.\n",
"\u001b[34m\u001b[1mwandb\u001b[0m: Waiting for W&B process to finish... \u001b[31m(failed 1).\u001b[0m Press Control-C to abort syncing.\n",
"\u001b[34m\u001b[1mwandb\u001b[0m: \n",
"\u001b[34m\u001b[1mwandb\u001b[0m: Run history:\n",
"\u001b[34m\u001b[1mwandb\u001b[0m: eval/loss █▅▄▃▃▃▂▂▂▂▂▂▂▂▂▂▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n",
"\u001b[34m\u001b[1mwandb\u001b[0m: eval/runtime ▁▆▅█▅█▅█▅█▅█▅▇▆█▅▇▅█▅█▅▇▅█▅█▅█▅█▅█\n",
"\u001b[34m\u001b[1mwandb\u001b[0m: eval/samples_per_second █▂▄▁▄▁▄▁▄▁▄▁▄▁▃▁▄▁▄▁▄▁▄▁▄▁▄▁▄▁▄▁▄▁\n",
"\u001b[34m\u001b[1mwandb\u001b[0m: eval/steps_per_second █▃▄▁▄▁▄▁▄▁▄▁▅▁▄▁▅▁▄▁▄▁▄▁▄▁▄▁▄▁▄▁▄▁\n",
"\u001b[34m\u001b[1mwandb\u001b[0m: train/epoch ▁▁▁▂▂▂▂▂▃▃▃▃▃▃▄▄▄▄▄▄▄▅▅▅▅▆▆▆▆▆▆▇▇▇▇▇████\n",
"\u001b[34m\u001b[1mwandb\u001b[0m: train/global_step ▁▁▁▁▂▂▂▂▂▃▃▃▃▃▃▄▄▄▄▄▅▅▅▅▅▅▆▆▆▆▆▇▇▇▇▇▇███\n",
"\u001b[34m\u001b[1mwandb\u001b[0m: train/learning_rate ▁▅██████████████████████████████████████\n",
"\u001b[34m\u001b[1mwandb\u001b[0m: train/loss █▇▇▄▅▄▃▅▃▅▃▂▃▃▂▃▃▃▃▃▁▃▂▂▃▁▃▃▃▃▁▃▃▃▃▃▄▁▃▁\n",
"\u001b[34m\u001b[1mwandb\u001b[0m: \n",
"\u001b[34m\u001b[1mwandb\u001b[0m: Run summary:\n",
"\u001b[34m\u001b[1mwandb\u001b[0m: eval/loss 0.5186\n",
"\u001b[34m\u001b[1mwandb\u001b[0m: eval/runtime 30.2925\n",
"\u001b[34m\u001b[1mwandb\u001b[0m: eval/samples_per_second 2.476\n",
"\u001b[34m\u001b[1mwandb\u001b[0m: eval/steps_per_second 0.627\n",
"\u001b[34m\u001b[1mwandb\u001b[0m: train/epoch 0.18\n",
"\u001b[34m\u001b[1mwandb\u001b[0m: train/global_step 170\n",
"\u001b[34m\u001b[1mwandb\u001b[0m: train/learning_rate 0.0002\n",
"\u001b[34m\u001b[1mwandb\u001b[0m: train/loss 0.4315\n",
"\u001b[34m\u001b[1mwandb\u001b[0m: \n",
"\u001b[34m\u001b[1mwandb\u001b[0m: 🚀 View run \u001b[33mpolar-water-20\u001b[0m at: \u001b[34m\u001b[4mhttps://wandb.ai/utensil/falcon-qlora/runs/i9jjo2rl\u001b[0m\n",
"\u001b[34m\u001b[1mwandb\u001b[0m: Synced 6 W&B file(s), 0 media file(s), 0 artifact file(s) and 0 other file(s)\n",
"\u001b[34m\u001b[1mwandb\u001b[0m: Find logs at: \u001b[35m\u001b[1m./wandb/run-20230602_040215-i9jjo2rl/logs\u001b[0m\n",
"Traceback (most recent call last):\n",
" File \"/root/miniconda3/envs/py3.9/bin/accelerate\", line 8, in \n",
" sys.exit(main())\n",
" File \"/root/miniconda3/envs/py3.9/lib/python3.9/site-packages/accelerate/commands/accelerate_cli.py\", line 45, in main\n",
" args.func(args)\n",
" File \"/root/miniconda3/envs/py3.9/lib/python3.9/site-packages/accelerate/commands/launch.py\", line 928, in launch_command\n",
" simple_launcher(args)\n",
" File \"/root/miniconda3/envs/py3.9/lib/python3.9/site-packages/accelerate/commands/launch.py\", line 588, in simple_launcher\n",
" raise subprocess.CalledProcessError(returncode=process.returncode, cmd=cmd)\n",
"subprocess.CalledProcessError: Command '['/root/miniconda3/envs/py3.9/bin/python3', 'scripts/finetune.py', 'examples/falcon/qlora.yml']' returned non-zero exit status 1.\n"
]
}
],
"source": [
"!accelerate launch scripts/finetune.py examples/falcon/qlora.yml"
]
},
{
"attachments": {},
"cell_type": "markdown",
"metadata": {},
"source": [
"## #2 \n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"scrolled": true
},
"outputs": [],
"source": [
"!cat examples/falcon/qlora.yml"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "jICMPJuomFsx",
"scrolled": true
},
"outputs": [],
"source": [
"!accelerate launch scripts/finetune.py examples/falcon/qlora.yml"
]
},
{
"attachments": {},
"cell_type": "markdown",
"metadata": {},
"source": [
"## #3"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"scrolled": true
},
"outputs": [],
"source": [
"!cat examples/falcon/qlora.yml"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"scrolled": true
},
"outputs": [],
"source": [
"!accelerate launch scripts/finetune.py examples/falcon/qlora.yml"
]
},
{
"attachments": {},
"cell_type": "markdown",
"metadata": {},
"source": [
"## #4"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"scrolled": true
},
"outputs": [],
"source": [
"!cat examples/falcon/qlora.yml"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"scrolled": true
},
"outputs": [],
"source": [
"!accelerate launch scripts/finetune.py examples/falcon/qlora.yml"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
},
{
"attachments": {},
"cell_type": "markdown",
"metadata": {},
"source": [
"## Below are ad hoc cells handling issues during training"
]
},
{
"attachments": {},
"cell_type": "markdown",
"metadata": {},
"source": [
"### Force release VRAM"
]
},
{
"cell_type": "code",
"execution_count": 24,
"metadata": {},
"outputs": [],
"source": [
"# First interupt the kernel, wait a few seconds then run this to kill finetune to release VRAM\n",
"!ps aux|grep python|grep finetune|awk '{print $2}'|xargs kill"
]
},
{
"attachments": {},
"cell_type": "markdown",
"metadata": {},
"source": [
"### Clean the finetuned model and all checkpoints"
]
},
{
"cell_type": "code",
"execution_count": 28,
"metadata": {},
"outputs": [],
"source": [
"# Only run this to start over\n",
"!rm -rf ./qlora-out"
]
},
{
"attachments": {},
"cell_type": "markdown",
"metadata": {},
"source": [
"### Zip the prepared dataset"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"!apt install zip\n",
"!zip -r last_run_prepared.zip -xi last_run_prepared"
]
},
{
"attachments": {},
"cell_type": "markdown",
"metadata": {},
"source": [
"### Monitoring GPU"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# Run this in a seperate terminal\n",
"!nvitop -m full"
]
},
{
"attachments": {},
"cell_type": "markdown",
"metadata": {},
"source": [
"### Fix DISK FULL"
]
},
{
"cell_type": "code",
"execution_count": 14,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"/\n"
]
}
],
"source": [
"%cd /"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"!du -d 2 -h|grep G"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"!du -d 2 -h /root/.local"
]
},
{
"cell_type": "code",
"execution_count": 13,
"metadata": {},
"outputs": [],
"source": [
"!rm -rf /root/.local/share/Trash/"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"!rm -rf /root/.local/share/wandb/"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"!rm -rf /root/.cache/wandb/"
]
},
{
"attachments": {},
"cell_type": "markdown",
"metadata": {},
"source": [
"### Check who is using GPU"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"scrolled": true
},
"outputs": [],
"source": [
"!apt install lsof"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"scrolled": true
},
"outputs": [],
"source": [
"!lsof /dev/nvidia*"
]
},
{
"attachments": {},
"cell_type": "markdown",
"metadata": {},
"source": [
"### Upload checkpoints to HF"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"!cp -r /workspace/axolotl/qlora-out/checkpoint-120 /workspace/llm-playground/storage/falcon-qlora/"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"!python /workspace/llm-playground/helper/storage.py -u"
]
},
{
"attachments": {},
"cell_type": "markdown",
"metadata": {},
"source": [
"### Update axolotl"
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"/workspace\n"
]
}
],
"source": [
"%cd /workspace/"
]
},
{
"cell_type": "code",
"execution_count": 7,
"metadata": {
"scrolled": true
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Cloning into 'axolotl-update'...\n",
"remote: Enumerating objects: 2484, done.\u001b[K\n",
"remote: Counting objects: 100% (1234/1234), done.\u001b[K\n",
"remote: Compressing objects: 100% (356/356), done.\u001b[K\n",
"remote: Total 2484 (delta 907), reused 1082 (delta 812), pack-reused 1250\u001b[K\n",
"Receiving objects: 100% (2484/2484), 1.30 MiB | 3.02 MiB/s, done.\n",
"Resolving deltas: 100% (1539/1539), done.\n"
]
}
],
"source": [
"!git clone https://github.com/OpenAccess-AI-Collective/axolotl axolotl-update"
]
},
{
"cell_type": "code",
"execution_count": 8,
"metadata": {},
"outputs": [],
"source": [
"!cp -r axolotl-update/* axolotl"
]
},
{
"cell_type": "code",
"execution_count": 9,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"/workspace/axolotl\n"
]
}
],
"source": [
"%cd /workspace/axolotl"
]
},
{
"cell_type": "code",
"execution_count": 10,
"metadata": {
"scrolled": true
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"On branch main\n",
"Your branch is up to date with 'origin/main'.\n",
"\n",
"Changes not staged for commit:\n",
" (use \"git add ...\" to update what will be committed)\n",
" (use \"git restore ...\" to discard changes in working directory)\n",
"\t\u001b[31mmodified: README.md\u001b[m\n",
"\t\u001b[31mmodified: configs/cerebras_1_3B_alpaca.yml\u001b[m\n",
"\t\u001b[31mmodified: configs/galactica_1_3B.yml\u001b[m\n",
"\t\u001b[31mmodified: configs/gpt_neox_20b.yml\u001b[m\n",
"\t\u001b[31mmodified: configs/llama_13B_alpaca.yml\u001b[m\n",
"\t\u001b[31mmodified: configs/llama_65B_alpaca.yml\u001b[m\n",
"\t\u001b[31mmodified: configs/llama_7B_4bit.yml\u001b[m\n",
"\t\u001b[31mmodified: configs/llama_7B_alpaca.yml\u001b[m\n",
"\t\u001b[31mmodified: configs/llama_7B_jeopardy.yml\u001b[m\n",
"\t\u001b[31mmodified: configs/pythia_1_2B_alpaca.yml\u001b[m\n",
"\t\u001b[31mmodified: configs/quickstart.yml\u001b[m\n",
"\t\u001b[31mmodified: configs/sample.yml\u001b[m\n",
"\t\u001b[31mmodified: configs/stability_3b.yml\u001b[m\n",
"\t\u001b[31mmodified: configs/vicuna_13B_4bit_reflect.yml\u001b[m\n",
"\t\u001b[31mmodified: docker/Dockerfile-base\u001b[m\n",
"\t\u001b[31mmodified: examples/gptq-lora-7b/config.yml\u001b[m\n",
"\t\u001b[31mmodified: examples/mpt-7b/config.yml\u001b[m\n",
"\t\u001b[31mmodified: requirements.txt\u001b[m\n",
"\t\u001b[31mmodified: scripts/finetune.py\u001b[m\n",
"\t\u001b[31mmodified: scripts/runpod-entrypoint.sh\u001b[m\n",
"\t\u001b[31mmodified: src/axolotl/datasets.py\u001b[m\n",
"\t\u001b[31mmodified: src/axolotl/flash_attn.py\u001b[m\n",
"\t\u001b[31mmodified: src/axolotl/utils/data.py\u001b[m\n",
"\t\u001b[31mmodified: src/axolotl/utils/models.py\u001b[m\n",
"\t\u001b[31mmodified: src/axolotl/utils/validation.py\u001b[m\n",
"\t\u001b[31mmodified: tests/test_prompt_tokenizers.py\u001b[m\n",
"\t\u001b[31mmodified: tests/test_validation.py\u001b[m\n",
"\n",
"Untracked files:\n",
" (use \"git add ...\" to include in what will be committed)\n",
"\t\u001b[31mexamples/falcon/lora.yml\u001b[m\n",
"\t\u001b[31mexamples/falcon/qlora.yml\u001b[m\n",
"\t\u001b[31mexamples/huggyllama/\u001b[m\n",
"\t\u001b[31mexamples/openllama/\u001b[m\n",
"\t\u001b[31mimage/axolotl-badge-web.png\u001b[m\n",
"\t\u001b[31msrc/axolotl/monkeypatch/\u001b[m\n",
"\t\u001b[31mtests/fixtures/alpaca/\u001b[m\n",
"\t\u001b[31mtests/test_packed_dataset.py\u001b[m\n",
"\n",
"no changes added to commit (use \"git add\" and/or \"git commit -a\")\n"
]
}
],
"source": [
"!git status"
]
},
{
"cell_type": "code",
"execution_count": 11,
"metadata": {
"scrolled": true
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Obtaining file:///workspace/axolotl\n",
" Preparing metadata (setup.py) ... \u001b[?25ldone\n",
"\u001b[?25hCollecting transformers@ git+https://github.com/huggingface/transformers.git (from axolotl==0.1)\n",
" Cloning https://github.com/huggingface/transformers.git to /tmp/pip-install-w8_gs79m/transformers_d7bb90e3e98d4c5c855b5488e6abe26d\n",
" Running command git clone --filter=blob:none --quiet https://github.com/huggingface/transformers.git /tmp/pip-install-w8_gs79m/transformers_d7bb90e3e98d4c5c855b5488e6abe26d\n",
" Resolved https://github.com/huggingface/transformers.git to commit f49a3453caa6fe606bb31c571423f72264152fce\n",
" Installing build dependencies ... \u001b[?25ldone\n",
"\u001b[?25h Getting requirements to build wheel ... \u001b[?25ldone\n",
"\u001b[?25h Preparing metadata (pyproject.toml) ... \u001b[?25ldone\n",
"\u001b[?25hRequirement already satisfied: bitsandbytes>=0.39.0 in /root/miniconda3/envs/py3.9/lib/python3.9/site-packages/bitsandbytes-0.39.0-py3.9.egg (from axolotl==0.1) (0.39.0)\n",
"Requirement already satisfied: accelerate in /root/miniconda3/envs/py3.9/lib/python3.9/site-packages (from axolotl==0.1) (0.20.0.dev0)\n",
"Requirement already satisfied: addict in /root/miniconda3/envs/py3.9/lib/python3.9/site-packages (from axolotl==0.1) (2.4.0)\n",
"Requirement already satisfied: fire in /root/miniconda3/envs/py3.9/lib/python3.9/site-packages (from axolotl==0.1) (0.5.0)\n",
"Requirement already satisfied: PyYAML==6.0 in /root/miniconda3/envs/py3.9/lib/python3.9/site-packages (from axolotl==0.1) (6.0)\n",
"Requirement already satisfied: datasets in /root/miniconda3/envs/py3.9/lib/python3.9/site-packages (from axolotl==0.1) (2.12.0)\n",
"Requirement already satisfied: sentencepiece in /root/miniconda3/envs/py3.9/lib/python3.9/site-packages (from axolotl==0.1) (0.1.99)\n",
"Requirement already satisfied: wandb in /root/miniconda3/envs/py3.9/lib/python3.9/site-packages (from axolotl==0.1) (0.15.3)\n",
"Requirement already satisfied: einops in /root/miniconda3/envs/py3.9/lib/python3.9/site-packages (from axolotl==0.1) (0.6.1)\n",
"Requirement already satisfied: xformers in /root/miniconda3/envs/py3.9/lib/python3.9/site-packages (from axolotl==0.1) (0.0.20)\n",
"Requirement already satisfied: bert-score==0.3.13 in /root/miniconda3/envs/py3.9/lib/python3.9/site-packages (from axolotl==0.1) (0.3.13)\n",
"Requirement already satisfied: evaluate==0.4.0 in /root/miniconda3/envs/py3.9/lib/python3.9/site-packages (from axolotl==0.1) (0.4.0)\n",
"Requirement already satisfied: rouge-score==0.1.2 in /root/miniconda3/envs/py3.9/lib/python3.9/site-packages (from axolotl==0.1) (0.1.2)\n",
"Requirement already satisfied: scipy in /root/miniconda3/envs/py3.9/lib/python3.9/site-packages (from axolotl==0.1) (1.10.1)\n",
"Requirement already satisfied: scikit-learn==1.2.2 in /root/miniconda3/envs/py3.9/lib/python3.9/site-packages (from axolotl==0.1) (1.2.2)\n",
"Requirement already satisfied: torch>=1.0.0 in /root/miniconda3/envs/py3.9/lib/python3.9/site-packages (from bert-score==0.3.13->axolotl==0.1) (2.0.1)\n",
"Requirement already satisfied: pandas>=1.0.1 in /root/miniconda3/envs/py3.9/lib/python3.9/site-packages (from bert-score==0.3.13->axolotl==0.1) (2.0.2)\n",
"Requirement already satisfied: numpy in /root/miniconda3/envs/py3.9/lib/python3.9/site-packages (from bert-score==0.3.13->axolotl==0.1) (1.24.3)\n",
"Requirement already satisfied: requests in /root/miniconda3/envs/py3.9/lib/python3.9/site-packages (from bert-score==0.3.13->axolotl==0.1) (2.31.0)\n",
"Requirement already satisfied: tqdm>=4.31.1 in /root/miniconda3/envs/py3.9/lib/python3.9/site-packages (from bert-score==0.3.13->axolotl==0.1) (4.65.0)\n",
"Requirement already satisfied: matplotlib in /root/miniconda3/envs/py3.9/lib/python3.9/site-packages (from bert-score==0.3.13->axolotl==0.1) (3.7.1)\n",
"Requirement already satisfied: packaging>=20.9 in /root/miniconda3/envs/py3.9/lib/python3.9/site-packages (from bert-score==0.3.13->axolotl==0.1) (23.1)\n",
"Requirement already satisfied: dill in /root/miniconda3/envs/py3.9/lib/python3.9/site-packages (from evaluate==0.4.0->axolotl==0.1) (0.3.6)\n",
"Requirement already satisfied: xxhash in /root/miniconda3/envs/py3.9/lib/python3.9/site-packages (from evaluate==0.4.0->axolotl==0.1) (3.2.0)\n",
"Requirement already satisfied: multiprocess in /root/miniconda3/envs/py3.9/lib/python3.9/site-packages (from evaluate==0.4.0->axolotl==0.1) (0.70.14)\n",
"Requirement already satisfied: fsspec[http]>=2021.05.0 in /root/miniconda3/envs/py3.9/lib/python3.9/site-packages (from evaluate==0.4.0->axolotl==0.1) (2023.5.0)\n",
"Requirement already satisfied: huggingface-hub>=0.7.0 in /root/miniconda3/envs/py3.9/lib/python3.9/site-packages (from evaluate==0.4.0->axolotl==0.1) (0.14.1)\n",
"Requirement already satisfied: responses<0.19 in /root/miniconda3/envs/py3.9/lib/python3.9/site-packages (from evaluate==0.4.0->axolotl==0.1) (0.18.0)\n",
"Requirement already satisfied: absl-py in /root/miniconda3/envs/py3.9/lib/python3.9/site-packages (from rouge-score==0.1.2->axolotl==0.1) (1.4.0)\n",
"Requirement already satisfied: nltk in /root/miniconda3/envs/py3.9/lib/python3.9/site-packages (from rouge-score==0.1.2->axolotl==0.1) (3.8.1)\n",
"Requirement already satisfied: six>=1.14.0 in /root/miniconda3/envs/py3.9/lib/python3.9/site-packages (from rouge-score==0.1.2->axolotl==0.1) (1.16.0)\n",
"Requirement already satisfied: joblib>=1.1.1 in /root/miniconda3/envs/py3.9/lib/python3.9/site-packages (from scikit-learn==1.2.2->axolotl==0.1) (1.2.0)\n",
"Requirement already satisfied: threadpoolctl>=2.0.0 in /root/miniconda3/envs/py3.9/lib/python3.9/site-packages (from scikit-learn==1.2.2->axolotl==0.1) (3.1.0)\n",
"Requirement already satisfied: psutil in /root/miniconda3/envs/py3.9/lib/python3.9/site-packages (from accelerate->axolotl==0.1) (5.9.5)\n",
"Requirement already satisfied: pyarrow>=8.0.0 in /root/miniconda3/envs/py3.9/lib/python3.9/site-packages (from datasets->axolotl==0.1) (12.0.0)\n",
"Requirement already satisfied: aiohttp in /root/miniconda3/envs/py3.9/lib/python3.9/site-packages (from datasets->axolotl==0.1) (3.8.4)\n",
"Requirement already satisfied: filelock in /root/miniconda3/envs/py3.9/lib/python3.9/site-packages (from transformers@ git+https://github.com/huggingface/transformers.git->axolotl==0.1) (3.12.0)\n",
"Requirement already satisfied: regex!=2019.12.17 in /root/miniconda3/envs/py3.9/lib/python3.9/site-packages (from transformers@ git+https://github.com/huggingface/transformers.git->axolotl==0.1) (2023.5.5)\n",
"Requirement already satisfied: tokenizers!=0.11.3,<0.14,>=0.11.1 in /root/miniconda3/envs/py3.9/lib/python3.9/site-packages (from transformers@ git+https://github.com/huggingface/transformers.git->axolotl==0.1) (0.13.3)\n",
"Requirement already satisfied: safetensors>=0.3.1 in /root/miniconda3/envs/py3.9/lib/python3.9/site-packages (from transformers@ git+https://github.com/huggingface/transformers.git->axolotl==0.1) (0.3.1)\n",
"Requirement already satisfied: termcolor in /root/miniconda3/envs/py3.9/lib/python3.9/site-packages (from fire->axolotl==0.1) (2.3.0)\n",
"Requirement already satisfied: Click!=8.0.0,>=7.0 in /root/miniconda3/envs/py3.9/lib/python3.9/site-packages (from wandb->axolotl==0.1) (8.1.3)\n",
"Requirement already satisfied: GitPython!=3.1.29,>=1.0.0 in /root/miniconda3/envs/py3.9/lib/python3.9/site-packages (from wandb->axolotl==0.1) (3.1.31)\n",
"Requirement already satisfied: sentry-sdk>=1.0.0 in /root/miniconda3/envs/py3.9/lib/python3.9/site-packages (from wandb->axolotl==0.1) (1.24.0)\n",
"Requirement already satisfied: docker-pycreds>=0.4.0 in /root/miniconda3/envs/py3.9/lib/python3.9/site-packages (from wandb->axolotl==0.1) (0.4.0)\n",
"Requirement already satisfied: pathtools in /root/miniconda3/envs/py3.9/lib/python3.9/site-packages (from wandb->axolotl==0.1) (0.1.2)\n",
"Requirement already satisfied: setproctitle in /root/miniconda3/envs/py3.9/lib/python3.9/site-packages (from wandb->axolotl==0.1) (1.3.2)\n",
"Requirement already satisfied: setuptools in /root/miniconda3/envs/py3.9/lib/python3.9/site-packages (from wandb->axolotl==0.1) (67.8.0)\n",
"Requirement already satisfied: appdirs>=1.4.3 in /root/miniconda3/envs/py3.9/lib/python3.9/site-packages (from wandb->axolotl==0.1) (1.4.4)\n",
"Requirement already satisfied: typing-extensions in /root/miniconda3/envs/py3.9/lib/python3.9/site-packages (from wandb->axolotl==0.1) (4.6.2)\n",
"Requirement already satisfied: protobuf!=4.21.0,<5,>=3.15.0 in /root/miniconda3/envs/py3.9/lib/python3.9/site-packages (from wandb->axolotl==0.1) (4.23.2)\n",
"Requirement already satisfied: pyre-extensions==0.0.29 in /root/miniconda3/envs/py3.9/lib/python3.9/site-packages (from xformers->axolotl==0.1) (0.0.29)\n",
"Requirement already satisfied: typing-inspect in /root/miniconda3/envs/py3.9/lib/python3.9/site-packages (from pyre-extensions==0.0.29->xformers->axolotl==0.1) (0.9.0)\n",
"Requirement already satisfied: sympy in /root/miniconda3/envs/py3.9/lib/python3.9/site-packages (from torch>=1.0.0->bert-score==0.3.13->axolotl==0.1) (1.12)\n",
"Requirement already satisfied: networkx in /root/miniconda3/envs/py3.9/lib/python3.9/site-packages (from torch>=1.0.0->bert-score==0.3.13->axolotl==0.1) (3.1)\n",
"Requirement already satisfied: jinja2 in /root/miniconda3/envs/py3.9/lib/python3.9/site-packages (from torch>=1.0.0->bert-score==0.3.13->axolotl==0.1) (3.1.2)\n",
"Requirement already satisfied: nvidia-cuda-nvrtc-cu11==11.7.99 in /root/miniconda3/envs/py3.9/lib/python3.9/site-packages (from torch>=1.0.0->bert-score==0.3.13->axolotl==0.1) (11.7.99)\n",
"Requirement already satisfied: nvidia-cuda-runtime-cu11==11.7.99 in /root/miniconda3/envs/py3.9/lib/python3.9/site-packages (from torch>=1.0.0->bert-score==0.3.13->axolotl==0.1) (11.7.99)\n",
"Requirement already satisfied: nvidia-cuda-cupti-cu11==11.7.101 in /root/miniconda3/envs/py3.9/lib/python3.9/site-packages (from torch>=1.0.0->bert-score==0.3.13->axolotl==0.1) (11.7.101)\n",
"Requirement already satisfied: nvidia-cudnn-cu11==8.5.0.96 in /root/miniconda3/envs/py3.9/lib/python3.9/site-packages (from torch>=1.0.0->bert-score==0.3.13->axolotl==0.1) (8.5.0.96)\n",
"Requirement already satisfied: nvidia-cublas-cu11==11.10.3.66 in /root/miniconda3/envs/py3.9/lib/python3.9/site-packages (from torch>=1.0.0->bert-score==0.3.13->axolotl==0.1) (11.10.3.66)\n",
"Requirement already satisfied: nvidia-cufft-cu11==10.9.0.58 in /root/miniconda3/envs/py3.9/lib/python3.9/site-packages (from torch>=1.0.0->bert-score==0.3.13->axolotl==0.1) (10.9.0.58)\n",
"Requirement already satisfied: nvidia-curand-cu11==10.2.10.91 in /root/miniconda3/envs/py3.9/lib/python3.9/site-packages (from torch>=1.0.0->bert-score==0.3.13->axolotl==0.1) (10.2.10.91)\n",
"Requirement already satisfied: nvidia-cusolver-cu11==11.4.0.1 in /root/miniconda3/envs/py3.9/lib/python3.9/site-packages (from torch>=1.0.0->bert-score==0.3.13->axolotl==0.1) (11.4.0.1)\n",
"Requirement already satisfied: nvidia-cusparse-cu11==11.7.4.91 in /root/miniconda3/envs/py3.9/lib/python3.9/site-packages (from torch>=1.0.0->bert-score==0.3.13->axolotl==0.1) (11.7.4.91)\n",
"Requirement already satisfied: nvidia-nccl-cu11==2.14.3 in /root/miniconda3/envs/py3.9/lib/python3.9/site-packages (from torch>=1.0.0->bert-score==0.3.13->axolotl==0.1) (2.14.3)\n",
"Requirement already satisfied: nvidia-nvtx-cu11==11.7.91 in /root/miniconda3/envs/py3.9/lib/python3.9/site-packages (from torch>=1.0.0->bert-score==0.3.13->axolotl==0.1) (11.7.91)\n",
"Requirement already satisfied: triton==2.0.0 in /root/miniconda3/envs/py3.9/lib/python3.9/site-packages (from torch>=1.0.0->bert-score==0.3.13->axolotl==0.1) (2.0.0)\n",
"Requirement already satisfied: wheel in /root/miniconda3/envs/py3.9/lib/python3.9/site-packages (from nvidia-cublas-cu11==11.10.3.66->torch>=1.0.0->bert-score==0.3.13->axolotl==0.1) (0.40.0)\n",
"Requirement already satisfied: cmake in /root/miniconda3/envs/py3.9/lib/python3.9/site-packages (from triton==2.0.0->torch>=1.0.0->bert-score==0.3.13->axolotl==0.1) (3.26.3)\n",
"Requirement already satisfied: lit in /root/miniconda3/envs/py3.9/lib/python3.9/site-packages (from triton==2.0.0->torch>=1.0.0->bert-score==0.3.13->axolotl==0.1) (16.0.5)\n",
"Requirement already satisfied: attrs>=17.3.0 in /root/miniconda3/envs/py3.9/lib/python3.9/site-packages (from aiohttp->datasets->axolotl==0.1) (23.1.0)\n",
"Requirement already satisfied: charset-normalizer<4.0,>=2.0 in /root/miniconda3/envs/py3.9/lib/python3.9/site-packages (from aiohttp->datasets->axolotl==0.1) (3.1.0)\n",
"Requirement already satisfied: multidict<7.0,>=4.5 in /root/miniconda3/envs/py3.9/lib/python3.9/site-packages (from aiohttp->datasets->axolotl==0.1) (6.0.4)\n",
"Requirement already satisfied: async-timeout<5.0,>=4.0.0a3 in /root/miniconda3/envs/py3.9/lib/python3.9/site-packages (from aiohttp->datasets->axolotl==0.1) (4.0.2)\n",
"Requirement already satisfied: yarl<2.0,>=1.0 in /root/miniconda3/envs/py3.9/lib/python3.9/site-packages (from aiohttp->datasets->axolotl==0.1) (1.9.2)\n",
"Requirement already satisfied: frozenlist>=1.1.1 in /root/miniconda3/envs/py3.9/lib/python3.9/site-packages (from aiohttp->datasets->axolotl==0.1) (1.3.3)\n",
"Requirement already satisfied: aiosignal>=1.1.2 in /root/miniconda3/envs/py3.9/lib/python3.9/site-packages (from aiohttp->datasets->axolotl==0.1) (1.3.1)\n",
"Requirement already satisfied: gitdb<5,>=4.0.1 in /root/miniconda3/envs/py3.9/lib/python3.9/site-packages (from GitPython!=3.1.29,>=1.0.0->wandb->axolotl==0.1) (4.0.10)\n",
"Requirement already satisfied: python-dateutil>=2.8.2 in /root/miniconda3/envs/py3.9/lib/python3.9/site-packages (from pandas>=1.0.1->bert-score==0.3.13->axolotl==0.1) (2.8.2)\n",
"Requirement already satisfied: pytz>=2020.1 in /root/miniconda3/envs/py3.9/lib/python3.9/site-packages (from pandas>=1.0.1->bert-score==0.3.13->axolotl==0.1) (2023.3)\n",
"Requirement already satisfied: tzdata>=2022.1 in /root/miniconda3/envs/py3.9/lib/python3.9/site-packages (from pandas>=1.0.1->bert-score==0.3.13->axolotl==0.1) (2023.3)\n",
"Requirement already satisfied: idna<4,>=2.5 in /root/miniconda3/envs/py3.9/lib/python3.9/site-packages (from requests->bert-score==0.3.13->axolotl==0.1) (3.4)\n",
"Requirement already satisfied: urllib3<3,>=1.21.1 in /root/miniconda3/envs/py3.9/lib/python3.9/site-packages (from requests->bert-score==0.3.13->axolotl==0.1) (1.26.16)\n",
"Requirement already satisfied: certifi>=2017.4.17 in /root/miniconda3/envs/py3.9/lib/python3.9/site-packages (from requests->bert-score==0.3.13->axolotl==0.1) (2023.5.7)\n",
"Requirement already satisfied: contourpy>=1.0.1 in /root/miniconda3/envs/py3.9/lib/python3.9/site-packages (from matplotlib->bert-score==0.3.13->axolotl==0.1) (1.0.7)\n",
"Requirement already satisfied: cycler>=0.10 in /root/miniconda3/envs/py3.9/lib/python3.9/site-packages (from matplotlib->bert-score==0.3.13->axolotl==0.1) (0.11.0)\n",
"Requirement already satisfied: fonttools>=4.22.0 in /root/miniconda3/envs/py3.9/lib/python3.9/site-packages (from matplotlib->bert-score==0.3.13->axolotl==0.1) (4.39.4)\n",
"Requirement already satisfied: kiwisolver>=1.0.1 in /root/miniconda3/envs/py3.9/lib/python3.9/site-packages (from matplotlib->bert-score==0.3.13->axolotl==0.1) (1.4.4)\n",
"Requirement already satisfied: pillow>=6.2.0 in /root/miniconda3/envs/py3.9/lib/python3.9/site-packages (from matplotlib->bert-score==0.3.13->axolotl==0.1) (9.5.0)\n",
"Requirement already satisfied: pyparsing>=2.3.1 in /root/miniconda3/envs/py3.9/lib/python3.9/site-packages (from matplotlib->bert-score==0.3.13->axolotl==0.1) (3.0.9)\n",
"Requirement already satisfied: importlib-resources>=3.2.0 in /root/miniconda3/envs/py3.9/lib/python3.9/site-packages (from matplotlib->bert-score==0.3.13->axolotl==0.1) (5.12.0)\n",
"Requirement already satisfied: smmap<6,>=3.0.1 in /root/miniconda3/envs/py3.9/lib/python3.9/site-packages (from gitdb<5,>=4.0.1->GitPython!=3.1.29,>=1.0.0->wandb->axolotl==0.1) (5.0.0)\n",
"Requirement already satisfied: zipp>=3.1.0 in /root/miniconda3/envs/py3.9/lib/python3.9/site-packages (from importlib-resources>=3.2.0->matplotlib->bert-score==0.3.13->axolotl==0.1) (3.15.0)\n",
"Requirement already satisfied: MarkupSafe>=2.0 in /root/miniconda3/envs/py3.9/lib/python3.9/site-packages (from jinja2->torch>=1.0.0->bert-score==0.3.13->axolotl==0.1) (2.1.2)\n",
"Requirement already satisfied: mpmath>=0.19 in /root/miniconda3/envs/py3.9/lib/python3.9/site-packages (from sympy->torch>=1.0.0->bert-score==0.3.13->axolotl==0.1) (1.3.0)\n",
"Requirement already satisfied: mypy-extensions>=0.3.0 in /root/miniconda3/envs/py3.9/lib/python3.9/site-packages (from typing-inspect->pyre-extensions==0.0.29->xformers->axolotl==0.1) (1.0.0)\n",
"Installing collected packages: axolotl\n",
" Attempting uninstall: axolotl\n",
" Found existing installation: axolotl 0.1\n",
" Uninstalling axolotl-0.1:\n",
" Successfully uninstalled axolotl-0.1\n",
" Running setup.py develop for axolotl\n",
"Successfully installed axolotl-0.1\n",
"\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
"\u001b[0m"
]
}
],
"source": [
"!pip install -e ."
]
},
{
"attachments": {},
"cell_type": "markdown",
"metadata": {},
"source": [
"### A new bash without tmux etc."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"!bash --norc --noprofile"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### Clean up all checkpoints but last one"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"cd /workspace/axolotl/qlora-out/ && ls -lhta |grep checkpoint- | awk 'NR > 1 {print $9}' | xargs rm -rf"
]
}
],
"metadata": {
"accelerator": "GPU",
"colab": {
"authorship_tag": "ABX9TyP0jHt4WNuLaC5ecmX0YtWl",
"gpuType": "T4",
"include_colab_link": true,
"provenance": [],
"toc_visible": true
},
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.9.16"
}
},
"nbformat": 4,
"nbformat_minor": 4
}