![\"Open](\"https://colab.research.google.com/assets/colab-badge.svg\"){kind=icon}
"
]
},
{
"cell_type": "markdown",
"source": [
"# Install Dependencies"
],
"metadata": {
"id": "aKracwOClKuS"
}
},
{
"cell_type": "code",
"source": [
"!pip install datasets peft -qq\n",
"!pip install accelerate -qq\n",
"!pip install bitsandbytes -qq"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "2b5BOF6oG2Cz",
"outputId": "696dc0a6-21bb-4d75-fe9e-bf7c7643a7ac"
},
"execution_count": null,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"\u001b[?25l \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m0.0/542.0 kB\u001b[0m \u001b[31m?\u001b[0m eta \u001b[36m-:--:--\u001b[0m\r\u001b[2K \u001b[91m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[91m╸\u001b[0m\u001b[90m━\u001b[0m \u001b[32m522.2/542.0 kB\u001b[0m \u001b[31m15.7 MB/s\u001b[0m eta \u001b[36m0:00:01\u001b[0m\r\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m542.0/542.0 kB\u001b[0m \u001b[31m11.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m251.6/251.6 kB\u001b[0m \u001b[31m33.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m116.3/116.3 kB\u001b[0m \u001b[31m17.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m194.1/194.1 kB\u001b[0m \u001b[31m23.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m134.8/134.8 kB\u001b[0m \u001b[31m19.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m302.6/302.6 kB\u001b[0m \u001b[31m35.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m21.3/21.3 MB\u001b[0m \u001b[31m75.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m119.8/119.8 MB\u001b[0m \u001b[31m13.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
"\u001b[?25h"
]
}
]
},
{
"cell_type": "markdown",
"source": [
"# Import Packages"
],
"metadata": {
"id": "cqly9oSflQuQ"
}
},
{
"cell_type": "code",
"source": [
"from google.colab import userdata\n",
"from huggingface_hub import login\n",
"from datasets import load_dataset\n",
"import torch\n",
"\n",
"from transformers import (\n",
" AutoTokenizer,\n",
" AutoModelForCausalLM,\n",
" TrainingArguments,\n",
" DataCollatorForLanguageModeling,\n",
" Trainer,\n",
" BitsAndBytesConfig)\n",
"\n",
"from peft import prepare_model_for_kbit_training, LoraConfig, get_peft_model"
],
"metadata": {
"id": "ykbsZ-1-NahC"
},
"execution_count": null,
"outputs": []
},
{
"cell_type": "markdown",
"source": [
"# Load Model and Tokenizer"
],
"metadata": {
"id": "PgJ4Idaelv-v"
}
},
{
"cell_type": "code",
"source": [
"# Define the model ID for the pre-trained model\n",
"model_id = \"google/gemma-2b\"\n",
"\n",
"# Configure BitsAndBytes for 4-bit quantization\n",
"bnb_config = BitsAndBytesConfig(\n",
" load_in_4bit=True,\n",
" bnb_4bit_quant_type=\"nf4\",\n",
" bnb_4bit_compute_dtype=torch.bfloat16)\n",
"\n",
"# Load the tokenizer\n",
"tokenizer = AutoTokenizer.from_pretrained(model_id)\n",
"\n",
"# Load the model with the specified quantization configuration and device mapping\n",
"base_model = AutoModelForCausalLM.from_pretrained(model_id, quantization_config=bnb_config, device_map={\"\":0})\n",
"\n",
"# Check if the tokenizer has a pad token defined\n",
"if tokenizer.pad_token is None:\n",
" # If not, assign the end of sequence (eos) token as the pad token\n",
" tokenizer.pad_token = tokenizer.eos_token"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/",
"height": 49,
"referenced_widgets": [
"c2b0dd991c894171a8b182f663096e1c",
"ae92fdb44f2844b0ac0341c7ac9101bf",
"8a2ed972123843d7b96688adfdf0c3e6",
"be25e2566e0f4150a46bbe10fb82dd6e",
"54588ba416534694b2c584cd4278ff44",
"4088585d52f04f7682a5bb15112eccad",
"b6441e2be61a4d72bd2572eea6857e4c",
"65325dcae63440c8b6e4493b88fac7a9",
"82653ed2585d403a844d0495a8e9ff10",
"2794c9576ad9463c829a6e8ed8e89b9d",
"2998098d68504f3985f4e1cb10098eaf"
]
},
"id": "wJ7MkBL4DYIw",
"outputId": "a7b6e667-500d-4a6e-b9d9-2e6f5171ff45"
},
"execution_count": null,
"outputs": [
{
"output_type": "display_data",
"data": {
"text/plain": [
"Loading checkpoint shards: 0%| | 0/2 [00:00"
],
"text/html": [
"\n",
" | Step | \n", "Training Loss | \n", "
|---|---|
| 1 | \n", "1.537000 | \n", "
| 2 | \n", "1.443500 | \n", "
" ] }, "metadata": {} }, { "output_type": "stream", "name": "stderr", "text": [ "/usr/local/lib/python3.10/dist-packages/peft/utils/save_and_load.py:180: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.\n", " warnings.warn(\"Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.\")\n", "/usr/local/lib/python3.10/dist-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n", " warnings.warn(\n", "/usr/local/lib/python3.10/dist-packages/peft/utils/save_and_load.py:180: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.\n", " warnings.warn(\"Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.\")\n" ] }, { "output_type": "execute_result", "data": { "text/plain": [ "TrainOutput(global_step=2, training_loss=1.4902946949005127, metrics={'train_runtime': 17.0333, 'train_samples_per_second': 0.117, 'train_steps_per_second': 0.117, 'total_flos': 550276276224.0, 'train_loss': 1.4902946949005127, 'epoch': 0.0007974481658692185})" ] }, "metadata": {}, "execution_count": 29 } ] }, { "cell_type": "markdown", "source": [ "# Save Model and Push to Hub" ], "metadata": { "id": "vfrjVIN2rOWf" } }, { "cell_type": "code", "source": [ "# # Renable warnings for inference\n", "model.config.use_cache = True\n", "\n", "# Save trained model\n", "trainer.model.save_pretrained(\"gemma-ft\", save_embedding_layers=True)" ], "metadata": { "id": "s5v6uT-PDYCO" }, "execution_count": null, "outputs": [] }, { "cell_type": "code", "source": [ "write_key = userdata.get('HF_TOKEN')\n", "login(write_key)" ], "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "jAyJq-3XDX9r", "outputId": "b5115872-c0fa-4e57-9ce5-68a5084ca95c" }, "execution_count": null, "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ "The token has not been saved to the git credentials helper. Pass `add_to_git_credential=True` in this function directly or `--add-to-git-credential` if using via `huggingface-cli` if you want to set the git credential as well.\n", "Token is valid (permission: write).\n", "Your token has been saved to /root/.cache/huggingface/token\n", "Login successful\n" ] } ] }, { "cell_type": "code", "source": [ "# from huggingface_hub import notebook_login\n", "# notebook_login()" ], "metadata": { "id": "paRZrxqCbjxI" }, "execution_count": null, "outputs": [] }, { "cell_type": "code", "source": [ "# Push LoRA Adapters\n", "\n", "trainer.push_to_hub(\"bmartinc80/gemma-ft\")" ], "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 255, "referenced_widgets": [ "517ba61eab604dbeafe8e9ed3da92ac3", "ddc20b0d5b684ea08584f35533876228", "7cdf2d82b2fa4579867cd54f4259cfd0", "9c5c7d47a5eb49e585372099c853db72", "aa1cbc4b5cf04e75b9495096bd116802", "f9fe1a6e91764a558d408e79609d92df", "6268ffdedcd6461c8a35a8578280a2fd", "85df3a34c5ea437793ad2eb01b8ee985", "08641e9c7b8e4befb6d32752a31908a2", "ef0714869adf49f58b2acc4cbfd07512", "803ca1b223194d93adeb89b7d2172547", "e5d7de61b1af4ebcaf6e4cb22b25464d", "6bf56bf6dc4b4a9081d172013974d734", "08bd68a82ac441ca843f48043b068159", "9374cc005ffc4814adccdc21e95ce4be", "1f4cff4178ff4d29957225a2bfdfb873", "f99180ceb12c45bd8dc5ba0533d5bab7", "638a09691932455e8d51a8527eb5d9cb", "e1635fbf4f7148e9bb067af8c5449c75", "63683fe1b8734939867081a0876a3964", "4b7a7f1c356e4ade9022be7fa3bc8ed5", "b053e218c8d04ea48339bb2f07c79558", "23035072acb1423da18015e29931fa8f", "718b2d7294194f1fa93677d9dca43e80", "dcb560c5bf6949778f8ef8f4dbb4ab3a", "7acd40975f2745e7af694285e80e56df", "7f34e5ee46d744a085e012a6b83a0f57", "7c977c78b05c4443a67d4f98a71d4383", "ecee013cdc444d92b894ea8ea76fed4d", "cb4f08ce2edb4cd0aa70bf610bfa8312", "22fae968553b407db9d9cadea46a06f9", "a626aa9e63f54a61b80693fac3f56e78", "e00a031b7ed44c3484f779a0d41c8c1f", "6727615955a8426ea3dcc425f8697334", "d8e8478b162d4016ac5f50ef35c65ad5", "64fe0bce16d74815b97edbb3351f7038", "6c5b03899a214de48413d70af5771959", "fc768a909f9348418b7810b41869fba5", "603fdc7a558242b98f5baba284527c08", "a46dd19154f3454e8e8c2d10d5708204", "a310fbf9b8f2499babf38f16aae54ee5", "e4c22055b01d4165af078c86a66940ea", "4c7e1a97ef13435e8dfd28c4d3546e6b", "991fdb8a2c6d45ee873d8edeeb3d68b4" ] }, "id": "D6UenhhuDX7Z", "outputId": "75d54c24-6d96-43e6-9017-059a70ecbabf" }, "execution_count": null, "outputs": [ { "output_type": "stream", "name": "stderr", "text": [ "/usr/local/lib/python3.10/dist-packages/peft/utils/save_and_load.py:180: UserWarning: Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.\n", " warnings.warn(\"Setting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.\")\n" ] }, { "output_type": "display_data", "data": { "text/plain": [ "Upload 3 LFS files: 0%| | 0/3 [00:00