![\"Open](\"https://colab.research.google.com/assets/colab-badge.svg\"){kind=icon}
"
]
},
{
"cell_type": "code",
"source": [
"!rm -rf outputs wandb sample_data"
],
"metadata": {
"id": "TorAghWGcV9t"
},
"execution_count": 1,
"outputs": []
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {
"id": "FuXIFTFapAMI",
"colab": {
"base_uri": "https://localhost:8080/"
},
"outputId": "f1ed378e-6534-4b9a-de52-a27a07ee88d6"
},
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m92.2/92.2 MB\u001b[0m \u001b[31m9.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
"\u001b[?25h Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n",
" Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n",
" Preparing metadata (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m224.5/224.5 kB\u001b[0m \u001b[31m7.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m7.8/7.8 MB\u001b[0m \u001b[31m94.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.3/1.3 MB\u001b[0m \u001b[31m71.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
"\u001b[?25h Building wheel for transformers (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",
" Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n",
" Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n",
" Preparing metadata (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m219.1/219.1 kB\u001b[0m \u001b[31m8.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
"\u001b[?25h Building wheel for peft (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",
" Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n",
" Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n",
" Preparing metadata (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",
" Building wheel for accelerate (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m474.6/474.6 kB\u001b[0m \u001b[31m14.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m110.5/110.5 kB\u001b[0m \u001b[31m14.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m212.5/212.5 kB\u001b[0m \u001b[31m15.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m134.3/134.3 kB\u001b[0m \u001b[31m17.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.0/1.0 MB\u001b[0m \u001b[31m48.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m114.5/114.5 kB\u001b[0m \u001b[31m10.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m268.8/268.8 kB\u001b[0m \u001b[31m32.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m149.6/149.6 kB\u001b[0m \u001b[31m20.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
"\u001b[?25h"
]
}
],
"source": [
"!pip install -q -U bitsandbytes\n",
"!pip install -q -U git+https://github.com/huggingface/transformers.git \n",
"!pip install -q -U git+https://github.com/huggingface/peft.git\n",
"!pip install -q -U git+https://github.com/huggingface/accelerate.git\n",
"!pip install -q datasets"
]
},
{
"cell_type": "code",
"source": [
"!pip install einops scipy"
],
"metadata": {
"id": "Jc9_x9rp1fdD",
"outputId": "a8ee8e06-4586-461a-fcf6-a5063d213179",
"colab": {
"base_uri": "https://localhost:8080/"
}
},
"execution_count": 3,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
"Collecting einops\n",
" Downloading einops-0.6.1-py3-none-any.whl (42 kB)\n",
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m42.2/42.2 kB\u001b[0m \u001b[31m2.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
"\u001b[?25hRequirement already satisfied: scipy in /usr/local/lib/python3.10/dist-packages (1.10.1)\n",
"Requirement already satisfied: numpy<1.27.0,>=1.19.5 in /usr/local/lib/python3.10/dist-packages (from scipy) (1.22.4)\n",
"Installing collected packages: einops\n",
"Successfully installed einops-0.6.1\n"
]
}
]
},
{
"cell_type": "code",
"source": [
"import torch\n",
"from transformers import AutoTokenizer, AutoModelForCausalLM, BitsAndBytesConfig\n",
"\n",
"model_id = \"tiiuae/falcon-rw-1b\"\n",
"bnb_config = BitsAndBytesConfig(\n",
" load_in_4bit=True,\n",
" bnb_4bit_use_double_quant=True,\n",
" bnb_4bit_quant_type=\"nf4\",\n",
" bnb_4bit_compute_dtype=torch.bfloat16\n",
")\n",
"\n",
"tokenizer = AutoTokenizer.from_pretrained(model_id)\n",
"model = AutoModelForCausalLM.from_pretrained(model_id, quantization_config=bnb_config, device_map={\"\":0}, trust_remote_code=True)"
],
"metadata": {
"id": "E0Nl5mWL0k2T",
"colab": {
"base_uri": "https://localhost:8080/",
"height": 1000,
"referenced_widgets": [
"16eee0550bd24bb4979d041f1c4b91b7",
"df45826855694504b8b79b1703ce7c03",
"73afff2a05aa4c4abc1a92dad9d9abfc",
"613fe052b15b4f51bc8e5f94ae2a0c28",
"90034726561743f6931d43765babd8b5",
"0585eabee80f4b0e8cdf5629781596b1",
"34d4b568dfea41b0ae72ce96971fc3fc",
"5e021ce1a485446f95d1823ff6b9fa68",
"3ecc2bfe16b9481dae89039c36a25f3b",
"24281a28e2c84c83a26ba7b2e5d36841",
"428bdb06162641ab8d2514070d7ecff5",
"382724e322e542fa98f67152117cb790",
"09c28dd36a994d85b1b3acdc8f1b6418",
"6b05818571154be6acba9b971884541d",
"6fedb3026e9247f98009f40a00f93a01",
"3eb238623a0447a0bebede01ec18a13d",
"a9a691a7fa99473893a8dcf6040fd019",
"c33bd7f963ad492e872ab274381ee6d9",
"66e025732b3d4198b1d1d0a6d819ce94",
"7cceec2a2ba24d0894cf4aa704fb4fc3",
"850278f2ff8046bda9702552d3ac664c",
"27fcec9661f846018d3a23bc3eea984b",
"6195e67f37ea4080b2c4150eb158a9fe",
"0d1ebeda4a594769ab48cc514b465dc4",
"92dccb14582b4e75a74c9d8876853198",
"ef11bdc1e5ca4d3386cc1b83cab3644a",
"bea265fe73fb4915ab736432e6a6b894",
"40a9286740134408816a8a854e7f83ed",
"f86e8fab9df94d689134cf7a02d260ce",
"c8571657e4874de99deae23956c3bc16",
"d4aa690d1d1943c5957459a6dc19cee3",
"fb83fcfc8c3142ad9456570d9217814e",
"011475d22ed84237826f2f4e80afb412",
"e65f63f9dcbd497b9043e5c733091002",
"e7d06c2ca87d47b4afb2f00acbb8f314",
"a156f96cc4624b82a6ce9dcb1171b194",
"df9348096eea4c339a636107de4d3e65",
"0faaaec05f1e4c4eb86bab67026f9e84",
"9f1494098291447ead8aea2a77c4d138",
"e27c60dcf22b47c7bca70efc6b0fba7e",
"9d35d5a95e884528ad826fd9fccf6deb",
"8e426505d7f247d086ba6f000e222887",
"353fb86076bf4ae7845e5e5179cd1f03",
"0bde86521c9f4180b6329bf6f655da20",
"a1561e3219d84a1591ffc748c5c6c83d",
"01fe0179a70e49c7a00c2edb1099063a",
"5df7c5d831cd418c9adb7ff0a797a0a4",
"e1e64f80d07e4996920a46f4731a63a3",
"c13742548c0d4eb6819bf8b01f2465ae",
"f53ad883bcba4af3b55b096bce29fd54",
"923622700def4e389db093da3fa1ce3f",
"f1684d8564ee473faae860d8bf2c6f94",
"bd8bbdac6079445180f80e77209c1813",
"ce709727b8f34aa7ba7a63206773f891",
"16e77f709e1e4529b50d64c2f688a136",
"0fc67d35b5d7414791818285a7a1fa7a",
"7390423092da48cdb8a638638207afdd",
"dd600494ac33429d9f0b24eb19b3fc68",
"7dd6148a31f6448fb012f7832d630f0d",
"a4b84e9428804ccc861a5efbe6968e60",
"db8f37b05e3144ea90d399b6449a3a98",
"77b53c0ac6574e06b7c01759d739fd4d",
"d408ce01b9af4ab2bbf0b030fc1b5403",
"5c7a1e34841a40ffa30e7e710e1ac24a",
"ef606b9ea8484ce6ac0844f68a18b0bc",
"cb9b6eea0a1742869006326e84c1c0c1",
"18927e9a11844f5e8673018ce36f9794",
"d21207ea6ba84c42a41069d2c0cedd50",
"ad104fd1afdd457d8f18cdb1ed8fe3fa",
"1cf1cc5eb5aa4ac38647784d4cd14eb2",
"0a28ce5e3b1749e7bddfe26e57f0b734",
"a7bc29129a424f339943cbacae87cd85",
"d771ba3cedeb42edaf3906160a93e61d",
"3a995e6999814f9dbfc0f943d4ad2954",
"cc02bcab15c446599d91ec156b917183",
"ca3910fbcb394a1aa205d267083b6944",
"fee236499aca45e793c966c45eb3bf68",
"e31f55a31de24314b8e603cecd5b8f8c",
"7a2657899509470cacd925df87cb6e42",
"0a5049bfc3534621833ac36897edd9c4",
"80c99229e785488e9be32fd3edd4e956",
"1b22eba055c0426ca99685a36e22709f",
"398fe4502655480cae4e08a429a9b142",
"46c484520a8f42959ddb6668a287e8fb",
"0ae115e551844a78a80eb56fa5eec06b",
"4c16e0fd600e43cd80ad3fef661175fe",
"7f76ae37fc3e4a9ea8312b3bc752e40b",
"f5623e55e0c54681b6b84fead2c10a74",
"8644e2c2d36c4f4bb3e28c5a672900a5",
"2b3a05e5c17c45a6bfbd4872b85f0163",
"8216bc7f64b34afc8586351a228ee764",
"aa56818e7c3d4c13bfb53e1e4a293e3d",
"c0bb03632da948a2924f25621683f81a",
"6a093b5f75e54529a6b5faa24faa2cb0",
"1c70bb25fd9f4cd1981f98e701fe52e3",
"392888c8592048d6a43c2d3917bd0a3a",
"ba24b55f2c644b399d0ed16c03c1f9d1",
"05ebcd21b4524e718e890dd907a48907",
"e37f332fbf3e4cd9b56d1f7916116789",
"f4ba4fbf1321452682095f2458ff31e1",
"ff841a502351416db8cd8e0422b83e86",
"ec07c7901f1a4dd9a0c7051e351ed52e",
"f361f8b7df244b73b07c97248f37a4d0",
"784cd8ab0ca64f1cb65d367c0cc9ba95",
"ec1dc7ceb7684331a98cf4f617179a9b",
"c0dc0987555c4962844f54e1c32706f8",
"4c73f8e6c26e49c1bd75fa9111fab0d7",
"067de3d35bba4ba1b34e3b75d2a84541",
"743d0b8f7fd640a78ecee39dd4353dce",
"95c1f7092eda4372a06a7a5d21b2f6b5"
]
},
"outputId": "f22d977a-21a1-4a8e-9490-0c718b29cc0b"
},
"execution_count": 4,
"outputs": [
{
"output_type": "display_data",
"data": {
"text/plain": [
"Downloading (…)okenizer_config.json: 0%| | 0.00/255 [00:00 1024). Running this sequence through the model will result in indexing errors\n"
]
},
{
"output_type": "display_data",
"data": {
"text/plain": [
"Map: 0%| | 0/518 [00:00=7.0 in /usr/local/lib/python3.10/dist-packages (from wandb) (8.1.3)\n",
"Collecting GitPython!=3.1.29,>=1.0.0 (from wandb)\n",
" Downloading GitPython-3.1.31-py3-none-any.whl (184 kB)\n",
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m184.3/184.3 kB\u001b[0m \u001b[31m22.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
"\u001b[?25hRequirement already satisfied: requests<3,>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from wandb) (2.27.1)\n",
"Requirement already satisfied: psutil>=5.0.0 in /usr/local/lib/python3.10/dist-packages (from wandb) (5.9.5)\n",
"Collecting sentry-sdk>=1.0.0 (from wandb)\n",
" Downloading sentry_sdk-1.24.0-py2.py3-none-any.whl (206 kB)\n",
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m206.5/206.5 kB\u001b[0m \u001b[31m10.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
"\u001b[?25hCollecting docker-pycreds>=0.4.0 (from wandb)\n",
" Downloading docker_pycreds-0.4.0-py2.py3-none-any.whl (9.0 kB)\n",
"Requirement already satisfied: PyYAML in /usr/local/lib/python3.10/dist-packages (from wandb) (6.0)\n",
"Collecting pathtools (from wandb)\n",
" Downloading pathtools-0.1.2.tar.gz (11 kB)\n",
" Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
"Collecting setproctitle (from wandb)\n",
" Downloading setproctitle-1.3.2-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (30 kB)\n",
"Requirement already satisfied: setuptools in /usr/local/lib/python3.10/dist-packages (from wandb) (67.7.2)\n",
"Requirement already satisfied: appdirs>=1.4.3 in /usr/local/lib/python3.10/dist-packages (from wandb) (1.4.4)\n",
"Requirement already satisfied: protobuf!=4.21.0,<5,>=3.19.0 in /usr/local/lib/python3.10/dist-packages (from wandb) (3.20.3)\n",
"Requirement already satisfied: six>=1.4.0 in /usr/local/lib/python3.10/dist-packages (from docker-pycreds>=0.4.0->wandb) (1.16.0)\n",
"Collecting gitdb<5,>=4.0.1 (from GitPython!=3.1.29,>=1.0.0->wandb)\n",
" Downloading gitdb-4.0.10-py3-none-any.whl (62 kB)\n",
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m62.7/62.7 kB\u001b[0m \u001b[31m8.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
"\u001b[?25hRequirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests<3,>=2.0.0->wandb) (1.26.15)\n",
"Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests<3,>=2.0.0->wandb) (2022.12.7)\n",
"Requirement already satisfied: charset-normalizer~=2.0.0 in /usr/local/lib/python3.10/dist-packages (from requests<3,>=2.0.0->wandb) (2.0.12)\n",
"Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests<3,>=2.0.0->wandb) (3.4)\n",
"Collecting smmap<6,>=3.0.1 (from gitdb<5,>=4.0.1->GitPython!=3.1.29,>=1.0.0->wandb)\n",
" Downloading smmap-5.0.0-py3-none-any.whl (24 kB)\n",
"Building wheels for collected packages: pathtools\n",
" Building wheel for pathtools (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
" Created wheel for pathtools: filename=pathtools-0.1.2-py3-none-any.whl size=8791 sha256=b27691c28003b69fe677abef24f386b27cd0ced353c004c65f8ee7af91cc0dd3\n",
" Stored in directory: /root/.cache/pip/wheels/e7/f3/22/152153d6eb222ee7a56ff8617d80ee5207207a8c00a7aab794\n",
"Successfully built pathtools\n",
"Installing collected packages: pathtools, smmap, setproctitle, sentry-sdk, docker-pycreds, gitdb, GitPython, wandb\n",
"Successfully installed GitPython-3.1.31 docker-pycreds-0.4.0 gitdb-4.0.10 pathtools-0.1.2 sentry-sdk-1.24.0 setproctitle-1.3.2 smmap-5.0.0 wandb-0.15.3\n"
]
}
]
},
{
"cell_type": "code",
"source": [
"import wandb\n",
"wandb.login()"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/",
"height": 124
},
"id": "RCDioBmOYpEE",
"outputId": "51f46017-2bdd-4cf4-c6dc-73aa45bb9bfc"
},
"execution_count": 10,
"outputs": [
{
"output_type": "display_data",
"data": {
"text/plain": [
"/content/wandb/run-20230528_174740-fdsvtgrs"
]
},
"metadata": {}
},
{
"output_type": "display_data",
"data": {
"text/plain": [
"| Step | \n", "Training Loss | \n", "
|---|---|
| 1 | \n", "1.997600 | \n", "
| 2 | \n", "2.265900 | \n", "
| 3 | \n", "1.809600 | \n", "
| 4 | \n", "2.020400 | \n", "
| 5 | \n", "3.371100 | \n", "
| 6 | \n", "1.832700 | \n", "
| 7 | \n", "2.383400 | \n", "
| 8 | \n", "1.730600 | \n", "
| 9 | \n", "2.520400 | \n", "
| 10 | \n", "2.620500 | \n", "
| 11 | \n", "2.613400 | \n", "
| 12 | \n", "2.683500 | \n", "
| 13 | \n", "2.354000 | \n", "
| 14 | \n", "1.917100 | \n", "
"
]
},
"metadata": {}
},
{
"output_type": "display_data",
"data": {
"text/plain": [
"\u001b[31m╭─\u001b[0m\u001b[31m──────────────────────────────\u001b[0m\u001b[31m \u001b[0m\u001b[1;31mTraceback \u001b[0m\u001b[1;2;31m(most recent call last)\u001b[0m\u001b[31m \u001b[0m\u001b[31m───────────────────────────────\u001b[0m\u001b[31m─╮\u001b[0m\n",
"\u001b[31m│\u001b[0m in \u001b[92m╭─────────────────────────────── Traceback (most recent call last) ────────────────────────────────╮\n",
"│ in <cell line: 29>:29 │\n",
"│ │\n",
"│ /usr/local/lib/python3.10/dist-packages/transformers/trainer.py:1696 in train │\n",
"│ │\n",
"│ 1693 │ │ inner_training_loop = find_executable_batch_size( │\n",
"│ 1694 │ │ │ self._inner_training_loop, self._train_batch_size, args.auto_find_batch_size │\n",
"│ 1695 │ │ ) │\n",
"│ ❱ 1696 │ │ return inner_training_loop( │\n",
"│ 1697 │ │ │ args=args, │\n",
"│ 1698 │ │ │ resume_from_checkpoint=resume_from_checkpoint, │\n",
"│ 1699 │ │ │ trial=trial, │\n",
"│ │\n",
"│ /usr/local/lib/python3.10/dist-packages/transformers/trainer.py:1973 in _inner_training_loop │\n",
"│ │\n",
"│ 1970 │ │ │ │ │ with model.no_sync(): │\n",
"│ 1971 │ │ │ │ │ │ tr_loss_step = self.training_step(model, inputs) │\n",
"│ 1972 │ │ │ │ else: │\n",
"│ ❱ 1973 │ │ │ │ │ tr_loss_step = self.training_step(model, inputs) │\n",
"│ 1974 │ │ │ │ │\n",
"│ 1975 │ │ │ │ if ( │\n",
"│ 1976 │ │ │ │ │ args.logging_nan_inf_filter │\n",
"│ │\n",
"│ /usr/local/lib/python3.10/dist-packages/transformers/trainer.py:2787 in training_step │\n",
"│ │\n",
"│ 2784 │ │ │ return loss_mb.reduce_mean().detach().to(self.args.device) │\n",
"│ 2785 │ │ │\n",
"│ 2786 │ │ with self.compute_loss_context_manager(): │\n",
"│ ❱ 2787 │ │ │ loss = self.compute_loss(model, inputs) │\n",
"│ 2788 │ │ │\n",
"│ 2789 │ │ if self.args.n_gpu > 1: │\n",
"│ 2790 │ │ │ loss = loss.mean() # mean() to average on multi-gpu parallel training │\n",
"│ │\n",
"│ /usr/local/lib/python3.10/dist-packages/transformers/trainer.py:2819 in compute_loss │\n",
"│ │\n",
"│ 2816 │ │ │ labels = inputs.pop(\"labels\") │\n",
"│ 2817 │ │ else: │\n",
"│ 2818 │ │ │ labels = None │\n",
"│ ❱ 2819 │ │ outputs = model(**inputs) │\n",
"│ 2820 │ │ # Save past state if it exists │\n",
"│ 2821 │ │ # TODO: this needs to be fixed and made cleaner later. │\n",
"│ 2822 │ │ if self.args.past_index >= 0: │\n",
"│ │\n",
"│ /usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py:1501 in _call_impl │\n",
"│ │\n",
"│ 1498 │ │ if not (self._backward_hooks or self._backward_pre_hooks or self._forward_hooks │\n",
"│ 1499 │ │ │ │ or _global_backward_pre_hooks or _global_backward_hooks │\n",
"│ 1500 │ │ │ │ or _global_forward_hooks or _global_forward_pre_hooks): │\n",
"│ ❱ 1501 │ │ │ return forward_call(*args, **kwargs) │\n",
"│ 1502 │ │ # Do not call functions when jit is used │\n",
"│ 1503 │ │ full_backward_hooks, non_full_backward_hooks = [], [] │\n",
"│ 1504 │ │ backward_pre_hooks = [] │\n",
"│ │\n",
"│ /usr/local/lib/python3.10/dist-packages/peft/peft_model.py:686 in forward │\n",
"│ │\n",
"│ 683 │ ): │\n",
"│ 684 │ │ peft_config = self.active_peft_config │\n",
"│ 685 │ │ if not isinstance(peft_config, PromptLearningConfig): │\n",
"│ ❱ 686 │ │ │ return self.base_model( │\n",
"│ 687 │ │ │ │ input_ids=input_ids, │\n",
"│ 688 │ │ │ │ attention_mask=attention_mask, │\n",
"│ 689 │ │ │ │ inputs_embeds=inputs_embeds, │\n",
"│ │\n",
"│ /usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py:1501 in _call_impl │\n",
"│ │\n",
"│ 1498 │ │ if not (self._backward_hooks or self._backward_pre_hooks or self._forward_hooks │\n",
"│ 1499 │ │ │ │ or _global_backward_pre_hooks or _global_backward_hooks │\n",
"│ 1500 │ │ │ │ or _global_forward_hooks or _global_forward_pre_hooks): │\n",
"│ ❱ 1501 │ │ │ return forward_call(*args, **kwargs) │\n",
"│ 1502 │ │ # Do not call functions when jit is used │\n",
"│ 1503 │ │ full_backward_hooks, non_full_backward_hooks = [], [] │\n",
"│ 1504 │ │ backward_pre_hooks = [] │\n",
"│ │\n",
"│ /usr/local/lib/python3.10/dist-packages/accelerate/hooks.py:165 in new_forward │\n",
"│ │\n",
"│ 162 │ │ │ with torch.no_grad(): │\n",
"│ 163 │ │ │ │ output = old_forward(*args, **kwargs) │\n",
"│ 164 │ │ else: │\n",
"│ ❱ 165 │ │ │ output = old_forward(*args, **kwargs) │\n",
"│ 166 │ │ return module._hf_hook.post_forward(module, output) │\n",
"│ 167 │ │\n",
"│ 168 │ module.forward = new_forward │\n",
"│ │\n",
"│ /root/.cache/huggingface/modules/transformers_modules/tiiuae/falcon-rw-1b/104655c0c067936f1ae2b4 │\n",
"│ 73625fe5161375591a/modelling_RW.py:753 in forward │\n",
"│ │\n",
"│ 750 │ │ │\n",
"│ 751 │ │ return_dict = return_dict if return_dict is not None else self.config.use_return │\n",
"│ 752 │ │ │\n",
"│ ❱ 753 │ │ transformer_outputs = self.transformer( │\n",
"│ 754 │ │ │ input_ids, │\n",
"│ 755 │ │ │ past_key_values=past_key_values, │\n",
"│ 756 │ │ │ attention_mask=attention_mask, │\n",
"│ │\n",
"│ /usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py:1501 in _call_impl │\n",
"│ │\n",
"│ 1498 │ │ if not (self._backward_hooks or self._backward_pre_hooks or self._forward_hooks │\n",
"│ 1499 │ │ │ │ or _global_backward_pre_hooks or _global_backward_hooks │\n",
"│ 1500 │ │ │ │ or _global_forward_hooks or _global_forward_pre_hooks): │\n",
"│ ❱ 1501 │ │ │ return forward_call(*args, **kwargs) │\n",
"│ 1502 │ │ # Do not call functions when jit is used │\n",
"│ 1503 │ │ full_backward_hooks, non_full_backward_hooks = [], [] │\n",
"│ 1504 │ │ backward_pre_hooks = [] │\n",
"│ │\n",
"│ /usr/local/lib/python3.10/dist-packages/accelerate/hooks.py:165 in new_forward │\n",
"│ │\n",
"│ 162 │ │ │ with torch.no_grad(): │\n",
"│ 163 │ │ │ │ output = old_forward(*args, **kwargs) │\n",
"│ 164 │ │ else: │\n",
"│ ❱ 165 │ │ │ output = old_forward(*args, **kwargs) │\n",
"│ 166 │ │ return module._hf_hook.post_forward(module, output) │\n",
"│ 167 │ │\n",
"│ 168 │ module.forward = new_forward │\n",
"│ │\n",
"│ /root/.cache/huggingface/modules/transformers_modules/tiiuae/falcon-rw-1b/104655c0c067936f1ae2b4 │\n",
"│ 73625fe5161375591a/modelling_RW.py:640 in forward │\n",
"│ │\n",
"│ 637 │ │ │ │ │ │\n",
"│ 638 │ │ │ │ │ return custom_forward │\n",
"│ 639 │ │ │ │ │\n",
"│ ❱ 640 │ │ │ │ outputs = torch.utils.checkpoint.checkpoint( │\n",
"│ 641 │ │ │ │ │ create_custom_forward(block), │\n",
"│ 642 │ │ │ │ │ hidden_states, │\n",
"│ 643 │ │ │ │ │ alibi, │\n",
"│ │\n",
"│ /usr/local/lib/python3.10/dist-packages/torch/utils/checkpoint.py:249 in checkpoint │\n",
"│ │\n",
"│ 246 │ │ raise ValueError(\"Unexpected keyword arguments: \" + \",\".join(arg for arg in kwar │\n",
"│ 247 │ │\n",
"│ 248 │ if use_reentrant: │\n",
"│ ❱ 249 │ │ return CheckpointFunction.apply(function, preserve, *args) │\n",
"│ 250 │ else: │\n",
"│ 251 │ │ return _checkpoint_without_reentrant( │\n",
"│ 252 │ │ │ function, │\n",
"│ │\n",
"│ /usr/local/lib/python3.10/dist-packages/torch/autograd/function.py:506 in apply │\n",
"│ │\n",
"│ 503 │ │ if not torch._C._are_functorch_transforms_active(): │\n",
"│ 504 │ │ │ # See NOTE: [functorch vjp and autograd interaction] │\n",
"│ 505 │ │ │ args = _functorch.utils.unwrap_dead_wrappers(args) │\n",
"│ ❱ 506 │ │ │ return super().apply(*args, **kwargs) # type: ignore[misc] │\n",
"│ 507 │ │ │\n",
"│ 508 │ │ if cls.setup_context == _SingleLevelFunction.setup_context: │\n",
"│ 509 │ │ │ raise RuntimeError( │\n",
"│ │\n",
"│ /usr/local/lib/python3.10/dist-packages/torch/utils/checkpoint.py:107 in forward │\n",
"│ │\n",
"│ 104 │ │ ctx.save_for_backward(*tensor_inputs) │\n",
"│ 105 │ │ │\n",
"│ 106 │ │ with torch.no_grad(): │\n",
"│ ❱ 107 │ │ │ outputs = run_function(*args) │\n",
"│ 108 │ │ return outputs │\n",
"│ 109 │ │\n",
"│ 110 │ @staticmethod │\n",
"│ │\n",
"│ /root/.cache/huggingface/modules/transformers_modules/tiiuae/falcon-rw-1b/104655c0c067936f1ae2b4 │\n",
"│ 73625fe5161375591a/modelling_RW.py:636 in custom_forward │\n",
"│ │\n",
"│ 633 │ │ │ │ def create_custom_forward(module): │\n",
"│ 634 │ │ │ │ │ def custom_forward(*inputs): │\n",
"│ 635 │ │ │ │ │ │ # None for past_key_value │\n",
"│ ❱ 636 │ │ │ │ │ │ return module(*inputs, use_cache=use_cache, output_attentions=ou │\n",
"│ 637 │ │ │ │ │ │\n",
"│ 638 │ │ │ │ │ return custom_forward │\n",
"│ 639 │\n",
"│ │\n",
"│ /usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py:1501 in _call_impl │\n",
"│ │\n",
"│ 1498 │ │ if not (self._backward_hooks or self._backward_pre_hooks or self._forward_hooks │\n",
"│ 1499 │ │ │ │ or _global_backward_pre_hooks or _global_backward_hooks │\n",
"│ 1500 │ │ │ │ or _global_forward_hooks or _global_forward_pre_hooks): │\n",
"│ ❱ 1501 │ │ │ return forward_call(*args, **kwargs) │\n",
"│ 1502 │ │ # Do not call functions when jit is used │\n",
"│ 1503 │ │ full_backward_hooks, non_full_backward_hooks = [], [] │\n",
"│ 1504 │ │ backward_pre_hooks = [] │\n",
"│ │\n",
"│ /usr/local/lib/python3.10/dist-packages/accelerate/hooks.py:165 in new_forward │\n",
"│ │\n",
"│ 162 │ │ │ with torch.no_grad(): │\n",
"│ 163 │ │ │ │ output = old_forward(*args, **kwargs) │\n",
"│ 164 │ │ else: │\n",
"│ ❱ 165 │ │ │ output = old_forward(*args, **kwargs) │\n",
"│ 166 │ │ return module._hf_hook.post_forward(module, output) │\n",
"│ 167 │ │\n",
"│ 168 │ module.forward = new_forward │\n",
"│ │\n",
"│ /root/.cache/huggingface/modules/transformers_modules/tiiuae/falcon-rw-1b/104655c0c067936f1ae2b4 │\n",
"│ 73625fe5161375591a/modelling_RW.py:385 in forward │\n",
"│ │\n",
"│ 382 │ │ residual = hidden_states │\n",
"│ 383 │ │ │\n",
"│ 384 │ │ # Self attention. │\n",
"│ ❱ 385 │ │ attn_outputs = self.self_attention( │\n",
"│ 386 │ │ │ layernorm_output, │\n",
"│ 387 │ │ │ layer_past=layer_past, │\n",
"│ 388 │ │ │ attention_mask=attention_mask, │\n",
"│ │\n",
"│ /usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py:1501 in _call_impl │\n",
"│ │\n",
"│ 1498 │ │ if not (self._backward_hooks or self._backward_pre_hooks or self._forward_hooks │\n",
"│ 1499 │ │ │ │ or _global_backward_pre_hooks or _global_backward_hooks │\n",
"│ 1500 │ │ │ │ or _global_forward_hooks or _global_forward_pre_hooks): │\n",
"│ ❱ 1501 │ │ │ return forward_call(*args, **kwargs) │\n",
"│ 1502 │ │ # Do not call functions when jit is used │\n",
"│ 1503 │ │ full_backward_hooks, non_full_backward_hooks = [], [] │\n",
"│ 1504 │ │ backward_pre_hooks = [] │\n",
"│ │\n",
"│ /usr/local/lib/python3.10/dist-packages/accelerate/hooks.py:165 in new_forward │\n",
"│ │\n",
"│ 162 │ │ │ with torch.no_grad(): │\n",
"│ 163 │ │ │ │ output = old_forward(*args, **kwargs) │\n",
"│ 164 │ │ else: │\n",
"│ ❱ 165 │ │ │ output = old_forward(*args, **kwargs) │\n",
"│ 166 │ │ return module._hf_hook.post_forward(module, output) │\n",
"│ 167 │ │\n",
"│ 168 │ module.forward = new_forward │\n",
"│ │\n",
"│ /root/.cache/huggingface/modules/transformers_modules/tiiuae/falcon-rw-1b/104655c0c067936f1ae2b4 │\n",
"│ 73625fe5161375591a/modelling_RW.py:306 in forward │\n",
"│ │\n",
"│ 303 │ │ │ │ attention_scores = attention_scores.to(torch.float32) │\n",
"│ 304 │ │ │ # attn_weights = torch.masked_fill(attention_scores, attention_mask, torch.f │\n",
"│ 305 │ │ │ attention_probs = F.softmax( │\n",
"│ ❱ 306 │ │ │ │ (attention_scores + alibi.view(batch_size, self.num_heads, 1, -1)) * sel │\n",
"│ 307 │ │ │ │ dim=-1, │\n",
"│ 308 │ │ │ │ dtype=hidden_states.dtype, │\n",
"│ 309 │ │ │ ) │\n",
"╰──────────────────────────────────────────────────────────────────────────────────────────────────╯\n",
"OutOfMemoryError: CUDA out of memory. Tried to allocate 2.74 GiB (GPU 0; 14.75 GiB total capacity; 8.16 GiB already\n",
"allocated; 2.63 GiB free; 11.06 GiB reserved in total by PyTorch) If reserved memory is >> allocated memory try \n",
"setting max_split_size_mb to avoid fragmentation. See documentation for Memory Management and \n",
"PYTORCH_CUDA_ALLOC_CONF\n",
"
\n"
]
},
"metadata": {}
}
]
},
{
"cell_type": "code",
"source": [
"trainer = transformers.Trainer(\n",
" model=model,\n",
" train_dataset=data[\"train\"],\n",
" args=transformers.TrainingArguments(\n",
" per_device_train_batch_size=1,\n",
" gradient_accumulation_steps=4,\n",
" warmup_steps=40,\n",
" max_steps=1000,\n",
" learning_rate=2e-4,\n",
" fp16=True,\n",
" save_strategy=\"steps\",\n",
" save_steps=100,\n",
" logging_steps=1,\n",
" output_dir=output_dir,\n",
" optim=\"paged_adamw_8bit\"\n",
" ),\n",
" data_collator=transformers.DataCollatorForLanguageModeling(tokenizer, mlm=False),\n",
")\n",
"model.config.use_cache = False # silence the warnings. Please re-enable for inference!\n",
"trainer.train(resume_from_checkpoint=True)\n",
"model.save_pretrained(output_dir)"
],
"metadata": {
"id": "K_CcqVS424Gr"
},
"execution_count": null,
"outputs": []
},
{
"cell_type": "code",
"source": [],
"metadata": {
"id": "jY53rV1wcd7K"
},
"execution_count": null,
"outputs": []
}
]
}