{ "nbformat": 4, "nbformat_minor": 0, "metadata": { "colab": { "name": "Speed_Up_Your_Algo_PyTorch.ipynb", "version": "0.3.2", "provenance": [], "collapsed_sections": [ "nJyq4ZJ3CL5-", "IOpLFtlLj_Sf", "zLBPSTrUkEDN", "ma-vz8-qCSwK", "70ana3V1EIFi", "4oxBruH9pPFV", "ZrV3b8zWGgoX", "KGE6gB_CJ4Oi" ] }, "kernelspec": { "name": "python3", "display_name": "Python 3" }, "accelerator": "GPU" }, "cells": [ { "metadata": { "id": "FkDg9fic_5Rl", "colab_type": "text" }, "cell_type": "markdown", "source": [ "This is a part of series I am writing, named \"**Speed Up Your Algorithms**\". \n", "\n", "Here is the list of posts with files I have written:\n", "[Github-SpeedUpYourAlgorithms](https://github.com/PuneetGrov3r/MediumPosts/tree/master/SpeedUpYourAlgorithms)" ] }, { "metadata": { "id": "nJyq4ZJ3CL5-", "colab_type": "text" }, "cell_type": "markdown", "source": [ "# Initial" ] }, { "metadata": { "id": "IOpLFtlLj_Sf", "colab_type": "text" }, "cell_type": "markdown", "source": [ "### Torch, Torchvision:" ] }, { "metadata": { "id": "m9K9aQp_CEw5", "colab_type": "code", "outputId": "a89fda99-0c33-4dbd-f6a3-6b3d240e0d41", "colab": { "base_uri": "https://localhost:8080/", "height": 34 } }, "cell_type": "code", "source": [ "!cat /usr/local/cuda/version.txt" ], "execution_count": 0, "outputs": [ { "output_type": "stream", "text": [ "CUDA Version 9.2.148\n" ], "name": "stdout" } ] }, { "metadata": { "id": "srtGQ9C1AdLT", "colab_type": "code", "colab": {} }, "cell_type": "code", "source": [ "!pip install http://download.pytorch.org/whl/cu92/torch-0.4.1-cp36-cp36m-linux_x86_64.whl\n", "!pip install torchvision" ], "execution_count": 0, "outputs": [] }, { "metadata": { "id": "zLBPSTrUkEDN", "colab_type": "text" }, "cell_type": "markdown", "source": [ "### Pycuda:" ] }, { "metadata": { "id": "8RpbRcfdeuk7", "colab_type": "code", "colab": {} }, "cell_type": "code", "source": [ "# https://medium.com/@iphoenix179/running-cuda-c-c-in-jupyter-or-how-to-run-nvcc-in-google-colab-663d33f53772\n", "# https://developer.nvidia.com/cuda-90-download-archive?target_os=Linux&target_arch=x86_64&target_distro=Ubuntu&target_version=1704&target_type=deblocal\n", "!apt update -qq;\n", "!wget https://developer.nvidia.com/compute/cuda/9.0/Prod/local_installers/cuda-repo-ubuntu1704-9-0-local_9.0.176-1_amd64-deb\n", "!mv cuda-repo-ubuntu1704-9-0-local_9.0.176-1_amd64-deb cuda-repo-ubuntu1704-9-0-local_9.0.176-1_amd64.deb \n", "!dpkg -i cuda-repo-ubuntu1704-9-0-local_9.0.176-1_amd64.deb\n", "!apt-key add /var/cuda-repo-9-0-local/7fa2af80.pub\n", "!apt-get update -qq;\n", "!apt-get install cuda gcc-5 g++-5 -y -qq;\n", "!ln -s /usr/bin/gcc-5 /usr/local/cuda/bin/gcc;\n", "!ln -s /usr/bin/g++-5 /usr/local/cuda/bin/g++;\n", "#!apt install cuda-9.2;" ], "execution_count": 0, "outputs": [] }, { "metadata": { "id": "pUBT0zfUFlz5", "colab_type": "code", "colab": {} }, "cell_type": "code", "source": [ "# http://alisonrowland.com/articles/installing-pycuda-via-pip\n", "# https://codeyarns.com/2015/07/31/pip-install-error-with-pycuda/\n", "import os\n", "PATH = os.environ[\"PATH\"]\n", "os.environ[\"PATH\"] = \"/usr/local/cuda-9.2/bin:/usr/local/cuda/bin:\" + PATH\n", "os.environ[\"LD_LIBRARY_PATH\"] = \"/usr/local/cuda/lib64\"\n", "os.environ[\"CUDA_ROOT\"] = \"/usr/local/cuda/\"" ], "execution_count": 0, "outputs": [] }, { "metadata": { "id": "He-T3N_uGB8i", "colab_type": "code", "colab": {} }, "cell_type": "code", "source": [ "!pip -q install --ignore-installed pycuda" ], "execution_count": 0, "outputs": [] }, { "metadata": { "id": "ma-vz8-qCSwK", "colab_type": "text" }, "cell_type": "markdown", "source": [ "# Import" ] }, { "metadata": { "id": "u0CQHYVcKjKs", "colab_type": "code", "colab": {} }, "cell_type": "code", "source": [ "import torchvision\n", "import torch\n", "import numpy as np\n", "import pandas as pd\n", "from torch import nn\n", "\n", "import matplotlib.pyplot as plt" ], "execution_count": 0, "outputs": [] }, { "metadata": { "id": "dWWAJMSVDjLN", "colab_type": "code", "colab": {} }, "cell_type": "code", "source": [ "import pycuda.driver as cuda\n", "cuda.init()" ], "execution_count": 0, "outputs": [] }, { "metadata": { "id": "70ana3V1EIFi", "colab_type": "text" }, "cell_type": "markdown", "source": [ "# 2. How to check the availability of cuda?" ] }, { "metadata": { "id": "aJTVCSj2A9jS", "colab_type": "code", "outputId": "a6079b17-92c3-4ef6-f6dc-cbfedaa0dbe1", "colab": { "base_uri": "https://localhost:8080/", "height": 34 } }, "cell_type": "code", "source": [ "torch.cuda.is_available() # We are on GPU instance, so cuda is already installed." ], "execution_count": 0, "outputs": [ { "output_type": "execute_result", "data": { "text/plain": [ "True" ] }, "metadata": { "tags": [] }, "execution_count": 8 } ] }, { "metadata": { "id": "T--ZU71EEkFr", "colab_type": "text" }, "cell_type": "markdown", "source": [ "# 3. How to get more info on your cuda devices?" ] }, { "metadata": { "id": "h5ujkSp2nVOe", "colab_type": "code", "outputId": "13140521-a763-42e3-83dd-b1eaef2de072", "colab": { "base_uri": "https://localhost:8080/", "height": 34 } }, "cell_type": "code", "source": [ "torch.cuda.current_device() # Get Id of current cuda device" ], "execution_count": 0, "outputs": [ { "output_type": "execute_result", "data": { "text/plain": [ "0" ] }, "metadata": { "tags": [] }, "execution_count": 9 } ] }, { "metadata": { "id": "dfyIYR9rE4Pg", "colab_type": "code", "outputId": "9cdc0288-a47a-4dd1-8c1b-0000380f3a68", "colab": { "base_uri": "https://localhost:8080/", "height": 34 } }, "cell_type": "code", "source": [ "cuda.Device(0).name() # get name of device using pycuda" ], "execution_count": 0, "outputs": [ { "output_type": "execute_result", "data": { "text/plain": [ "'Tesla K80'" ] }, "metadata": { "tags": [] }, "execution_count": 10 } ] }, { "metadata": { "id": "PdfRQs65m_7I", "colab_type": "code", "outputId": "5a2569b1-d95e-4e3a-a059-969fa14c384b", "colab": { "base_uri": "https://localhost:8080/", "height": 34 } }, "cell_type": "code", "source": [ "torch.cuda.get_device_name(0) # or using torch" ], "execution_count": 0, "outputs": [ { "output_type": "execute_result", "data": { "text/plain": [ "'Tesla K80'" ] }, "metadata": { "tags": [] }, "execution_count": 11 } ] }, { "metadata": { "id": "A8ALco5IngOu", "colab_type": "code", "outputId": "5f44950f-b69a-4e7c-80c3-667e3efcd728", "colab": { "base_uri": "https://localhost:8080/", "height": 67 } }, "cell_type": "code", "source": [ "# Simple class for getting info on all cuda compatible devices\n", "import pycuda.autoinit\n", "\n", "class aboutCudaDevices():\n", " def __init__(self):\n", " pass\n", " \n", " def num_devices(self):\n", " \"\"\"Return number of devices connected.\"\"\"\n", " return cuda.Device.count()\n", " \n", " def devices(self):\n", " \"\"\"Get info on all devices connected.\"\"\"\n", " num = cuda.Device.count()\n", " print(\"%d device(s) found:\"%num)\n", " for i in range(num):\n", " print(cuda.Device(i).name(), \"(Id: %d)\"%i)\n", " \n", " def mem_info(self):\n", " \"\"\"Get available and total memory of all devices.\"\"\"\n", " available, total = cuda.mem_get_info()\n", " print(\"Available: %.2f GB\\nTotal: %.2f GB\"%(available/1e9, total/1e9))\n", " \n", " def attributes(self, device_id=0):\n", " \"\"\"Get attributes of device with device Id = device_id\"\"\"\n", " return cuda.Device(device_id).get_attributes()\n", " \n", " def __repr__(self):\n", " \"\"\"Class representation as number of devices connected and about them.\"\"\"\n", " num = cuda.Device.count()\n", " string = \"\"\n", " string += (\"%d device(s) found:\\n\"%num)\n", " for i in range(num):\n", " string += ( \" %d) %s (Id: %d)\\n\"%((i+1),cuda.Device(i).name(),i))\n", " string += (\" Memory: %.2f GB\\n\"%(cuda.Device(i).total_memory()/1e9))\n", " return string\n", "\n", "# You can print output just by typing its name (__repr__):\n", "aboutCudaDevices()" ], "execution_count": 0, "outputs": [ { "output_type": "execute_result", "data": { "text/plain": [ "1 device(s) found:\n", " 1) Tesla K80 (Id: 0)\n", " Memory: 12.00 GB" ] }, "metadata": { "tags": [] }, "execution_count": 12 } ] }, { "metadata": { "id": "BAyU2477owkl", "colab_type": "code", "outputId": "4a4f096c-1c28-46a3-a391-974d5d2d1403", "colab": { "base_uri": "https://localhost:8080/", "height": 34 } }, "cell_type": "code", "source": [ "# Lets check how much memory is allocated:\n", "torch.cuda.memory_allocated()" ], "execution_count": 0, "outputs": [ { "output_type": "execute_result", "data": { "text/plain": [ "0" ] }, "metadata": { "tags": [] }, "execution_count": 13 } ] }, { "metadata": { "id": "Equ6KD99oEMT", "colab_type": "code", "outputId": "7333a930-b7f7-4b13-8ea8-d1dd1015b0f2", "colab": { "base_uri": "https://localhost:8080/", "height": 34 } }, "cell_type": "code", "source": [ "# Let allocate some arrays to GPU:\n", "mat_a = torch.cuda.FloatTensor((100., 100.))\n", "mat_a" ], "execution_count": 0, "outputs": [ { "output_type": "execute_result", "data": { "text/plain": [ "tensor([100., 100.], device='cuda:0')" ] }, "metadata": { "tags": [] }, "execution_count": 14 } ] }, { "metadata": { "id": "O6b6mf59oXhe", "colab_type": "code", "outputId": "6ff6867f-f802-4d3a-d4f9-f15041a8ee6d", "colab": { "base_uri": "https://localhost:8080/", "height": 34 } }, "cell_type": "code", "source": [ "# Lets check how muc memory is allocated now:\n", "torch.cuda.memory_allocated()" ], "execution_count": 0, "outputs": [ { "output_type": "execute_result", "data": { "text/plain": [ "512" ] }, "metadata": { "tags": [] }, "execution_count": 15 } ] }, { "metadata": { "id": "Al9L1fBooqx-", "colab_type": "code", "outputId": "ef47df7f-07e3-49b1-fd75-2a45e3a8d9ad", "colab": { "base_uri": "https://localhost:8080/", "height": 34 } }, "cell_type": "code", "source": [ "# Memory used by Cache Allocator:\n", "torch.cuda.memory_cached()" ], "execution_count": 0, "outputs": [ { "output_type": "execute_result", "data": { "text/plain": [ "1048576" ] }, "metadata": { "tags": [] }, "execution_count": 16 } ] }, { "metadata": { "id": "V0JshoUtDPM6", "colab_type": "code", "colab": {} }, "cell_type": "code", "source": [ "# These memory methods are only available for GPUs. And that's where they are actually needed." ], "execution_count": 0, "outputs": [] }, { "metadata": { "id": "4oxBruH9pPFV", "colab_type": "text" }, "cell_type": "markdown", "source": [ "# 4. How to store Tensors and run Models on GPU?" ] }, { "metadata": { "id": "54xGuTWvo5Z3", "colab_type": "code", "outputId": "8b36a64e-f254-4af8-e78f-00848db43ea6", "colab": { "base_uri": "https://localhost:8080/", "height": 34 } }, "cell_type": "code", "source": [ "# For storing something on CPU:\n", "Mat_cpu = torch.FloatTensor([1., 2.])\n", "Mat_cpu" ], "execution_count": 0, "outputs": [ { "output_type": "execute_result", "data": { "text/plain": [ "tensor([1., 2.])" ] }, "metadata": { "tags": [] }, "execution_count": 18 } ] }, { "metadata": { "id": "4HzPhPO5EIVm", "colab_type": "code", "outputId": "ee17ef47-f8c7-4f0a-b912-fbfaa9b037a6", "colab": { "base_uri": "https://localhost:8080/", "height": 34 } }, "cell_type": "code", "source": [ "# To put this on GPU:\n", "Mat_gpu = Mat_cpu.cuda()\n", "Mat_gpu, Mat_cpu" ], "execution_count": 0, "outputs": [ { "output_type": "execute_result", "data": { "text/plain": [ "(tensor([1., 2.], device='cuda:0'), tensor([1., 2.]))" ] }, "metadata": { "tags": [] }, "execution_count": 19 } ] }, { "metadata": { "id": "4EXubaSKETq2", "colab_type": "code", "outputId": "6392d2b2-e7e6-4251-fd11-28e2ce904a4b", "colab": { "base_uri": "https://localhost:8080/", "height": 34 } }, "cell_type": "code", "source": [ "# Or you could have done:\n", "Mat_gpu2 = torch.cuda.FloatTensor([1., 2.])\n", "Mat_gpu2" ], "execution_count": 0, "outputs": [ { "output_type": "execute_result", "data": { "text/plain": [ "tensor([1., 2.], device='cuda:0')" ] }, "metadata": { "tags": [] }, "execution_count": 20 } ] }, { "metadata": { "id": "WX3ROV-vFAMG", "colab_type": "code", "colab": {} }, "cell_type": "code", "source": [ "# Putting Model on GPU:\n", "sq = nn.Sequential(\n", " nn.Linear(20, 20),\n", " nn.ReLU(),\n", " nn.Linear(20, 4),\n", " nn.Softmax()\n", ")" ], "execution_count": 0, "outputs": [] }, { "metadata": { "id": "oiLwGOdtGJqM", "colab_type": "code", "colab": {} }, "cell_type": "code", "source": [ "model = sq.cuda()" ], "execution_count": 0, "outputs": [] }, { "metadata": { "id": "PfZlJIUUGSoJ", "colab_type": "code", "outputId": "7e2f2c17-32e1-4cc2-c1ba-b0383899db44", "colab": { "base_uri": "https://localhost:8080/", "height": 34 } }, "cell_type": "code", "source": [ "# To check if model is on GPU:\n", "# From the discussions here: discuss.pytorch.org/t/how-to-check-if-model-is-on-cuda\n", "next(model.parameters()).is_cuda" ], "execution_count": 0, "outputs": [ { "output_type": "execute_result", "data": { "text/plain": [ "True" ] }, "metadata": { "tags": [] }, "execution_count": 23 } ] }, { "metadata": { "id": "FwIa7hAk7DDb", "colab_type": "text" }, "cell_type": "markdown", "source": [ "# CPU vs GPU" ] }, { "metadata": { "id": "_4QjDwoa1Xw7", "colab_type": "code", "outputId": "60ac43b7-81a3-459b-e212-4237c7746f72", "colab": { "base_uri": "https://localhost:8080/", "height": 34 } }, "cell_type": "code", "source": [ "model_cpu = nn.Sequential(\n", " nn.Linear(20, 20),\n", " nn.ReLU(),\n", " nn.Linear(20, 4),\n", " nn.Softmax()\n", ").float()\n", "model_gpu = model.float()\n", "next(model_cpu.parameters()).is_cuda, next(model_gpu.parameters()).is_cuda" ], "execution_count": 0, "outputs": [ { "output_type": "execute_result", "data": { "text/plain": [ "(False, True)" ] }, "metadata": { "tags": [] }, "execution_count": 37 } ] }, { "metadata": { "id": "s21Imowu3woJ", "colab_type": "code", "colab": {} }, "cell_type": "code", "source": [ "cpu_inp = torch.ones((100000, 20), dtype=torch.float32)\n", "gpu_inp = torch.ones((100000, 20), dtype=torch.float32).cuda()" ], "execution_count": 0, "outputs": [] }, { "metadata": { "id": "6zTaBUiS4E5u", "colab_type": "code", "outputId": "895c3847-96b3-4a3d-8ca2-6e65f5f66535", "colab": { "base_uri": "https://localhost:8080/", "height": 87 } }, "cell_type": "code", "source": [ "%timeit res = model_cpu(cpu_inp)" ], "execution_count": 0, "outputs": [ { "output_type": "stream", "text": [ "/usr/local/lib/python3.6/dist-packages/torch/nn/modules/container.py:91: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n", " input = module(input)\n" ], "name": "stderr" }, { "output_type": "stream", "text": [ "10 loops, best of 3: 28 ms per loop\n" ], "name": "stdout" } ] }, { "metadata": { "id": "jX0ENIk_4K5D", "colab_type": "code", "outputId": "4e24e52b-2caf-434b-bb15-318ebb7cbb0a", "colab": { "base_uri": "https://localhost:8080/", "height": 87 } }, "cell_type": "code", "source": [ "%%timeit \n", "for i in range(10): # Because we don't have enough memory on GPU\n", " res = model_gpu(gpu_inp[i*1000:i*1000+1000])" ], "execution_count": 0, "outputs": [ { "output_type": "stream", "text": [ "/usr/local/lib/python3.6/dist-packages/torch/nn/modules/container.py:91: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n", " input = module(input)\n" ], "name": "stderr" }, { "output_type": "stream", "text": [ "100 loops, best of 3: 3.61 ms per loop\n" ], "name": "stdout" } ] }, { "metadata": { "id": "7-U_851D4UWx", "colab_type": "code", "outputId": "2d2f6fd4-4d32-4b32-ad59-794e28b0f0a2", "colab": { "base_uri": "https://localhost:8080/", "height": 417 } }, "cell_type": "code", "source": [ "fig, ax = plt.subplots(1, 1)\n", "ax.plot([\"a\", \"b\"], [28, 3.61], \"-o\")\n", "ax.set_xticklabels([\"CPU Model\", \"GPU Model\"])\n", "fig.suptitle(\"CPU vs GPU\", fontsize=17)\n", "ax.set_ylabel(\"Time (ms)\")\n", "ax.set_xlabel(\"Model Type\")" ], "execution_count": 0, "outputs": [ { "output_type": "execute_result", "data": { "text/plain": [ "Text(0.5,0,'Model Type')" ] }, "metadata": { "tags": [] }, "execution_count": 54 }, { "output_type": "display_data", "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAAfIAAAF/CAYAAACyv0vWAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJzs3XlcVXX+x/HXuZddUVYBEXcEFDVX\nxDWXXFvcMjfS1GmZyWymZWrMprLG0bYx2xUzNbXUNAtNM7MxF8wtRXFXxAVERQHZ4f7+sPjFJCIK\nXC68n49Hf3A495zPbR7Nm/f5nnuuYbFYLIiIiIhNMll7ABEREbl1CnIREREbpiAXERGxYQpyERER\nG6YgFxERsWEKchERERumIBe5Tdu3b+fxxx+nU6dOhIaG0rlzZ8aPH8+3335baL8vv/ySoKCgQv8E\nBwcTHh7Ok08+yZEjRwr2PX36NEFBQcybN++650xJSSEoKIhZs2aV5Vu7obS0ND766CMGDx5MmzZt\naNasGZ07d+bJJ59k27Ztf9i/R48ef3j/zZs3595772XevHnk5eUV7Pvcc8/Rtm3bIs/92muvERQU\nVCbvS8TW2Fl7ABFb9sEHHzBz5kyGDh3KO++8g6+vL+fOnWPFihVMmjSJBx54gFdeeaXQa+bNm0fj\nxo0ByMnJ4fjx47z11lsMHz6clStXEhAQYI23UiJnzpxh3Lhx5Ofn89hjj9GqVSvMZjPHjx9n/vz5\njB07lueff54xY8YUel2XLl2YNm1awc+XLl1iw4YNTJ8+nfj4eKZMmVLeb0XE5inIRW7R5s2b+c9/\n/sNTTz3Fww8/XLDd39+ftm3bUqdOHWbOnMnAgQNp3bp1we/d3Nzw9vYu+Ll27doEBgbStWtXlixZ\nwjPPPFOu7+NWPPXUU+Tk5LB8+XLc3d0LttetW5euXbvyl7/8hWXLljFkyBCqV69e8HsHB4dC793b\n25ugoCDOnj3LkiVL+Otf/1pofxEpni6ti9yiuXPn4u/vz4QJE677+/Hjx7Nx48ZCIV4UHx8fPDw8\nOH36dKnMFh8fT1BQEHPnzv3D7x577DF69uyJxWIhISGBp59+mi5duhAaGkrXrl2ZMmUKKSkpRR57\nx44d7N69m4kTJxYK8d+YTCZmzZrF119/fdOhHBwcTG5uLgkJCTf/JkUEUJCL3JLc3Fx+/vlnunTp\ngsl0/f+MHBwc8PX1vanjXb58mcuXL1O7du1SmS8gIIBWrVqxZs2aQttTUlLYtGkTAwcOxDAMnn76\naQ4ePMjMmTP57rvvmDFjBjt27ODZZ58t8tjR0dEAdOvWrch97OxKdrHv5MmTmM1matWqVaLXiYgu\nrYvcksuXL5OVlYW/v/9tH+v06dNMnToVR0dH7r///lKY7pr77ruPl156iTNnzhTMuXbtWnJzcxk4\ncCAA+/btK3Tp38/Pj8jISNLS0oo8bmJiIk5OTnh4eNz2jFlZWWzYsIEvvviCQYMGUaNGjds+pkhV\noyAXuQWGYdzya4cPH17Q4nNzc8nOziY0NJS5c+fSsGHD0hqRfv368dprr7FmzZqCy/9RUVG0adOm\n4Ia6u+66iy+++ILs7Gy6d+9O+/bti70qYDKZCt1h/psPP/yQjz76qNC2Rx55hEcffbTg540bN9Kq\nVauCnzMyMnBxcWHkyJH89a9/veX3KlKVKchFboGbmxsuLi6cPHmyxK99++23adSoEXAtFGvWrPmH\nJmo2m294jN+C9EaXsN3c3LjzzjsLgjwpKYnt27fz8ssvF+wzbdo0WrduzapVq1i1ahUWi4Xw8HD+\n8Y9/FMz4v2rXrk1OTg5nz54tFPrDhw+nX79+BT/37t2bnJycQq8NCwvjpZdeKvj5t5vf/vd9mEwm\nbvTFjPn5+SW+fC9SWWmNXOQWmM1mwsLC2LhxI1lZWdfdx2KxsHDhwj/cOObn50e9evWoV68eAQEB\n172c7O7ujslk4sKFC9c99m83hRW3Bn/fffcRExNDfHw8a9aswd7evlDY2tvbM3LkSJYsWUJ0dDSv\nv/46R48e5U9/+lORQdqpUycA1q1bV2i7m5tbwfuqV6/edV/r7OxcaB8/P7/rBrKXlxfp6elkZGRc\n9zjnzp3Dx8fnhu9dpKpQkIvcogkTJnDp0iVef/316/4+MjKSqVOnsnv37hIf28nJibCwMFavXk16\nevoffj9v3jycnJzo0qXLDY/TrVs33Nzc+P7771m9ejW9evUquJM8OTmZr776iqtXrwJQvXp1BgwY\nwJgxYzhz5gwXL1687jGbNWtG165d+eCDD4iPj7/uPseOHSvJ2/2Drl27kp+fz/Lly6977E2bNnHX\nXXfd1jlEKgvzS7+/ziUiN6127do4OTnxwQcfcOjQITw8PDAMg+PHjzNr1iwiIyN58sknGTp0KACx\nsbF8//33DB8+vNBnqYvSrFkzFi9ezObNm/H19S049jvvvMPKlSuZPHkyYWFhNzyG2Wzm9OnT/Pjj\nj8TExPD3v/+dunXrAtfWp8eNG8ehQ4cICAjAYrFw6NAhZs+ejYeHB+PHjy/yXoBOnTqxYcMGPv30\nU+zt7alWrRoZGRkcPnyYBQsWMGXKFGrXrs2kSZPw9PQE4NNPP8XLy4sBAwbc1L/bxMRE5s2bh8lk\nwtXVlZSUFDZu3MjkyZPx8PBg+vTpODk5FXsskcrOsNxoIUpEirVr1y7mzZvHrl27uHz5Ml5eXjRr\n1owxY8bQvn37gv2+/PJLnn/+eVauXElISMhNHTs+Pp7333+fzZs3c/HiRapXr06LFi0YN24c4eHh\nNz3fiBEj8PHxYePGjYU+Lrd3715mzpxJTEwMV69excvLi/DwcCZNmlTsZfvs7GwWL17MmjVrOHr0\nKJmZmbi7uxMcHMxdd93FwIEDcXBwKNi/R48eBAcH8/7779/U3BaLhWXLlvHFF19w9OhRcnJyqFOn\nDr179+bhhx/Wg2NEfqUgFxERsWFaIxcREbFhCnIREREbpiAXERGxYQpyERERG6YgFxERsWEKchER\nERumIBcREbFhCnIREREbpiAXERGxYQpyERERG6YgFxERsWEKchERERumIBcREbFhCnIREREbpiAX\nERGxYQpyERERG6YgFxERsWEKchERERumIBcREbFhCnIREREbpiAXERGxYQpyERERG6YgFxERsWEK\nchERERumIBcREbFhCnIREREbZmftAW4kKSm11I/p7u5CcnJ6qR9XRESqptLOFW9v1xLtX+UauZ2d\n2dojiIhIJWLtXKlyQS4iIlKZKMhFRERsmIJcRETEhinIRUREbJiCXERExIYpyEVERGyYglxERMSG\nKchFRERsWIV+sltpij6QSNTWk5y9mE5tTxcGhNcnrKmPtccSERG5LVUiyKMPJPLRqv0FP59Oulrw\ns8JcRERsWZW4tB619WQR2+PKdQ4REZHSViWC/OyF6z/M/tzFq+U8iYiISOmqEkFe28vlutu93ZzL\neRIREZHSVSWCfEB4/etuP5+cwZroOPLzLeU7kIiISCmpEkEe1tSHR+5tRh3v6phNBnW8q9O7XQDV\nne1Y+sMxpn22U5fZRUTEJhkWi6XC1tGkpNRSP6a3t2vBcVPTs/nsu8Nsjz2PvZ2JQV0a0rtdACaT\nUernFRGRyun3uVJaxyuJKtHIi+Lq4sCj94Xy54GhODmY+eKHo/z7s10kXLr+zXEiIiIVTZVu5L+X\nkp7NZ+sO8/PBa+18SLdG9GpbB5Ohdi4iIkVTI68garg48NjAUB4bGIqjvZkl3x9hxme7SExWOxcR\nkYpLQf4/2gXX4tUJYbQJ8ubw6Sv8M3I763fEk19xL1yIiEgVpiC/jhrVHPjzwFAeva8ZDvZmFq0/\nwuuLdnP+coa1RxMRESlEa+TFuHI1mwVrD7HrcBIO9ibuv7Mx3Vv7a+1cREQArZFXeDWrOfCXQaE8\nfE9T7M0mPvvuMG8s3k2S2rmIiFQAauQlcCUti/lrD7H7yAUc7c0M696Ibq3UzkVEqjI1chtSs7oj\njw9uzp/uboqd2WDBusO8uWQPF66onYuIiHUoyEvIMAzCQ315ZXwYLRt5EhuXzJTI7WzcfYYKfHFD\nREQqKQX5LXJ3deSJoS0YPyAEs2Ewf+0h3vp8DxevZFp7NBERqUIU5LfBMAw6Nfdj6oQwWjTyZP/J\nZKZERvPfX86qnYuISLlQkJcCd1dHJg1twbj+IRgGzFtzkLe/+IVLKWrnIiJSthTkpcQwDDq38GPq\n+DBCG3oQc+ISUyKj2aR2LiIiZUhBXso8ajjx1/tbMrZfMACfrDnIf5buJTk1y8qTiYhIZaQgLwOG\nYdC1ZW2mjg+jWQMP9h2/yAtzotm875zauYiIlCoFeRnyqOHE34a1ZEzfICwWC5FRscxcpnYuIiKl\nR0FexgzDoNsd/rwyvj0h9dzZe+wiU+ZEsyVG7VxERG6fgryceNV05unhdxDRJ4i8fAtzvoll1vJ9\nXE5TOxcRkVunZ61bwYXLGXyy5iCxcclUc7Jj5F1N6NDUB0PPbBcRsTl61noV5OXmzFPD72B07ybk\n5lmY/fUB3v1yH1euZlt7NBERsTF21h6gqjIZBj1a1yG0oSefRMWy+8gFDsdvY1TvJoSFqJ2LiMjN\nUSO3slpuzjwzshWj7mpCTl4+H686wPsrYkhROxcRkZugRl4BmAyDnm3q0LyhB3OjYtl5OIlD8ZcZ\n3bsJ7UN8rD2eiIhUYGrkFUgtdxeeHdWaEb0Cyc7J48Ov9vP+yhhS0tXORUTk+tTIKxiTYXBX2wBa\nNPQkcnUsOw6e59CpZCJ6B9E2uJa1xxMRkQqmTD9+NmPGDHbu3Elubi6PPPIIGzZsYP/+/bi5uQEw\nfvx47rzzziJfX1k/fnaz8vMtrN8Rz/L/HicnN5/2IbUYdVcTXF0crD2aiIj8ytofPyuzRr5t2zaO\nHDnC559/TnJyMoMGDaJDhw787W9/o3v37mV12krFZDLo3b4uzRt5Mnd1LNtjz3MwLpmIPsG0CfK2\n9ngiIlIBlNkaebt27Zg5cyYANWrUICMjg7y8vLI6XaXm51mN50e1YVj3xqRn5fHein18tGo/aRk5\n1h5NRESsrFye7Pb555+zY8cOzGYzSUlJ5OTk4OnpyZQpU/Dw8Cjydbm5edjZmct6PJsSn5jKzCW7\nOXQqGTdXR/4ytCUdQv2sPZaIiFhJmQf5+vXr+eijj5g7dy4xMTG4ubkREhLCxx9/TEJCAi+++GKR\nr63qa+RFyc+3sHb7KVZsOkFuXj7hzXwY0asJ1Z3trT2aiEiVY+018jL9+NmmTZv48MMPmT17Nq6u\nroSHhxMSEgJAjx49OHz4cFmevtIymQz6dajHPx9qRwM/V7buT2TKnGj2HLlg7dFERKSclVmQp6am\nMmPGDD766KOCu9QnTpxIfHw8ANHR0QQGBpbV6asEf69q/COiDUO6NeRqZg7vLN/LnG8OcDVTa+ci\nIlVFmd21vnr1apKTk3nyyScLtg0ePJgnn3wSZ2dnXFxcmDZtWlmdvsowm0wMCK/PHY29mBMVy5aY\nBA6cvMTYfsG0aORl7fFERKSM6WtMK5G8/HzWbDvFVz+dIC/fQqfmvozoGYiLk9bORUTKirXXyPVk\nt0rEbDJxd8dr7TwyKpbN+xI4cDKZsf2Cad7Q09rjiYhIGdCz1iuhOrWqM/nBNgzs0oCUq9m8/cUv\nfLI6lvTMXGuPJiIipUyX1iu5U4mpzI2K5dT5NNxdHXmofzChDdTORURKi7UvrauRV3J1fVx5YUxb\n7ut8rZ2/9fkvzFtzkIwstXMRkcpAjbwKOZWYypxvYjmdlIZnDUfG9g+hWf2in6wnIiLFUyOXclPX\nx5UXx7blno71SU7N5s0le5j/rdq5iIgtUyOvok4mpBAZFcuZpKt41nBiXP9gQtTORURKTI1crKK+\nbw1eHNOOuzvWIzk1i9eX7GHBukNkZqudi4jYEjVy4cS5a+387IWreNV0Ylz/EILruVt7LBERm2Dt\nRq4gFwBycvNZtfkEq7fFYbFAz9Z1GHpnIxwd9DWyIiI3oiC/AQV5+Tt+NoXIqAOcu5iOt9u1dh5U\nV+1cRKQoCvIbUJBbR05uHis3neDb7afAAj3b1mFIt0Y42qudi4j8LwX5DSjIrevYmStERsWScCmd\nWu7OjOsfQpMAN2uPJSJSoSjIb0BBbn3ZOdfa+drtpwC4q10Ag7o2VDsXEfmVgvwGFOQVx9HTV4iM\nOkBicgY+7s6MH9CUxnVqWnssERGrU5DfgIK8YsnKyWPFf4/z3c/xAPRpX5eBXRrgoHYuIlWYgvwG\nFOQV0+H4y8xdHcv55Ax8PVwYPyCERv5q5yJSNSnIb0BBXnFl5eSx/MdjfL/jNBjQ99d2bm+ndi4i\nVYu1g1yPaJVb4mhvZmSvJjw7shVeNZ1YE32Klz75meNnU6w9mohIlaJGLrctKzuPZT8e4/udpzEM\n6BdWj/s6N8DeTn8nikjlp0YuNs/Rwcyou5rw7IhWeNZwYvW2OF6Z9zMnzqmdi4iUNTVyKVWZ2bks\n23iMDbvOYDIM+nWoy72d1M5FpPJSI5dKxcnBjtG9g3hm+B24uzoStTWOVz79mbgE/fEkIlIW1Mil\nzGRk5bJ04zE27r7WzgeE1+OeTvWxM+vvRxGpPNTIpdJydrTjwT5BPDX8DtxdHfh6y0lembeDU4n6\nQ0pEpLSokUu5yMjK5YsfjvLjnrOYTdfa+d0d1c5FxPapkUuV4Oxox5i+wfztgZbUrO7Aqs0nefVT\ntXMRkdulRi7lLj0zl883HGHT3nOYTQb3dKpP/w711M5FxCapkUuV4+Jkx0P9Q3jy/pbUqObAyk0n\neG3+Tk6fT7P2aCIiNkeNXKwqPTOHJd8f5ad919r5vZ0b0L9DXcwm/Y0pIrZBjVyqNBcne8YNCOHJ\n+1vg6mLPiv8e57X5OzmTpHYuInIz1MilwriamcPi9UfYEpOAndngvs4N6Bumdi4iFZsaucivqjnZ\nM+HupjwxpAXVnOxZ/uNx/rVgJ2cuXLX2aCIiFZaCXCqcOwK9mDohjPBmPpw4l8rLn/zMmm1x5OdX\n2ItHIiJWoyCXCqm6sz1/uqcZEwc3x8XJjqUbj/GvhTs5d1HtXETk97RGLhVeWkYOi747zLYDidiZ\nTQzu2pDe7QIwmQxrjyYiojVykeJUd7bn4Xub8ZdBzXFxNPPFD0eZ9tlOEi6lW3s0ERGrUyMXm5Ka\nns1n3x1me+x57O1MDOnakF5t1c5FxHrUyEVKwNXFgUfvC+XPA0NxcjCzZMNR/r1oF4lq5yJSRSnI\nxSa1Da7F1AlhtA2uxdHTV/jn3O1893M8+RX3ApOISJlQkIvNquHiwJ8HhvLofc1wsDez+PsjzPhs\nF4nJauciUnUoyMXmtQ/xYeqEMNo08ebwr+18/Q61cxGpGhTkUinUrObAnweF8si9zbA3m1i0/giv\nL9rN+csZ1h5NRKRM6a51qXSupGUxf+0hdh+5gKO9maF3NqJ7a39Mhu5sF5HSp7vWRUpZzeqOPD64\nOQ/f0xQ7s8Fn3x3mjcW7uaB2LiKVkIJcKiXDMOjQzJepE8K4o7EXB09dZsrc7fyw+wwV+CKUiEiJ\nKcilUnOr7sjEIc2ZcHcIZsNgwdpDvPn5Hi5cUTsXkcqhTNfIZ8yYwc6dO8nNzeWRRx6hefPmPPvs\ns+Tl5eHt7c3rr7+Og4NDka/XGrmUpuTULOZ/e5Bfjl3EycHMsB6N6dayNobWzkXkNlh7jbzMgnzb\ntm1ERkYye/ZskpOTGTRoEOHh4XTt2pV+/frx1ltv4evry8iRI4s8hoJcSpvFYmFLTAKL1h8hIyuX\nZvXdGdsvBM+aTtYeTURsVKUN8ry8PLKysnBxcSEvL4+OHTtSrVo1vv32WxwcHNi9ezdz585l1qxZ\nRR5DQS5lJTk1i3lrDrLv+LV2PrxnIF1a+Kmdi0iJWTvI7UrtzP/DbDbj4uICwLJly+jatSs//fRT\nwaV0T09PkpKSbngMd3cX7OzMpT5bSf8lSeXj7e3Ka3/uxPrtp5izKoZ5aw6y98QlJt5/B15uztYe\nT0RsjDVzpcyC/Dfr169n2bJlzJ07l969exdsv5kLAcll8KhNNXL5vTsaevDKuPbMW3OQXQfP8+cZ\nGxjeszGdm6udi8jNsXYjL9O71jdt2sSHH37I7NmzcXV1xcXFhczMTAASExOpVatWWZ5e5KZ41HDi\nr8NaMrZfMBaLhU9WH2Tmsr0kp2ZZezQRkWKVWZCnpqYyY8YMPvroI9zc3ADo2LEja9euBWDdunV0\n6dKlrE4vUiKGYdC1ZW2mjg+jaX139h67yAtzotm875w+dy4iFVqZ3ez2+eefM2vWLBo0aFCw7d//\n/jcvvPACWVlZ1K5dm2nTpmFvb1/kMXSzm1iDxWLhx1/O8vmGo2Rl59GykScP9g3G3dXR2qOJSAVk\n7Uvreta6SBEuXM7gkzUHiY1LppqTHSN7NaFDMx+tnYtIIdYOcj3ZTaQIXm7OPD38DiL6BJGbZ2H2\nNweYtXwfV9K0di4iFYeCXOQGDMOgeyt/XhnfnuC6buw5eoEX5kSzbX+C1s5FpEJQkIvcBG83Z54e\n0YpRdzUhJy+fj78+wHsrYrhyNdvao4lIFVfmnyMXqSxMhkHPNnVo3tCDuasPsutwEofjLzO6dxPa\nBdfS2rmIWIUauUgJ1XJ34dmRrRjRK5DsnDw+/Go/76+MIUXtXESsQI1c5BaYDIO72gbQopEnc6Ni\n2XkoiUOnLhPRJ4h2wXrQkYiUHzVykdvg4+7C30e1ZnjPa+38g5Ux19p5utq5iJQPNXKR22QyDHq3\n+/92vuPgeQ6dSiaidxBt1c5FpIypkYuUEl8PF54b1ZoHejQmMzuP91fG8OFXMaRl5Fh7NBGpxNTI\nRUqRyWTQp33dgna+PfY8B+OSiegTTJsgb2uPJyKVkBq5SBnw86zG86PbMKx7Y9Kz8nhvxT4+XrVf\n7VxESp0auUgZMZkM+oZda+eRUbFsO5BIbFwyD/YNolWg2rmIlA41cpEyVturGv+IaM3QOxtxNTOH\nWcv3MfvrA1zNVDsXkdunRi5SDswmE/071KNlYy/mRh1g6/4EDsRdYkzfYO5o7GXt8UTEhqmRi5Qj\nf69q/COiDUO6NSQtPYd3lu0l8psDpKudi8gtUiMXKWdmk4kB4fVp2diLyKhYNscksP/kJcb2C6ZF\nI7VzESkZNXIRK6njXZ3JEW0Y1KUBqek5/GfpXuZGxZKemWvt0UTEhhiWCvylyklJqaV+TG9v1zI5\nrsjtiD+fRmTUAU4lpuHu6shD/YIJbehp7bFE5CaUdq54e7uWaP8bBnlCQgJz585l06ZNnD17FgB/\nf3+6dOnC2LFj8fPzu71pi6Egl6okNy+f1Vvj+HrLSfLyLXRt6ccDPQJxdtQKmEhFVmGDfNmyZURG\nRjJixAg6duxI7dq1ATh79ixbtmxhyZIljB8/niFDhtz+1EVQkEtVdCoxlcioWOLPp+FRw5GH+oXQ\nrIGHtccSkSJYO8iL/FP/yJEjrFq1Cnt7+0LbGzduTOPGjRk+fDhvvvnmrU0pIkWq6+PKlDFt+WbL\nSaK2xvHm53vodkdthnVvrHYuIn9Q7Br5lStXOH/+PIGBgWzatIm9e/cybNgwvL3L/slUauRS1cUl\npBIZdYDTSVfxrOHEQ/2DaVpf7VykIrF2Iy/2rvVnnnmG8+fPc/LkSf7973/j5ubG5MmTb3lAEbl5\n9XxdeXFsO+7pWJ/k1CzeWLKH+WsPkZGlO9tF5JpigzwjI4NOnTrx7bffMnr0aEaNGkVOjh5eIVJe\n7MwmBnVtyAtj2uDvVY2Nu8/wz7nbiY1LtvZoIlIB3FSQX7p0ibVr13LnnXdisVi4cuVKecwmIr9T\n37cGL45tx4DwelxMyeT1xbtZuO4Qmdlq5yJVWbFBfs8999C7d286dOiAn58f7733HmFhYeUxm4j8\nD3s7E0O6NeKFB9tS26saG3ad4cXI7Rw6pXYuUlWV+IEwqampuLqWbCH+VulmN5Gi5eTmsfKnE3wb\nfQqLBXq2qcPQbo1wdDBbezSRKsXaN7sVG+Rbtmxh0aJFpKam8vtd58+ff2sTloCCXKR4x85eYW5U\nLOcuplPLzZlxA0JoEuBm7bFEqowKH+S9e/fmsccew9fXt9D28PDwkk9XQgpykZuTk5vHik0nWLv9\nFFigV9sABndriKO92rlIWbN2kBf7dIn69eszaNCgWx5IRMqevZ2ZYd0b07qJN5FRsXy3I569xy4w\nbkAIgXXUzkUqs2Ib+fr16/nhhx9o1aoVdnb/n/sDBw4s8+HUyEVKLjsnjxWbjrNuezwAd7ULYHDX\nhjionYuUiQrfyD/88EOcnZ3Jzs4u2GYYRrkEuYiUnIO9mQd6BNK6iTdzo2JZ93M8vxy7yPgBITT2\nr2nt8USklBXbyEeMGMHixYvLa55C1MhFbk9WTh4r/nuc736OBwP6tKvLwC4N1M5FSpG1G7n5pZde\neulGOyQnJ5OTk0OtWrUwDAOLxYLFYsEwjNuZ86akp2cXv1MJVavmWCbHFamI7MwmQht6ElLPncOn\nLvPLsYvsPJxEfT9XPFydrD2eSKVQ2rlSrZpjifYvtpG3atWKjIyMazv/GuSGYRAbG3vrU94kNXKR\n0pOVncfyH4+xfudpDAP6htVlYOcG2NupnYvcDms38iKDPCcn5w9fYXor+9wOBblI6Tt0Kpm5q2NJ\nupxJba9qjB8QQgO/GtYeS8RmWTvIi3xE64QJEzhx4kSRLzx27BgTJkwo0clExPqC6rrzyrgwerau\nw9kLV3l1/g6W/3iMnNx8a48mIregyEZ+5MgRnn32WXx9fenSpQt+fn4AnDt3jk2bNpGYmMj06dMJ\nDAwss+HUyEXKVmxcMp+sjuXClUz8vaoxTu1cpMSs3chvuEZusVj4/vvv+e9//0tCQgIAvr6+dO3a\nlZ49e5b5DW8KcpGyl5mdy9IfjvHD7jOYDIP+4fW4t1N97MzFfqeSiFDBg9zaFOQi5efAyUt8svog\nF1MyqeNdjfEDmlLPt3y+IEnElinIb0BBLlK+MrJyWfrDUTbuOYvJMLi7Yz3u7qh2LnIj1g5y/dcp\nIgWcHe14sG8wTz1wB26uDqyOuQtjAAAgAElEQVTafJKpn+7gVKL++BWpqG4qyPPz80lKSirrWUSk\ngmjWwIOp48Po2tKP+PNpTP10B1/9dILcPN3ZLlLRFBvkW7dupVevXkRERADwr3/9ix9++KHMBxMR\n63J2tGNsvxD+NqwlNao58NVPJ3h1/g7iz6dZezQR+Z1ig/ztt9/miy++wNvbG4BHH32UDz74oMwH\nE5GKIbShJ1PHh9G5hR+nEtN4Zd7PrNqsdi5SURQb5C4uLnh5eRX87OHhUaZPcxORisfFyY5x/UN4\n8v6WuLrYs3LTCV6bv5PTSWrnItZWbJA7OTmxfft2AK5cucKiRYtwdLy5B7ofPnyYXr16sXDhQgCe\ne+457rnnHiIiIoiIiGDjxo23PrmIlLsWjTx5dUIYnZr7EpeYysuf/Mw3W06Sl692LmItxX787Ny5\nc7z00ktER0fj4OBAmzZtmDx5MnXq1LnhgdPT03nkkUeoX78+QUFBjB49mueee44+ffrQvXv3mxpO\nHz8Tqbh+OXqBed8e5EpaNvV9XRl/d1P8vapZeyyRcmftj5/ZFbeDn58fH330UYkHcXBwYPbs2cye\nPbvErxWRiq9lYy9enRDGou+OsHV/Ai9/sp37Ojegb1hdzCZ9slWkvBTbyLds2cKiRYtITU3l97vO\nnz//pk4wa9Ys3N3dCxp5UlISOTk5eHp6MmXKFDw8PIp8bW5uHnb6ikWRCm/7/gTeXbqH5NQsmtR1\n48nhrQnw0VPhRMpDsUHeu3dvHnvsMXx9fQttDw8Pv6kT/D7It27dipubGyEhIXz88cckJCTw4osv\nFvlaXVoXsR1pGTksWn+YbfsTsTObGNS1AX3a1cVkKtvvZBCxtgp/ab1+/foMGjTolgf6vd+Hf48e\nPXjppZdK5bgiYn3Vne15+J5mtA2qxfxvD7L0h2PsOpTEuAEh+Hlq7VykrBQb5MOGDWPy5Mm0atUK\nO7v/333gwIElPtnEiRN59tlnCQgIIDo6uky/AlVErKN1E2+aBLjx2XeHiT6QyEuf/MygLg3p3S5A\n7VykDBR7aX3o0KE4OzsXurRuGAYzZsy44YFjYmKYPn06Z86cwc7ODh8fH0aPHs3HH3+Ms7MzLi4u\nTJs2DU9PzyKPoUvrIrZt56HzzF97iNT0HBr712TcgBB8PVysPZZIqbL2pfVig3zEiBEsXrz4toa6\nVQpyEduXkp7NZ+sO8/PB89jbmRjStSG92qqdS+Vh7SAv9jMiPXr0YNu2bWRnZ5Ofn1/wj4jIzajh\n4sBjA0N5bGAojvZmlmw4yvRFu0hMTrf2aCKVQrGNvFWrVmRkZBR+kWEQGxtbpoOBGrlIZZNyNZsF\n6w6x81ASDnYmhtzZiJ5t6mAy1M7Fdlm7kRcb5NakIBepfCwWCz8fPM/CdYdJy8ihSYAb4/oHU8td\na+dimypskC9fvpwhQ4Ywc+bM675w0qRJJZ+uhBTkIpXXlavZLFh7iF2Hk3CwN3H/nY3p3tpf7Vxs\njrWDvMg1ctOvj1g0m83X/UdE5HbUrObAXwaF8vC9TbE3m/jsu8O8sXg3SZczin+xiBQo8nPkv4X1\n448/Xm7DiEjVYhgGHZr6ElLXnflrD7H7yAVejNzOsO6N6NZK7VzkZhTZyJctW1aec4hIFVazuiOP\nD27On+5uip3ZYMG6w7y5ZA8X1M5FiqWvKBKRCsEwDMJDfZk6IYw7GnsRG5fMlLnb2bj7DBX4nlwR\nqyvyZrfmzZtf96lrFosFwzDYuHFjWc+mm91EqiiLxcKWmAQWrz9CelYuTeu7M7ZfMF41na09msgf\nWPtmtyLXyJs2bcpbb7112wOJiJSUYRh0au5H0/oefPrtQfYeu8iLkdt5oEdjurasjaG1c5ECRQa5\ng4MD/v7+5TmLiEgh7q6OTBrags37Elj8/WE+/fbaw2TG9gvGo4aTtccTqRCKXCNv0aJFec4hInJd\nhmHQuYUfU8eHEdrQg5gTl5gSGc1/fzmrtXMR9GQ3EbEhFouFn/aeY8mGI2Rk5RHa0IOxfdXOxbqs\nvUauu9ZFxGYYhkGXlrWZOj6MZg08iDl+iSmR2/lp7zm1c6myFOQiYnM8ajjxt2EtGdM3CIvFwtzV\nscxctpfk1CxrjyZS7nRpXURs2oUrGcxbc5ADJ5NxcbRjRK9AOob66s52KTe6tC4ichu8ajrz1AN3\n8GCfIPIsFiKjYpm1fB+X09TOpWpQkIuIzTMMgztb+TN1XHtC6rmz5+gFpsyJZuv+BK2dS6WnIBeR\nSsPLzZmnht/B6N5NyM2zMPvrA7z75T6uqJ1LJVbkA2FERGyRyTDo0boOoQ09+SQqlt1HLnA4/jKj\nejchLMRHa+dS6aiRi0ilVMvNmWdGtmLUXU3Iycvn41UHeG9FDFeuZlt7NJFSpUYuIpWWyTDo2aYO\nzRt6MDcqll2Hkzgcf5nRvZvQPsTH2uOJlAo1chGp9Gq5u/DsqNaM6BVIdk4eH361n/dX7CMlXe1c\nbJ8auYhUCSbD4K62AbRo6Enk6lh2HEriUPxlInoH0Ta4lrXHE7llauQiUqX4eLjw3MjWDO/RmMzs\nPN5fGcMHK2NIVTsXG6VGLiJVjslk0Lt9XVo09mJuVCw/HzzPoVPJRPQJok2Q2rnYFjVyEamyfD1c\neG5Ua4Z1b0x6Vh7vrYjho1X7ScvIsfZoIjdNjVxEqjSTyaBvWF1aNvYkMiqW6AOJxMYlM6ZPEK2a\neFt7PJFiqZGLiAB+ntX4x+g23H9nI9Izc5n15T4+/lrtXCo+NXIRkV+ZTAb9OtQrWDvftj+R2JPJ\nPNg3iFaBaudSMamRi4j8D3+vavwjojVDujXkamYOs5bvY/bXB7iaqXYuFY8auYjIdZhNJgaE1+eO\nxl7MiYpl6/4EYuMuMaZvMC0be1l7PJECauQiIjfg712dFx5sw+CuDUlNz2Hmsr1ERh0gXe1cKgg1\nchGRYphNJu7ueK2dR0bFsnlfAgdOJjOmbzAtGnlaezyp4tTIRURuUp1a1Zn8YBsGdWlAytVs/rP0\nF+aujiU9M9fao0kVZlgsFou1hyhKUlJqqR/T29u1TI4rIlVL/Pk0Ir85wKnzabi7OvJQ/2BCG6id\nV0WlnSve3q4l2l+NXETkFgTUqs4LY9pyX+dr7fytz39h3pqDZGSpnUv5UiMXEblNpxJTmfNNLKeT\n0vCs4cjYfiE0a+Bh7bGknKiRi4jYuLo+rrw4ti33dqpPcmo2b36+h/nfqp1L+VAjFxEpRXEJqURG\nHeB00lU8azjxUP9gmtZXO6/M1MhFRCqRer6uTBnTjrs71iM5NYs3luxhwbpDZGarnUvZUCMXESkj\nJ86lEBkVy9kLV/Gq6cS4/iEE13O39lhSyqzdyBXkIiJlKCc3n1WbT7B6WxwWC/RsXYehdzbC0cFs\n7dGklCjIb0BBLiKVxfGzKURGHeDcxXS83a6186C6aueVgYL8BhTkIlKZ5OTmsfKnE3wbfQqLBXq1\nqcOQbmrntk5BfgMKchGpjI6duUJkVCwJl9Kp5e7MuP4hNAlws/ZYcosU5DegIBeRyio7J4+Vm06w\ndvspAHq1DWBwt4Y42qud2xprB3mZfvzs8OHD9OrVi4ULFwJw7tw5IiIiGDlyJJMmTSI7O7ssTy8i\nUmE52JsZ1qMxz49uQy0PF77bEc9Lc7dz9PQVa48mNqbMgjw9PZ2pU6cSHh5esO2dd95h5MiRLFq0\niHr16rFs2bKyOr2IiE1oXKcmLz/Ujt7tAjifnMG0hTv5fMMRsnPyrD2a2IgyC3IHBwdmz55NrVq1\nCrZFR0fTs2dPALp3787WrVvL6vQiIjbDwd7M8J6B/H1Ua7zdnVm7PZ6XPvmZY2fUzqV4dmV2YDs7\n7OwKHz4jIwMHBwcAPD09SUpKuuEx3N1dsLMr/fWikq4/iIiUB29vV9qE+rFgdSxf/3ScaQt3MrBb\nY0b1DcZBa+cVmjVzpcyCvDg3c49dcnJ6qZ9XN7uJSEU3sFN9QgJqMnd1LF9uPMrWfWcZNyCERrVr\nWns0uY5KfbPb/3JxcSEzMxOAxMTEQpfdRUTk/wXVdeeVcWH0bFOHcxfT+deCnSzbeIyc3HxrjyYV\nTLkGeceOHVm7di0A69ato0uXLuV5ehERm+LoYGbUXU14dkQrPGs4sXpbHC/P+5kT51KsPZpUIGX2\nOfKYmBimT5/OmTNnsLOzw8fHhzfeeIPnnnuOrKwsateuzbRp07C3ty/yGPocuYjINZnZuSzbeIwN\nu85gMgz6dajLvZ0aYG+nL7G0NmtfWtcDYUREbEjsyUt8suYgF65k4u9djfEDQqjvW8PaY1VpCvIb\nUJCLiPxRRlYuSzceY+Pua+28f3g97u1UHzuz2rk1WDvI9b+6iIiNcXa048E+QTw1/A7cXR34ZstJ\nXpm3g7gElZSqSI1cRMSGZWTl8sUPR/lxz1nMJoMB4fW4u6PaeXlSIxcRkVvm7GjHmL7B/O2BltSs\n7sCqzSeZ+ukOTiWqsFQVauQiIpVEemYun284wqa95zCbDO7pWJ/+4fXUzsuYGrmIiJQKFyc7Huof\nwl+HtaRGNQdW/nSCV+fv4PT5NGuPJmVIQS4iUsk0b+jJ1PHt6dzcj1OJabw872e+3nKSvHw9Fa4y\nUpCLiFRCLk72jBsQwpP3t8DVxZ4V/z3Oq/N3cjpJ7byy0Rq5iEgldzUzhyXrj7A5JgE7s8F9nRvQ\nN6wuZpO6XGnQGrmIiJSpak72jL+7KU8MbUE1Z3uW/3icfy3YyZkLV609mpQCNXIRkSokLSOHxesP\ns3V/InZmg4FdGtKnfYDa+W1QIxcRkXJT3dmeP93TjImDm+PiZM+yjceYtnAX5y6qndsqNXIRkSoq\nLSOHRd8dZtuBROzMJgZ1bUCfdnUxmQxrj2ZT1MhFRMQqqjvb8/C9zfjLoOa4OJpZ+sMxpn22U+3c\nxqiRi4gIqenZfPbdYbbHnsfezsTgrg25q22A2vlNUCMXERGrc3Vx4NH7QvnzwFCcHMx8vuEo/160\ni8RL6dYeTYqhRi4iIoWkpGezcN1hdhw8j4OdicHdGtGrbR1Mhtr59aiRi4hIhVLDxYE/Dwzl0fua\n4WBvZsn3R5jx2S4Sk9XOKyI1chERKVLK1WwWrDvEzkNJONiZGHpnI3q0UTv/PTVyERGpsGpUu9bO\nH7m3GfZ2JhatP8Lri3Zz/nKGtUeTX6mRi4jITbmSlsX8tYfYfeQCjvZmht7ZiO6t/at8O1cjFxER\nm1CzuiOPD27Ow/c0xc5s8Nl3h3lj8W6S1M6tSo1cRERK7HJaFvO/PcSeo9fa+bDujejWqmq2czVy\nERGxOW7VHZk4pDl/urspZpPBgnWHeXPJHi5cUTsvbwpyERG5JYZhEB7qy9QJYbRs5ElsXDJTIrez\ncc8ZKvDF3kpHQS4iIrfF3dWRJ4a2YPyAEEyGwfxvD/HW53u4eCXT2qNVCQpyERG5bYZh0Km5H69O\nCKN5Q0/2n0xmSmQ0//3lrNp5GVOQi4hIqXF3deTJ+1vwUP9gDAPmrTnI21/8wqUUtfOyoiAXEZFS\nZRgGXVrUZur4MEIbeBBz4hJTIqPZtFftvCwoyEVEpEx41HDir8NaMrZfMBYLfLL6IDOX7SU5Ncva\no1UqCnIRESkzhmHQteW1dt6svjt7j13khTnRbN53Tu28lCjIRUSkzHnWdOJvD9zBg32DyLdYiIyK\nVTsvJQpyEREpF4ZhcOcd/kwd356Qetfa+ZQ50WyJUTu/HQpyEREpV141nXl6+B1E9AkiL9/CnG9i\nmbV8H5fT1M5vhYJcRETKnWEYdG/lzyvj2xNc1409Ry8wZU40W/cnqJ2XkIJcRESsxtvNmadHtGLU\nXU3Iyctn9tcHePfLfVy5mm3t0WyGnbUHEBGRqs1kGPRsU4fmjTz5JCqW3UcucOR0NKPuakL7kFoY\nVfAb1UpCjVxERCqEWm7OPDOyFSN7BZKdk8dHq/bz/soYUtTOb0iNXEREKgyTYdCrbQDNG3kyNyqW\nnYeSOHTqMqN7N6F9iI+1x6uQ1MhFRKTC8XF34e+jWjO857V2/uFXv7bzdLXz/6VGLiIiFZLJMOjd\nLoCWjTyJXB3LjoPnOXQqmYjeQbQNrmXt8SoMNXIREanQfDxceG5kax7o0ZjM7DzeXxnDh1/FkKp2\nDqiRi4iIDTCZDPq0r0uLX9fOt8ee52BcMhF9gmkT5G3t8axKjVxERGyGn2c1nh/dhmHdG5Oelcd7\nK/bx0ar9pGXkWHs0q1EjFxERm2IyGfQN+7Wdr44l+kAisXHJjOkTRKsmVa+dq5GLiIhNqu1VjedH\nt+b+OxuRnpnDrC/3MfvrqtfO1chFRMRmmU0m+nWoR4vGXsyNOsDW/YkciEtmTN9g7mjsZe3xyoVh\nKcen00dHRzNp0iQCAwMBaNKkCVOmTCly/6Sk1FKfwdvbtUyOKyIi1pWXn8+30adYuekEefkWOob6\nMqJXINWc7Mv0vKWdK97eriXav9wbefv27XnnnXfK+7QiIlLJmU0mBoTXp2VjLyKjYtkSk8CBk5cY\n2y+YFo0qbzvXGrmIiFQqdbyrMzmiDYO6NiQ1PYf/LN3L3KhY0jMr59p5uV9af/nll6lbty5Xrlzh\n8ccfp1OnTkXun5ubh52dubzGExGRSubE2Sv8Z8lujp+5gmdNJyYOu4M2wZXrme3lGuSJiYns3LmT\nfv36ER8fz4MPPsi6detwcHC47v5aIxcRkduVm5fP6q1xfL3lJHn5Frq08OOBHoG4OJXO6rK118jL\n9dK6j48P/fv3xzAM6tati5eXF4mJieU5goiIVDF2ZhP3dm7AlDFtCahVnU17zzElMpqYExetPVqp\nKNcgX7VqFZGRkQAkJSVx8eJFfHwq1yUOERGpmOr6uDJlTFvu7VSflKvZvPX5L8xbc5CMrFxrj3Zb\nyvXSelpaGk8//TQpKSnk5OTw+OOP061btyL316V1EREpC3EJqURGHeB00lU8azgytn8Izep73NKx\nrH1pvVyDvKQU5CIiUlZy8/L5evNJorbGkW+xcGcrf+6/sxHOjiVbO1eQ34CCXEREytrJhBQiv4nl\nzIWreNZwYlz/YEJK0M6tHeT6HLmIiFRp9X1r8OLYdgwIr0dyahavL9nDgnWHyMy2jbVzNXIREZFf\nnTiXQmRULGcvXMWrphPjB4QQVNf9hq9RIxcREakgGvjV4J9j29K/Qz0upmQyfdFuPvvuMFnZedYe\nrUhq5CIiItdx7OwV5kbFcu5iOrXcnHmof/B127m1G7mCXEREpAg5uXms2HSCtdtPgQV6tq3DkG6N\ncLT//8eHK8hvQEEuIiIVwdEzV4iMiiXxUjq13J0Z1z+E5NQsorae5OzFdGp7ujAgvD5hTW//IWcK\n8mIoyEVE5FZk5+SxYtNx1m2Pp6jgfOTeZrcd5rrZTUREpAw42Jt5oEcgz41ujZ3ZuO4+UVvjynkq\nBbmIiEiJBNZxIz//+p383MWr5TyNglxERKTEantVu+52P8/rby9LCnIREZESGhBev4jt9cp3EKB0\nvlVdRESkCvnthraorXGcu3gVP89qDAivVyp3rZeU7loXERG5Ddb+HLkurYuIiNgwBbmIiIgNU5CL\niIjYMAW5iIiIDVOQi4iI2DAFuYiIiA1TkIuIiNgwBbmIiIgNU5CLiIjYsAr9ZDcRERG5MTVyERER\nG6YgFxERsWEKchERERumIBcREbFhCnIREREbpiAXERGxYQpyERERG1ahgvzkyZM8/PDDDB06lMGD\nBzN16lSys7MB6NGjByNHjiQiIoJhw4Yxc+ZMAKKjo3niiScKHWfWrFksXLiw0LbTp08TFBTEnj17\nCm0fMmQIzz333E3Nd/XqVXr06FHk7683i4iIWE9cXByPPvoo999/P/fffz+TJk3i0qVLAHz55Zd0\n69aNiIgIRo8eTUREBEePHgUgIiKCw4cPFzpWWFjYH47/3HPPMX78+ELbfvjhB4KCgjh9+vRNzTh9\n+nS+/PLLIn9/vVl+r8IEeV5eHhMnTmTChAksW7aM5cuXA/Dee+8V7DN79mwWLFjAkiVL2LVrFzt2\n7CjROQICAvjmm28Kfo6LiyMlJaV03oCIiFQov8+VpUuXsnTpUpo1a8Zrr71WsE///v1ZsGABCxcu\nZOLEibz66qslPs/p06cL/jgAWL16NQEBAaXyHm6GXbmdqRibN2+mYcOGtG/fHgDDMHjmmWcwmf74\nt4bJZKJ58+bExcVRp06dmz5Hy5Yt2bJlC3l5eZjNZqKioujUqROZmZnAtUb99ttvY2dnh4+PD9Om\nTSM7O5uJEyeSlZVFmzZtCo61Y8cO3nrrLezs7PDz82Pq1Km3+W9ARERK0+bNmwkMDKRt27YF2yZM\nmEBRDzRt2bIlcXFxJT5P586dWbNmDaNGjSIzM5OTJ0/i5+cHQE5ODi+++CLx8fFkZ2fzxBNP0Llz\nZ7766ivmzJmDj48PTk5OBAYGkpeXx5QpU4iPjyc3N5cnnniC8PDwYs9fYRr58ePHCQkJKbTNyckJ\nBweHP+ybmZlJdHQ0zZs3L9E57O3tadmyJdHR0QB8//33dOvWreD3//znP3n77bdZuHAhNWvW5Ouv\nv+arr74iMDCQRYsWFZrv1Vdf5f3332f+/Pl4enry7bfflmgWEREpW8ePHycoKKjQNpPJhNlsvu7+\nP/zwQ4lzBaB3795ERUUBsHHjRjp27Fjwu6ioKBwcHFi4cCGzZs1i6tSpWCwW3n77bebNm8cHH3xQ\n8MfD119/jbe3NwsWLOC9997jX//6102dv8I0csMwyMvLu+E+f/rTnwr+Bxg2bBhNmjQpCOXrHe96\n+vbtyzfffIOXlxc+Pj64uLgAcPnyZQzDKPgrKiwsjJ9//pn8/HzatWsHUHC14MKFC8TFxTFx4kQA\n0tPTcXd3x8fHp4TvWkREyorJZCI3N7fg58cee4y0tDQSEhJYtWoVcO0yeExMDBaLBW9vbyZPnlzk\n8YrKFX9/f3Jycjh79iyrV6/mscceY9euXQDExMQUrK37+Pjg4ODApUuXqFatGp6engC0bt0agN27\nd7Nz586C12ZlZRXcJ3YjFSbIGzZsyGeffVZoW3Z2NidPnqRJkybAtTXyatWqFdrHw8PjD+vcly5d\n+sNfYb8JDw/nlVdewdvbmz59+hRsNwyj0OWWnJycgm2/Xd7Pz88HrjX7WrVqsWDBgkLHLuqPChER\nKX+BgYHMnz+/4OcPPvgAuHbz9G//f96/f3/+/ve//+G17u7uhbLl0qVLeHt7F3muPn36sGLFCk6c\nOPGHq8u/z5bs7GwMwyi0bPzb7+3t7Xn00Ue5++67S/I2K86l9U6dOnHmzBk2bNgAXAvN119/ndWr\nV9/wdfXr1ychIaHg0sSlS5eIjo4u+Avnfzk4ONCuXTuWL19e6A70mjVrYhgGZ8+eBWD79u2EhobS\noEEDYmJigP8P6po1awIU3N24YMECDh48eKtvXUREykCHDh1ISEgoyBWA/fv3c/Xq1SIvr/8mPDy8\noLUDLF26lK5duxa5f58+fZg/f/4f9mnevHlBdpw7dw6TyYSbmxupqamkpKSQk5NT0MBbtmzJ999/\nD8DFixd56623bup9VphGbjKZiIyM5MUXX+Tdd9/FwcGBjh078vjjj9/wdfb29rzxxhtMmTIFi8WC\nxWLhhRdewMvLq8jX9O3bl0uXLuHq6lpo+9SpU3nqqaews7MjICCAAQMGkJ6ezl/+8hfGjBlT6Ga3\n1157jeeff76gnT/wwAPs3r379v4liIhIqTEMgzlz5vDKK6/w3nvvYW9vj4uLCx988AFOTk43fO0D\nDzzAm2++yfDhwzGbzTRq1Ijnn3++yP0DAgKoU6dOoSu9AAMGDGD79u1ERESQk5PDK6+8gslk4vHH\nH2f06NH4+/sTGBgIQL9+/di2bRvDhw8nLy+v2PwreJ/6PnIRERHbVWEurYuIiEjJKchFRERsmIJc\nRETEhinIRUREbJiCXERExIYpyEVswG/f3rd48eJC23fs2EFQUFCJHka0dOnSYr/xLyIigi1bthTa\nNmPGjIJvHwwNDSUiIoKIiAhWrlx5829EREpdhfkcuYjcWP369fnyyy8ZMWJEwbYvv/ySBg0alMv5\nn332WeDaHxUjR478w5MNRcQ61MhFbEStWrUwDIMjR44AkJGRwc6dO2nRokXBPsuWLWPo0KFERETw\n5JNPkpaWBsBnn33G3XffzYQJEwo9uOjgwYM89NBDREREMHz4cA4cOFDiuVJTU+nYsSNXr14Frj2C\nslOnTly+fJmmTZvy7rvvEhERweDBgwu+U7k0zisi1yjIRWzIfffdx/LlywFYu3YtXbt2LXhm89mz\nZ5k1axbz5s1jwYIF+Pn5MW/ePFJTU3nnnXdYsGABc+bMITk5ueB4zzzzDC+//DILFizgpZde4oUX\nXijxTK6urnTr1o21a9cC8NNPP9GhQwfc3NzIy8sjMDCQBQsWMGLECN55551SO6+IXKMgF7Eh/fr1\nY82aNeTm5rJixQruvffegt8dOHCAZs2aUb16deDat/Xt27ePuLg4/P39cXd3Byj4JqaLFy9y4sQJ\nJk+eTEREBK+99hppaWkFXyZREsOHD2fFihUArFmzhqFDhxb8rnPnzsC1b3g6evRoqZ5XRLRGLmJT\nPDw8aNq0KcuWLSMpKemG351ssVgKvsHv91+/+FtgOjg4YG9vXypr3S1btiQ1NZXjx49z5MgROnTo\nUGiO3xiGUarnFRE1chGbc9999/H2228zYMCAQttDQ0PZv39/wbr4li1baNmyJXXr1uX06dOkpKRg\nsVjYunUrcO2SeJ06dfjxxx8BOHHiBO++++4tzzVs2DAmT55M7969C/3hsG3bNgB27txJUFBQqZ9X\npKpTIxexMT169ODFF18sdFkdwNfXl0mTJvHQQw/h4OCAr68vf/vb33BxceHRRx9l1KhR+Pv74+/v\nT2ZmJgDTp0/n1Vdf5Rc0dMoAAACRSURBVOOPPyY3N7fYj6XdyL333su0adP4z3/+U2j7gQMHWLx4\nMVeuXGH69Omlfl6Rqk7ffiYipWLNmjWsX7+eN998s2BbUFAQ+/fvx85OnUGkrOi/LhG5bRMnTuTi\nxYsFd6X/X7t2QAIAAIAwrH9rWwiHrcQRBH4scgAIc3YDgDAhB4AwIQeAMCEHgDAhB4CwAV0oBg/9\n7QrmAAAAAElFTkSuQmCC\n", "text/plain": [ "" ] }, "metadata": { "tags": [] } } ] }, { "metadata": { "id": "ZrV3b8zWGgoX", "colab_type": "text" }, "cell_type": "markdown", "source": [ "# 5. How to select and work on GPU(s) if you have multiple of them?" ] }, { "metadata": { "id": "g4B86rMUGbHI", "colab_type": "code", "outputId": "ccb3379d-0cd3-4753-e7b2-d1c225388fc9", "colab": { "base_uri": "https://localhost:8080/", "height": 35 } }, "cell_type": "code", "source": [ "# Sadly we only have one GPU here, so we will look into the dynamics with that GPU only.\n", "cuda0 = torch.device('cuda:0')\n", "\n", "x = torch.Tensor([1., 2.]).to(cuda0)\n", "x" ], "execution_count": 0, "outputs": [ { "output_type": "execute_result", "data": { "text/plain": [ "tensor([1., 2.], device='cuda:0')" ] }, "metadata": { "tags": [] }, "execution_count": 56 } ] }, { "metadata": { "id": "uvNDslc6IvUC", "colab_type": "code", "colab": {} }, "cell_type": "code", "source": [ "# To change the default Device:\n", "torch.cuda.set_device(0) # '0' is the Id of device" ], "execution_count": 0, "outputs": [] }, { "metadata": { "id": "bmsLfUk3JegG", "colab_type": "code", "outputId": "b5744a38-5744-4a8c-9401-7a0e8fd561a3", "colab": { "base_uri": "https://localhost:8080/", "height": 35 } }, "cell_type": "code", "source": [ "y = torch.Tensor([3., 4.]).to(cuda0)\n", "z = x + y\n", "z # This tensor is stored on the same device as x and y" ], "execution_count": 0, "outputs": [ { "output_type": "execute_result", "data": { "text/plain": [ "tensor([4., 6.], device='cuda:0')" ] }, "metadata": { "tags": [] }, "execution_count": 58 } ] }, { "metadata": { "id": "KGE6gB_CJ4Oi", "colab_type": "text" }, "cell_type": "markdown", "source": [ "# 6. Data Parallelism?" ] }, { "metadata": { "id": "cyVerBqXJxwB", "colab_type": "code", "colab": {} }, "cell_type": "code", "source": [ "# Again we only have one GPU, so we will only use that to see into Dynamics\n", "module = nn.Sequential(\n", " nn.Linear(20, 20),\n", " nn.ReLU(),\n", " nn.Linear(20, 4),\n", " nn.Softmax()\n", ").cuda()\n", "\n", "inp = torch.ones((30, 20)).cuda()" ], "execution_count": 0, "outputs": [] }, { "metadata": { "id": "Ik8h9tc3LTKQ", "colab_type": "code", "outputId": "575bf4f1-3933-4d89-8c1c-0837ccd9ba44", "colab": { "base_uri": "https://localhost:8080/", "height": 52 } }, "cell_type": "code", "source": [ "# Replicate module to devices in device_ids\n", "replicas = nn.parallel.replicate(module, [0])\n", "\n", "# Distribute input to devices in device_ids\n", "inputs = nn.parallel.scatter(inp, [0])\n", "\n", "# Apply the models to corresponding inputs\n", "outputs = nn.parallel.parallel_apply(replicas, inputs)\n", "\n", "# Gather result from all devices to output_device\n", "result = nn.parallel.gather(outputs, 0)" ], "execution_count": 0, "outputs": [ { "output_type": "stream", "text": [ "/usr/local/lib/python3.6/dist-packages/torch/nn/modules/container.py:91: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n", " input = module(input)\n" ], "name": "stderr" } ] }, { "metadata": { "id": "9hosvC37Lp1i", "colab_type": "code", "outputId": "1cb95020-60a9-484f-da99-13bd912cd37f", "colab": { "base_uri": "https://localhost:8080/", "height": 35 } }, "cell_type": "code", "source": [ "result.shape # Output from module" ], "execution_count": 0, "outputs": [ { "output_type": "execute_result", "data": { "text/plain": [ "torch.Size([30, 4])" ] }, "metadata": { "tags": [] }, "execution_count": 63 } ] }, { "metadata": { "id": "vR4KkgxCLtWC", "colab_type": "code", "outputId": "50f9bfb4-1829-4634-fa67-d8626269ea27", "colab": { "base_uri": "https://localhost:8080/", "height": 70 } }, "cell_type": "code", "source": [ "# Or we could have done:\n", "model = nn.DataParallel(model, device_ids=[0])\n", "result = model(inp)\n", "result.shape" ], "execution_count": 0, "outputs": [ { "output_type": "stream", "text": [ "/usr/local/lib/python3.6/dist-packages/torch/nn/modules/container.py:91: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n", " input = module(input)\n" ], "name": "stderr" }, { "output_type": "execute_result", "data": { "text/plain": [ "torch.Size([30, 4])" ] }, "metadata": { "tags": [] }, "execution_count": 64 } ] } ] }