{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "# GPUs\n",
    "\n",
    "Check your CUDA driver and device. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-03T22:10:58.775829Z",
     "start_time": "2019-07-03T22:10:58.421457Z"
    },
    "attributes": {
     "classes": [],
     "id": "",
     "n": "1"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wed Jul  3 22:10:58 2019       \n",
      "+-----------------------------------------------------------------------------+\n",
      "| NVIDIA-SMI 418.67       Driver Version: 418.67       CUDA Version: 10.1     |\n",
      "|-------------------------------+----------------------+----------------------+\n",
      "| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |\n",
      "| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |\n",
      "|===============================+======================+======================|\n",
      "|   0  Tesla V100-SXM2...  Off  | 00000000:00:1B.0 Off |                    0 |\n",
      "| N/A   70C    P0   228W / 300W |   7684MiB / 16130MiB |     78%      Default |\n",
      "+-------------------------------+----------------------+----------------------+\n",
      "|   1  Tesla V100-SXM2...  Off  | 00000000:00:1C.0 Off |                    0 |\n",
      "| N/A   44C    P0    38W / 300W |     11MiB / 16130MiB |      0%      Default |\n",
      "+-------------------------------+----------------------+----------------------+\n",
      "|   2  Tesla V100-SXM2...  Off  | 00000000:00:1D.0 Off |                    0 |\n",
      "| N/A   43C    P0    59W / 300W |    978MiB / 16130MiB |     14%      Default |\n",
      "+-------------------------------+----------------------+----------------------+\n",
      "|   3  Tesla V100-SXM2...  Off  | 00000000:00:1E.0 Off |                    0 |\n",
      "| N/A   40C    P0    40W / 300W |     11MiB / 16130MiB |      0%      Default |\n",
      "+-------------------------------+----------------------+----------------------+\n",
      "                                                                               \n",
      "+-----------------------------------------------------------------------------+\n",
      "| Processes:                                                       GPU Memory |\n",
      "|  GPU       PID   Type   Process name                             Usage      |\n",
      "|=============================================================================|\n",
      "|    0    118587      C   ...iconda3/envs/d2l-en-numpy2-0/bin/python  7673MiB |\n",
      "|    2    119109      C   ...iconda3/envs/d2l-en-numpy2-1/bin/python   967MiB |\n",
      "+-----------------------------------------------------------------------------+\n"
     ]
    }
   ],
   "source": [
    "!nvidia-smi"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "Number of available GPUs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-03T22:11:00.147240Z",
     "start_time": "2019-07-03T22:10:58.778752Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from mxnet import np, npx\n",
    "from mxnet.gluon import nn\n",
    "npx.set_np()\n",
    "\n",
    "npx.num_gpus()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "Computation devices"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-03T22:11:00.157196Z",
     "start_time": "2019-07-03T22:11:00.149532Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cpu(0) gpu(0) gpu(1)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(gpu(0), cpu(0), [gpu(0), gpu(1)])"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(npx.cpu(), npx.gpu(), npx.gpu(1))\n",
    "\n",
    "def try_gpu(i=0):\n",
    "    return npx.gpu(i) if npx.num_gpus() >= i + 1 else npx.cpu()\n",
    "\n",
    "def try_all_gpus():\n",
    "    ctxes = [npx.gpu(i) for i in range(npx.num_gpus())]\n",
    "    return ctxes if ctxes else [npx.cpu()]\n",
    "\n",
    "try_gpu(), try_gpu(3), try_all_gpus()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "Create ndarrays on the 1st GPU"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-03T22:11:04.523547Z",
     "start_time": "2019-07-03T22:11:00.159618Z"
    },
    "attributes": {
     "classes": [],
     "id": "",
     "n": "5"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "gpu(0)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([[1., 1., 1.],\n",
       "       [1., 1., 1.]], ctx=gpu(0))"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x = np.ones((2, 3), ctx=try_gpu())\n",
    "print(x.context)\n",
    "x"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "Create on the 2nd GPU"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-03T22:11:08.769323Z",
     "start_time": "2019-07-03T22:11:04.525346Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.59119   , 0.313164  , 0.76352036],\n",
       "       [0.9731786 , 0.35454726, 0.11677533]], ctx=gpu(1))"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y = np.random.uniform(size=(2, 3), ctx=try_gpu(1))\n",
    "y"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "Copying between devices"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-03T22:11:08.779134Z",
     "start_time": "2019-07-03T22:11:08.770982Z"
    },
    "attributes": {
     "classes": [],
     "id": "",
     "n": "7"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[1. 1. 1.]\n",
      " [1. 1. 1.]] @gpu(0)\n",
      "[[1. 1. 1.]\n",
      " [1. 1. 1.]] @gpu(1)\n"
     ]
    }
   ],
   "source": [
    "z = x.copyto(try_gpu(1))\n",
    "print(x)\n",
    "print(z)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "The inputs of an operator must be on the same device, then the computation will run on that device."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-03T22:11:08.786457Z",
     "start_time": "2019-07-03T22:11:08.781557Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[1.59119  , 1.313164 , 1.7635204],\n",
       "       [1.9731786, 1.3545473, 1.1167753]], ctx=gpu(1))"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y + z"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "Initialize parameters on the first GPU."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-03T22:11:08.795260Z",
     "start_time": "2019-07-03T22:11:08.789855Z"
    },
    "attributes": {
     "classes": [],
     "id": "",
     "n": "12"
    }
   },
   "outputs": [],
   "source": [
    "net = nn.Sequential()\n",
    "net.add(nn.Dense(1))\n",
    "net.initialize(ctx=try_gpu())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "When the input is an ndarray on the GPU, Gluon will calculate the result on the same GPU."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-03T22:11:08.818131Z",
     "start_time": "2019-07-03T22:11:08.797675Z"
    },
    "attributes": {
     "classes": [],
     "id": "",
     "n": "13"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.04995865],\n",
       "       [0.04995865]], ctx=gpu(0))"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "net(x)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "Let us confirm that the model parameters are stored on the same GPU."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-03T22:11:08.825572Z",
     "start_time": "2019-07-03T22:11:08.820386Z"
    },
    "attributes": {
     "classes": [],
     "id": "",
     "n": "14"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.0068339 , 0.01299825, 0.0301265 ]], ctx=gpu(0))"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "net[0].weight.data()"
   ]
  }
 ],
 "metadata": {
  "celltoolbar": "Slideshow",
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}