{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "# GPUs\n",
    "\n",
    "查看 CUDA 和驱动。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-03T22:10:58.775829Z",
     "start_time": "2019-07-03T22:10:58.421457Z"
    },
    "attributes": {
     "classes": [],
     "id": "",
     "n": "1"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wed Sep  4 11:20:44 2019       \r\n",
      "+-----------------------------------------------------------------------------+\r\n",
      "| NVIDIA-SMI 418.67       Driver Version: 418.67       CUDA Version: 10.1     |\r\n",
      "|-------------------------------+----------------------+----------------------+\r\n",
      "| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |\r\n",
      "| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |\r\n",
      "|===============================+======================+======================|\r\n",
      "|   0  Tesla V100-SXM2...  Off  | 00000000:00:1B.0 Off |                    0 |\r\n",
      "| N/A   58C    P0    58W / 300W |      0MiB / 16130MiB |      0%      Default |\r\n",
      "+-------------------------------+----------------------+----------------------+\r\n",
      "|   1  Tesla V100-SXM2...  Off  | 00000000:00:1C.0 Off |                    0 |\r\n",
      "| N/A   52C    P0    58W / 300W |    322MiB / 16130MiB |      0%      Default |\r\n",
      "+-------------------------------+----------------------+----------------------+\r\n",
      "|   2  Tesla V100-SXM2...  Off  | 00000000:00:1D.0 Off |                    0 |\r\n",
      "| N/A   52C    P0    61W / 300W |      0MiB / 16130MiB |      0%      Default |\r\n",
      "+-------------------------------+----------------------+----------------------+\r\n",
      "|   3  Tesla V100-SXM2...  Off  | 00000000:00:1E.0 Off |                    0 |\r\n",
      "| N/A   51C    P0    43W / 300W |     11MiB / 16130MiB |      0%      Default |\r\n",
      "+-------------------------------+----------------------+----------------------+\r\n",
      "                                                                               \r\n",
      "+-----------------------------------------------------------------------------+\r\n",
      "| Processes:                                                       GPU Memory |\r\n",
      "|  GPU       PID   Type   Process name                             Usage      |\r\n",
      "|=============================================================================|\r\n",
      "|    1     66377      C   /home/ubuntu/miniconda3/bin/python           311MiB |\r\n",
      "+-----------------------------------------------------------------------------+\r\n"
     ]
    }
   ],
   "source": [
    "!nvidia-smi"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "查看有多少 GPU 可以用。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-03T22:11:00.147240Z",
     "start_time": "2019-07-03T22:10:58.778752Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from mxnet import np, npx\n",
    "from mxnet.gluon import nn\n",
    "npx.set_np()\n",
    "\n",
    "npx.num_gpus()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "使用不同的设备。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-03T22:11:00.157196Z",
     "start_time": "2019-07-03T22:11:00.149532Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cpu(0) gpu(0) gpu(1)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(gpu(0), cpu(0), [gpu(0), gpu(1)])"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(npx.cpu(), npx.gpu(), npx.gpu(1))\n",
    "\n",
    "def try_gpu(i=0):\n",
    "    return npx.gpu(i) if npx.num_gpus() >= i + 1 else npx.cpu()\n",
    "\n",
    "def try_all_gpus():\n",
    "    ctxes = [npx.gpu(i) for i in range(npx.num_gpus())]\n",
    "    return ctxes if ctxes else [npx.cpu()]\n",
    "\n",
    "try_gpu(), try_gpu(3), try_all_gpus()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "在第一个 GPU 上创建数据。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-03T22:11:04.523547Z",
     "start_time": "2019-07-03T22:11:00.159618Z"
    },
    "attributes": {
     "classes": [],
     "id": "",
     "n": "5"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "gpu(0)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([[1., 1., 1.],\n",
       "       [1., 1., 1.]], ctx=gpu(0))"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x = np.ones((2, 3), ctx=try_gpu())\n",
    "print(x.context)\n",
    "x"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "在第二个 GPU 上创建数据。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-03T22:11:08.769323Z",
     "start_time": "2019-07-03T22:11:04.525346Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.67478997, 0.07540122, 0.9956977 ],\n",
       "       [0.09488854, 0.415456  , 0.11231736]], ctx=gpu(1))"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y = np.random.uniform(size=(2, 3), ctx=try_gpu(1))\n",
    "y"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "在不同的设备间复制数据。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-03T22:11:08.779134Z",
     "start_time": "2019-07-03T22:11:08.770982Z"
    },
    "attributes": {
     "classes": [],
     "id": "",
     "n": "7"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[1. 1. 1.]\n",
      " [1. 1. 1.]] @gpu(0)\n",
      "[[1. 1. 1.]\n",
      " [1. 1. 1.]] @gpu(1)\n"
     ]
    }
   ],
   "source": [
    "z = x.copyto(try_gpu(1))\n",
    "print(x)\n",
    "print(z)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "一个操作子的输入必须在同一个设备上，计算也将会在那个设备上。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-03T22:11:08.786457Z",
     "start_time": "2019-07-03T22:11:08.781557Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[1.6747899, 1.0754012, 1.9956977],\n",
       "       [1.0948886, 1.415456 , 1.1123173]], ctx=gpu(1))"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y + z"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "在第一个 GPU 上初始化模型参数。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-03T22:11:08.795260Z",
     "start_time": "2019-07-03T22:11:08.789855Z"
    },
    "attributes": {
     "classes": [],
     "id": "",
     "n": "12"
    }
   },
   "outputs": [],
   "source": [
    "net = nn.Sequential()\n",
    "net.add(nn.Dense(1))\n",
    "net.initialize(ctx=try_gpu())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "当输入数据也在 GPU 上时，Gluon 会在那个 GPU 上计算结果。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-03T22:11:08.818131Z",
     "start_time": "2019-07-03T22:11:08.797675Z"
    },
    "attributes": {
     "classes": [],
     "id": "",
     "n": "13"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.04995865],\n",
       "       [0.04995865]], ctx=gpu(0))"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "net(x)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "确认模型参数和结果都在同一个 GPU 上。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-03T22:11:08.825572Z",
     "start_time": "2019-07-03T22:11:08.820386Z"
    },
    "attributes": {
     "classes": [],
     "id": "",
     "n": "14"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.0068339 , 0.01299825, 0.0301265 ]], ctx=gpu(0))"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "net[0].weight.data()"
   ]
  }
 ],
 "metadata": {
  "celltoolbar": "Slideshow",
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}