{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## CIFAR 10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "%reload_ext autoreload\n",
    "%autoreload 2\n",
    "\n",
    "from fastai.conv_learner import *\n",
    "PATH = 'data/cifar/'\n",
    "os.makedirs(PATH, exist_ok=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "You can get the data via:\n",
    "\n",
    "    wget http://pjreddie.com/media/files/cifar.tgz"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "classes = ('plane', 'car', 'bird', 'cat', 'deer', 'dog', 'frog', 'horse', 'ship', 'truck')\n",
    "stats = (np.array([ 0.4914 ,  0.48216,  0.44653]), np.array([ 0.24703,  0.24349,  0.26159]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def to_label_subdirs(path, subdirs, classes, labelfn):\n",
    "    for sd in subdirs:\n",
    "        for rf in os.listdir(os.path.join(path, sd)):\n",
    "            af = os.path.join(path, sd, rf)\n",
    "            if not os.path.isfile(af):\n",
    "                continue\n",
    "            lb = labelfn(rf)\n",
    "            if not lb:\n",
    "                continue\n",
    "            os.renames(af, os.path.join(path, sd, lb, rf))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "to_label_subdirs(PATH, 'train test'.split(), classes, lambda f: f[f.find('_') + 1 : f.find('.')])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_data(sz,bs):\n",
    "    tfms = tfms_from_stats(stats, sz, aug_tfms=[RandomFlip()], pad=sz // 8)\n",
    "    return ImageClassifierData.from_paths(PATH, val_name='test', tfms=tfms, bs=bs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "bs=256"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Look at data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = get_data(32, 4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "x, y = next(iter(data.trn_dl))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.image.AxesImage at 0x7fafdc107828>"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAP8AAAD8CAYAAAC4nHJkAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4xLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvAOZPmwAAHZpJREFUeJztnVuMXNd1pv91TlV1Vd9INq8tSiIlRZGlcWxZ0yM4cSB44plAYwSQDUwC+8HQgxEGgxiIgeRB8ABjDzAPzmBsw08O6EiIEji+TGTDwsAYx9A4o/EAkS3JMkWZkkXTlESR7CYpkn1hdV3OWXmoEtJq7X93sZtdTWX/H0Cw+qza56zadVadU/uvtZa5O4QQ6ZFttQNCiK1BwS9Eoij4hUgUBb8QiaLgFyJRFPxCJIqCX4hEUfALkSgKfiESpbKRwWZ2P4AvA8gB/KW7fz72/KySeaWWh21ZeDsAdLvd4HYv1/frRMuM2ioVPiVlWQS3F0V4e2/MOn9BGRmW53yuyrIM7269fvCpitvYEOODYrtb7y9RK5XwXFUj7zM73wCg4CbEXoEj/L70CNtq1ZHIocLH6rQ66Ha7A70ztt5JNbMcwC8A/HsApwD8BMDH3f3nbExttOq77pgK2sYbE/RYF85dCG5vN9uDO7zSj0aN2nbu3klti82F4PbL85fomPYVfrZ45ETygr8v27dto7bm4jLxg8+VRz5pynrk5jDjJ3RGPmBHajzoLHIqdtr8AzY2bs+u7cHt03t30THnZs9T2/zFyPvpPFi7WZPaCoRtN+87SMfkWfgc/uWLJ9Bcag4U/Bu57b8XwHF3P+HubQDfAPDABvYnhBgiGwn+/QBeW/H3qf42IcQ7gI185w/dWrztBszMDgE4BAB5VeuLQlwvbCQaTwG4acXfNwI4vfpJ7n7Y3WfcfSarKPiFuF7YSDT+BMDtZnaLmdUAfAzA49fGLSHEZrPu235375rZpwB8Hz2p7xF3f2GNUQDCq7at1hIdNT7RCG4fbE3z7TTGw/tbyw/me57H5Cu+FF1E5LeYJLa0uMjHOZEBI3OVG5cOa12+gl2UHWqrj4avK9blq/bIIvsb58OImtfbZYXIsxGJYGp6D7Xlda6anH39MrXVq1xdYNN/2x2/QccwLfjUr952803ZkM7v7t8D8L2N7EMIsTXoS7gQiaLgFyJRFPxCJIqCX4hEUfALkSgbWu2/agywjEgsxiWgZjOcrFKQLLu1aC7zcY3ROrUx3ytVrjXFkmY8kuhVqXJtLs8jbxt5aVnGP+ereZXaRitj1NZphd8XAMiIpFdE3uex7dyPSN4XugWXCDvd8PxfJglQAPD+3/xNamteaVHb5YUfU9t4PZKMdSU8J8eO/4qOaXeuBLdfWeYJRKvRlV+IRFHwC5EoCn4hEkXBL0SiKPiFSJThrvYDYMvRnQ4vj9Ruh1dmvbO+EmRFJMsltspulbCtiKw2xwSJSNlCFAWXAipMMQFYOTiUkf15ZH+tNk90qo7w1fmSzHGrzVfLJyqT1MZq1gGAVfj8Z92wWnH5Ep+Pej1cag4Amss8cWZklCsIzaVT1ObdcNbS/AL3Ma+QWo0xCWkVuvILkSgKfiESRcEvRKIo+IVIFAW/EImi4BciUYYq9bk7ut2wLJNHXGEtr9bdbWhwNeQtMN9bLS41RRQq5JGWUUW0ZRS3VbKw/GaRxJ6SZQMB6Fb5sSwiixYFeW35KB3j2Q5qm1/ikuPoKE8+qlm4s01jlCfazJ3mtfhOzR6ntvpIuKMTAHikz9dyK5yk0xjl85GTbkmXruJyriu/EImi4BciURT8QiSKgl+IRFHwC5EoCn4hEmVDUp+ZnQSwgF6qXtfdZ2LPL0tHsxmWxeokYw4AcpL+1om0i4pRyfjL7na47LVMpL5qlWe3WY3LkTH38zyS8ldefZ+yPOef850ub0GV825dKJzXi8stLOllVb7D5SbP+KtG6iQ2Knz+0Q7Lb41IBuHShXlq27ONj9s/tZPa3nj9PLXNd8Kve7zBswQvnL8YNkSyN1dzLXT+f+vu/JUJIa5LdNsvRKJsNPgdwN+b2TNmduhaOCSEGA4bve3/gLufNrM9AH5gZi+6+5Mrn9D/UDgEALYFdYOEEGE2dOV399P9/+cAfAfAvYHnHHb3GXefsUgfeyHEcFl38JvZmJlNvPkYwO8COHqtHBNCbC4buRHfC+A71ktbqwD4W3f/39ERDpRF+Orf7ESKMJLUOLavtSgjagiTIgHAs/DxRhqRQpbLkVZSkcKfsfZaiGQzdogc6TypjEqpPRuXPo1VCwWAIixfZUVEVlwOZ7cBQN34fDSKN6ht73R43K4dPBOw2uAFPG84wOW8fXu28322LlFbdz7s4/lIAc+zr58Nbn/s/Ot0zGrWHfzufgLAe9c7XgixtUjqEyJRFPxCJIqCX4hEUfALkSgKfiESZbgFPMGLcVYirnTbYckjUscyinG1CXkke6wowwdcvMyzr7wbyeojcwEARFXs7TMyrizCtjLix/gEL6qZeyT1sOAyIJM4x6rcjxt3cMn0xn08G/Bdd/FMu1tvD9sM03TMT5/j7+erL71KbZMZPyEPTkX6Q46GX/dopLDqwTt3Bbd//0ezdMxqdOUXIlEU/EIkioJfiERR8AuRKAp+IRJluKv9paN1hayIRlbZO63wqidb2V7TD7I/AKiCr8q2OmHfY62wLNKvK6/wcdVIsk0nsspeqYXf0iKS2eNdvr/RSL3DqvH5331DeHX+zlt58stvvIuv2t/16+PUNjrB22RVJsP1+M6e46vik6O7uW2MqwSX545Q24lZ3m6sJAk8rTZXHVCG6ye2moNLYLryC5EoCn4hEkXBL0SiKPiFSBQFvxCJouAXIlGGXkw7K8OfN0U7IkWVRIqK1LKLEUuMiZSYQ+bkszIi5xWRisW1iFQ50ua2bqQIYadGDBkfMw5+rLunec26gzfzce9+b1iqPHATb8k1UT9NbfVI3fc80r7MF8PNpEYibcO6tX3Utn38Rmpbvvgytf3/p05QW9ltBLePOZdgMxIvy8uDx4Su/EIkioJfiERR8AuRKAp+IRJFwS9Eoij4hUiUNaU+M3sEwO8BmHP3d/e3TQH4JoCDAE4C+AN3v7jm0RwAkdnKWOsnIumtU+mDRQaWkYw5kIy/0rnvHskS9E4k4y/ihuXcfzcyMPIxP97gfnzovh3UdscdPPNwbEc4084tnGUHALEkzabz9lqVIiyVAcAkmY5uRNJFhWcXlh3+Xu/bwWXMu+/kdRLPXgzrs9U2r5/YJdOY57EX9lYGufL/FYD7V217CMAT7n47gCf6fwsh3kGsGfzu/iSA1Z0QHwDwaP/xowA+co39EkJsMuv9zr/X3c8AQP//PdfOJSHEMNj0n/ea2SEAhwBoeVGI64j1huOsmU0DQP//OfZEdz/s7jPuPhNZ+xJCDJn1Bv/jAB7sP34QwHevjTtCiGExiNT3dQAfBLDLzE4B+CyAzwP4lpl9EsCrAH5/4CMakUoiMg9X5tZ3K+ERqc8iRSlBbLHWWqVzYxnJzutGxkUVTmaM+Gh1ritu27d6rfefubzMZaXLszcEt4+Mv4uOaZYsJRG4sLBIbe2ly9R2724iA+a8IGjW5n7Uq7xY6PQ4vQHGe/41D7WF7kRw+3LBpdTuUtjH//PKSTpmNWsGv7t/nJg+NPBRhBDXHVqCEyJRFPxCJIqCX4hEUfALkSgKfiESZagFPM2ACiloWRYRSYxl9RXr++yynEtsWaQYZ0ay6SzmRqTwZMYy8ACUEckx4iLAdhnJEmxHZMULF3k22t6pO6jtzNlwT7tfvMLn/tyVcP85AJi8cYraahUuOe4rwkVBJw9yqY/1wQOAmvP+eRMZf8/GIvOfVcKyXSUyqLYjnG1ZqZ3iB1p93IGfKYT4F4WCX4hEUfALkSgKfiESRcEvRKIo+IVIlKH36mN98jxWvZHJgJEebevFWdYhAKaIxbL6rOD7i3mfj1SprR5518pOuOhjN1Zk1HgW2+z5vdRm3QN83Fy44OZLL52kY5YznjGXb69TW6fC5a2sOhvcbsZ79WUZf82VjPtRrfFxyLdT08+OXgob9vCYmJ4O+1FcRUzoyi9Eoij4hUgUBb8QiaLgFyJRFPxCJMpwE3scyP3qD+mkMJ1FM2oifkSq4GWRNXjWJSuSz4Hd2/gqb9HlK/A7dvBxvLIb8PorrwW3N9uRZKZY0o/xo3Ui9eyqE+GEmn/17vDqOwA0xnjSzM238QSj0Spva3VwZ9j/C85975a87dYytlFbMX6Q2n7yMq9B+Oj3zwa3e2OJjrnvt8M1EhevDN7DTld+IRJFwS9Eoij4hUgUBb8QiaLgFyJRFPxCJMog7boeAfB7AObc/d39bZ8D8IcAzvWf9hl3/97ahzNkCCesVCr8c6hDCtB5RKKKepHzY1UrXNoqSXKMd7p0TC1Sw69b47JMEUvEaXIparwT3udElb+uakRynNrDJzmrv0RtN+8K18G7bx9PjBmLvC9146+51uH1+Boe3ufFDmnjBaAowklJALAcSQi6QM5tAHj+DS7bvbxMxr0RkTefD8/HUvPaSn1/BeD+wPYvufvd/X8DBL4Q4npizeB39ycB8G6NQoh3JBv5zv8pMztiZo+YWbiOsBDiumW9wf8VALcBuBvAGQBfYE80s0Nm9rSZPR3pjC2EGDLrCn53n3X3wt1LAF8FcG/kuYfdfcbdZ6LNJoQQQ2VdwW9mK9uxfBTA0WvjjhBiWAwi9X0dwAcB7DKzUwA+C+CDZnY3AAdwEsAfDXa4DBnC9eLKbkS3I0pUts4vLREVDWU31q4r7HueR+rBEakJAMa3cYmqU/IWVNUOn6spMr9Fh0tllVidwfYctU1sv0Jt20bCtpGCS471Lp/HRrGT2vIWn8ci3x3eDn4sd76EdXGev+Zl8DmebZ6ntnyKxESbZxD+6rWLwe3tNpedV7Nm8Lv7xwObHx74CEKI6xL9wk+IRFHwC5EoCn4hEkXBL0SiKPiFSJThtutyoEtqNOYV7kotD8tveSWi2UUouvwzryy5FNVdDv9EMa/xdldlRHkpSi7ZNRd5McvJSBHUmybDklhriWeV1bZz/0eWuB/VRe5/azH8wmcvcAnzEiLyZuTnoXnBJbaWhTPmZusTdEw30vKqzqq4AuheIW23ANyyl8t2878efs9+euplOqa5FD5WWfJipqvRlV+IRFHwC5EoCn4hEkXBL0SiKPiFSBQFvxCJMlSpzx0oaVYX18S6RALKisFljZWUBS+0aJFsr7IIT9eO7XvpmDfO/5Lamgs8Y84jxQ9Gmtx2oBbOSNtdjxS5HOeFIm/dyQtu1iK9+vJaeJ+1Bve97fPUVhivJNfxcIYbAHTaYRntShHJIq2Gi48CQI0UkwWARotLz22ShQcAlfNh2fHXbuCFRFukst7Sq3TI29CVX4hEUfALkSgKfiESRcEvRKIo+IVIlKGu9lsGVOvhFdFqNdLWitSY83XWAs8rfLW/knM/Op2wH1dafCXXI1Ncz7iyMNLgtm3b+cr9IlFTTl+6TMdkZ3jSz2+19lPbjh1cCbCRsEJTHeNJOJN8d6yMIwAgj6zAWxFWkVrNSTrmlQtcRcoi59yVc7y+34FJ7uPkZDh5Kn8f97EkNQ3PPMZrBa5GV34hEkXBL0SiKPiFSBQFvxCJouAXIlEU/EIkyiDtum4C8NcA9qGnuBx29y+b2RSAbwI4iF7Lrj9wj2RYAMhzw+RUWM/xMiLbkTpsBi55xXBEjhUxNbJwUkqsFdb45HZqq1a5tlWtcjmysWsXtY3sCCf2LJ58hY6Zv/A6tZ2Y4/LVyDbu4+hIeCJrtcj1JqLn1boNaqt0I7X/ssXg9hHw/bWMS31t4ydI5wofd/sBPm6pfja4/fwiH7NzNHwu1nk5xrcxyJW/C+BP3f1OAO8H8MdmdheAhwA84e63A3ii/7cQ4h3CmsHv7mfc/dn+4wUAxwDsB/AAgEf7T3sUwEc2y0khxLXnqr7zm9lBAO8D8BSAve5+Buh9QADYc62dE0JsHgP/vNfMxgE8BuDT7j5vkWITq8YdAnAIAPKK1heFuF4YKBrNrIpe4H/N3b/d3zxrZtN9+zSAYFkadz/s7jPuPpNlCn4hrhfWjEbrXeIfBnDM3b+4wvQ4gAf7jx8E8N1r754QYrMY5Lb/AwA+AeB5M3uuv+0zAD4P4Ftm9kkArwL4/bV2lOcVTE5MBW1lyXWeQb9iXAtimYLsziWWCZiDy3lFh2d6xfwoSJYjAJx89SqKuPVpd3j9xHNLvH3Z7iWeddaaD0tsF5/jtfju+jXe0mqixmW0qQb3o6iEx3ULnjVZzbmEOV7n18ubd01T29RopG/bMpFTSy4TjzTD7bqyq+hgt2bwu/uPALDo+9DghxJCXE/oS7gQiaLgFyJRFPxCJIqCX4hEUfALkShDLeCZZTlGG2E5JyaXMYqIPBgjX+ePjbqkGGRmfH+1SJHOfIyPW27zTLVOm8te85fCLbSKWHuqjEtb/3h0ltqa4Clk1Twscf7iGG/xNTJyM7VNVMLSIQAsTPLzYNtEOPOzXeetsKyMvC9XuB+tCi+EWpa8Bdg2corUaWs7oNEIZyVWMl6odTW68guRKAp+IRJFwS9Eoij4hUgUBb8QiaLgFyJRhi71jTXCmUr1ES6JGZHmovJVhDznEopH5MPlVrhQZyznMI9YY/LmaIMXJ+12eYbYzh3hHm6dDpcHi5Lbmi1e+PPZ47xe6/JS+HXfuv8eOubSlbDvAHDrXdxWA+9Pd+ZiuDjm6TLcHw8AvMOLe5YZn6slcn4AwCIpJAoA5uF9epPLvZfLsK0gcnQIXfmFSBQFvxCJouAXIlEU/EIkioJfiEQZ6mo/YDCEk0jKkq/As5pqlkd6a8W8iNQE7ERWvpnv1SqfxmokichLrlZkGZ+PkRGeiMOUjHYkUSg2j93lcM1FAOhUuP9vnJsPbv+tgzN0zP975v9S24svvkhtM+/ZR237b7gluL1S8Pp4tYiI1OnyuVpo8fNq2wiv5Yh2OOmn2uBJUMtEKfKrKHepK78QiaLgFyJRFPxCJIqCX4hEUfALkSgKfiESZU2pz8xuAvDXAPYBKAEcdvcvm9nnAPwhgHP9p37G3b8X31eGWi0secSSbVjqjPv6avhZpOZepcLr0pmFfYypK5UKn+IsMjAmR8bbl4Vt9RE+v13nMuDYBG+hVRnlr60odwS3Hz91ho45eYYn6Pz8jXCCDgA889wJarvttnDn+APvei8dU5vk7b98hJ9zzRE+br7Fa/hVOuF2XVmNJwpdIIp05yrU70F0/i6AP3X3Z81sAsAzZvaDvu1L7v4/Bj+cEOJ6YZBefWcAnOk/XjCzYwD2b7ZjQojN5aq+85vZQQDvA/BUf9OnzOyImT1iZuH7PCHEdcnAwW9m4wAeA/Bpd58H8BUAtwG4G707gy+QcYfM7Gkze7rT5t9hhBDDZaDgN7MqeoH/NXf/NgC4+6y7F95bdfsqgHtDY939sLvPuPtMtcar9QghhsuawW+9peWHARxz9y+u2D694mkfBXD02rsnhNgsBlnt/wCATwB43sye62/7DICPm9ndABzASQB/tObB8hzbtoWlo5hsx+rPdbvrq+FXqUQyCKs8Y45JhGWklqBFMvdiuHPNZj0yYBkZU3Uub2ZVLvXVifQJAKM3hE+t83NczpvaF5blAKBSCcthAJBHahoePX4uuP3IiR/SMbumeb3AepW/L3fexO9sZ26boLaJPGwb283PxdlWWDLv+CU6ZjWDrPb/CGHxOKrpCyGub/QLPyESRcEvRKIo+IVIFAW/EImi4BciUYZawNPhKEsiy0SkLSbNxbLiYmTRdl1cmnMLy5GNBi/OmMVy/iKvuaegholJnKxQZ0wejGUyZl1+fcgiPuZ5+Hj79nIZzfbyrLjywI3UdvFCWM4DgLnz4ZZiC5cu0DFnX5+jtiLynr3GO5vhhZ/y+f83d4Tl1Dvu4Sk0yxaeK/cXuBOr0JVfiERR8AuRKAp+IRJFwS9Eoij4hUgUBb8QiTJcqa90tNvLQVusgCeTqQpfX8YcSi7XxLLpCpq9xzMSaxnPzIr144vNR6xXX7UStrUivfoyRLLzKjyrr8z4/Dd9Kbh9pM6P1Wnza9HYzt3U1qjtpbbRibCkt7jAC4LOnnmN2i4uhs9fAGi1+HlwboEX8PyHfwxn4h3ldUkxOhXuhbiwwHtNrkZXfiESRcEvRKIo+IVIFAW/EImi4BciURT8QiTKUKU+GJe3YlJUSaW5q2hMttoRQhZJFaxVw9lvZaRlYDtS3LNCMt/6nnBLFrERiTAmsY1UeFbiaD5GbcgiEmcR9rEL/j6jMU5NZYefqhPbuI9V0k9wcoLLpeNj3I/zC7z3xOISf23dS+HsQgDoXgzLka/M8fOjnAvvrwlezHQ1uvILkSgKfiESRcEvRKIo+IVIFAW/EImy5mq/mdUBPAlgpP/8v3P3z5rZLQC+AWAKwLMAPuHukaVcAG4oSrbqzFdfiyKcrMBVgDixFf0skojDfLdI26qYi92IrdPmCRqxcRWy2p9X+FvdJbUJAWA5X6Q20r2s5wc5Xr5OganIuY8lOT8AoFENt9DKc95RPq+NUtv49kgyU5Mn/SxN8ePNXw7b7GI4eQcA5hdIW65FXptwNYNc+VsAfsfd34teO+77zez9AP4cwJfc/XYAFwF8cuCjCiG2nDWD33u8+fFf7f9zAL8D4O/62x8F8JFN8VAIsSkM9J3fzPJ+h945AD8A8EsAl9z9zV8UnALA6wwLIa47Bgp+dy/c/W4ANwK4F8CdoaeFxprZITN72syeZoU8hBDD56pW+939EoB/APB+ANvN7M3VmxsBnCZjDrv7jLvP1Gr8Z6RCiOGyZvCb2W4z295/3ADw7wAcA/BDAP+x/7QHAXx3s5wUQlx7BtFdpgE8aj09KwPwLXf/X2b2cwDfMLP/BuCnAB5ea0cOR1GEJRu2vUdYmrOY1hSFS31xP8K2WCusmNZn0VqC3I9YncGSSWyRY5WRFmWdSGJSJL+IzgmTAIF43cK4PMvHFUX4dVcjd6GWcR/rdT6Po6MRiXCcJx+NT4YTieoTPBlo7HJ4zCsnFuiY1awZ/O5+BMD7AttPoPf9XwjxDkS/8BMiURT8QiSKgl+IRFHwC5EoCn4hEsVistE1P5jZOQCv9P/cBeD80A7OkR9vRX68lXeaHwfcnfc2W8FQg/8tBzZ72t1ntuTg8kN+yA/d9guRKgp+IRJlK4P/8BYeeyXy463Ij7fyL9aPLfvOL4TYWnTbL0SibEnwm9n9ZvaSmR03s4e2woe+HyfN7Hkze87Mnh7icR8xszkzO7pi25SZ/cDMXu7/zys+bq4fnzOz1/tz8pyZfXgIftxkZj80s2Nm9oKZ/Ul/+1DnJOLHUOfEzOpm9mMz+1nfj//a336LmT3Vn49vmlm4f9yguPtQ/wHI0SsDdiuAGoCfAbhr2H70fTkJYNcWHPc+APcAOLpi238H8FD/8UMA/nyL/PgcgD8b8nxMA7in/3gCwC8A3DXsOYn4MdQ5QS/nfLz/uArgKfQK6HwLwMf62/8CwH/ayHG24sp/L4Dj7n7Ce6W+vwHggS3wY8tw9ycBvLFq8wPoFUIFhlQQlfgxdNz9jLs/23+8gF6xmP0Y8pxE/Bgq3mPTi+ZuRfDvB/Dair+3svinA/h7M3vGzA5tkQ9vstfdzwC9kxDAni305VNmdqT/tWDTv36sxMwOolc/4ils4Zys8gMY8pwMo2juVgR/qCTLVkkOH3D3ewD8BwB/bGb3bZEf1xNfAXAbej0azgD4wrAObGbjAB4D8Gl35x0rhu/H0OfEN1A0d1C2IvhPAbhpxd+0+Odm4+6n+//PAfgOtrYy0ayZTQNA//+5rXDC3Wf7J14J4KsY0pyYWRW9gPuau3+7v3nocxLyY6vmpH/sqy6aOyhbEfw/AXB7f+WyBuBjAB4fthNmNmZmE28+BvC7AI7GR20qj6NXCBXYwoKobwZbn49iCHNivYJ/DwM45u5fXGEa6pwwP4Y9J0MrmjusFcxVq5kfRm8l9ZcA/vMW+XArekrDzwC8MEw/AHwdvdvHDnp3Qp8EsBPAEwBe7v8/tUV+/A2A5wEcQS/4pofgx2+jdwt7BMBz/X8fHvacRPwY6pwAeA96RXGPoPdB819WnLM/BnAcwP8EMLKR4+gXfkIkin7hJ0SiKPiFSBQFvxCJouAXIlEU/EIkioJfiERR8AuRKAp+IRLlnwCJh9zgo31oowAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fafdd14ee80>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.imshow(data.trn_ds.denorm(x)[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.image.AxesImage at 0x7fafdd064da0>"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAP8AAAD8CAYAAAC4nHJkAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4xLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvAOZPmwAAHVRJREFUeJztnWmMnNeVnt9TW1f1RrK5tkhJpGQqska2JbmjeKLJwJlJBooxgGwgM7ARGPphjAbBGIiByQ/DAWIHyA/PILbhH4EDOhZGDhwvGduwMNAsjmJHMQLIpmXtsmQtTYqL2Fy6yV6qu7aTH1UMqPZ9Txe72dXU3PcBCHZ/p+/9Tt36Tn1V961zjrk7hBD5UdhqB4QQW4OCX4hMUfALkSkKfiEyRcEvRKYo+IXIFAW/EJmi4BciUxT8QmRKaSODzex+AF8GUATwX93989Hfj0/s8t03HkzPFZ4ofbgQDLLAFo4L3CgyP6IxgS06V2RrB9/KXGk0k8erlTIdUwgWqxP4ETHIu0r0HVVmix5XK7BF49bjR8R6ro9T09OYO3cuDKfLrDv4zawI4D8D+OcATgD4mZk96u4vsjG7bzyIP3/saNJWKPJlLRbSl1Klwv2rBI+sFtiqJe7HKLmix4IrfZSbMBSESPTEzC0vU9v06XPJ44f376FjRoKFXAr8sCAUhq9x+EfB0wr8YIE8H8x3ocN9rwcPqxHM2ehc/ctohVz3AMCesX81NdX3/Bt5hu4F8Kq7v+7uDQDfAvDABuYTQgyQjQT/fgBvXvH7id4xIcQ7gI0Ef+pzxa+9OzOzh8zsqJkdvXT+7AZOJ4S4lmwk+E8AuPGK3w8AOLX6j9z9iLtPufvU+M7dGzidEOJaspHg/xmAw2Z2yMwqAD4K4NFr45YQYrNZ926/u7fM7JMA/hZdRethd39hjVFoWXr/tRSIE0OV9E5poF6hXOC7q6VgF7UQ7Bw7MRVafL5G8PJqgagU7Q03A//33rAreTx6lW+1uB/LgSfFYNbKukXCqyc6E7MFlw5qwfPigRLggSe2jvtsqcP9qJL5CldRnGdDOr+7PwbgsY3MIYTYGvQNPyEyRcEvRKYo+IXIFAW/EJmi4BciUxT8QmSKgl+ITFHwC5EpCn4hMkXBL0SmKPiFyJQNfbf/6jGUPH3KAsuaAdAi2TFR2aTokUVJRKUgIcjIS2UnOFdQaQyVwMnoiSkHZbxOkjJe40EZr1KJn60a+BGV8SoNsIxXlFDDvEhXOuxSD57Q5eBhRXOup4yXF7gf7BLuRMUrV6E7vxCZouAXIlMU/EJkioJfiExR8AuRKQp+ITJlsFJfB/B6WvLwoGNPm7xErQTaSmeI2yx61IGtSHS7YjCmHZxqeXaB2uqzc9R2YHKC2l5/5dXk8YrzenC/cctBaiu1uKBaCGoJrqeCX9QmK5L6uPAJzBJPzgUOXgw05FYwrhk42V7HikSt3spE0Yv8W43u/EJkioJfiExR8AuRKQp+ITJFwS9Epij4hciUDUl9ZjYNYB5dRavl7lPhAAecyXNMzwNoCpMHOloksa0Exk40J5FRWqSdGACUAjny/Nwitf38x09Q231TfJlnL11KHv/xj35Cxxw6sI/ayhWel9gOdCXWpixqk8VXI5bzojy2JXJ/qwe64uIKtzWC57MdzNkOWroxIgmZ2dg1muJa6Pz/1N3TeaRCiOsWve0XIlM2GvwO4O/M7Odm9tC1cEgIMRg2+rb/Pnc/ZWZ7APzQzH7p7m/7sNp7UXgIAHZN3rTB0wkhrhUbuvO7+6ne/zMAvg/g3sTfHHH3KXefGp/YvZHTCSGuIesOfjMbMbOxyz8D+D0Az18rx4QQm8tG3vbvBfB96xYMLAH47+7+N9EAd0eDZImVAleK5DWqGMguHkiHzeBRt4ICng0yrm58zFCQrVib2EZt5y+ep7ZvfP0RaqtsS88598YrdMy7bpikttt/8x9R257tw9TG1LJWlLoX3Irmg2GVIGOu2ElPWgnONTzE54uyC82DScNqs2lKQVpfqZz20SzycNUcV+vQZdz9dQDvW+94IcTWIqlPiExR8AuRKQp+ITJFwS9Epij4hciUgRbwbHsbl1bSWWfloMdYpZDWPErGtZBKi9tKrBIngGJQ3bNDpmxU+Jh2kPG3OL/E/aiMU9uLv/qf1OaltPw2XuA5czPP/F9qO/4Wz9m64e73UtvkoZuTxys13v0vWCoQxQ4AMMtNqJPMw+UoAy84WdSfsBDIusWgoCyfj9v678gXzH8N5hBCvANR8AuRKQp+ITJFwS9Epij4hciUge72d9pt1BcvJm2tMt8OnV9Jp3UUg53X4eGd1DYUFEerBEpAydK2UovP11zk28orCzxdpTrKW3JdWOYV7RpL6d35Qo1vHS/OHOPzzdep7ZGnXqC2W95zZ/L4He+/h47Zt/8GaqtV+fPSCnJZmiTZphkkhbWiGo/B/TIqn+fNq2/X1YkSheig/nUA3fmFyBQFvxCZouAXIlMU/EJkioJfiExR8AuRKQOV+horF3Hi9b9N2sbH0okgALB08a3k8VaLJ6vsP/SPqW1kZA+1NStcmit2FpLHy6w3FQC0uKbUCjSl6Vd/SW22NEdt20rpp3Q8qO3WXuFy3mgtSD6a41Ll8Z9eSB5/7UUuD955N29DdvjO91PbUCCLdogA1wmUt6jlVdChLBzXjk5IKBb4dVUkpqiF2mp05xciUxT8QmSKgl+ITFHwC5EpCn4hMkXBL0SmrCn1mdnDAH4fwIy739k7NgHg2wAOApgG8IfuHpVSAwA0VuZx7LX/lbRVyqN0XGsl3evIgzyqC5emqW33ztupbWRkV+BHuv5goc2aUyHsT7Uwx8edeOWn1HZwOMgibKbPNzrEW2tVqyPcFlwhN/ApsYPYnjzOpb7j7bSUCgDzb05T2/itXAbcc/Ph5PF2m8uszUCz6wRZc5Gc1/L+22hdphTUtWQyYCQfr6afO/9fALh/1bFPA3jc3Q8DeLz3uxDiHcSawe/uTwBY/Y2NBwBc7hb5CIAPX2O/hBCbzHo/8+9199MA0Puff2VOCHFdsukbfmb2kJkdNbOjK0vBZ2MhxEBZb/CfMbNJAOj9P8P+0N2PuPuUu08NDa+jc4EQYlNYb/A/CuDB3s8PAvjBtXFHCDEo+pH6vgnggwB2mdkJAJ8F8HkA3zGzTwA4DuAP+jlZ21uYX05ne/lC+jgALNfT8lW5zN0/P3eG2mbPvkJtQxWuX9XraclxeIgXx+wscOnl9Os8O68xx7PpDvCOV5gcTfs/WqvRMfPL6ccFAKPbx6ht/04uix4YSd9XFuZ5JmBtlK+jNfjz+fQT3+N+3Hlf8vj+m97Nz1Xi0udKg2eSetA+rriOfl31QEI2T19XnUDCXM2awe/uHyOm3+37LEKI6w59w0+ITFHwC5EpCn4hMkXBL0SmKPiFyJTB9uprtbE4m+7VVylx/crb6YyolTaXqKJebN7mmXblIpeiaIJeI5D6Znk218IFLhvVF7kMuMCT33DPwe3J43fs5EUuR6tchmo2uf/VKpcPx4fT4961bzcdsxDIVGOjvFff06+9Sm3/568fTh6/855/Qsccuv091PbGSV5YtVLj0ufEjoPUxrgwO01tjXq6J+Pycjq+UujOL0SmKPiFyBQFvxCZouAXIlMU/EJkioJfiEwZqNQ3Uh3B1G33Jm1RfcPlZjrDrRD1n2txW6HACy2WuGoHeHq5Wk0uOU6fOEFt1Q7P3NtR5cUbG0T6BICKLyePb9/Gs/NGRnkW2/kl/ti8wLPOFufSklOpw+W80SDzrdzk59pT45fxsZl00dXXnn+CjqnXX6S20gj3f3mJ+7+y9Atqo2Pqad8BoOjp9fDgmlqN7vxCZIqCX4hMUfALkSkKfiEyRcEvRKYMdLe/UqngxpsPECvPxGm168njxeClq2T8oS3UedJMp5PeLQeA8eGdyeOzx/iu7GzzOJ+vyqWF7RWe6FQgrZoA4ObtO5LHi+UyHVOqciWgfuk8tdVK/DlbOHcseXzR+WMe3bWfn8v4LvtEhSsjVUurFfV53l3uwkmuBr1nil2/wMgYTz5auLBmN7tfY/QGrsIszqevgXIUFKvQnV+ITFHwC5EpCn4hMkXBL0SmKPiFyBQFvxCZ0k+7rocB/D6AGXe/s3fscwD+CMDZ3p99xt0fW/t0bRQ8XSOvGdTVA1FeiuAST6HI5wtzd4LEk85cuuZe+xhvJXUwaOU1Msbr2S2vcMlxLmivVRlJy3adYK2adV4U8MTMW9Tmo7xm3eFmeh1nZtISIAA0guQdn+BrtTOQTHeU0/e3xXleP/HSeb5W59/kbeXGb+M+dhppuTqi2OJyL/Oj1QjiaBX93Pn/AsD9ieNfcve7ev/6CHwhxPXEmsHv7k8A4C93Qoh3JBv5zP9JM3vWzB42s/TXyoQQ1y3rDf6vALgVwF0ATgP4AvtDM3vIzI6a2dHF+av/3COE2BzWFfzufsbd2+7eAfBVAOnyPN2/PeLuU+4+NTLGmzwIIQbLuoLfzCav+PUjAJ6/Nu4IIQZFP1LfNwF8EMAuMzsB4LMAPmhmdwFwANMA/rivszkAIgEhqOFX9LT0Uoqkvg6fcEd1G7WdPMOlkiWSvbe/wGu3tcaGqa00xMe9cIzvsVqFv4Maq6XlIW/x2m7zK7xF2dkF/lGtvXiS2naOpeW3S3M8u82DQo4l59LcyNg+anvXRFr6fOtCut0VAKDMs/Nm3uT+V4f4vXTnzaP8fITTx3hGJfOj2eBS9WrWDH53/1ji8Nf6PoMQ4rpE3/ATIlMU/EJkioJfiExR8AuRKQp+ITJloAU8m+0mzsyeTdqWGjyLrVZOS2KjQZHLlnPJo1oIinu+/Ca1nTidTi+0vTyb69YxLvGcn+Wy0cmzPJvu9v03UNsI0hl/Cys8822uzddjdyBV7lniGXorSK9JocqLUrYaPLuw0A4Kmi7zdTw0lj7fy0G25ckgy/FSEDHP/YzLhwdmJqmNceLYaWqzUjomOkFW6mp05xciUxT8QmSKgl+ITFHwC5EpCn4hMkXBL0SmDFbqazVw6kK6d127w/u+lctpmWeRyB0AsLjC5Zrxs7xQ5L3GM/5+42Da9r9P8ay4yh4uA960g2cl/tZtt1LbUlDA8+RcOguvEfTjK7V478IdK7w4aX2Bj1supJ+zHTsn6Jil2VPUVizyS9U7PPNw21BaqnzPTbzn3sqbPFtxpcXXPiq6OnOcy3aMVjBflWW0BpmRq9GdX4hMUfALkSkKfiEyRcEvRKYo+IXIlIHu9gMGRzlpabb4Dnx9ibQmKqfnAoC5cxepbeE430V9dozvlr73pnSSzm9O8iSRv3r5NWpr3HoLtd16A09kefmVaWq78Fr6fO/bzev+tZZ5fbz5En9sr14KWkNdSicm/cNDvN5ebYj76AX+XFeDJJ3KUHrce2/dT8eMlUl/OAAnzgfJWERZAIB6kCzE2DbOk8L2V9L37VcKXEFaje78QmSKgl+ITFHwC5EpCn4hMkXBL0SmKPiFyJR+2nXdCODrAPYB6AA44u5fNrMJAN8GcBDdll1/6O5cBwFQrQzhHxxMy1srK1xuaiylW03Vytz9X83y+nJvLPD6eMcv8ESW0Yl0UspN+3hNvffv4Y/rrdPT1HbTPi5t7SoHUuXZ9Jze4rLRnh08QWroJi7NYYTfO974xUzy+OE93Pd9JS5TLdd58s5ojXeIHxpOP+7icFCbkKusGJvk55oEH/jL6Vf5pITbb+Zy5Hak17Fa4bLnavq587cA/Km7vxvABwD8iZndAeDTAB5398MAHu/9LoR4h7Bm8Lv7aXd/qvfzPICXAOwH8ACAR3p/9giAD2+Wk0KIa89VfeY3s4MA7gbwJIC97n4a6L5AANhzrZ0TQmwefQe/mY0C+C6AT7l7uld1etxDZnbUzI4uLvDPe0KIwdJX8JtZGd3A/4a7f693+IyZTfbskwCSOzzufsTdp9x9amQ02EkRQgyUNYPfzAzA1wC85O5fvML0KIAHez8/COAH1949IcRm0U9W330APg7gOTN7unfsMwA+D+A7ZvYJAMcB/MFaE7m30VpOZ9vVggy97Tt2Jo8HCWeojfCadcuddMswANg9ypfkLKmdV5/jH2cO7ODtqRYuvE5t9QUuv+3cyW37bkvLjuctyOob5hlno4W0zAoAPspbQy0jPW76+Bt0zI49fO3rDX6u6A3l8DZSQ9F55p4F98TKEF/7PaNcBjx39urf9e4J6h1iIa2qF6z/bbw1g9/dfwKwaoH43b7PJIS4rtA3/ITIFAW/EJmi4BciUxT8QmSKgl+ITBloAU93R5O0O+oE7bqsnC4U2Qw6E43sCNo7FXjhyWKVZ7+dXEz7vrPEv/A4to0/rs5BLuWcHecSYXGEP/B9e3clj+9pczny/Ay3BSU6URrl8uwQUb1emeaJnweCTLtKOZD6LnJbdTzdUmx4JL1OAFCs8LVvgUuEhQK/5rYPc6l1PfN1rkLSo/NveAYhxDsSBb8QmaLgFyJTFPxCZIqCX4hMUfALkSkDlfqsUEBpOJ3d1GlwSWypme7j54HWZzUu/+y9iWf8NS7wVMHGcrqI5IGJoADmQS4pjRe5tLUSSHPtFpebdo2nc7DKBe7j+DiX7Bptfq7lDhcCb7srbXvmLd678LVz/DGPj/DrY6jEfRyaPZU8biNcZm22eSZjp8jXqr3Ci4wO165e6msG8xnS13egfv8auvMLkSkKfiEyRcEvRKYo+IXIFAW/EJky0N3+jrexTNpyFQv8dahIivW1gt3QqL7f7Xfz9lqL5/iu8vzZdGJP7Ua+a38paK21NM/9n6vzOUdG+NM21EonsiBobVauBTXrguSSQiPYZb85rajsO8zr3B1//jS17WzzfezhYI13bJtPHq/WyToBqM+fozbUtlNTJ1BhamPb+JyExlK63iUAFNpEAQtqE/7aHFftkRDi7wUKfiEyRcEvRKYo+IXIFAW/EJmi4BciU9aU+szsRgBfB7APQAfAEXf/spl9DsAfAbjc++oz7v5YOJkDRqSIVosn4rilbUsraRkHACoF3h6pWApqvu3miRvDE+k5W6TGIAB0jCfUtIIsjHOLXKvcvZv7Xy6kx9UbaWkIAApWobZGm8uRK02eAHNx4ULyeHkkLZcCQL3DGkMBs3W+WCcXuDx7qJB+bI1gPZYXefsyC0KmEtQ0HB3jCV6M+ZU3qW2lnvbRO/zaWE0/On8LwJ+6+1NmNgbg52b2w57tS+7+n/o+mxDiuqGfXn2nAZzu/TxvZi8B2L/ZjgkhNper+sxvZgcB3A3gyd6hT5rZs2b2sJnxr24JIa47+g5+MxsF8F0An3L3SwC+AuBWAHeh+87gC2TcQ2Z21MyO1hf45z0hxGDpK/jNrIxu4H/D3b8HAO5+xt3b3v0y8VcB3Jsa6+5H3H3K3adqo3xjSQgxWNYMfjMzAF8D8JK7f/GK45NX/NlHADx/7d0TQmwW/ez23wfg4wCeM7One8c+A+BjZnYXumXDpgH88VoTmQGlcvr1pt3kEkWJtCYaKvHMNwuLmXFjqcDlpov1dJZVNVjGUlBn8OJCIDfVudS3VOcS50iNvLuKFoRIqQAwVOGSadTMa3kp/dw0goy5elAvsB58YnSusGG2kX5shbnzdMxCPZ15CgDVYD1qZS7rVmq8DRyjGMzXbKYzGd37r+LXz27/TwCkIiLW9IUQ1zX6hp8QmaLgFyJTFPxCZIqCX4hMUfALkSmDbdeFAkqdtBQ1FLwMMdNQMKjlPNNrucElqkKb21jG1NJKIB0al41aJAMPAArFwH8i8wA8A7JWHaFjGs2gLVSgHJUrXIras3tv8vjxzgwds8KXHh3nEuzMJS4RnjqflkU9WMPFRW6bJO3QAKAQSHMInuv1zMfWQ+26hBBrouAXIlMU/EJkioJfiExR8AuRKQp+ITJlwFKfoejprKgSV1CANpG9gt5oKPIJa9V0H7nuuXhRylo57ccK8w+ABVlWI+VxaqtXuO5VX+EFJhvttJS60uLzNVpcjhwe4ilzixf5Gi9cSqfhLQQFXYKkvlBzXA4kwvMX089nNehB2GhGjnD/rcNt3uHXyHrm4370L/bpzi9Epij4hcgUBb8QmaLgFyJTFPxCZIqCX4hMGajU5x2HL6V1mfYQlzWGkJavnGQIAsB8mxe5LFa5fGWVQJojfd9qLT5fI5B4xmtcoxoqcj/OL/Css2otnT22GPSfawVrNTLEe7G0gqKry6Rw5jLp1Qig2wmS4MZlRe/wgW/Np4ukbq9x32uVoHdhkA3YDPr/WSC1rmc+5odH67sK3fmFyBQFvxCZouAXIlMU/EJkioJfiExZc7ffzKoAngAw1Pv7v3T3z5rZIQDfAjAB4CkAH3f3sA1vAYYKSajwFt99rRfSyRmdAh/TrvNdWTjfeTULaq010zv3FeO11pabPFHIne/AV4K2YTu384SgYjG9vktt3p6qWOL+N9pcdVh0/tgwnL4UhqpBgkuwox/RCXJZzsynn+u9QfeskWrwfLb4ydqB+51WsFbrmI/5cRXduvq6868A+B13fx+67bjvN7MPAPgzAF9y98MAZgF8ov/TCiG2mjWD37tcvkWVe/8cwO8A+Mve8UcAfHhTPBRCbAp9feY3s2KvQ+8MgB8CeA3AnPv/f/98AsD+zXFRCLEZ9BX87t5297sAHABwL4B3p/4sNdbMHjKzo2Z2dHGRf2NJCDFYrmq3393nAPwYwAcAbDezy7tLBwCcImOOuPuUu0+NjARNDYQQA2XN4Dez3Wa2vfdzDcA/A/ASgB8B+Je9P3sQwA82y0khxLWnn8SeSQCPWFcDKwD4jrv/lZm9COBbZvYfAfwCwNfWmsgLgA8R/aLBXWk10oknjVDXSNcKBIBWK6qnxmXAIlmuQiBRbasMU1unVaO2etBurNHhNffKpHbh2Ng2OqYZXAYW1IQrl3kCTJHYJvbzx3X6Ff646iQhDAAskEXrzbTtXKC8TU5wW6HIpeB2myvdC4tcal3PfNSPq5BL1wx+d38WwN2J46+j+/lfCPEORN/wEyJTFPxCZIqCX4hMUfALkSkKfiEyxfxq0oA2ejKzswCO9X7dBeDcwE7OkR9vR368nXeaHze7++5+Jhxo8L/txGZH3X1qS04uP+SH/NDbfiFyRcEvRKZsZfAf2cJzX4n8eDvy4+38vfVjyz7zCyG2Fr3tFyJTtiT4zex+M3vZzF41s09vhQ89P6bN7Dkze9rMjg7wvA+b2YyZPX/FsQkz+6GZ/ar3P++Ttbl+fM7MTvbW5Gkz+9AA/LjRzH5kZi+Z2Qtm9m96xwe6JoEfA10TM6ua2U/N7JmeH/+hd/yQmT3ZW49vmxlPq+wHdx/oPwBFdMuA3QKgAuAZAHcM2o+eL9MAdm3BeX8bwD0Anr/i2J8D+HTv508D+LMt8uNzAP7tgNdjEsA9vZ/HALwC4I5Br0ngx0DXBIABGO39XAbwJLoFdL4D4KO94/8FwL/eyHm24s5/L4BX3f1175b6/haAB7bAjy3D3Z8AcGHV4QfQLYQKDKggKvFj4Lj7aXd/qvfzPLrFYvZjwGsS+DFQvMumF83diuDfD+DNK37fyuKfDuDvzOznZvbQFvlwmb3ufhroXoQA9myhL580s2d7Hws2/ePHlZjZQXTrRzyJLVyTVX4AA16TQRTN3YrgT5Ua2SrJ4T53vwfAvwDwJ2b221vkx/XEVwDcim6PhtMAvjCoE5vZKIDvAviUu18a1Hn78GPga+IbKJrbL1sR/CcA3HjF77T452bj7qd6/88A+D62tjLRGTObBIDe/zNb4YS7n+ldeB0AX8WA1sTMyugG3Dfc/Xu9wwNfk5QfW7UmvXNfddHcftmK4P8ZgMO9ncsKgI8CeHTQTpjZiJmNXf4ZwO8BeD4etak8im4hVGALC6JeDrYeH8EA1sTMDN0akC+5+xevMA10TZgfg16TgRXNHdQO5qrdzA+hu5P6GoB/t0U+3IKu0vAMgBcG6QeAb6L79rGJ7juhTwDYCeBxAL/q/T+xRX78NwDPAXgW3eCbHIAfv4XuW9hnATzd+/ehQa9J4MdA1wTAe9Etivssui80//6Ka/anAF4F8D8ADG3kPPqGnxCZom/4CZEpCn4hMkXBL0SmKPiFyBQFvxCZouAXIlMU/EJkioJfiEz5fyke17IncxjHAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fafdd158940>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.imshow(data.trn_ds.denorm(x)[1])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Fully connected model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = get_data(32, bs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "lr = 1e-2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "From [this notebook](https://github.com/KeremTurgutlu/deeplearning/blob/master/Exploring%20Optimizers.ipynb) by our student Kerem Turgutlu:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "class SimpleNet(nn.Module):\n",
    "    def __init__(self, layers):\n",
    "        super().__init__()\n",
    "        self.layers = nn.ModuleList([\n",
    "            nn.Linear(layers[i], layers[i + 1]) for i in range(len(layers) - 1)])\n",
    "        \n",
    "    def forward(self, x):\n",
    "        x = x.view(x.size(0), -1)\n",
    "        for l in self.layers:\n",
    "            l_x = l(x)\n",
    "            x = F.relu(l_x)\n",
    "        return F.log_softmax(l_x, dim=-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn = ConvLearner.from_model_data(SimpleNet([32*32*3, 40, 10]), data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(SimpleNet(\n",
       "   (layers): ModuleList(\n",
       "     (0): Linear(in_features=3072, out_features=40)\n",
       "     (1): Linear(in_features=40, out_features=10)\n",
       "   )\n",
       " ), [122880, 40, 400, 10])"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "learn, [o.numel() for o in learn.model.parameters()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "OrderedDict([('Linear-1',\n",
       "              OrderedDict([('input_shape', [-1, 3072]),\n",
       "                           ('output_shape', [-1, 40]),\n",
       "                           ('trainable', True),\n",
       "                           ('nb_params', 122920)])),\n",
       "             ('Linear-2',\n",
       "              OrderedDict([('input_shape', [-1, 40]),\n",
       "                           ('output_shape', [-1, 10]),\n",
       "                           ('trainable', True),\n",
       "                           ('nb_params', 410)]))])"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "learn.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "61a8c98dab2042049de01ea0db471779",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>HBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " 76%|███████▌  | 148/196 [00:15<00:05,  9.40it/s, loss=10]  "
     ]
    }
   ],
   "source": [
    "learn.lr_find()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYwAAAEOCAYAAACaQSCZAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4xLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvAOZPmwAAIABJREFUeJzt3Xd4VFX+x/H3N42QkIQWWiihSe8BQWyw/hTLir2ja1kWy6pr367r7tpduyuLrg3XtWAvKCoqICX0EhAElE5oIZCenN8fM7oxDjCB3LmT5PN6nnm4c++5M98c8vDhtnPMOYeIiMiBxPhdgIiI1A4KDBERCYsCQ0REwqLAEBGRsCgwREQkLAoMEREJiwJDRETCosAQEZGwKDBERCQsCgwREQlLnN8F1KTmzZu7zMxMv8sQEak15s6du805lx5O2zoVGJmZmWRnZ/tdhohIrWFm34bbVqekREQkLAoMEREJiwJDRETCosAQEZGwKDBERCQsCgwREQmLAqOa8gpKmbl6O0Wl5X6XIiISUXXqOYyD9egnK0luEEertERapjagZWoicTExfLJ8C5/mbMXMaNukIRt3FTJ1RS4l5RUkJcQysnsLrjy2M73apPn9I4iIeK7eB0ZFhePJz7+hoCT0EUOHZkkkxsUyc/V2khJiuWhoB4Z0bMIXK7fx/uJNTF66mZuO78Yvj+pETIxFuHoRkcgx55zfNdSYrKwsdzBPejvn2FlQypbdRWzeXcSWvCL2FJdxZNfmdGuZgpnhnMPsx4GwY28Jv520iMlLt5Ce0oAmSfGkJMbTKjWRNo0TGdC+CcO7NCetYTwA5RWONdv2sHhDHovW57F0w26Kyyto16Qh3VqmcOHQDjRNTqiRvhARCYeZzXXOZYXVVoFxaJxzvLVgI1+u3Mbe4jLyCkvZvLuIjbsKKS6rIDbGaJWaSFlFBbsLyygMXvtIjI+hZ+tUkhLiWLezgO92FJAUH8vFR2TSqEEca7btJa1hPCO6tWBIx6YkxOlyk4jUPAVGFCgrr2D+ul18viKXjbsKiY+NIblBHD3bpNInI43O6cnExf4vBFZuyeehKSt5b/EmAFqkNGBXYSklZRUAxMYYcTFGSmIcaQ3jaZ3WkM7pyXRu0YjO6YFXy9QGPzkKEhHZHwVGLbY1v4jkhDiSG8RRUFLGjFXbWbQhj/KKCsrKHbuLysgrLGHDzkK+yd3LnuKyH/ZtlZpIVmYTemek0bF5Mp2aJ9O+WRIN4mJ9/IlEJJopMOoJ5xxb84v5JncPX2/OZ953u8heu4ONeUU/tDGDdk2SOKprc07o1YrurVJoEB9LckLsj45wRKR+UmDUc7uLSlm7bS9rgq+cTbv5cuW2H90J1jA+liEdm3Jkl+YM79Kc7q1SdJeXSD1UncCo97fV1kWpifH0bduYvm0b/7CuqLSc6au2sTGviOLScr7bUcD0Vdv42/s5ADRvlMDPurdk9IA2DO3YTOEhIj+hwKgnEuNj+VmPlj9ZvymvkOmrtvPlylzeXbSR/2avo2F8LCmJcaQkxpGe0oDWaQ05rGUKWZlN6JORRmK8romI1Ec6JSU/KCwp5+OcLSxct4u9xWXsLiply+5iNucVsWFXIRC4HfjILs05rkdLTh+YoQvqIrWcrmFIjdu+p5h53+1i+qptTMnZwvqdhXRolsQfT+7Jz3q00O28IrWUAkM85Zxj2qpt3PHOMlZt3UOXFo34WY8WjOrViv7tGis8RGoRBYZERGl5Ba9kr+P9xZuYtXoHZRWOXm1SGTO0A6f2b0NSgi6RiUS7qAgMM2sHPA+0AiqA8c65h6u0GQ3cGdxeBlzvnJsW3HYJ8Idg078655470HcqMPyzu6iUdxZu5IWvvmX55nxSEuM4a1BbjuzSnMzmybRvmkS8nvsQiTrREhitgdbOuXlmlgLMBU5zzi2r1KYRsNc558ysL/CKc667mTUFsoEswAX3HeSc27m/71Rg+M85R/a3O3nhq2/5YMkmSssDv1/NGyVw8bBMxgztQBMNsCgSNaLiOQzn3CZgU3A538xygAxgWaU2eyrtkkwgHABOAD52zu0AMLOPgVHAf7yqV2qGmTE4symDM5uSV9ibVVv3sGbbXt5dtJEHP/6axz5bxRGdmzGyewtO6dtGo/OK1CIROclsZpnAAGBWiG2nA3cBLYCTg6szgHWVmq0PrpNaJK1hPIM6NGFQhyacNagtKzbn8/Kc7/hs+Vb+9NZS7v5gORcMac/YozvRIjXR73JF5AA8P6kcPO30OoHrE7urbnfOveGc6w6cRuB6BkCo22xCnjszs7Fmlm1m2bm5uTVVtnigW6sU/vzzXky9eQQfXn8UJ/RqxTPT13DMfVN5aMrXFJSUHfhDRMQ3ngaGmcUTCIuJzrlJ+2vrnPsC6GxmzQkcUbSrtLktsHEf+413zmU557LS09NrqHLxWvdWqfzj3P58euOxjOzegoemrGTE/VN5be56Kirqzp17InWJZ4FhgZvxnwZynHMP7qNNl2A7zGwgkABsByYDx5tZEzNrAhwfXCd1TGbzZB6/cCCvjRtGq7SG3PTqQk59fBrzv9vv/Q0i4gMvjzCGA2OAkWa2IPg6yczGmdm4YJszgSVmtgB4HDjXBewgcHpqTvD1l+8vgEvdlJXZlDeuPIKHz+vP9j0lnPnkDO76IIei0tBzrYtI5OnBPYk6+UWl/P39HP4zex1dWjTi/rP70b9d4wPvKCLVVp3bavUklUSdlMR47jqjL89dNoS9xWWc8cR07vogh/yiUr9LE6nXFBgStY45LJ3Jvzmaswe146nPV3PUvZ/x5NRvdJpKxCcKDIlqqYnx3HNWX9655kgGtGvMPR8u59THppGz6Sd3aIuIxxQYUiv0aZvGvy8dwnOXDWFnQSmjH5vO+C++oay8wu/SROoNBYbUKsccls6H1x3FMd3S+fv7yzn9iRks2ZDnd1ki9YICQ2qdZo0aMH7MIB67YACb8oo4/YnpfLhks99lidR5CgyplcyMU/q2YcoNR9M7I42rX5rHOwtDDgYgIjVEgSG1WuOkBF64/HAGtW/CdS/P5+XZ3/ldkkidpcCQWq9RgzievWwwR3VN57ZJi3nw46+pSw+kikQLBYbUCUkJcUy4JItzstryyCcrueTfc/hs+VbKNZChSI3RpMtSZ8THxnDPmX3p2iKFp774hkufnUPrtERO7tOak/u2pn+7xgTHuhSRg6CxpKROKimrYErOFl6fu54vVuZSWu4Y3qUZfzqlF91apfhdnkjUiIo5vf2gwJBQ8gpLmTRvPQ9NWcme4jJO65/BmGEd6Nc2TUccUu8pMERC2Lm3hIc/Wckr2esoKCmnd0YqVx7ThVG9WxEbo+CQ+kmBIbIf+UWlvLVgI89MX8Pq3L10Tk/mkfMH0KtNmt+liUSchjcX2Y+UxHguGtqBj39zDI9dMICCknIufno2q7bu8bs0kaimwJB6KzYm8LT4xCsOx8y4aMIsvtte4HdZIlFLgSH1Xqf0Rrx4xRAKS8v5+WPTeGvBBj34JxKCAkME6N4qlTevHk7n9GSue3kB17w0n92a4U/kRxQYIkEdmyfz6rgjuPmEbny4dDM/f3QaSzdq6HSR7ykwRCqJjTGuHtGF/44dSnFpBac/MYOXZ3+nU1QiKDBEQsrKbMp71x7J4R2bctukxdz06iIKSzSXuNRvCgyRfWjWqAHPXjqE64/ryqT567lgwkxd15B6TYEhsh+xMcb1xx3GkxcOYsmGPMY8PZu8QoWG1E+eBYaZtTOzz8wsx8yWmtl1IdpcaGaLgq8ZZtav0ra1ZrbYzBaYmR7fFl+N6t2KJy8cxLKNeVz89CydnpJ6ycsjjDLgRudcD2AocLWZ9azSZg1wjHOuL3AnML7K9hHOuf7hPrYu4qXjerbk8QsGsmhDHje9ulAXwqXe8SwwnHObnHPzgsv5QA6QUaXNDOfczuDbmUBbr+oRqQnH92rFbaO6897iTTzyySq/yxGJqIhcwzCzTGAAMGs/zS4HPqj03gEfmdlcMxvrXXUi1TP26E6cMSCDf0z5mo+XbfG7HJGI8TwwzKwR8DpwvXNu9z7ajCAQGLdWWj3cOTcQOJHA6ayj97HvWDPLNrPs3NzcGq5e5KfMjL+f0YfeGanc+MoC1u3Q+FNSP3gaGGYWTyAsJjrnJu2jTV9gAjDaObf9+/XOuY3BP7cCbwBDQu3vnBvvnMtyzmWlp6fX9I8gElJifCyPXzAQ5+Cal+ZRXKaL4FL3eXmXlAFPAznOuQf30aY9MAkY45z7utL6ZDNL+X4ZOB5Y4lWtIgejQ7Nk7ju7LwvX53HLa4soLa/wuyQRT8V5+NnDgTHAYjNbEFz3O6A9gHPun8CfgGbAE8GpMsuCd0S1BN4IrosDXnLOfehhrSIHZVTv1tx8Qjfum7yCPUVlPH7hQBLjY/0uS8QTmnFPpAa8MPNb/vTWEg7v2JTnLhtCgziFhtQOmnFPJMLGDO3Ag+f0Y+bqHfz5raV6RkPqJC9PSYnUK6cPaMuqrXt4/LNv6NUmlTHDMv0uSaRGKTBEatCN/9eN5ZvyueOdZTSIi+Wcwe38LkmkxuiUlEgNiokxHjqvP0M7NeOW1xdx62uLKCrVLbdSNygwRGpYSmI8z102hF+P7MJ/s9dx4ysad0rqBgWGiAdiY4wbj+/GzSd0473Fm3h74Ua/SxI5ZAoMEQ+NO6Yzgzo04Y9vLmFzXpHf5YgcEgWGiIdiY4wHzu5Habnj2v/MZ/ueYr9LEjloCgwRj2U2T+buM/uwYN0uRj38JVNXbPW7JJGDosAQiYDR/TN465rhNEmK5xf/nsPDU1bqQrjUOgoMkQjp0TqVt6858oe5NK59eYFuuZVaRQ/uiURQYnwsD5zTjy4tG3HvhyvYubeECZdkacBCqRV0hCESYWbGVcd24f6z+zFt1TaumjiPkjINjS7RT4Eh4pOzBrXlb6f35tPlW7n6pXkUlJT5XZLIfikwRHx04eEduOPUXkzJ2cJZT37Fhl2Ffpcksk8KDBGfXXJEJs9cMph1Owo49dFpLNmQ53dJIiEpMESiwIjuLXjj6uEkxsdywb9msnDdLr9LEvkJBYZIlOjSohEvjx1KasN4LpowS6EhUUeBIRJF2jVN4pVfDaNxcjy/fD6brbs1/pREDwWGSJRp07gh48dkkV9UxpW65VaiiAJDJAr1aJ3KfWf3Ze63O7nz3WV+lyMCKDBEotYpfdvwy6M68sLMb/lwyWa/yxFRYIhEs5tP6E6fjDRum7RI82mI7zwLDDNrZ2afmVmOmS01s+tCtLnQzBYFXzPMrF+lbaPMbIWZrTKz27yqUySaJcTF8NB5/SkureDGVxdQXqERbsU/Xh5hlAE3Oud6AEOBq82sZ5U2a4BjnHN9gTuB8QBmFgs8DpwI9ATOD7GvSL3QOb0Rt5/ak+mrtnP9fxdQWq6L4OIPz0ardc5tAjYFl/PNLAfIAJZVajOj0i4zgbbB5SHAKufcagAzexkYXXlfkfrk3MHt2VVQyl0fLKe0rIJHzh9AQpzOKEtkReQ3zswygQHArP00uxz4ILicAayrtG19cJ1IvfWrYzrzp1N68uHSzVz54lzNpSER53lgmFkj4HXgeufc7n20GUEgMG79flWIZiFP3prZWDPLNrPs3NzcmihZJGpddmRH/npabz5ZvpVfPp+t0JCI8jQwzCyeQFhMdM5N2kebvsAEYLRzbntw9XqgXaVmbYGNofZ3zo13zmU557LS09NrrniRKHXR0A7ce2Zfpq3axsXPzCavoNTvkqSe8PIuKQOeBnKccw/uo017YBIwxjn3daVNc4CuZtbRzBKA84C3vapVpLY5Z3A7Hjq3Pwu+28XpT07nu+0Ffpck9YCXRxjDgTHASDNbEHydZGbjzGxcsM2fgGbAE8Ht2QDOuTLgGmAykAO84pxb6mGtIrXO6P4ZvHD5ELbvKeH0J6aTsynkGV+RGmPO1Z37urOyslx2drbfZYhE1De5e7howiyKSsuZeMVQerZJ9bskqUXMbK5zLiuctrovT6SW65weGBY9MT6WCyfM1ARM4hkFhkgd0KFZMi+PHUpSQhznPvUVX3ytOwal5ikwROqIDs2SmXTVEbRrmsRlz87hrQUb/C5J6hgFhkgd0jI1kVfHDSMrswk3vrKQWau3H3gnkTCFFRhmdp2ZpVrA02Y2z8yO97o4Eam+lMR4xl+cRftmSVw5cR7rduiWW6kZ4R5hXBZ8Svt4IB24FLjbs6pE5JCkJsYz4eIsysoruOzZOUxbuY26dEek+CPcwPh+qI6TgH875xYSevgOEYkSndIb8eRFg9hZUMJFT8/ixIe/ZOlG3UElBy/cwJhrZh8RCIzJZpYCaIxlkSg3vEtzpt06knvP6suuglKueC6b3Pxiv8uSWircwLgcuA0Y7JwrAOIJnJYSkSiXGB/LOVntmHBJFjsLSrhq4lxKyvT/Pam+cANjGLDCObfLzC4C/gDo2FakFumdkcY9Z/Zlztqd3PVBjt/lSC0UbmA8CRQEp1C9BfgWeN6zqkTEE6P7Z3DJsA48O2Mt87/b6Xc5UsuEGxhlLnCLxWjgYefcw0CKd2WJiFduHtWdlimJ/O6NJZRpulephnADI9/Mfktg9Nn3gnNux3tXloh4pVGDOG4/tSc5m3bz7Iy1fpcjtUi4gXEuUEzgeYzNBKZLvc+zqkTEUyf0asXI7i148OOv2bCr0O9ypJYIKzCCITERSDOzU4Ai55yuYYjUUmbGHaf2osI5bn9bU81IeMIdGuQcYDZwNnAOMMvMzvKyMBHxVrumSVx/3GF8vGwLHy3d7Hc5UguEe0rq9wSewbjEOXcxMAT4o3dliUgkXH5kR7q1TOH2t5eSV6i5wWX/wg2MGOfc1krvt1djXxGJUvGxMfzt9N5s3l3EUfd8yj0fLmfbHj0JLqGF+4/+h2Y22cx+YWa/AN4D3veuLBGJlKzMprxx1XCO7Nqcpz7/htGPTWejLoRLCGHP6W1mZwLDCQw6+IVz7g0vCzsYmtNb5NAsXLeLiybMIj2lAa+MG0bzRg38Lkk85smc3s65151zNzjnfhONYSEih65fu8Y8c+lgNuYVcvHTsykuK/e7JIki+w0MM8s3s90hXvlmtjtSRYpI5AzObMqj5w9k2abdTPhyjd/lSBTZb2A451Kcc6khXinOudRIFSkikfV/PVtyYu9WPPrpSs3YJz/w7E4nM2tnZp+ZWY6ZLTWz60K06W5mX5lZsZndVGXbWjNbbGYLzEwXJkQi7I+n9MQw7nx3md+lSJTw8tbYMuBG51wPYChwtZn1rNJmB3AtcP8+PmOEc65/uBdkRKTmtGnckGt/1pWPlm1hyrItfpcjUcCzwHDObXLOzQsu5wM5BMagqtxmq3NuDqAnhkSi0OVHdqR7qxR+98ZidhWU+F2O+CwiD9+ZWSYwAJhVjd0c8JGZzTWzsV7UJSL7lxAXw/1n92PH3hLueEenpuo7zwPDzBoBrwPXO+eqc2fVcOfcQOBEAqezjt7H5481s2wzy87Nza2BikWkst4ZaVw9ogtvzN/AZI05Va95GhhmFk8gLCY65yZVZ1/n3Mbgn1uBNwiMXxWq3XjnXJZzLis9Pf1QSxaREK4e0YVebVK59fVFegq8HvPyLikDngZynHMPVnPfZDNL+X4ZOB5YUvNVikg4EuJiePT8AZSVO655aR6lmqmvXvLyCGM4gRn6RgZvjV1gZieZ2TgzGwdgZq3MbD1wA/AHM1tvZqlAS2CamS0kMKz6e865Dz2sVUQOoFN6I+4+sw/zvtvFPR8s97sc8UGcVx/snJtGYNyp/bXZDLQNsWk30M+LukTk4J3Stw1z1uxgwrQ1dExP5sLDO/hdkkSQZ4EhInXTH07pyXc7Cvjjm0to3qgBJ/Rq5XdJEiGa00JEqiU+NobHLxxI37aNufY/81mwbpffJUmEKDBEpNqSEuJ45heDad6oAdf+Zz75RXr2tj5QYIjIQWmanMBD5/Vn/c4C/vzWUr/LkQhQYIjIQRuc2ZRfj+zKpPkbeGP+er/LEY8pMETkkPx6ZBeGZDblt5MWs2yjpsmpyxQYInJI4mJjeOzCAaQ1jOdXL2azc68GKayrFBgicshapCTyz4sGsSWvmGtfnk+ZngSvkxQYIlIjBrRvwp2n9eLLldu476MVfpcjHtCDeyJSY84d3J5F6/N46vPV9G6Txs/7tfG7JKlBOsIQkRr155/3IqtDE255bRGrtub7XY7UIAWGiNSohLgYnrhwIAlxMfz+jSU45/wuSWqIAkNEalyL1ERuO7E7s9bsYNK8DX6XIzVEgSEinjg3qx0D2zfmb+/naD7wOkKBISKeiIkx/nZ6H/IKS7np1YUUl5X7XZIcIgWGiHimR+tUbv95T6bkbOWK57LZW1zmd0lyCBQYIuKpMcMyeeDsfkxftY1L/z2H8gpdBK+tFBgi4rkzB7XlnjP7MnvtDibO+tbvcuQgKTBEJCLOGtSW4V2acd/kFWzbU+x3OXIQFBgiEhFmxh2n9qaotJy7P1judzlyEBQYIhIxXVo04oqjOvHa3PXMWr3d73KkmhQYIhJRvx7ZhfZNk7jl9UUUluhW29pEgSEiEZWUEMe9Z/Xl2+0F3DtZp6ZqEwWGiETc0E7NuGRYB56dsZbZa3b4XY6EybPAMLN2ZvaZmeWY2VIzuy5Em+5m9pWZFZvZTVW2jTKzFWa2ysxu86pOEfHHLaO607ZJQ255baFOTdUSXh5hlAE3Oud6AEOBq82sZ5U2O4BrgfsrrzSzWOBx4ESgJ3B+iH1FpBZLbhDHvWf2Y+32Au6brAmXagPPAsM5t8k5Ny+4nA/kABlV2mx1zs0BSqvsPgRY5Zxb7ZwrAV4GRntVq4j4Y1jnZlw8rAP/nrGGOWt1airaReQahpllAgOAWWHukgGsq/R+PVXCRkTqhltHdSejcUOufHEez3+1lqJSnZ6KVp4Hhpk1Al4HrnfO7Q53txDrQg5AY2ZjzSzbzLJzc3MPtkwR8UlygzjGj8miQ7Mk/vTWUkbcP5X1Owv8LktC8DQwzCyeQFhMdM5Nqsau64F2ld63BTaGauicG++cy3LOZaWnpx98sSLim55tUnlt3DBevPxwtu8p4anPV/tdkoTg5V1SBjwN5DjnHqzm7nOArmbW0cwSgPOAt2u6RhGJHmbGkV2bM7p/G16du44dezXpUrTx8ghjODAGGGlmC4Kvk8xsnJmNAzCzVma2HrgB+IOZrTezVOdcGXANMJnAxfJXnHNLPaxVRKLE2KM7UVRawQtfaVTbaBPn1Qc756YR+lpE5TabCZxuCrXtfeB9D0oTkSjWtWUKI7u34Pmv1vKrYzqRGB/rd0kSpCe9RSTq/PKoTmzfW8ITn63COU24FC0UGCISdYZ2asrJfVrzyKerGPvCXHYV6HpGNFBgiEjUMTMeu2AAfzi5B1NXbOW0x6ezUxfBfafAEJGoZGZccVQnXvrlUDbuKuKqifMoLa/wu6x6TYEhIlFtcGZT7jqjD1+t3s5f313mdzn1mmd3SYmI1JQzB7VlxZZ8xn+xmm6tUrng8PZ+l1Qv6QhDRGqFW0d155jD0vnTW0s0vatPFBgiUivExhiPnD+A9s2SuHLiPNbt0HhTkabAEJFaI61hPBMuzqK0vIKrX5pHcZlGto0kBYaI1Cqd0htx/9n9WLQ+j7ve15zgkaTAEJFa54Rerbh0eCbPzljLB4s3+V1OvaHAEJFa6bcn9qBf2zR+/Z/5PDxlpZ7RiAAFhojUSglxMTx/2eGc3Lc1/5jyNWc9OUNDontMgSEitVZaUjwPnzeAxy8YSM7mfK6aOFdHGh5SYIhIrXdy39bcc2YfZq7ewV/e0dPgXtGT3iJSJ5w+oC3LN+Xz1Ber6dkmlfOH6GnwmqYjDBGpM24Z1Z2jD0vnz28vZcmGPL/LqXMUGCJSZ8TGGA+d259myQlcOXEueYWlfpdUpygwRKROaZqcwGMXDGTTriKue3m+ngavQQoMEalzBnVowl9G92bqilzGPj+XotK6GxpLNuQxe80OKiq8n8pWgSEiddIFh7fn7jP68MXKXC799xz2Fpf5XZInnvz8G658cS4VEZj7XIEhInXWeUPa849z+jN77Q7GPD2L3UV165pGQUkZn+Zs5cQ+rYiL9f6fcwWGiNRppw3I4LHzB7B4Qx4X/msWm/OK/C6pxkzJ2UphaTmn9G0Tke/zLDDMrJ2ZfWZmOWa21MyuC9HGzOwRM1tlZovMbGClbeVmtiD4eturOkWk7juxT2ueGjOIFVvyOereT7nhlQWszt3jd1mH7N2FG2mR0oDBmU0j8n1eHmGUATc653oAQ4GrzaxnlTYnAl2Dr7HAk5W2FTrn+gdfp3pYp4jUAyO7t2TKb47hwsM78MHizZz/r5kUltTei+G7i0qZ+nUuJ/dtTWyMReQ7PQsM59wm59y84HI+kANkVGk2GnjeBcwEGptZa69qEpH6rX2zJG4/tRfPXjqYLbuLeWb6Gr9LOmgfL91CSVlFxE5HQYSuYZhZJjAAmFVlUwawrtL79fwvVBLNLNvMZprZaZ4XKSL1xuGdmvGz7i3459Rvau0It+8u2khG44YMbN84Yt/peWCYWSPgdeB659zuqptD7PL9vWHtnXNZwAXAQ2bWeR+fPzYYLNm5ubk1VreI1G23ntidvSVlPP7ZKr9Lqbb8olKmrdrGSX1aYRaZ01HgcWCYWTyBsJjonJsUosl6oF2l922BjQDOue//XA1MJXCE8hPOufHOuSznXFZ6enoNVi8iddlhLVM4e1A7nv9qLS98tRYXgecYasqXK7dRWu44rkfLiH6vl3dJGfA0kOOce3Afzd4GLg7eLTUUyHPObTKzJmbWIPg5zYHhgMYsFpEadduJ3RnWuTl/fGspFz8zu9bccvtJzlbSGsYzqEOTiH6vl0cYw4ExwMhKt8eeZGbjzGxcsM37wGpgFfAv4Krg+h5AtpktBD4D7nbOKTBEpEY1SU7guUsH89fTepO9difH/+Nz3lqwIaqPNsorHJ+t2MqIbukReVivMs/mw3DOTSP0NYrKbRxwdYj1M4A+HpUmIvIDM+OioR04sktsvdY+AAALfklEQVRzbnhlAde9vIDJSzfz19P60DQ5we/yfmLBul3s2FvCyAifjgI96S0iAkBm82ReHXcEt4zqxsfLtnD8Pz5nyrItfpf1E5/kbCEuxjjmsMhfs1VgiIgExcYYVx3bhbevOZL0lESueD6bDxZv8rusH/l0+VYGZzYlrWF8xL9bgSEiUkWP1qm8cdURDGjfmBteWRg1s/et21HA8s35/KxHC1++X4EhIhJCYnwsT40ZRJOkeMY+n83WfP/voHo1ex1mcEKvVr58vwJDRGQfWqQkMv7iLHYWlPKrF/ydiKmkrIKXZq9jRLcWtGua5EsNCgwRkf3onZHGg+f0Y/53u/jdpMU/ueV23Y6CiMx29+HSzWzbU8yYYR08/6598ey2WhGRuuLEPq254f8O48GPv6bcOY7qmo4Bz8/8loXrdnH6gAzuP7ufp6PGvvjVt7RvmsQxXf0b0UKBISIShl+P7MK2PcW8mr2etxZsBKBj82TOGJDBpPmBh/0eOKe/J6GxfPNuZq/dwe9O6k5MhIYyD0WBISISBjPjL6N78+ef92LNtj3kFZYyoF0TYmKMTunJ3P/R1zjggbP71egT2DNWbeP3by4hMT6Gswe1O/AOHlJgiIhUQ2yM0aVFyo/WXTOyK2bGfZNX4Bw8eM6hh0ZRaTl/fHMJr85dT/umSTxzyWCa+PzkuQJDRKQGXD2iC2Zw74crqHCO+8/uR2J8LAAFJWUYRsOE2P1+RnmFIzbGyM0vZuwL2cz/bhdXHduZa3/W9YfP8pMCQ0Skhlx1bBdizbjrg+Ws21HAYxcM5ONlW3jgoxXExBgXD+vARUM70Co18UfzWKzams+f317KjG+20yKlAaXljoKSMv550UBG9Y6eSUgtmkdlrK6srCyXnZ3tdxkiUs9NXrqZG19ZSEFJGRUOjjksneQGsXywZDPOQUJsDOkpDUhPaUDjpHimr9pGw/hYzslqR15hKflFZVw1ojN923o/m56ZzQ1OVndAOsIQEalhJ/RqReerG/HARys4uW9rTu7TGjNjde4epq7IZUt+Ebm7i9maX8zGXYWcMaAtN4/qRvNGDfwufb90hCEiUo9V5whDT3qLiEhYFBgiIhIWBYaIiIRFgSEiImFRYIiISFgUGCIiEhYFhoiIhEWBISIiYalTD+6ZWS7w7UHsmgYczCzv4ex3oDb72h5qfdV1+3vfHNh2gNoOlvqrevzqr0Ppq1Drorm/wt1H/fXTfTo458Kblck5V+9fwHiv9jtQm31tD7W+6rr9vQey1V/1u78Opa9qW3+Fu4/669B+J3VKKuAdD/c7UJt9bQ+1vuq6A733ivqrevzqr0Ppq1Drorm/wt1H/XXw+9StU1LyP2aW7cIcH0bUX9Wl/qqeutJfOsKou8b7XUAto/6qHvVX9dSJ/tIRhoiIhEVHGCIiEhYFhoiIhEWBISIiYVFg1FNmlmxmc83sFL9riXZm1sPM/mlmr5nZlX7XE+3M7DQz+5eZvWVmx/tdT7Qzs05m9rSZveZ3LQeiwKhlzOwZM9tqZkuqrB9lZivMbJWZ3RbGR90KvOJNldGjJvrLOZfjnBsHnAPU+lsj96eG+utN59wvgV8A53pYru9qqL9WO+cu97bSmqG7pGoZMzsa2AM875zrHVwXC3wN/B+wHpgDnA/EAndV+YjLgL4EhipIBLY5596NTPWRVxP95ZzbamanArcBjznnXopU/ZFWU/0V3O8BYKJzbl6Eyo+4Gu6v15xzZ0Wq9oMR53cBUj3OuS/MLLPK6iHAKufcagAzexkY7Zy7C/jJKSczGwEkAz2BQjN73zlX4WnhPqmJ/gp+ztvA22b2HlBnA6OGfr8MuBv4oC6HBdTc71dtocCoGzKAdZXerwcO31dj59zvAczsFwSOMOpkWOxHtfrLzI4FzgAaAO97Wll0qlZ/Ab8GjgPSzKyLc+6fXhYXhar7+9UM+BswwMx+GwyWqKTAqBssxLoDnmt0zj1b86XUCtXqL+fcVGCqV8XUAtXtr0eAR7wrJ+pVt7+2A+O8K6fm6KJ33bAeaFfpfVtgo0+11Abqr+pRf1VPne0vBUbdMAfoamYdzSwBOA942+eaopn6q3rUX9VTZ/tLgVHLmNl/gK+Abma23swud86VAdcAk4Ec4BXn3FI/64wW6q/qUX9VT33rL91WKyIiYdERhoiIhEWBISIiYVFgiIhIWBQYIiISFgWGiIiERYEhIiJhUWCIb8xsTwS+49Qwh3uvye881syOOIj9BpjZhODyL8zssZqvrvrMLLPq8N0h2qSb2YeRqkn8ocCQWi84nHRIzrm3nXN3e/Cd+xuH7Vig2oEB/A549KAK8plzLhfYZGbD/a5FvKPAkKhgZjeb2RwzW2Rmd1Ra/2ZwZsClZja20vo9ZvYXM5sFDDOztWZ2h5nNM7PFZtY92O6H/6mb2bNm9oiZzTCz1WZ2VnB9jJk9EfyOd83s/e+3Valxqpn93cw+B64zs5+b2Swzm29mU8ysZXCo63HAb8xsgZkdFfzf9+vBn29OqH9UzSwF6OucWxhiWwcz+yTYN5+YWfvg+s5mNjP4mX8JdcRmgZkV3zOzhWa2xMzODa4fHOyHhWY228xSgkcSXwb7cF6ooyQzizWz+yr9Xf2q0uY3gQtD/gVL3eCc00svX17AnuCfxwPjCYzyGQO8Cxwd3NY0+GdDYAnQLPjeAedU+qy1wK+Dy1cBE4LLvyAw6RHAs8Crwe/oSWDOAoCzCAxbHgO0AnYCZ4WodyrwRKX3TfjfaAlXAA8El28HbqrU7iXgyOByeyAnxGePAF6v9L5y3e8AlwSXLwPeDC6/C5wfXB73fX9W+dwzgX9Vep8GJACrgcHBdakERq5OAhKD67oC2cHlTGBJcHks8IfgcgMgG+gYfJ8BLPb790ov714a3lyiwfHB1/zg+0YE/sH6ArjWzE4Prm8XXL8dKAder/I5k4J/ziUwf0Uob7rA/B/LzKxlcN2RwKvB9ZvN7LP91PrfSsttgf+aWWsC/wiv2cc+xwE9zX4Y9TrVzFKcc/mV2rQGcvex/7BKP88LwL2V1p8WXH4JuD/EvouB+83sHuBd59yXZtYH2OScmwPgnNsNgaMR4DEz60+gfw8L8XnHA30rHYGlEfg7WQNsBdrs42eQOkCBIdHAgLucc0/9aGVg4qLjgGHOuQIzm0pgWlmAIudceZXPKQ7+Wc6+f7eLKy1blT/DsbfS8qPAg865t4O13r6PfWII/AyF+/ncQv73sx1I2APAOee+NrNBwEnAXWb2EYFTR6E+4zfAFqBfsOaiEG2MwJHc5BDbEgn8HFJH6RqGRIPJwGVm1gjAzDLMrAWB/73uDIZFd2CoR98/DTgzeC2jJYGL1uFIAzYEly+ptD4fSKn0/iMCo5cCEPwffFU5QJd9fM8MAkNkQ+AawbTg8kwCp5yotP1HzKwNUOCce5HAEchAYDnQxswGB9ukBC/ipxE48qgAxhCYg7qqycCVZhYf3Pew4JEJBI5I9ns3ldRuCgzxnXPuIwKnVL4ys8XAawT+wf0QiDOzRcCdBP6B9MLrBCa9WQI8BcwC8sLY73bgVTP7EthWaf07wOnfX/QGrgWygheJlxFidjXn3HICU5qmVN0W3P/SYD+MAa4Lrr8euMHMZhM4pRWq5j7AbDNbAPwe+KtzrgQ4F3jUzBYCHxM4OngCuMTMZhL4x39viM+bACwD5gVvtX2K/x3NjQDeC7GP1BEa3lwEMLNGzrk9FphfeTYw3Dm3OcI1/AbId85NCLN9ElDonHNmdh6BC+CjPS1y//V8AYx2zu30qwbxlq5hiAS8a2aNCVy8vjPSYRH0JHB2NdoPInCR2oBdBO6g8oWZpRO4nqOwqMN0hCEiImHRNQwREQmLAkNERMKiwBARkbAoMEREJCwKDBERCYsCQ0REwvL/a6A4U+MMMggAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fafdd2340b8>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r",
      " 76%|███████▌  | 148/196 [00:30<00:09,  4.93it/s, loss=10]"
     ]
    }
   ],
   "source": [
    "learn.sched.plot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "dcbf4554bf6345479a904afdc84eb02b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>HBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=2), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " 13%|█▎        | 26/196 [00:03<00:20,  8.28it/s, loss=2.06]\n",
      " 13%|█▎        | 26/196 [00:03<00:20,  8.22it/s, loss=2.05]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Exception in thread Thread-4:\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/paperspace/anaconda3/envs/fastai/lib/python3.6/threading.py\", line 916, in _bootstrap_inner\n",
      "    self.run()\n",
      "  File \"/home/paperspace/anaconda3/envs/fastai/lib/python3.6/site-packages/tqdm/_tqdm.py\", line 144, in run\n",
      "    for instance in self.tqdm_cls._instances:\n",
      "  File \"/home/paperspace/anaconda3/envs/fastai/lib/python3.6/_weakrefset.py\", line 60, in __iter__\n",
      "    for itemref in self.data:\n",
      "RuntimeError: Set changed size during iteration\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss   accuracy                   \n",
      "    0      1.766196   1.642816   0.419     \n",
      "    1      1.675517   1.568509   0.4466                     \n",
      "\n",
      "CPU times: user 1min 38s, sys: 2min 51s, total: 4min 29s\n",
      "Wall time: 44.3 s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[array([1.56851]), 0.4466]"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%time learn.fit(lr, 2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8717af90861d4c74af82cc5b87af1457",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>HBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=2), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss   accuracy                   \n",
      "    0      1.617357   1.515796   0.4654    \n",
      "    1      1.582096   1.496592   0.4684                     \n",
      "\n",
      "CPU times: user 1min 37s, sys: 2min 46s, total: 4min 23s\n",
      "Wall time: 43.5 s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[array([1.49659]), 0.4684]"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%time learn.fit(lr, 2, cycle_len=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## CNN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "class ConvNet(nn.Module):\n",
    "    def __init__(self, layers, c):\n",
    "        super().__init__()\n",
    "        self.layers = nn.ModuleList([\n",
    "            nn.Conv2d(layers[i], layers[i + 1], kernel_size=3, stride=2)\n",
    "            for i in range(len(layers) - 1)])\n",
    "        self.pool = nn.AdaptiveMaxPool2d(1)\n",
    "        self.out = nn.Linear(layers[-1], c)\n",
    "        \n",
    "    def forward(self, x):\n",
    "        for l in self.layers: x = F.relu(l(x))\n",
    "        x = self.pool(x)\n",
    "        x = x.view(x.size(0), -1)\n",
    "        return F.log_softmax(self.out(x), dim=-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn = ConvLearner.from_model_data(ConvNet([3, 20, 40, 80], 10), data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "OrderedDict([('Conv2d-1',\n",
       "              OrderedDict([('input_shape', [-1, 3, 32, 32]),\n",
       "                           ('output_shape', [-1, 20, 15, 15]),\n",
       "                           ('trainable', True),\n",
       "                           ('nb_params', 560)])),\n",
       "             ('Conv2d-2',\n",
       "              OrderedDict([('input_shape', [-1, 20, 15, 15]),\n",
       "                           ('output_shape', [-1, 40, 7, 7]),\n",
       "                           ('trainable', True),\n",
       "                           ('nb_params', 7240)])),\n",
       "             ('Conv2d-3',\n",
       "              OrderedDict([('input_shape', [-1, 40, 7, 7]),\n",
       "                           ('output_shape', [-1, 80, 3, 3]),\n",
       "                           ('trainable', True),\n",
       "                           ('nb_params', 28880)])),\n",
       "             ('AdaptiveMaxPool2d-4',\n",
       "              OrderedDict([('input_shape', [-1, 80, 3, 3]),\n",
       "                           ('output_shape', [-1, 80, 1, 1]),\n",
       "                           ('nb_params', 0)])),\n",
       "             ('Linear-5',\n",
       "              OrderedDict([('input_shape', [-1, 80]),\n",
       "                           ('output_shape', [-1, 10]),\n",
       "                           ('trainable', True),\n",
       "                           ('nb_params', 810)]))])"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "learn.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "95db0c19899d459da8c873cb990dc061",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>HBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " 98%|█████████▊| 192/196 [00:18<00:00, 10.29it/s, loss=10.1]"
     ]
    }
   ],
   "source": [
    "learn.lr_find(end_lr=100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEOCAYAAABmVAtTAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4xLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvAOZPmwAAIABJREFUeJzt3XucXWV97/HPd26ZXCaT24SEJEOQcLUC0UBFsGL1gNVapFKBWqpom6NVCy312GJfvmxt1dYeTlWqmMI52Eqtl0QFvAC1gEQ0kMSEQAbkEpCEyZVkMpn73vt3/thrNsOwZzITsvZl5vt+vfZr1l7rWXv91t4z+zfPep71PIoIzMzMAGrKHYCZmVUOJwUzMytwUjAzswInBTMzK3BSMDOzAicFMzMrcFIwM7MCJwUzMytwUjAzswInBTMzK6grdwDjNW/evFi6dGm5wzAzqyobNmzYGxEthytXdUlh6dKlrF+/vtxhmJlVFUnPjKWcLx+ZmVmBk4KZmRU4KZiZWYGTgpmZFTgpmJlZgZOCmZkVOCmYmVWBu7bu4vFdnakfx0nBzKwKfOiWjaz5xY7Uj+OkYGZW4SKC/myOhtr0v7KdFMzMKlx/NgdAQ52TgpnZpNefySeFKU4KZmY2mBRcUzAzsxcuH1Vzm4KkRkkPSNos6RFJf1OkzBRJ35D0hKR1kpamFY+ZWbWaKDWFPuA3I+IM4EzgLZJeO6zM+4H9EbEM+D/AP6QYj5lZVZoQSSHyDiVP65NHDCt2EfDVZPnbwJskKa2YzMyqUV9mAlw+ApBUK2kTsBu4KyLWDSuyCHgWICIyQAcwN82YzMyqzYTpkhoR2Yg4E1gMnC3p14YVKVYrGF6bQNJKSeslrd+zZ08aoZqZVawJcfloqIg4ANwDvGXYpu3AEgBJdUAz8HyR/VdFxIqIWNHSctgpRs3MJpQJcZ+CpBZJs5LlqcCbgUeHFbsVeE+yfAnw3xHxkpqCmdlkVqgp1Namfqy6FF97IfBVSbXkk883I+J2SX8LrI+IW4GbgH+X9AT5GsJlKcZjZlaV+kp4+Si1pBARDwHLi6z/xJDlXuD30orBzGwi6M9mgQnUpmBmZkduwjU0m5nZkeufKPcpmJnZy1fKNgUnBTOzCjd481pVd0k1M7Ojw5ePzMysoD+To75W1NSkPzSck4KZWYXrz5RmfmZwUjAzq3j92VxJGpnBScHMrOL1Z5wUzMws4aRgZmYFfVm3KZiZWSJfU0h/hFRwUjAzq3i+fGRmZgX9mRxTfPnIzMxggnRJlbRE0t2S2iQ9IumqImVmS/qOpIckPVBkDmczs0lvolw+ygDXRMSpwGuBD0k6bViZa4FNEXE68IfA51OMx8ysKvVlstXf+ygi2iNiY7LcCbQBi4YVOw34cVLmUWCppGPSisnMrBpNlJpCgaSl5KfmXDds02bgd5MyZwPHAYtLEZOZWbWYUElB0gxgNXB1RBwctvmzwGxJm4CPAL8gf9lp+GuslLRe0vo9e/akHbKZWUUpZUNzXZovLqmefEK4JSLWDN+eJIkrk7ICtiWP4eVWAasAVqxYEWnGbGZWafomwiipyZf8TUBbRFw3QplZkhqSp38E/KRIbcLMbFLrz+SYUl/9NYVzgSuALcnlIcj3NmoFiIgbgFOBf5OUBbYC708xHjOzqhMR9GdLd/NaakkhItYCo04TFBE/A05MKwYzs2qXyQURTJyGZjMzO3KF+ZmdFMzMrJAUqr2h2czMXr7+7GBNwUNnm5lNer58ZGZmBX1OCmZmNshtCmZmVjDYpjDFNQUzM3ObgpmZFTgpmJlZQX82C7hNwczMcE3BzMyGcJdUMzMr6HOXVDMzGzR4+chdUs3MzG0KZmb2ghcGxKvypCBpiaS7JbVJekTSVUXKNEu6TdLmpMyVacVjZlaNSj3MRZrTcWaAayJio6QmYIOkuyJi65AyHwK2RsTbJbUAj0m6JSL6U4zLzKxq9Gdy1NaIumpvaI6I9ojYmCx3Am3AouHFgCZJAmYAz5NPJmZmRv7yUalqCZBuTaFA0lJgObBu2KbrgVuB54Am4NKIyJUiJjOzatCfyZWsPQFK0NAsaQawGrg6Ig4O23whsAk4FjgTuF7SzCKvsVLSeknr9+zZk3bIZmYVo28iJQVJ9eQTwi0RsaZIkSuBNZH3BLANOGV4oYhYFRErImJFS0tLmiGbmVWU/kxpLx+l2ftIwE1AW0RcN0KxXwFvSsofA5wMPJVWTGZm1aY/myvZjWuQbpvCucAVwBZJm5J11wKtABFxA/Ap4GZJWwABH4uIvSnGZGZWVfoz2ZJePkotKUTEWvJf9KOVeQ64IK0YzMyqXV+mtDUF39FsZlbBevqzTG2oLdnxnBTMzCpYz0CWqfVOCmZmhmsKZmY2RL6mUJL7jAEnBTOzipavKbih2czMcJuCmZklIiKfFBp8+cjMbNLry+SIwDUFMzPLtycATK13m4KZ2aTXPZBPCtN8+cjMzAZrCo2+T8HMzHoHBi8fOSmYmU163f2Dl4+cFMzMJr2epKbQ6JqCmZm90PtoAiQFSUsk3S2pTdIjkq4qUuajkjYlj4clZSXNSSsmM7Nq0jOQASbO5aMMcE1EnAq8FviQpNOGFoiIz0XEmRFxJvBXwL0R8XyKMZmZVY2e/hzAxBglNSLaI2JjstwJtAGLRtnlcuDracVjZlZtuvvzNYUJ16YgaSmwHFg3wvZpwFuA1aWIx8ysGvQOTMDeR5JmkP+yvzoiDo5Q7O3AT0e6dCRppaT1ktbv2bMnrVDNzCpKz0CWuhpRXztBhrmQVE8+IdwSEWtGKXoZo1w6iohVEbEiIla0tLQc7TDNzCpSd4lnXYN0ex8JuAloi4jrRinXDLwB+F5asZiZVaPeEs+lAJDmKEvnAlcAWyRtStZdC7QCRMQNybqLgTsjoivFWMzMqk6p52eGFJNCRKwFNIZyNwM3pxWHmVm16u4vfU3BdzSbmVWo/KxrTgpmZkZ52hScFMzMKlR3f7ak9yiAk4KZWcXqGciW9G5mGGNSkHSVpJnKu0nSRkkXpB2cmdlk1lvBDc3vS+5GvgBoAa4EPptaVGZmRvdA5V4+Guxa+lbg/0XEZsbQ3dTMzI5cT3+2pPMzw9iTwgZJd5JPCndIagJy6YVlZja5ZXNBXyZXsXc0vx84E3gqIrqTiXCuTC8sM7PJrRwjpMLYawrnAI9FxAFJfwD8NdCRXlhmZpPb4PzMldrQ/GWgW9IZwP8CngH+LbWozMwmucH5mSuySyqQiYgALgI+HxGfB5rSC8vMbHLrKVw+SnPc0pca69E6Jf0V+VFPXy+pFqhPLywzs8ltsKYwtaG09xiP9WiXAn3k71fYSX6u5c+lFpWZ2STXPZgU6ktbUxhTUkgSwS1As6TfBnojwm0KZmYpGex9VJGjpEp6F/AA8HvAu4B1ki45zD5LJN0tqU3SI5KuGqHc+ZI2JWXuHe8JmJlNROXqfTTWesnHgbMiYjeApBbgv4Bvj7JPBrgmIjYmN7ttkHRXRGwdLCBpFvAl4C0R8StJ84/oLMzMJpjBy0eVep9CzWBCSOw73L4R0R4RG5PlTqCNfFvEUL8PrImIXyXldmNmZoWaQqm7pI61pvAjSXcAX0+eXwr8YKwHkbQUWA6sG7bpJKBe0j3ku7h+3m0VZmbQ2TsAQFNjBXZJjYiPSnoncC75gfBWRcR3xrKvpBnAauDqZKTV4cd/DfAmYCrwM0k/j4hfDnuNlcBKgNbW1rEc1sysqh3oHqCxvqZiawpExGryX+5jJqk+2eeWiFhTpMh2YG9EdAFdkn4CnAG8KClExCpgFcCKFStiPDGYmVWjA939zJraUPLjjtouIKlT0sEij05Jw//rH76vgJuAtoi4boRi3yN/M1ydpGnAr5NvezAzm9T2dw8wa1rp7xEetaYQES9nKItzyd8BvUXSpmTdtUBr8to3RESbpB8BD5EfivvGiHj4ZRzTzGxCONDdX3lJ4eWIiLWMYSKeiPgcvjvazOxFDnQPsGz+jJIft7SDapiZ2ZjkLx9VWJuCmZmVXkTQ0VOey0dOCmZmFaarP8tANpjtpGBmZvu7+gEqr0uqmZmVXkdP/m5mXz4yMzP2d+drCrOnu6ZgZjbpHehOagpTXVMwM5v0DiQ1BXdJNTOzQk2h2TUFMzPb3z3AjCl1NNSV/ivaScHMrMIc6O4vSy0BnBTMzCrOgZ4BZk93UjAzM/JdUmeXoZEZnBTMzCpOR/eALx+ZmVnehKwpSFoi6W5JbZIekXRVkTLnS+qQtCl5fCKteMzMqkEuF3T0lGfWNUhxkh0gA1wTERslNQEbJN0VEVuHlbsvIn47xTjMzKpGZ2+GXJTnxjVIsaYQEe0RsTFZ7iQ/9/KitI5nZjYRDI57VI4hLqBEbQqSlgLLgXVFNp8jabOkH0p6ZSniMTOrVLs7+wCYP3NKWY6f5uUjACTNAFYDV0fEwWGbNwLHRcQhSW8FvgucWOQ1VgIrAVpbW1OO2MysfNo7egBY2NxYluOnWlOQVE8+IdwSEWuGb4+IgxFxKFn+AVAvaV6RcqsiYkVErGhpaUkzZDOzstrZ0QvAguapZTl+mr2PBNwEtEXEdSOUWZCUQ9LZSTz70orJzKzStXf00jSljhlTUr+QU1SaRz0XuALYImlTsu5aoBUgIm4ALgE+KCkD9ACXRUSkGJOZWUXb2dHLgjJdOoIUk0JErAV0mDLXA9enFYOZWbVpP1jepOA7ms3MKsjOjp6yNTKDk4KZWcUYyObY3dlXtkZmcFIwM6sYuzv7iChfd1RwUjAzqxg7k3sU3KZgZma0J/couKZgZmaFG9cWznSbgpnZpNfe0cvU+lpmTi3PjWvgpGBmVjF2dvSysLmRZKCHsnBSMDOrEO0dPWVtZAYnBTOzirHjgJOCmZkBuzt72XWwj9MWzixrHE4KZmYV4KFnOwA4ffGsssbhpGBmVgEe2n6AGsGvLXJNwcxs0tu8vYOTjmliWkP5uqOCk4KZWdlFBA9tP8Dpi5vLHYqTgplZuW3f38P+7oGytydAutNxLpF0t6Q2SY9IumqUsmdJykq6JK14zMwq1ebtBwA4owKSQpoXrzLANRGxUVITsEHSXRGxdWghSbXAPwB3pBiLmVnFemh7Bw21NZy8oKncoaRXU4iI9ojYmCx3Am3AoiJFPwKsBnanFYuZWSVb+/hezmydRUNd+a/olyQCSUuB5cC6YesXARcDN5QiDjOzSrPrYC9b2w9y/skt5Q4FKEFSkDSDfE3g6og4OGzzPwMfi4jsYV5jpaT1ktbv2bMnrVDNzEru3sfy32lvPHl+mSPJS7VDrKR68gnhlohYU6TICuA/kxEB5wFvlZSJiO8OLRQRq4BVACtWrIg0YzYzK6W7H9vNgpmNnFIB7QmQYlJQ/pv+JqAtIq4rViYijh9S/mbg9uEJwcxsohrI5lj7+F7edvrCsg6XPVSaNYVzgSuALZI2JeuuBVoBIsLtCGY2qa1/ej+dfZmKaU+AFJNCRKwFxpz6IuK9acViZlaJbnvoORrrazjvxMpJCuXv/2RmNgn1ZbJ8/6F2LnzlAmZMKe94R0M5KZiZlcHdj+6mo2eAi5cXu32rfJwUzMzKYPXGHbQ0TeG8ZfPKHcqLOCmYmZXYzo5e7nlsNxedcSx1tZX1NVxZ0ZiZTQJf+O/HAXjP65aWN5AinBTMzEro6b1dfPPBZ7n87FaWzJlW7nBewknBzKxEIoLP/vBR6mtr+PBvLit3OEU5KZiZlchNa7fxo0d28uHfXMb8psZyh1OUk4KZWQnc/8RePv2DNi447Rg++IYTyh3OiJwUzMxS9nxXP1d/YxOvaJnBdZeeSU1NZYxzVIyTgplZiiKCv1z9EPu7+/n8ZWdW1N3LxTgpmJmlpKNngI98/RfcuXUXH73wZF55bHO5Qzqsyk5ZZmZV6pe7OnnfzQ+ys6OXj154Mn903ivKHdKYOCmYmR0lEcGvnu/m/if38envtzG1oZZvfeAclrfOLndoY+akYGb2MnX1Zfjepuf42s+fYWt7ftbhUxfO5Kb3rODYWVPLHN34pDnz2hLg34AFQA5YFRGfH1bmIuBTyfYM+Xmc16YVk5nZ0dI7kOX+J/dy19Zd3La5nUN9GU5Z0MQn334arz1hLifOb6K2gnsZjSTNmkIGuCYiNkpqAjZIuisitg4p82Pg1ogISacD3wROSSOYtY/v5e++v5U50xuYPb2BOdPyP5un1tPVl6GjZ4BsLshFkM0FmWwwkM3Rn829aHkgm2Mgmy/TWF/DtIY6pjbUMqW2htoavehRI1E3bF1tjaiVqKkZxza9tFwEBPnqagAEBJFfP2xbRAxZN6TcON6/kX61ayTqa0V9XQ0NtTXU1Yi62hoGZxYUFKYZHHwNKb/f8PMeXK6reeE9KPwc8l5WyrSFNrn0DmS557E9tLUf5OEdHfz0yb30DuSY3lDLha9cwLtfexyvbp1V9b+fac681g60J8udktqARcDWIWUODdllOuP7nhqXKfU1LJkzjf1d/bS1H2R/Vz8HegaI5IjTGmpf9AVVX1tDXa1oqK2hvraG+rr8uvraGqbW1yJB30CO3Z29dPdl6c/myOWCTJJYMrl84hh8DK6L1M5w8qgR1NXUUFOT/BTU1daMmIQLSSX5PBvqaphSl//ZUFf7wnJtfv0L22qYUldLY30NTY31zJhSR1NjHU2N9cxMfjbW11T9l4CNLCLY+Kv9fHvDdm7f3E5nXwYJls6dzmVntfKmU+dz9vFzmFJXW+5Qj5qStClIWgosB9YV2XYx8BlgPvC2tGI4a+kczlo650XrMtkch/oyTGuoo6GuNL1zc7kgG0MSRsQLySQ3LJlE8XXZXCT/gQMI6YX/yAfXa3D9SMtD9j+8kTNZNkdSe8rXoAaXC3vF4CvkFwZrMrkonjyzw9+f5PGSciOUeWlizr1oe38mR38mR2dvJr+czdE3kE/qfZn8oz+TG/PnWVejQqJ4adLIL8+fOYVjm6eyaPZUlsyZVvH91CejTDbH9v09bNvbxVN7u3h8VyeP7erklzs76erPMrW+lt961QIuefViXn3cbBrrJ04SGE6R8r+ukmYA9wJ/HxFrRin3G8AnIuLNRbatBFYCtLa2vuaZZ55JK1wzIoKBbBQSRs9AlkN9GTp7M3T2DtDZm+Fgb4ZDQ54Xfg4r19k7QG7Yn9jx86Zz3NxpzJnWwInHNHHmklmcvriZ6U4WqYoIDvZkaD/YQ3tHL7s6ennm+W7WPbWPLTs6GMi+8EHNnlbPyQuaOPmYJs5YMosLKmzKzCMhaUNErDhsuTSTgqR64Hbgjoi4bgzltwFnRcTekcqsWLEi1q9ffxSjNEtPLhfs7epjx/4edhzoYdueLh557iA7DvSw71Afz3X0AvlLYicd08Ty1lmcuWQWy1tns6xlRkUPh1Bpegey7DjQw/b9PWzf3832/T3s7+rnYO8A2/Z28/TeLnoGsi/ap7ZGnL64mbOPn8OylhkcP286S+dNZ+70hgl3WXCsSSHN3kcCbgLaRkoIkpYBTyYNza8GGoB9acVkVmo1NWJ+UyPzmxqL9lV/vqufzc8e4BfPHmDTswf4/kPtfP2BZwGYO72Bc5fN452vWczrl82btAkik82x82Avuw720tOfI5PLkYtgb2c/G3+1n8d3H+LZ57vZ3dn3ov3qa8XsaQ3MaKyjdc40znnFXI6d1ciC5kYWNjeyoHkq85umUF9hM5+VW2o1BUnnAfcBW8h3OQW4FmgFiIgbJH0M+ENgAOgBPnq4LqmuKdhElssF2/Z1sfGZ/fzsqX3c/ehu9ncP0NI0hdedMJeLzjyWN548f8L9FwtwqC/Dk7sP8eSe5LG7iyf2HOKZfV0vurQzVPPUek5d2MSS2dNYMmcai2dPZfHsaSyZM5X5TY1V2SU0LRVx+SgNTgo2mfRlstzxyC7+a+sufvrEXvZ19XPKgiY+eP4JvO1VCytuft/DiQiefb6Hx3d3sm1vF0/u6WLb3kNs29vFroMv/KdfVyOOmzuNE1pmcML8GRw3ZxrHNDcyY0pdoZdZU2MdS+dOn7Q1qPFyUjCbYAayOW7d9BxfvvdJnth9iNY501j5G6/gktcsrtjeMO0dPWx4Zj9bdnTw8I4OHt5xkI6egcL2OdMbOH7e9MJj2fwZnNAyg+PmTvNlnaPMScFsgsrlgrvadvGle55k87MHmDdjCu8/73guPWsJc6Y3lDs89h3q495f7uE7v9jBfY/n+4zU14pTFszk1xY186pFzZy8oIkTWqYza1r5450snBTMJriI4GdP7ePL9zzJfY/vpbZGnLtsHle+binnn9xS8naHA939fPoHbXxrw3Yi4NjmRt511hLedMoxnLRgxoS6wasaOSmYTSJt7Qe5dfNzfGfjDnYe7GXRrKmcvriZM5fM4rwT53HawpmpJYn2jh6+9vNn+PoDz9LRM8B7zlnKxcsX8cpjZ/p6fwVxUjCbhAayOW7b/Bx3PrKLR3ce5Ol93QAsmz+DP/j1Vi47u/Wotj9ksjne8Ll7aO/o4Y0nz+fPLzipKiaSmYycFMyMnR293P3Ybv7zwWfZ/OwBFjY3cvWbT+Sdr158VHou/fSJvbz7xnV88fLlvP2MY49CxJaWsSYFN++bTWALmhu5/OxWvvehc/n6H7+WY2Y28rHVW7jwn3/Cjx5u5+X+U3jb5ueY3lDL/zjtmKMUsZWbk4LZJHHOCXP5zp+8jq9c8Rok8YGvbeQdX7qf+58ccVSZUfVncvzw4Z1c8MoFFdsl1sbPScFsEpHEha9cwI+uej3/+M7T2X2wl9//13VccdM6tmzvGNdrrX1iDx09A7z9jIUpRWvl4KRgNgnV1dbwrrOWcPdfnM/H33oqD+/o4O3Xr+WDX9vAE7s7R923L5Plq/c/zcdWb2H2tHrOW9ZSoqitFKp7LFgze1ka62v54994BZedvYQb79vGTWu3cccjO7l4+WKufvOJLJkzrVA2IvjWhu38812/5LmOXs5eOodr33ZqyeYisdJw7yMzK3i+q58b7n2Sr97/NLkILj+7lQ+/cRktTVP4pzsf41/ufpIzl8zimgtO4rxl8ybkwHwTlbukmtkR29nRyxf/+3G+8eCz5CJ4RcsMnth9iMvPXsLfv+NVvimtCjkpmNnL9sy+LtZs3MHPntzH8uNm8bELT3FCqFJOCmZmVlD2m9ckLZF0t6Q2SY9IuqpImXdLeih53C/pjLTiMTOzw0uz91EGuCYiNkpqAjZIuisitg4psw14Q0Tsl/RbwCrg11OMyczMRpFaUoiIdqA9We6U1AYsArYOKXP/kF1+DixOKx4zMzu8knQwlrQUWA6sG6XY+4EfliIeMzMrLvWb1yTNAFYDV0fEwRHKvJF8UjhvhO0rgZUAra2tKUVqZmap1hQk1ZNPCLdExJoRypwO3AhcFBH7ipWJiFURsSIiVrS0+JZ6M7O0pNn7SMBNQFtEXDdCmVZgDXBFRPwyrVjMzGxs0rx8dC5wBbBF0qZk3bVAK0BE3AB8ApgLfCm5XT4zln60ZmaWjqq7eU3SHuCZce7WDIw2LvBI24utH75u6PNiy0PXzQOOZPD6w8U/WpnR4h3+/HDLRxr/aPEdbvvR/AzKEf/h4h3+PK3P4Gj+Do0W50jbJsrvEFTGZzDW50PXnxgRh58rNSIm/ANYdSTbi60fvm7o82LLw9atTyP+8ZzDeOMfdi5HFH+lfAbliL9SPoOj+Ts0WpyV+BlMtr/jUeI+7PEjYtLMp3DbEW4vtn74utsOs3y4Y4/FWF5jrOcw3vjHevzDmeifwWjbK+EzOJq/Q8PXjeV8/DtUur/joc/H/TtUdZePqpmk9VHFbSaOv/yq/RyqPX6YGOcwmslSU6gUq8odwMvk+Muv2s+h2uOHiXEOI3JNwczMClxTMDOzAicFMzMrcFIwM7MCJ4UKIWm6pA2SfrvcsRwJSadKukHStyV9sNzxjJekd0j6V0nfk3RBueM5EpJeIekmSd8udyxjlfzefzV5799d7njGqxrf88NxUniZJP1fSbslPTxs/VskPSbpCUl/OYaX+hjwzXSiHN3ROIeIaIuIDwDvAkraXe8oxf/diPhj4L3ApSmGW9RROoenIuL96UZ6eOM8l98Fvp28979T8mCLGE/8lfKeH01OCi/fzcBbhq6QVAv8C/BbwGnA5ZJOk/QqSbcPe8yX9Gbykw/tKnXwiZt5meeQ7PM7wFrgx6UN/+jEn/jrZL9Su5mjdw7ldjNjPBfyE2s9mxTLljDG0dzM2OOfcFKfT2Gii4ifJJMIDXU28EREPAUg6T/JDw3+GeAll4eS+SSmk/9l65H0g4jIpRr4EEfjHJLXuRW4VdL3gf9IL+KXHPdofAYCPgv8MCI2phvxSx2tz6ASjOdcgO3kE8MmKuSf1HHGv5UJpiI+hAloES/89wP5X/xFIxWOiI9HxNXkv0j/tZQJYRTjOgdJ50v6gqSvAD9IO7gxGFf8wEeANwOXSPpAmoGNw3g/g7mSbgCWS/qrtIMbp5HOZQ3wTklf5ugMJZGWovFX+Ht+RFxTSIeKrDvsXYIRcfPRD+WIjescIuIe4J60gjkC443/C8AX0gvniIz3HPYBlZLQhit6LhHRBVxZ6mCOwEjxV/J7fkRcU0jHdmDJkOeLgefKFMuRqvZzqPb4YWKcw6BqP5dqj3/MnBTS8SBwoqTjJTUAlwG3ljmm8ar2c6j2+GFinMOgaj+Xao9/zJwUXiZJXwd+Bpwsabuk90dEBvgwcAfQBnwzIh4pZ5yjqfZzqPb4YWKcw6BqP5dqj//l8oB4ZmZW4JqCmZkVOCmYmVmBk4KZmRU4KZiZWYGTgpmZFTgpmJlZgZOCpU7SoRIc43cON7R0Csc8X9LrjmC/5ZJuTJbfK+n6ox/d+ElaOny46CJlWiT9qFQxWek5KVjVSIYvLioibo2Iz6ZwzNHGBzsfGHdSAK4FvnhEAZVZROwB2iWdW+5YLB1OClZSkj4q6UFJD0n6myHrv6v8zHOPSFo5ZP0hSX8raR1wjqSnJf2NpI2Stkg6JSlX+I9b0s3JiK33S3pK0iXJ+hpJX0qOcbukHwxuGxamuEA0AAAEMElEQVTjPZI+Lele4CpJb5e0TtIvJP2XpGOSoZU/APyZpE2SXp/8F706Ob8Hi31xSmoCTo+IzUW2HSfpx8l782NJrcn6EyT9PHnNvy1W81J+BrPvS9os6WFJlybrz0reh82SHpDUlNQI7kvew43FajuSaiV9bshn9T+HbP4uUHWzpNkYRYQffqT6AA4lPy8AVpEfcbIGuB34jWTbnOTnVOBhYG7yPIB3DXmtp4GPJMt/AtyYLL8XuD5Zvhn4VnKM08iPgw9wCflhvWuABcB+4JIi8d4DfGnI89m8cPf/HwH/O1n+JPAXQ8r9B3BestwKtBV57TcCq4c8Hxr3bcB7kuX3Ad9Nlm8HLk+WPzD4fg573XeSH3Z98Hkz0AA8BZyVrJtJfmTkaUBjsu5EYH2yvBR4OFleCfx1sjwFWA8cnzxfBGwp9++VH+k8PHS2ldIFyeMXyfMZ5L+UfgL8qaSLk/VLkvX7yM/GtXrY66xJfm4gP51jMd+N/LwUWyUdk6w7D/hWsn6npLtHifUbQ5YXA9+QtJD8F+22EfZ5M3CaVBhleaakpojoHFJmIbBnhP3PGXI+/w7845D170iW/wP4pyL7bgH+SdI/ALdHxH2SXgW0R8SDABFxEPK1CuB6SWeSf39PKvJ6FwCnD6lJNZP/TLYBu4FjRzgHq3JOClZKAj4TEV950UrpfPJfqOdERLeke4DGZHNvRAyfprEv+Zll5N/hviHLGvZzLLqGLH8RuC4ibk1i/eQI+9SQP4eeUV63hxfO7XDGPDBZRPxS0muAtwKfkXQn+cs8xV7jz8hP/XpGEnNvkTIiXyO7o8i2RvLnYROQ2xSslO4A3idpBoCkRcrPLdwM7E8SwinAa1M6/lrys3zVJLWH88e4XzOwI1l+z5D1nUDTkOd3kh9JE4DkP/Hh2oBlIxznfvJDMkP+mv3aZPnn5C8PMWT7i0g6FuiOiK+Rr0m8GngUOFbSWUmZpqThvJl8DSIHXAEUa8C/A/igpPpk35OSGgbkaxaj9lKy6uWkYCUTEXeSv/zxM0lbgG+T/1L9EVAn6SHgU+S/BNOwmvxkKQ8DXwHWAR1j2O+TwLck3QfsHbL+NuDiwYZm4E+BFUnD7FaKzMgVEY8CzUmD83B/ClyZvA9XAFcl668G/lzSA+QvPxWL+VXAA5I2AR8H/i4i+oFLgS9K2gzcRf6//C8B75H0c/Jf8F1FXu9G8vMPb0y6qX6FF2plbwS+X2QfmwA8dLZNKpJmRMQhSXOBB4BzI2JniWP4M6AzIm4cY/lpQE9EhKTLyDc6X5RqkKPH8xPgoojYX64YLD1uU7DJ5nZJs8g3GH+q1Akh8WXg98ZR/jXkG4YFHCDfM6ksJLWQb19xQpigXFMwM7MCtymYmVmBk4KZmRU4KZiZWYGTgpmZFTgpmJlZgZOCmZkV/H9TjstNAWUZOQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fafdd5d1dd8>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r",
      " 98%|█████████▊| 192/196 [00:30<00:00,  6.39it/s, loss=10.1]"
     ]
    }
   ],
   "source": [
    "learn.sched.plot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "419db733bd1544da848214f87d983030",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>HBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=2), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " 15%|█▍        | 29/196 [00:03<00:18,  9.18it/s, loss=2.21] \n",
      " 16%|█▋        | 32/196 [00:03<00:17,  9.50it/s, loss=2.2] "
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Exception in thread Thread-10:\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/paperspace/anaconda3/envs/fastai/lib/python3.6/threading.py\", line 916, in _bootstrap_inner\n",
      "    self.run()\n",
      "  File \"/home/paperspace/anaconda3/envs/fastai/lib/python3.6/site-packages/tqdm/_tqdm.py\", line 144, in run\n",
      "    for instance in self.tqdm_cls._instances:\n",
      "  File \"/home/paperspace/anaconda3/envs/fastai/lib/python3.6/_weakrefset.py\", line 60, in __iter__\n",
      "    for itemref in self.data:\n",
      "RuntimeError: Set changed size during iteration\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss   accuracy                   \n",
      "    0      1.711504   1.737088   0.3824    \n",
      "    1      1.52142    1.558574   0.4381                     \n",
      "\n",
      "CPU times: user 1min 38s, sys: 2min 51s, total: 4min 29s\n",
      "Wall time: 44 s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[array([1.55857]), 0.4381]"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%time learn.fit(1e-1, 2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "800ec2126d1847a1be3ce641f495f068",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>HBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss   accuracy                   \n",
      "    0      1.456638   1.38682    0.5034    \n",
      "    1      1.357452   1.284294   0.5388                     \n",
      "    2      1.296569   1.239791   0.5547                     \n",
      "    3      1.264639   1.205657   0.5701                     \n",
      "\n",
      "CPU times: user 3min 21s, sys: 5min 45s, total: 9min 7s\n",
      "Wall time: 1min 27s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[array([1.20566]), 0.5701]"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%time learn.fit(1e-1, 4, cycle_len=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Refactored"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "class ConvLayer(nn.Module):\n",
    "    def __init__(self, ni, nf):\n",
    "        super().__init__()\n",
    "        self.conv = nn.Conv2d(ni, nf, kernel_size=3, stride=2, padding=1)\n",
    "        \n",
    "    def forward(self, x): return F.relu(self.conv(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "class ConvNet2(nn.Module):\n",
    "    def __init__(self, layers, c):\n",
    "        super().__init__()\n",
    "        self.layers = nn.ModuleList([ConvLayer(layers[i], layers[i + 1])\n",
    "            for i in range(len(layers) - 1)])\n",
    "        self.out = nn.Linear(layers[-1], c)\n",
    "        \n",
    "    def forward(self, x):\n",
    "        for l in self.layers: x = l(x)\n",
    "        x = F.adaptive_max_pool2d(x, 1)\n",
    "        x = x.view(x.size(0), -1)\n",
    "        return F.log_softmax(self.out(x), dim=-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn = ConvLearner.from_model_data(ConvNet2([3, 20, 40, 80], 10), data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "OrderedDict([('Conv2d-1',\n",
       "              OrderedDict([('input_shape', [-1, 3, 32, 32]),\n",
       "                           ('output_shape', [-1, 20, 16, 16]),\n",
       "                           ('trainable', True),\n",
       "                           ('nb_params', 560)])),\n",
       "             ('ConvLayer-2',\n",
       "              OrderedDict([('input_shape', [-1, 3, 32, 32]),\n",
       "                           ('output_shape', [-1, 20, 16, 16]),\n",
       "                           ('nb_params', 0)])),\n",
       "             ('Conv2d-3',\n",
       "              OrderedDict([('input_shape', [-1, 20, 16, 16]),\n",
       "                           ('output_shape', [-1, 40, 8, 8]),\n",
       "                           ('trainable', True),\n",
       "                           ('nb_params', 7240)])),\n",
       "             ('ConvLayer-4',\n",
       "              OrderedDict([('input_shape', [-1, 20, 16, 16]),\n",
       "                           ('output_shape', [-1, 40, 8, 8]),\n",
       "                           ('nb_params', 0)])),\n",
       "             ('Conv2d-5',\n",
       "              OrderedDict([('input_shape', [-1, 40, 8, 8]),\n",
       "                           ('output_shape', [-1, 80, 4, 4]),\n",
       "                           ('trainable', True),\n",
       "                           ('nb_params', 28880)])),\n",
       "             ('ConvLayer-6',\n",
       "              OrderedDict([('input_shape', [-1, 40, 8, 8]),\n",
       "                           ('output_shape', [-1, 80, 4, 4]),\n",
       "                           ('nb_params', 0)])),\n",
       "             ('Linear-7',\n",
       "              OrderedDict([('input_shape', [-1, 80]),\n",
       "                           ('output_shape', [-1, 10]),\n",
       "                           ('trainable', True),\n",
       "                           ('nb_params', 810)]))])"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "learn.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fe4104c61a234e75a4bd64b53fe39081",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>HBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=2), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss   accuracy                   \n",
      "    0      1.728346   1.639025   0.4117    \n",
      "    1      1.513297   1.399134   0.4903                     \n",
      "\n",
      "CPU times: user 1min 42s, sys: 2min 48s, total: 4min 31s\n",
      "Wall time: 44.2 s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[array([1.39913]), 0.4903]"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%time learn.fit(1e-1, 2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a4f3f7246fb54ba08a5e9702becaa95b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>HBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=2), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss   accuracy                   \n",
      "    0      1.322032   1.260205   0.5485    \n",
      "    1      1.274674   1.20203    0.5723                     \n",
      "\n",
      "CPU times: user 1min 38s, sys: 2min 52s, total: 4min 30s\n",
      "Wall time: 44.3 s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[array([1.20203]), 0.5723]"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%time learn.fit(1e-1, 2, cycle_len=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## BatchNorm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "class BnLayer(nn.Module):\n",
    "    def __init__(self, ni, nf, stride=2, kernel_size=3):\n",
    "        super().__init__()\n",
    "        self.conv = nn.Conv2d(ni, nf, kernel_size=kernel_size, stride=stride,\n",
    "                              bias=False, padding=1)\n",
    "        self.a = nn.Parameter(torch.zeros(nf, 1, 1))\n",
    "        self.m = nn.Parameter(torch.ones(nf, 1, 1))\n",
    "        \n",
    "    def forward(self, x):\n",
    "        x = F.relu(self.conv(x))\n",
    "        x_chan = x.transpose(0, 1).contiguous().view(x.size(1), -1)\n",
    "        if self.training:\n",
    "            self.means = x_chan.mean(1)[:, None, None]\n",
    "            self.stds  = x_chan.std (1)[:, None, None]\n",
    "        return (x-self.means) / self.stds * self.m + self.a"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "class ConvBnNet(nn.Module):\n",
    "    def __init__(self, layers, c):\n",
    "        super().__init__()\n",
    "        self.conv1 = nn.Conv2d(3, 10, kernel_size=5, stride=1, padding=2)\n",
    "        self.layers = nn.ModuleList([BnLayer(layers[i], layers[i + 1])\n",
    "            for i in range(len(layers) - 1)])\n",
    "        self.out = nn.Linear(layers[-1], c)\n",
    "        \n",
    "    def forward(self, x):\n",
    "        x = self.conv1(x)\n",
    "        for l in self.layers: x = l(x)\n",
    "        x = F.adaptive_max_pool2d(x, 1)\n",
    "        x = x.view(x.size(0), -1)\n",
    "        return F.log_softmax(self.out(x), dim=-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn = ConvLearner.from_model_data(ConvBnNet([10, 20, 40, 80, 160], 10), data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "OrderedDict([('Conv2d-1',\n",
       "              OrderedDict([('input_shape', [-1, 3, 32, 32]),\n",
       "                           ('output_shape', [-1, 10, 32, 32]),\n",
       "                           ('trainable', True),\n",
       "                           ('nb_params', 760)])),\n",
       "             ('Conv2d-2',\n",
       "              OrderedDict([('input_shape', [-1, 10, 32, 32]),\n",
       "                           ('output_shape', [-1, 20, 16, 16]),\n",
       "                           ('trainable', True),\n",
       "                           ('nb_params', 1800)])),\n",
       "             ('BnLayer-3',\n",
       "              OrderedDict([('input_shape', [-1, 10, 32, 32]),\n",
       "                           ('output_shape', [-1, 20, 16, 16]),\n",
       "                           ('nb_params', 0)])),\n",
       "             ('Conv2d-4',\n",
       "              OrderedDict([('input_shape', [-1, 20, 16, 16]),\n",
       "                           ('output_shape', [-1, 40, 8, 8]),\n",
       "                           ('trainable', True),\n",
       "                           ('nb_params', 7200)])),\n",
       "             ('BnLayer-5',\n",
       "              OrderedDict([('input_shape', [-1, 20, 16, 16]),\n",
       "                           ('output_shape', [-1, 40, 8, 8]),\n",
       "                           ('nb_params', 0)])),\n",
       "             ('Conv2d-6',\n",
       "              OrderedDict([('input_shape', [-1, 40, 8, 8]),\n",
       "                           ('output_shape', [-1, 80, 4, 4]),\n",
       "                           ('trainable', True),\n",
       "                           ('nb_params', 28800)])),\n",
       "             ('BnLayer-7',\n",
       "              OrderedDict([('input_shape', [-1, 40, 8, 8]),\n",
       "                           ('output_shape', [-1, 80, 4, 4]),\n",
       "                           ('nb_params', 0)])),\n",
       "             ('Conv2d-8',\n",
       "              OrderedDict([('input_shape', [-1, 80, 4, 4]),\n",
       "                           ('output_shape', [-1, 160, 2, 2]),\n",
       "                           ('trainable', True),\n",
       "                           ('nb_params', 115200)])),\n",
       "             ('BnLayer-9',\n",
       "              OrderedDict([('input_shape', [-1, 80, 4, 4]),\n",
       "                           ('output_shape', [-1, 160, 2, 2]),\n",
       "                           ('nb_params', 0)])),\n",
       "             ('Linear-10',\n",
       "              OrderedDict([('input_shape', [-1, 160]),\n",
       "                           ('output_shape', [-1, 10]),\n",
       "                           ('trainable', True),\n",
       "                           ('nb_params', 1610)]))])"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "learn.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1f1acb285f2c47aba9a6a1cd688a557d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>HBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=2), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss   accuracy                   \n",
      "    0      1.474475   1.421361   0.4984    \n",
      "    1      1.264842   1.144034   0.5881                     \n",
      "\n",
      "CPU times: user 1min 56s, sys: 3min 24s, total: 5min 21s\n",
      "Wall time: 47.6 s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[array([1.14403]), 0.5881]"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%time learn.fit(3e-2, 2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "93b61cc1195446a0ba7e55e8aeeac4e3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>HBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss   accuracy                   \n",
      "    0      1.168034   1.030074   0.6267    \n",
      "    1      1.030772   0.96697    0.6655                     \n",
      "    2      0.964813   0.872289   0.696                       \n",
      "    3      0.905667   0.837793   0.7079                      \n",
      "\n",
      "CPU times: user 3min 53s, sys: 6min 43s, total: 10min 36s\n",
      "Wall time: 1min 34s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[array([0.83779]), 0.7079]"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%time learn.fit(1e-1, 4, cycle_len=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Deep BatchNorm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "class ConvBnNet2(nn.Module):\n",
    "    def __init__(self, layers, c):\n",
    "        super().__init__()\n",
    "        self.conv1 = nn.Conv2d(3, 10, kernel_size=5, stride=1, padding=2)\n",
    "        self.layers = nn.ModuleList([BnLayer(layers[i], layers[i + 1])\n",
    "            for i in range(len(layers) - 1)])\n",
    "        self.layers2 = nn.ModuleList([BnLayer(layers[i + 1], layers[i + 1], 1)\n",
    "            for i in range(len(layers) - 1)])\n",
    "        self.out = nn.Linear(layers[-1], c)\n",
    "        \n",
    "    def forward(self, x):\n",
    "        x = self.conv1(x)\n",
    "        for l,l2 in zip(self.layers, self.layers2):\n",
    "            x = l(x)\n",
    "            x = l2(x)\n",
    "        x = F.adaptive_max_pool2d(x, 1)\n",
    "        x = x.view(x.size(0), -1)\n",
    "        return F.log_softmax(self.out(x), dim=-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn = ConvLearner.from_model_data(ConvBnNet2([10, 20, 40, 80, 160], 10), data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1d32e84eb0af4e2ea27310bb794b7c6a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>HBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=2), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss   accuracy                   \n",
      "    0      1.505403   1.369886   0.4972    \n",
      "    1      1.292517   1.193988   0.5743                     \n",
      "\n",
      "CPU times: user 2min 7s, sys: 3min 40s, total: 5min 47s\n",
      "Wall time: 50.9 s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[array([1.19399]), 0.5743]"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%time learn.fit(1e-2, 2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2305d35f716c481cbe358bda04d3604d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>HBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=2), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss   accuracy                   \n",
      "    0      1.114137   1.040168   0.6291    \n",
      "    1      1.034688   0.982892   0.6514                     \n",
      "\n",
      "CPU times: user 2min 5s, sys: 3min 42s, total: 5min 47s\n",
      "Wall time: 51.3 s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[array([0.98289]), 0.6514]"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%time learn.fit(1e-2, 2, cycle_len=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Resnet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "class ResnetLayer(BnLayer):\n",
    "    def forward(self, x): return x + super().forward(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Resnet(nn.Module):\n",
    "    def __init__(self, layers, c):\n",
    "        super().__init__()\n",
    "        self.conv1 = nn.Conv2d(3, 10, kernel_size=5, stride=1, padding=2)\n",
    "        self.layers = nn.ModuleList([BnLayer(layers[i], layers[i + 1])\n",
    "            for i in range(len(layers) - 1)])\n",
    "        self.layers2 = nn.ModuleList([ResnetLayer(layers[i + 1], layers[i + 1], 1)\n",
    "            for i in range(len(layers) - 1)])\n",
    "        self.layers3 = nn.ModuleList([ResnetLayer(layers[i + 1], layers[i + 1], 1)\n",
    "            for i in range(len(layers) - 1)])\n",
    "        self.out = nn.Linear(layers[-1], c)\n",
    "        \n",
    "    def forward(self, x):\n",
    "        x = self.conv1(x)\n",
    "        for l,l2,l3 in zip(self.layers, self.layers2, self.layers3):\n",
    "            x = l3(l2(l(x)))\n",
    "        x = F.adaptive_max_pool2d(x, 1)\n",
    "        x = x.view(x.size(0), -1)\n",
    "        return F.log_softmax(self.out(x), dim=-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn = ConvLearner.from_model_data(Resnet([10, 20, 40, 80, 160], 10), data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "wd = 1e-5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "aa9d90215bb948eb9691c7fbea8020fb",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>HBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=2), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss   accuracy                   \n",
      "    0      1.555916   1.497878   0.4593    \n",
      "    1      1.286486   1.179735   0.5811                     \n",
      "\n",
      "CPU times: user 2min 11s, sys: 3min 50s, total: 6min 2s\n",
      "Wall time: 56.1 s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[array([1.17973]), 0.5811]"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%time learn.fit(1e-2, 2, wds=wd)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5814d5dedcb54752bffe08266467f768",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>HBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=7), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss   accuracy                   \n",
      "    0      1.075466   1.038314   0.6273    \n",
      "    1      1.047439   0.991257   0.6495                     \n",
      "    2      0.931519   0.911734   0.6783                      \n",
      "    3      0.96682    0.917621   0.6752                      \n",
      "    4      0.860942   0.831846   0.7079                      \n",
      "    5      0.760845   0.758946   0.7312                      \n",
      "    6      0.723117   0.757247   0.7335                      \n",
      "\n",
      "CPU times: user 7min 40s, sys: 13min 20s, total: 21min\n",
      "Wall time: 3min 14s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[array([0.75725]), 0.7335]"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%time learn.fit(1e-2, 3, cycle_len=1, cycle_mult=2, wds=wd)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "aa1e415a596e4955bd199c07778152c2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>HBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=16), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss   accuracy                    \n",
      "    0      0.821457   0.828655   0.7095    \n",
      "    1      0.735682   0.729878   0.7426                      \n",
      "    2      0.67023    0.709068   0.7555                      \n",
      "    3      0.614033   0.68396    0.7587                      \n",
      "    4      0.72356    0.731519   0.7446                      \n",
      "    5      0.646835   0.666082   0.7637                      \n",
      "    6      0.587808   0.630712   0.7811                      \n",
      "    7      0.538311   0.63509    0.7782                      \n",
      "    8      0.653226   0.719071   0.7544                      \n",
      "    9      0.587378   0.638724   0.779                       \n",
      "    10     0.53147    0.606534   0.791                       \n",
      "    11     0.486855   0.574349   0.8018                      \n",
      "    12     0.60116    0.674546   0.7682                      \n",
      "    13     0.536271   0.590718   0.793                       \n",
      "    14     0.478524   0.577702   0.8039                      \n",
      "    15     0.439396   0.589477   0.7972                      \n",
      "\n",
      "CPU times: user 17min 51s, sys: 30min 39s, total: 48min 31s\n",
      "Wall time: 7min 37s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[array([0.58948]), 0.7972]"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%time learn.fit(1e-2, 4, cycle_len=4, wds=wd)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "## Resnet 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "class Resnet2(nn.Module):\n",
    "    def __init__(self, layers, c, p=0.5):\n",
    "        super().__init__()\n",
    "        self.conv1 = BnLayer(3, 16, stride=1, kernel_size=7)\n",
    "        self.layers = nn.ModuleList([BnLayer(layers[i], layers[i + 1])\n",
    "            for i in range(len(layers) - 1)])\n",
    "        self.layers2 = nn.ModuleList([ResnetLayer(layers[i + 1], layers[i + 1], 1)\n",
    "            for i in range(len(layers) - 1)])\n",
    "        self.layers3 = nn.ModuleList([ResnetLayer(layers[i + 1], layers[i + 1], 1)\n",
    "            for i in range(len(layers) - 1)])\n",
    "        self.out = nn.Linear(layers[-1], c)\n",
    "        self.drop = nn.Dropout(p)\n",
    "        \n",
    "    def forward(self, x):\n",
    "        x = self.conv1(x)\n",
    "        for l,l2,l3 in zip(self.layers, self.layers2, self.layers3):\n",
    "            x = l3(l2(l(x)))\n",
    "        x = F.adaptive_max_pool2d(x, 1)\n",
    "        x = x.view(x.size(0), -1)\n",
    "        x = self.drop(x)\n",
    "        return F.log_softmax(self.out(x), dim=-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "learn = ConvLearner.from_model_data(Resnet2([16, 32, 64, 128, 256], 10, 0.2), data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "wd = 1e-6"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {
    "hidden": true,
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9d68521acef0497fbb9e22097935f14f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>HBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=2), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss   accuracy                   \n",
      "    0      1.726595   1.476193   0.477     \n",
      "    1      1.511903   1.594056   0.5248                     \n",
      "\n",
      "CPU times: user 2min 33s, sys: 4min 11s, total: 6min 44s\n",
      "Wall time: 1min 7s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[array([1.59406]), 0.5248]"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%time learn.fit(1e-2, 2, wds=wd)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ec4b9952e70d4b18912edb32a7e0f371",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>HBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=7), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss   accuracy                   \n",
      "    0      1.246219   1.123039   0.5994    \n",
      "    1      1.205521   1.079559   0.6165                     \n",
      "    2      1.047397   0.982982   0.6518                     \n",
      "    3      1.11306    1.042084   0.643                      \n",
      "    4      0.986444   0.938702   0.6705                      \n",
      "    5      0.86359    0.827887   0.7107                      \n",
      "    6      0.820836   0.859191   0.6998                      \n",
      "\n",
      "CPU times: user 8min 53s, sys: 14min 50s, total: 23min 43s\n",
      "Wall time: 3min 58s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[array([0.85919]), 0.6998]"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%time learn.fit(1e-2, 3, cycle_len=1, cycle_mult=2, wds=wd)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6d855b76ada04385a95bfafe788e6faf",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>HBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=16), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss   accuracy                    \n",
      "    0      0.922289   0.922782   0.6783    \n",
      "    1      0.843655   0.815614   0.7125                      \n",
      "    2      0.733954   0.732746   0.7458                      \n",
      "    3      0.695225   0.732575   0.7457                      \n",
      "    4      0.826921   0.759739   0.7323                      \n",
      "    5      0.731842   0.704877   0.7553                      \n",
      "    6      0.642404   0.659563   0.7721                      \n",
      "    7      0.605025   0.728076   0.7616                      \n",
      "    8      0.72092    0.719592   0.7534                      \n",
      "    9      0.652721   0.653841   0.776                       \n",
      "    10     0.583139   0.606309   0.7903                      \n",
      "    11     0.535503   0.64212    0.7817                      \n",
      "    12     0.656404   0.654129   0.7783                      \n",
      "    13     0.587746   0.655965   0.7777                      \n",
      "    14     0.518367   0.601116   0.7925                      \n",
      "    15     0.489359   0.597911   0.7945                      \n",
      "\n",
      "CPU times: user 20min 17s, sys: 33min 52s, total: 54min 9s\n",
      "Wall time: 8min 58s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[array([0.59791]), 0.7945]"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%time learn.fit(1e-2, 4, cycle_len=4, wds=wd)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  },
  "toc": {
   "colors": {
    "hover_highlight": "#DAA520",
    "navigate_num": "#000000",
    "navigate_text": "#333333",
    "running_highlight": "#FF0000",
    "selected_highlight": "#FFD700",
    "sidebar_border": "#EEEEEE",
    "wrapper_background": "#FFFFFF"
   },
   "moveMenuLeft": true,
   "nav_menu": {
    "height": "266px",
    "width": "252px"
   },
   "navigate_menu": true,
   "number_sections": true,
   "sideBar": true,
   "threshold": 4,
   "toc_cell": false,
   "toc_section_display": "block",
   "toc_window_display": false,
   "widenNotebook": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}