{ "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "# Lesson 2 - Image Classification Models from Scratch" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "#Run once per session\n", "!pip install fastai -q --upgrade" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Grab our vision related libraries" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "from fastai.vision.all import *" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "And our data" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "data": { "text/html": [ "\n", "
\n", " \n", " \n", " 100.00% [15683414/15683414 00:00<00:00]\n", "
\n", " " ], "text/plain": [ "" ] }, "metadata": { "tags": [] }, "output_type": "display_data" } ], "source": [ "path = untar_data(URLs.MNIST)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Working with the data" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "items = get_image_files(path)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "PosixPath('/root/.fastai/data/mnist_png/training/0/12497.png')" ] }, "execution_count": null, "metadata": { "tags": [] }, "output_type": "execute_result" } ], "source": [ "items[0]" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Create an image object. Done automatically with `ImageBlock`." ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "im = PILImageBW.create(items[0])" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "" ] }, "execution_count": null, "metadata": { "tags": [] }, "output_type": "execute_result" }, { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAAOcAAADnCAYAAADl9EEgAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjIsIGh0\ndHA6Ly9tYXRwbG90bGliLm9yZy8li6FKAAAEFElEQVR4nO3d0U1iQRiA0cvGXrQPO9AupAMLsAPt\nQjugD6yGfdqnxTvuwoUPOefReXAS8mUSJv+w2u12E9Dz69wbAPYTJ0SJE6LECVHihKibwbqvcmF5\nq31/dHJClDghSpwQJU6IEidEiROixAlR4oQocUKUOCFKnBAlTogSJ0SJE6LECVHihChxQpQ4IUqc\nECVOiBInRIkTosQJUeKEKHFClDghSpwQJU6IEidEiROixAlR4oQocUKUOCFKnBAlTogSJ0SJE6LE\nCVHihChxQpQ4IUqcECVOiBInRIkTosQJUeKEKHFClDghSpwQJU6IEidEiROixAlR4oQocUKUOCFK\nnBAlTogSJ0TdnHsDHNfb29vs+mazmV3/+Pg45naO5uHhYXb95eVldv329vaY2zkJJydEiROixAlR\n4oQocUKUOCFKnBC12u12c+uzixzf4+Pj7Hr1HrJuu93Orp/5HnS1749OTogSJ0SJE6LECVHihChx\nQpQ4Ico85xnM3WWW7zEPnakczZKu1+t/3tN3jf53cd7TyQlR4oQocUKUOCFKnBAlTogSJ0SZ5/wP\no7dhl7yvO9Qhd5VL3wV+fn5+uXZ3d7fo/x50sDTznHBJxAlR4oQocUKUOCFKnBB1lVcpc1/ZT9M0\nPT8/z66fc6zr9fV1dv3p6elEOzmt1WrvbcPRuEoBvk2cECVOiBInRIkTosQJUeKEqKt8GnPp8aOR\nubGt0fOSxSccj2U0indtnJwQJU6IEidEiROixAlR4oQocULUxd5zlp+nHM1c3t/ff7l2zfeYS35m\no8+kyMkJUeKEKHFClDghSpwQJU6IEidEZd+tXfqd0kOMfkbv/f39RDs5rdE95WazmV1f8r3fC/9M\nvFsLl0ScECVOiBInRIkTosQJUeKEqIud51zSOX8Dc/TboaO7xHPeNS7pwu8x/4uTE6LECVHihChx\nQpQ4IUqcEGVkjIztdvvl2k9+MnQyMgaXRZwQJU6IEidEiROixAlR4oQoI2N822hsa+6nDadp2VG7\nn8jJCVHihChxQpQ4IUqcECVOiBInRGXvOedm+6Zp/ATker0+5nZOxl0ifzg5IUqcECVOiBInRIkT\nosQJUeKEqOy7tXBFvFsLl0ScECVOiBInRIkTosQJUeKEKHFClDghSpwQJU6IEidEiROixAlR4oQo\ncUKUOCFKnBAlTogSJ0SJE6LECVHihChxQpQ4IUqcECVOiBInRIkTosQJUeKEKHFClDghSpwQJU6I\nEidEiROixAlR4oQocUKUOCFKnBAlTogSJ0SJE6LECVHihChxQpQ4IUqcECVOiBInRIkTosQJUeKE\nKHFClDghSpwQJU6IEidEiROibgbrq5PsAviLkxOixAlR4oQocUKUOCFKnBD1G8pmimVDi7kyAAAA\nAElFTkSuQmCC\n", "text/plain": [ "
" ] }, "metadata": { "tags": [] }, "output_type": "display_data" } ], "source": [ "im.show()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Split our data with `GrandparentSplitter`, which will make use of a `train` and `valid` folder." ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "splits = GrandparentSplitter(train_name='training', valid_name='testing')" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "(#3) [/root/.fastai/data/mnist_png/training/0/12497.png,/root/.fastai/data/mnist_png/training/0/5475.png,/root/.fastai/data/mnist_png/training/0/33023.png]" ] }, "execution_count": null, "metadata": { "tags": [] }, "output_type": "execute_result" } ], "source": [ "items[:3]" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Splits need to be applied to some items" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "splits = splits(items)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "([0, 1, 2, 3, 4], [60000, 60001, 60002, 60003, 60004])" ] }, "execution_count": null, "metadata": { "tags": [] }, "output_type": "execute_result" } ], "source": [ "splits[0][:5], splits[1][:5]" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "* Make a `Datasets`\n", "\n", "* Expects items, transforms for describing our problem, and a splitting method" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "dsrc = Datasets(items, tfms=[[PILImageBW.create], [parent_label, Categorize]], \n", " splits=splits)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "We can look at an item in our `Datasets` with `show_at`" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "" ] }, "execution_count": null, "metadata": { "tags": [] }, "output_type": "execute_result" }, { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAAOcAAAD3CAYAAADmIkO7AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjIsIGh0\ndHA6Ly9tYXRwbG90bGliLm9yZy8li6FKAAAHQ0lEQVR4nO3dP4iU2x3H4fPqgobEK4pwEcEuKTSg\nhYhYCCv4D6wEFQM2msIErUSIoqCwlcViY6GdLGhAMLAYxUIRGxELuUhA7ERESbGycQMadCdFrpB7\n7+4Z3Z3Z+c76POX8eGcOwsdX5njeaVqtVgHyLOj1AoCpiRNCiRNCiRNCiRNCiRNCiRNCiXOeaJpm\nedM0f2ua5t9N07xomuYPvV4TszPQ6wXQMRdLKf8ppXxfSllfSvl70zQ/tFqtf/R2WcxU438I9b+m\naX5dSnlbSvl9q9V6/uNrI6WUV61W6y89XRwz5p+188PvSikfP4f5ox9KKWt7tB46QJzzw29KKf/6\n2WvjpZQlPVgLHSLO+WGilPLdz177rpTyrgdroUPEOT88L6UMNE3z2/97bV0pxZdBfcwXQvNE0zR/\nLaW0Sil/LP/7tvZWKWWzb2v7lzvn/PHnUsqvSin/LKVcK6X8SZj9zZ0TQrlzQihxQihxQihxQqh2\n//Hdt0XQfc1UL7pzQihxQihxQihxQihxQihxQihxQihxQihxQihxQihxQihxQihxQihxQihxQihx\nQihxQihxQihxQihxQihxQihxQihxQihxQihxQihxQihxQihxQihxQihxQihxQihxQihxQihxQihx\nQihxQihxQihxQqiBXi+An5qYmKjOHz9+XJ1funSpOh8eHq7OR0ZGpp0dPHiweu2SJUtmNeen3Dkh\nlDghlDghlDghlDghlDghlDghVNNqtWrz6pCZuXHjxrSzixcvVq+9f/9+h1fTOVu2bKnOjxw5Up3v\n37+/k8vpJ81UL7pzQihxQihxQihxQihxQihxQihHxmbg5cuX1fmVK1eq83Pnzk07m5ycnNGaEjx4\n8KA6Hx8fr85Xrlw57azdNs185M4JocQJocQJocQJocQJocQJocQJoRwZm8KLFy+q88HBwVldz9QW\nLJj+XnH16tXqtXv37u30cuaSI2PQT8QJocQJocQJocQJocQJocQJob7J85ztzmPax+yN2lnWZ8+e\nzeFKMrhzQihxQihxQihxQihxQihxQihxQqhv8jznihUrqvO3b9/O0Ur4UosWLarOb968WZ1v3bq1\nk8vpNOc5oZ+IE0KJE0KJE0KJE0KJE0KJE0L17XnOT58+Vee138hs9zuR3TYwMP0f+9mzZ6vXjo6O\nVufHjh2rzp88eVKdDw8PV+e98uHDh+p837591fndu3er83Xr1n31mrrNnRNCiRNCiRNCiRNCiRNC\niRNC9e1WyuXLl6vzo0ePztFKfmnjxo3V+YkTJ6ad7dmzp3rtyZMnZ7Smz9qtrSZ1m6WU9sf83rx5\nU53bSgG+mDghlDghlDghlDghlDghlDghVN8+GnP79u3VebsjQt3Ubg/28OHDc7SSr1c7ivf+/ftZ\nvffr16+r86GhoWlnIyMjs/rsW7duVec7duyY1fvPkkdjQj8RJ4QSJ4QSJ4QSJ4QSJ4QSJ4SK3ed8\n+vRpdd7uJ93GxsZm/Nm1R1eWUsrp06er81OnTlXnCxcu/Oo1fQs+fvw47azdTwC2Y58T6BhxQihx\nQihxQihxQihxQihxQqjY59ZeuHChOp/NPmY7a9eurc7PnDnTtc+mO65fv16d93ifc0runBBKnBBK\nnBBKnBBKnBBKnBBKnBAq9jznggX1vzeaZsojcB3x6NGj6nzDhg1d++xvWTfPc7ZTe17vHHCeE/qJ\nOCGUOCGUOCGUOCGUOCFU7JGxXlq8eHGvl9A1te2KUuqP7Zzt9tXk5GR1fu3atVm9f027x5kmcueE\nUOKEUOKEUOKEUOKEUOKEUOKEULH7nG2OsnX1yFi7xySeP3++a589W4ODg9X5iRMnqvNNmzZNO1u+\nfPmM1vTZ8+fPq/OhoaFZvX/N5s2bu/be3eLOCaHECaHECaHECaHECaHECaHECaFiH415586d6vzA\ngQPV+fj4eCeX0zdWrVpVnb969WqOVjK3li5dWp3fu3evOl+/fn0nl/O1PBoT+ok4IZQ4IZQ4IZQ4\nIZQ4IZQ4IVTsPmc7u3fvrs5v3749RythLtSep1tKKaOjo9X5zp07O7mcTrPPCf1EnBBKnBBKnBBK\nnBBKnBCqb7dS3r17V53v2rVr2tnDhw87vRy+wOrVq6vzQ4cOTTvbtm1b9draIz37gK0U6CfihFDi\nhFDihFDihFDihFDihFB9u8/ZTm0fdGJionrt2NhYdX78+PEZranftfv5wDVr1lTnixcvrs6XLVv2\n1WuaJ+xzQj8RJ4QSJ4QSJ4QSJ4QSJ4QSJ4Sat/uc0Efsc0I/ESeEEieEEieEEieEEieEEieEEieE\nEieEEieEEieEEieEEieEEieEEieEEieEEieEEieEEieEEieEEieEEieEEieEEieEEieEEieEEieE\nEieEEieEEieEEieEEieEEieEEieEEieEEieEEieEEieEGmgzb+ZkFcAvuHNCKHFCKHFCKHFCKHFC\nKHFCqP8CcLlCLaKitvEAAAAASUVORK5CYII=\n", "text/plain": [ "
" ] }, "metadata": { "tags": [] }, "output_type": "display_data" } ], "source": [ "show_at(dsrc.train, 3)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "We can see that it's a `PILImage` of a zero, along with a label of `0`" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Next we need to give ourselves some transforms on the data! These will need to:\n", "1. Ensure our images are all the same size\n", "2. Make sure our output are the `tensor` our models are wanting\n", "3. Give some image augmentation" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "tfms = [ToTensor(), CropPad(size=34, pad_mode=PadMode.Zeros), RandomCrop(size=28)]" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "* `ToTensor`: Converts to tensor\n", "* `CropPad` and `RandomCrop`: Resizing transforms\n", "* Applied on the `CPU` via `after_item`" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "gpu_tfms = [IntToFloatTensor(), Normalize()]" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "* ``: Enables GPU usage\n", "* `IntToFloatTensor`: Converts to a float\n", "* `Normalize`: Normalizes data" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "dls = dsrc.dataloaders(bs=128, after_item=tfms, after_batch=gpu_tfms)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "And show a batch" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAAgAAAAIHCAYAAADpfeRCAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjIsIGh0\ndHA6Ly9tYXRwbG90bGliLm9yZy8li6FKAAAgAElEQVR4nO3de3RW5ZXH8f0QIsj9JqAiIF2gIjMo\n0harHbG2lUtRhFVFW2/DRWC0OAzWsWqrSIc66pJVBG/1UosWUaFFsbQqooKlKhrtwDgIFhFBESxE\nLkKAZ/6IXat178BJ3uQ952R/P2v5B7+c854dfJLsHPb7nBBjFAAA4EuDtAsAAADFRwMAAIBDNAAA\nADhEAwAAgEM0AAAAOEQDAACAQzQAAAA4RANQAyGEWSGEjSGE8hDCqhDCqLRrApIKITQKIdwXQngv\nhPBpCKEshDAw7bqA6gohdA0hPB1C+GsI4cMQwh0hhIZp15UXNAA1M1VEusYYW4jIWSIyJYRwUso1\nAUk1FJH3ReQ0EWkpIteJyJwQQtcUawJqYqaIbBKRw0XkBKlc0+NTrShHaABqIMa4Isa4+29//Py/\nL6VYEpBYjHFHjPGGGOPaGOP+GONTIvIXEaGJRd4cLSJzYoyfxRg/FJGFInJ8yjXlBg1ADYUQZoYQ\ndorI2yKyUUSeTrkkoEZCCB1EpIeIrEi7FqCaponIiBBCkxDCkSIyUCqbACRAA1BDMcbxItJcRL4u\nInNFZPeBzwCyJ4RQKiIPi8gvY4xvp10PUE0vSuVv/OUisl5EXhOR36RaUY7QABQgxrgvxrhERDqJ\nyLi06wGqI4TQQER+JSJ7ROTylMsBquXz9btQKn8Bayoi7USktYjcnGZdeUIDUDsaCjMAyJEQQhCR\n+0Skg4gMjzFWpFwSUF1tRKSziNwRY9wdY9wiIg+IyKB0y8oPGoBqCiG0DyGMCCE0CyGUhBDOFJHz\nReS5tGsDquFOETlORIbEGHelXQxQXTHGzVI5vDouhNAwhNBKRC4WkbfSrSw/Qowx7RpyJYRwmIg8\nLiK9pbKBek9Efh5jvDfVwoCEQghdRGStVM6t7P27D10WY3w4laKAGgghnCCVg4C9RWSfiCwSkSti\njB+lWlhO0AAAAOAQ/wQAAIBDNAAAADhEAwAAgEM0AAAAOEQDAACAQwd7bCJvEUAhQtoFCGsYhcnC\nGhZhHaMw5jrmDgAAAA7RAAAA4BANAAAADtEAAADgEA0AAAAO0QAAAOAQDQAAAA7RAAAA4BANAAAA\nDtEAAADgEA0AAAAO0QAAAOAQDQAAAA7RAAAA4BANAAAADtEAAADgEA0AAAAO0QAAAOAQDQAAAA41\nTLsAAMUxY8YMlU2ZMkVlo0aNUtnll1+usg4dOtROYYCIlJWVqeyWW25R2ezZs2t8jf3796usQQP7\n9+BJkyapbNy4cSrr2rVrjetJG3cAAABwiAYAAACHaAAAAHCIBgAAAIdCjPFAHz/gB/Nk4cKFZv7O\nO++obMKECSobMmSIykaOHKmyU045RWVt27ZNUmJ9FNIuQOrRGi5Unz59VGYNRb311lsqa9Kkicre\nfvtt8zqdOnWqQXWZlYU1LJLTdVxeXq6ySy+91Dx28eLFKtu2bVut1mP9vAsh+f/ijh07qmz48OEq\n+9nPfqayQw89NPF16oD5SXIHAAAAh2gAAABwiAYAAACHaAAAAHAo9zsBWkN8F110kcpee+0183xr\nCMoaCnnyySdV9tRTT6nszjvvVNmYMWPMawN1Zd26dSpr3769yhYsWKCyDRs2qOzoo49W2dy5c81r\nW7sGVrXbGvLJ+r752GOPqWzmzJkqW7p0aeLrWINzjRs3Tnz+F1nD3CtWrDCPHTZsmMqsIe9zzz1X\nZXv27FGZ9bMhbXxVAgDgEA0AAAAO0QAAAOAQDQAAAA5lYgjQ2p3p5ZdfVtmcOXNUNn/+fJVZA1DV\nYe16tmPHjkTnWrsIHn/88SqzhkmA2tK5c2eVWTv0lZSUqOyoo45S2YgRI1R25ZVXmte2hqfq2e6A\n7r377rsq+973vlfQa44dO1ZlP/jBD1TWo0ePgq5TCOvzrqioUFm7du2KUU7BuAMAAIBDNAAAADhE\nAwAAgEM0AAAAOJSJxwF/9tlnKmvatGmtXuPLX/6ymVvDTUOHDlVZt27dVJb0MZK33XabyqoaoKpn\nsvAo1Vw+RrUuWMNKpaWlic7dvHmzyqxhQRF7x7NLLrkk0XUyKAtrWCRj63j16tUqO/bYYxOdO2XK\nFDP/j//4D5UlXZ/FsmzZMpXdfffdKrvnnntUlvLnwuOAAQBAJRoAAAAcogEAAMAhGgAAABwq+k6A\nu3btUtkVV1xR49ezdu0bPny4yqZPn26e37x5c5Vt3LixxvUMGjRIZdYOV0CxFTKEZO1sduaZZxZS\nDpyyHtcukvqQXCL9+vVTWdeuXVVm7bC5bds2lVmfs/Uzra5wBwAAAIdoAAAAcIgGAAAAh2gAAABw\nqOhDgB988IHKHnjggRq/3syZM1V24YUXJj7//fffV9kZZ5xR43p++MMfqqxx48Y1fj0gC/bv36+y\nPXv2pFAJ8m7AgAFm/txzz6nssMMOq+tyCrZw4UKVrV27VmXWDojWAPztt99eK3UlwR0AAAAcogEA\nAMAhGgAAAByiAQAAwCEaAAAAHCr6uwDat2+vsjvuuENll19+ucpOPPFElQ0ePDjRda13H4iInHba\naSp77733Er2mxXpHw2uvvZb4/O9///sqy8MkLOq3jz/+WGXW9LOI/Xx01C8tW7ZUWY8ePVS2atUq\nla1cudJ8zYceekhl1vfDDh06JCkxsR07dpj5yJEjVfbss8+qrLy8XGXWu2ayiDsAAAA4RAMAAIBD\nNAAAADhEAwAAgEMhxnigjx/wg7XFqsHaZrRBA92vWM9Ttgb++vTpY1578+bNSUo0awwhJDq3Orp0\n6aKyBx98UGWnnHKKyqxnUKes9v+Cqq8oa7i++/DDD1V2xBFHmMfu3LlTZTneDjsLa1gkB+t4w4YN\nKvva176msvXr1yd+ze7du6vMGj61hsu3bt2qsrFjx6psy5Yt5rWXLVuWpERp1aqVyo499liVXXDB\nBSo799xzVdauXbtE160mcx1zBwAAAIdoAAAAcIgGAAAAh2gAAABwKBNDgIW45ZZbVHbnnXeqrJDd\n/USKNwSY1PTp01U2fvz4FCo5oCwMUGV+DedBv379VPbKK6+YxzIEWCdyuY6tgb++ffuaxyYdyLac\nffbZKvvtb3+rsup8H2/SpInKHnnkEZUdddRRKuvdu7f5miliCBAAAFSiAQAAwCEaAAAAHKIBAADA\noaI/DrgQGzduVNm9996rskIH/mrb+eefr7JnnnnGPDbpIIz1mE2grrz++usqu+mmm8xjDznkkLou\nBznRqVMnlb3xxhvmsdaj2desWZPoOr/5zW8SHWc9pvff/u3fzGOvuuoqlVk7teYZdwAAAHCIBgAA\nAIdoAAAAcIgGAAAAh3I1BGg9knT16tUFveZhhx2mMuuRkTfeeGNB1/miqgZhTjrppETnH2QHRyCR\niooKlU2cOFFlHTt2VNkPfvAD8zWtx3YDf3P44YebufXo4HfffbdWr22tzfLycvNY62dDfcNXKgAA\nDtEAAADgEA0AAAAO0QAAAOBQroYAe/bsqbLzzjtPZbt27VLZySefbL7m6NGjVdamTZsaVFc9VQ0v\nJn3EcJqPIkbtmjp1qsqaN2+usssvv7zWr71s2TKVzZgxQ2XWMJZVI3AwVQ32vfjiizV+zZKSEpW1\naNFCZZ988onKrEf8itgDsg888IDKGjVqpLK8fH/mDgAAAA7RAAAA4BANAAAADtEAAADgUK6GAK1h\ni5/97Gcqy8MjG6+55pq0S0BGWLtCLl26VGWXXXaZeX5paWmi65SVlalsyJAhKnv66adVloevKeTD\nfffdZ+br1q2r8WtauwsuWbJEZY8//rjKbr31VvM1H3vssUTZggULVHbmmWear5k13AEAAMAhGgAA\nAByiAQAAwCEaAAAAHKIBAADAoVy9C8CSh+lka+vLrVu3plAJsqhbt24qs6aVx40bZ54/duxYlVmT\n/Na08/e+9z2VDRgwwLwOUBu++93vmvnNN99c49ccPHiwytq3b6+yK6+8UmWjRo0yX3PHjh0qO/LI\nI1VmfT6nnnqqyqyvybRxBwAAAIdoAAAAcIgGAAAAh2gAAABwKMQYD/TxA34Q2r59+1R29dVXq+z2\n229P/Jonnniiyp5//nmVZfD57Fl4KHbm1/CHH36osn//939X2ezZs83zGzTQfXy/fv1UZg38jR49\nWmVJtxZ2IgtrWCQH6zgpa0tqEZG+ffsmOn/YsGEqe+ihh1TWuHHj6hX2Bdb3cmvr+SlTpqjM+hr6\nwx/+oDLr67SOmOuYOwAAADhEAwAAgEM0AAAAOEQDAACAQ7kfArR2a/rd736nsv79+5vnW4MizZo1\nq3E91rOux4wZk/j8jh07quzPf/6zytq0aVO9wtKRhQGqzK9hS0VFhcqq2glwzpw5KtuwYYPKClnX\njmVhDYvkdB1bPvnkEzP/9re/rTJrYHDJkiUqK+IwndKjRw+VWbu/3n///Sq76KKL6qQmA0OAAACg\nEg0AAAAO0QAAAOAQDQAAAA7l/nHA48ePV9msWbMSn28NjyxdujTRuY8++qjK7rrrrsTXtlifT04G\n/lCLrJ3EZs6caR67ePFilQ0aNCjR+b169ap+cUABqvp+Zq1Fawhw2rRpKrO+byb1wAMPmPmIESNU\nZu2mWV5erjJrh80LLrigBtXVLe4AAADgEA0AAAAO0QAAAOAQDQAAAA7lfgiwUK+88orK2rZtq7L9\n+/erbPv27YmOs3Tp0sXML7nkkkTnw59DDjnEzK2d0ax11Lt3b5VNmDBBZdawUqGDqOvXr1fZU089\npbJrrrlGZa1bty7o2siHTp06JTru8ccfT5RZrJ1vQ7A3e/zVr36lsiOOOEJl3//+91VmPTa4YcPs\n/bjlDgAAAA7RAAAA4BANAAAADtEAAADgUO4fB7x69WqVXXzxxSqzhv1Ekg/tVWd45IusR/y++OKL\n5rFf+tKXEr1mTmThUaqZX8N1Ydu2bSqzdjz73//9X5Xde++9tV6PNShlfZ2eccYZtX7tAmVhDYs4\nWMfWmrUGsgthfR+fPn26eWznzp1VZu1W2LVr14LrKgIeBwwAACrRAAAA4BANAAAADtEAAADgUO6H\nAJN65plnzHzixIkqW7lypcqsYcEGDXT/dNVVV6ls7NixKsvJ4EihsjBAVW/WMFKRhTUswjpGYRgC\nBAAAlWgAAABwiAYAAACHaAAAAHCIBgAAAIfcvAsAqcjCBDVrGIXIwhoWYR2jMLwLAAAAVKIBAADA\nIRoAAAAcogEAAMAhGgAAAByiAQAAwCEaAAAAHKIBAADAIRoAAAAcogEAAMAhGgAAAByiAQAAwCEa\nAAAAHKIBAADAoYM9DhgAANRD3AEAAMAhGgAAAByiAQAAwCEaAAAAHKIBAADAIRoAAAAcogEAAMAh\nGgAAAByiAQAAwCEaAAAAHKIBAADAIRoAAAAcogEAAMAhGgAAAByiAaiBEMKsEMLGEEJ5CGFVCGFU\n2jUB1RFCaBNCmBdC2BFCeC+EcEHaNQE1EULoHkL4LIQwK+1a8qZh2gXk1FQRGRlj3B1COFZEFocQ\n3ogxLk+7MCChGSKyR0Q6iMgJIrIghPBmjHFFumUB1TZDRF5Nu4g84g5ADcQYV8QYd//tj5//96UU\nSwISCyE0FZHhInJ9jHF7jHGJiMwXkQvTrQyonhDCCBHZKiLPpV1LHtEA1FAIYWYIYaeIvC0iG0Xk\n6ZRLApLqISJ7Y4yr/i57U0SOT6keoNpCCC1EZLKITEy7lryiAaihGON4EWkuIl8XkbkisvvAZwCZ\n0UxEyr+QbZPK9QzkxU0icl+McX3aheQVDUABYoz7Pr992klExqVdD5DQdhFp8YWshYh8mkItQLWF\nEE4QkW+KyO1p15JnDAHWjobCDADyY5WINAwhdI8xvvN51ltEGABEXvQXka4isi6EIFJ5V6skhNAz\nxtgnxbpyJcQY064hV0II7UXkGyLylIjsksoudK6InB9jnJ9mbUBSIYTZUjm8Okoq3wXwtIh8jXcB\nIA9CCE3kH+9iTZLKhmBcjPHjVIrKIe4AVF+Uytv9d0nlP6G8JyJX8sMfOTNeRO4XkU0iskUqv3Hy\nwx+5EGPcKSI7//bnEMJ2EfmMH/7Vwx0AAAAcYggQAACHaAAAAHCIBgAAAIdoAAAAcOhg7wJgQhCF\nCGkXIKxhFCYLa1iEdYzCmOuYOwAAADhEAwAAgEM0AAAAOEQDAACAQ2wFDKBWVVRUqKy0tFRle/bs\nUVlJSUmiDEDhuAMAAIBDNAAAADhEAwAAgEM0AAAAOMQQIICD2rZtm8reeust89jJkyerrG/fvir7\n7W9/q7I+ffqobODAgSobOnSoypo2bWrWA8DGHQAAAByiAQAAwCEaAAAAHKIBAADAoRDjAZ8yySMo\nUYgsPEqVNXwAmzdvVlnz5s1VdvLJJ6usrKysTmpK4tJLL1XZvffeax7boEFBv+dkYQ2LsI5RGB4H\nDAAAKtEAAADgEA0AAAAO0QAAAOAQDQAAAA7xLgDUpSxMULOGP/f222+rrH///io75phjVPb666+r\nbMeOHYmvbX2fCaF2l8cbb7xh5r179y7kZbOwhkVYxygM7wIAAACVaAAAAHCIBgAAAIdoAAAAcKhh\n2gUAKMxLL72ksv/8z/9U2apVq1S2ZcsWlW3atCnRdasarrv22mtV9uabb6psxIgRKrv77rsTXdsa\nSjzuuOMSnQugEncAAABwiAYAAACHaAAAAHCIBgAAAIfYCRB1KQu7qNWbNXzTTTeZ+Q033KCyg3xd\nH9CoUaNUNmTIEJV961vfMs9v3Lhxja+d1O7du1XWqFGjurhUFtawSD1ax0gFOwECAIBKNAAAADhE\nAwAAgEM0AAAAOMQQIOpSFgaoMr+Gra9Ba3e/008/PfH5JSUlKjviiCNUNmPGDJWVlpaq7C9/+YvK\nli1bZtZz9dVXq8x6xHCDBrn4/SMLa1gkB+u4vunbt6/KrB0o586dq7KhQ4fWSU0FYAgQAABUogEA\nAMAhGgAAAByiAQAAwCEeB5wxO3fuVFlFRYXKysrKVPbOO+8kusZZZ52lsvbt2yc6F4XZt2+fyqZO\nnaqyH//4x4lfs1WrViqzHr971FFHqay8vFxl1vDT6tWrE9fz0EMPqWz48OEqGzhwoMrOP/98lR16\n6KGJr43iePDBB1W2efNmlV1xxRUqq6MdG4siBD1L17Vr1+IXUku4AwAAgEM0AAAAOEQDAACAQzQA\nAAA4xE6AtcwaxHv88cdV9swzz5jnr1+/XmXbt29X2aZNm1TWpEmTJCXKyJEjVXb77bcnOreasrCL\nWmpreMeOHSobN26cymbNmlXQdd59912VJR1MsoYAX375ZZX9+c9/Vpm1u5+IyJIlS1R26623Jqqn\nU6dOKrN2RezSpUui16sFWVjDIimu461bt6qsW7duKtu2bZvKtmzZojJraDVN+/fvN/MTTjhBZStW\nrFCZ9Xk3a9as8MJqFzsBAgCASjQAAAA4RAMAAIBDNAAAADiUiZ0AZ8+erbKJEyeq7IUXXlBZ9+7d\n66Smmvrv//5vlb3xxhsqs3ZGExHp3bt3outYn3ePHj1UZj1ylV3/imP37t0qswb+rEFc63Gi119/\nvXmdzp0716C6Si1atFDZgAEDEmVV+c53vqMyaxfC5557TmXz589X2de//nWVTZs2TWXDhg1LWiKq\nwXrsc+vWrVVmDcMtX75cZWeccUbtFFZLdu3aZebWoHW/fv1UluedKrkDAACAQzQAAAA4RAMAAIBD\nNAAAADhEAwAAgEOZeBeAtU2oNYE5ePBglV199dUqs7a6LRbrOe4dO3ZUWWlpaeLX3Llzp8oWL16s\nMmvi33p+ddJ3GiC5ffv2qWzq1KmJzp0wYYLKrrvuOpW1a9eu+oWlwFqH1nPhV61alej1rO2xre21\nTznlFPP8Dh06JLoObNY7QP74xz+q7PDDD1fZokWLVJa1dwE0bGj/GLS2Vv/4449VZn3tl5SUFF5Y\nEXAHAAAAh2gAAABwiAYAAACHaAAAAHAoE0OAkydPVtnDDz+c6Nzzzz+/tsspiLXlqcXaNrMqF198\nscqswagpU6aozNr2d9CgQYmvjWSsLX5vu+22ROeOHTtWZXkZ+CvE+PHjVTZjxoxE51rbh1c1zPXQ\nQw9VrzC40qhRIzNv06aNyqxt3ffs2aOyQw45pPDCioA7AAAAOEQDAACAQzQAAAA4RAMAAIBDmRgC\ntIbS8jqo9otf/EJlK1asUNmjjz5qnv/Vr3410XUuueQSlf3rv/6ryqzdrFD7rB3CLKeeeqrKunfv\nXtvl5EKPHj1UNmzYMJXNnTs30etZg5giDAHWBWuHP0uvXr3quBIUgjsAAAA4RAMAAIBDNAAAADhE\nAwAAgEOZGALMK+uRxWPGjFGZ9UjeGKP5mvPnz0907MSJE1X26quvqqxbt24qS7pbIZKrqKhIdJy1\n61heHh1aCOuR1j/96U9VNm/evBpf4+abb67xuaie7du3p11CrbF28hMRKS8vL3IlxccdAAAAHKIB\nAADAIRoAAAAcogEAAMAhhgALYD1qd9SoUSq7//77E7+mNTD4ox/9SGXWY5CtYUFrZ8HBgwer7Oyz\nzzbrOeyww8wc/+jNN99MdNyuXbtUtnfvXpVV9WjbtFiDUtajUUVEfve736ls4cKFKnvllVdqXE+z\nZs1UNmTIkBq/HqpmDbjOnDkz0bnXXnutyqp6VO63vvUtlbVo0SLRdQrx2WefmfnmzZtV1rNnT5VV\n9TjhPOAOAAAADtEAAADgEA0AAAAO0QAAAOBQtiaN6oGhQ4eqbNq0aSor9DG9N910U6Ljpk6dqjJr\nt8LmzZub55933nnVK8ypc845R2XPP/+8yl5++WWVPfvssyo744wzVFZaWmpee8OGDSqraqfJJB5+\n+GGVWY9//cMf/lDja1THRRddpLIbb7xRZV26dClGOe5Y37+SDr2+9957Kjv33HPNY63hQGuo+r/+\n679UVtX3rySqGgLctm2byk455RSVVfV1mQfcAQAAwCEaAAAAHKIBAADAIRoAAAAcYgiwlg0aNCjt\nEv7B//zP/6jM2m3QOk6EIcCkrL8na4ho7NixKrPWzIABA1TWqlUr89pz5sxR2f79+81j0/LP//zP\nKjv11FNVdtlll6mse/fuKmvcuHHtFIZ/sGzZMpVdffXVic61djy95JJLVFZWVmaebw2f3nHHHYmy\n4cOHq+yqq65SWd++fVVW1deKNUhbyHBtFnEHAAAAh2gAAABwiAYAAACHaAAAAHCIBgAAAIfCQaYa\nMzXyaD2T3Jpoz/PWjIXYuXOnyhYvXqyyefPmqWzBggXma1rbzFaD/p9TfKmt4b1796rshRdeUJn1\nHPSssbaUHjFiROLz27Vrp7KmTZsWVFORZGENixRpHVvb9D7xxBMq69mzp8qWL1+uMmt73+r44IMP\nVGZN91vf06yfF8ccc4zKGjVqZF77rbfeUpn1jp0nn3zSPD9jzHXMHQAAAByiAQAAwCEaAAAAHKIB\nAADAocxuBfynP/1JZb/4xS9UNnPmzGKUUzSbNm1SWbNmzVRmDfdZW2n26tVLZRdeeKHKbr311oQV\nIqmGDfWX12mnnaayu+++W2Vr165V2fTp083rWNsLv/jiiyo78sgjVWYNIFqDYC1btlRZSUmJWQ/q\nvwkTJqis0IE/i7VmH3nkEZVZw8rWFtnW97mNGzcmrmfNmjUqs4YN6+Lvoi5wBwAAAIdoAAAAcIgG\nAAAAh2gAAABwKLM7AVq72lnDcNZOgNdcc43KqnqufVXPWE/C2n3qnHPOSXScNewnYj9T2xp+bNu2\nrcoWLVqkspEjR6qsffv25rXrQBZ2UcvUbpaF+Oyzz8zc2snM2oXQ+lqxBhXxD7KwhkWKtI6tIblV\nq1ap7I477lBZHgbfrK+LL3/5y+ax1k6Ajz32mMrOPvtslWVwQJadAAEAQCUaAAAAHKIBAADAIRoA\nAAAcytUQ4JVXXqkya2jO+pysAajqHFuM46o6dv78+SqzHktp/Z01adLEvE6RZGGAqt4MASIVWVjD\nIqzjOnP66aebubWb5pYtW1RWyCB5ETEECAAAKtEAAADgEA0AAAAO0QAAAOBQZrcBs4bXpk2bprLV\nq1er7Pnnn1dZgwZ2r3OQIchaOe6ss85S2ejRo83zu3fvrjLrUayWlAf+AKDeaNq0qcoyuMNfQbgD\nAACAQzQAAAA4RAMAAIBDNAAAADiU2Z0Ak7J2v1uzZo3K5syZU4xyZMyYMSpr0aKFypIO9uVcFnZR\ny/waRqZlYQ2LsI5rRXUeB2zt8GcNmOcEOwECAIBKNAAAADhEAwAAgEM0AAAAOJT7IUBkWhYGqFjD\nKEQW1rAI67hWbN26VWVt27Y1j500aZLKbr755lqvqUgYAgQAAJVoAAAAcIgGAAAAh2gAAABwiAYA\nAACHGqZdAAAAxbBo0aLEx06YMKEOK8kG7gAAAOAQDQAAAA7RAAAA4BANAAAADjEECABw4f7771fZ\nbbfdZh57+OGH13U5qeMOAAAADtEAAADgEA0AAAAO0QAAAOBQiPGAj5nmGdQoRBaepc4aRiGysIZF\nWMcojLmOuQMAAIBDNAAAADhEAwAAgEM0AAAAOHSwIUAAAFAPcQcAAACHaAAAAHCIBgAAAIdoAAAA\ncIgGAAAAh2gAAABwiAYAAACHaAAAAHCIBgAAAIdoAAAAcIgGAAAAh2gAAABwiAYAAACHaAAAAHCI\nBqCaQgiNQgj3hRDeCyF8GkIoCyEMTLsuoDpCCJeHEF4LIewOITyYdj1ATYQQZoUQNoYQykMIq0II\no9KuKU9oAKqvoYi8LyKniUhLEblOROaEELqmWBNQXRtEZIqI3J92IUABpopI1xhjCxE5S0SmhBBO\nSrmm3KABqKYY444Y4w0xxrUxxv0xxqdE5C8iwqJDbsQY58YYfyMiW9KuBaipGOOKGOPuv/3x8/++\nlGJJuUIDUKAQQgcR6SEiKzwTMycAAArMSURBVNKuBQC8CSHMDCHsFJG3RWSjiDydckm5QQNQgBBC\nqYg8LCK/jDG+nXY9AOBNjHG8iDQXka+LyFwR2X3gM/A3NAA1FEJoICK/EpE9InJ5yuUAgFsxxn0x\nxiUi0klExqVdT140TLuAPAohBBG5T0Q6iMigGGNFyiUBACp/pjEDkBB3AGrmThE5TkSGxBh3pV0M\nUF0hhIYhhMYiUiIiJSGExiEEfiFAboQQ2ocQRoQQmoUQSkIIZ4rI+SLyXNq15UWIMaZdQ66EELqI\nyFqp/HemvX/3octijA+nUhRQTSGEG0TkJ1+Ib4wx3lD8aoDqCyEcJiKPi0hvqfxl9j0R+XmM8d5U\nC8sRGgAAABzinwAAAHCIBgAAAIdoAAAAcIgGAAAAhw72th8mBFGIkHYBwhpGYbKwhkVYxyiMuY65\nAwAAgEM0AAAAOEQDAACAQzQAAAA4RAMAAIBDNAAAADhEAwAAgEM0AAAAOEQDAACAQzQAAAA4RAMA\nAIBDNAAAADhEAwAAgEM0AAAAOEQDAACAQzQAAAA4RAMAAIBDNAAAADhEAwAAgEM0AAAAONQw7QIA\npOfOO+9U2ezZs1X2k5/8RGXf+MY36qQm+FRRUVHn1ygpKVFZgwZ+fw/2+5kDAOAYDQAAAA7RAAAA\n4BANAAAADtXLIcAYo8rWrVtnHjtr1iyVvf766yqbO3euyi699FKVffe731XZgAEDVBZCMOsBimn5\n8uUq+/jjj1XWtm3bYpSDHFu1apXKduzYkfj8YcOGqayq79s1tXTpUpX169evVq+RJ9wBAADAIRoA\nAAAcogEAAMAhGgAAABwK1sDc3zngB7Nqz549KmvSpElBr2n9PSUd5Fu8eLHKTj311ILqyYksTDrm\ncg3XhU8++URl7dq1U9nMmTNVNnbs2DqpKQeysIZFirSO9+3bp7Jbbrkl0bk///nPVfbRRx8VXFNt\ncjwEaK5j7gAAAOAQDQAAAA7RAAAA4BANAAAADtXLnQAt1rCTiMjIkSNV9pWvfEVlX/3qV1V2zz33\nqGzy5MkqGzNmjMqWLFli1tOmTRszBwpl7fBnueuuu1TmeAgw93bu3Gnm1hDzpEmTVGatB9QP3AEA\nAMAhGgAAAByiAQAAwCEaAAAAHKqXQ4ClpaUqe//99xMfm9R1112nsgULFqjMeuTqypUrzdd0skMg\nUlBWVpbouP79+9dtISiqZs2amXkxHkl+9NFHq+zMM880j33kkUdUVl5eXqv1vPDCCyqr6u+nV69e\ntXrtLOIOAAAADtEAAADgEA0AAAAO0QAAAOBQvRwCtIZbChn2q0qDBrp/sjLLBx98UNvlAAf0xBNP\nJDrun/7pn+q4EhRTVY98L2QI0NoZcuDAgSrr1q2bynr27Gm+5ksvvaSyFStW1KC6qv3oRz9S2bRp\n08xj582bp7L69uhg7gAAAOAQDQAAAA7RAAAA4BANAAAADtXLIcA8+OUvf2nm5513XpErQX1UUVGh\nsnXr1qVQCdK2dOlSM7/oootq/JpnnXWWyqra4S+pzp07q6y2hwAtmzZtMvMhQ4ao7Mknn1SZNTTb\ntGnTwgsrAu4AAADgEA0AAAAO0QAAAOAQDQAAAA7RAAAA4FCoapvIzx3wg9BOPvlklb366qsqW7hw\noXn+N7/5zVqvKUV1/8Dxg3O5hq3nqLdq1SrRuX/9619V1rJly4JryqksrGERB+t4165dKvvxj3+c\n6Nxt27ap7L777iu4piSOP/54lT399NMq69SpUzHKqYq5jrkDAACAQzQAAAA4RAMAAIBDNAAAADjE\nEGAB9uzZo7K+ffuqbO/evSqranvO1q1bF15YdmRhgMrlGi4rK1NZnz59VGZt37pgwQKVNWjg9neF\nLKxhEafrOKndu3er7I9//GOic4cOHWrmn376aY3r6devn8rmzZunsvbt29f4GtXEECAAAKhEAwAA\ngEM0AAAAOEQDAACAQw3TLiDP1qxZo7KVK1eq7J577lFZPRv2Q8a89tpriY5r3ry5yhwP/CGnGjVq\npLL+/fsnOreqYcFevXrVuJ5ly5ap7JNPPlFZEYcATXylAwDgEA0AAAAO0QAAAOAQDQAAAA4xBJiQ\n9XjVAQMGJDr3pJNOqu1ygAP64IMPVGbt+nnaaacVoxygTj3//PMq2759e6Jz165dW8vV5Ad3AAAA\ncIgGAAAAh2gAAABwiAYAAACHGAJM6MMPP1TZ+vXrVTZixAiVFbKjFFATTzzxhMpC0E8EPe6444pR\nDlAj//d//6eyadOmqWzWrFkq27lzZ53UVFPXXnutyqyv02LiDgAAAA7RAAAA4BANAAAADtEAAADg\nULB2B/s7B/xgfbV8+XKVfeUrX1GZNVRlDQZ27NixdgrLH/0XVHz1fg1v3LhRZd27d1eZNRS1ePFi\nlf3Lv/xLrdRVT2RhDYs4WMeffvqpyo466qhEx+XVvn37inUpcx1zBwAAAIdoAAAAcIgGAAAAh2gA\nAABwyP1OgGVlZSobPXp0onOvuuoqlbVr167gmoDq+P3vf68ya+Dv6KOPVhkDf8iK0tJSlX3nO99R\n2a9//etilJNY586dVTZu3LgUKqk+7gAAAOAQDQAAAA7RAAAA4BANAAAADtEAAADgUL18F4C1veLg\nwYPNYxctWqSy/fv3q6xPnz4q++lPf6qyBg3oqVBca9asSXTcOeecU8eVADXXuHFjlc2cOVNlkyZN\nUtnAgQNVtmnTptop7CCOP/54lf3whz8syrULxU8rAAAcogEAAMAhGgAAAByiAQAAwKHcDwH+6U9/\nUpk1lPHss88WdJ1jjjlGZQz8AUDdadGihcpOOOEElb3//vvFKMcUQkjt2oXiJxgAAA7RAAAA4BAN\nAAAADtEAAADgUK6GAK1nnI8ePVplrVu3rvVrv/7667X+mgCAwjVsmKsfZZnBHQAAAByiAQAAwCEa\nAAAAHKIBAADAocxOTsQYVXb99derbOXKlQVd54orrlDZRx99pLLHHntMZWVlZSqzdqmyWI8cFhHZ\nu3evykpLS1WW592nULus9Wq59NJL67gSAHnCHQAAAByiAQAAwCEaAAAAHKIBAADAoWAN2/2dA36w\nLu3YsUNlLVu2rPHrNW7c2MxXrFihskaNGqnsxBNPTPSav//97xO93uTJk8165s+fr7J169ap7NBD\nDzXPz5gsTCqmtoaLpVmzZiqzds1csGCBygYOHFgnNdUjWVjDIg7WMeqUuY65AwAAgEM0AAAAOEQD\nAACAQzQAAAA4lNmdAAvRt29flT366KPmsV26dEn0mtOnT1fZiBEjVNazZ89Er9euXTszf+mll1SW\nk4E/pKRXr14qa9OmjcpOP/30YpQDICe4AwAAgEM0AAAAOEQDAACAQzQAAAA4lNmdAHfv3q2yk08+\nOdG5L7zwgsqaN29eUD27du1S2bRp0xKd279/f5VZg1sihdeZMVnYRY0d1FCILKxhEdYxCsNOgAAA\noBINAAAADtEAAADgEA0AAAAO0QAAAOBQZt8FgHohCxPUrGEUIgtrWIR1jMLwLgAAAFCJBgAAAIdo\nAAAAcIgGAAAAh2gAAABwiAYAAACHaAAAAHCIBgAAAIdoAAAAcIgGAAAAh2gAAABwiAYAAACHaAAA\nAHCIBgAAAIcO9jhgAABQD3EHAAAAh2gAAABwiAYAAACHaAAAAHCIBgAAAIdoAAAAcOj/ARFVh/h6\nC4zTAAAAAElFTkSuQmCC\n", "text/plain": [ "
" ] }, "metadata": { "tags": [] }, "output_type": "display_data" } ], "source": [ "dls.show_batch()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "From here we need to see what our model will expect" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "xb, yb = dls.one_batch()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "And now the shapes:" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "(torch.Size([128, 1, 28, 28]), torch.Size([128]))" ] }, "execution_count": null, "metadata": { "tags": [] }, "output_type": "execute_result" } ], "source": [ "xb.shape, yb.shape" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "10" ] }, "execution_count": null, "metadata": { "tags": [] }, "output_type": "execute_result" } ], "source": [ "dls.c" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "So our input shape will be a [128 x 1 x 28 x 28] and our output shape will be a [128] tensor that we need to condense into 10 classes" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## The Model\n", "\n", "Our models are made up of **layers**, and each layer represents a matrix multiplication to end up with our final `y`. For this image problem, we will use a **Convolutional layer**, a **Batch Normalization layer**, an **Activation Function**, and a **Flattening layer**" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Convolutional Layer\n", "\n", "These are always the first layer in our network. I will be borrowing an analogy from [here](https://adeshpande3.github.io/A-Beginner%27s-Guide-To-Understanding-Convolutional-Neural-Networks/) by Adit Deshpande.\n", "\n", "Our example Convolutional layer will be 5x5x1\n", "\n", "Imagine a flashlight that is shining over the top left of an image, which covers a 5x5 section of pixels at one given moment. This flashlight then slides crosses our pixels at all areas in the picture. This flashlight is called a **filter**, which can also be called a **neuron** or **kernel**. The region it is currently looking over is called a **receptive field**. This filter is also an array of numbers called **weights** (or **parameters**). The depth of this filter **must** be the same as the depth of our input. In our case it is 1 (in a color image this is 3). Now once this filter begins moving (or **convolving**) around the image, it is multiplying the values inside this filter with the original pixel value of our image (also called **element wise multiplications**). These are then summed up (in our case this is just one multiplication of 28x28) to an individual value, which is a representation of **just** the top left of our image. Now repeat this until every unique location has a number and we will get what is called an **activation** or **feature map**. This feature map will be 784 different locations, which turns into a 28x28 array\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "def conv(ni, nf): return nn.Conv2d(ni, nf, kernel_size=3, stride=2, padding=1)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Here we can see our `ni` is equivalent to the depth of the filter, and `nf` is equivalent to how many filters we will be using. (Fun fact this always has to be divisible by the size of our image)." ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Batch Normalization\n", "\n", "As we send our tensors through our model, it is important to normalize our data throughout the network. Doing so can allow for a much larger improvement in training speed, along with allowing each layer to learn independantly (as each layer is then re-normalized according to it's outputs)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "def bn(nf): return nn.BatchNorm2d(nf)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "`nf` will be the same as the filter output from our previous convolutional layer" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Activation functions\n", "\n", "They give our models non-linearity and work with the `weights` we mentioned earlier along with a `bias` through a process called **back-propagation**. These allow our models to learn and perform more complex tasks because they can choose to fire or activate one of those neurons mentioned earlier. On a simple sense, let's look at the `ReLU` activation function. It operates by turning any negative values to zero, as visualized below:\n", "\n", "![](https://miro.medium.com/max/2052/1*DfMRHwxY1gyyDmrIAd-gjQ.png)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "From \"A Practical Guide to ReLU by Danqing Liu [URL](https://medium.com/@danqing/a-practical-guide-to-relu-b83ca804f1f7)." ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "def ReLU(): return nn.ReLU(inplace=False)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Flattening\n", "\n", "The last bit we need to do is take all these activations and this outcoming matrix and flatten it into a single dimention of predictions. We do this with a `Flatten()` module" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "Flatten??" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Making a Model\n", "\n", "* Five convolutional layers\n", "* `nn.Sequential`\n", "* 1 -> 32 -> 10" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "model = nn.Sequential(\n", " conv(1, 8),\n", " bn(8),\n", " ReLU(),\n", " conv(8, 16),\n", " bn(16),\n", " ReLU(),\n", " conv(16,32),\n", " bn(32),\n", " ReLU(),\n", " conv(32, 16),\n", " bn(16),\n", " ReLU(),\n", " conv(16, 10),\n", " bn(10),\n", " Flatten()\n", ")" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Now let's make our `Learner`" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "learn = Learner(dls, model, loss_func=CrossEntropyLossFlat(), metrics=accuracy)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "We can then also call `learn.summary` to take a look at all the sizes with thier **exact** output shapes" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "Sequential (Input shape: 128 x 1 x 28 x 28)\n", "================================================================\n", "Layer (type) Output Shape Param # Trainable \n", "================================================================\n", "Conv2d 128 x 8 x 14 x 14 80 True \n", "________________________________________________________________\n", "BatchNorm2d 128 x 8 x 14 x 14 16 True \n", "________________________________________________________________\n", "ReLU 128 x 8 x 14 x 14 0 False \n", "________________________________________________________________\n", "Conv2d 128 x 16 x 7 x 7 1,168 True \n", "________________________________________________________________\n", "BatchNorm2d 128 x 16 x 7 x 7 32 True \n", "________________________________________________________________\n", "ReLU 128 x 16 x 7 x 7 0 False \n", "________________________________________________________________\n", "Conv2d 128 x 32 x 4 x 4 4,640 True \n", "________________________________________________________________\n", "BatchNorm2d 128 x 32 x 4 x 4 64 True \n", "________________________________________________________________\n", "ReLU 128 x 32 x 4 x 4 0 False \n", "________________________________________________________________\n", "Conv2d 128 x 16 x 2 x 2 4,624 True \n", "________________________________________________________________\n", "BatchNorm2d 128 x 16 x 2 x 2 32 True \n", "________________________________________________________________\n", "ReLU 128 x 16 x 2 x 2 0 False \n", "________________________________________________________________\n", "Conv2d 128 x 10 x 1 x 1 1,450 True \n", "________________________________________________________________\n", "BatchNorm2d 128 x 10 x 1 x 1 20 True \n", "________________________________________________________________\n", "Flatten 128 x 10 0 False \n", "________________________________________________________________\n", "\n", "Total params: 12,126\n", "Total trainable params: 12,126\n", "Total non-trainable params: 0\n", "\n", "Optimizer used: \n", "Loss function: FlattenedLoss of CrossEntropyLoss()\n", "\n", "Callbacks:\n", " - TrainEvalCallback\n", " - Recorder\n", " - ProgressCallback" ] }, "execution_count": null, "metadata": { "tags": [] }, "output_type": "execute_result" } ], "source": [ "learn.summary()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "`learn.summary` also tells us:\n", "* Total parameters\n", "* Trainable parameters\n", "* Optimizer\n", "* Loss function\n", "* Applied `Callbacks`" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "data": { "text/html": [], "text/plain": [ "" ] }, "metadata": { "tags": [] }, "output_type": "display_data" }, { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAENCAYAAAACHGKEAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjIsIGh0\ndHA6Ly9tYXRwbG90bGliLm9yZy8li6FKAAAgAElEQVR4nO3dd3ic5ZnF4d+jbqu5SO5FNu69yKY3\n00nA9JJAQl8IBFhYWmBDQkJbIISShPVCEkIMCQFDqCEETEsMQe5F7mDjLjcVW13P/jGTYIQkS7ZG\n32jm3Nc1F6Nv3pk5GmQdffU1d0dEROJXQtABREQkWCoCEZE4pyIQEYlzKgIRkTinIhARiXMqAhGR\nOBexIjCzvmY208yWmNliM7uugTHZZvaqmc0Pj7k4UnlERKRhFqnzCMysJ9DT3eeYWSYwGzjN3Zfs\nMeYHQLa732JmucAyoIe7V0UklIiIfE3E1gjcfaO7zwnfLwUKgd71hwGZZmZABrAdqIlUJhER+bqk\ntngTM8sDxgOf1HvoceAVYAOQCZzr7nVNvVZOTo7n5eW1fkgRkRg2e/bsre6e29BjES8CM8sAXgSu\nd/eSeg+fAMwDpgAHAG+b2Yf1x5nZFcAVAP369aOgoCDSsUVEYoqZrWnssYgeNWRmyYRKYLq7z2hg\nyMXADA9ZCXwGDKs/yN2nuXu+u+fn5jZYaCIiso8iedSQAU8Bhe7+s0aGrQWOCY/vDgwFVkcqk4iI\nfF0kNw0dClwILDSzeeFlPwD6Abj7E8BPgN+a2ULAgFvcfWsEM4mISD0RKwJ3/4jQL/emxmwAjo9U\nBhER2TudWSwiEudUBCIicU5FECG7KmtYuqmEWau2UV3b5KkRIiKBapMTyuLJy3PX89PXC9laVvnv\nZZMHdOHxb42nW2ZagMlERBqmImjCZ1t3MXPpFs6Z1JeM1L1/VH9ZtJEbnp/H2L6duOSwPPp16cjO\n3dX89PUlnPLYR/zy2xOZ2L9zGyQXgIrqWgo3lrBsUynds9MY16cTndNTgo4lEnVUBA2orq1j2ger\neeSdFVTV1PHkh6u5+4zRHD20W6PP+WB5Ed9/bi7j+nbi95cdSMeULz/aif07c+XvZ3PetFkcOiiH\nrumpdM1IYWj3TE4Y1aPBknF3dlXVUlRayfZdlezYVc2O3VXk5aST378zodM0okttnbN4QzEfrtjK\nuh3lJCcaiQlGWnIi3TJT6ZaZRo/sNIb3zPzK59OUmto6Fm8ooayyhqqaOipr6thaVsnmkgo2Flfg\nDl3Sk+mSnkpSgrGxuIINO8tZs303KzaXUlP31Ysq9u/akRE9sxiQk87A3Ax6dUojIzWJjilJpCYl\nUFZZQ2lFDeXVtYzomUVuZmokPiqRqBKxq49GSn5+vkfyEhNLN5Vw/R/msXRTKSeN6sFZE/tw75tL\nWbmljNPH9+aY4d3IyUglJyOVqpo6NpdW8MX23dz7xlLyctL5wxUHkd0h+WuvW1xezT2vF7JkYwnb\nyirZtquKypo60pITOH5EDw4c2IW123azfHMpq7fuYktJJeXVtQ1mnJTXmWumDOaIwTmBFkJVTR2L\nNxQze80OZq/ZwazV29i5uxqArukp1LlTU+dUVNdSXfvlz1ligjGyVxYT+3cmJSmBbWVVbCurpENK\nIqN7d2Jsn2xSkxN4df5GXluwga1lX78YbYJBbmYqSQkJbNtVSUV1aD9Mx5REenXqQO9OHRjZK4sx\nfbIZ3jOLDTsrmPfFTuZ9sYMVm8tYu33310qiIWP7ZHPU0G6cNr43A3LSW+mTE2l7Zjbb3fMbfExF\n8KWSimpO+vmHVNXWcfdpozh+ZA8AKmtq+cW7K/nle6sa/eUxqFsGz11+ULP/gnR35qzdwYw563lt\nwUaKy6tJTjQG5mQwqHsGPbPSyM0MFU7XjBS6pKeQ3SGZ95cX8cR7q9hQXMGQ7hlMyuvC2L6dmNCv\nEwfkZrRKMVTV1JFgkJT49WMJqmvr+HBFEX+et4G3l2xmd1WorPp26cCBA7py+OAcDjkg5yufg7uz\nc3c1m0srWLe9nLlf7KDg8x3MX7eTujrIyUihS0YKJeU1rN2++9/PS0lK4Jhh3Th5dE+6ZaaSkpRA\ncmJCuIhTvpKvvKqWqto6stKSmvUZVNfWsW5HORt3lrO7qpbd1bVUVteSnppEZloSSQkJzF6znZnL\nipi7dgcJZlxwUH+uO2awNi9Ju6QiaKYbn5/PS3PX8cJVhzCh39e35ZdUVLOpuIKi0kqKSitJTUqg\nW1Zok0fP7LQGf3E2R2VNLRt3VtC7cweSm/EaVTV1vDhnHa8t2MCCL4oprQxduXtEzyzOndSX08b1\nJrvj19dK9vSvTU9pSQkkJSZQV+d8/Nk2ZsxZz5sLNwIwrl8nJvbrTE5mKquLdrGqqIyF64vZubua\nTh2TOWlUTw4fnMPE/p3pntXyHeF1dY4ZX/nFvWNXVeg9yqs5amguWWlNfx9tYUtJBT9/ZwV/+Oda\nMlKT+P6UwVx4cH/SkhODjibSbCqCZnhz4Uaumj6Ha6cM4objh7b660dKXZ2zeusu/rFqK88XfMGi\n9SWkJCVwQG4GvbLT6NWpA+mpSdS5U1vnFJdXs6qojFVbyiipCBVIh+REkhKM0soaMlKTOHl0Dzok\nJ1KwZgeFG0uoc0hPSWRgbgZDe2Ry0qgeHD44l5Sk+Dr6eNmmUu55o5D3lxfRKzuN648bwhnje+/z\nHwAibUlFwJebYib27/K1x7aUVHDCzz+gb5eOvHjVIc36qzxaLVpfzJ/nrWd10S7W7yxnw85yKsKb\nehLNSE9N4oDcDA7olk6fzh2prK6jrLKa8upaJg/oynHDu9Mh5cu/dHdV1lBWWUO3zNSo3EEdhH+s\n3Mr9by1j/hc7GZCTzsmjezBlWDfG9e1MYoI+I4lOKgLgj5+u5ZYXF/LS9w5hfL3NPpc9/SkfrdzK\n69cezgG5Ga0VVWKYu/PW4k38+u+fM3vNDmrrnM4dkzlySC5ThnfnyMG5e908J9KWmiqCuDl89Jtj\nenHfm0t59J0V/Obiyf9e/vHqbfytcAu3njRMJSDNZmacOKonJ47qSfHuaj5YUcS7S7fw3rItvDxv\nA4kJxgkju3P9sUMY0j0z6LgiTYqbIkhPTeKywwfyQHiVfmzfTrg7D761jO5ZqVx0SF7QEaWdyu6Y\nzClje3HK2F7U1jnzvtjJXxdvYvona3lz0SZOGdOLG44bQp4OP5Uo1X43hu+D7xzcn+wOyTz27goA\n3ltWRMGaHVx7zGAdASKtIjHBmNi/M7edPJwPbz6aK488gLeXbOYbj37Ie8u2BB1PpEFxVQSZaclc\ndtgA/la4hYXrinngrWX069KRc/L7Bh1NYlDn9BRuOXEY7/7XkfTvms6lTxfw3D/XBh1L5GviqggA\nvntoHllpSfzHMwUs2VjCDccNaddHCUn065ndgeevPJjDBuVw24yF/M9fllLbjLOaRdpK3P0GzEpL\n5pLDBrChuIKh3TM5ZWyvoCNJHMhITeKp7+Zz/uR+/PK9VXz31/9kS2lF0LFEgDgsAoCLDx3ApLzO\n/PCUETruW9pMUmIC95w+ivvPHM2nn2/n5Ec+4u8rNUW3BC9uziMQiSbLNpVy9bNzWFVUxjVHD+K6\nYwbrDGWJqKbOI9BPnkgAhvbI5JVrDuXMCX147N2VnP9/H7NhZ3nQsSROqQhEAtIxJYkHzx7Lw+eO\nZcmGEk5+9ENmLtUhptL2VAQiATt9fB9e/f5h9MzuwCVPf8rj766gvW2ylfZNRSASBQbmZjDjqkM4\ndWwvHvzrcr43fQ67wpcXF4m0uLnEhEi065CSyM/PHceoXtnc+2Yh63eW89zlB5HejPmyRfaH1ghE\nooiZcfkRA3nigoksWl/M96bPobq2LuhYEuNUBCJR6PiRPbj79NG8v7yIH8xYqH0GElERKwIz62tm\nM81siZktNrPrGhl3lJnNC495P1J5RNqb8yf349opg/jT7HU8/PbyoONIDIvkxsca4EZ3n2NmmcBs\nM3vb3Zf8a4CZdQJ+CZzo7mvNrFsE84i0O/953BA2lVTw6LsrceCG44ZopjhpdRErAnffCGwM3y81\ns0KgN7Bkj2HfAma4+9rwOB1ELbIHM+PeM8ZgGI+9u5KS8mruPGUkCbo0irSiNjkcwczygPHAJ/Ue\nGgIkm9l7QCbwiLv/ri0yibQXiQnGfWeOJrtjMtM+WE1JRQ0PnDVGl6SQVhPxIjCzDOBF4Hp3L2ng\n/ScCxwAdgFlm9rG7L6/3GlcAVwD069cv0pFFoo6ZcdtJw8jukMwDby0jLTmBe04frc1E0ioiWgRm\nlkyoBKa7+4wGhqwDtrn7LmCXmX0AjAW+UgTuPg2YBqGLzkUys0i0MjOuPnoQu6tq+MXMVfTp3JGr\njx4UdCyJAZE8asiAp4BCd/9ZI8P+DBxmZklm1hE4ECiMVCaRWPBfxw9l6rhePPDWMl6auy7oOBID\nIrlGcChwIbDQzOaFl/0A6Afg7k+4e6GZ/QVYANQBT7r7oghmEmn3zIz/OWsMm0squPmFBXTLTOPQ\nQTlBx5J2TPMRiLRTxeXVnP3EP1i3o5xnLz+IcX07BR1JopjmIxCJQdkdknnm0gPJyUjlot/8k2Wb\nSoOOJO2UikCkHeuelcbvLz2QlMQELnzqE9Zu2x10JGmHVAQi7Vy/rh35/WUHUlVbxwVPfUJRaWXQ\nkaSdURGIxIAh3TP5zUWTKCqt5OLf/pMyzWUgLaAiEIkR4/t15hffHk/hxlKu+v1sqmp0+WppHhWB\nSAyZMqw7950xmg9XbOXmF+br8tXSLJr6SCTGnJ3fl03FFTz09nIm9O/Mdw7OCzqSRDmtEYjEoKuP\nHsTRQ3P56WuFLFpfHHQciXIqApEYlJBgPHTOOLqkp3DNs3MoragOOpJEMRWBSIzqkp7Co+eP54sd\n5fzgpUXaXyCNUhGIxLDJA7pww3FDeHX+Bn71/qqg40iU0s5ikRh31ZEHsGxTKf/zl2X0yErjjAl9\ngo4kUUZFIBLjEhKMB84ew9aySm5+YQG5makcPjg36FgSRbRpSCQOpCYl8sSFExnULYMrn5nNkg31\nJwuUeKYiEIkTWWnJPH3JZDLTkrn62Tm6DIX8m4pAJI50z0rjkfPGsWbbLn74suaAkhAVgUicOXBg\nV647Zggz5q7nxdma6lJUBCJx6ZopgzhwQBf++8+LWFVUFnQcCZiKQCQOJSYYj5w3nrTkRK55di4V\n1bVBR5IAqQhE4lSP7DQePHsMhRtLuOeNwqDjSIBUBCJxbMqw7lx22AB+N2sNf1m0Keg4EhAVgUic\nu/nEYYzpk83NL8xn3Q7NeRyPVAQicS4lKYHHzh9PncO1z82lplYzm8UbFYGI0L9rOvecMZo5a3fy\n+MyVQceRNqYiEBEATh3bizPG9+bRd1Ywe832oONIG1IRiMi//XjqSHp37sD1f5ynyWziiIpARP4t\nMy2Zh88Zx/od5dz5yuKg40gbiVgRmFlfM5tpZkvMbLGZXdfE2ElmVmNmZ0Uqj4g0T35eF66ZMpgZ\nc9bz18U6pDQeRHKNoAa40d1HAAcBV5vZiPqDzCwRuB/4awSziEgLfH/KIIZ2z+SHf16sTURxIGJF\n4O4b3X1O+H4pUAj0bmDo94EXgS2RyiIiLZOcmMC9Z45mc2kFD/11edBxJMLaZB+BmeUB44FP6i3v\nDZwO/Govz7/CzArMrKCoqChSMUVkDxP6deY7B/Xn6VmfM3ftjqDjSARFvAjMLIPQX/zXu3v9aZF+\nDtzi7k2eweLu09w9393zc3M1xZ5IW7npxGH0yErjthkLqdaJZjErokVgZsmESmC6u89oYEg+8Acz\n+xw4C/ilmZ0WyUwi0nwZqUncNXUUSzeV8ug7K4KOIxESyaOGDHgKKHT3nzU0xt0HuHueu+cBLwDf\nc/eXI5VJRFruuBHdOXtiHx6fuZIPV2jTbCyK5BrBocCFwBQzmxe+nWxmV5rZlRF8XxFpZXdNHcXg\nbhlc/4d5bC6pCDqOtDJz96AztEh+fr4XFBQEHUMk7qzYXMqpj/+dMX2ymX7ZgSQl6nzU9sTMZrt7\nfkOP6f+kiDTL4O6Z/PS0UXzy2XYe/psOKY0lKgIRabYzJ/bhvEl9+cXMVbyls45jhopARFrkR6eO\nZGyfbG58fj4rt2ji+1igIhCRFklLTuRXF0wkNSmB/3imQJegiAEqAhFpsV6dOvDYt8bz+bbd3PzC\nAtrbQSfyVSoCEdknhxyQw00nDOXNRZt4cc76oOPIflARiMg+u/zwgUzO68KPX1nMhp3lQceRfaQi\nEJF9lphgPHj2WGrdufmFBdTVaRNRe6QiEJH90q9rR27/xnA+WrmV6Z+sCTqO7AMVgYjst29N7scR\nQ3K5542lLNtUGnQcaSEVgYjsNzPjgbPGkJmWxOW/K2Dn7qqgI0kLqAhEpFV0z0rjiQsnsqm4gmue\nnUuN5i9oN1QEItJqJvTrzE9PG8VHK7dy35tLg44jzZQUdAARiS3nTOrLko0lPPnRZ4zuk83UcQ1N\nVS7RRGsEItLqbv/GcPL7d+YHMxby2dZdQceRvVARiEirS05M4NHzx5OclMDV0+dQUV0bdCRpgopA\nRCKiV6cOPHjWWJZsLOGeNwqDjiNNUBGISMQcO6I7lx02gN/NWsMbCzcGHUcaoSIQkYi6+cRhjOvb\niZtfWMDqIs1fEI1UBCISUSlJCfzi2xNITjSu+v0cdlfVBB1J6lERiEjE9e7UgUfOG8/yLaXc/tIi\nzV8QZVQEItImjhiSy38eO4SX5q7n95+sDTqO7KFZRWBmB5hZavj+UWZ2rZl1imw0EYk11xw9iKOG\n5vKTV5ewaH1x0HEkrLlrBC8CtWY2CJgG9AWejVgqEYlJCQnGz84ZR5f0FK55dg5lldpfEA2aWwR1\n7l4DnA485u43AT0jF0tEYlWX9BQePX88a7fv5gczFmp/QRRobhFUm9n5wHeB18LLkiMTSURi3eQB\nXbjhuCG8Mn8Df/z0i6DjxL3mFsHFwMHA3e7+mZkNAJ6JXCwRiXVXHTWIwwblcOcri3U9ooA1qwjc\nfYm7X+vuz5lZZyDT3e9v6jlm1tfMZprZEjNbbGbXNTDm22a2wMwWmtk/zGzsPn4fItLOJCYYD50z\nlpSkBG59UfMdB6m5Rw29Z2ZZZtYFmAP8n5n9bC9PqwFudPcRwEHA1WY2ot6Yz4Aj3X008BNCO6JF\nJE50z0rj9pOH88ln2/mDNhEFprmbhrLdvQQ4A/idux8IHNvUE9x9o7vPCd8vBQqB3vXG/MPdd4S/\n/Bjo05LwItL+nTupLwcP7Mq9bxSyqbgi6DhxqblFkGRmPYFz+HJncbOZWR4wHvikiWGXAm828vwr\nzKzAzAqKiopa+vYiEsXMjHvPGE11XR13vKyzjoPQ3CK4C3gLWOXun5rZQGBFc55oZhmEzkO4PrxW\n0dCYowkVwS0NPe7u09w9393zc3NzmxlZRNqLvJx0bjhuCH8r3MyMOeuDjhN3mruz+E/uPsbdrwp/\nvdrdz9zb88wsmVAJTHf3GY2MGQM8CUx1923Njy4iseSSQwcweUAX7nh5ESu3lAYdJ640d2dxHzN7\nycy2hG8vmlmT2/PNzICngEJ3b3DHspn1A2YAF7r78paGF5HYkZSYwGPnj6djSiJXT59LeZVmNWsr\nzd009BvgFaBX+PZqeFlTDgUuBKaY2bzw7WQzu9LMrgyP+SHQFfhl+PGCln8LIhIrumel8fC541i+\npZQfvbI46DhxI6mZ43Ldfc9f/L81s+ubeoK7fwTYXsZcBlzWzAwiEgeOGJLL1UcN4vGZKzl0cA6n\nju0VdKSY19w1gm1mdoGZJYZvFwDani8iEXH9sYMZ17cTd726mOLy6qDjxLzmFsElhA4d3QRsBM4C\nLopQJhGJc0mJCdx9+ii276riob8uCzpOzGvuUUNr3P1Ud891927ufhqw16OGRET21che2Xzn4Dye\n+XgNC9dp7oJI2p8Zym5otRQiIg244fghdE1P5Y4/L9K1iCJof4qgyR3BIiL7KystmTu+MZz5X+zk\nuU81vWWk7E8RqJ5FJOKmjuvFwQO7ct8bS1m/szzoODGpySIws1IzK2ngVkrofAIRkYgyM+4/cwx1\n7tz0p/naRBQBTRaBu2e6e1YDt0x3b+45CCIi+6Vf14789zdH8I9V23h61udBx4k5+7NpSESkzZw7\nqS9ThnXjvjeXsnJLWdBxYoqKQETaBTPjvjNH0zElkRuen0dNbV3QkWKGikBE2o1umWncffpoFqwr\n5lfvrQo6TsxQEYhIu3Ly6J6cOrYXj7yzgsUbdKJZa1ARiEi7c9fUkXRJT+HG5+dTWaPLVe8vFYGI\ntDudOqZw/5ljWLqplJ//rVmTJUoTVAQi0i4dPawb503qy/++v4rZa3YEHaddUxGISLt1+zeG0zO7\nAzc+P4/dVTVBx2m3VAQi0m5lpiXz0DljWbN9N/e+sTToOO2WikBE2rWDBnbl0kMH8MzHa3h/eVHQ\ncdolFYGItHv/dcJQBnfL4OYX5lO8WzOatZSKQETavbTkRB4+dxzbyqr46etLgo7T7qgIRCQmjOqd\nzaWHD+BPs9dR8Pn2oOO0KyoCEYkZ104ZTM/sNO54eZGuRdQCKgIRiRnpqUncecoIlm4q5elZa4KO\n026oCEQkppwwsgdHDsnl4beXs7mkIug47YKKQERiipnx41NHUlVbx12vacdxc6gIRCTm5OWkc/VR\ng3h9wUZmLtsSdJyoF7EiMLO+ZjbTzJaY2WIzu66BMWZmj5rZSjNbYGYTIpVHROLLlUcNZFC3DO54\naZEuP7EXkVwjqAFudPcRwEHA1WY2ot6Yk4DB4dsVwK8imEdE4khqUiL3njGa9TvLdYXSvYhYEbj7\nRnefE75fChQCvesNmwr8zkM+BjqZWc9IZRKR+DIprwvnT+7LUx99xqL1msSmMW2yj8DM8oDxwCf1\nHuoNfLHH1+v4elmIiOyzW08cTueOKdw2Y6HOLWhExIvAzDKAF4Hr3b1kH1/jCjMrMLOCoiJdVEpE\nmi+7YzJ3TR3JwvXFPPG+5jluSESLwMySCZXAdHef0cCQ9UDfPb7uE172Fe4+zd3z3T0/Nzc3MmFF\nJGadPLonp2ie40ZF8qghA54CCt39Z40MewX4TvjooYOAYnffGKlMIhK/7jp1JJ06ap7jhkRyjeBQ\n4EJgipnNC99ONrMrzezK8Jg3gNXASuD/gO9FMI+IxLHO6Sncf+Zolm4q5REdRfQVSZF6YXf/CLC9\njHHg6khlEBHZ05Rh3Tk3vy9PvL+Kk0b1ZHSf7KAjRQWdWSwiceX2b4aOIvrxq4sJ/S0qKgIRiStZ\nacncdMJQCtbs4NUF2iUJKgIRiUNn5/dlZK8s7n2jkPIq7ThWEYhI3ElMMO48ZSQbiyt0bgEqAhGJ\nU5MHdOGbY3ryxPurWLdjd9BxAqUiEJG4ddvJwzGDe99cGnSUQKkIRCRu9e7UgauODM1bMGvVtqDj\nBEZFICJx7T+OHEjvTh348auL4/aidCoCEYlracmJ3P6N4SzdVMpz/1wbdJxAqAhEJO6dNKoHBw/s\nykNvL2fHrqqg47Q5FYGIxD0z485TR1BSXs3P3l4edJw2pyIQEQGG9cji2wf259l/rmXllrKg47Qp\nFYGISNh1xw6mQ3Ii9/8lvg4nVRGIiITlZKRy1VEH8PaSzXy8On4OJ1URiIjs4dLDBtAzO4173iik\nri4+rk6qIhAR2UNaciL/dfxQFqwr5tUFG4KO0yZUBCIi9Zw+vjcjemZx/5tL2V1VE3SciFMRiIjU\nk5Bg/HjqSDYUV/DIO7E/raWKQESkAZPyunBOfh+e+vAzlm4qCTpORKkIREQacetJw8lMS+KOlxbF\n9I5jFYGISCO6pKdw28nDKVizg+cLvgg6TsSoCEREmnD2xD5MHtCFe99cytayyqDjRISKQESkCWbG\nPaePoryqljv/vDjoOBGhIhAR2YtB3TK57tjBvL5wI28u3Bh0nFanIhARaYYrjhjIyF5Z/PefF8Xc\npapVBCIizZCcmMADZ41l5+5q7nptSdBxWpWKQESkmUb0yuJ7Rw/ipbnreadwc9BxWk3EisDMfm1m\nW8xsUSOPZ5vZq2Y238wWm9nFkcoiItJarjl6EEO6Z3DHy4soq4yNy09Eco3gt8CJTTx+NbDE3ccC\nRwEPmVlKBPOIiOy3lKQE7jtzDJtKKnjwrWVBx2kVESsCd/8A2N7UECDTzAzICI+NjXoVkZg2oV9n\nvnNQf56e9Tlz1u4IOs5+C3IfwePAcGADsBC4zt3rAswjItJsN504jB5Zadz64gKqatr3r64gi+AE\nYB7QCxgHPG5mWQ0NNLMrzKzAzAqKioraMqOISIMyUpP46WmjWL65jF+9tyroOPslyCK4GJjhISuB\nz4BhDQ1092nunu/u+bm5uW0aUkSkMccM786pY3vx2LsrWLBuZ9Bx9lmQRbAWOAbAzLoDQ4HVAeYR\nEWmxn0wdRU5GKv/5x3mUV9UGHWefRPLw0eeAWcBQM1tnZpea2ZVmdmV4yE+AQ8xsIfAOcIu7b41U\nHhGRSMjumMyDZ49lVdEu7v/L0qDj7JOkSL2wu5+/l8c3AMdH6v1FRNrKYYNzuPjQPH7z98+ZMqwb\nRwxpX5uwdWaxiEgruOXEYQzqlsENz8/ni+27g47TIioCEZFWkJacyBMXTKCqppZLfvspxeXVQUdq\nNhWBiEgrGdQtkycunMjn23Zx5TOz2835BSoCEZFWdMgBOdx/5hhmrd7GrS8uwD365zqO2M5iEZF4\ndcaEPqzbUc7P3l5O/67pXHfs4KAjNUlFICISAd+fMojPt+3i4b8tJy+nI1PH9Q46UqO0aUhEJALM\njHvPGM3kAV246U8LKPi8qWtwBktFICISIalJifzvBRPp3bkDVzwzO2oPK1URiIhEUOf0FH590SSq\na+u49g9zqamNviOJVAQiIhE2ICede04fzdy1O3n0nRVBx/kaFYGISBs4ZWwvzp7Yh8dnruST1duC\njvMVKgIRkTbyo1NH0r9rOv/5x3kU746eM49VBCIibSQ9NYlHzhvHltJKrvtj9OwvUBGIiLShMX06\ncdfUUby3rIg7Xl4UFWce6x12UoQAAAklSURBVIQyEZE29q0D+7FhZzmPz1xJ704d+P4xwZ55rCIQ\nEQnAjccPYcPOch56ezndslI5d1K/wLKoCEREAmBm3HfmGLaUVnLLiwtZuqmU204aTkpS22+x1z4C\nEZGApCQl8OuLJnHRIaHZzc7531ms31ne5jlUBCIiAUpJSuBHp47kF9+awMotZXzz0Q8p3FjSphlU\nBCIiUeAbY3ryyjWHkpqUyAVPfsLKLWVt9t4qAhGRKDEwN4Pplx+ImfHtJz9mzbZdbfK+Fg3HsLZE\nfn6+FxQUBB1DRCRilm0q5bxps0hLTmRSXhdq3amrc44f2Z3Tx/fZp9c0s9nunt/QY1ojEBGJMkN7\nZPLMpQfSJT2FBet2UrixhJVbythWVhWR99PhoyIiUWhU72xev/bwNnkvrRGIiMQ5FYGISJxTEYiI\nxLmIFYGZ/drMtpjZoibGHGVm88xssZm9H6ksIiLSuEiuEfwWOLGxB82sE/BL4FR3HwmcHcEsIiLS\niIgVgbt/AGxvYsi3gBnuvjY8fkuksoiISOOC3EcwBOhsZu+Z2Wwz+06AWURE4laQ5xEkAROBY4AO\nwCwz+9jdl9cfaGZXAFcA9OsX3DW7RURiUZBFsA7Y5u67gF1m9gEwFvhaEbj7NGAagJkVmdma8EPZ\nQHET9+svSwa2tjDnnq/RnMfqL2tuxn/9N6eFGdsq37+W6TOMrnztIWO059ufjE0ti7bPsH+j7+zu\nEbsBecCiRh4bDrxDqIw6AouAUS18/WlN3a+/DCjYh+9hWkseq7+suRn3+G+LMrZVPn2G0ZmvPWSM\n9nz7k3EvWaPqM2zqFrE1AjN7DjgKyDGzdcCdhP6axN2fcPdCM/sLsACoA55090YPNW3Eq3u5X3/Z\nhBa+fv3XaM5j9Zc1N2NT79OUtsr3r/v6DKMrX2OPR1PGaM/X2OPNybi3ZS0R6c+wUe3u6qP7w8wK\nvJGr70WLaM8Y7fkg+jNGez6I/ozRng/aR8Z/ibczi6cFHaAZoj1jtOeD6M8Y7fkg+jNGez5oHxmB\nOFsjEBGRr4u3NQIREalHRSAiEudUBCIicU5FEGZmh5vZE2b2pJn9I+g8DTGzBDO728weM7PvBp2n\nvvDVZD8Mf45HBZ2nIWaWbmYFZvbNoLM0xMyGhz+/F8zsqqDzNMTMTjOz/zOzP5rZ8UHnqc/MBprZ\nU2b2QtBZ/iX8c/d0+HP7dtB56ouJImjsktdmdqKZLTOzlWZ2a1Ov4e4fuvuVwGvA09GYEZgK9AGq\nCZ2ZHW35HCgD0qI0H8AtwPOtma01M7p7Yfjn8Bzg0CjN+LK7Xw5cCZwbhflWu/ulrZmrIS3Megbw\nQvhzOzXS2VqsJWe+ResNOILQiU6L9liWCKwCBgIpwHxgBDCa0C/7PW/d9nje80BmNGYEbgX+I/zc\nF6IwX0L4ed2B6VGY7zjgPOAi4JvR+P84/JxTgTeBb0VrxvDzHgImRHG+Vv03sp9ZbwPGhcc8G8lc\n+3KLicnr3f0DM8urt3gysNLdVwOY2R+Aqe5+L9DgZgEz6wcUu3tpNGYMn6FdFf6yNtry7WEHkBpt\n+cKbq9IJ/cMsN7M33L0umjKGX+cV4BUzex14trXytVZGMzPgPuBNd58TbfnaSkuyElpD7gPMIwq3\nxMREETSiN/DFHl+vAw7cy3MuBX4TsURf19KMM4DHzOxw4INIBgtrUT4zOwM4AegEPB7ZaEAL87n7\n7QBmdhGwtTVLoAkt/QyPIrQZIRV4I6LJvtTSn8PvA8cC2WY2yN2fiGQ4Wv4ZdgXuBsab2W3hwmgr\njWV9FHjczL7Bvl+CImJiuQhazN3vDDpDU9x9N6GyikruPoNQWUU1d/9t0Bka4+7vAe8FHKNJ7v4o\noV9sUcndtxHafxE1PHSV5YuDztGYqFtFaUXrgb57fN0nvCyaRHtG5dt/yrj/oj3fntpT1n+L5SL4\nFBhsZgPMLIXQTsJXAs5UX7RnVL79p4z7L9rz7ak9Zf1S0HurW2nv/XPARr48rPLS8PKTCU10swq4\nXRmVTxmjO2O052uvWfd200XnRETiXCxvGhIRkWZQEYiIxDkVgYhInFMRiIjEORWBiEicUxGIiMQ5\nFYHEBDMra+P3e9LMRrTSa9Wa2TwzW2Rmr5pZp72M72Rm32uN9xYBTV4vMcLMytw9oxVfL8nda1rr\n9fbyXv/ObmZPA8vd/e4mxucBr7n7qLbIJ7FPawQSs8ws18xeNLNPw7dDw8snm9ksM5trZv8ws6Hh\n5ReZ2Stm9i7wjoVmXHvPQrOFLTWz6eFLMBNenh++X2ahmePmm9nHZtY9vPyA8NcLzeynzVxrmUXo\nCpaYWYaZvWNmc8KvMTU85j7ggPBaxAPhsTeFv8cFZvbjVvwYJQ6oCCSWPQI87O6TgDOBJ8PLlwKH\nu/t44IfAPXs8ZwJwlrsfGf56PHA9oTkMBtLwrGHpwMfuPpbQ5cEv3+P9H3H30TRjxjYzSwSO4ctr\n01QAp7v7BOBo4KFwEd0KrHL3ce5+k4WmixxM6Fr444CJZnbE3t5P5F90GWqJZccCI8J/xANkmVkG\nkA08bWaDCU2vmbzHc9529+17fP1Pd18HYGbzgDzgo3rvU0VodiyA2YRmQgM4GDgtfP9Z4MFGcnYI\nv3ZvoBB4O7zcgHvCv9Trwo93b+D5x4dvc8NfZxAqhraYs0JigIpAYlkCcJC7V+y50MweB2a6++nh\n7e3v7fHwrnqvUbnH/Voa/jdT7V/ubGtsTFPK3X2cmXUE3gKuJnS9/28DucBEd682s88JzQddnwH3\nuvv/tvB9RQBtGpLY9ldCs2kBYGbjwnez+fIa8RdF8P0/JrRJCkKXI26ShyYeuha40cySCOXcEi6B\no4H+4aGlQOYeT30LuCS8toOZ9Tazbq30PUgcUBFIrOhoZuv2uN1A6JdqfngH6hK+nLXqf4B7zWwu\nkV0rvh64wcwWAIOA4r09wd3nAguA84HphPIvBL5DaN8GHpqB6+/hw00fcPe/Etr0NCs89gW+WhQi\nTdLhoyIREt7UU+7ubmbnAee7+9S9PU+krWkfgUjkTCQ0YbkBO4FLAs4j0iCtEYiIxDntIxARiXMq\nAhGROKciEBGJcyoCEZE4pyIQEYlzKgIRkTj3/4kjxBX+6ojkAAAAAElFTkSuQmCC\n", "text/plain": [ "
" ] }, "metadata": { "tags": [] }, "output_type": "display_data" } ], "source": [ "learn.lr_find()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Let's use a learning rate around 1e-1 (0.1)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "data": { "text/html": [ "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
epochtrain_lossvalid_lossaccuracytime
00.2091750.1591020.95120001:14
10.1236670.0664940.97850001:14
20.0665890.0388870.98830001:14
" ], "text/plain": [ "" ] }, "metadata": { "tags": [] }, "output_type": "display_data" } ], "source": [ "learn.fit_one_cycle(3, lr_max=1e-1)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Simplify it" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "* Try to make it more like `ResNet`.\n", "* `ConvLayer` contains a `Conv2d`, `BatchNorm2d`, and an activation function" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "def conv2(ni, nf): return ConvLayer(ni, nf, stride=2)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "And make a new model" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "net = nn.Sequential(\n", " conv2(1,8),\n", " conv2(8,16),\n", " conv2(16,32),\n", " conv2(32,16),\n", " conv2(16,10),\n", " Flatten()\n", ")" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Great! That looks much better to read! Let's make sure we get (roughly) the same results with it." ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "learn = Learner(dls, net, loss_func=CrossEntropyLossFlat(), metrics=accuracy)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "data": { "text/html": [ "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
epochtrain_lossvalid_lossaccuracytime
00.2207340.1971680.93390001:15
10.1303330.0757140.97580001:14
20.0787640.0411040.98700001:15
" ], "text/plain": [ "" ] }, "metadata": { "tags": [] }, "output_type": "display_data" } ], "source": [ "learn.fit_one_cycle(3, lr_max=1e-1)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Almost the exact same! Perfect! Now let's get a bit more advanced" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# ResNet (kinda)\n", "\n", "The ResNet architecture is built with what are known as ResBlocks. Each of these blocks consist of two `ConvLayers` that we made before, where the number of filters do not change. Let's generate these layers. " ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "class ResBlock(Module):\n", " def __init__(self, nf):\n", " self.conv1 = ConvLayer(nf, nf)\n", " self.conv2 = ConvLayer(nf, nf)\n", " \n", " def forward(self, x): return x + self.conv2(self.conv1(x))" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "* Class notation\n", "* `__init__`\n", "* `foward`" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Let's add these in between each of our `conv2` layers of that last model. " ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "net = nn.Sequential(\n", " conv2(1,8),\n", " ResBlock(8),\n", " conv2(8,16),\n", " ResBlock(16),\n", " conv2(16,32),\n", " ResBlock(32),\n", " conv2(32,16),\n", " ResBlock(16),\n", " conv2(16,10),\n", " Flatten()\n", ")" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "Sequential(\n", " (0): ConvLayer(\n", " (0): Conv2d(1, 8, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n", " (1): BatchNorm2d(8, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", " (2): ReLU()\n", " )\n", " (1): ResBlock(\n", " (conv1): ConvLayer(\n", " (0): Conv2d(8, 8, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n", " (1): BatchNorm2d(8, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", " (2): ReLU()\n", " )\n", " (conv2): ConvLayer(\n", " (0): Conv2d(8, 8, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n", " (1): BatchNorm2d(8, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", " (2): ReLU()\n", " )\n", " )\n", " (2): ConvLayer(\n", " (0): Conv2d(8, 16, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n", " (1): BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", " (2): ReLU()\n", " )\n", " (3): ResBlock(\n", " (conv1): ConvLayer(\n", " (0): Conv2d(16, 16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n", " (1): BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", " (2): ReLU()\n", " )\n", " (conv2): ConvLayer(\n", " (0): Conv2d(16, 16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n", " (1): BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", " (2): ReLU()\n", " )\n", " )\n", " (4): ConvLayer(\n", " (0): Conv2d(16, 32, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n", " (1): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", " (2): ReLU()\n", " )\n", " (5): ResBlock(\n", " (conv1): ConvLayer(\n", " (0): Conv2d(32, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n", " (1): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", " (2): ReLU()\n", " )\n", " (conv2): ConvLayer(\n", " (0): Conv2d(32, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n", " (1): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", " (2): ReLU()\n", " )\n", " )\n", " (6): ConvLayer(\n", " (0): Conv2d(32, 16, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n", " (1): BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", " (2): ReLU()\n", " )\n", " (7): ResBlock(\n", " (conv1): ConvLayer(\n", " (0): Conv2d(16, 16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n", " (1): BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", " (2): ReLU()\n", " )\n", " (conv2): ConvLayer(\n", " (0): Conv2d(16, 16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n", " (1): BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", " (2): ReLU()\n", " )\n", " )\n", " (8): ConvLayer(\n", " (0): Conv2d(16, 10, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n", " (1): BatchNorm2d(10, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", " (2): ReLU()\n", " )\n", " (9): Flatten()\n", ")" ] }, "execution_count": null, "metadata": { "tags": [] }, "output_type": "execute_result" } ], "source": [ "net" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Awesome! We're building a pretty substantial model here. Let's try to make it **even simpler**. We know we call a convolutional layer before each `ResBlock` and they all have the same filters, so let's make that layer!" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "def conv_and_res(ni, nf): return nn.Sequential(conv2(ni, nf), ResBlock(nf))" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "net = nn.Sequential(\n", " conv_and_res(1,8),\n", " conv_and_res(8,16),\n", " conv_and_res(16,32),\n", " conv_and_res(32,16),\n", " conv2(16,10),\n", " Flatten()\n", ")" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "And now we have something that resembles a ResNet! Let's see how it performs" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "learn = Learner(dls, net, loss_func=CrossEntropyLossFlat(), metrics=accuracy)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "data": { "text/html": [], "text/plain": [ "" ] }, "metadata": { "tags": [] }, "output_type": "display_data" }, { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEKCAYAAAAfGVI8AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjIsIGh0\ndHA6Ly9tYXRwbG90bGliLm9yZy8li6FKAAAgAElEQVR4nO3dd3wc9Z3G8c93Va3uInfLFXdjGwsb\nYjqEFoJpR0IoR0kcCAkQyoWEO7j0QsIBgYOYTkLJAaZDIBCDKaYI94Z7b3KVq+r3/tgFjFC1NJqV\n9nm/XvtiNfPbncfC1qOZ2ZmfuTsiIpK4ImEHEBGRcKkIREQSnIpARCTBqQhERBKcikBEJMGpCERE\nElxy2AEaq1OnTt6nT5+wY4iItCqffPLJZnfPr2ldqyuCPn36UFRUFHYMEZFWxcxW1rZOh4ZERBKc\nikBEJMGpCEREEpyKQEQkwakIREQSnIpARCTBqQgkru0pq+C9JZtZsmkn5ZVVYccRaZNa3XUE0vbt\n3FfOrNU7mDxjDf+Yu4E9ZZUAJEeMPp0yyW2XQnLESE2OkJeRSu8OGRR0zKB7bjsy0pLISE0iOz2F\nbjnpRCIW8p9GJP4FVgRm1gt4FOgCODDJ3e+oNuYY4HlgeWzRZHf/RVCZ5Mv2llXy6cadzF9XwpJN\nu+iUncrAztkM7JJNl9w0UpMimBmlFZXMX1fCrNXbWVK8i7KKKiqqnKoqp11qEpmpyWSmJZOWEiEl\nEiE5yUhJipCaHCE1KUJ6ShJ5GSm0z0ilXUoS89eXMGP1Nmat3s6eskoiZiRFjF37Kli3Yy8791UA\nkJ2ezOkju3PSsK5s21PG4k27WLppF7vLKiivdHaVVrByyx5embOeyqqvTrCUnZ7MsO45DO+ey+H9\nO3JYv45kpul3H5HqgvxXUQFc5+7TzSwb+MTM/unu86uNe8fdTwswR0Jzd/aVV1FWWUVZRRWrtu7h\nncXFvLN4MzNXb//8B2h6SoR95V8+9JIcMTJSk9hbXkl5ZXRcXkYK6clJJEWMSAT2lVexu7Ti89/a\nGyolyRjaPZcOmalUVjnu0D4jlcP6daBbXjv6dcrkqIH5pKck1fte5ZVVrNu+lw079rGnvJK9ZZVs\n21PGgvUlzFlbwqMfrOT+d5eTkmQc2qcDQ7rlkNsuhdx2KXTITKVrbjpdc9LJz07DDNzBDNKS69+2\nSFsQWBG4+3pgfez5TjNbAPQAqhdBiyjZV86UhZv45sHd4/ZwwY495Ux6Zymvzt3AWaN78L2j+n3+\nw8jdmbl6O7NWb2dDSSkbS/axp6yCrjnpdMtrR8/27RjTuz3dctsBsLu0gr9/vJoH3l3O2u17v7Qd\nMzi4Zx6XH92PET3yGNY9h57t27GztILFG3exeONOtuwuY09ZBbtLK0lPSWJUr1xG9sr7/P2rq6py\nyiqrKK+soqLSKa/8onz2lleyY085W/eUsbu0ggGdsxnWPadBP+QbIiUpQu+OmfTumFnj+tKKSopW\nbOPtRcVMXVTM4x+uYm95/cWVkZpEfnYaXbLTGVWQx4RR3RnaLQez+Pz7I3KgrCXmLDazPsBUYLi7\nl+y3/BjgGWANsA643t3n1fVehYWFfiD3Gnp2xhp+/PdZPHPF1xjTu32jXx+kbbvL+OsHK7nvnWXs\nKq1gWPcc5q4toW+nTP7rtCFs3lXGo9NWMHdt9FuXkmR0zk6nXWoSG0v2fX4oBaB3xwxG9MjlncWb\n2bG3nLF9OnD0oHzSkiOkJUfomJXG4f060j4zNaQ/bXwoq6hix95ytu4uY0PJPjbs2MvmXWVfGrN1\ndxmbdpayYcdeZqzaTkWVc1DnLE4e3pVh3XM/L1AVg7QGZvaJuxfWtC7wA6ZmlkX0h/01+5dAzHSg\nt7vvMrNTgeeAg2p4j4nARICCgoIDynH8kC6kJkV4Zc76uCiC3aUVvLFgI8/PXMfURcVUVDknDu3C\ntScOZHDXHKYuKuaWF+Zx6cPR0hvYJYtfnTGcE4d1oVNm2pf2anbuK2fllj18sGwLHyzbygfLtnJY\nvw58/+j+HFIQ/p81HqUmR8jPTiM/O41BXbPrHb9tdxkvz1nPczPWcveUJXx2SiIvI4VxfTvwtf6d\nGD+gI/3zs1QM0uoEukdgZinAS8Br7n5bA8avAArdfXNtYw50jwDgu48UMW/dDt77yXEtcnho2+4y\nnvpkNdnpKfTumEGv9hnMWrOdl2evZ8qnm9hXXkX33HS+ObI7Z4zuwZBuOV96fWlFJS/OWk/3vHQO\n79dRP2DixGcn2eet28HMVdt5f+mWzw+/5bZLYWSvPEb1yuPogfkcUpCn/28SF+raIwisCCz6t/8R\nYKu7X1PLmK7ARnd3MxsLPE10D6HWUE0pgtoOD1VVOWY02z9Yd+e5mWv55UsL2Lq77Cvr87PTOHV4\nV75xcHcKe7eP23MW0nCrt+5h2tItzFi9jRmrtrNo406qHHp1aMeEkT04e0xP+naq+RyGSEsI69DQ\neOBCYI6ZzYwt+xlQAODu9wLnAFeYWQWwF/h2XSXQVCcM6UJqcoSXZ39xeKisoooJd7/H1t2lnDys\nKycP78bYvh1IasAPZ3eneFcpq7bsoXhnKTtLK9i1r4Ipn27incWbGdUrj79dNo6cdsms2LyHVVv3\n0LdTZoPfX1qPXh0y6NUhg3MP7QVEP5zw+ryNPD9zLf/71hLufmsJJw7twuVH92e0DtdJnGmRk8XN\nqSl7BPDVw0N/eXspv311IeMHdKRoxTZKK6oY0DmLhy85lJ7tM77y+lVb9vDq3PW8Nm8DCzfsrPFj\nk9lpydxw8iDOH9dbP/CFTSX7+OsHK3l02kp27C1nXN8O/PC4ARwxoJMOG0mLCeXQUFCaWgTPzVjL\nNX+fyTNXHE6PvAyO/9NbHN6/I/f/+6Gfn8D9r+fmkpmWzF8vG8uAztm4O1M+3cTtbyxm9podAIzo\nkUthn/b07pBB746ZdMlJJzs9mez0ZLLSkklO0t075Mt2l1bwxEeruP+d5Wwo2cfInrlceewAThjS\nRYcHJXAqgv3s3FfOmF+9wQXjerNldymvzt3AP3981Jc+g75gfQkXPvARlVVV3PLNYfxf0WreX7qF\nPh0zOH9cb04e3pVeHb66tyDSEKUVlUyevpZ73lrKqq17GNA5i+8d2ZcJo3o027UVItWpCKr53qNF\nTFu6hV2lFVx13ACuPXHQV8as3LKbCx74kNVb99I+I4VrThjId8YVkKLf9KWZVFRW8fKc9fzl7WXM\nX19Cp6w0fnBMfy48vLf+nkmzUxFU89nhoR557Xjj2qNpl1rzb2GbSvbx2vyNTBjVnZz0lCZtU6Q2\n7s77S7dw95QlvL90C/3yM7np1CEcN7izziFIs1ERVLOrtIIL7v+Qa044iGMGdW6mZCJN89m5qF+9\nvIBlxbs5YkAnbvrGkK9cXyJyIFQEIq1IeWUVf/tgJXe8uZgde8s5d0wvrjtxIJ1z0sOOJq1YXUWg\nA5EicSYlKcIl4/vy9vXHctn4vkyesYbj//Q27y6u9YJ7kSZREYjEqdyMFP7ztKG8/uOj6Z7Xjosf\n+oinilaHHUvaIBWBSJzr2ymTp644nMP6deSGp2dz2z8X0doO6Up8UxGItAI56Sk8dMmh/NuYntz5\n5mKuenImexs5GZBIbTRvn0grkZIU4Q/nHEy//Cz+8NpCVmzezaSLxtQ6WZBIQ2mPQKQVMTOuOKY/\n911YyLLiXZx+13vMXbsj7FjSyqkIRFqhE4Z24dkrx5OaFOGiBz9iafGusCNJK6YiEGmlBnbJ5m/f\nHUfE4KIHPmL9jr31v0ikBioCkVasb6dMHr5kLDv2lnPRAx+xrYaJkETqoyIQaeWG98jlvosKWbl1\nD5c98jH7yvVpImmcwIrAzHqZ2RQzm29m88zs6jrGHmpmFWZ2TlB5RNqyw/t35I5vjWL6qu1c/9Qs\nqqp0nYE0XJB7BBXAde4+FDgMuNLMhlYfZGZJwO+B1wPMItLmnTKiGzeeMpiXZq/ntn8uCjuOtCKB\nXUfg7uuB9bHnO81sAdADmF9t6I+AZ4BDg8oikii+f1Q/Vm7ZzV1TllDQMYNzC3uFHUlagRa5oMzM\n+gCjgQ+rLe8BnAkcSx1FYGYTgYkABQUFQcUUafXMjF9MGM6abXu58ZnZVFU53x6rfzNSt8BPFptZ\nFtHf+K9x95Jqq28HfuLuVXW9h7tPcvdCdy/Mz88PKqpIm5CSFOHeC8Zw1MB8bpw8h9vf0L2JpG6B\nFoGZpRAtgcfcfXINQwqBJ81sBXAO8L9mdkaQmUQSQWZaMvddVMg5Y3py+xuL+dmzc6jUCWSpRWCH\nhiw6x94DwAJ3v62mMe7ed7/xDwMvuftzQWUSSSQpSRFuPedguuSkcfeUpaQlJ/Hfpw8LO5bEoSDP\nEYwHLgTmmNnM2LKfAQUA7n5vgNsWEaLnDG44aTD7yqt44N3lFHTI4NIj+tb/QkkoQX5q6F2gwTNv\nu/vFQWURSXQ/O3UIa7bt4Zcvz6dH+3acNKxr2JEkjujKYpEEkBQxbv/WaEb2zOPqJ2cwfdW2sCNJ\nHFERiCSIdqlJ3P/vhXTOTueShz5m0cadYUeSOKEiEEkgnbLS+Ntl40hLjnDhAx+yeuuesCNJHFAR\niCSYgo4Z/PWycewrr+LCBz6keGdp2JEkZCoCkQQ0qGs2D158KBtLSvnuo0WUVdR5Tae0cSoCkQQ1\npnd7bjt3JLNWb+e3ry4IO46ESEUgksBOGdGNS8b34aH3VvDqnPVhx5GQqAhEEtxPTxnCyF55/MfT\ns1mxeXfYcSQEKgKRBJeaHOHu74wmEjF+8Nh0zXCWgFQEIkLP9hncdu5I5q8v4ecvVp8yRNo6FYGI\nAHD8kC5cfnR/nvhoFc/OWBN2HGlBKgIR+dz1Jw5kbN8O/GzyXBbryuOEoSIQkc8lJ0X483mjyUxL\n4orHprOnrCLsSNICVAQi8iVdctK549ujWVq8i1/ofEFCUBGIyFeMH9CJK47uz5Mfr+bl2bq+oK0L\nrAjMrJeZTTGz+WY2z8yurmHMBDObbWYzzazIzI4IKo+INM6Pvz6QUb3yuHHybNZs083p2rIg9wgq\ngOvcfShwGHClmQ2tNuZNYKS7jwIuBe4PMI+INEJKUoQ7vz0ad7j6yZlUVOp+RG1VYEXg7uvdfXrs\n+U5gAdCj2phd7v7ZjNqZgGbXFokjBR0z+PWZw/lk5TZuff3TsONIQFrkHIGZ9QFGAx/WsO5MM1sI\nvEx0r6Cm10+MHToqKi4uDjKqiFQzYVQPvjOugL+8vYyXZq8LO44EIPAiMLMs4BngGncvqb7e3Z91\n98HAGcAva3oPd5/k7oXuXpifnx9sYBH5iv/+5jDG9G7PDU/NZsH6r/wzllYu0CIwsxSiJfCYu0+u\na6y7TwX6mVmnIDOJSOOlJke45/xDyGmXzMS/FrF9T1nYkaQZBfmpIQMeABa4+221jBkQG4eZHQKk\nAVuCyiQiB65zTjr3XDCGjTtKuerJmVRW6ZReWxHkHsF44ELguNjHQ2ea2almdrmZXR4bczYw18xm\nAncD39rv5LGIxJlDCtpzy+lDmbqomDvfXBx2HGkmyUG9sbu/C1g9Y34P/D6oDCLS/L4ztoDpK7dz\n578WM6ogj2MHdQ47kjSRriwWkUYxM351xnAGd83hmidnsnqrLjZr7VQEItJo7VKTuPeCQ6hy58rH\np+tis1ZORSAiB6R3x0x+d9bBzF6zg0nvLAs7jjSBikBEDtg3Du7GKcO7cvsbi1myaVfYceQAqQhE\npEl+PmEY7VKS+Mkzs/WR0lZKRSAiTdI5O52bTxvKJyu38ei0FWHHkQOgIhCRJjvrkB4cPTCfP/zj\nU5YV6xBRa6MiEJEmMzN+d/YI0lIi/OiJGZRWVIYdSRpBRSAizaJbbjtuPWck89aV8PtXdcvq1kRF\nICLN5utDu3Dx1/rw4HvLeXPBxrDjSAOpCESkWd14ymCGdsvh+qdmsWHHvrDjSAOoCESkWaWnJPHn\n74xmX3kVP3lmNrqPZPxTEYhIs+ufn8VPTx3M24uKeeKj1WHHkXqoCEQkEBeM6834AR351cvzWbVF\nN6aLZyoCEQlEJGL84ZyRJJlx/dOzqNJVx3EryBnKepnZFDObb2bzzOzqGsacb2azzWyOmb1vZiOD\nyiMiLa9HXjtu/uZQPlq+lQffWx52HKlFkHsEFcB17j4UOAy40syGVhuzHDja3UcQnbh+UoB5RCQE\n54zpyfGDO3Pra5+yVFcdx6XAisDd17v79NjzncACoEe1Me+7+7bYlx8APYPKIyLhMDN+c9YI0lOS\nuOGpWboxXRxqkXMEZtYHGA18WMewy4BXWyKPiLSsLjnp/PfpQ5m+ajsPvqtDRPEm8CIwsyzgGeAa\ndy+pZcyxRIvgJ7Wsn2hmRWZWVFxcHFxYEQnMGaN6cMKQLvzxdR0iijeBFoGZpRAtgcfcfXItYw4G\n7gcmuPuWmsa4+yR3L3T3wvz8/OACi0hgooeIhtMuVYeI4k2Qnxoy4AFggbvfVsuYAmAycKG7Lwoq\ni4jEh87Z6dzyzeghor9OWxF2HIkJco9gPHAhcJyZzYw9TjWzy83s8tiYm4GOwP/G1hcFmEdE4sAZ\no2JzF7z2KWu37w07jgDW2u4DUlhY6EVF6guR1mzNtj2c+D9TGdu3Aw9dfCjRAwgSJDP7xN0La1qn\nK4tFpMX1bJ/BDScN4q1Pi3lh1rqw4yQ8FYGIhOKiw/swqlceP39xPlt3l4UdJ6GpCEQkFEkR4/dn\nH0zJ3nJ+88qCsOMkNBWBiIRmUNdsJh7Vj6c/WcO0pTV+elxagIpAREJ11fEHUdAhg5uencO+ck16\nHwYVgYiEKj0liV+dMZxlm3dzz1tLw46TkFQEIhK6owbmM2FUd+55aylLNun2Ey1NRSAiceG/ThtK\nekqEm5+fq3mOW1iDisDM+ptZWuz5MWZ2lZnlBRtNRBJJp6w0/uPkwby/dIuuLWhhDd0jeAaoNLMB\nRCeP6QU8HlgqEUlI540tYGTPXH750gJ27C0PO07CaGgRVLl7BXAm8Gd3vwHoFlwsEUlESRHj12eO\nYOvuUv70+qdhx0kYDS2CcjM7D/h34KXYspRgIolIIhveI5eLDu/DXz9Yyew128OOkxAaWgSXAIcD\nv3b35WbWF/hrcLFEJJFde+JAOmWl8V/PzaVK8xYErkFF4O7z3f0qd3/CzNoD2e7++4CziUiCyklP\n4aZThzBrzQ6e/Hh12HHavIZ+augtM8sxsw7AdOA+M6txshkRkeYwYVR3xvXtwB9eW6ib0gWsoYeG\ncmPzDZ8FPOru44ATgoslIonOzPjlGcPZta+CP/xjYdhx2rSGFkGymXUDzuWLk8V1MrNeZjbFzOab\n2Twzu7qGMYPNbJqZlZrZ9Y3ILSIJYGCXbC49oi9Pfrya6au2hR2nzWpoEfwCeA1Y6u4fm1k/YHE9\nr6kArnP3ocBhwJVmNrTamK3AVcAfG5FZRBLIVccfRJecNG55fp5OHAekoSeLn3L3g939itjXy9z9\n7Hpes97dp8ee7wQWAD2qjdnk7h8DunJERGqUlZbMT08Zwpy1O5g8Y23Ycdqkhp4s7mlmz5rZptjj\nGTPr2dCNmFkfYDTw4YGENLOJZlZkZkXFxcUH8hYi0oqdPrI7I3vlcetrC9lTVhF2nDanoYeGHgJe\nALrHHi/GltXLzLKI3qLimtgJ50Zz90nuXujuhfn5+QfyFiLSikUixs2nDWFjSSn3vr0s7DhtTkOL\nIN/dH3L3itjjYaDen8hmlkK0BB5z98lNyCkiCW5M7w6cdnA3Jk1dyrrte8OO06Y0tAi2mNkFZpYU\ne1wA1DmvnJkZ8ACwwN11zYGINNmNpwymytHHSZtZQ4vgUqIfHd0ArAfOAS6u5zXjgQuB48xsZuxx\nqpldbmaXA5hZVzNbA1wL/KeZrTGznAP5g4hI29ezfQbfPaIvz81cx8INB3SkWWqQ3JBB7r4SOH3/\nZWZ2DXB7Ha95F7B63ncD0OCTziIiE4/qx6PTVvLnfy3h7u8cEnacNqEpM5Rd22wpREQaKC8jlYsO\n780rc9azZNPOsOO0CU0pgjp/2xcRCcp3j+xHu5Qk7vrXkrCjtAlNKQJd4icioeiQmcqFh/XmhVnr\nWL55d9hxWr06i8DMdppZSQ2PnUSvJxARCcV3j+xHanKEu6dor6Cp6iwCd89295waHtnu3qATzSIi\nQcjPTuM7Y3vz7Iy1rNyivYKmaMqhIRGRUF1+dD9SkyL88fVFYUdp1VQEItJqdc5J57Ij+vLirHWa\n37gJVAQi0qp9/+h+dMhM5XevLsRdn2E5ECoCEWnVstNT+NFxA3h/6RbeXqS7Ex8IFYGItHrnj+tN\nQYcMfvfqQio1eU2jqQhEpNVLTY5w/UmDWLhhJ8/P1OQ1jaUiEJE24bQR3RjSLYe7pizRXkEjqQhE\npE2IRIwfHTeAZcW7eXXu+rDjtCoqAhFpM04e1pUBnbO4619LNNF9I6gIRKTNiESMK4/tz8INO3lz\n4aaw47QagRWBmfUysylmNt/M5pnZ1TWMMTO708yWmNlsM9PNxUWkSb55cHcKOmRw178W67qCBgpy\nj6ACuM7dhwKHAVea2dBqY04BDoo9JgL3BJhHRBJAclKEHxzTn1lrdjB18eaw47QKgRWBu6939+mx\n5zuBBUCPasMmAI961AdAnpl1CyqTiCSGsw7pSbfcdO54Y5H2ChqgRc4RmFkfYDTwYbVVPYDV+329\nhq+WBWY20cyKzKyouFhXDopI3VKTI/zwuAFMX7WdKZ/qXEF9Ai8CM8sCngGucfcDmm3a3Se5e6G7\nF+bn5zdvQBFpk84t7EVBhwz++NoifYKoHoEWgZmlEC2Bx9x9cg1D1gK99vu6Z2yZiEiTpCRF+PHX\nD2L++hJe0XUFdQryU0MGPAAscPfbahn2AnBR7NNDhwE73F3/x0SkWZw+sgcDu2Rx2z8XUVFZFXac\nuBXkHsF44ELgODObGXucamaXm9nlsTGvAMuAJcB9wA8CzCMiCSYpYlz79UEsK97N5Bk62FCbwKab\ndPd3AatnjANXBpVBROSkYV04uGcud7yxmNNHdic9JSnsSHFHVxaLSJtmZtx4ymDWbt/LA+8uDztO\nXFIRiEib97X+nThpWBfunrKEjSX7wo4Td1QEIpIQbjp1KBWVzu//sTDsKHFHRSAiCaGgYwaXHdmX\nydPXMnO1Jrrfn4pARBLGlccOID87jZ+/OE+3ntiPikBEEkZWWjI3nDSIGau28+rcDWHHiRsqAhFJ\nKGcf0pMBnbP4n38u0pSWMSoCEUkoSRHjxycMZPGmXbw4a13YceKCikBEEs4pw7sypFsOt7+xiHLd\nekJFICKJJxIxrvv6QFZs2cPk6WvCjhM6FYGIJKTjh3RmZK887nxzCaUVlWHHCZWKQEQSkll0r2Dt\n9r08/Uli7xWoCEQkYR15UCeG98jhkfdXJPR1BSoCEUlYZsZFh/dh0cZdTFu2Jew4oVERiEhCO31k\nd9pnpPDo+yvDjhKaIGcoe9DMNpnZ3FrWtzezZ81stpl9ZGbDg8oiIlKb9JQkvnVoAa/P38Da7XvD\njhOKIPcIHgZOrmP9z4CZ7n4wcBFwR4BZRERqdcFhBQA89kFi7hUEVgTuPhXYWseQocC/YmMXAn3M\nrEtQeUREatOzfQYnDOnCkx+vZl954n2UNMxzBLOAswDMbCzQG+hZ00Azm2hmRWZWVFxc3IIRRSRR\nXPy1PmzdXcZLs9eHHaXFhVkEvwPyzGwm8CNgBlBjFbv7JHcvdPfC/Pz8lswoIgni8P4dOahzFg++\nuzzhPkoaWhG4e4m7X+Luo4ieI8gHloWVR0QSm5nx3SP7Mn99Ce8tSayPkoZWBGaWZ2apsS+/C0x1\n95Kw8oiInDG6B52y0vjL1KVhR2lRQX589AlgGjDIzNaY2WVmdrmZXR4bMgSYa2afAqcAVweVRUSk\nIdKSk7hkfB/eWbyZBesT5/fS5KDe2N3Pq2f9NGBgUNsXETkQ548r4O4pS7hv6jJu+9aosOO0CF1Z\nLCKyn7yMVM4t7MULs9axfkdiXGCmIhARqeayI/riwEPvrQg7SotQEYiIVNOrQwanjujG3z5YycaS\nfWHHCZyKQESkBtefOJCKSue3rywIO0rgVAQiIjXo3TGTiUf147mZ6/hoeV13y2n9VAQiIrX4wbH9\n6Z6bzi0vzKOyqu1ebawiEBGpRUZqMjd9YygL1pfw+Idt986kKgIRkTqcOqIrX+vfkVtf+5TnZ65t\nk3sGKgIRkTqYGb8+cwRdc9O5+smZnHDb2zxVtJqqNlQIKgIRkXr07ZTJP64+insvOIR2KUnc8PRs\nJr3Tdu6RqSIQEWmASMQ4eXg3Xr7qCI4ZlM89by2lZF952LGahYpARKQRzIzrvj6IHXvLefDd5WHH\naRYqAhGRRhrRM5cTh3bhgXeWs31PWdhxmkxFICJyAK49cSC7yiq4rw2cK1ARiIgcgMFdc/jGiG48\n9N4KtuwqDTtOkwQ5Mc2DZrbJzObWsj7XzF40s1lmNs/MLgkqi4hIEK45YSD7yiu5a8qSsKM0SZB7\nBA8DJ9ex/kpgvruPBI4B/rTf1JUiInFvQOcsvj22gEenrWTeuh1hxzlggRWBu08F6rpTkwPZZmZA\nVmxsRVB5RESC8JOTBtM+I4WfPTu31V51HOY5gruIzlu8DpgDXO3uVSHmERFptNyMFP7zG0OZtXp7\nq70fUZhFcBIwE+gOjALuMrOcmgaa2UQzKzKzouLi4pbMKCJSrwmjunPEgE784R+fsqkVTmQTZhFc\nAkz2qCXAcmBwTQPdfZK7F7p7YX5+fouGFBGpj5nxyzOGU1pZxc3Pz8O9dR0iCrMIVgHHA5hZF2AQ\n0Po/kCsiCalvp0yu/fpA/jFvA3e8uTjsOI2SHNQbm9kTRD8N1MnM1gC3ACkA7n4v8EvgYTObAxjw\nE3ffHFQeEZGgff+ofizZtIvb31hMr/YZnD2mZ9iRGiSwInD38+pZvw44Majti4i0NDPjN2eOYN32\nvdw4eTZdc9Pp3TGDVVv2ULyrlKMOyqd9Zvx9Sj6wIhARSUSpyRHuuWAMZ9/zPuff/+GX1mWnJTPx\nqH5cekRfMtPi58evtbaTGmq12yQAAAndSURBVIWFhV5UVBR2DBGROm3YsY+nP1lNx6w0enfIIC0l\nib+8vZTX52+kU1Yq/3HSYP6tsCfRS6mCZ2afuHthjetUBCIiLWf6qm389pUFfLxiG8cP7sxvzxpB\n55z0wLdbVxHopnMiIi3okIL2/H3i4dx82lDeXbKZE2+fyuvzNoSaSUUgItLCIhHj0iP68vJVR9Kr\nfQY/fHwGc9eGd68iFYGISEgGdM7ikUvH0iEzlR8+Pp1dpeHcbk1FICISog6Zqdx53mhWbd3DTc/O\nCeWqZBWBiEjIxvbtwI9PGMjzM9fxVNGaFt++ikBEJA784NgBjB/QkZtfmMuy4l0tum0VgYhIHEiK\nGP9z7ijSkpO4/qlZLTq3gYpARCROdM5J5xcThjF91Xbue6fl7sGpIhARiSOnj+zOycO6ctvri1i0\ncWeLbFNFICISR8yMX505nKz0ZK77v1mUVwY/caOKQEQkznTKSuPXZwxnztod3D1lSeDbUxGIiMSh\nU0Z044xR3fnzv5Ywa/X2QLelIhARiVM/nzCcztlp/PjvM9lbVhnYdgIrAjN70Mw2mdncWtbfYGYz\nY4+5ZlZpZh2CyiMi0trktkvhj/82kmWbd/O7VxcEtp0g9wgeBk6ubaW73+ruo9x9FPBT4G133xpg\nHhGRVmf8gE5cOr4vj0xbyTuLiwPZRmBF4O5TgYb+YD8PeCKoLCIirdl/nDyIQV2ymb+uJJD3D32u\nNDPLILrn8MM6xkwEJgIUFBS0UDIRkfiQnpLECz8aT1pyUiDvHw8ni78JvFfXYSF3n+Tuhe5emJ+f\n34LRRETiQ1AlAPFRBN9Gh4VEREITahGYWS5wNPB8mDlERBJZYOcIzOwJ4Bigk5mtAW4BUgDc/d7Y\nsDOB1919d1A5RESkboEVgbuf14AxDxP9mKmIiIQkHs4RiIhIiFQEIiIJTkUgIpLgzL3lpkNrDmZW\nDKyMfZkL7KjjefVlKcDmRm5y//doyLrqyxqa8bP/dmpkxpbK99kyfQ/jK19ryBjv+ZqSsa5l8fY9\n7O3uNV+I5e6t9gFMqut59WVAUVO20ZB11Zc1NON+/21UxpbKp+9hfOZrDRnjPV9TMtaTNa6+h3U9\nWvuhoRfreV7b+gPdRkPWVV/W0Izxnq++bdVF38P6t1OX+l4X7xnjPV9t6xuSsb5ljRH097BWre7Q\nUFOYWZG7F4adoy7xnjHe80H8Z4z3fBD/GeM9H7SOjJ9p7XsEjTUp7AANEO8Z4z0fxH/GeM8H8Z8x\n3vNB68gIJNgegYiIfFWi7RGIiEg1KgIRkQSnIhARSXAqghgzO9LM7jWz+83s/bDz1MTMImb2azP7\ns5n9e9h5qjOzY8zsndj38Ziw89TEzDLNrMjMTgs7S03MbEjs+/e0mV0Rdp6amNkZZnafmf3dzE4M\nO091ZtbPzB4ws6fDzvKZ2N+7R2Lft/PDzlNdmygCM3vQzDaZ2dxqy082s0/NbImZ3VjXe7j7O+5+\nOfAS8Eg8ZgQmAD2BcmBNHOZzYBeQHqf5AH4C/F9zZmvOjO6+IPb38FxgfJxmfM7dvwdcDnwrDvMt\nc/fLmjNXTRqZ9Szg6dj37fSgszVaY658i9cHcBRwCDB3v2VJwFKgH5AKzAKGAiOI/rDf/9F5v9f9\nH5AdjxmBG4Hvx177dBzmi8Re1wV4LA7zfZ3ojHgXA6fF4//j2GtOB14FvhOvGWOv+xNwSBzna9Z/\nI03M+lNgVGzM40HmOpBH6JPXNwd3n2pmfaotHgsscfdlAGb2JDDB3X8L1HhYwMwKgB3uvjMeM8Ym\n+CmLfVkZb/n2sw1Ii7d8scNVmUT/Ye41s1fcvSqeMsbe5wXgBTN7GXi8ufI1V0YzM+B3wKvuPj3e\n8rWUxmQluofcE5hJHB6JaRNFUIsewOr9vl4DjKvnNZcBDwWW6Ksam3Ey8GczOxKYGmSwmEblM7Oz\ngJOAPOCuYKMBjczn7jcBmNnFwObmLIE6NPZ7eAzRwwhpwCuBJvtCY/8e/gg4Acg1swH+xYyDQWns\n97Aj8GtgtJn9NFYYLaW2rHcCd5nZNzjwW1AEpi0XQaO5+y1hZ6iLu+8hWlZxyd0nEy2ruObRmfHi\nkru/BbwVcow6ufudRH+wxSV330L0/EXc8Oh0vJeEnaM2cbeL0ozWAr32+7pnbFk8ifeMytd0yth0\n8Z5vf60p6+fachF8DBxkZn3NLJXoScIXQs5UXbxnVL6mU8ami/d8+2tNWb8Q9tnqZjp7/wSwni8+\nVnlZbPmpwCKiZ/FvUkblU8b4zhjv+Vpr1voeuumciEiCa8uHhkREpAFUBCIiCU5FICKS4FQEIiIJ\nTkUgIpLgVAQiIglORSBtgpntauHt3W9mQ5vpvSrNbKaZzTWzF80sr57xeWb2g+bYtgho8nppI8xs\nl7tnNeP7Jbt7RXO9Xz3b+jy7mT0CLHL3X9cxvg/wkrsPb4l80vZpj0DaLDPLN7NnzOzj2GN8bPlY\nM5tmZjPM7H0zGxRbfrGZvWBm/wLetOiMa29ZdLawhWb2WOwWzMSWF8ae77LozHGzzOwDM+sSW94/\n9vUcM/tVA/daphG9gyVmlmVmb5rZ9Nh7TIiN+R3QP7YXcWts7A2xP+NsM/t5M34bJQGoCKQtuwP4\nH3c/FDgbuD+2fCFwpLuPBm4GfrPfaw4BznH3o2NfjwauITqHQT9qnjUsE/jA3UcSvT349/bb/h3u\nPoIGzNhmZknA8Xxxb5p9wJnufghwLPCnWBHdCCx191HufoNFp4s8iOi98EcBY8zsqPq2J/IZ3YZa\n2rITgKGxX+IBcswsC8gFHjGzg4hOr5my32v+6e5b9/v6I3dfA2BmM4E+wLvVtlNGdHYsgE+IzoQG\ncDhwRuz548Afa8nZLvbePYAFwD9jyw34TeyHelVsfZcaXn9i7DEj9nUW0WJoiTkrpA1QEUhbFgEO\nc/d9+y80s7uAKe5+Zux4+1v7rd5d7T1K93teSc3/Zsr9i5NttY2py153H2VmGcBrwJVE7/d/PpAP\njHH3cjNbQXQ+6OoM+K27/6WR2xUBdGhI2rbXic6mBYCZjYo9zeWLe8RfHOD2PyB6SAqityOuk0cn\nHroKuM7Mkonm3BQrgWOB3rGhO4Hs/V76GnBpbG8HM+thZp2b6c8gCUBFIG1Fhpmt2e9xLdEfqoWx\nE6jz+WLWqj8AvzWzGQS7V3wNcK2ZzQYGADvqe4G7zwBmA+cBjxHNPwe4iOi5DTw6A9d7sY+b3uru\nrxM99DQtNvZpvlwUInXSx0dFAhI71LPX3d3Mvg2c5+4T6nudSEvTOQKR4IwhOmG5AduBS0POI1Ij\n7RGIiCQ4nSMQEUlwKgIRkQSnIhARSXAqAhGRBKciEBFJcCoCEZEE9//xD2puxcHP5wAAAABJRU5E\nrkJggg==\n", "text/plain": [ "
" ] }, "metadata": { "tags": [] }, "output_type": "display_data" } ], "source": [ "learn.lr_find()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Let's do 1e-1 again" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "data": { "text/html": [ "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
epochtrain_lossvalid_lossaccuracytime
00.1568250.9045460.75620001:16
10.0898130.0630870.98030001:16
20.0409060.0256750.99220001:18
" ], "text/plain": [ "" ] }, "metadata": { "tags": [] }, "output_type": "display_data" } ], "source": [ "learn.fit_one_cycle(3, lr_max=1e-1)" ] } ], "metadata": { "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" } }, "nbformat": 4, "nbformat_minor": 1 }