{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "a73b2bf8-3926-4861-b274-329a055ce241",
   "metadata": {},
   "source": [
    "# Convolutional Neural Networks (CNNs) with PyTorch\n",
    "\n",
    "**Authors:** Jeffrey Huang and Alex Michels\n",
    "\n",
    "In this notebook, we will use PyTorch CNNs to recognize text from images. We use CNNs in this use case because the individual values of pixels don't tell us very much, but convolutions can help us extract features."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "67b15a5d-2602-4912-b674-3b5cf1473c95",
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from pathlib import Path\n",
    "import time\n",
    "import torch\n",
    "from torchvision import datasets\n",
    "from torchvision.transforms import ToTensor\n",
    "from torch.utils.data import DataLoader\n",
    "from torch import optim\n",
    "from torch.autograd import Variable\n",
    "import torch.nn as nn"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4d437ff8",
   "metadata": {},
   "source": [
    "## Data Wrangling\n",
    "\n",
    "First, we need to download the built in PyTorch MNIST dataset:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "5bb898ec-3b9c-41f3-9369-8919f80d71a4",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data = datasets.MNIST(\n",
    "    root = 'data',\n",
    "    train = True,\n",
    "    transform = ToTensor(),\n",
    "    download = True,\n",
    ")\n",
    "test_data = datasets.MNIST(\n",
    "    root = 'data', \n",
    "    train = False,\n",
    "    transform = ToTensor()\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "98009d0e",
   "metadata": {},
   "source": [
    "Next, we will examine the data. The data is of arrays based on the color of the pixel. We can also plot our data to see the images they make:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "5bd8ea61-60d1-45ed-8a39-43b9245b7172",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(tensor([[[0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
       "           0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
       "           0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
       "           0.0000, 0.0000, 0.0000, 0.0000],\n",
       "          [0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
       "           0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
       "           0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
       "           0.0000, 0.0000, 0.0000, 0.0000],\n",
       "          [0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
       "           0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
       "           0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
       "           0.0000, 0.0000, 0.0000, 0.0000],\n",
       "          [0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
       "           0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
       "           0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
       "           0.0000, 0.0000, 0.0000, 0.0000],\n",
       "          [0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
       "           0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
       "           0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
       "           0.0000, 0.0000, 0.0000, 0.0000],\n",
       "          [0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
       "           0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
       "           0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
       "           0.0000, 0.0000, 0.0000, 0.0000],\n",
       "          [0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
       "           0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
       "           0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
       "           0.0000, 0.0000, 0.0000, 0.0000],\n",
       "          [0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.3294, 0.7255,\n",
       "           0.6235, 0.5922, 0.2353, 0.1412, 0.0000, 0.0000, 0.0000, 0.0000,\n",
       "           0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
       "           0.0000, 0.0000, 0.0000, 0.0000],\n",
       "          [0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.8706, 0.9961,\n",
       "           0.9961, 0.9961, 0.9961, 0.9451, 0.7765, 0.7765, 0.7765, 0.7765,\n",
       "           0.7765, 0.7765, 0.7765, 0.7765, 0.6667, 0.2039, 0.0000, 0.0000,\n",
       "           0.0000, 0.0000, 0.0000, 0.0000],\n",
       "          [0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.2627, 0.4471,\n",
       "           0.2824, 0.4471, 0.6392, 0.8902, 0.9961, 0.8824, 0.9961, 0.9961,\n",
       "           0.9961, 0.9804, 0.8980, 0.9961, 0.9961, 0.5490, 0.0000, 0.0000,\n",
       "           0.0000, 0.0000, 0.0000, 0.0000],\n",
       "          [0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
       "           0.0000, 0.0000, 0.0000, 0.0667, 0.2588, 0.0549, 0.2627, 0.2627,\n",
       "           0.2627, 0.2314, 0.0824, 0.9255, 0.9961, 0.4157, 0.0000, 0.0000,\n",
       "           0.0000, 0.0000, 0.0000, 0.0000],\n",
       "          [0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
       "           0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
       "           0.0000, 0.0000, 0.3255, 0.9922, 0.8196, 0.0706, 0.0000, 0.0000,\n",
       "           0.0000, 0.0000, 0.0000, 0.0000],\n",
       "          [0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
       "           0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
       "           0.0000, 0.0863, 0.9137, 1.0000, 0.3255, 0.0000, 0.0000, 0.0000,\n",
       "           0.0000, 0.0000, 0.0000, 0.0000],\n",
       "          [0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
       "           0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
       "           0.0000, 0.5059, 0.9961, 0.9333, 0.1725, 0.0000, 0.0000, 0.0000,\n",
       "           0.0000, 0.0000, 0.0000, 0.0000],\n",
       "          [0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
       "           0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
       "           0.2314, 0.9765, 0.9961, 0.2431, 0.0000, 0.0000, 0.0000, 0.0000,\n",
       "           0.0000, 0.0000, 0.0000, 0.0000],\n",
       "          [0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
       "           0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
       "           0.5216, 0.9961, 0.7333, 0.0196, 0.0000, 0.0000, 0.0000, 0.0000,\n",
       "           0.0000, 0.0000, 0.0000, 0.0000],\n",
       "          [0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
       "           0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0353,\n",
       "           0.8039, 0.9725, 0.2275, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
       "           0.0000, 0.0000, 0.0000, 0.0000],\n",
       "          [0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
       "           0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.4941,\n",
       "           0.9961, 0.7137, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
       "           0.0000, 0.0000, 0.0000, 0.0000],\n",
       "          [0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
       "           0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.2941, 0.9843,\n",
       "           0.9412, 0.2235, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
       "           0.0000, 0.0000, 0.0000, 0.0000],\n",
       "          [0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
       "           0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0745, 0.8667, 0.9961,\n",
       "           0.6510, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
       "           0.0000, 0.0000, 0.0000, 0.0000],\n",
       "          [0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
       "           0.0000, 0.0000, 0.0000, 0.0000, 0.0118, 0.7961, 0.9961, 0.8588,\n",
       "           0.1373, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
       "           0.0000, 0.0000, 0.0000, 0.0000],\n",
       "          [0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
       "           0.0000, 0.0000, 0.0000, 0.0000, 0.1490, 0.9961, 0.9961, 0.3020,\n",
       "           0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
       "           0.0000, 0.0000, 0.0000, 0.0000],\n",
       "          [0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
       "           0.0000, 0.0000, 0.0000, 0.1216, 0.8784, 0.9961, 0.4510, 0.0039,\n",
       "           0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
       "           0.0000, 0.0000, 0.0000, 0.0000],\n",
       "          [0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
       "           0.0000, 0.0000, 0.0000, 0.5216, 0.9961, 0.9961, 0.2039, 0.0000,\n",
       "           0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
       "           0.0000, 0.0000, 0.0000, 0.0000],\n",
       "          [0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
       "           0.0000, 0.0000, 0.2392, 0.9490, 0.9961, 0.9961, 0.2039, 0.0000,\n",
       "           0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
       "           0.0000, 0.0000, 0.0000, 0.0000],\n",
       "          [0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
       "           0.0000, 0.0000, 0.4745, 0.9961, 0.9961, 0.8588, 0.1569, 0.0000,\n",
       "           0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
       "           0.0000, 0.0000, 0.0000, 0.0000],\n",
       "          [0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
       "           0.0000, 0.0000, 0.4745, 0.9961, 0.8118, 0.0706, 0.0000, 0.0000,\n",
       "           0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
       "           0.0000, 0.0000, 0.0000, 0.0000],\n",
       "          [0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
       "           0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
       "           0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
       "           0.0000, 0.0000, 0.0000, 0.0000]]]),\n",
       " 7)"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_data[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "57cb6a65-df9c-4835-8dab-aa2cc30e5244",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAv8AAAKQCAYAAAAIWRWtAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAABoB0lEQVR4nO3deZyN9f//8dfYxr4OZdTQQpJQZEuWFLIUyZZPtpAsH/Qle/YlJUkRkUKUJS0Kn5TIGkm2fBQGoTC2wSDG74/Pret3Xu+aM2fMOec657wf99ut2+39nOvMdV7Mu5m3a17X+4q6fv36dQEAAAAQ8TK4XQAAAACA4GDxDwAAAFiCxT8AAABgCRb/AAAAgCVY/AMAAACWYPEPAAAAWILFPwAAAGAJFv8AAACAJVj8AwAAAJawbvGfmJgoL774otSpU0cKFiwoUVFRMmzYMLfLgiXatWsnUVFRKf63ceNGt0uEBdauXSv169eXfPnySbZs2aR48eIycuRIt8tChNu2bZs0aNBA4uLiJFu2bJI/f36pUqWKzJ071+3SYInz589Lr169JDY2VrJmzSrlypWTDz/80O2ygi6T2wUEW0JCgkyfPl3Kli0rjRs3lhkzZrhdEiwyZMgQ6dKly98+3qhRI4mOjpYHHnjAhapgk3nz5skzzzwjzZs3l9mzZ0vOnDll3759cvToUbdLQ4Q7c+aM3HrrrdKqVSspUqSIXLhwQT744AN55plnJD4+XgYPHux2iYhwTz75pGzevFnGjRsnJUqUkHnz5kmrVq0kOTlZnn76abfLC5qo69evX3e7iGD6648bFRUlJ0+elIIFC8rQoUO5+g/XrF69WmrWrCmDBw/m6isC6siRI3LXXXdJmzZtZMqUKW6XA4iISOXKleXo0aNy6NAht0tBBPvyyy+lQYMGzoL/L3Xq1JFdu3bJoUOHJGPGjC5WGDzWtf381V4BhIqZM2dKVFSUdOjQwe1SEOFmzJghFy5ckH79+rldCuCIiYmRTJmsa0RAkC1ZskRy5swpzZo1Ux9v3769HD16VDZt2uRSZcFn3eIfCCVnz56VRYsWSe3ateW2225zuxxEuDVr1kj+/Pllz549Uq5cOcmUKZMUKlRIunTpIufOnXO7PFgiOTlZrl69KidOnJApU6bIihUr+AcpAm7nzp1y9913/+0fmmXKlHGO24LFP+Ci+fPnS1JSkjz77LNulwILHDlyRC5evCjNmjWTFi1ayMqVK6Vv374ye/ZsqV+/vljWBQqXdO3aVTJnziyFChWS3r17yxtvvCHPPfec22UhwiUkJEj+/Pn/9vG/PpaQkBDsklzD79kAF82cOVMKFCggTZo0cbsUWCA5OVkuXbokQ4cOlf79+4uISM2aNSVLlizSq1cv+frrr+WRRx5xuUpEuoEDB0rHjh3l+PHj8vnnn0v37t3lwoUL0qdPH7dLQ4Tz1vZtU0s4V/4Bl2zfvl22bNki//rXvyQ6OtrtcmCBAgUKiIhI3bp11ccfe+wxERHZunVr0GuCfeLi4qRChQpSv359mTp1qnTu3FkGDBggJ06ccLs0RLACBQr849X9U6dOiYj8428FIhWLf8AlM2fOFBGRjh07ulwJbPFXb6vpr3afDBn4kYDgq1ixoly9elX279/vdimIYPfee6/8/PPPcvXqVfXxHTt2iIhI6dKl3SjLFXynB1xw+fJlmTt3rlSsWNGqbzhwV9OmTUVEZNmyZerjX375pYj8b8tFINhWrVolGTJkkNtvv93tUhDBmjRpIufPn5fFixerj7///vsSGxsrlSpVcqmy4LOy53/ZsmVy4cIFSUxMFBGR3bt3y6JFi0REpH79+pI9e3Y3y4MFPvnkEzl16hRX/RFUderUkUaNGsmIESMkOTlZKleuLFu2bJHhw4dLw4YNpVq1am6XiAjWuXNnyZ07t1SsWFFuuukmOXnypCxcuFA++ugj6du3rxQsWNDtEhHBHnvsMXn00Ufl+eefl3Pnzsmdd94p8+fPl+XLl8vcuXOt2eNfxMKHfImIFCtWTA4ePPiPxw4cOCDFihULbkGwTp06dWT9+vVy7NgxyZUrl9vlwCJJSUkyfPhwmTdvnhw7dkxiY2OldevWMnToUO49QUDNmjVLZs2aJT///LOcOXNGcubMKWXLlpWOHTvKv/71L7fLgwXOnz8vgwYNkgULFsipU6ekZMmSMmDAAGnZsqXbpQWVlYt/AAAAwEb0/AMAAACWYPEPAAAAWILFPwAAAGAJFv8AAACAJVj8AwAAAJZg8Q8AAABYgsU/AAAAYAmfn/AbFRUVyDrgR5H46AbmX/iIxPknwhwMJ5E4B5l/4YP5Bzf5Mv+48g8AAABYgsU/AAAAYAkW/wAAAIAlWPwDAAAAlmDxDwAAAFiCxT8AAABgCRb/AAAAgCVY/AMAAACWYPEPAAAAWMLnJ/zi70qWLKnyrl27Unzt3XffrfLevXsDUhMABEuuXLlU7tatmzMeM2aMOtaiRQuVFy5cGLjCACAVderUUXnFihUqz5s3T+UtW7aoPHHixMAUFgRc+QcAAAAsweIfAAAAsASLfwAAAMAS9PynQ6tWrVROTk5O8bXejgFAOKhUqZLK48ePV7latWrO+Pr16+rYyy+/rPLy5ctVTkxM9EeJAOCT8uXLq2yu01q2bOk1R0dHqzxu3Dg/VhdYXPkHAAAALMHiHwAAALAEi38AAADAEvT8p8GgQYNU7tOnT4qvnTx5ssq///57QGoCAH+pUKGCyu3bt1fZ7HnNmzdviuc6ffq0yuZzUJKSkm6gQoS6AgUKqGw+D6dEiRLOuH///upY8eLFVT548KDKw4YNU/n999+/0TIBKV26dLo+/4UXXlB55cqVzth8JkCo4co/AAAAYAkW/wAAAIAlWPwDAAAAloi6bm7GnNILo6ICXUvI6dGjh8rmHq5ZsmRR+ezZs864U6dO6tiSJUv8XF3KfPyShhUb5l+GDPrf4hUrVlTZnI9PPfWUymvXrnXGtWvX9nN1vovE+ScSmXPQ3Of6zTffVNmcg+bfwalTp1R+5ZVXnPGMGTPUsYSEhBuuM60icQ6G6vzLly+fyj/99JPKRYoUSfFzzT9Tal+3K1euqFyrVi2VN27c6PXzg4X5Fx6effZZladPn56u8x09etQZm89E8TwWaL7MP678AwAAAJZg8Q8AAABYgsU/AAAAYAn2+fcia9asKps9/qbFixc742D2+MM9N998s8rmPug7duxQed++fc64WbNm6liTJk1UNnsG01oLkJpevXqpnNqc27lzp8rmfN+9e7df6kLoGj9+vMr33HOPyt56/NPL/Bls9myHSs8/Qke1atVU7t69uzOuU6eOOvbDDz+o/Ntvv6n8xBNPeH2v2NhYZ2ze9zl8+PDUiw0irvwDAAAAlmDxDwAAAFiCxT8AAABgCXr+Pdx5550qN23a1KVK4KaYmBiVP/vsM2ecLVs2r6/17PlLjblv8i+//KLyxIkTVW7fvr3KefPmVTl79uw+vzfsNWLECGds9uyb+6hPnjxZ5TFjxqh8+vRpP1eHUNS6dWtn3KdPH3XMzT3tn376aZXNPmtEPs+5KSJy8eJFlT/55BOV33nnHWf8xhtvqGPms0nOnDmj8kcffaRyvXr1UqyrXLlyKmfMmFHla9eupfi5wcCVfwAAAMASLP4BAAAAS9D246F48eIqly9f3qVK4KaTJ0+qPHv2bGdcuXJlr5/7448/+vw+5rZi8fHxKt9+++0q9+jRw+v5GjVq5PN7I3KZv14eOHCgyoMHD3bGZpvPW2+9pXLfvn39XB3CUWrbXHtjtk58/PHHznjWrFnqWKtWrVTu2rWr13NPmTLlhutCeMqfP7/KvXv3Vvm+++5TuXHjxirffffdN/zeTz75pMpmi5Gnxx9/XGVzPfn999/fcB3+wJV/AAAAwBIs/gEAAABLsPgHAAAALBF13cd9usxtCSORufVScnKy19efP39eZc9eWrN3Npjc3HotUGyYfyZzizKzp//o0aMq33rrrYEuySeROP9EwmcODhkyROVhw4ap7PnnmDBhgjoWKT3+kTgHQ2X+LVq0SGWzDzohIUHlf//73yrPnz/fGefOndvruR955BGV9+zZo3KpUqV8qDj4mH/+VbFiRWe8Zs0adSxz5sxeP9fcztO8RyAtzL+Dl156yWv2tGrVKpXNue1Pvsw/rvwDAAAAlmDxDwAAAFiCxT8AAABgCav3+X/++efT9fm//fabym72+SP8FSlSROUaNWqobPYbPvroowGvCaGvUqVKKvfq1Utlc95MmzbNGXvrUQX+SZ8+fVT+5ZdfVP70009V3rhxo8p58+Z1xgsWLFDHateurfLly5dVHjp0aJpqRWS4//77nXFqPf6m0aNH+60Os5d++/btKl+9etUZZ8qkl9fm81fcxpV/AAAAwBIs/gEAAABLsPgHAAAALGFdz79nn78/e8GA9Jo4caLK5h7Yhw8fVvngwYMBrwmhz3PfdBHdUy0i8tVXX6n8wgsvOOOkpKSA1YXIFB8fr/KAAQO8vv6ee+5R2bPP/+6771bHzJ5qz/tTREQWLlzoa5mIIC1btvT5tfv27VP57Nmz/i7HsWTJEpU97wHwvE9BRCRfvnwqFyxYUOUTJ074uTrvuPIPAAAAWILFPwAAAGAJFv8AAACAJSKu59/cA9azv1VEZMyYMSl+boYMafu30K5du9L0esBThQoVVK5bt67X1/ft21dl+rXt9Nhjj6lcoEABlS9cuKByjx49VGbeIJCqVq2q8ueff66yeU+Kp3//+98qv/POO36rC+GjcuXKKt97770pvvbatWsqjx8/XuU///zTf4WlYubMmc7Y7Pk3/wwVK1ZU+YsvvghcYf+AK/8AAACAJVj8AwAAAJaIuLYfz608RURGjRqlcnJyss/nMl/78ccfq9yhQ4c0Vgf8f71791Y5Z86cKq9Zs0Zlc/7BTjNmzFDZnDfPPvusynv37r3h93rkkUdUfvDBB1W+7777VPZsKWrVqtUNvy/Ch9m++Omnn6rsrc3H1K9fP5XN7RDHjRun8qVLl3w+N8JHTEyMyt7m0NGjR1U2vz8GU548eVx777Tiyj8AAABgCRb/AAAAgCVY/AMAAACWiLpuPk87pRdGRQW6Fr/YvXu3ysWLF/f5c82tPhctWqSyuQ3ZH3/8kcbqgsPHL2lYCZf5l5rChQs74/j4eHUsUyZ9C06lSpVU3rJlS8Dq8qdInH8ioTMHzXuRli9frnL9+vV9Ple9evVULleunMretkZOzYgRI1QeNmzYDZ8rrSJxDobK/DP9/PPPKt91110+f675Z0rt63bmzBmVzfugZs2a5Yy3bt2qjgVzi1vmX/o0btxY5cWLF6f42q5du6o8bdq0QJT0j8yf2QkJCc7YvBdr3759KtepU0dlcz2QHr7MP678AwAAAJZg8Q8AAABYgsU/AAAAYImw2+ff3O914sSJKt9+++03fO6LFy+qvHHjRpVDtccfocvsCfzggw+ccZYsWdQx87Hk4dLjj8Dr3LlzisfMeVOoUCGVPe8zERH517/+5YzbtWunjuXPn19ls3c0LT3affv2VTmYPf8Ink2bNqlsziHP73ki3p9XMmDAAJWrVq2qsrmPevv27VX2fPaOm/ecIH1Gjx7tdgk+Meer2efv6ciRIyr7s8f/RnDlHwAAALAEi38AAADAEiz+AQAAAEuEXc9/kyZNVPbsX02rpUuXqmz2Lpr3EwBp1bZtW5Vr1KjhjC9duqSOvfPOO0GpCeHnySefTPHY1atXVf76669VLlWqlMqeffup7Qd9+vTpFD9XROTUqVMqe95zlTVrVnXsnnvuUXnXrl1e3xvhwbxvJD0aNGig8s0336zyo48+qnKvXr1Uvu+++5xxv3791LEvv/xS5e+///5Gy4SlMmbMqHL58uV9/lzzHlK3ceUfAAAAsASLfwAAAMASLP4BAAAAS4RFz7/nPtVmD3V6vPzyyyqHWk8Wwp+5T7Wn1157TeVff/010OUgTGTLlk1lc69+T6tXr/Z6rv3796u8fv16Z7xo0SJ1zLwPxez5f+qpp1Q29/L3xrw/AEjN77//rvKCBQtUvuWWW1T27Pk3n6Ni3j8ApMb8Pjxt2jSVGzVqlOLnmvv6z5w503+F+QFX/gEAAABLsPgHAAAALMHiHwAAALBEWPT8586d2xk/+OCD6TrXK6+84ox/+umndJ0LMFWvXl1lb8+h2Lx5c6DLQZhKSkpS2bP3uUyZMl4/95tvvlH5mWeeUfnYsWPOuESJEupY/fr1VTZ7+r3deyAismfPHmfcqVOnFN8XuBF16tRRedSoUSm+9syZMyqvXbs2ECUhAMz7lEqWLJnia4sVK+a39zW/t/bs2VPl1q1b+3wu8/tuqN3Tx5V/AAAAwBIs/gEAAABLhEXbjz8lJCQ4Y/NX60BaZc+eXeXhw4ernCmT/l/s6NGjzviLL74IXGGIKAMGDHDGjz76qDp2/fp1lc1Wnt69e6tcrVo1Z3zvvfeqY+Z8Ts3nn3+usmebW2JiYprOBZhbd5ptFmPHjvX6+Z5zznPbTxG2mg0nHTt2VPmDDz5wxrVq1VLHXnjhBZVLly59w+9bsWJFlWNiYry+/uTJkyp7fj8M9bZervwDAAAAlmDxDwAAAFiCxT8AAABgCet6/gF/GjlypMo1atTw+vo2bdo44z///DMgNSHybN261Rl7bvspIlKwYEGVixQpovL//d//qRwVFeWMr1y5oo798ccfKu/bt0/lCRMmqLx06VKVr169+rfaAU958+Z1xoMHD1bHzK2Rzblt3t9i9lU3b97cGR86dCg9ZcJF5vehfv36OePly5erY/nz51fZ3K44PZKTk1X2vGdUROT1119Xedy4cX5770Djyj8AAABgCRb/AAAAgCVY/AMAAACWoOcfSINChQqp3K5dO5XNntQ333xT5e+++y4gdcEesbGxKpt77VepUkXlfPnyqbxu3Tpn/O6776pjs2bN8keJCDODBg1S2fO+EBGRKVOmOGNzv/zo6GiV69Wrp3KrVq1Url27tjMuUKCAOmZ+//R8LoqIyMqVK1U272dhL//I9MMPPzjj4sWLq2Pdu3dX2XzWTlq88847Km/btk3lt99++4bPHWq48g8AAABYgsU/AAAAYAkW/wAAAIAloq6bTXYpvdDoAUTo8vFLGlZCZf6ZPf4zZ870+vr77rtP5e3bt/u7pJATifNPJHTmIFIXiXMwkPNv8uTJKrds2VLljBkzOuPVq1erY2XKlFG5WLFiXt/r4MGDztjs4V+8eLHK69evVzkxMdHruUMF8w9u8mX+ceUfAAAAsASLfwAAAMASLP4BAAAAS7DPP+BHR44cUTk+Pt6dQgDARz169FB5woQJKns+W6J///7q2M0336yy+eyIefPmqey5Z/u5c+fSXiyAdOPKPwAAAGAJFv8AAACAJVj8AwAAAJag5x9Ig2PHjnk9/v7776tMTyuAcGPeq+SZH3/88eAWA8DvuPIPAAAAWILFPwAAAGCJqOs+PoeaRzuHDx4tDjdF4vwTYQ6Gk0icg8y/8MH8g5t8mX9c+QcAAAAsweIfAAAAsASLfwAAAMASLP4BAAAAS7D4BwAAACzB4h8AAACwBIt/AAAAwBI+7/MPAAAAILxZdeX/22+/laioqH/8b+PGjW6Xhwj3zTffSIcOHaRkyZKSI0cOKVKkiDzxxBPyww8/uF0aLJGYmCgvvvii1KlTRwoWLChRUVEybNgwt8uCJfgZjFCwdu1aqV+/vuTLl0+yZcsmxYsXl5EjR7pdVlBlcrsAN4wZM0Zq1aqlPla6dGmXqoEtpk6dKgkJCdKzZ08pVaqUnDhxQiZMmCCVK1eWFStWyMMPP+x2iYhwCQkJMn36dClbtqw0btxYZsyY4XZJsBA/g+GWefPmyTPPPCPNmzeX2bNnS86cOWXfvn1y9OhRt0sLKisX/8WLF5fKlSu7XQYs89Zbb0mhQoXUx+rVqyd33nmnjBkzhsU/Aq5o0aJy+vRpiYqKkpMnT7L4hyv4GQw3HDlyRDp37izPPfecTJkyxfm4+Q9RG1i5+AfcYC78RURy5swppUqVksOHD7tQEWwTFRXldgkA4IoZM2bIhQsXpF+/fm6X4jqrev7/0q1bN8mUKZPkzp1b6tatK2vXrnW7JFjq7NmzsnXrVrnnnnvcLgUAgoKfwXDDmjVrJH/+/LJnzx4pV66cZMqUSQoVKiRdunSRc+fOuV1eUFm1+M+TJ4/07NlTpk2bJqtWrZJJkybJ4cOHpWbNmrJixQq3y4OFunXrJhcuXJBBgwa5XQoABBQ/g+GmI0eOyMWLF6VZs2bSokULWblypfTt21dmz54t9evXF5s2v7R+q88zZ87IvffeK/nz55effvrJ7XJgkSFDhsioUaNk8uTJ0r17d7fLgWVOnjwpBQsWlKFDh7LjD1zDz2AES4kSJeSXX36RsWPHSv/+/Z2PT5o0SXr16iVfffWVPPLIIy5WGDxWXfn/J3nz5pWGDRvK9u3bJSkpye1yYInhw4fLqFGjZPTo0Sz8AViLn8EIlgIFCoiISN26ddXHH3vsMRER2bp1a9Brcov1i38RcX7Vw81wCIbhw4fLsGHDZNiwYTJw4EC3ywEAV/EzGMFQpkyZf/z4X/MvQwZ7lsT2/ElTcPr0aVm6dKmUK1dOsmbN6nY5iHAjR46UYcOGyeDBg2Xo0KFulwMAruJnMIKladOmIiKybNky9fEvv/xSRMSq7Wet2urz6aeflri4OKlQoYLExMTIL7/8IhMmTJA//vhD3nvvPbfLQ4SbMGGCvPTSS1KvXj1p0KDB355oadM3Hrhn2bJlcuHCBUlMTBQRkd27d8uiRYtERKR+/fqSPXt2N8tDBONnMNxUp04dadSokYwYMUKSk5OlcuXKsmXLFhk+fLg0bNhQqlWr5naJQWPVDb/jxo2Tjz76SA4cOCDnz5+X/PnzS7Vq1WTAgAHywAMPuF0eIlzNmjVl9erVKR636H9FuKhYsWJy8ODBfzx24MABKVasWHALgjX4GQy3JSUlyfDhw2XevHly7NgxiY2NldatW8vQoUMlOjra7fKCxqrFPwAAAGAz63v+AQAAAFuw+AcAAAAsweIfAAAAsASLfwAAAMASLP4BAAAAS7D4BwAAACzB4h8AAACwhM9P+I2KigpkHfCjSHx0A/MvfETi/BNhDoaTSJyDzL/wwfyDm3yZf1z5BwAAACzB4h8AAACwBIt/AAAAwBIs/gEAAABLsPgHAAAALMHiHwAAALAEi38AAADAEiz+AQAAAEuw+AcAAAAsweIfAAAAsEQmtwsAQs3999+v8urVq51xjhw50nSu0aNHqzxo0KAbrqtFixYqb968WeX4+PgbPjcAAAiMCRMmqFy7dm2V27Vrp/K2bdsCWg9X/gEAAABLsPgHAAAALMHiHwAAALBE1PXr16/79MKoqEDXAj/x8UsaVgI5/2bNmqVy1apVVb7zzjsD9t7p0a1bN5XffvttlyrRInH+ifA98J+ULVtW5eXLlztj8++rXLlyKv/+++8BqysS5yDzL23Kly+vcubMmb2+fufOnSqfP3/+ht+b+WenunXrOuOmTZuqYx07dlTZnCPvvPOOyl26dLnhOnyZf1z5BwAAACzB4h8AAACwBIt/AAAAwBLs858OlStXVvmxxx5zxjVq1FDH0toD+OGHH6o8bdq0NFYHX7Vt21Zl82t19uxZZ/zTTz+pY2PHjlV5wIABKvft21flX3/91RkPGzZMHTP7p++++26VCxYsqLJ5L4JnT3Wg9wgGRETat2+vcqFChZzx/v371bFr164FpSaElkyZ9DJj7ty5zviTTz5Rxz7//HOV7733XpWffPJJlS9fvqzyE0884YzN758ZM2b0Wqf53p7nAkREoqOjVa5Xr57K77//vjPOlSuX13OZ95iY64FA48o/AAAAYAkW/wAAAIAlWPwDAAAAlmCf/zTw7OkXEfn4449VzpIlizM2/77Mv+aLFy+qnJyc7PW98+TJ43Od7DGcNubfvfn359nXPHv27IDVYZo/f77KzZs39/p6z33TzfsYVq5c6b/CUhGJ80+E74EiIkWLFlV5+/btKufMmdMZm/c9rV27NnCFGSJxDobL/MuRI4fKH3zwgcqPP/54MMtJ0Zo1a1Q2e/4nTJhww+dm/kWG7Nmzq1ymTBmV161b5/O5vvvuO5WHDh2q8urVq9NYXcrY5x8AAACAg8U/AAAAYAkW/wAAAIAlrNvnP1++fM7Y3KO9S5cuXj+3SZMmKpt9+xs2bHDGU6dOVcfMHqzdu3erfOXKFa/vjcBp3bq1ypMnT1b5s88+C2Y5jl69eql83333qVy8eHGVb775Zmf8wAMPqGPB7PlH5DD3aO/atavKZk+s5zzbsmVL4ApDyDB/Lk6aNEnlW2655YbPfebMGZX/85//qGw+S8Lz+KFDh7yeOz4+XuXU7ruDfQYPHqxyv379fP7c1157TWXzmT9u48o/AAAAYAkW/wAAAIAlWPwDAAAAlrCu579s2bLOuFOnTurYm2++qfLOnTtV9rxfQESkfPnyKps9hAgP5n76ly5dUvny5cvBLMfxxx9/qNyzZ0+Vv/zyy2CWAwuZ98P06dNH5YMHD6rcrFkzZ2z+f4TIYP7ce/vtt1UuWLCg18/3/Lm6fPlydczcN/2bb75ROTEx0ec6gdSY9yy1aNFC5QEDBqic2v75nmvIV199NZ3VBRZX/gEAAABLsPgHAAAALBHxbT/lypVT2XMbMnO7TbPNx+T5K21EriVLlrhdAhASmjZtqnJUVJTK5hay586dC3hNcNfIkSNVTq3Nx+TZZvnBBx+oY6dOnVL5/PnzaawO8J25dae5tafZ5nPt2jWVFyxYoHL//v2dsbkVfKjhyj8AAABgCRb/AAAAgCVY/AMAAACWiPie/+eff17l6OhoZ1y1atVglwPcsMOHD6u8a9cule+5555gloMIdNNNN6mcK1culc3tjOfNmxfokhBizPvo0mr06NH/OBYR+frrr1U2t5b96aef0vXegOe9my+++KLX1yYkJKjsec+oiMioUaP8V1iQceUfAAAAsASLfwAAAMASLP4BAAAAS0R8z//p06dVPnr0qDM29xQGQpn5KPI8efKk+Np27dqpbPZmHzx40G91IXI0b95c5fvvv19l8z6T9evXB7wmhJbXXntN5Q4dOnh9vfl9qnDhwim+tnbt2ipv2rRJ5S+//FLlF154QWXznhSgZMmSKrdp08YZZ8mSRR07c+aMym+99ZbK4dzjb+LKPwAAAGAJFv8AAACAJVj8AwAAAJaIun79+nWfXhgVFehaAmL27Nkq16lTxxk/8cQT6pjZXxiufPyShpVwnX/+VLduXZXN/ldvypQpo7LZu+1PkTj/RCJzDlaoUEHlNWvWqHz16lWVq1SponIg51F6ROIcDNf5Zz474s4773TGTZo0UcfMn8l33HGH13Pv3btXZc/n+qxatSpNdfoT8889TZs2Vfm9995T2fPeObPHv1GjRiqH6z1Nvsw/rvwDAAAAlmDxDwAAAFiCxT8AAABgiYjv+W/QoIHKEydOdMaee/6LiNSsWTMYJQUc/YaRiZ5/d0XKHMybN68znjBhgjrWvn17ldetW6fyQw89FLC6/CkS52CkzD9vihQpovKgQYNUbtmypcqec1lEf907duyojs2aNcsPFfqG+ece8+/e29fC/H73/vvvB6SmYKPnHwAAAICDxT8AAABgiUxuFxBo33zzjcpffPGFM+7Vq5c6NnLkSJXHjRun8oULF/xbHBAgw4cPV/nnn392qRKEGs/t7UqUKKGOmb/aN7fJAwLpyJEjKnft2lVlcyvauXPnqpwhw/+/njl16lR1bMeOHSpv2bLlhuuEe3LkyKGy2c5ltrxcu3ZN5c6dOztjc/5kzZpV5Zw5c3o9d0JCgg8Vhyau/AMAAACWYPEPAAAAWILFPwAAAGCJiO/5T0pKUtmzF7p06dLqmLmtmNlb9sILL/i5OsB3zz33nM+v/fPPP1VOTk72dzkIU4MHD3bG5cqVU8fOnTun8o8//hiMkgCffPjhhyofO3ZM5VWrVjnjLFmyqGMNGzZUmZ7/8FG0aFFnvHTpUnWsVKlSXj9348aNKi9fvtwZf/DBB+pY8eLFVb7vvvtUvnz5ssp9+/ZVefr06c74ypUrXutyG1f+AQAAAEuw+AcAAAAsweIfAAAAsETUdR+fQx0uj3ZOi2zZsqm8bNkylStVqqRykyZNVPbsHQslPFrcv+6//35n7LmPtC8mTJig8u23337DdRQuXFhlb38nU6ZMUdl8bPkff/yh8uHDh2+4LlMkzj+R8P0eaM4bz2c+5MqVSx3r2LGjyuYe2uEiEudgeuZf+/btVTbv7Vi8ePENn9tNmTLp2xZXrlzpjKtXr66Omf3dzzzzTMDqYv751z333OOMzR7+7Nmzq2zWae7z7/m8JvP7n8k8V2pf1wYNGjhjN9eHvsw/rvwDAAAAlmDxDwAAAFiCxT8AAABgiYjf598b8xkAZn+22Sdt3iOAyNS0aVOVZ8+e7YyzZs0a7HJuSNeuXb3mTZs2qfzFF1+oPHr06MAUhoAzn08yefJklT37XD17pEVE5s+fH7jC4Jp3331X5UmTJqkcrj3/JUuWVLlEiRIuVYJA+ve//+2MzR7/1Jj36Xl+/ztz5ow69vHHH3s9l7k2yJMnj8rR0dFpqs1NXPkHAAAALMHiHwAAALAEi38AAADAEq7s858/f36VH3jgAZVXrFjht/dKj1WrVql84sQJlZs3bx7McnzGHsNps2PHDpVjY2NVzps3b8DeO1ASEhJUPn/+vMpFixZV2dwLefPmzSr37t3bGX///fde3zsS559I+OzzX7VqVZW/++47lT37XM1nmfz6668BqyuYInEOpmf+mX8fp0+fVtncE3/nzp03/F7pYX6vNXuqBw4cqHKLFi1Uzp07tzM2v6d57sEuIvKf//znRstMFfMvfWJiYlQ+fvy4z5+b2t78nvdA9evXTx27dOmSys8//7zKb731ltf39nwW1Keffpp6sQHCPv8AAAAAHCz+AQAAAEuw+AcAAAAs4co+/0uWLFHZ7CH+8ccfVU5Lv1d6mP1cNWrUULl27dpBqQPBVbp0aZWTk5NdqiRtzD3af/rpJ2ds3jfjeUxE5MUXX/R67i1btqicWp8/3OPZ5ywi0rp1a5XNHthwuXcB/nPw4EGV4+LiVPZ8lomIyOuvv67yV1995YyPHTvm9b3M76d33nlniq/17JEWEXn44YdVLlKkiNf38sbckz2QPf7wL/MZTJ7Poalfv36aztWzZ0+VP/jgA2ecM2dOdWzQoEFes3kfyWeffaayeX9VKOPKPwAAAGAJFv8AAACAJVzZ6tNz20ARkVdffVXlPXv2qDx8+HBnbG5Ft3Xr1jS9d+XKlZ3xY489po516NBB5VdeeUXlqVOnqvznn3+m6b2DhW3G0ubcuXMq58iRI2DvlRbm9pybNm1SuW3btiqn9uv4YInE+ScSOu0yWbNmVdn81XNq7Ym7d+92xhUrVlTHzF+3h6tInIPpmX/m94pp06apnCVLFq+fn5iY6IyvXLni9bXm909zvqaHOT/37dun8pgxY5zxggUL1LFgtnMy//xr0qRJzrh79+5eX2vW+fXXX6vs2epjfv9L7VyXL19W2WwND5X2WLb6BAAAAOBg8Q8AAABYgsU/AAAAYAlXev4zZdI7jLZs2VLl9957L8X3Nh+/fPjwYZV37Nihstlv6PkY882bN6tjo0ePVnnt2rUqh2qPv4l+w7QpV66cyh9++KHKxYsXD8j7Xr16VeU5c+aoPH36dJVDpZ8wNZE4/0RCp+e/ffv2Ks+cOVNl8+//6NGjKterV88Z79q1y8/VhYZInIP+nH/VqlVTec2aNX47t3kPlXmPQExMjDM2v07z5s1TeefOnSovX75cZXML41DB/PMvz+1iV61apY7FxsaqbNaZnq/FxYsXVX722WdV/uSTT1RO7X6YYKHnHwAAAICDxT8AAABgCRb/AAAAgCVc6flPzeOPP65yo0aNnPEdd9yRpnOZ+wKPHz/eGa9evfoGqgt99Bumj3kPQFxc3A2fK3/+/Cp79me/9NJL6ph5z0m4isT5J+Juz2uhQoWc8ZIlS9SxKlWqqGzeS/LEE0+ovGzZMj9XF3oicQ76c/6Z50ptL/5HH33UGRctWlQdK1mypMobNmxQedu2bSofOHDAGZt77/OcidAVKvc8lS1bVmVz3/+HHnpIZW/37Jn3fX788ccqf/nllyqb96CEKnr+AQAAADhY/AMAAACWYPEPAAAAWCIke/6RPvQbwk2ROP9E3J2Dns8nMfe5Pn78uMrmcyp69+4duMJCVCTOQb4Hhg/mH9xEzz8AAAAAB4t/AAAAwBIs/gEAAABL0PMfgeg3hJsicf6JMAfDSSTOQeZf+GD+wU30/AMAAABwsPgHAAAALMHiHwAAALAEi38AAADAEiz+AQAAAEuw+AcAAAAsweIfAAAAsASLfwAAAMASLP4BAAAAS7D4BwAAACzB4h8AAACwBIt/AAAAwBIs/gEAAABLsPgHAAAALBF1/fr1624XAQAAACDwrLryv23bNmnQoIHExcVJtmzZJH/+/FKlShWZO3eu26XBAsw/uC0xMVFefPFFqVOnjhQsWFCioqJk2LBhbpcFi3z//fdSt25dyZUrl+TMmVNq1aol69atc7ssWKJdu3YSFRWV4n8bN250u8SgyOR2AcF05swZufXWW6VVq1ZSpEgRuXDhgnzwwQfyzDPPSHx8vAwePNjtEhHBmH9wW0JCgkyfPl3Kli0rjRs3lhkzZrhdEiyyefNmqV69ulSsWFHmzJkj169fl/Hjx0vt2rVl1apVUqVKFbdLRIQbMmSIdOnS5W8fb9SokURHR8sDDzzgQlXBR9uPiFSuXFmOHj0qhw4dcrsUWIj5h2D569t9VFSUnDx5UgoWLChDhw7l6j+Col69erJt2zbZv3+/ZM+eXUT+99uo22+/XUqUKMFvAOCK1atXS82aNWXw4MEycuRIt8sJCqvaflISExMjmTJZ9UsQhBDmH4Llr19tA25Yt26d1KxZ01n4i4jkypVLqlevLuvXr5djx465WB1sNXPmTImKipIOHTq4XUrQWLniSE5OluTkZDl9+rQsXLhQVqxYIW+++abbZcESzD8ANrpy5YpER0f/7eN/fWzHjh1SuHDhYJcFi509e1YWLVoktWvXlttuu83tcoLGysV/165dZdq0aSIikiVLFnnjjTfkueeec7kq2IL5B8BGpUqVko0bN0pycrJkyPC/xoOrV6/Kpk2bROR/96QAwTR//nxJSkqSZ5991u1SgsrKtp+BAwfK5s2b5YsvvpAOHTpI9+7d5dVXX3W7LFiC+QfARj169JC9e/dK9+7d5ciRI3L48GHp0qWLHDx4UETE+QcBECwzZ86UAgUKSJMmTdwuJaisvPIfFxcncXFxIiJSv359EREZMGCAtG3bVgoWLOhmabAA8w+AjTp06CAnTpyQUaNGydSpU0VEpEqVKtKnTx95+eWXpUiRIi5XCJts375dtmzZIj179vzHdrRIxj+zRaRixYpy9epV2b9/v9ulwELMPwC26Nevn5w8eVJ27Ngh8fHxsn79ejl9+rTkyJFDypcv73Z5sMjMmTNFRKRjx44uVxJ8Vl75N61atUoyZMggt99+u9ulwELMPwA2iY6OltKlS4uIyKFDh+Sjjz6STp06SbZs2VyuDLa4fPmyzJ07VypWrOjMRZtYtfjv3Lmz5M6dWypWrCg33XSTnDx5UhYuXCgfffSR9O3bl5YLBBTzD6Fg2bJlcuHCBUlMTBQRkd27d8uiRYtE5H9taJ7bMAL+tHPnTlm8eLFUqFBBoqOj5aeffpJx48ZJ8eLFrdlfHaHhk08+kVOnTll51V/Esod8zZo1S2bNmiU///yznDlzRnLmzClly5aVjh07yr/+9S+3y0OEY/4hFBQrVsy5wdJ04MABKVasWHALgjX27t0rnTp1kp07d8r58+clLi5OWrZsKf3795ccOXK4XR4sUqdOHefZErly5XK7nKCzavEPAAAA2IwbfgEAAABLsPgHAAAALMHiHwAAALAEi38AAADAEiz+AQAAAEuw+AcAAAAsweIfAAAAsITPT/iNiooKZB3wo0h8dAPzL3xE4vwTYQ6Gk0icg8y/8MH8g5t8mX9c+QcAAAAsweIfAAAAsASLfwAAAMASLP4BAAAAS7D4BwAAACzB4h8AAACwBIt/AAAAwBIs/gEAAABLsPgHAAAALMHiHwAAALAEi38AAADAEiz+AQAAAEuw+AcAAAAskcntAgD8z7Bhw1QeOnRomj7/22+/dca1atXyQ0UAACDScOUfAAAAsASLfwAAAMASLP4BAAAAS0Rdv379uk8vjIoKdC0+KV++vMrVq1f3+vo2bdqoXKZMGWf83XffpXhMRCRPnjxez/3pp5+qPGnSJGe8evVqr58bSD5+ScNKqMy/tKpZs2aKOa09/ekRzL+/SJx/IuEzB1P7+x8+fLgzNu8ziRSROAfDZf6B+Qd3+TL/uPIPAAAAWILFPwAAAGAJFv8AAACAJcKu5793794qv/LKKzd8LvPPlN4+vQsXLjhjs+d/4sSJKq9atSpd7+UN/YaB462HX0SkRo0aXo+7xbPPWySwvd6ROP9EQmcOpiYtf//h8mdKq0icg5H6tYpEts+/ggULqjx16lSVGzdurHKGDPo6dHJyss/vtXfvXpWfeeYZr6//+eefnfHFixd9fp9wQs8/AAAAAAeLfwAAAMASLP4BAAAAS2Ryu4BIkjNnTmdcv359dSwuLk7l2rVrq5yQkBC4wpAunvdnuNnD/+2336ocKvcTILTUqlVL5UDeXwSkpnTp0iq/++67zviBBx7w+rlm7/KPP/6o8kMPPaRypPZwh5smTZqo/Pjjj6ucWk9/Wnr+S5QoofKGDRu8vn7cuHHO+KWXXvL5fSINV/4BAAAAS7D4BwAAACzB4h8AAACwRNj1/J8/f17lw4cPq3zTTTepnDFjRpWPHj3qjL/77jt17LPPPlO5aNGiKnfv3l3lvHnzqpw7d+4Uqv5736O5z+3MmTNT/Fy4K1i99WZPv7k3v4mef/wTcx55Yz7vIZDPf4AdHnzwQZW//PJLlT378s174zz3YBfR99GJ/P35OX369FF5xIgRaSsWAbFmzRqV165dq3K1atWCWY7Sv39/Z3z58mV1bPTo0cEuxzVc+QcAAAAsweIfAAAAsETUdR+fQx0ujxZv2bKlyjly5FA5Pe01ZpvPV199pfL999/vjFP7a33rrbdU7tmz5w3XZbL90eL+FqitPs22HrPlwnyv9GzZGMy/v0icfyLh8z3QlJavR7j+GU2ROAdD9WuTKZPuHja/T918880qP/zww87YbNtNzYoVK1TOnz+/ymPGjHHGlStXVscGDhyo8rVr19L03mnB/NPMrc7Nr9uWLVtUTstWnxky6GvYaflcs+3Hc/6IiIwdO9bnc4USX+YfV/4BAAAAS7D4BwAAACzB4h8AAACwRNht9ZmaDz/80G/nio2NVblz584qe/b4i+jes9T6zsxtRhG6atWq5YxT6/lPzzaLZq9seu4v8KwZdjPvLRk6dKhLlSAStWjRQmVzq88yZcqonJY+f3Or7lKlSqmcNWtWld99911nbG4bGol9+OHi0KFDXrN534g3c+bMUbl8+fIqlyhRwudzmfeEjhw5UuVs2bKpPG7cOJU9t60NN1z5BwAAACzB4h8AAACwBIt/AAAAwBIR1/OfHoMGDVK5e/fuKhcsWFBls4fQs8/fPPbDDz+o/MUXX9xwnXBPWnr6Rf7e11+jRg1n7M9nBojo3u601gmI/H2+mhkoUqSIyuPHj1d59erVKpu992lRp04dr+/tzeTJk1VOy/7vCF3PPPOMykWLFlX5xRdfVPmhhx5SuWTJkime25wj/fv391rLSy+95PV4KOPKPwAAAGAJFv8AAACAJVj8AwAAAJaI+J5/c8/X/PnzO2Oz59rs+c+SJYvf6pgwYYLKSUlJfjs3gsecM+ae6f7u4/fG3L+d/mwAgdayZUuVCxcurLK57/+1a9d8Prfns3JEREaNGpWm2tatW+eMFy5cmKbPRXg6ePCgyt26dVO5WrVqKr///vvO2LxfIDXmPQD0/AMAAAAIeSz+AQAAAEuw+AcAAAAsEXY9/7feeqvX4z169FC5efPmKt9yyy3OOCoqSh0z9+ZPq99++80ZjxgxQh37+uuv03VuuMezj3/VqlXuFWLwfGYAAARDrly5VD58+LDKx48fv+FzN2zYUOU777wzTZ+/f/9+Z3z16tUbrgORY+3atSo3aNDAGe/cuTPY5YQMrvwDAAAAlmDxDwAAAFgiLNp+nnjiCWf88ccfq2PpbdVJizNnzqhsbkP2+uuvB60WBE8wt+9MC7Muz5akWrVqBbkaAJEqe/bszrh79+7q2NKlS1Xeu3evz+c1t1ocOXKkymaLkenKlSsq9+3b1+f3hp327NnjjCtVqqSObdq0KU3n2rx5s8oPPPDAjRcWZFz5BwAAACzB4h8AAACwBIt/AAAAwBJh0fP/6aefOmPz8d/Jyck3fN60nmvevHkq0+Nvh2HDhjljc3tNs+/+22+/VXn16tUpHjdfa24jmtZ7DTxfn1pdsIfn/BXRczhU72dBaPHcFjtjxozqWIECBbx+rtlXPXjwYGdcvXp1dezSpUsqz5o1S+X27durPG3aNJXTs80okNb1pDn3q1Wr5ozNLUZDDVf+AQAAAEuw+AcAAAAsweIfAAAAsETUdR83yvfs+XNTq1atVJ4+fbrK2bJl8/lc5p8ptb+Ka9euqdytWzeVZ8yY4fN7B1Iwn30QLKEy/4LJ2z7+aWXu+x/IewAicf6JRM4c9LwHYOjQoV5fG65/5kicg6HytTD30n/55ZdVNn9OmvfWJSQkOONXXnlFHVu2bJnKCxYsULlQoUIqlyhRQuVTp06lVHZQMf/CQ0xMjMqvvvqqyk8//bTXzzfntuc9KOb6MJh8mX9c+QcAAAAsweIfAAAAsASLfwAAAMASYbHPv6f58+ernDVrVpUffvhhld98880Uz2X2VDds2FDl0qVLq5wrVy6VS5UqpXKxYsWccXx8fIrvC3eZ+56b/e+hsie+Wcfw4cNVTq1f2xP7/gPwh8mTJ6t87tw5lQsXLqzyjz/+qLLns0/OnDmjjvXr10/lkiVLqjxy5EiVQ6XHH+Hp5MmTKq9fv17l1Hr+wxlX/gEAAABLsPgHAAAALMHiHwAAALCEK/v8T506VWWz72rIkCF+e6/0eO6551R+6623vL7+119/dcZVq1ZVx4LZm8gew5q5P77Z/24K5p74aZGeryv7/KdfpOxznZZ9/s37TMz7ZUJVJM7BSJl/nu666y6Vt2zZovLmzZtVrl+/vsqXLl0KTGHpxPwLT507d1Z5ypQpXl9v7vO/ePFiZ/z888+rY+Y6N5DY5x8AAACAg8U/AAAAYAkW/wAAAIAlXNnn3+zbO3TokBtlpOqnn35K0+uLFy/ujLNly+bvcnCDUuvxN5n3CASq19Hsn65Ro4bKaa3bk9mrHSr3LcB9nnMhLc+KAPztnXfeUTk6OlrlV155ReVQ7fFHZEpOTk7T65944glnvGLFCnVsxowZfqnJX7jyDwAAAFiCxT8AAABgiaC0/ZjtDHnz5lU5te2U3OL5KxyR1Ns/vvvuO2d89uzZgNSEtDNbXtLaTuPZnpPaVofmuc22ivS08qTGs9UnXLZkRPDRAgY31a5d2xk/+OCD6ti8efNUXrZsWVBqAvzBcw24du1aFytJHVf+AQAAAEuw+AcAAAAsweIfAAAAsERQev5Xr16tcosWLVQ2t/t6+umnVR41alRgCjMMGTJE5YIFC6qc2iOTH3roIWecJ08edez8+fPprA43ytz2Mq199559+25ujWj2arOdJ4Bw47l95/Hjx9Wx0aNHB7scwG9KlSrljEuWLKmO7dmzJ9jleMWVfwAAAMASLP4BAAAAS7D4BwAAACwRlJ5/0/Lly1U+dOiQypUqVVJ5/vz5ztjcaz+1Pnxv/HkuEZH169c74ytXrqTrXPAfsxc+vfv+B4tZZ61atdwpBABukHn/W5EiRZzxli1b1LFQ64uG3TJk8H59PGPGjCp73icaExMTkJr8hSv/AAAAgCVY/AMAAACWYPEPAAAAWMKVnn/TjBkzVI6NjU3xtXFxcYEux2fvv/++ys8++6xLlSAtzN75VatWqezWPQDmvv3Dhg1zpQ7gLzVq1HC7BIS5cePGqezZF+15Px8QapKTk9P0+t27dzvjUL9/hSv/AAAAgCVY/AMAAACWYPEPAAAAWCLquo+b25t74rulV69eKg8ZMkRlc09hb1Lb5z8+Pl7lN998U+U1a9aovHXrVp/fO5DS+7yCUBQq8w+pi8T5JxKZc9D8WqX2DIxw+TuIxDkYLn/3xYoVU/m///2vyomJic74pptuUseuXbsWsLqCifkXnjp37qyyueYz/fbbbyq3adPGGa9du9Z/haWRL/OPK/8AAACAJVj8AwAAAJZg8Q8AAABYIux6/pE6+g3hpkicfyLMwXASiXMwXObfyJEjVR40aJDK7777rjPu2LFjUGoKNuZfeIqJiVF5ypQpKjdu3FjlMmXKqBwqe/vT8w8AAADAweIfAAAAsARtPxGIXznCTZE4/0SYg+EkEudguMy/hg0bqjx//nyVK1Wq5Ix3794dlJqCjfkHN9H2AwAAAMDB4h8AAACwBIt/AAAAwBL0/Ecg+g3hpkicfyLMwXASiXOQ+Rc+mH9wEz3/AAAAABws/gEAAABLsPgHAAAALOFzzz8AAACA8Gb9lf8ZM2ZIVFSU5MyZ0+1SYIHExER58cUXpU6dOlKwYEGJioqSYcOGuV0WLPH9999L3bp1JVeuXJIzZ06pVauWrFu3zu2yYIl27dpJVFRUiv9t3LjR7RIR4X788Udp3LixxMbGSvbs2aVkyZIyYsQIuXjxotulBZXVi/8jR45Inz59JDY21u1SYImEhASZPn26XL58WRo3bux2ObDI5s2bpXr16pKUlCRz5syROXPmyKVLl6R27dqyYcMGt8uDBYYMGSIbNmz4238xMTFSpEgReeCBB9wuERFs9+7dUrVqVYmPj5fXX39dli5dKi1btpQRI0ZIq1at3C4vqDK5XYCbunTpItWrV5f8+fPLokWL3C4HFihatKicPn1aoqKi5OTJkzJjxgy3S4IlhgwZInnz5pXly5dL9uzZRUTkkUcekdtvv1369OnDbwAQcHfccYfccccd6mOrV6+WkydPyuDBgyVjxowuVQYbzJs3Ty5duiSLFy925uHDDz8sx44dk+nTp8vp06clX758LlcZHNZe+Z87d66sXr1apkyZ4nYpsMhfv94Ggm3dunVSs2ZNZ+EvIpIrVy6pXr26rF+/Xo4dO+ZidbDVzJkzJSoqSjp06OB2KYhwmTNnFhGRPHnyqI/nzZtXMmTIIFmyZHGjLFdYufg/fvy49OrVS8aNGye33HKL2+UAQMBduXJFoqOj//bxvz62Y8eOYJcEy509e1YWLVoktWvXlttuu83tchDh2rZtK3nz5pXnn39e9u/fL4mJibJ06VKZNm2adOvWTXLkyOF2iUFjZdtP165d5a677pLnn3/e7VIAIChKlSolGzdulOTkZMmQ4X/Xfa5evSqbNm0Skf/djwIE0/z58yUpKUmeffZZt0uBBYoVKyYbNmyQJk2aqPazf//73/L666+7V5gLrLvyv3jxYvn888/lnXfeof0CgDV69Oghe/fule7du8uRI0fk8OHD0qVLFzl48KCIiPMPAiBYZs6cKQUKFJAmTZq4XQosEB8fL40aNZICBQrIokWLZPXq1TJ+/Hh57733pGPHjm6XF1RWXfk/f/68dOvWTXr06CGxsbFy5swZEfnfr8NFRM6cOSOZM2e26lc/AOzQoUMHOXHihIwaNUqmTp0qIiJVqlSRPn36yMsvvyxFihRxuULYZPv27bJlyxbp2bPnP7ajAf7Wv39/OXfunGzbts1Z51WvXl1iYmKkQ4cO0qZNG6lRo4bLVQaHVZd6Tp48KX/88YdMmDBB8uXL5/w3f/58uXDhguTLl09at27tdpkAEBD9+vWTkydPyo4dOyQ+Pl7Wr18vp0+flhw5ckj58uXdLg8WmTlzpoiIdVdc4Z5t27ZJqVKl/naB968tZnfu3OlGWa6w6sr/zTffLKtWrfrbx8eNGyerV6+WZcuWSUxMjAuVAUBwREdHS+nSpUVE5NChQ/LRRx9Jp06dJFu2bC5XBltcvnxZ5s6dKxUrVnTmIhBosbGxsnPnTjl//rx6sOtfzzmxaQMYqxb/WbNmlZo1a/7t4++9955kzJjxH48B/rZs2TK5cOGCJCYmisj/Hjzy13Mm6tevr7ZiBPxl586dsnjxYqlQoYJER0fLTz/9JOPGjZPixYvLyJEj3S4PFvnkk0/k1KlTXPVHUPXq1UsaN24sjz76qPTu3VtiYmJk48aNMnbsWClVqpQ89thjbpcYNFHXr1+/7nYRbmvXrp0sWrRIzp8/73YpsECxYsWcmyxNBw4ckGLFigW3IFhh79690qlTJ+fKV1xcnLRs2VL69+/PfU4Iqjp16jjPlsiVK5fb5cAiq1atknHjxsn27dvl7Nmzcuutt0qjRo1kwIABUqBAAbfLCxoW/wAAAIAlrLrhFwAAALAZi38AAADAEiz+AQAAAEuw+AcAAAAsweIfAAAAsASLfwAAAMASLP4BAAAAS/j8hN+oqKhA1gE/isRHNzD/wkckzj8R5mA4icQ5yPwLH8w/uMmX+ceVfwAAAMASLP4BAAAAS7D4BwAAACzB4h8AAACwBIt/AAAAwBIs/gEAAABLsPgHAAAALMHiHwAAALAEi38AAADAEiz+AQAAAEuw+AcAAAAsweIfAAAAsASLfwAAAMASLP4BAAAAS2RyuwAAAAAgrTJnzqzySy+9pHK/fv1UzpQp5WXv77//rvKwYcNUnj59+g1UGJq48g8AAABYgsU/AAAAYAkW/wAAAIAloq5fv37dpxdGRQW6lhuyfft2lZcvX67yvHnzVN62bZvf3jt//vwqnz9/3hlfuXLFb++TVj5+ScNKqM4/N5UoUULllStXOuODBw+qY40aNVL5zJkzAasrEuefCHMwnETiHGT+hQ/mX+Dkzp1bZbPn//jx4yr/8ccfKp89ezbFc+fIkUPlm266SWWz53/gwIEqJyYmpnjuYPJl/nHlHwAAALAEi38AAADAEiz+AQAAAEuEfc//xYsXVc6aNavKTzzxhMqff/75Db9Xhgz630pz585V+fLly864S5cuKR4LNPoN7fDf//5X5eLFi6f42mbNmqm8ePHigNQkEpnzT4Q5GE4icQ7aMP9iY2NV7ty5s8q33Xabyg0bNlQ5X758znj8+PHq2NChQ1UO5M9k5p9/tW/f3hmvX79eHdu/f7/Kffr0Ufn9999X+ejRoym+T548eVSuU6eOyh9++KHK33zzjcqe99ZdunQpxfcJNHr+AQAAADhY/AMAAACWYPEPAAAAWCLse/7N3vrXX39d5cqVK6ucnn3+o6OjVTZ7ruPi4pxx1apV1bGNGzfe8PumFf2GkWnixIkq9+zZM8XXmve2NG3aVOWrV6/6rzBDJM4/kdCdg3nz5lXZ7FM1v/ae93+Yf6YFCxaoPHv2bJW/+OKLGy0zqCJxDobq/EtN2bJlVfacj0899ZQ6VrhwYZXNHmzz67p3794UP9/cD948t7n/uz8x//zro48+csaHDh1Sx/r27Ruw982VK5fKH3/8scoPP/ywyvfdd58zNp9BFUz0/AMAAABwsPgHAAAALMHiHwAAALBEJrcLSKuYmBiVzZ7/FStWqJyeHn+TuS/wpk2bVC5atKjf3gsw51Pbtm19/twlS5aoHMgefwSX2eP/2Wefqfzggw96/Xxv/aBmD3aTJk1UNvdKHzt2rNf3QuQx+6ArVKigstmDXa9ePZU9519CQoI6Zt6rZPZYm3u6//LLLyq/9tprzrhdu3bqWHJysiA8ee6vb+61/95776m8a9cuv71vYmKiygMGDFDZXAN63gPgZs+/L7jyDwAAAFiCxT8AAABgibBr+/F8fLKISJkyZVT2/LWfv2XOnFnlkiVLqhyJ23vBPeaj6812D9OpU6ec8bJlywJREkLA5MmTVU6tzcfk+atsc7tD83tYsWLFVB44cKDKCxcuVPnXX39NUy0ITbfffrvKntvDPvfcc+pYau2uc+bMUXnr1q3O+N1331XHzDaL1OTLl09lz21EV69erY6dOHEiTedG6PDcVt1sYR00aJDKZrvXlStX/FbHvn37vB43/78JZVz5BwAAACzB4h8AAACwBIt/AAAAwBJh0fOfP39+Z/zyyy97fe0PP/wQsDrMnv977703YO8F+9SpU0flN954Q2Xz8epJSUkqe26pF8hH1yP4PL+2rVu3VsfMPv3jx4+rPHv2bJWnT5/ujFPrYV20aJHK5tafd955p8r0/Icn8/61r7/+WuXo6GhnbG53eOnSJZVXrlyp8rFjx/xR4j8aPHiwyp5bgW/YsCFg74vg2r17tzM272dr0aKFyt99953KU6dO9Vsd5ja2phkzZvjtvQKNK/8AAACAJVj8AwAAAJZg8Q8AAABYIix6/j37DT17+kREDh48qHIg+wuBQDKfYWH2+Ju93Z9++qnKW7ZsCUxhcF2BAgVSPLZ8+XKVe/bsqXJ6+vD79++vstnzX7NmTa+1IDyYe+Cbz8vx3I//9OnTQanpn9xzzz0qm/e/eD7rxHwGBSJD8+bNVfZ8boSIyKRJk1Tes2ePyqtWrfL5vapWraqy2fP/zTffqLx3716fz+02rvwDAAAAlmDxDwAAAFiCxT8AAABgibDo+W/cuHGKx8w9rT17/vytdOnSXo979kL+/vvvAasDkaFatWoqP/30015fb/bldurUye81ITR59jab94IMGTJEZX/utV+kSBGVzfdu1qyZyuY9AggPCQkJKk+YMMGlSrzr0qWLygULFlT5nXfeccaee8MjciQnJ6s8dOhQlefOnavyvHnzVO7du7cz/vDDD9Ux856S+fPnq3zlyhWVn3zySZXNZ16EMq78AwAAAJZg8Q8AAABYgsU/AAAAYImQ7Pk397Ru27Ztiq8tUaKEyp999pnK5nMAPPeATWtvbNOmTb0e9+wti4+PT9O5YYeMGTM641deeUUdy5cvn9fPnThxosrnz5/3X2EIaZ7PePjtt9/UMX/uLV23bl2V33vvvRTrEBEpVqyYyhUqVHDGPHcC6dWvXz+VO3bsqPIPP/yg8qBBgwJeE0KL+bybGTNmqPzvf/9b5ZkzZzrjWrVqqWMNGzZU+eabb1bZvBcmMTExbcWGEK78AwAAAJZg8Q8AAABYIuq6+XvclF5obPEWSOYjldeuXeu3cx87dswZd+/eXR1bsmSJyoULF1b5xx9/VLlQoUIqDxw40BmPGzcuXXWmh49f0rASzPkXSKNHj3bGAwYM8Pra1atXq2z+ijJUReL8E3F3Dk6ePNkZd+vWTR1bsGCByub3HnM7xAMHDjhj83H17du3V9mzTc0Xbdq0ccYffPBBmj7XnyJxDkbK90BvYmNjVTZb3EzmVouffPKJv0u6Icy/0LFv3z6Vb7vtthRfa/4Z//vf/6pcsmRJ/xUWQL7MP678AwAAAJZg8Q8AAABYgsU/AAAAYImQ3OrTfMSyp6tXr6ps9qiaPVvDhw9X2bPf64033lDHzN6w7Nmzq2z2+Jv+/PNPr8dhH/O+EXO+etO6dWt/l4Mw9csvv6R4rFmzZl6zN+b3y0jsVUboypw5s8pz5szx+vqvvvpK5a+//lplz+2ST58+nc7qEAmmTp2q8ssvv+zz55rzM1u2bConJSXdeGEu48o/AAAAYAkW/wAAAIAlWPwDAAAAlgjJnv/z58+rfPnyZWf89NNPq2Pm3vymX3/9VeWRI0c649q1a6tj27ZtS0uZf+PmvtYIDWZP4DfffKOy+bhwT6+99prKns+kgN1++OEHZ+y5T7+ISLFixfz2PgsXLlR58eLFKpv3UN11111+e2+EB7MPukCBAiqb87FatWoqN23a1BnnzJlTHStVqpTX937kkUdU/v7771X2/P6bnl5vRI7HHnssxWPmPaTm3Dbn8rvvvqtyu3btVPZcq4Y6rvwDAAAAlmDxDwAAAFiCxT8AAABgiajrPm7sbO4HHUhm35VnX2B69+7NlOn/3+bw7LPPqmMDBgxQOS4uzuu5zH7YFi1aOOPk5OQbLTHdInGv7mDOv/TImzevyqdOnUrxteaxihUrqrx//36/1RVMkTj/REJ3DsbExKhct25dlUuUKKFyhQoVnPH8+fPVMbPn3/P7pYjIli1bVDZ7/tu0aeOM586d663sgIrEORgq82/y5Mkqd+3a1evr0/MsifXr16tsPovHFB8f74w/++wzdWzr1q0+v296Mf/c079/f5VHjBihsrd7SMuXL6/ykCFDvL7X0aNHVb777rudsXnvajD5Mv+48g8AAABYgsU/AAAAYAkW/wAAAIAlQrLn3y2PP/64yp988onK5h6uRYoUUdlbf3cw0W/ono8++kjlZs2apfjatm3bqjxnzpyA1BRskTj/RMJnDvrTnXfeqfJ///tfr6/37Pl387knkTgH3Zx/sbGxznjXrl3q2Nq1a1Xu3LmzyuazT9atW+eMb7rpJnXMvOfE8z66cML8C578+fOrfOTIEZWzZMmi8quvvuqM+/Xrp46Z95t6PpNCJPXvaZ73RNWsWVMdS0pK8vq5/kTPPwAAAAAHi38AAADAEiz+AQAAAEtkSv0l9jB7+E3m3v2h0uMP93jumS4i0qhRI6+v//PPP51xpPT4A3/ZsWOH2yUgADz3MzefG3Hp0iWVExMTVW7SpInKns+lOHfunDr24osvpqtO2Me8V9Ps8Tf34p85c2aK5/L8+SwismTJEpWrVKmi8rfffquy53pgxowZ6ljr1q1TfF83cOUfAAAAsASLfwAAAMASVrf9ZMyYUWXz15OmDz/8MJDlIAyYc2bAgAEqZ82a1evn9+rVy98lASFj+/btbpeAADtx4kSaXm+2P2TI8P+vOQ4fPlwdO3jw4I0XBivly5fP6/FZs2apvHfvXp/PbW7v/v3336vcsmVLlT23+m7QoIE6VrhwYZWPHTvmcx2BwJV/AAAAwBIs/gEAAABLsPgHAAAALGF1z3/u3LlVfuSRR7y+ftGiRYEsB2Fg4MCBKj/55JMqm4/V3rVrl8qpPR4cCCcHDhxwuwSEOLMn23OLbH6mIr1Suwfl7rvvVjlbtmzOOCkpKU3vZd7zd/z4cZU9t731vLdF5O9bkLqNK/8AAACAJVj8AwAAAJZg8Q8AAABYwuqe/7Zt23o9fvbsWZX/+9//BrIchCjPx9mbj583e/zNfYG7d++usvk4eyCcREVFqbxhwwaXKkGoiI6OVnnOnDkqm3OmR48ezvjw4cOBKwxW+OKLL1S+ePGiyuZ9effff78z3rNnjzp25513qmz+fDfnsvn6nTt3OuMxY8aoY6H2DAuu/AMAAACWYPEPAAAAWILFPwAAAGAJq3v+U2P2b3vu4Qp7lClTxhnnyJFDHTN7AOPj41VevXp1wOoCAi0mJkZlswfWzLDPbbfdpnLTpk1VNufIN998E/CaYI/Tp0+r/OCDD6psPlunVKlSzrhYsWLqmPnzPLV7+sxnQ61du9YZ//nnn16qdh9X/gEAAABLsPgHAAAALMHiHwAAALCE1T3/u3fvVvnQoUMqm/u0Hj16NOA1IfTs37/fGZv3fZw4cULlRo0aBaUmIBjq1avndgkIcX379vV63HwWREJCQiDLgeW2b9+u8r333utSJaGNK/8AAACAJVj8AwAAAJZg8Q8AAABYwuqe///85z8qm3u+AiIiW7dudcbZs2d3sRIguA4cOOD1eK1atYJUCUKV5/dHEZF27dqpPHHiRJWvXr0a6JIApIIr/wAAAIAlWPwDAAAAloi67uPz2c3HHiN0+fglDSvMv/ARifNPxM45WL16dZVfeukllT/44AOVZ82aFfCafBGJc9DG+ReumH9wky/zjyv/AAAAgCVY/AMAAACWYPEPAAAAWIKe/whEvyHcFInzT4Q5GE4icQ4y/8IH8w9uoucfAAAAgIPFPwAAAGAJFv8AAACAJXzu+QcAAAAQ3qy68v/tt99KVFTUP/63ceNGt8uDJdauXSv169eXfPnySbZs2aR48eIycuRIt8uCBb7//nupW7eu5MqVS3LmzCm1atWSdevWuV0WLMIchJt+/PFHady4scTGxkr27NmlZMmSMmLECLl48aLbpQWVVYv/v4wZM0Y2bNig/itdurTbZcEC8+bNkxo1akiePHlk9uzZ8uWXX0q/fv0icncIhJbNmzdL9erVJSkpSebMmSNz5syRS5cuSe3atWXDhg1ulwcLMAfhpt27d0vVqlUlPj5eXn/9dVm6dKm0bNlSRowYIa1atXK7vKCyqu3n22+/lVq1asnChQvlqaeecrscWObIkSNy1113SZs2bWTKlClulwPL1KtXT7Zt2yb79++X7Nmzi4hIYmKi3H777VKiRAmuviLgmINw0+DBg2X06NHy66+/yh133OF8/LnnnpPp06fLqVOnJF++fC5WGDxWXvkH3DBjxgy5cOGC9OvXz+1SYKF169ZJzZo1nUWXiEiuXLmkevXqsn79ejl27JiL1cEGzEG4KXPmzCIikidPHvXxvHnzSoYMGSRLlixulOUKKxf/3bp1k0yZMknu3Lmlbt26snbtWrdLggXWrFkj+fPnlz179ki5cuUkU6ZMUqhQIenSpYucO3fO7fIQ4a5cuSLR0dF/+/hfH9uxY0ewS4JlmINwU9u2bSVv3rzy/PPPy/79+yUxMVGWLl0q06ZNk27dukmOHDncLjForFr858mTR3r27CnTpk2TVatWyaRJk+Tw4cNSs2ZNWbFihdvlIcIdOXJELl68KM2aNZMWLVrIypUrpW/fvjJ79mypX78+ff8IqFKlSsnGjRslOTnZ+djVq1dl06ZNIiKSkJDgVmmwBHMQbipWrJhs2LBBdu7cKXfccYfkzp1bGjVqJG3btpVJkya5XV5QWbX4v+++++T111+Xxo0by0MPPSTt27eX9evXS+HCheXFF190uzxEuOTkZLl06ZIMHDhQBgwYIDVr1pS+ffvK2LFjZd26dfL111+7XSIiWI8ePWTv3r3SvXt3OXLkiBw+fFi6dOkiBw8eFBGRDBms+nEAFzAH4ab4+Hhp1KiRFChQQBYtWiSrV6+W8ePHy3vvvScdO3Z0u7ygsv7/tLx580rDhg1l+/btkpSU5HY5iGAFChQQEZG6deuqjz/22GMiIrJ169ag1wR7dOjQQcaNGydz5syRW265ReLi4mT37t3Sp08fEREpUqSIyxUi0jEH4ab+/fvLuXPnZMWKFdK0aVOpXr269O3bV15//XV59913ZfXq1W6XGDTWL/5FxGm3iIqKcrkSRLIyZcr848f/mn9c9UKg9evXT06ePCk7duyQ+Ph4Wb9+vZw+fVpy5Mgh5cuXd7s8WIA5CLds27ZNSpUq9bfe/gceeEBERHbu3OlGWa6wfrVx+vRpWbp0qZQrV06yZs3qdjmIYE2bNhURkWXLlqmPf/nllyIiUrly5aDXBPtER0dL6dKlpWjRonLo0CH56KOPpFOnTpItWza3S4MlmINwQ2xsrOzatUvOnz+vPv7XMyZuueUWN8pyhVX7/D/99NMSFxcnFSpUkJiYGPnll19kwoQJsm/fPlm2bJk88sgjbpeICPf444/Lf/7zHxk8eLBUrlxZtmzZIsOHD5dHHnlEPv/8c7fLQwTbuXOnLF68WCpUqCDR0dHy008/ybhx46RYsWKyatUqyZkzp9slIsIxB+Gmzz77TBo3biyVKlWS3r17S0xMjGzcuFHGjh0rcXFx8uOPP1qz3adVi/9x48bJRx99JAcOHJDz589L/vz5pVq1ajJgwADn1z5AICUlJcnw4cNl3rx5cuzYMYmNjZXWrVvL0KFD/3ELPMBf9u7dK506dZKdO3fK+fPnJS4uTlq2bCn9+/e3aos7uIc5CLetWrVKxo0bJ9u3b5ezZ8/KrbfeKo0aNZIBAwY49+XZwKrFPwAAAGAz63v+AQAAAFuw+AcAAAAsweIfAAAAsASLfwAAAMASLP4BAAAAS7D4BwAAACzB4h8AAACwRCZfXxgVFRXIOuBHkfjoBuZf+IjE+SfCHAwnkTgHmX/hg/kHN/ky/7jyDwAAAFiCxT8AAABgCRb/AAAAgCVY/AMAAACWYPEPAAAAWILFPwAAAGAJFv8AAACAJVj8AwAAAJZg8Q8AAABYwucn/ALwrwoVKqg8efJklW+++WaVly1bpnLXrl0DUxgAAIhYXPkHAAAALMHiHwAAALAEi38AAADAEvT8A0FUtmxZZ7x48WJ1bN++fSpXqVJF5d9//z1whQEAACtw5R8AAACwBIt/AAAAwBIs/gEAAABLhF3Pf4ECBVTOnj27ys8++6zKuXLlUrlBgwbOuHjx4urYjBkzVDZ7rN966y2Vjx8/7kPFsFmWLFlU7tOnjzO+9dZb1bFq1aqpTI8/gEi2YMEClZs1a6ZyixYtvL4eMJUsWVLlXr16qdykSRNnvGTJEq/neuedd1T++eefVb548eINVBgauPIPAAAAWILFPwAAAGAJFv8AAACAJaKuX79+3acXRkUFuhZHbGysym3atHHGzz//vDpWpEgRr+cy6/bxj/uPn7t+/XqVH3roIZ/PFUxp+TOGi2DOP39q27atyrNmzXLGK1euVMfq1auncnJycuAKC6BInH8i4TsHM2fOrHKrVq2ccb9+/dSxUqVKqfzNN9+ovGjRIpW/+uorlX/99dcbrtOfInEOhuv8M+9tWrduXYrHTIcPH1Y5Li7Of4UFEPPPPZs3b1b5/vvvV9nza5Pa+tA8bt4jMHjwYJX37NmTtmIDxJf5x5V/AAAAwBIs/gEAAABLsPgHAAAALBESPf9dunRR+cUXX1S5aNGizjitvXTbt29XeeLEic74l19+SdO5EhMTVd61a1eaPj9Y6DcMHdOmTVP50UcfdcZVq1ZVxyJlX/9InH8i4TMHo6OjVTb3uR47dqzf3uv9999XuX379n47d3pE4hwMl/lnOnTokMqeff5mT//ChQtVfuGFF1QOl78D5p97OnfurPLbb7+tsufXxryHdPfu3So/99xzKjdu3FjlpKQklR977DFn/MMPP/hWcADQ8w8AAADAweIfAAAAsASLfwAAAMASIdHz77nvr4hIpUqVUnzvo0ePqmMffvihyjNnzlT5yJEjKpt9+5GIfkP3mH38q1atUrlnz57O2OxFjBSROP9EQncOFitWTOV58+apXLlyZZU9e7DN+63MZ03kz59f5WbNmql89epVlXPlypV6wUEQiXMwVOefqXfv3iq/9tprKnv29Tdv3lwdM/f9N+8XqFKlisobN2684ToDifnnHrPnf+rUqSp7fm0yZcqUpnM3adJEZfO5JydOnHDGNWvWVMeC+QwAev4BAAAAOFj8AwAAAJYIibafEiVKqGz+Ki9fvnzOeP369erYgw8+GLC6whW/cgye7Nmzq7xp0yaVL126pLLnr63NlolIEYnzTyR052CPHj1UnjRpkso///yzymXKlHHG165d83ruhg0bqvzpp5+q/PHHH6tstgW5JRLnYKjOv9RadcztPOPi4nw+t/l1NFuI/u///s/ncwUT8889adnqM2PGjOl6r5EjR6o8cOBAZ2y2Xz7zzDPpeq+0oO0HAAAAgIPFPwAAAGAJFv8AAACAJdK2z1GA7N27V2VzO848efI446VLlwalJsAXDzzwgMr33HOPyiNGjFA5Uvv84R7PR8r/k2effVbl1Pr8PT355JMqm32/5tZ3nrUsW7bM5/dB+Hrqqae8Hk/LfXnm/QKmF154wevxUL0HAMFj3ofkub22iMhdd93ljEuWLKmOpbYdp/l6k2evfbVq1dSxmJgYlU+ePOn1XIHGlX8AAADAEiz+AQAAAEuw+AcAAAAsERI9/2nxyy+/uF0C4Hj55ZdVPnjwoMoTJkwIZjnA35j3UHmTOXNmlUuVKuX19X/++afK27Zt8/m9EBl69+6t8sKFC1U29/n3ZD4jwMypSevrEfnMXvqxY8eq/P777ztj8/4Acy/+6tWrq9y/f3+VzXtUfvzxR2c8ZswYr3W5jSv/AAAAgCVY/AMAAACWYPEPAAAAWCLsev47duyo8vbt21Vu0KDBDZ/7/vvvV7lGjRoqm88YMO8/mDFjhjNOS58twkexYsVUrlChgspmz3Mg58Edd9yhcrly5ZyxuV/xrl27AlYHQluvXr1U7tSpU4qvjYuLU7lixYpez/3ee++pfOzYsTTVhvBj9vibffdp2dffnJuAv/38888qe+7F77nnv4jIpk2bVDafa7J7926VvT1jJdR6/E1c+QcAAAAsweIfAAAAsASLfwAAAMASIdnzv2bNGpVbt27tjB999FF1zOznMpk9W579XqkxP/e5557z+vpXX33VGT/55JPq2FdffaXyxYsXfa4DoSNDBv3v5YwZM6q8ePHiGz63eS7PeS/y92cKxMTEqHz16lVnfOXKFXWsbt26Km/cuPGG60Ro+fTTT1WuV6+eym3btlW5cuXKzvjXX39VxypVquT1vTznmIjI1q1bfa4TkeG1115T2dzH39u+/qb07tO/YcOGdH0+It8PP/ygsude/OXLl/f6uZMmTVL5hRde8F9hLuPKPwAAAGAJFv8AAACAJaKu+9gHY7bABFKJEiVU7t69uzNu2bKlOmaWnytXLpU9f8UjIpKcnJzi+86cOVNlc+vP++67T2Wz7aJ48eLO2Pz7Mrc0mzx5cop1pFdaWpvCRTDnnzfmVp/79+9XuVSpUiqbW25mzpzZGVerVk0de+WVV1S+9957VZ4+fbrKb7/9tsoHDhxwxmYriPn/hdk+588tSSNx/omEzhw0mVsrej6+XkTk9ttv99t7ffHFFyo3atTIb+f2p0icg6Ey/9avX6/yb7/9pnLz5s1v+FxVqlRJUy2h8ndiYv6FrtmzZztjs7XW/LrdfPPNKof69p1/8WX+ceUfAAAAsASLfwAAAMASLP4BAAAAS4Rkz7835v0AZg9/7ty5VQ7kVnQFChRQeefOnc64UKFC6pjZ2/jQQw8FrC76DQMntZ5/8z4R83HgI0eOdMZ9+/ZVx8wtydq1a6fyrl27fK6zT58+Ko8fP15lsw88Pj7e53OnJhLnn0jozMHU3HLLLSqb2w57KlKkiMrmnDSZxydMmJDG6oIjEudgqMw/8++2RYsWKi9YsOCGz5VWofJ3YmL+hQ7Pn7kiIgMHDnTGqW0FX6tWLZXNbehDFT3/AAAAABws/gEAAABLsPgHAAAALJHJ7QLSau/evW6X4EhISFD5ypUrLlWCUGHek2LuW+35zIpBgwapYy+//LLK3p5JkZpffvlF5T/++EPlc+fO3fC5EdrMfdffeOONFF9brlw5lVPr+TefHwGkVajeJ4LIULJkSZU9e/xFvPfDm8fMc4VLz78vuPIPAAAAWILFPwAAAGAJFv8AAACAJcKu5z+UZM+eXeVMmf7/X6e5f2yGDPw7ywZ33HGHykOHDlV57dq1znjs2LF+fe+4uDhnPGbMGHVs5cqVKp86dcqv743wZO5jnZrTp08HqBKEq169eqn81FNPqdysWTOfz7VhwwaVzWdWbNy4MW3FwTrmvXTmWmzJkiXO+OTJk+pYp06dAldYiGFFCgAAAFiCxT8AAABgCRb/AAAAgCXo+U+Hjh07qnzTTTc5Y3O/2KVLlwalJrircOHCKn///fcqm/vvp0fmzJlVHj9+vDPOly+fOvbKK6/47X0ROcz+bNOuXbtUvnjxYiDLQRg4fPiwyuazTFLj2df/wgsvqGNmT7/5c3ThwoVpei9EvoIFC6pcrVo1lX/++WeV27Rp44wfeughdcxc0zVp0kTl6dOn33CdoYYr/wAAAIAlWPwDAAAAlmDxDwAAAFiCnv80KF26tMojRoxI8bXr1q1TOZJ6xWx25coVlQ8dOqTybbfdpnKdOnVUvnTpkt9q6dmzp8rNmzd3xi1btlTHtm/f7rf3RXjz3Nu/YsWKXl/7xhtvqJyUlBSQmhA+HnzwQZUnTJigsrlX/8SJE/323ps2bfLbuRAZihYtqrLn825ERLZu3aqy531LBw8eVMfM729169ZV+f777/d67nDClX8AAADAEiz+AQAAAEvQ9uNFrly5VJ4yZYrKOXPmTPFzPR8hLSKSkJDgv8LgmqNHj6rcvXt3lT/77DOVPbcVE0lb+1fGjBlVNh89PmrUKJVnzpzpjM35B/zl3nvvdcbmHPvzzz9VNrf6BMytPj3bDYFge+KJJ1Q2t4f1Zs+ePV6z2eYTSbjyDwAAAFiCxT8AAABgCRb/AAAAgCXo+fdg9vivWrVK5fvuu8/r548cOdIZ+3N7M4SuZcuWqbxgwQKV33zzTZVLlizpjL/++mt1LDY2VuVmzZqp/Mgjj6j87rvvqvz8888746tXr3orGxYx+/rNHllPv//+u8rr168PSE0A4A+FChVSOSoqymv2lNo2oRkyRO718cj9kwEAAABQWPwDAAAAlmDxDwAAAFgi7Hr+a9SooXLXrl1VnjFjhsqJiYkpnuuuu+5SuWfPniqXLVtWZXP/2I8//ljll19+OcX3QmS6du2ayk8//bTKgwYNUtlz3/9evXp5Pfe6detUbtGihcqLFy9WOTk52ev5YKd8+fKpXKtWrRRfO27cuECXA6SIZwYgrcx1WWrZ02uvvaZygQIFVDZ/xprPAQhnXPkHAAAALMHiHwAAALAEi38AAADAEmHX82/udd60aVOv2dzj1Vv/l+nixYsqv/POOyoPHTpU5aSkJJ/Pjchk9t17PvvhnzIQaA899JDPr92wYUMAKwG8Y/4hrdauXatyp06dVK5QoYLKnvfpmfv4mz+/Bw8erLK5JgxnXPkHAAAALMHiHwAAALAEi38AAADAEmHX8z9q1CiVp02bpnLHjh1Vzpkzp8qePf8lSpRQx44ePary0qVLVf7iiy/SViwAuOyOO+5I8djly5dVvnTpUqDLAVJ0+PBhr8cPHToUpEoQLsznLZnPb3ryySdTPH78+HF17JlnnlE5kvb1N3HlHwAAALAEi38AAADAEiz+AQAAAEtEXfdx43tzv3yErrQ8yyBcMP/CRyTOP5HwnYONGzdW2bNHduHChepYixYtglFSwEXiHAzX+Wcj5h/c5Mv848o/AAAAYAkW/wAAAIAlaPuJQPzKEW6KxPknwhwMJ5E4B5l/4YP5BzfR9gMAAADAweIfAAAAsASLfwAAAMASLP4BAAAAS7D4BwAAACzB4h8AAACwBIt/AAAAwBI+7/MPAAAAILxx5R8AAACwBIt/AAAAwBIs/gEAAABLsPgHAAAALMHiHwAAALAEi38AAADAEiz+AQAAAEuw+AcAAAAsweIfAAAAsMT/A9JiQd1QHzMvAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 1000x800 with 25 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "figure = plt.figure(figsize=(10, 8))\n",
    "cols, rows = 5, 5\n",
    "for i in range(1, cols * rows + 1):\n",
    "    sample_idx = torch.randint(len(train_data), size=(1,)).item()\n",
    "    img, label = train_data[sample_idx]\n",
    "    figure.add_subplot(rows, cols, i)\n",
    "    plt.title(label)\n",
    "    plt.axis(\"off\")\n",
    "    plt.imshow(img.squeeze(), cmap=\"gray\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "832bcf0e",
   "metadata": {},
   "source": [
    "DataLoaders are necessary for letting the model process large datasets."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "d99724e6-e386-45c7-af80-e3d21e2a097e",
   "metadata": {},
   "outputs": [],
   "source": [
    "loaders = {\n",
    "    'train' : torch.utils.data.DataLoader(train_data,\n",
    "                                          batch_size=100,\n",
    "                                          shuffle=True,\n",
    "                                          num_workers=1),\n",
    "    'test' : torch.utils.data.DataLoader(test_data,\n",
    "                                         batch_size=100,\n",
    "                                         shuffle=True,\n",
    "                                         num_workers=1)\n",
    "}"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e470faa8",
   "metadata": {},
   "source": [
    "## Creating the Model\n",
    "\n",
    "Next, we define the model as well as its forward pass function. This is a fairly simple CNN with 2 convolutional layers."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "0d1ea6d6-e052-48dd-93a6-b521b38dadbd",
   "metadata": {},
   "outputs": [],
   "source": [
    "class CNN(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(CNN, self).__init__()\n",
    "        self.conv1 = nn.Sequential(\n",
    "            nn.Conv2d(\n",
    "                in_channels=1,  # (int) -> number of channels in input image    =1 because output is grayscale image\n",
    "                out_channels=16, # (int) -> number of channels produced by the convolution\n",
    "                kernel_size=5, # (int, tuple) -> size of convolving kernel\n",
    "                stride=1, # (int, tuple, optional) -> stride of convolution, default is 1\n",
    "                padding=2, # (int, tuple, optional) -> zero-padding added to both sides of input, default is 0\n",
    "            ),\n",
    "            nn.ReLU(),\n",
    "            nn.MaxPool2d(kernel_size=2),\n",
    "        )\n",
    "        self.conv2 = nn.Sequential(\n",
    "            nn.Conv2d(16, 32, 5, 1, 2),\n",
    "            nn.ReLU(),\n",
    "            nn.MaxPool2d(2),\n",
    "        )\n",
    "        self.out = nn.Linear(32 * 7 * 7, 10)\n",
    "    def forward(self, x):\n",
    "        x = self.conv1(x)\n",
    "        x = self.conv2(x)\n",
    "        x = x.view(x.size(0), -1)\n",
    "        output = self.out(x)\n",
    "        return output, x       "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "bf629382-9dc6-417d-ac6e-9bfc8d929e3f",
   "metadata": {},
   "outputs": [],
   "source": [
    "cnn = CNN()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "703a643c",
   "metadata": {},
   "source": [
    "Next we define the loss function and optimizer. Cross Entropy Loss is a loss function for classification problems, while Adam is a popular and powerful optimizer that extends the already effective stochastic gradient descent algorithm."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "8242ffcc-d539-4bfb-bf5c-d6a48734a9f0",
   "metadata": {},
   "outputs": [],
   "source": [
    "loss_func = nn.CrossEntropyLoss()\n",
    "optimizer = optim.Adam(cnn.parameters(), lr=0.01)  "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "df753edf",
   "metadata": {},
   "source": [
    "It is worthwhile to check and make sure what device we are running on. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "3c7e42c3-a6ff-4bf2-8952-799307d39269",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Using device: cpu\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# train on the GPU or on the CPU, if a GPU is not available\n",
    "device = torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu')\n",
    "\n",
    "print('Using device:', device)\n",
    "print()\n",
    "\n",
    "#Additional Info when using cuda\n",
    "if device.type == 'cuda':\n",
    "    print(torch.cuda.get_device_name(0))\n",
    "    print('Memory Usage:')\n",
    "    print('Allocated:', round(torch.cuda.memory_allocated(0)/1024**3,1), 'GB')\n",
    "    print('Cached:   ', round(torch.cuda.memory_reserved(0)/1024**3,1), 'GB')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b29c3200",
   "metadata": {},
   "source": [
    "## Training the Model\n",
    "\n",
    "Next is the training step. \n",
    "\n",
    "**As this is a convolutional neural network being trained on a large image dataset, expect training time to take much longer than for a simple linear regression model. It is also worthwhile to time the training loop.**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "61975a57-f7ae-4a47-90ad-27f7b03b89c1",
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch [1/5], Step[1/600], Loss: 2.3010\n",
      "Epoch [1/5], Step[2/600], Loss: 2.7873\n",
      "Epoch [1/5], Step[3/600], Loss: 2.2077\n",
      "Epoch [1/5], Step[8/600], Loss: 1.4460\n",
      "Epoch [1/5], Step[9/600], Loss: 1.1443\n",
      "Epoch [1/5], Step[10/600], Loss: 0.8667\n",
      "Epoch [1/5], Step[11/600], Loss: 0.9214\n",
      "Epoch [1/5], Step[16/600], Loss: 1.0815\n",
      "Epoch [1/5], Step[17/600], Loss: 0.5077\n",
      "Epoch [1/5], Step[18/600], Loss: 1.0005\n",
      "Epoch [1/5], Step[19/600], Loss: 0.6165\n",
      "Epoch [1/5], Step[24/600], Loss: 0.4259\n",
      "Epoch [1/5], Step[25/600], Loss: 0.3769\n",
      "Epoch [1/5], Step[26/600], Loss: 0.3689\n",
      "Epoch [1/5], Step[27/600], Loss: 0.3474\n",
      "Epoch [1/5], Step[128/600], Loss: 0.0309\n",
      "Epoch [1/5], Step[129/600], Loss: 0.0608\n",
      "Epoch [1/5], Step[130/600], Loss: 0.0554\n",
      "Epoch [1/5], Step[131/600], Loss: 0.2005\n",
      "Epoch [1/5], Step[136/600], Loss: 0.1554\n",
      "Epoch [1/5], Step[137/600], Loss: 0.2146\n",
      "Epoch [1/5], Step[138/600], Loss: 0.0350\n",
      "Epoch [1/5], Step[139/600], Loss: 0.0702\n",
      "Epoch [1/5], Step[144/600], Loss: 0.1806\n",
      "Epoch [1/5], Step[145/600], Loss: 0.0505\n",
      "Epoch [1/5], Step[146/600], Loss: 0.0413\n",
      "Epoch [1/5], Step[147/600], Loss: 0.0866\n",
      "Epoch [1/5], Step[152/600], Loss: 0.1066\n",
      "Epoch [1/5], Step[153/600], Loss: 0.0252\n",
      "Epoch [1/5], Step[154/600], Loss: 0.1037\n",
      "Epoch [1/5], Step[155/600], Loss: 0.0856\n",
      "Epoch [1/5], Step[256/600], Loss: 0.0678\n",
      "Epoch [1/5], Step[257/600], Loss: 0.0756\n",
      "Epoch [1/5], Step[258/600], Loss: 0.1582\n",
      "Epoch [1/5], Step[259/600], Loss: 0.1289\n",
      "Epoch [1/5], Step[264/600], Loss: 0.0499\n",
      "Epoch [1/5], Step[265/600], Loss: 0.0876\n",
      "Epoch [1/5], Step[266/600], Loss: 0.0833\n",
      "Epoch [1/5], Step[267/600], Loss: 0.0905\n",
      "Epoch [1/5], Step[272/600], Loss: 0.1186\n",
      "Epoch [1/5], Step[273/600], Loss: 0.0602\n",
      "Epoch [1/5], Step[274/600], Loss: 0.0861\n",
      "Epoch [1/5], Step[275/600], Loss: 0.1177\n",
      "Epoch [1/5], Step[280/600], Loss: 0.0497\n",
      "Epoch [1/5], Step[281/600], Loss: 0.0421\n",
      "Epoch [1/5], Step[282/600], Loss: 0.1780\n",
      "Epoch [1/5], Step[283/600], Loss: 0.0327\n",
      "Epoch [1/5], Step[384/600], Loss: 0.0285\n",
      "Epoch [1/5], Step[385/600], Loss: 0.0828\n",
      "Epoch [1/5], Step[386/600], Loss: 0.0458\n",
      "Epoch [1/5], Step[387/600], Loss: 0.0500\n",
      "Epoch [1/5], Step[392/600], Loss: 0.2066\n",
      "Epoch [1/5], Step[393/600], Loss: 0.1402\n",
      "Epoch [1/5], Step[394/600], Loss: 0.0493\n",
      "Epoch [1/5], Step[395/600], Loss: 0.0344\n",
      "Epoch [1/5], Step[400/600], Loss: 0.0806\n",
      "Epoch [1/5], Step[401/600], Loss: 0.0828\n",
      "Epoch [1/5], Step[402/600], Loss: 0.1046\n",
      "Epoch [1/5], Step[403/600], Loss: 0.0535\n",
      "Epoch [1/5], Step[408/600], Loss: 0.0878\n",
      "Epoch [1/5], Step[409/600], Loss: 0.0360\n",
      "Epoch [1/5], Step[410/600], Loss: 0.0552\n",
      "Epoch [1/5], Step[411/600], Loss: 0.0530\n",
      "Epoch [1/5], Step[512/600], Loss: 0.0371\n",
      "Epoch [1/5], Step[513/600], Loss: 0.0273\n",
      "Epoch [1/5], Step[514/600], Loss: 0.0427\n",
      "Epoch [1/5], Step[515/600], Loss: 0.0510\n",
      "Epoch [1/5], Step[520/600], Loss: 0.0423\n",
      "Epoch [1/5], Step[521/600], Loss: 0.0098\n",
      "Epoch [1/5], Step[522/600], Loss: 0.0454\n",
      "Epoch [1/5], Step[523/600], Loss: 0.0378\n",
      "Epoch [1/5], Step[528/600], Loss: 0.0534\n",
      "Epoch [1/5], Step[529/600], Loss: 0.1308\n",
      "Epoch [1/5], Step[530/600], Loss: 0.0389\n",
      "Epoch [1/5], Step[531/600], Loss: 0.0375\n",
      "Epoch [1/5], Step[536/600], Loss: 0.0539\n",
      "Epoch [1/5], Step[537/600], Loss: 0.0870\n",
      "Epoch [1/5], Step[538/600], Loss: 0.0444\n",
      "Epoch [1/5], Step[539/600], Loss: 0.0144\n",
      "Epoch [2/5], Step[1/600], Loss: 0.1345\n",
      "Epoch [2/5], Step[2/600], Loss: 0.1259\n",
      "Epoch [2/5], Step[3/600], Loss: 0.0329\n",
      "Epoch [2/5], Step[8/600], Loss: 0.0653\n",
      "Epoch [2/5], Step[9/600], Loss: 0.0308\n",
      "Epoch [2/5], Step[10/600], Loss: 0.0378\n",
      "Epoch [2/5], Step[11/600], Loss: 0.0493\n",
      "Epoch [2/5], Step[16/600], Loss: 0.1019\n",
      "Epoch [2/5], Step[17/600], Loss: 0.0834\n",
      "Epoch [2/5], Step[18/600], Loss: 0.0103\n",
      "Epoch [2/5], Step[19/600], Loss: 0.0963\n",
      "Epoch [2/5], Step[24/600], Loss: 0.0314\n",
      "Epoch [2/5], Step[25/600], Loss: 0.0304\n",
      "Epoch [2/5], Step[26/600], Loss: 0.0737\n",
      "Epoch [2/5], Step[27/600], Loss: 0.0337\n",
      "Epoch [2/5], Step[128/600], Loss: 0.0157\n",
      "Epoch [2/5], Step[129/600], Loss: 0.0345\n",
      "Epoch [2/5], Step[130/600], Loss: 0.0390\n",
      "Epoch [2/5], Step[131/600], Loss: 0.0328\n",
      "Epoch [2/5], Step[136/600], Loss: 0.0401\n",
      "Epoch [2/5], Step[137/600], Loss: 0.0749\n",
      "Epoch [2/5], Step[138/600], Loss: 0.0254\n",
      "Epoch [2/5], Step[139/600], Loss: 0.0194\n",
      "Epoch [2/5], Step[144/600], Loss: 0.0575\n",
      "Epoch [2/5], Step[145/600], Loss: 0.1244\n",
      "Epoch [2/5], Step[146/600], Loss: 0.0719\n",
      "Epoch [2/5], Step[147/600], Loss: 0.2173\n",
      "Epoch [2/5], Step[152/600], Loss: 0.0419\n",
      "Epoch [2/5], Step[153/600], Loss: 0.0076\n",
      "Epoch [2/5], Step[154/600], Loss: 0.0256\n",
      "Epoch [2/5], Step[155/600], Loss: 0.0540\n",
      "Epoch [2/5], Step[256/600], Loss: 0.0578\n",
      "Epoch [2/5], Step[257/600], Loss: 0.0705\n",
      "Epoch [2/5], Step[258/600], Loss: 0.0657\n",
      "Epoch [2/5], Step[259/600], Loss: 0.0462\n",
      "Epoch [2/5], Step[264/600], Loss: 0.0519\n",
      "Epoch [2/5], Step[265/600], Loss: 0.0296\n",
      "Epoch [2/5], Step[266/600], Loss: 0.0405\n",
      "Epoch [2/5], Step[267/600], Loss: 0.0738\n",
      "Epoch [2/5], Step[272/600], Loss: 0.0106\n",
      "Epoch [2/5], Step[273/600], Loss: 0.0147\n",
      "Epoch [2/5], Step[274/600], Loss: 0.0693\n",
      "Epoch [2/5], Step[275/600], Loss: 0.1410\n",
      "Epoch [2/5], Step[280/600], Loss: 0.0135\n",
      "Epoch [2/5], Step[281/600], Loss: 0.0027\n",
      "Epoch [2/5], Step[282/600], Loss: 0.0105\n",
      "Epoch [2/5], Step[283/600], Loss: 0.0218\n",
      "Epoch [2/5], Step[384/600], Loss: 0.1432\n",
      "Epoch [2/5], Step[385/600], Loss: 0.0150\n",
      "Epoch [2/5], Step[386/600], Loss: 0.1172\n",
      "Epoch [2/5], Step[387/600], Loss: 0.0214\n",
      "Epoch [2/5], Step[392/600], Loss: 0.0736\n",
      "Epoch [2/5], Step[393/600], Loss: 0.0790\n",
      "Epoch [2/5], Step[394/600], Loss: 0.0813\n",
      "Epoch [2/5], Step[395/600], Loss: 0.0265\n",
      "Epoch [2/5], Step[400/600], Loss: 0.0591\n",
      "Epoch [2/5], Step[401/600], Loss: 0.0764\n",
      "Epoch [2/5], Step[402/600], Loss: 0.1025\n",
      "Epoch [2/5], Step[403/600], Loss: 0.0495\n",
      "Epoch [2/5], Step[408/600], Loss: 0.0133\n",
      "Epoch [2/5], Step[409/600], Loss: 0.0040\n",
      "Epoch [2/5], Step[410/600], Loss: 0.0600\n",
      "Epoch [2/5], Step[411/600], Loss: 0.0564\n",
      "Epoch [2/5], Step[512/600], Loss: 0.0248\n",
      "Epoch [2/5], Step[513/600], Loss: 0.1097\n",
      "Epoch [2/5], Step[514/600], Loss: 0.0120\n",
      "Epoch [2/5], Step[515/600], Loss: 0.0348\n",
      "Epoch [2/5], Step[520/600], Loss: 0.0252\n",
      "Epoch [2/5], Step[521/600], Loss: 0.0141\n",
      "Epoch [2/5], Step[522/600], Loss: 0.0725\n",
      "Epoch [2/5], Step[523/600], Loss: 0.0303\n",
      "Epoch [2/5], Step[528/600], Loss: 0.0250\n",
      "Epoch [2/5], Step[529/600], Loss: 0.0133\n",
      "Epoch [2/5], Step[530/600], Loss: 0.0673\n",
      "Epoch [2/5], Step[531/600], Loss: 0.1008\n",
      "Epoch [2/5], Step[536/600], Loss: 0.0387\n",
      "Epoch [2/5], Step[537/600], Loss: 0.0968\n",
      "Epoch [2/5], Step[538/600], Loss: 0.0580\n",
      "Epoch [2/5], Step[539/600], Loss: 0.0297\n",
      "Epoch [3/5], Step[1/600], Loss: 0.0949\n",
      "Epoch [3/5], Step[2/600], Loss: 0.0547\n",
      "Epoch [3/5], Step[3/600], Loss: 0.0498\n",
      "Epoch [3/5], Step[8/600], Loss: 0.0582\n",
      "Epoch [3/5], Step[9/600], Loss: 0.0140\n",
      "Epoch [3/5], Step[10/600], Loss: 0.0189\n",
      "Epoch [3/5], Step[11/600], Loss: 0.0566\n",
      "Epoch [3/5], Step[16/600], Loss: 0.0660\n",
      "Epoch [3/5], Step[17/600], Loss: 0.0192\n",
      "Epoch [3/5], Step[18/600], Loss: 0.0157\n",
      "Epoch [3/5], Step[19/600], Loss: 0.0537\n",
      "Epoch [3/5], Step[24/600], Loss: 0.0737\n",
      "Epoch [3/5], Step[25/600], Loss: 0.0106\n",
      "Epoch [3/5], Step[26/600], Loss: 0.0207\n",
      "Epoch [3/5], Step[27/600], Loss: 0.0169\n",
      "Epoch [3/5], Step[128/600], Loss: 0.0203\n",
      "Epoch [3/5], Step[129/600], Loss: 0.0098\n",
      "Epoch [3/5], Step[130/600], Loss: 0.0613\n",
      "Epoch [3/5], Step[131/600], Loss: 0.0507\n",
      "Epoch [3/5], Step[136/600], Loss: 0.1746\n",
      "Epoch [3/5], Step[137/600], Loss: 0.0173\n",
      "Epoch [3/5], Step[138/600], Loss: 0.0164\n",
      "Epoch [3/5], Step[139/600], Loss: 0.0727\n",
      "Epoch [3/5], Step[144/600], Loss: 0.0210\n",
      "Epoch [3/5], Step[145/600], Loss: 0.0289\n",
      "Epoch [3/5], Step[146/600], Loss: 0.0423\n",
      "Epoch [3/5], Step[147/600], Loss: 0.1073\n",
      "Epoch [3/5], Step[152/600], Loss: 0.0219\n",
      "Epoch [3/5], Step[153/600], Loss: 0.0165\n",
      "Epoch [3/5], Step[154/600], Loss: 0.0172\n",
      "Epoch [3/5], Step[155/600], Loss: 0.0520\n",
      "Epoch [3/5], Step[256/600], Loss: 0.0422\n",
      "Epoch [3/5], Step[257/600], Loss: 0.0447\n",
      "Epoch [3/5], Step[258/600], Loss: 0.0892\n",
      "Epoch [3/5], Step[259/600], Loss: 0.0150\n",
      "Epoch [3/5], Step[264/600], Loss: 0.0835\n",
      "Epoch [3/5], Step[265/600], Loss: 0.0137\n",
      "Epoch [3/5], Step[266/600], Loss: 0.1238\n",
      "Epoch [3/5], Step[267/600], Loss: 0.0161\n",
      "Epoch [3/5], Step[272/600], Loss: 0.0355\n",
      "Epoch [3/5], Step[273/600], Loss: 0.1550\n",
      "Epoch [3/5], Step[274/600], Loss: 0.0348\n",
      "Epoch [3/5], Step[275/600], Loss: 0.0518\n",
      "Epoch [3/5], Step[280/600], Loss: 0.0117\n",
      "Epoch [3/5], Step[281/600], Loss: 0.0162\n",
      "Epoch [3/5], Step[282/600], Loss: 0.0951\n",
      "Epoch [3/5], Step[283/600], Loss: 0.0658\n",
      "Epoch [3/5], Step[384/600], Loss: 0.1450\n",
      "Epoch [3/5], Step[385/600], Loss: 0.1073\n",
      "Epoch [3/5], Step[386/600], Loss: 0.0034\n",
      "Epoch [3/5], Step[387/600], Loss: 0.1261\n",
      "Epoch [3/5], Step[392/600], Loss: 0.0082\n",
      "Epoch [3/5], Step[393/600], Loss: 0.0144\n",
      "Epoch [3/5], Step[394/600], Loss: 0.0348\n",
      "Epoch [3/5], Step[395/600], Loss: 0.0034\n",
      "Epoch [3/5], Step[400/600], Loss: 0.0244\n",
      "Epoch [3/5], Step[401/600], Loss: 0.0274\n",
      "Epoch [3/5], Step[402/600], Loss: 0.0612\n",
      "Epoch [3/5], Step[403/600], Loss: 0.0084\n",
      "Epoch [3/5], Step[408/600], Loss: 0.0263\n",
      "Epoch [3/5], Step[409/600], Loss: 0.1690\n",
      "Epoch [3/5], Step[410/600], Loss: 0.0497\n",
      "Epoch [3/5], Step[411/600], Loss: 0.0342\n",
      "Epoch [3/5], Step[512/600], Loss: 0.0184\n",
      "Epoch [3/5], Step[513/600], Loss: 0.0869\n",
      "Epoch [3/5], Step[514/600], Loss: 0.0501\n",
      "Epoch [3/5], Step[515/600], Loss: 0.0334\n",
      "Epoch [3/5], Step[520/600], Loss: 0.0065\n",
      "Epoch [3/5], Step[521/600], Loss: 0.0680\n",
      "Epoch [3/5], Step[522/600], Loss: 0.0803\n",
      "Epoch [3/5], Step[523/600], Loss: 0.1726\n",
      "Epoch [3/5], Step[528/600], Loss: 0.0060\n",
      "Epoch [3/5], Step[529/600], Loss: 0.0189\n",
      "Epoch [3/5], Step[530/600], Loss: 0.1532\n",
      "Epoch [3/5], Step[531/600], Loss: 0.0268\n",
      "Epoch [3/5], Step[536/600], Loss: 0.0517\n",
      "Epoch [3/5], Step[537/600], Loss: 0.0222\n",
      "Epoch [3/5], Step[538/600], Loss: 0.0674\n",
      "Epoch [3/5], Step[539/600], Loss: 0.0449\n",
      "Epoch [4/5], Step[1/600], Loss: 0.0334\n",
      "Epoch [4/5], Step[2/600], Loss: 0.0218\n",
      "Epoch [4/5], Step[3/600], Loss: 0.0115\n",
      "Epoch [4/5], Step[8/600], Loss: 0.0506\n",
      "Epoch [4/5], Step[9/600], Loss: 0.0566\n",
      "Epoch [4/5], Step[10/600], Loss: 0.0143\n",
      "Epoch [4/5], Step[11/600], Loss: 0.0344\n",
      "Epoch [4/5], Step[16/600], Loss: 0.0552\n",
      "Epoch [4/5], Step[17/600], Loss: 0.0691\n",
      "Epoch [4/5], Step[18/600], Loss: 0.0226\n",
      "Epoch [4/5], Step[19/600], Loss: 0.0551\n",
      "Epoch [4/5], Step[24/600], Loss: 0.0192\n",
      "Epoch [4/5], Step[25/600], Loss: 0.0141\n",
      "Epoch [4/5], Step[26/600], Loss: 0.0177\n",
      "Epoch [4/5], Step[27/600], Loss: 0.0807\n",
      "Epoch [4/5], Step[128/600], Loss: 0.0812\n",
      "Epoch [4/5], Step[129/600], Loss: 0.0597\n",
      "Epoch [4/5], Step[130/600], Loss: 0.0027\n",
      "Epoch [4/5], Step[131/600], Loss: 0.0721\n",
      "Epoch [4/5], Step[136/600], Loss: 0.0038\n",
      "Epoch [4/5], Step[137/600], Loss: 0.0641\n",
      "Epoch [4/5], Step[138/600], Loss: 0.0209\n",
      "Epoch [4/5], Step[139/600], Loss: 0.0039\n",
      "Epoch [4/5], Step[144/600], Loss: 0.0399\n",
      "Epoch [4/5], Step[145/600], Loss: 0.0050\n",
      "Epoch [4/5], Step[146/600], Loss: 0.0550\n",
      "Epoch [4/5], Step[147/600], Loss: 0.0069\n",
      "Epoch [4/5], Step[152/600], Loss: 0.0056\n",
      "Epoch [4/5], Step[153/600], Loss: 0.0557\n",
      "Epoch [4/5], Step[154/600], Loss: 0.1290\n",
      "Epoch [4/5], Step[155/600], Loss: 0.0248\n",
      "Epoch [4/5], Step[256/600], Loss: 0.0065\n",
      "Epoch [4/5], Step[257/600], Loss: 0.0109\n",
      "Epoch [4/5], Step[258/600], Loss: 0.0144\n",
      "Epoch [4/5], Step[259/600], Loss: 0.0839\n",
      "Epoch [4/5], Step[264/600], Loss: 0.0247\n",
      "Epoch [4/5], Step[265/600], Loss: 0.0234\n",
      "Epoch [4/5], Step[266/600], Loss: 0.0582\n",
      "Epoch [4/5], Step[267/600], Loss: 0.0041\n",
      "Epoch [4/5], Step[272/600], Loss: 0.1262\n",
      "Epoch [4/5], Step[273/600], Loss: 0.0061\n",
      "Epoch [4/5], Step[274/600], Loss: 0.0117\n",
      "Epoch [4/5], Step[275/600], Loss: 0.0508\n",
      "Epoch [4/5], Step[280/600], Loss: 0.0058\n",
      "Epoch [4/5], Step[281/600], Loss: 0.1394\n",
      "Epoch [4/5], Step[282/600], Loss: 0.1032\n",
      "Epoch [4/5], Step[283/600], Loss: 0.0093\n",
      "Epoch [4/5], Step[384/600], Loss: 0.0099\n",
      "Epoch [4/5], Step[385/600], Loss: 0.0244\n",
      "Epoch [4/5], Step[386/600], Loss: 0.0634\n",
      "Epoch [4/5], Step[387/600], Loss: 0.0861\n",
      "Epoch [4/5], Step[392/600], Loss: 0.0415\n",
      "Epoch [4/5], Step[393/600], Loss: 0.0244\n",
      "Epoch [4/5], Step[394/600], Loss: 0.0073\n",
      "Epoch [4/5], Step[395/600], Loss: 0.0727\n",
      "Epoch [4/5], Step[400/600], Loss: 0.0344\n",
      "Epoch [4/5], Step[401/600], Loss: 0.0526\n",
      "Epoch [4/5], Step[402/600], Loss: 0.0469\n",
      "Epoch [4/5], Step[403/600], Loss: 0.0225\n",
      "Epoch [4/5], Step[408/600], Loss: 0.0051\n",
      "Epoch [4/5], Step[409/600], Loss: 0.0374\n",
      "Epoch [4/5], Step[410/600], Loss: 0.1101\n",
      "Epoch [4/5], Step[411/600], Loss: 0.0204\n",
      "Epoch [4/5], Step[512/600], Loss: 0.0102\n",
      "Epoch [4/5], Step[513/600], Loss: 0.0353\n",
      "Epoch [4/5], Step[514/600], Loss: 0.0593\n",
      "Epoch [4/5], Step[515/600], Loss: 0.0230\n",
      "Epoch [4/5], Step[520/600], Loss: 0.1317\n",
      "Epoch [4/5], Step[521/600], Loss: 0.0152\n",
      "Epoch [4/5], Step[522/600], Loss: 0.1528\n",
      "Epoch [4/5], Step[523/600], Loss: 0.0796\n",
      "Epoch [4/5], Step[528/600], Loss: 0.0619\n",
      "Epoch [4/5], Step[529/600], Loss: 0.0280\n",
      "Epoch [4/5], Step[530/600], Loss: 0.0453\n",
      "Epoch [4/5], Step[531/600], Loss: 0.0276\n",
      "Epoch [4/5], Step[536/600], Loss: 0.0162\n",
      "Epoch [4/5], Step[537/600], Loss: 0.0576\n",
      "Epoch [4/5], Step[538/600], Loss: 0.0131\n",
      "Epoch [4/5], Step[539/600], Loss: 0.0536\n",
      "Epoch [5/5], Step[1/600], Loss: 0.0300\n",
      "Epoch [5/5], Step[2/600], Loss: 0.0062\n",
      "Epoch [5/5], Step[3/600], Loss: 0.0746\n",
      "Epoch [5/5], Step[8/600], Loss: 0.0050\n",
      "Epoch [5/5], Step[9/600], Loss: 0.0073\n",
      "Epoch [5/5], Step[10/600], Loss: 0.0374\n",
      "Epoch [5/5], Step[11/600], Loss: 0.0194\n",
      "Epoch [5/5], Step[16/600], Loss: 0.0032\n",
      "Epoch [5/5], Step[17/600], Loss: 0.0114\n",
      "Epoch [5/5], Step[18/600], Loss: 0.0017\n",
      "Epoch [5/5], Step[19/600], Loss: 0.0296\n",
      "Epoch [5/5], Step[24/600], Loss: 0.0107\n",
      "Epoch [5/5], Step[25/600], Loss: 0.1005\n",
      "Epoch [5/5], Step[26/600], Loss: 0.0535\n",
      "Epoch [5/5], Step[27/600], Loss: 0.0150\n",
      "Epoch [5/5], Step[128/600], Loss: 0.0950\n",
      "Epoch [5/5], Step[129/600], Loss: 0.0121\n",
      "Epoch [5/5], Step[130/600], Loss: 0.0068\n",
      "Epoch [5/5], Step[131/600], Loss: 0.0046\n",
      "Epoch [5/5], Step[136/600], Loss: 0.0561\n",
      "Epoch [5/5], Step[137/600], Loss: 0.0932\n",
      "Epoch [5/5], Step[138/600], Loss: 0.0313\n",
      "Epoch [5/5], Step[139/600], Loss: 0.0569\n",
      "Epoch [5/5], Step[144/600], Loss: 0.0050\n",
      "Epoch [5/5], Step[145/600], Loss: 0.0543\n",
      "Epoch [5/5], Step[146/600], Loss: 0.1123\n",
      "Epoch [5/5], Step[147/600], Loss: 0.0038\n",
      "Epoch [5/5], Step[152/600], Loss: 0.0188\n",
      "Epoch [5/5], Step[153/600], Loss: 0.0004\n",
      "Epoch [5/5], Step[154/600], Loss: 0.0089\n",
      "Epoch [5/5], Step[155/600], Loss: 0.0101\n",
      "Epoch [5/5], Step[256/600], Loss: 0.0143\n",
      "Epoch [5/5], Step[257/600], Loss: 0.0368\n",
      "Epoch [5/5], Step[258/600], Loss: 0.1298\n",
      "Epoch [5/5], Step[259/600], Loss: 0.0296\n",
      "Epoch [5/5], Step[264/600], Loss: 0.0167\n",
      "Epoch [5/5], Step[265/600], Loss: 0.0628\n",
      "Epoch [5/5], Step[266/600], Loss: 0.0583\n",
      "Epoch [5/5], Step[267/600], Loss: 0.0081\n",
      "Epoch [5/5], Step[272/600], Loss: 0.1128\n",
      "Epoch [5/5], Step[273/600], Loss: 0.0334\n",
      "Epoch [5/5], Step[274/600], Loss: 0.0126\n",
      "Epoch [5/5], Step[275/600], Loss: 0.0020\n",
      "Epoch [5/5], Step[280/600], Loss: 0.0065\n",
      "Epoch [5/5], Step[281/600], Loss: 0.0387\n",
      "Epoch [5/5], Step[282/600], Loss: 0.0177\n",
      "Epoch [5/5], Step[283/600], Loss: 0.0286\n",
      "Epoch [5/5], Step[384/600], Loss: 0.0409\n",
      "Epoch [5/5], Step[385/600], Loss: 0.0167\n",
      "Epoch [5/5], Step[386/600], Loss: 0.0175\n",
      "Epoch [5/5], Step[387/600], Loss: 0.0345\n",
      "Epoch [5/5], Step[392/600], Loss: 0.0106\n",
      "Epoch [5/5], Step[393/600], Loss: 0.0108\n",
      "Epoch [5/5], Step[394/600], Loss: 0.0212\n",
      "Epoch [5/5], Step[395/600], Loss: 0.0150\n",
      "Epoch [5/5], Step[400/600], Loss: 0.0364\n",
      "Epoch [5/5], Step[401/600], Loss: 0.0001\n",
      "Epoch [5/5], Step[402/600], Loss: 0.0165\n",
      "Epoch [5/5], Step[403/600], Loss: 0.0735\n",
      "Epoch [5/5], Step[408/600], Loss: 0.0078\n",
      "Epoch [5/5], Step[409/600], Loss: 0.0587\n",
      "Epoch [5/5], Step[410/600], Loss: 0.0046\n",
      "Epoch [5/5], Step[411/600], Loss: 0.0513\n",
      "Epoch [5/5], Step[512/600], Loss: 0.0328\n",
      "Epoch [5/5], Step[513/600], Loss: 0.0730\n",
      "Epoch [5/5], Step[514/600], Loss: 0.0896\n",
      "Epoch [5/5], Step[515/600], Loss: 0.0244\n",
      "Epoch [5/5], Step[520/600], Loss: 0.1046\n",
      "Epoch [5/5], Step[521/600], Loss: 0.0129\n",
      "Epoch [5/5], Step[522/600], Loss: 0.0958\n",
      "Epoch [5/5], Step[523/600], Loss: 0.1078\n",
      "Epoch [5/5], Step[528/600], Loss: 0.0319\n",
      "Epoch [5/5], Step[529/600], Loss: 0.0606\n",
      "Epoch [5/5], Step[530/600], Loss: 0.0835\n",
      "Epoch [5/5], Step[531/600], Loss: 0.0390\n",
      "Epoch [5/5], Step[536/600], Loss: 0.0489\n",
      "Epoch [5/5], Step[537/600], Loss: 0.0168\n",
      "Epoch [5/5], Step[538/600], Loss: 0.0122\n",
      "Epoch [5/5], Step[539/600], Loss: 0.0445\n",
      "Training took 690.5935745239258 seconds\n"
     ]
    }
   ],
   "source": [
    "start_time = time.time()\n",
    "\n",
    "num_epochs = 5\n",
    "def train(num_epochs, cnn, loaders):\n",
    "    cnn.train()\n",
    "    total_step = len(loaders['train'])\n",
    "    for epoch in range(num_epochs):\n",
    "        for i, (images, labels) in enumerate(loaders['train']):\n",
    "            # gives batch data, normalizes x when iterating train_loader\n",
    "            b_x = Variable(images)\n",
    "            b_y = Variable(labels)\n",
    "            \n",
    "            output = cnn(b_x)[0]\n",
    "            loss = loss_func(output, b_y)\n",
    "\n",
    "            # clear gradients for this trainign step\n",
    "            optimizer.zero_grad()\n",
    "            # backpropogation, computing gradients\n",
    "            loss.backward()\n",
    "            # apply gradients\n",
    "            optimizer.step()\n",
    "            if (i+1) & 100 == 0:\n",
    "                print('Epoch [{}/{}], Step[{}/{}], Loss: {:.4f}'\n",
    "                      .format(epoch + 1, num_epochs, i+1, total_step, loss.item()))\n",
    "                pass\n",
    "            pass\n",
    "        pass\n",
    "train(num_epochs, cnn, loaders)\n",
    "\n",
    "curr_time = time.time()\n",
    "print(f\"Training took {curr_time - start_time} seconds\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ccb6166e",
   "metadata": {},
   "source": [
    "## Testing the Model\n",
    "\n",
    "Let's define a testing function for model evaluation:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "05c7199d-5500-4673-b8c0-488514d84edf",
   "metadata": {},
   "outputs": [],
   "source": [
    "def test():\n",
    "    cnn.eval()  # set to evaluation mode\n",
    "    with torch.no_grad():  # don't update gradients during testing\n",
    "        correct = 0\n",
    "        total = 0\n",
    "        for images, labels in loaders['test']:  # for each item in the test set, test it\n",
    "            test_output, last_layer = cnn(images)\n",
    "            pred_y = torch.max(test_output, 1)[1].data.squeeze()\n",
    "            accuracy = (pred_y == labels).sum().item() / float(labels.size(0))\n",
    "    print('Test accuracy of model on 10000 test images: %.2f' % accuracy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "ecf1dbc9-82e7-4ffc-b6e0-a4a4e452b727",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test accuracy of model on 10000 test images: 1.00\n"
     ]
    }
   ],
   "source": [
    "test()  # this takes a minute"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "658188de-b88a-4877-9fed-dcf1a705a6d3",
   "metadata": {},
   "source": [
    "Let's grab a few samples from our test set, plot them, and then manually check our accuracy:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "a1f7e3f7-6d92-4a70-8b45-359b8e06edcb",
   "metadata": {},
   "outputs": [],
   "source": [
    "sample = next(iter(loaders['test']))\n",
    "imgs, lbls = sample\n",
    "actual_number = lbls[:10].numpy()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b78692b8-1f68-401e-bde2-f5bca7ea0d2d",
   "metadata": {},
   "source": [
    "Plotting the samples:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "ccfeb183-5de2-4082-ac2d-1526ca66d219",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAxsAAAFVCAYAAACHE/L8AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA+WElEQVR4nO3dd3iVRfr/8TsQkpAEAek1Iiy9rRLBBRFESlhhRbo0wcYXRYRllSIJqKCAuGJhWRFFWNcCUgMqgihKUVYQAUGk924iRQLI/P7YH1mfcw/kEDI5OTnv13VxXc4n8zyZhHGSm3PmmTBjjBEAAAAAyGJ5Aj0AAAAAALkTxQYAAAAAJyg2AAAAADhBsQEAAADACYoNAAAAAE5QbAAAAABwgmIDAAAAgBMUGwAAAACcoNgAAAAA4ESOLzZWrlwpI0eOlJSUlEAPJUtNnz5dihUrJidPnkzPzp07J4mJiVKhQgWJiIiQuLg4GTp0qPz666+ea5cuXSqxsbGyf//+7B52yAmV+ffLL7/I6NGjpUmTJlKyZEmJjY2VWrVqydixY+Xs2bOea5l/2StU5qCvX3/9VSpXrixhYWHywgsveD7GHMw+oTT/0tLSZPz48VKzZk2JiYmREiVKSEJCgqxcudJzLfMv+zD/csn8Mznc+PHjjYiYnTt3BnooWeb06dOmTJkyZvz48Z78nnvuMVFRUWbMmDHm008/NU8//bSJiIgwbdq0Ufdo2rSp6dmzZ3YNOWSFyvzbsGGDKVq0qBk4cKCZN2+eWbp0qRk5cqSJiooyzZo1MxcvXvTcg/mXfUJlDvr661//akqXLm1ExNqPOZg9Qmn+9ejRw+TJk8cMHz7cLF261MycOdPcfPPNJjw83Hz99deevsy/7MH8yx3zj2Ijm5w7d86cP3/eGGPMpEmTTFRUlPn555/TP75q1SojImbChAme68aMGWNExCxevNiTz5o1y+TNm9fs2bPH+dhDWajMv1OnTplTp06p6y59/V9++aUnZ/5ln1CZg7/39ddfm4iICDNz5szLFhvMwewRKvPv7NmzJm/evKZ79+6e6w4cOGBExDz22GOenPmXPZh/uWP+5ehiIykpyYiI+rNs2TJjjDHvvfeeadCggYmOjjYxMTGmRYsWZu3atZ579OrVy8TExJiffvrJJCQkmJiYGFO2bFkzaNAgc/bsWU/fSZMmmdq1a5uYmBgTGxtrqlSpYoYOHerps2HDBtO2bVtTqFAhExkZaerUqWOmTZvm6bNs2TIjImb69Olm0KBBpnTp0iYsLMxs3rzZGGNMrVq1TMeOHT3XvPDCC0ZE0vtc8v333xsRMQ8//LAnT0tLMwULFjQjRoy4um8q/BZK8+9yvvjiCyMi5t///rcnZ/5lj1Ccg2lpaaZGjRpm4MCBZufOnZctNpiD7oXS/Dt37pzJly+feeSRRzz5qVOnTJ48ecwTTzzhyZl/7jH/cs/8y9HFxt69e03//v2NiJjZs2ebVatWmVWrVpnU1FQzevRoExYWZvr06WOSk5PN7Nmzza233mpiYmLMpk2b0u/Rq1cvExERYapVq2ZeeOEFs2TJEpOYmGjCwsLMqFGj0vu9++67RkRM//79zeLFi82SJUvM5MmTPdXkli1bTIECBUzFihXN9OnTzcKFC03Xrl2NiJixY8em97s00cqUKWM6dOhg5s+fb5KTk83x48fN3r17jYiYSZMmeb7WS69g7Nixw5P/+OOPRkTMrbfeqr4/CQkJ5qabbrrm7zPsQmn+Xc6lxX79+vXqY8w/90JxDg4fPtzccMMN5tSpU1csNoxhDroWavNvwIABJjY21syZM8ekpqaanTt3mq5du5rChQubn376SfVn/rnF/Ms98y9HFxvG2F9C27NnjwkPDzf9+/f39D158qQpWbKk6dSpU3rWq1cvIyLmgw8+8PRt3bq1qVKlSnr70UcfNYUKFbriWLp06WIiIyPVy1YJCQkmOjrapKSkGGP+N9EaN26s7vH+++8bETGrV6/25HPnzjUiYmbMmOHJp06dakTEVK5cWd1r+PDhJk+ePNa3vyBrhMr8s1m/fr3Jnz+/adeunfXjzL/sEUpzcN26dSZfvnzm448/NsaYDIsN5qB7oTT/Ll68aBITE02ePHnS/xW9fPnyZt26ddbxMP/cY/7ljvmX459GZfPJJ5/IhQsXpGfPnnLhwoX0P1FRUXL77bfL559/7ukfFhYmbdq08WS1a9eW3bt3p7dvueUWSUlJka5du8q8efPk2LFj6vN+9tln0qxZMylXrpwnv+++++TMmTOyatUqT96+fXt1jwMHDoiISPHixT15QkKCVKpUSZ588kn59NNPJSUlRT7++GMZNmyY5M2bV/Lk0X9VxYsXl4sXL8qhQ4cs3yW4khvnn69du3bJXXfdJeXKlZM33njD2of5Fzi5cQ5euHBB+vTpI507d5aWLVtm/E0Q5mCg5Mb5JyIyevRoeeGFF2TkyJGybNkymTdvnlSpUkWaN28u69atU/2Zf4HB/JP0ewTL/AsP9AAy4/DhwyIiEh8fb/247y/m0dHREhUV5ckiIyM9j/Ts0aOHXLhwQaZMmSLt27eXixcvSnx8vDz77LPSvHlzERE5fvy4lCpVSn2+0qVLp3/892x9Lz3G1nc8ERER8tFHH0mPHj2kRYsWIiISExMjY8aMkWeeeUbKlCmj7nXpHr6PxoVbuXH+/d7u3buladOmEh4eLkuXLpXrr7/e2o/5Fzi5cQ6+9NJLsmPHDvnggw/SH3P5yy+/iIjI2bNnJSUlRQoUKCB58+ZNv4Y5GBi5cf5t3rxZEhMTZdy4cTJ48OD0PCEhQapXry6DBg2SZcuWea5h/gUG80889wiG+ReUxUbRokVFRGTWrFkSFxeXZfft3bu39O7dW06fPi3Lly+XpKQkueuuu2Tr1q0SFxcnRYoUkYMHD6rrLlWql8Z1SVhY2GXHfuLECTURK1WqJKtWrZL9+/fLiRMnpGLFipKamioDBgyQxo0bq3udOHHC+nnhVm6dfyL/LTSaNGkixhj5/PPPpWzZspcdL/MvcHLjHNy4caOkpqbKH/7wB3XNiBEjZMSIEbJu3TqpW7dues4cDIzcOP/Wr18vxhj1C2y+fPmkTp068sUXX6h7Mf8Cg/kn6fewfd6cKMcXG5GRkSLirdxatmwp4eHhsn37duvLVNcqJiZGEhIS5Ny5c3L33XfLpk2bJC4uTpo1ayZz5syRAwcOpFeyIv89nCU6OloaNGiQ4b2rVq0qIiLbt2+XGjVqWPuUKVMm/ZWMp556SmJiYuT+++9X/Xbs2CFFihSREiVKZObLhB9Caf7t2bNHmjRpIr/99pt8/vnnGS7izL/sESpzcMiQIXLfffd5+h46dEi6du0qffv2lc6dO0ulSpU8H2cOuhcq8+/S/VavXi233357ep6WliZr1661/sML88895l/umH85vtioVauWiIhMnDhRevXqJfny5ZMqVarI008/LcOHD5cdO3ZIq1atpHDhwnL48GH55ptvJCYmRkaNGnVVn+fBBx+U/PnzS8OGDaVUqVJy6NAhee6556RgwYLplWZSUpIkJydL06ZNJTExUa6//np55513ZOHChTJu3DgpWLBghp+nfv36kj9/flm9erW0bdvW87Fx48ZJyZIlpXz58nL48GH54IMPZO7cuTJjxgzr26guTUpb9YysESrz78iRI9K0aVM5ePCgTJ06VY4cOSJHjhxJ/3jZsmXVYsf8yx6hMgerVq2a/oP4kl27domISMWKFaVJkybqXsxB90Jl/jVq1Eji4+Nl5MiRcubMGWncuLGkpqbKK6+8Ijt37pQZM2aoezH/3GP+5ZL5F+AN6n4ZOnSoKV26dPoO/UvPWJ47d65p2rSpue6660xkZKSJi4szHTp0MEuWLEm/9tIzln1deqTnJW+//bZp2rSpKVGihImIiDClS5c2nTp1Mt9//73nug0bNpg2bdqYggULmoiICFOnTh3z1ltvefpcehLBzJkzrV9Pjx49TPXq1VU+atQoU7FiRRMZGWkKFSpkWrVqZZYvX269x7Zt24yImA8//ND6cWSdUJh/l6653J+kpCRPf+Zf9gqFOWhzpadRMQezT6jMv5SUFDN8+HBTrVo1Ex0dbYoXL26aNGliFi1apPoy/7IP8y/4519QFBu5zZo1a/x+/OjlPPXUU6Z8+fLpJ1IC/mL+IdCYgwgk5h8CKRTnX5gxxjh/+QRK586d5fTp05KcnHzV16akpMiNN94or7zyinTr1s3B6JDbMf8QaMxBBBLzD4EUavMvKM/ZyA0mTJgg8fHxcvLkyau+dufOnTJ06FC59957HYwMoYD5h0BjDiKQmH8IpFCbf7yyAQAAAMAJXtkAAAAA4ATFBgAAAAAnKDYAAAAAOOH3oX5BcWgIsl12bflh/sEmO7ecMQdhwxqIQGL+IZD8nX+8sgEAAADACYoNAAAAAE5QbAAAAABwgmIDAAAAgBMUGwAAAACcoNgAAAAA4ATFBgAAAAAnKDYAAAAAOEGxAQAAAMAJig0AAAAATlBsAAAAAHCCYgMAAACAExQbAAAAAJyg2AAAAADgBMUGAAAAACcoNgAAAAA4QbEBAAAAwAmKDQAAAABOUGwAAAAAcIJiAwAAAIATFBsAAAAAnKDYAAAAAOAExQYAAAAAJ8IDPQAA/xUZGamywoULq6xv374qi4mJUZkxxtOOj49XfU6dOqWy1q1bq2z58uWe9po1a1Sff//73yr77rvvVAb36tatq7I2bdqobNKkSZ728ePHXQ0JCGoFChTwtCtWrKj6vPTSSyqLiIhQ2Z/+9KcsGxcQDHhlAwAAAIATFBsAAAAAnKDYAAAAAOAExQYAAAAAJ8KM7y7Sy3UMC3M9FgQhP6fPNQum+VejRg2VhYfrZzEkJCR42nfccYfqY8tsbN+frPy78b2/7d4HDx5UmW0j5N69e7NsXNk1/0RyxhwcOHCgyhITE1Vme2BA3rx5Vfbee+952t26dbuG0YUm1sDcJzY2VmUzZ870tBs1aqT62P6/e+ihh1T2xhtvXMPovJh//ilWrJjKunfv7mnffffdqs+xY8dUZutn+/7885//9LRta/XRo0dVFkz8nX+8sgEAAADACYoNAAAAAE5QbAAAAABwImQO9bvppptU9pe//MXTLl26tOpTuXJllTVu3DjLxrVv3z6VPfrooyqbN29eln1OZE6RIkVUZjvEqX379iqzHdjnz3sdbfsgNm7cqDLb+0WPHDniac+YMSPDz3c50dHRnvarr76q+pQqVUplBQsWVFlW7tnITerXr6+ywYMHe9r33HOPX/fatGmTypYsWaKyF1980c/R5Tz58+dX2cMPP6yyL7/8UmXffvutkzHh2tjWyUGDBqnMdkClzfbt2z1t2/v2+/Xrp7LXXntNZS1btvS0bev3Rx99pLI333wzw3HCvaFDh6pswIABnrbt7/Ra9kO2a9fO077hhhtUn6JFi/p1rx49enjaW7Zs8eu6nIJXNgAAAAA4QbEBAAAAwAmKDQAAAABOUGwAAAAAcCLoD/Wzjcu2wXrChAkq8z1o7fz586rPuXPnVLZy5UqVbd68WWUlS5b0tG2b2mybHHfv3q2y5s2be9rbtm1TfQIhlA4UatGihcoWLVrk17W28Z84ccLT7tOnj+rz3XffqSwQG6x95+knn3yi+lSoUEFlHOrnvxEjRqhs5MiRGV731ltvqezxxx9X2alTpzIzrBzD9wEEGzZsUH0KFy6ssjNnzqgsOTlZZV27ds3UuEJpDbwWtodF3HnnnZ72yy+/rPrYHjyRE0ycOFFltkM3XWP+afXq1VPZwoULVeb7wADb99J2EJ/tQEffBwjY7ufvZnN/+tnGNXr0aJW5xqF+AAAAAAKKYgMAAACAExQbAAAAAJyg2AAAAADgRNCfIG47fdG2cctm/fr1nnavXr1Un++//z5zA7OIiIhQ2fLly1V2yy23qGzcuHGetr8nCSPrfPHFFyrz3bgvItKqVSuVNWnSRGX79+/3tG0bXnPKaduzZs3ytG0bvz/99FOV5ZTxB4M//vGPGfaxnQxu21ge7JvBCxQooLI33njD07ZtBreJjo5WWaVKlTI3MPjF9rP0oYceUtmtt96aZZ/T9tCUIkWKeNr+zhkb3wcx/O1vf8v0veDWAw88oDLfuSCiNzfPnj1b9bFturb93tm9e3eV3X333Z72bbfdluEYLse3X5UqVfy6LqfglQ0AAAAATlBsAAAAAHCCYgMAAACAExQbAAAAAJwI+g3i1+KJJ57wtLNyM7jNhQsXVObvBnEEXlpamsqWLVvmV5ZT2U6wf/LJJ1XWunVrT/vcuXOqz5IlS7JuYCHIdzOhiN4UaPseHzx40NWQsoXtZOl//etfKvN98ILt/8f7779fZatWrVJZSkrKVYwQV2I75f6xxx5TWaFChTK8l+3v1PbgCd8HpoiIfPfddyr77LPPPG3bydI2mzdvVllSUpKnbft5jpzh6NGjKrOdyu2b+fvgnWPHjqnspZdeUtnatWs97c8//9yv+9vG6vs558yZ49e9cgpe2QAAAADgBMUGAAAAACcoNgAAAAA4QbEBAAAAwImg3yB+9uxZle3evVtlcXFxKitfvryTMV3OgAEDVDZ48OBsHQPwe3379lXZU089pbKLFy962rYNmhMmTMi6gYUg26ZA3w3ithPEg13z5s1V5vtAAhvbKenvvvtulowJ/mvRooXK/NkMLiKyYMECT3vUqFGqj+8mWxH7qfC2n6/+bAj/8ccfVXbnnXeqLNgfxBBKnnvuOZVVq1ZNZe3atfO0bad5v/jiiyobM2aMymybxocOHZrh/W3Z66+/rrIpU6Z42rb/L3IyXtkAAAAA4ATFBgAAAAAnKDYAAAAAOBH0ezZOnz6tsv/85z8qs+3ZeOWVVzztu+66S/XZsGGDymyHtn311Vcq8907UqxYMdXH5tChQyqzHZwEXA3b+0z79evn17W++zimTZuWFUPC79gOEqtSpYqnvX79+uwajhO2A/xs77X3x2uvvXatw0EWWL16tcoaNGigMtshZB07dvS0ffeGiYgUKFBAZTNnzlSZbe+Irz179qisS5cuKmN/RnA7c+aMymx7Ef05xO/xxx9X2bfffquyqlWrqsx3TtrG1bNnT5UF24F9/uCVDQAAAABOUGwAAAAAcIJiAwAAAIATFBsAAAAAnAgzthNFbB0tB07lVLaN2Fu2bFFZ4cKFM3X/nTt3qmzNmjUq69SpU4b3Onz4sMpatWqlspy6MdTP6XPNgmn+BUL+/PlVNnXqVE/bdlBabGysymbNmqUy2ybKnCC75p+I+zk4YsQIlfk+GOLIkSOqT5s2bVRme0hGTlC7dm2VrVu3zq9rjx496mnXqlUrwz7ZIdTXwIYNG6rsyy+/VFlaWprK/u///s/T/uCDD1Qf28++ihUr+jU234e03HbbbarPvn37/LpXThXq8+9a+P6su/vuu1Uffw5b9bef7wMRRIJ/M7i/849XNgAAAAA4QbEBAAAAwAmKDQAAAABOUGwAAAAAcCJXbhC3KVWqlMr82fR6yy23qKxz586ZGoPtZHDbqacbN27M1P0Dgc1p2a9evXoqGzVqlMpatmyZ4b2GDBmishdffFFltpN9c4LctEG8XLlyKlu8eLGnXblyZdVnwYIFKrP9vdoekpHdXn75ZZU98sgjfl371Vdfedq33357lozpWoX6Gpg3b16VLVmyRGW2v6/z58972j/++KPqU7NmzUyPzfchLbaHXwS7UJ9/18L3YTwLFy5UffzdIL527VqVDRw40NP2XcNyAzaIAwAAAAgoig0AAAAATlBsAAAAAHCCYgMAAACAEyGzQTyzSpQoobLJkyer7C9/+UuG9+rbt6/KXn/99cwNLIdgc1rWiYyMVNmf/vQnlc2fP19ltpPAz50752nbHkbwxRdfXM0Qc5zctEHcpnz58p72xx9/rPpUqVJFZQcOHFCZ72nNIiLbtm3ztF1vIp87d67KbCeg206b9j1917aZMxBYA/3zxBNPqOz555/P8Drb13327FmV9ejRQ2W5cUO4L+Zf5vn+/GvYsKHq4+8G8X79+qks2H+/8wcbxAEAAAAEFMUGAAAAACcoNgAAAAA4QbEBAAAAwInwQA8gpzt8+LDKKlasmKl7BftmXGStm2++2dN+7rnnVJ877rjDr3v99NNPKvM9vZT5F3z27NnjaSckJKg+ixYtUlnVqlVVNm/ePJX5rm/r16/3a1y2jY979+5VWdmyZT3tBg0a+HX/48ePq+zrr7/261rkTAUKFMjUdbYNqEeOHFFZKGwGh398f7aK2NfJYsWKedq2uebvxvjHH39cZf/617887TNnzvh1r9yIVzYAAAAAOEGxAQAAAMAJig0AAAAATrBnIwO2A6dq1qzp17WHDh3ytFNSUrJiSAhCdevWVZnvAWelSpXy6167d+9Wme3Avl27dvl1PwQP29/9qFGjVDZ+/HiV+e6fENGHltrmkY2//TKraNGiKmvUqJGnbTsgEDlD3rx5VWZbA/1hO+CxePHiKhs5cqRfGXK/Bx98UGVFihRRme8ejdmzZ6s+K1asUNmQIUNUZjtctV27dp72O++8owcbInhlAwAAAIATFBsAAAAAnKDYAAAAAOAExQYAAAAAJ9ggnoHq1atn+tqpU6d62rYDApH7PP/88yrr3bu3ynw3rB08eFD1efXVV1U2ffp0ldmuRWj44IMPVPbNN9+orF69eip77LHHPG3bxmyb6OholZUrV86va/3x4osvqowN4cHDdkDpn//85wyvW7Vqlcp8N9mKiCxYsEBliYmJKouMjPS0X3jhBdXHdoAkgodtfjz00EMqsx3Yt3jxYk+7Y8eOfn3Obt26qcz20ALfh1qwQRwAAAAAshjFBgAAAAAnKDYAAAAAOEGxAQAAAMAJNoj/TtWqVVVm23Rmc+HCBZXZNqMheN1www0qmz9/vsps88h2ou6RI0c8bdtp9d99953/AwT+P9vp8bZs1qxZmbp/yZIlVdasWTOV/e1vf/O0a9Wq5df9f/jhh0yNCznDdddd51c/35+bSUlJqo/vOili32xuu/bJJ5/0tG2nPPfo0UNlp0+f1oNFjjRs2DCV2TaD29YU2999Ztk+Z7Vq1bLs/sGOVzYAAAAAOEGxAQAAAMAJig0AAAAATlBsAAAAAHCCDeK/88ADD6gsf/78fl27b98+laWmpl7zmBA4Xbt29bRHjx6t+sTFxfl1r7fffltlffr0ydS4YmJiVFaiRAmVdejQwdOeOHGi6pOWlpapMSC0HTp0SGW203FjY2M97UmTJjkbE3KO2bNnq8z289V3/bH9HLU5duyYyvr3768y31PtbQ8x6NSpk8reeustv8aBwPvyyy9VdvPNN6ssTx79b+u2eeSPKVOmqGzy5Mkqu+222zztxo0bqz7Lly/P1BiCDa9sAAAAAHCCYgMAAACAExQbAAAAAJxgz8bv3HTTTZm+1vYeVQS3Vq1aedrly5dXfWwH+djeO/yPf/xDZb77gSpXrqz6VK9eXWWDBg1SmW3unjlzxtO2vTd09erVKgOyypw5czztvn37qj61a9dWWXR0tLMxwb3Fixer7MMPP1RZx44dPe2lS5eqPr179/br/jYPP/ywp71x40bVx/fgPxH2bAQT2+9eAwYMUJntQEffA3i3bNmS6XHYfhfwzWwH/rJnAwAAAACuAcUGAAAAACcoNgAAAAA4QbEBAAAAwImQ3iDevXt3T7tRo0Z+XWc7rI/DqnCJ76ZHEZG7775bZb4HntWvX9+v++/evVtlc+fOVdn48eM9bTaDI7sdOXLE0966davqY9sgPm7cOJX5HtD27LPPqj7nzp272iEimyQlJaksPj7e077hhhtUn0WLFqns3XffVdmpU6dUVqZMGU+7SJEiqk9UVJTKEDxsB/MdP35cZba/e9+Denv06KH6+D5oRUQf1ne5z+m7Qfz1119XfUIFr2wAAAAAcIJiAwAAAIATFBsAAAAAnKDYAAAAAOBEmLEde2jrGBbmeizZ7pNPPvG0mzdv7td1a9euVVm9evWyZEzBxs/pc80CMf+GDBniads2pPrLNv7Mfu8eeOABlU2bNi1T9wp22TX/RHLnGpjdatasqbI1a9aoLDxcP7skTx7vv43NnDlT9enSpcs1jC5zcvMa6FrJkiU9bduDAbp166Yyf78Xvv1sf1efffaZyu68806/7p8TMP+0li1bqmzhwoUq8/2afvjhB9Xnu+++U5ntgS+2jeRjxozxtCdOnKj6BDt/5x+vbAAAAABwgmIDAAAAgBMUGwAAAACcoNgAAAAA4AQbxH/H3w3ito3CiYmJWTKmYJObN6fly5fP0/7jH/+o+rRv396ve/melCuiN8Z++OGHqs+uXbtUZjsx9eLFi36NI7dhg3juZNuAGRcX52nnlM2WuXkNzAk6duzoV9ahQweVrVixwtMeO3as6vPll1+qLDU19WqGGFDMP61YsWIqGzp0qMoGDhzoadt+jvo+mOJy/Wzzb86cOVccZ27ABnEAAAAAAUWxAQAAAMAJig0AAAAATlBsAAAAAHAiZDaI+55UKiKyZcsWT/u6665Tfb755huVNW7cWGXnzp27htEFLzanIZDYII5AYw1EIDH/Ms/3pHHbgykeeughlY0ePVplPCToynhlAwAAAIATFBsAAAAAnKDYAAAAAOBEyOzZsPHnUL9hw4ap7Pnnn3c2pmDD+0URSOzZQKCxBiKQmH8IJPZsAAAAAAgoig0AAAAATlBsAAAAAHCCYgMAAACAEyG9QRzXjs1pCCQ2iCPQWAMRSMw/BBIbxAEAAAAEFMUGAAAAACcoNgAAAAA4QbEBAAAAwAm/N4gDAAAAwNXglQ0AAAAATlBsAAAAAHCCYgMAAACAExQbAAAAAJyg2AAAAADgBMUGAAAAACcoNgAAAAA4QbEBAAAAwAmKDQAAAABOUGwAAAAAcIJiAwAAAIATFBsAAAAAnKDYAAAAAOAExQYAAAAAJ3J8sbFy5UoZOXKkpKSkBHooWWr69OlSrFgxOXnypIiI7Nq1S8LCwi77p1WrVunXLl26VGJjY2X//v2BGj6AbBIqa6CISFpamowfP15q1qwpMTExUqJECUlISJCVK1d6rmUNBEID618uWf9MDjd+/HgjImbnzp2BHkqWOX36tClTpowZP358enb27FmzatUq9efJJ580ImImT57suUfTpk1Nz549s3voIWfFihUmKSnJ/Pzzz4EeSpZ6++23TdGiRc0vv/ySng0bNszUrVvXFC5c2ERGRpoKFSqYBx980Ozatctz7ZIlS0xMTIzZt29fdg87JIXKGmiMMT169DB58uQxw4cPN0uXLjUzZ840N998swkPDzdff/21py9rYPYIpTVwwYIFpkePHqZmzZomPDzcXO5XJNbA7MP6lzvWP4qNbHLu3Dlz/vx5Y4wxkyZNMlFRUX4t3k2aNDHR0dEmNTXVk8+aNcvkzZvX7Nmzx8Vw8f/llvn3e5db6Pr162fGjh1r5s+fb5YtW2Zee+01U6pUKVOiRAlz7NgxT99gW+iCWW6ZgxmtgWfPnjV58+Y13bt391x34MABIyLmscce8+Ssgdkjt8y/37vcGtinTx/zhz/8wXTq1MncfPPNly02jGENzC65Zf6F+vqXo4uNpKQkIyLqz7Jly4wxxrz33numQYMGJjo62sTExJgWLVqYtWvXeu7Rq1cvExMTY3766SeTkJBgYmJiTNmyZc2gQYPM2bNnPX0nTZpkateubWJiYkxsbKypUqWKGTp0qKfPhg0bTNu2bU2hQoVMZGSkqVOnjpk2bZqnz7Jly4yImOnTp5tBgwaZ0qVLm7CwMLN582ZjjDG1atUyHTt2zPDr37ZtmwkLCzP33Xef+lhaWpopWLCgGTFiRIb3QeaFykJ3OYsWLTIiYqZOnerJg22hC1ahtAaeO3fO5MuXzzzyyCOe/NSpUyZPnjzmiSee8OSsgdkjlNbA3377Lf2/H3nkkSsWG6yB7rH+5Z71L0cXG3v37jX9+/c3ImJmz56d/tai1NRUM3r0aBMWFmb69OljkpOTzezZs82tt95qYmJizKZNm9Lv0atXLxMREWGqVatmXnjhBbNkyRKTmJhowsLCzKhRo9L7vfvuu0ZETP/+/c3ixYvNkiVLzOTJkz3V5JYtW0yBAgVMxYoVzfTp083ChQtN165djYiYsWPHpve7NNHKlCljOnToYObPn2+Sk5PN8ePHzd69e42ImEmTJmX49Q8bNsyIiPnqq6+sH09ISDA33XRTZr618EMoLXSXs2bNGiMi5u233/bkwbbQBatQWwMHDBhgYmNjzZw5c0xqaqrZuXOn6dq1qylcuLD56aefVH/WQLdCeQ3MqNhgDXSP9S/3rH85utgwxv6vKnv27DHh4eGmf//+nr4nT540JUuWNJ06dUrPevXqZUTEfPDBB56+rVu3NlWqVElvP/roo6ZQoUJXHEuXLl1MZGSk+peMhIQEEx0dbVJSUowx/5tojRs3Vvd4//33jYiY1atXX/FzXbhwwZQpU8ZUrVr1sn2GDx9u8uTJY06dOnXFeyFzQm2hu+T8+fPmzJkzZu3ataZhw4amcuXK5uTJk6pfMC10wSyU1sCLFy+axMREkydPnvRfbMuXL2/WrVtnHQ9roFuhugYak3GxYQxrYHZg/csd619QFhtTpkwxImLWrFljzp8/7/nTuXNnU7x48fS+vXr1MmFhYebXX3/13HfIkCEmKioqvT19+nQjIqZLly5m7ty55ujRo2osxYsXN61bt1b5pcnz0UcfGWP+N9EmTpyo+v797383ImJ27Nhxxa87OTnZiIh6T+nvTZw40YiI2bZt2xXvhcwLpYXOGGMOHjzo+RfM+vXrm/3791v7BtNCF8xCaQ185plnTHR0tHn66afNsmXLzLx580zz5s1N0aJF1b+YG8MamB1CbQ28xJ9igzXQPda/3LH+5fhH39ocPnxYRETi4+MlX758nj/vv/++HDt2zNM/OjpaoqKiPFlkZKScPXs2vd2jRw958803Zffu3dK+fXspXry41K9fXz799NP0PsePH5dSpUqp8ZQuXTr9479n6/vrr7+KiKjx+Jo6darky5dPevbsedk+l+5x6Z7IHp988olcuHBBevbsKRcuXEj/ExUVJbfffrt8/vnnnv5hYWHSpk0bT1a7dm3ZvXt3evuWW26RlJQU6dq1q8ybN0/NYRGRzz77TJo1ayblypXz5Pfdd5+cOXNGVq1a5cnbt2+v7nHgwAERESlevLj1aytatKisWbNGvvrqK5kyZYqcOHFCmjZtKgcPHlR9ixcvLhcvXpRDhw5Z7wV3cuMauHnzZklMTJRRo0bJiBEjpEmTJtK2bVtZuHChFCpUSAYNGqTuxRoYGLl5DbwarIGBwfonnnsEw/oXHugBZEbRokVFRGTWrFkSFxeXZfft3bu39O7dW06fPi3Lly+XpKQkueuuu2Tr1q0SFxcnRYoUsf7SdWnxujSuS8LCwi479hMnTlgnoojIkSNHJDk5Wdq2bXvFBfHEiRPWzwu3fr/Q2eTJ463h/V3oLly4IFOmTJH27dvLxYsXJT4+Xp599llp3ry5iGRPsRseHi716tUTEZGGDRtKq1atpEKFCvL888/LxIkTPX2DaaHLbXLjGrh+/Xoxxqj/r/Llyyd16tSRL774Qt2LNTAwcvMaeDVYAwOD9U/S72H7vDlRji82IiMjRcT7P3PLli0lPDxctm/fbv2Xi2sVExMjCQkJcu7cObn77rtl06ZNEhcXJ82aNZM5c+bIgQMH0hc3kf8ezhIdHS0NGjTI8N5Vq1YVEZHt27dLjRo1rH2mT58u58+fl/vvv/+K99qxY4cUKVJESpQocRVfHa5VblzoLqds2bJSunRp2bp1q/pYMC10wSxU1sBL91u9erXcfvvt6XlaWpqsXbtWypYtq+7FGhgYobQGXglroHusf7lj/cvxxUatWrVERGTixInSq1cvyZcvn1SpUkWefvppGT58uOzYsUNatWolhQsXlsOHD8s333wjMTExMmrUqKv6PA8++KDkz59fGjZsKKVKlZJDhw7Jc889JwULFkyvNJOSkiQ5OVmaNm0qiYmJcv3118s777wjCxculHHjxknBggUz/Dz169eX/Pnzy+rVq6Vt27bWPlOnTpVy5cpJy5Ytr3ivS5PStqAia4TKQnc527Ztk3379lnnajAtdMEsVNbARo0aSXx8vIwcOVLOnDkjjRs3ltTUVHnllVdk586dMmPGDHUv1kD3Qn0NvBLWQPdY/3LJ+hfoTSP+GDp0qCldunT6Dv1Lj92bO3euadq0qbnuuutMZGSkiYuLMx06dDBLlixJv/bSY/d8XXqk3yVvv/22adq0qSlRooSJiIgwpUuXNp06dTLff/+957oNGzaYNm3amIIFC5qIiAhTp04d89Zbb3n6XNocNHPmTOvX06NHD1O9enXrx1asWGFExCQmJl7xe7Jt2zYjIubDDz+8Yj9cm0t/lw8//LBZuXKlWbNmjfnll1/MmDFjTHh4uHn44YfNnDlzzOeff27ef/9989e//tXzd+fv/HvggQdM//79zXvvvWe++OIL8/7775u6deuaggULmiNHjhhj/vcklsqVK5t//etfZtGiRaZbt25GRMy4cePUmG3zLy0tzeTPn189TnL9+vXmjjvuMJMmTTIff/yxWbx4sZkwYYIpW7asKVasmDpF3Jj/Pj7ynnvuufpvKq5aqKyBKSkpZvjw4aZatWomOjraFC9e3DRp0sQsWrRI9WUNzB6hsgYaY8yuXbvMzJkzzcyZM02rVq3S7zFz5kyzZs0a1Z81MHuw/gX/+hcUxUZuc+nsgoyehnElTz31lClfvnz6IUVwJxQWukOHDpnu3bubihUrmujoaBMREWFuvPFG07dvX+uhVcG20CFnYQ0MLqGwBhpjzFtvvWU9V0RETK9evTx9WQORWaG4/oUZY4zDF05wGZ07d5bTp09LcnLyVV+bkpIiN954o7zyyivSrVs3B6NDbvaf//xH4uPjZfXq1VK/fv1M3WPEiBEyffp02b59u4SH5/h3YyIHYg1EoLAGItBCbf0Lykff5gYTJkyQ+Ph4OXny5FVfu3PnThk6dKjce++9DkaG3K5evXrSqVMneeaZZzJ1fUpKirz22msyZswYfsgi01gDESisgQi0UFv/eGUDCEH79u2TqVOnyqBBg6RAgQJXde26detkyZIlMnjw4ODYmAYAPlgDgexDsQEAAADACd5GBQAAAMAJig0AAAAATvi9s4n3JcImu96Fx/yDTXa+C5Q5CBvWQAQS8w+B5O/845UNAAAAAE5QbAAAAABwgmIDAAAAgBMUGwAAAACcoNgAAAAA4ATFBgAAAAAnKDYAAAAAOEGxAQAAAMAJig0AAAAATlBsAAAAAHCCYgMAAACAExQbAAAAAJyg2AAAAADgBMUGAAAAACcoNgAAAAA4QbEBAAAAwAmKDQAAAABOUGwAAAAAcCI80AMAAABwrU2bNiqbP3++ylq3bq2yjz76yMmYkDvUqFFDZW+++aanXa9ePb/utXHjRpXVqVMncwPLIXhlAwAAAIATFBsAAAAAnKDYAAAAAOAExQYAAAAAJ9ggDgC4Kvfff79fme+GyJkzZ6o+/fr1U1lqauo1jA74r8jISE978ODBqs/FixdVVqBAAWdjQs5l2+TdvHlzlbVt21Zlt99+u8r27t3radvWv2LFivl1r0qVKnna27ZtU31yMl7ZAAAAAOAExQYAAAAAJyg2AAAAADhBsQEAAADACTaIAznYddddp7KYmBiVDRs2TGXffvutp/3rr7+qPrGxsSo7duyYylauXOlpHz16VA8WuVL//v1V9tJLL6ksLCxMZVu3bvW077rrLtWnXLlyKrOd9MymcVytgQMHetqNGjVSfWybdmfNmuVsTMg5oqKiPG3bKfGlS5f2616nTp1SWe/evT3tZcuWqT4REREqa9iwocr279/v1zhyKl7ZAAAAAOAExQYAAAAAJyg2AAAAADgRZowxfnW0vB83s4YOHaoy23t5k5OTM3X/d999V2W7du3K1L1wZX5On2uWlfMvs2yH7/geWiYiUrx4cZU1bdpUZddff72nbTs8yPZ127J8+fKpLCstWLDA0/7LX/7i9PP5K7vmn0jOmIOuVa5cWWXffPONymx7iWyH87355puedsuWLVWfefPmqey5555T2fDhw1Xmj7Jly6qsevXqKvN9n7TtvdoPPvigykJpDQw2J06c8LRt66Tv++pFgmvPBvPPP7a9jtOmTfO027Vr59e93nrrLZU9//zzKtu+fbt/g8siVatWVdlDDz2kshEjRqjs9OnTmfqc/s4/XtkAAAAA4ATFBgAAAAAnKDYAAAAAOEGxAQAAAMCJHHOoX/369f3K/GE74Oy3337L1L1sbBulRo4cqbJz585l2efs1q2bp23btPn4449n2efLzerUqaOyP/7xjyrzPQDKNh9r1KiRdQPLwXwPWbv//vtVn6lTp2bXcODIzTffrDLbZvA33nhDZVOmTFGZ77q7aNEi1eezzz5TWXh4xj+abBu/ExISVPbkk0+qLC4uTmUzZszwtG2HvSHn6tu3r8p8Dy1dvXq16hNMm8GRebZ1wHdDuO3g23vvvVdlixcvVllaWto1jC5rdO7cWWWPPfaYymzr9w8//OBkTJfwygYAAAAAJyg2AAAAADhBsQEAAADACYoNAAAAAE4EZIP4hg0bVGbb8HzLLbdk6v758+fP1HX+sm0QHz9+vNPP6cu2kfPPf/6zymwnYm7cuNHJmILFP/7xD5U1aNAgACPxOnLkiMp27typssOHD6vsxRdfVJnv/2d9+vRRfTI7b/3ZwIvgs3fvXr/6xcfHqywiIkJlvhsubQ/qePXVV1XWtm1blfluYLRtEI+KilLZ6NGjVTZp0iSVHT16VGXImerWrauyl156SWV58+b1tG0nPyM0FC5cWGW+v8vZTtFesGCBszFdi4IFC6qsadOmKsspJ7/zygYAAAAAJyg2AAAAADhBsQEAAADACYoNAAAAAE4EZJdncnKyylasWKGy4sWLe9q2E7KLFSvm1+e0nf5cqlQpv67NiXw3vomIVKhQQWW2TeOhvkE8sw8QSElJUdm0adNUZjuR9qeffsrw/rYTSM+cOaOyCxcuqMz3hHkRkddff93TvvHGGzMcw+UsX77c02ajZWirU6eOyjp06KAy3w3bAwcOVH3KlCmjsgIFCmQ4hrlz56rMdor9woULM7wXgovtpPh8+fKpbMeOHZ72smXLnI0JwccYc8V2TuK7IXzEiBGqT6NGjVRme/DML7/8knUD8xOvbAAAAABwgmIDAAAAgBMUGwAAAACcoNgAAAAA4ESY8XNHTE45hTCzKlWqpLJChQpl2f2TkpJU5nui7q233qr6REdHZ9kYbBuH77vvPpW99957WfY5s2tDVVbOP9spm7ZThn1PtX/uuedUH9tp3lnJthGyffv2KuvZs6fKMnvK94wZM1Q2YMAAT9u2WT4QsnNDX7Cvgf6IjIxU2bp161RWtWpVldlO342JicnUOGbPnq2yZ5991tPevHmz6mN7yIJrwbgGBhPbwwJsD9ywPSymRo0anvaWLVuybmA5BPPPP+XLl1eZ7wMDbHPI9vvCt99+m2Xjsp0E3qtXL5X5/o5pu+748eMqa926tcqycvz+zj9e2QAAAADgBMUGAAAAACcoNgAAAAA4ETJ7NgKhd+/envbEiRNVn8zu2Th27JjKOnXqpDLfw9iyGu8Xzbxy5cp52rZD0Wx7ga677jpnYxIR+fnnn1Xm+x5S28GQu3btcjWky2LPxrXxnYN33nmn6mPbz1SyZEm/7n/27FlP+8MPP1R9/v73v6ts06ZNKgvEfgx/sAa6NXjwYJWNHTtWZRs2bFBZgwYNPG3f+ZgbMP8yb8iQIZ62774wEZGjR4+qLD4+XmX79u1Tme/B1La9GI8++qjKbAed+n7/bYcHjxo1SmU//PCDyrISezYAAAAABBTFBgAAAAAnKDYAAAAAOEGxAQAAAMCJzJ36BcV2+NrLL7/saefPnz/T9/c9WOupp55SfVxvBkfW8j0saMKECZm+17lz51Tme+jZV199pfrYNt3aHlqQmJjoaf/hD39QfXw324mI/POf/9SDRUDYNieOGzfO07YdanUtfA/N6tGjR5beH7lf27Zt/eq3ePFileXGDeHIOpMmTfK0W7ZsqfrcdtttKvP93U5EZOnSpSrr3r27p23bWG7z7rvvqsz3UOHdu3erPraDVXMKXtkAAAAA4ATFBgAAAAAnKDYAAAAAOEGxAQAAAMAJThDPIr4bIUXsG4v8sWPHDpU1a9bM0967d2+m7p3VOL0086Kiojztf/zjHxn2EbFvJLedKJ+VJ3oXLFjQ0+7atavqY9sg3rdvX5V9/PHHWTYuThAXiY2NVdk777yjsj//+c8qy5Mn439v8n3QgIjIihUrVPbAAw+ozHeDbpcuXVSf+fPnZziGnIw1MOs0b95cZcnJySpLTU1Vme9p4SL2n6W5DfMv6/ie+C0icvDgQb+uta2lKSkpnvawYcNUH9vP/WDCCeIAAAAAAopiAwAAAIATFBsAAAAAnKDYAAAAAOAEJ4hnQps2bVRm25zmjx9//FFlSUlJKsspG8KRdXw3z/bu3TtAI8mY74bMyZMnqz7lypVT2fjx41WWlRvEIfLqq6+qzLZG+WP27NkqGzRokMpsDySIi4tTme+G38aNG6s+wb5BHFknISFBZeHh+teUOXPmqCwUNoMj82666SaVDRw40NO+4YYbVB9/N0BfvHhRZf369fO0bSeDhwpe2QAAAADgBMUGAAAAACcoNgAAAAA4QbEBAAAAwAk2iGfC3LlzVZbZUzzffPNNlc2aNStT9wKyS4ECBVRWqlQplRUqVCgbRhM66tatq7Lu3btn+n6vv/66p92/f3/V5/z5837d68svv1SZ7wZx2ynmgwcP9uv+yH1iY2M97WbNmvl1XbFixVwMBzmc7ZTu2rVrq8y2Ebt8+fIqi4yM9LT37dun+jz99NMqsz3MxXb/Fi1aZDiuUMErGwAAAACcoNgAAAAA4ATFBgAAAAAn2LPxO77v3xMR+eijj1Rme9+g7UAXX74HyIiIvPzyy36ODggc3zn/0EMPqT733Xefyr7//ntXQwpJQ4YMUZltPbKZMWOGyh599FFP+8KFC5kbmIgkJyerzPf9zoULF1Z9bO+/P3r0aKbHgeDx1FNPedo1a9ZUfb7++muVPfDAA87GhJyjePHinrbt587IkSP9ute2bdtUNmnSJE/7n//8p+qTlpamsnfeeUdlW7duVVn+/Pn9Glso4JUNAAAAAE5QbAAAAABwgmIDAAAAgBMUGwAAAACcYIP474wePVpljRs3VpltM7jtUL/Nmzd72u+///41jA7ZrUuXLp52u3btVB/bIWhHjhxxNqbsUL9+fZX5buS0Hc5mc/LkySwZE65dUlKSyjK7ITxfvnwqu+uuuzK87uzZsyo7c+ZMpsaA4BIREaGyVq1aZXjdp59+qrITJ05kyZiQc9h+1xo7dqynHR8fr/ocOHBAZV27dlWZ7+9jIpmfR9u3b1fZxo0bM3WvUMErGwAAAACcoNgAAAAA4ATFBgAAAAAnKDYAAAAAOBHSG8Sjo6M97Vq1amX6XrbNR76baA8fPpzp+yP7FSpUyNPu2LGj6rNs2TKVTZ482dWQrsr111/vabds2VL1adCggcp69uypsoIFC2b4+X799VeVLViwIMPr4L+KFStm+trSpUur7JdffvG0b7rpJtWnUqVKKmvYsKHKunXrluEYFi1apLLTp09neB2CX4UKFVR2LT9zkbvYfr7ecsstnvbPP/+s+thOnfdd17LD8uXLVdavXz9P2/ZAmaNHjzobU07CKxsAAAAAnKDYAAAAAOAExQYAAAAAJyg2AAAAADgR0hvEGzVq5Gk3a9Ys0/f67LPPVLZnz55M3w+BN3PmTE/7nnvuUX0ef/xxlUVFRals//79Klu6dKmn7bshXUSkWrVqKqtRo4bKbCfx1qtXz9OOjY1VfTIrNTVVZbYNfkuWLMmyzwn9dyoiYozx69qvvvoqq4eToSNHjnjab7zxRraPATnDjTfemKnrNm3alMUjQbDwXdtsPyNXrFihsjZt2qhs165dWTUsK9uDNPbu3etpp6WlOR1DTsYrGwAAAACcoNgAAAAA4ATFBgAAAAAnQnrPxr333hvoISAHO378uKdtO7jxzjvvVNmLL77obExZ7fz58yqzvUfa92v6+OOPVZ9jx45l3cBg9dtvv6ksT57s/zcj2wGltjkxceJET/u7775zNSTkcL57JG1s8yM5OdnBaJDTjB8/XmXXXXedp92iRQvVx7YX6D//+Y/KbAfwzps3z9O2rWG2AwJth5raxua77zMQhw3mFLyyAQAAAMAJig0AAAAATlBsAAAAAHCCYgMAAACAEyG9QTwrvfnmm4EeAhx75plnVNa4cWOV1alTx6/7hYWFedr+Hs5mY7t2/fr1nrbvZjURvUFOROSHH37I9DjgVvv27VXWoUMHlZUrV05lJUuWVFmFChU87bVr16o+f//731Vm67dt2zaVAZf07Nkzwz5bt25V2ZkzZ1wMBzmM7RDkXr16ZXhdgwYNVDZ48GCV2Q7lbdeunadtO3Tv66+/Vpnt576N7TDfUMUrGwAAAACcoNgAAAAA4ATFBgAAAAAnKDYAAAAAOMEG8UyYPXu2ymynLiN3sZ2Q3bx5c5W1adNGZXfccUeWjePkyZMq+/DDD1W2dOnSLPucyBlsG/ptGZDTDBo0SGWJiYme9vfff59dw0EusXr1apXZHppRvXp1lfnOyXr16qk+ts3gGzduVNmYMWNUtnDhQpWFKl7ZAAAAAOAExQYAAAAAJyg2AAAAADhBsQEAAADAiTDj57HFvqcd5wbTpk3ztLt37+7XdWPHjlXZ8OHDs2JIQedaTr2+Grlx/uHaZdf8E2EOwo41EIHE/EMg+Tv/eGUDAAAAgBMUGwAAAACcoNgAAAAA4ATFBgAAAAAnKDYAAAAAOEGxAQAAAMAJig0AAAAATlBsAAAAAHAiPNADCKRZs2Z52hUrVlR9br31VpUtXrzY2ZgAAACA3IJXNgAAAAA4QbEBAAAAwAmKDQAAAABOUGwAAAAAcCLMGGP86hgW5nosCEJ+Tp9rxvyDTXbNPxHmIOxYAxFIzD8Ekr/zj1c2AAAAADhBsQEAAADACYoNAAAAAE5QbAAAAABwwu8N4gAAAABwNXhlAwAAAIATFBsAAAAAnKDYAAAAAOAExQYAAAAAJyg2AAAAADhBsQEAAADACYoNAAAAAE5QbAAAAABwgmIDAAAAgBP/D4w/Yt8f1jjeAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 1000x400 with 10 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "figure = plt.figure(figsize=(10, 4))\n",
    "cols, rows = 5, 2\n",
    "for i in range(1, cols * rows + 1):\n",
    "    img, label = imgs[i], lbls[i]\n",
    "    figure.add_subplot(rows, cols, i)\n",
    "    plt.title(label)\n",
    "    plt.axis(\"off\")\n",
    "    plt.imshow(img.squeeze(), cmap=\"gray\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "02e45c02-e6cf-4f63-8fd2-64a633b5a8e5",
   "metadata": {},
   "source": [
    "Checking our prediction abilities on our 10 samples:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "fb2b657e-c7c4-40e8-be58-d32b87baa0f0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Prediction number: [0 9 2 9 8 8 7 3 8 1]\n",
      "Actual number: [0 9 2 4 8 8 7 3 8 1]\n"
     ]
    }
   ],
   "source": [
    "test_output, last_layer = cnn(imgs[:10])\n",
    "pred_y = torch.max(test_output, 1)[1].data.numpy().squeeze()\n",
    "print(f'Prediction number: {pred_y}')\n",
    "print(f'Actual number: {actual_number}')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a703075e",
   "metadata": {},
   "source": [
    "## Saving The Trained Model\n",
    "\n",
    "Next, we can save the trained model's state using the lines below:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "48653cab-eb2b-4a17-85c2-57de0b4732b8",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saving model to: models/01_pytorch_mnist_cnn.pth\n"
     ]
    }
   ],
   "source": [
    "# 1. Create models directory \n",
    "MODEL_PATH = Path(\"models\")\n",
    "MODEL_PATH.mkdir(parents=True, exist_ok=True)\n",
    "\n",
    "# 2. Create model save path \n",
    "MODEL_NAME = \"01_pytorch_mnist_cnn.pth\"\n",
    "MODEL_SAVE_PATH = MODEL_PATH / MODEL_NAME\n",
    "\n",
    "# 3. Save the model state dict \n",
    "print(f\"Saving model to: {MODEL_SAVE_PATH}\")\n",
    "torch.save(obj=cnn.state_dict(), # only saving the state_dict() only saves the models learned parameters\n",
    "           f=MODEL_SAVE_PATH)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "a3b27366-1acb-4e2a-a4e3-03667a66ffab",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<All keys matched successfully>"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Instantiate a new instance of our model (this will be instantiated with random weights)\n",
    "loaded_model_0 = CNN()\n",
    "\n",
    "# Load the state_dict of our saved model (this will update the new instance of our model with trained weights)\n",
    "loaded_model_0.load_state_dict(torch.load(f=MODEL_SAVE_PATH))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "ad626a94-4f29-4e3a-a774-5ea528848fc9",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test accuracy of model on 10000 test images: 0.99\n"
     ]
    }
   ],
   "source": [
    "loaded_model_0.eval()\n",
    "with torch.no_grad():\n",
    "    correct = 0\n",
    "    total = 0\n",
    "    for images, labels in loaders['test']:\n",
    "        test_output, last_layer = loaded_model_0(images)\n",
    "        pred_y = torch.max(test_output, 1)[1].data.squeeze()\n",
    "        accuracy = (pred_y == labels).sum().item() / float(labels.size(0))\n",
    "        pass\n",
    "print('Test accuracy of model on 10000 test images: %.2f' % accuracy)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3-0.9.4",
   "language": "python",
   "name": "python3-0.9.4"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}