{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "UEBilEjLj5wY"
   },
   "source": [
    "Deep Learning Models -- A collection of various deep learning architectures, models, and tips for TensorFlow and PyTorch in Jupyter Notebooks.\n",
    "- Author: Sebastian Raschka\n",
    "- GitHub Repository: https://github.com/rasbt/deeplearning-models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 119
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 536,
     "status": "ok",
     "timestamp": 1524974472601,
     "user": {
      "displayName": "Sebastian Raschka",
      "photoUrl": "//lh6.googleusercontent.com/-cxK6yOSQ6uE/AAAAAAAAAAI/AAAAAAAAIfw/P9ar_CHsKOQ/s50-c-k-no/photo.jpg",
      "userId": "118404394130788869227"
     },
     "user_tz": 240
    },
    "id": "GOzuY8Yvj5wb",
    "outputId": "c19362ce-f87a-4cc2-84cc-8d7b4b9e6007"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Sebastian Raschka \n",
      "\n",
      "CPython 3.7.3\n",
      "IPython 7.6.1\n",
      "\n",
      "torch 1.1.0\n"
     ]
    }
   ],
   "source": [
    "%load_ext watermark\n",
    "%watermark -a 'Sebastian Raschka' -v -p torch"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "rH4XmErYj5wm"
   },
   "source": [
    "# Model Zoo -- AlexNet CIFAR-10 Classifier"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Network Architecture"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "References\n",
    "    \n",
    "- [1] Krizhevsky, Alex, Ilya Sutskever, and Geoffrey E. Hinton. \"[Imagenet classification with deep convolutional neural networks.](https://papers.nips.cc/paper/4824-imagenet-classification-with-deep-convolutional-neural-networks.pdf)\" In Advances in Neural Information Processing Systems, pp. 1097-1105. 2012.\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "MkoGLH_Tj5wn"
   },
   "source": [
    "## Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "id": "ORj09gnrj5wp"
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import time\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "from torch.utils.data import DataLoader\n",
    "from torch.utils.data.dataset import Subset\n",
    "\n",
    "from torchvision import datasets\n",
    "from torchvision import transforms\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "from PIL import Image\n",
    "\n",
    "\n",
    "if torch.cuda.is_available():\n",
    "    torch.backends.cudnn.deterministic = True"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "I6hghKPxj5w0"
   },
   "source": [
    "## Model Settings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 85
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 23936,
     "status": "ok",
     "timestamp": 1524974497505,
     "user": {
      "displayName": "Sebastian Raschka",
      "photoUrl": "//lh6.googleusercontent.com/-cxK6yOSQ6uE/AAAAAAAAAAI/AAAAAAAAIfw/P9ar_CHsKOQ/s50-c-k-no/photo.jpg",
      "userId": "118404394130788869227"
     },
     "user_tz": 240
    },
    "id": "NnT0sZIwj5wu",
    "outputId": "55aed925-d17e-4c6a-8c71-0d9b3bde5637"
   },
   "outputs": [],
   "source": [
    "##########################\n",
    "### SETTINGS\n",
    "##########################\n",
    "\n",
    "# Hyperparameters\n",
    "RANDOM_SEED = 1\n",
    "LEARNING_RATE = 0.0001\n",
    "BATCH_SIZE = 256\n",
    "NUM_EPOCHS = 40\n",
    "\n",
    "# Architecture\n",
    "NUM_CLASSES = 10\n",
    "\n",
    "# Other\n",
    "DEVICE = \"cuda:0\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Files already downloaded and verified\n"
     ]
    }
   ],
   "source": [
    "train_indices = torch.arange(0, 48000)\n",
    "valid_indices = torch.arange(48000, 50000)\n",
    "\n",
    "\n",
    "train_transform = transforms.Compose([transforms.Resize((70, 70)),\n",
    "                                      transforms.RandomCrop((64, 64)),\n",
    "                                      transforms.ToTensor()])\n",
    "\n",
    "test_transform = transforms.Compose([transforms.Resize((70, 70)),\n",
    "                                     transforms.CenterCrop((64, 64)),\n",
    "                                     transforms.ToTensor()])\n",
    "\n",
    "train_and_valid = datasets.CIFAR10(root='data', \n",
    "                                   train=True, \n",
    "                                   transform=train_transform,\n",
    "                                   download=True)\n",
    "\n",
    "train_dataset = Subset(train_and_valid, train_indices)\n",
    "valid_dataset = Subset(train_and_valid, valid_indices)\n",
    "test_dataset = datasets.CIFAR10(root='data', \n",
    "                                train=False, \n",
    "                                transform=test_transform,\n",
    "                                download=False)\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "train_loader = DataLoader(dataset=train_dataset, \n",
    "                          batch_size=BATCH_SIZE,\n",
    "                          num_workers=4,\n",
    "                          shuffle=True)\n",
    "\n",
    "valid_loader = DataLoader(dataset=valid_dataset, \n",
    "                          batch_size=BATCH_SIZE,\n",
    "                          num_workers=4,\n",
    "                          shuffle=False)\n",
    "\n",
    "test_loader = DataLoader(dataset=test_dataset, \n",
    "                         batch_size=BATCH_SIZE,\n",
    "                         num_workers=4,\n",
    "                         shuffle=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Set:\n",
      "\n",
      "Image batch dimensions: torch.Size([256, 3, 64, 64])\n",
      "Image label dimensions: torch.Size([256])\n",
      "\n",
      "Validation Set:\n",
      "Image batch dimensions: torch.Size([256, 3, 64, 64])\n",
      "Image label dimensions: torch.Size([256])\n",
      "\n",
      "Testing Set:\n",
      "Image batch dimensions: torch.Size([256, 3, 64, 64])\n",
      "Image label dimensions: torch.Size([256])\n"
     ]
    }
   ],
   "source": [
    "# Checking the dataset\n",
    "print('Training Set:\\n')\n",
    "for images, labels in train_loader:  \n",
    "    print('Image batch dimensions:', images.size())\n",
    "    print('Image label dimensions:', labels.size())\n",
    "    break\n",
    "    \n",
    "# Checking the dataset\n",
    "print('\\nValidation Set:')\n",
    "for images, labels in valid_loader:  \n",
    "    print('Image batch dimensions:', images.size())\n",
    "    print('Image label dimensions:', labels.size())\n",
    "    break\n",
    "\n",
    "# Checking the dataset\n",
    "print('\\nTesting Set:')\n",
    "for images, labels in train_loader:  \n",
    "    print('Image batch dimensions:', images.size())\n",
    "    print('Image label dimensions:', labels.size())\n",
    "    break"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "##########################\n",
    "### MODEL\n",
    "##########################\n",
    "\n",
    "class AlexNet(nn.Module):\n",
    "\n",
    "    def __init__(self, num_classes):\n",
    "        super(AlexNet, self).__init__()\n",
    "        self.features = nn.Sequential(\n",
    "            nn.Conv2d(3, 64, kernel_size=11, stride=4, padding=2),\n",
    "            nn.ReLU(inplace=True),\n",
    "            nn.MaxPool2d(kernel_size=3, stride=2),\n",
    "            nn.Conv2d(64, 192, kernel_size=5, padding=2),\n",
    "            nn.ReLU(inplace=True),\n",
    "            nn.MaxPool2d(kernel_size=3, stride=2),\n",
    "            nn.Conv2d(192, 384, kernel_size=3, padding=1),\n",
    "            nn.ReLU(inplace=True),\n",
    "            nn.Conv2d(384, 256, kernel_size=3, padding=1),\n",
    "            nn.ReLU(inplace=True),\n",
    "            nn.Conv2d(256, 256, kernel_size=3, padding=1),\n",
    "            nn.ReLU(inplace=True),\n",
    "            nn.MaxPool2d(kernel_size=3, stride=2),\n",
    "        )\n",
    "        self.avgpool = nn.AdaptiveAvgPool2d((6, 6))\n",
    "        self.classifier = nn.Sequential(\n",
    "            nn.Dropout(0.5),\n",
    "            nn.Linear(256 * 6 * 6, 4096),\n",
    "            nn.ReLU(inplace=True),\n",
    "            nn.Dropout(0.5),\n",
    "            nn.Linear(4096, 4096),\n",
    "            nn.ReLU(inplace=True),\n",
    "            nn.Linear(4096, num_classes)\n",
    "        )\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = self.features(x)\n",
    "        x = self.avgpool(x)\n",
    "        x = x.view(x.size(0), 256 * 6 * 6)\n",
    "        logits = self.classifier(x)\n",
    "        probas = F.softmax(logits, dim=1)\n",
    "        return logits, probas\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "id": "_lza9t_uj5w1"
   },
   "outputs": [],
   "source": [
    "torch.manual_seed(RANDOM_SEED)\n",
    "\n",
    "model = AlexNet(NUM_CLASSES)\n",
    "model.to(DEVICE)\n",
    "\n",
    "optimizer = torch.optim.Adam(model.parameters(), lr=LEARNING_RATE)  "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "RAodboScj5w6"
   },
   "source": [
    "## Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 1547
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 2384585,
     "status": "ok",
     "timestamp": 1524976888520,
     "user": {
      "displayName": "Sebastian Raschka",
      "photoUrl": "//lh6.googleusercontent.com/-cxK6yOSQ6uE/AAAAAAAAAAI/AAAAAAAAIfw/P9ar_CHsKOQ/s50-c-k-no/photo.jpg",
      "userId": "118404394130788869227"
     },
     "user_tz": 240
    },
    "id": "Dzh3ROmRj5w7",
    "outputId": "5f8fd8c9-b076-403a-b0b7-fd2d498b48d7"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 001/040 | Batch 000/188 | Cost: 2.3029\n",
      "Epoch: 001/040 | Batch 150/188 | Cost: 1.7090\n",
      "Epoch: 001/040\n",
      "Train ACC: 31.92 | Validation ACC: 31.05\n",
      "Time elapsed: 0.22 min\n",
      "Epoch: 002/040 | Batch 000/188 | Cost: 1.7312\n",
      "Epoch: 002/040 | Batch 150/188 | Cost: 1.6115\n",
      "Epoch: 002/040\n",
      "Train ACC: 43.78 | Validation ACC: 44.35\n",
      "Time elapsed: 0.43 min\n",
      "Epoch: 003/040 | Batch 000/188 | Cost: 1.5096\n",
      "Epoch: 003/040 | Batch 150/188 | Cost: 1.4324\n",
      "Epoch: 003/040\n",
      "Train ACC: 53.03 | Validation ACC: 52.30\n",
      "Time elapsed: 0.64 min\n",
      "Epoch: 004/040 | Batch 000/188 | Cost: 1.3731\n",
      "Epoch: 004/040 | Batch 150/188 | Cost: 1.2505\n",
      "Epoch: 004/040\n",
      "Train ACC: 56.87 | Validation ACC: 57.30\n",
      "Time elapsed: 0.85 min\n",
      "Epoch: 005/040 | Batch 000/188 | Cost: 1.0734\n",
      "Epoch: 005/040 | Batch 150/188 | Cost: 1.1652\n",
      "Epoch: 005/040\n",
      "Train ACC: 60.97 | Validation ACC: 60.30\n",
      "Time elapsed: 1.07 min\n",
      "Epoch: 006/040 | Batch 000/188 | Cost: 1.0730\n",
      "Epoch: 006/040 | Batch 150/188 | Cost: 1.1333\n",
      "Epoch: 006/040\n",
      "Train ACC: 62.87 | Validation ACC: 60.90\n",
      "Time elapsed: 1.28 min\n",
      "Epoch: 007/040 | Batch 000/188 | Cost: 1.0317\n",
      "Epoch: 007/040 | Batch 150/188 | Cost: 1.0182\n",
      "Epoch: 007/040\n",
      "Train ACC: 67.03 | Validation ACC: 64.35\n",
      "Time elapsed: 1.50 min\n",
      "Epoch: 008/040 | Batch 000/188 | Cost: 1.0245\n",
      "Epoch: 008/040 | Batch 150/188 | Cost: 0.9324\n",
      "Epoch: 008/040\n",
      "Train ACC: 64.87 | Validation ACC: 64.55\n",
      "Time elapsed: 1.71 min\n",
      "Epoch: 009/040 | Batch 000/188 | Cost: 1.0012\n",
      "Epoch: 009/040 | Batch 150/188 | Cost: 0.8525\n",
      "Epoch: 009/040\n",
      "Train ACC: 70.84 | Validation ACC: 67.30\n",
      "Time elapsed: 1.93 min\n",
      "Epoch: 010/040 | Batch 000/188 | Cost: 0.7442\n",
      "Epoch: 010/040 | Batch 150/188 | Cost: 0.7908\n",
      "Epoch: 010/040\n",
      "Train ACC: 70.95 | Validation ACC: 67.10\n",
      "Time elapsed: 2.14 min\n",
      "Epoch: 011/040 | Batch 000/188 | Cost: 0.8389\n",
      "Epoch: 011/040 | Batch 150/188 | Cost: 0.8383\n",
      "Epoch: 011/040\n",
      "Train ACC: 74.18 | Validation ACC: 69.95\n",
      "Time elapsed: 2.36 min\n",
      "Epoch: 012/040 | Batch 000/188 | Cost: 0.7037\n",
      "Epoch: 012/040 | Batch 150/188 | Cost: 0.9285\n",
      "Epoch: 012/040\n",
      "Train ACC: 74.23 | Validation ACC: 66.70\n",
      "Time elapsed: 2.57 min\n",
      "Epoch: 013/040 | Batch 000/188 | Cost: 0.7205\n",
      "Epoch: 013/040 | Batch 150/188 | Cost: 0.7099\n",
      "Epoch: 013/040\n",
      "Train ACC: 76.88 | Validation ACC: 70.00\n",
      "Time elapsed: 2.78 min\n",
      "Epoch: 014/040 | Batch 000/188 | Cost: 0.6575\n",
      "Epoch: 014/040 | Batch 150/188 | Cost: 0.6311\n",
      "Epoch: 014/040\n",
      "Train ACC: 76.69 | Validation ACC: 70.00\n",
      "Time elapsed: 3.00 min\n",
      "Epoch: 015/040 | Batch 000/188 | Cost: 0.6724\n",
      "Epoch: 015/040 | Batch 150/188 | Cost: 0.8899\n",
      "Epoch: 015/040\n",
      "Train ACC: 80.01 | Validation ACC: 71.80\n",
      "Time elapsed: 3.22 min\n",
      "Epoch: 016/040 | Batch 000/188 | Cost: 0.6895\n",
      "Epoch: 016/040 | Batch 150/188 | Cost: 0.5913\n",
      "Epoch: 016/040\n",
      "Train ACC: 79.64 | Validation ACC: 70.75\n",
      "Time elapsed: 3.43 min\n",
      "Epoch: 017/040 | Batch 000/188 | Cost: 0.6096\n",
      "Epoch: 017/040 | Batch 150/188 | Cost: 0.5401\n",
      "Epoch: 017/040\n",
      "Train ACC: 82.48 | Validation ACC: 72.20\n",
      "Time elapsed: 3.65 min\n",
      "Epoch: 018/040 | Batch 000/188 | Cost: 0.5421\n",
      "Epoch: 018/040 | Batch 150/188 | Cost: 0.4187\n",
      "Epoch: 018/040\n",
      "Train ACC: 84.17 | Validation ACC: 73.60\n",
      "Time elapsed: 3.86 min\n",
      "Epoch: 019/040 | Batch 000/188 | Cost: 0.4490\n",
      "Epoch: 019/040 | Batch 150/188 | Cost: 0.4658\n",
      "Epoch: 019/040\n",
      "Train ACC: 84.06 | Validation ACC: 72.65\n",
      "Time elapsed: 4.08 min\n",
      "Epoch: 020/040 | Batch 000/188 | Cost: 0.4837\n",
      "Epoch: 020/040 | Batch 150/188 | Cost: 0.4519\n",
      "Epoch: 020/040\n",
      "Train ACC: 86.10 | Validation ACC: 72.90\n",
      "Time elapsed: 4.29 min\n",
      "Epoch: 021/040 | Batch 000/188 | Cost: 0.4615\n",
      "Epoch: 021/040 | Batch 150/188 | Cost: 0.5283\n",
      "Epoch: 021/040\n",
      "Train ACC: 85.61 | Validation ACC: 72.10\n",
      "Time elapsed: 4.51 min\n",
      "Epoch: 022/040 | Batch 000/188 | Cost: 0.4693\n",
      "Epoch: 022/040 | Batch 150/188 | Cost: 0.4589\n",
      "Epoch: 022/040\n",
      "Train ACC: 88.70 | Validation ACC: 73.90\n",
      "Time elapsed: 4.72 min\n",
      "Epoch: 023/040 | Batch 000/188 | Cost: 0.2818\n",
      "Epoch: 023/040 | Batch 150/188 | Cost: 0.4123\n",
      "Epoch: 023/040\n",
      "Train ACC: 89.58 | Validation ACC: 73.45\n",
      "Time elapsed: 4.94 min\n",
      "Epoch: 024/040 | Batch 000/188 | Cost: 0.3030\n",
      "Epoch: 024/040 | Batch 150/188 | Cost: 0.3685\n",
      "Epoch: 024/040\n",
      "Train ACC: 90.44 | Validation ACC: 73.60\n",
      "Time elapsed: 5.15 min\n",
      "Epoch: 025/040 | Batch 000/188 | Cost: 0.2399\n",
      "Epoch: 025/040 | Batch 150/188 | Cost: 0.3384\n",
      "Epoch: 025/040\n",
      "Train ACC: 90.85 | Validation ACC: 73.35\n",
      "Time elapsed: 5.37 min\n",
      "Epoch: 026/040 | Batch 000/188 | Cost: 0.2333\n",
      "Epoch: 026/040 | Batch 150/188 | Cost: 0.2852\n",
      "Epoch: 026/040\n",
      "Train ACC: 92.25 | Validation ACC: 72.20\n",
      "Time elapsed: 5.58 min\n",
      "Epoch: 027/040 | Batch 000/188 | Cost: 0.2728\n",
      "Epoch: 027/040 | Batch 150/188 | Cost: 0.3350\n",
      "Epoch: 027/040\n",
      "Train ACC: 91.94 | Validation ACC: 73.85\n",
      "Time elapsed: 5.80 min\n",
      "Epoch: 028/040 | Batch 000/188 | Cost: 0.2277\n",
      "Epoch: 028/040 | Batch 150/188 | Cost: 0.2987\n",
      "Epoch: 028/040\n",
      "Train ACC: 92.84 | Validation ACC: 72.85\n",
      "Time elapsed: 6.02 min\n",
      "Epoch: 029/040 | Batch 000/188 | Cost: 0.2115\n",
      "Epoch: 029/040 | Batch 150/188 | Cost: 0.2038\n",
      "Epoch: 029/040\n",
      "Train ACC: 92.28 | Validation ACC: 72.50\n",
      "Time elapsed: 6.23 min\n",
      "Epoch: 030/040 | Batch 000/188 | Cost: 0.1841\n",
      "Epoch: 030/040 | Batch 150/188 | Cost: 0.2074\n",
      "Epoch: 030/040\n",
      "Train ACC: 94.63 | Validation ACC: 74.00\n",
      "Time elapsed: 6.44 min\n",
      "Epoch: 031/040 | Batch 000/188 | Cost: 0.1490\n",
      "Epoch: 031/040 | Batch 150/188 | Cost: 0.2191\n",
      "Epoch: 031/040\n",
      "Train ACC: 94.67 | Validation ACC: 72.60\n",
      "Time elapsed: 6.66 min\n",
      "Epoch: 032/040 | Batch 000/188 | Cost: 0.1719\n",
      "Epoch: 032/040 | Batch 150/188 | Cost: 0.1990\n",
      "Epoch: 032/040\n",
      "Train ACC: 93.93 | Validation ACC: 71.60\n",
      "Time elapsed: 6.87 min\n",
      "Epoch: 033/040 | Batch 000/188 | Cost: 0.1839\n",
      "Epoch: 033/040 | Batch 150/188 | Cost: 0.1939\n",
      "Epoch: 033/040\n",
      "Train ACC: 95.61 | Validation ACC: 73.75\n",
      "Time elapsed: 7.09 min\n",
      "Epoch: 034/040 | Batch 000/188 | Cost: 0.0995\n",
      "Epoch: 034/040 | Batch 150/188 | Cost: 0.1726\n",
      "Epoch: 034/040\n",
      "Train ACC: 95.35 | Validation ACC: 73.85\n",
      "Time elapsed: 7.30 min\n",
      "Epoch: 035/040 | Batch 000/188 | Cost: 0.1451\n",
      "Epoch: 035/040 | Batch 150/188 | Cost: 0.1414\n",
      "Epoch: 035/040\n",
      "Train ACC: 95.90 | Validation ACC: 73.55\n",
      "Time elapsed: 7.52 min\n",
      "Epoch: 036/040 | Batch 000/188 | Cost: 0.0551\n",
      "Epoch: 036/040 | Batch 150/188 | Cost: 0.1009\n",
      "Epoch: 036/040\n",
      "Train ACC: 95.40 | Validation ACC: 72.45\n",
      "Time elapsed: 7.73 min\n",
      "Epoch: 037/040 | Batch 000/188 | Cost: 0.1616\n",
      "Epoch: 037/040 | Batch 150/188 | Cost: 0.1102\n",
      "Epoch: 037/040\n",
      "Train ACC: 96.08 | Validation ACC: 72.55\n",
      "Time elapsed: 7.95 min\n",
      "Epoch: 038/040 | Batch 000/188 | Cost: 0.1409\n",
      "Epoch: 038/040 | Batch 150/188 | Cost: 0.1090\n",
      "Epoch: 038/040\n",
      "Train ACC: 96.69 | Validation ACC: 73.85\n",
      "Time elapsed: 8.16 min\n",
      "Epoch: 039/040 | Batch 000/188 | Cost: 0.1088\n",
      "Epoch: 039/040 | Batch 150/188 | Cost: 0.1309\n",
      "Epoch: 039/040\n",
      "Train ACC: 96.11 | Validation ACC: 72.15\n",
      "Time elapsed: 8.38 min\n",
      "Epoch: 040/040 | Batch 000/188 | Cost: 0.1418\n",
      "Epoch: 040/040 | Batch 150/188 | Cost: 0.1745\n",
      "Epoch: 040/040\n",
      "Train ACC: 97.45 | Validation ACC: 74.00\n",
      "Time elapsed: 8.59 min\n",
      "Total Training Time: 8.59 min\n"
     ]
    }
   ],
   "source": [
    "def compute_acc(model, data_loader, device):\n",
    "    correct_pred, num_examples = 0, 0\n",
    "    model.eval()\n",
    "    for i, (features, targets) in enumerate(data_loader):\n",
    "            \n",
    "        features = features.to(device)\n",
    "        targets = targets.to(device)\n",
    "\n",
    "        logits, probas = model(features)\n",
    "        _, predicted_labels = torch.max(probas, 1)\n",
    "        num_examples += targets.size(0)\n",
    "        assert predicted_labels.size() == targets.size()\n",
    "        correct_pred += (predicted_labels == targets).sum()\n",
    "    return correct_pred.float()/num_examples * 100\n",
    "    \n",
    "\n",
    "start_time = time.time()\n",
    "\n",
    "cost_list = []\n",
    "train_acc_list, valid_acc_list = [], []\n",
    "\n",
    "\n",
    "for epoch in range(NUM_EPOCHS):\n",
    "    \n",
    "    model.train()\n",
    "    for batch_idx, (features, targets) in enumerate(train_loader):\n",
    "        \n",
    "        features = features.to(DEVICE)\n",
    "        targets = targets.to(DEVICE)\n",
    "            \n",
    "        ### FORWARD AND BACK PROP\n",
    "        logits, probas = model(features)\n",
    "        cost = F.cross_entropy(logits, targets)\n",
    "        optimizer.zero_grad()\n",
    "        \n",
    "        cost.backward()\n",
    "        \n",
    "        ### UPDATE MODEL PARAMETERS\n",
    "        optimizer.step()\n",
    "        \n",
    "        #################################################\n",
    "        ### CODE ONLY FOR LOGGING BEYOND THIS POINT\n",
    "        ################################################\n",
    "        cost_list.append(cost.item())\n",
    "        if not batch_idx % 150:\n",
    "            print (f'Epoch: {epoch+1:03d}/{NUM_EPOCHS:03d} | '\n",
    "                   f'Batch {batch_idx:03d}/{len(train_loader):03d} |' \n",
    "                   f' Cost: {cost:.4f}')\n",
    "\n",
    "        \n",
    "\n",
    "    model.eval()\n",
    "    with torch.set_grad_enabled(False): # save memory during inference\n",
    "        \n",
    "        train_acc = compute_acc(model, train_loader, device=DEVICE)\n",
    "        valid_acc = compute_acc(model, valid_loader, device=DEVICE)\n",
    "        \n",
    "        print(f'Epoch: {epoch+1:03d}/{NUM_EPOCHS:03d}\\n'\n",
    "              f'Train ACC: {train_acc:.2f} | Validation ACC: {valid_acc:.2f}')\n",
    "        \n",
    "        train_acc_list.append(train_acc)\n",
    "        valid_acc_list.append(valid_acc)\n",
    "        \n",
    "    elapsed = (time.time() - start_time)/60\n",
    "    print(f'Time elapsed: {elapsed:.2f} min')\n",
    "  \n",
    "elapsed = (time.time() - start_time)/60\n",
    "print(f'Total Training Time: {elapsed:.2f} min')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEGCAYAAABo25JHAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAgAElEQVR4nOzdd3jUVdbA8e+dkoRAaAm9BaT3EnqR3kUFVARRBAULYlnZRV8FxYara0FUFLCACnZFQZAqRVroVWqAUEMgCSVlyn3/mGRImUwmyUwSmPN5njw786s3kZ0zv1vOUVprhBBC+C9DYTdACCFE4ZJAIIQQfk4CgRBC+DkJBEII4eckEAghhJ8zFXYDcissLEyHh4cXdjOEEOKGsnXr1gta63Ku9t1wgSA8PJzIyMjCboYQQtxQlFLHs9snXUNCCOHnJBAIIYSfk0AghBB+7oYbIxBC5J3FYiE6OpqkpKTCborwkaCgIKpWrYrZbPb4HAkEQviR6OhoQkJCCA8PRylV2M0RXqa1JjY2lujoaGrWrOnxedI1JIQfSUpKIjQ0VILATUopRWhoaK6f+CQQCOFnJAjc3PLy39evAsHyfec4FZdY2M0QQogixW8Cwc6TcTw0N5KO01YWdlOE8GtKKUaOHOl8b7VaKVeuHAMHDgRg4cKFTJs2ze01Tp8+zdChQwH44osvGD9+fK7a8Prrr+d4zKhRo/jhhx9ydd282LFjB4sXL/b5fdzxm0Cw4WhsYTdBCAEUL16cPXv2kJjoeDpftmwZVapUce4fNGgQkyZNcnuNypUr5+tD2pNAUFAkEBSgR269pbCbIIRI1a9fPxYtWgTA/Pnzuffee5370n/DHzVqFBMmTKBDhw7UqlXL+eEfFRVF48aNneecPHmSvn37Uq9ePV5++WXn9jvuuINWrVrRqFEjPv30UwAmTZpEYmIizZs3Z8SIEQDMnTuXpk2b0qxZswxPK2vWrMly78xcnXv8+HF69OhB06ZN6dGjBydOnADg+++/p3HjxjRr1owuXbqQkpLC5MmT+fbbb2nevDnffvtt/v6weeRX00f7Na7IH3vOFnYzhCgSXv5tL/tOJ3j1mg0rl2TKbY1yPG7YsGFMnTqVgQMHsmvXLkaPHs3atWtdHnvmzBnWrVvHgQMHGDRokLNLKL3NmzezZ88egoODad26NQMGDCAiIoLPPvuMsmXLkpiYSOvWrRkyZAjTpk1jxowZ7NixA4C9e/fy2muvsX79esLCwrh48aLH987u3PHjx3P//ffzwAMP8NlnnzFhwgR++eUXpk6dytKlS6lSpQpxcXEEBAQwdepUIiMjmTFjhsd/Z2/zmycCgHoVQwCw2aVOsxCFqWnTpkRFRTF//nz69+/v9tg77rgDg8FAw4YNOXfunMtjevXqRWhoKMWKFWPw4MGsW7cOgOnTp9OsWTPatWvHyZMnOXToUJZzV65cydChQwkLCwOgbNmyHt87u3M3bNjA8OHDARg5cqSzPR07dmTUqFHMmjULm83m9vcuSH71RBBkNgKQbLURHOBXv7oQWXjyzd2XBg0axLPPPsvq1auJjc1+DC8wMND5WmvXX+IyT5lUSrF69WqWL1/Ohg0bCA4OpmvXri7n12uts51ymdO93Z3rqn0zZ85k06ZNLFq0iObNmzufSgqbXz0RBJocv26yxV7ILRFCjB49msmTJ9OkSZN8X2vZsmVcvHiRxMREfvnlFzp27Eh8fDxlypQhODiYAwcOsHHjRufxZrMZi8UCQI8ePfjuu++cwSh911BOsju3Q4cOLFiwAICvv/6aTp06AXDkyBHatm3L1KlTCQsL4+TJk4SEhHD58uV8/w3yw68CQdoTQZK16DySCeGvqlatypNPPumVa3Xq1ImRI0fSvHlzhgwZQkREBH379sVqtdK0aVNefPFF2rVr5zx+7NixNG3alBEjRtCoUSP+7//+j1tvvZVmzZrxzDPPeHzf7M6dPn06n3/+OU2bNmXevHm8//77AEycOJEmTZrQuHFjunTpQrNmzejWrRv79u0r1MFild2jVlEVERGh81qY5qdt0Tzz3U5WPduVmmHFvdwyIYq+/fv306BBg8JuhvAxV/+dlVJbtdYRro73qyeCgNSuoRSrdA0JIUQavwoEgSZH15AEAiGEuM6vAoHziaAITdsSQojC5l+BwOj4dcd/s72QWyKEEEWHfwWC1CeCM/FSnUkIIdL4VSBIW0cghBDiOr/6ZAyQQCBEoTMajTRv3pzGjRtz2223ERcX55P7dOjQwSfXvRn51Sej2Xj91z18/kohtkQI/1WsWDF27NjBnj17KFu2LB9++KFP7vP333/75LreUJTyDIGfBQKT4XpOkNFfbCnElgghANq3b8+pU6cAWL16tbM4DTgyeH7xxRcAhIeHM2XKFFq2bEmTJk04cOAAAC+99BKjR4+ma9eu1KpVi+nTpzvPL1GihPO6Xbt2ZejQodSvX58RI0Y48wYtXryY+vXr06lTJyZMmJDh/mmioqLo3LkzLVu2pGXLls4Ac88992SoIzBq1Ch+/PFHbDYbEydOpHXr1jRt2pRPPvnE2Y5u3boxfPhwZ1oNV2myAebMmUPdunXp2rUrDz/8sDMtd0xMDEOGDKF169a0bt2a9evX5+Ovf51fZV5L/0QgGUiF3/tjEpzd7d1rVmwC/dxXF0tjs9lYsWIFY8aM8ej4sLAwtm3bxkcffcTbb7/N7NmzAThw4ACrVq3i8uXL1KtXj0cffRSz2Zzh3O3bt7N3714qV65Mx44dWb9+PREREYwbN441a9ZQs2bNDDUR0itfvjzLli0jKCiIQ4cOce+99xIZGcmwYcP49ttv6d+/PykpKaxYsYKPP/6YOXPmUKpUKbZs2UJycjIdO3akd+/ewPV02TVr1gRwmSY7OTmZV155hW3bthESEkL37t1p1qwZAE8++SRPP/00nTp14sSJE/Tp04f9+/d79Pdzx68CgckoRbuFKGxpRWGioqJo1aoVvXr18ui8wYMHA9CqVSt++ukn5/YBAwYQGBhIYGAg5cuX59y5c1StWjXDuW3atHFuS7t3iRIlqFWrlvND+d57783wrTyNxWJh/Pjx7NixA6PRyMGDBwFHcZ0JEyaQnJzMkiVL6NKlC8WKFePPP/9k165dzkI28fHxHDp0iICAANq0aeO8HzhyEv38888AzjTZZ8+e5dZbb3WmtL7rrruc91y+fDn79u1znp+QkMDly5cJCQnx6G+YHb8KBKHFA5yvb7QcS0J4nYff3L0tbYwgPj6egQMH8uGHHzJhwgRMJhN2+/VV/5lTRqelhDYajVit1izbXe1zd4ynnwHvvvsuFSpUYOfOndjtdoKCggAICgqia9euLF26lG+//db5RKG15oMPPqBPnz4ZrrN69WqKFy+e4b2rNNnu2mW329mwYQPFihXzqO2e8qsxgvR5w0/LWgIhClWpUqWYPn06b7/9NhaLhRo1arBv3z6Sk5OJj49nxYoVPr1//fr1OXr0KFFRUQDZZv6Mj4+nUqVKGAwG5s2bl2Ggd9iwYXz++eesXbvW+cHfp08fPv74Y2ea64MHD3L16lWX13WVJrtNmzb89ddfXLp0CavVyo8//ug8p3fv3hkqmXmrnoFfBQIhRNHSokULmjVrxoIFC6hWrRp33323Mz10ixYtfHrvYsWK8dFHH9G3b186depEhQoVKFWqVJbjHnvsMb788kvatWvHwYMHM3yr7927N2vWrKFnz54EBDh6HB566CEaNmxIy5Ytady4MePGjXP5lJJdmuwqVarw/PPP07ZtW3r27EnDhg2d7Zo+fTqRkZE0bdqUhg0bMnPmTK/8LfwqDTVA+KRFztdR0wZ4o0lC3DAkDXVGV65coUSJEmitefzxx6lTpw5PP/10YTfL2S6r1cqdd97J6NGjufPOOz0+v8ikoVZKVVNKrVJK7VdK7VVKZalAoRymK6UOK6V2KaVa+qo9QgiR2axZs2jevDmNGjUiPj6ecePGFXaTAMe02LRFdzVr1uSOO+7w6f18OVhsBf6ltd6mlAoBtiqllmmt96U7ph9QJ/WnLfBx6v8KIYTPPf3000XiCSCzt99+u0Dv57MnAq31Ga31ttTXl4H9QJVMh90OzNUOG4HSSqlKvmpTZum7iYTwFzdad7DInbz89y2QwWKlVDjQAtiUaVcV4GS699FkDRYopcYqpSKVUpExMTG+aqYQN72goCBiY2MlGNyktNbExsY6p7h6yufrCJRSJYAfgae01gmZd7s4Jcu/UK31p8Cn4Bgszk97QosHEHs1JT+XEOKGVbVqVaKjo5EvVDevoKCgLAvqcuLTQKCUMuMIAl9rrX9ycUg0UC3d+6rAaV+2qWqZYhIIhN8ym80ZVrYKAb6dNaSAOcB+rfU72Ry2ELg/dfZQOyBea33GV20C6NekwIYghBDihuDLJ4KOwEhgt1Iqbfnb80B1AK31TGAx0B84DFwDHvRhe4QQQrjgs0CgtV6H6zGA9Mdo4HFftcEVSTsnhBAZ+X2KiYQkS2E3QQghCpX/BIKzu+GPSQRYL2fY/NAXkVxLyZoHRAgh/IX/BIKE07DpY+qoUxk2b466yKQfvVycQwghbiD+EwjK1QOgY6kLWXYt3OnTGatCCFGk+U8gKFUdTMVQFw5Sv2L+qvkIIcTNxH8CgcEAYXUg5gBP9ayTZXdiis3FSUIIcfPzn0AAUK4+xPxD7fIlsuz6ZccpFycIIcTNz88CQT2IP0ntrEWIuJosM4eEEP7JzwJBfcf/XjiYZderi/YXcGOEEKJo8M9AEPMPVcsUK9y2CCFEEeFfgaBMOBgDIOYAj3WtXditEUKIIsG/AoHRBKF1IOYfbFKYQwghAH8LBOAYMI45QLJFposKIQT4ZSCoD5eO07duySy7jsdeLYQGCSFE4fK/QFClFaCpenVPll2v/L6v4NsjhBCFzP8CQfW2oIwQtS7Lrt2n4jl/OakQGiWEEIXH/wJBYAhUagrH/86y61xCMm1eW0GSjB8IIfyI/wUCgBodITqSAFwXpan/4pICbpAQQhQe/w0EtmS2PSBZSIUQwj8DQa2uYAykxImVhd0SIYQodP4ZCAKCoVobOLYm20P6v7+WS1dTCrBRQghROPwzEADUvBXO7qYMCS537zuTQItXlhVwo4QQouD5byC4pTugWTTA9YCxEEL4C/8NBJWbQ3Aolc+ucntY+KRFGd6fiL2G1Wb3ZcuEEKJA+W8gMBih4e3wzx8E49kisvMJSXR5axWvLZbaBUKIm4f/BgKAJneBNZFehkiPDo9NHTz++3CsL1slhBAFyr8DQbV2ULIqdxjX5+o0pXzUHiGEKAT+HQgMBmh2D12Nu6mmzmV7WPikRXSctpKle88WYOOEEKJg+HcgAIgYAwqGGd0PGp+KS+S95YcKqFFCCFFwJBCUqoKu3Yu7jGswY83VqeGTFvHe8oM+apgQQhQMCQSAbj2W8iqOoca/PDpepRskkKcEIcSNTgIBwC3d2WG/hUeNC4GcaxnLWLEQ4mYigQDHN/wvrH2oboihvUGqlAkh/EuOgUApZSyIhhQmg0HRc+jDxOnijDCuyPH4nJ8ZhBDixuHJE8FhpdRbSqmGPm9NIRrYshaGFvfR17CZuuqk22P3n8mYqE5STgghbmSeBIKmwEFgtlJqo1JqrFKqZE4nKaU+U0qdV0plrRLv2N9VKRWvlNqR+jM5l233upK9J3GFYkw0fZfjsV+sP+Z8/fn6KB+2SgghfCvHQKC1vqy1nqW17gD8G5gCnFFKfamUqu3m1C+Avjlcfq3Wunnqz1SPW+0rwWWZb+tON8N2Qol3e+hLv10fS4iVugVCiBuYR2MESqlBSqmfgfeB/wG1gN+Axdmdp7VeA1z0VkMLysqA7piUnaHG7IvWZGbXMmoghLhxedI1dAi4HXhLa91Ca/2O1vqc1voHIL9V3tsrpXYqpf5QSjXK7qDU7qhIpVRkTExMPm/p3lFVjTW2Jow1/U4xD7OSWm0SCIQQNy6Pxgi01mO01n9n3qG1npCPe28DamitmwEfAL9kd6DW+lOtdYTWOqJcuXL5uGXOmlQtxXvWIYSqyx7NIAL4LN14gRBC3Gg8CQTllVK/KaUupA7+/qqUqpXfG2utE7TWV1JfLwbMSqmw/F43v2YMb8n/PTKKTboho0xLPU478cv2Uz5umRBC+IYngeAb4DugIlAZ+B6Yn98bK6UqqtRcDUqpNqltKfRE/yUCTbSqUYbP9CCqqguMNf7u0Xk/bovm8/XHsMhUUiHEDcbkwTFKaz0v3fuvlFLjczxJqflAVyBMKRWNY7aRGUBrPRMYCjyqlLICicAwrYvOqOt61ZKltgjGmX5jnq0XCRR3e/zaQxdYe+gCNrvmoc75fmASQogC40kgWKWUmgQswLGo9h5gkVKqLIDW2uXMIK31ve4uqrWeAczIXXMLToDJwPvXBtMnMJJRxqVMtw326LyTF6+x42QcJy9eo1rZYI7GXGFwy6o+bq0QQuSdyulLuFLK3Uio1loX6NffiIgIHRnpWWnJ/PhgxSH+t+wgs8xv09rwD52S3+cKwXm6VtS0AV5unRBC5I5SaqvWOsLVPk8WlNV083PT9oHUrRgCwAfWOymtrjLSuLyQWySEEL7hyYIys1JqglLqh9Sf8Uopc0E0rjClpZrepW/hL1tTHjIt8nhdgRBC3Eg8mTX0MdAK+Cj1p1XqNr/xvnVwrtYVCCHEjcSTQNBaa/2A1npl6s+DQGtfN6ywpa9Ctk3XZb2tEeNMvxNI7vMK2e2ad5Yd5PxlxxPFzpNxPP71Nmz2IjNJSgjhxzwJBDal1C1pb1IXk9l816SioUvdMFpWL80jtzp+9Q9sd1JOxXNfHsYKtp24xPQVh/jXdzsBeOzrbSzafYYz8YlebbMQQuSFJ9NHJ+KYQnoUR9d5DeBBn7aqCAg0GfnpsY4ATOpXnztmlGLduUY8alrIPFsvUvB8mCTZ6lhkdjnJsUr5XIKMNQghig63TwRKKQOOxV51gAmpP/W01qsKoG1FijIoPrHdRphK4G7j6lydO2L2JgB2nIzjSrIVq3QJCSGKELeBQGttB/6ntU7WWu/SWu/UWicXUNuKFAWsszdml70mL5m+pJHb5RXZ23c6IeeDhBCiAHkyRvCnUmqISj966ocGNauMxsDDKf/iEiX4yPw+Jbma6+scibnifN3pzVU0nrKUq8meJbYTQghf8CQQPIMj0VyyUipBKXVZKeV3X2sf6BDOgVf6co6yjEt5hsoqlrfMn5DbUvaZu4WuJFvZfcp9NTQhhPAlT1YWh2itDVrrAK11ydT3OdYsvtkopQgyGwHHdNJp1nvpY4zM9XjBi7+4LOEshBCFxpOVxVlWUbna5m/m2PoRaa/LK6bPqaOiC7s5QgiRZ9kGAqVUUGqG0TClVBmlVNnUn3AcdQn8UuVSQamvFONSnuYaQbxl/gQD3q1DYLNrkiw3/XINIUQR4O6JYBywFaif+r9pP78CH/q+aUXTgrHtna9jKcUUyyiaG46kjhfkjd1FBtinvt1B/RfzWxJaCCFylm0g0Fq/r7WuCTyrta6VLuNos9RaAn6pVHDGhWQL7e35xdaBIca1vGv+kNwOHgN8tfF4lm2/7TwNQBGq1SOEuEnluLJYa/2BUqoDEJ7+eK31XB+2q+jK8rms+LdlHN0N27nTuJ4ALDxjeYxkAjy+5OLdZ7PdF3n8Eq3Dy+atrUII4YEcA4FSah5wC7CD6zmGNOCXgUC7+MafgplWyZ/wsulzhptWUYIkHrBM8sr9rDZ5IhBC+JYnuYYigIZFqZ5wYTIbHb1pNcOKc+zC9QVlFkw8b32YWErxhOkXhttX8I2th8fXDZ+0iKhpA7DbNQ/PvV6BzVXgEUIIb/JkQdkeoKKvG3KjKB5o4rfxnZh5XyuX+9+3DiZah/G6eQ6jjX/k+vpnE5JYceB8ntu351Q8G4/G5vl8IYT/8eSJIAzYp5TaDDjzDGmtB/msVUVck6qlOB3nOoW0FRM9k9/iPfNHTDbPo6Y6wyvWkR5lK+305kr6NsoYc9MylqZ3Jj6RAKOB0BKBWfYN/GAdIHWShRCe8yQQvOTrRtyIyocE0rRqKaqVCWbR7jMZ9iURyBOWJ/iPns9Dpj/obYzk/pRJ/KOru71m9KVEZq/LmMxu3LytWT7U27+xEsj4YZ9ksUnOIiFEnrhbUFYfQGv9F7BRa/1X2g/pngz8lcloYOH4TvRr4rrXzIKJV60j+VfKI1RQcXwW8BZlyH+KpvmbTzhfpx+2Gf3FFlq9mrFozq7oOGeSu9lrj/LW0gP5vr8Q4ubjbozgm3SvN2Ta95EP2nJD6t+4ktv9P9q7MDj5JcKIZ3bA/wjAkut7zNsQRcdpK7lr5t8899Nu5/YPVh4G4LstJ/n7SNZxgUEz1tPjf38B8Oqi/Xy46kiu7y2EuPm5CwQqm9eu3vstg0ERUaOM22O26bq8aH2QVoZDvG2eSW4Xnb34615OxSWyJepShu2/73IsOvv3j7tydT0hhEjPXSDQ2bx29d6vtcwhEAB8Z+vGu5YhDDJuYK55mldyE526lMjXm7KuShZCiNxwFwiqKqWmK6U+SPc67X2VAmrfDeHffepl2VbMbKR+xZAM26bb7uR3Wzu6GHezPOBZjOQvqdzVFBv/97OktRZC5I+7QDARR5K5yHSv097/2/dNu3GYjAbqViiRYdv97Wsw6/6IDNs0Bp6yPMYWe11qGc5yJGgkHQzyQS6EKFzZTh/VWn9ZkA250Y1sH+4sOvPXxK5ULxvMyYtZ1xpYMXFXyks8alzIf8wL+CbgdUanPMtKe8uCbrIQQgCerSwWHhjZrgZzR7ehfsUQKpcuhlKKEkHZL9P42DaI5kmfcNhemQ/N02mj9nu1PeGTFjlf29OVx5QaB0KIzCQQeFGXuuVY8lQXZz6issUDWP1s12yPjyOEYSkvclaX4bvAV1gQ8IpPqp3NWnvU+XrHyTivX18IcWOTQOBj4WHF3e6/QCmGprzEDOvt1FUnWRb4b+aa36CTYbfb83Lj8/VRztfL9p3z2nWFEDcHT2oW/1cpVVIpZVZKrVBKXVBK3VcQjfMXsZTibes93J7yClH2CnQx7uargDeYbPJOpu+zCUnO13MypbAQQghPngh6a60TgIFANFAXxywi4WUndQW6prxL7+Q3sWgjo01LuMu42uv3mbshipMXr7ncN/H7nfR9b02O1zh8/gp7T8d7uWVCiMLgSSBIS5vZH5ivtb7oyYWVUp8ppc4rpVzOj1QO05VSh5VSu5RSN+20mXoVQnI+KJ2DuhoNkz9nja0Jr5vm0NGL3UQAk3/dy7BPN5KYYssSEL7fGs2Bs5dzvEbPd/5iwPR1Xm2XEKJweBIIflNKHcBRoGaFUqockJTDOQBfAH3d7O8H1En9GQt87ME1b0jfjbte8P6O5pU9OseCicctT3JEV+brgDe4w+DdD93LSRYenhtJ5/+uAhwJ6mRGkRD+KcdAoLWeBLQHIrTWFuAqcLsH560B3D093A7M1Q4bgdJKKfcZ3G5QaQXv29cK5b1hLZzbJ/Sog9GQfdqmywRzf8okTulQ3jbP9OoAckKSlXWHLwBwPPYqg2asZ/KvsrhNCH/kyWDxXYBVa21TSr0AfAV49rXWvSrAyXTvo8kmdYVSaqxSKlIpFRkTE+OFWxe8Tc/34PMHW2fY9kyvuhx5vb/b885Thr7Jb3JIV+Uz83/5NmAq75g/oqry3t/h3k83ArArOuc+/0W7zrDnlIwNCHEz8aQwzYta6++VUp2APsDbOLpx2ubz3q6+CrtMZqe1/hT4FCAiIuKGTHhXoWRQns+9TDD3pLzAa+bPuM3o+NAebFzHN9buzLb156iuRH4Swp6Od/T0pR8b+HTNEaIvJTL19sacikskMcXGmfhEHv9mW57vI4QomjwJBGkdxwOAj7XWvyqlXvLCvaOBauneVwVOe+G6N4ScUldnlkAJnrBM4AnLBCpzgR8CX2K4aSXDTSs5q8vwnnUIP9i6YPXoP2nOXl/sKGJzX7sa9H4351lEQogblyeDxaeUUp8AdwOLlVKBHp6Xk4XA/amzh9oB8VrrMzmddDNY++9uzB3TJs/nnyaMDskz+LflYZbZWlKSa0wzz+Zw0P28bpqNGe+VrJyRWvzGU5+tO8aWKI8mlgkhigiVvtyhywOUCsYx+2e31vpQ6oBuE631nzmcNx/oCoQB54AppE5F1VrPVEopYEbqta8BD2qtI3NqcEREhI6MzPGwG8qRmCvMWHmYe9tU52xCEhPmbwegW71yrPon57GAUOLZGvSo8/1CW3smWMbjjfpBAUYDKbbsaydkrqecluMo83YhROFSSm3VWke42pdjP4LW+ppS6gjQRynVB1ibUxBIPe/eHPZr4PGcruMPbilXgnfvae58nxYIyhQP4J27m/HMdzvdnh9LKcKTHJVFXzTNY4zpD6qpGIalvEAyAflqm7sgIIS4OXgya+hJ4GugfOrPV0qpJ3zdMAFouK1Z7iZovWYdwYfWQbQwHGam+V2CSPZR44QQNwtP+vrHAG211pO11pOBdsDDvm2Wf3ugfQ3n67RMpp6yY+At6zBetYygm3En6wKfpCwJ3m5ijqIvuU5hIYQoejz5lFGQoaaiDSle71NNq5YGXM+lrRVWnBFtq+d4jdm2AUyzDCNMJbAt6BFWBzzN3cZVXm5p9jq9uUpyEQlxg/BkruHnwCal1M+p7+8A5viuSaJv44r8secMz6bWQq5VrjhHY67yychW9GlUEYCeDSrwzrKD7HazuGumbRDr7Y35LmAq4YZz/Ncwi0QdyG/2Dl5v8/BZG9l0LONsoR+2RtOocimSLDYSkiwEmows3XOWu1tXy+Yq7g3+aD1x1yysdFPjQQiRe54MFr+jlFoNdMLxJPCg1nq7rxvmz4oHmpj9wPVVyGHFAzkac5XSxczObd3ql+fDVTlP7dyta9Eg+QsCsPBVwOt8EDCDFtbDTLWOxBsPdukroWX21cbjTLmtEQ/PjWTtoQv0bVSRJXvP0qBSSRpWLsnkX/cwtkstaoS6r9mQZtsJKaojhC+47RpSShmUUjaW+ZQAACAASURBVHu01tu01tO11u9LECg8mbuK7DlM/U0vBTNjUiayyV6f0aYlvGb6jIrEuriq96095MhpFB3nGDdItNiYtfYoX286wfhvHP+cnv52B01eWuqDe8cQPmkRF6+meP3aQtws3AYCrbUd2KmUyrlTWvjM4JaOFEw1M1U7+78BDbMcW6lU9qksHKkqXuRba1dGmFawMegJPjO/RXESvdvgVCrTE8eeU45B6zPxiUz7w7Fy2ZZaT/nn7ae4nOR6IdycdcfoOG1lntrw6RpHmU53XWhC+DtPBosrAXtTq5MtTPvxdcPEdfe0rsaxN/pnyVfUKlOaivHdalMqXfeRa4r/WMfyH8vDnLCXo7txB3uDxtDXsNnLrc7eleTrH/hWe8Z1ClprNmcaa3jl932civNNsBJCeBYIXsZRnWwq8L90P6KAKKVwLMTOqkzw9Q/+p3vVZUATzzJ5f2vrRpeU93nB8iAAMwPeY5RxCYqCXUB28NwVftga7Xz/9aYT3P3JBpbs8YtsI0IUCdkOFiulagMVtNZ/ZdreBTjl64YJz6QFiAc7hmM0KB7vVpthbarT+rXlHp3/la0X++w1+D/z17xknssI4wq22etgw8Bie1vW2ZvkuW0pNjtjvtiSZfuCzSczvH/2++srp49duArAyYvyBCBEQXH3RPAe4Kpm4bXUfaIISHtOeLxbbQAMBkW5kEBeHJh1/CA723RdhqZMYZLlIa4SxFDjXww3reSrgDeYbv6Aylwgr4PKKw6cz7LNXX99Wp0eXQCD2EIIB3fTR8O11rsyb9RaRyqlwn3WIpEnmScQZR4/SNO4SknnoG2G8zGwwNadBbbuGLBjxso08yzuNK5nkHEDh+xVmGAZz35dw8VVvWfW2mMA2PMYBzpOW0nJYmb+eLKzF1slxM3N3ROBu0oqxbzdEJE3vRtVACA4wJhhe3YrBH5/IucPSDsGkgngacvjPJXyGFvsdaljOMUfgc/xY8CU1GmnvmXXmjPxiew8mbu1A6fiEtl/puBTaghxI3MXCLYopbLkFFJKjQG2+q5JIjem3t6YTc/3oHhgxoe7bMaWc+0XeyfuSnmJESnPEa+DaWU4xKLA57nHx+kq9p1OoPObq7j9w/Vuj/v78AViLuecWK+wc6IkWWwZZksJUZS46xp6CvhZKTWC6x/8EUAAcKevGyY8YzYa3JbBTN8VVL9iCAA1QoM5Hpu7pHDr7U1oljybJuoo/zN/zJvmWQSSwlxbn7w33o3jsdew5tA/lGy1MXz2JmqEBvPXxG4+aYe3dP7vKmIuJ0udBlEkZftEoLU+p7XugGP6aFTqz8ta6/Za67MF0zyRV2ElAgFoVzOUd+5uBkCDSiUB+PPpLnm+7m5di34p01hua8FU85e8YvqMQLy/ateTBWD1XlgCkOugVhg8eWoRorB4kmtoFVBwaSuFV1QuXYxVz3alWpli7EjtZ29SpRQAgSaju1NzZMPI05bHWaBeYaRpOXUMp/jc2peq6jwndAWW2Vvh684Ya6aCOQt3nmZQLms3CCEcvFPpXBRJaSkpIsLLsvSpLtStUCLLMXc0r8wvO07TpmbZLCt63blMMANS3uA+4zKmmObSLmC/c98SW2ues4zhEiXz/0u4cPj8ZWfqiDQT5m/n1jrlfHI/IW52Egj8RL3U8YHMqqdm/ixmzttTwle2XqyyNaee4SQhXON24990N2zjz8B/+MR6G2d0KMEqiUh7PY5pz1Y956TnO2tcbu/836z5iNIS3t3/2WZ2TunNxqOxtL8llGCzkTPxSVQrG+w81m7XxCVaKFs8f+U9hbjRSCDwU38+3QWbXbNolyOVQ6saZehYO5TLSVY+WJlzeuv0TlGOU3bHt/Ff7Z1oqKKYbp7BC+avncfYtGKWbSDvWofku45ydhKySVqXZvU/53lywQ561C9PzbDizF53jOXPdGHB5pMMb1ud33ae4d3lB9nwXHcqlZIZ0sJ/SCDwU3UrOJ4QzsQ7Ujk4AkEYn607lu05IUGmbDOEprdPh9Mz5W0qc4GqKoZ4ijPW9DuPmH7jfuOfvGy9nx9tnbEW8D+/aymOQnvHL15zrjVIe7qYne73PhuflCUQxF5J5tO1R/l3n/oYDZ6Pf9jsGp2LdOFCFAYJBH6ue/0KbHuxl7M7pKSb7KVP9qjDq4v2Z7s/s9OEcVqHAfAvy2NstDfkTdMs3jTP4gnTz6y1NWGbrsP3tq75+h2yE3ct42ymtM/jw+evuD0vc4I/q81Oq1cduZva1ixL9/oVPG5D17dXcTouyePjhSgMuauMLm5K6fvEB7eowmNdb3F5nCHdB2S1srnvOvne1pXmyZ/wguVBjtkrcq9pFW+ZP2W2+S3uN3q3KE1k1EWm/r4vw7bnf96dp2t9G3k9SZ7d7ggM4ZMWuX16SnPyYqKz5oIQRZUEApGBwaD4d9/6fDWmLQAz72vl3Nc6vCwAz/Sqy5In87YWIYESfGXrxUjL83ROfped9lr0NG5nqvlLooKGM8k0H4MXUmEPnbmBn7Z5J0nu//28J8P7axZHF9M7yw565fpCFDYJBMKlTnXCOPRaP7rVdwwCKwVNqpbi0Gv9mNCjTpaUFnlxUlfg9pRX6ZL8LgusXQF4xPQbSwL+Q3ku5fv6eZXTCEDak9GVZKvbms1C3CgkEIhsmY2GLOUmzcas/2QaVsq4XuCeiGq5us8JXYFJ1rGEJ33DC5YHqWs4xZ+B/2aiaQGt1D8URF1lTxWdlgjhPRIIhFtpwwKuviVPvb0RfRpV4IdH22fY/vrgvBez+crWi4mWsZQgkcdNC/kx8GX+CRzFaOMfqXURfC/td/7rYIzLb/x2mQUkbjISCIRb7rpJ7m8fzicjIwgOMPHW0KbO7dlNr8y5nrLD97au1Ev+kgdTJrLc1oLLFGOyeR5/B01gqulzWqqD+PK7+dVkxxjAvA3Hs+xTgM40hJE2ePzhqtytvxCiqJBAINxK6w/PrtBNmpAg92MGIW7GFHo2yDod04aRVfYWPGSZSETyTAYmvwrA/aZl/BT4EnsCx/CE8Sef1Fi+d9ZGNhyJdZnK+/zl5CxPBCmpeY8+WHnI620RoiBIIBBuGQyK35/oxGejWrs/LtOnZu3yGfMaPde/gXNhVf1M6S7CQ4PJyR5di6ZJs3gy5TFet9zLXh3Ov8w/sCDgVSaZvqEk7tcG5Nana4643P78z7uzBAJfzA49FZfIvI1Zn0iE8AVZUCZy1Dg1a6k7mQPBogmdsNk1z36/k8W7z5K+t+jRrrfw5IIdzvdVy3i2JiGB4vxq7wTAp7aBjDH+wYvmr2hrOMAg49/Mtg7gpC7HX/ZmpOBZN1R2Vv0TgymbLq7MH/yNp3h3DQTAyDmbOBpzlYFNKlFGch8JH5MnAuEVhtR/SWk1DwJNRoIDTFROTdVgNhqYfm8LImqUoUxwxg+2+9uH5+GOijm2/tRL+oLJlgdI0gFMNs9jVsA7vGP+iGLkfzVvdoVxsksZkWTxXjdV3DULADYZmBYFQJ4IhFekpWUoHxKYYfszvetSpngAtzevjMlooGu98mw8mrHmscGg+HxUax78Ykuu75tMAHNtfZhr60054njZ/CUDjZsYaNwEwEJbezbb6/ONrQd2L33veTnTiuX0rqVYsdg0xcxGAkx5v19hl9YU/kUCgfCKtK6hzP3nwQEmHu9WO8O25tVK065WWXo2qOBMA515TCH3FDGU4THLU7Sy/sPDpsWU5Cr9DZsYZNzAi6Z5XKEYe+3hPGl5PF+1EtIytrrScLKjm6hL3XLMHd0my/6TF69lSH2dE3kgEAXBp11DSqm+Sql/lFKHlVKTXOwfpZSKUUrtSP15yJftEb6T1p3uyRz7ILORBWPb81DnWvRpVBGA8iUDCQky8eHwlqyZ2M1lER1PbdX1eMTyNMMtL1A7+StetYwggWBC1WW6GHezPegR3jbP9Om6hDUHYxg3LzLL9lfcPE2kl37I5Wx8EhO/30my1eat5gmRgc+eCJRSRuBDoBcQDWxRSi3UWmf+f8K3WuvxvmqHKBhpaa2HtKyap/MDTUZ2v9TH+T79iubyIYGcz0fN39m2Acy2OYrGt1CHeNS0kNsMGxgatIaVtua8ar0Phea0DiWRoDzfJ7Ole895dNwT87djMijevae5y/2Tf93Dn/vO0aNBefo29k5xHyHS82XXUBvgsNb6KIBSagFwO+DZVyJxQ6lQMoioaQO8dj2dbsFY06qlGXdrLe6auSHf192u6zDW8i9qq2hmmD+gu3EH3Y2OGUx2rZhuu5P3rEPwVS/9n/uyBoffdp4GyBAILlxxpNA+dP5yur+EjBwI3/BlIKgCnEz3Phpo6+K4IUqpLsBB4Gmt9UkXxwg/ExJ0ffqn0XA982l2Zt7Xkke+2ubx9Q/rqvRNeZMKXOQR029E6zBaGI7wlOknBhvWcoZQArByVQfyie021tqb4K0P4vRpK94fdv3DP2384MHPN1//vf46ypqDMQAuF7gJ4Q2+HCNw9c82cwfyb0C41ropsBz40uWFlBqrlIpUSkXGxMR4uZmiKPpoREvn67RuoqOv9+fYG/0zHPfSbQ059Fo/+jauxNcPufqe4d45yvKy9QHm2AbwhGU8n1v7EKISaWs4gA0DDQwnmBcwjX8CR9HJkLd6Bu6kX0/x8FzHmMKqf67/G08LApD3MBSfaJEqacItXz4RRAPp01BWBU6nP0BrnX4e4SzgTVcX0lp/CnwKEBERIf+i/UCFktf76ksHO54ODKkj0pEv9GTj0Vj6Na6EQV2futqxdli+7qkx8LL1AV62PkAJrnGFYIqTyATTT4wzLeKrgDf43NqHNfambLQ38Op4AkCSxf1gcObKadmJvnSNeRuO85++9Vn1z3nGfBnJCwMa8FDnWt5oprgJ+fKJYAtQRylVUykVAAwDFqY/QCmVfuRrEOB5HUThN14Y2DDD+7ASgQxsWhmjQXn84QgQmosVuldwTPG8SjHesI5gaPJkYnUID5qW8nnAW+wPGk0/wyaPr+eJqNhrXrnOE/O388mao+w9ncCYLx1PGa7GJoRI47NAoLW2AuOBpTg+4L/TWu9VSk1VSg1KPWyCUmqvUmonMAEY5av2iBtXiVwUwdn+Yi+2vtCTbx7O2k00Y3hLF2d4JlLXp23yh4xJ+RfPW8aw316djwPe57+mTxhlXEIpL+U6mrXmaLb7Hp4bSfw1CxeuZJ1BZbXZnd0/20/EARkH3D2xZM8ZTqQGo8QUG+GTFrHqwPlcXUPcmHy6oExrvRhYnGnb5HSvnwOe82UbhH9Jy8vToUQg88a0YeQcx8CrN2Y0WTGxwu4o3fmLrSMfmD/gbtNfALxknstvtnb8bmvHCntLrHn8v9Zri90/FDeb+ifgKBf6YMdwgsxGPlx1mPeWH+K1Oxszom0N57G5TYb3yFfbCA4wsm9qX6anZlJ98IstXp0NJoomWVksiqxxt9bik7+y/4ack851ymXZdk9EtQzF6PPqGkGMsUwk3HqGNoYDDDRs5Daj42evvQafWAeywd6IGErhi2mf7yw7yDvLDtImvCyboy4CjqeJaX8ccB6zYv/17iBXLbDbNWPnRTKmUy3a3xLq+L1SHOMUFqv303uLoksCgSiynuvXgOf6NcjXNTb/Xw+uJFmd73s0KJ8lEPRsUJ7l+89TtngAF6+m5Or6UboSUbZKfGfrhslipa9hC1PMc5ke8CEAydrMGntTXrMOJ0p7fzFYWhCArGMMH6y8Xihn0zHHcdtOXKJZ1dIYDYqEJAvL959nS9Qldk7pneFcmZHhXyQQiJta+ZAgyqcrf9C7UUXa1CzL5mPXP0BnP9CakxevERJkovnUZXm+lxUTv9vbszS5NW0M+6mvTnKLOsVQ4xq6BOzkJesDzLf1yM+vky9bj19kyMeORXlR0wY4C+rkpvRmitVOXGIK5UO8O2NKFC5JQy38zqz7I3gjU13lamWDKR2c/ayio6/3z3ZfZhZMrLc3YY6tP89bH+aelMkc0NV5wzyHT83/o4NhDxWJpZY67ZMKa9lJCwLgSKX9/E97ALic7okpJw0mL6HNayuw+aIajyg08kQg/E6pYmbubVOdmX8d8agoTt9GFZ1rGNLr3bCCR9Myt+s6jEx5jnGm3xhtXEJv41bnvnO6NHvsNamhzvGq9T4i7XW5QjF8nU7ikzVHWb4/+7Zn95CQFgDOJiRhMijneo/EFBsNJi/hP33r82jXW7zeXuFbEgiE3/prYrccj6kZVpyZI1tl2d65Thif3h9By1eWeTSukEBx3rIO41PrQB4z/UoQKRzWVehp2EZXww6MSvNFwH+dx39nvZXnrA9hw5i7X8pDP2yNzvD+5MXr4wtj50bmGOA6TlsJXJ+NFZfo+Bt88fcxCQQ3IOkaEiKd78a1ByCiRhkg+2pk04e1AKBLHcdq5rfvaubR9eMpwRvWEUyxPsg8W28esEziluSvGZz8Ei9bRjLb2o8YXZK7TX/xkfl9gsh71lV3Dp/PuO6h839XOV9nDgK7ouOYsfIQcdeyBrzMfx/lhSeZ+ERLhjEc4XsSCIRIp03Nsvz4aAc+SX0K6NWwgnNf+gRxaYV4RneqSYDJ4AwIebVN1+VzWz9etY6kdfJMplgeoJdhK+sCn+RV0xyaqiP5un5+DJqxnrf/PMhLC/dm2ffE/O1A7grohE9alCHxXmYPfxnJ3Z9sIDFF6i8UFAkEQmTSqkYZQksEEvlCTyalm756e/MqzoL2KvX/OU2rlubgq/0oXzKIkCDXPa21wornug1f2vpwd8qLHNcVuM+0goWBL7IucAJt1H7KEcf9xqU8ZfqBPobNVKFgEjH+suN0lm2/Z6rWlpbx439//uP2w96dPafjgdzNZsqPuGspGbrG/JGMEQiRjbASgVm29WtSid92nqZEQNb/6yx6ojM7o+Oc35IBvh3bjud/zlvW0khdnyEpL1OGBN4xf0w3406+C3zF5bF/2Fqz0d6QqwSxz16DfTo8T/fMC7v9ejKLtM/utDUMv+86zcCmlXN1vbRrZJdGSmvNh6sOM7xtDcrmIn9Udrq+vZq4axa/XkEtgUCIXPjfXc2YcltDl7OIqocGUz002BkI0j5YaoaV4EjM1Tzf8xIledDyHypYLjLI+DfBJLPE3ppjuhLdDDu43/gnXQ076Wfc4jxngbUrr1nv4zKe10fOK6tdO8cKziYk8euOU85947/Znm0gWHsoxuXq78TULKx7Tye4rEOx8ehF3v7zILui42lZowyXrqXwXL8GnE9IYu/pBLrVL5+r9sdds+Tq+JuRBAIhciHAZHD5pODOu/c0Y8ORWNrWDHXmCsqLc5Rllm1ghm1L7a1Zam9NICmUU3GU5BrDjSu4z7SCAcZNXKEYF3RJ5tt6cFUHkkQAW+z1uUgI3pqi+tS323m+//UutPQ1FgDir1kIMBloMHkJL6bLJDtyzuYs38ITkq5/KG8+djFLIFiy54yz3ddSbM6UGgObVOa2GesA7+SVyi3H4LumdvrVizcQCQRCeNm8MW04dO76rJyQIDO9G1UE4NGut/Dxau8P/CYTQLR2fBN+wTqGn2ydGW1aQkmu0tRwlNfNczIcf1qX5aouxikdxvOWMZwm74Pdi3efpZ+bWsrpg98rv7uvVHvxyvWZSW8t/YfHu9V2vt93OoFHvtrmMhtt+qeQvw9foEM+a1PkVs93HMkHb9TuJQkEQnhZ5zrlXHZ5ADSsVBKAjrVDebZ3PZpUKUXt//vD623YpuuyzVIXgGIkUUVdIEwlUIxkGqgTNDMcIUzF08ZwgLWBT/KFrS+fWgdwDvclQbOTflwkP8ymjPNXUqx2zEbF0r1nCUjddyXZsRI6fZrt2euOOV8fuXDVGQguXk3BrnWWp7g7P1rP4JZVGdnuerbWZKujSyrQ5Ju1G57aEnWR6EvXuLNF1QK7pwQCIQpQ2kdXmeAAWlQvg70AUjUkEsRhXZXDqbdaRQtInZnZQB1nvOlnHjQu4T7jMqZZ7+WELk+0LschXRV7AU0svJxkYf7mEzSpUjrD9rov/EH9iiEcOHvZ42vFp1vv0PIVR+6ozN/Ut5+IY/uJOAa3qOLc1mTKn6TY7M7kg4sndKZh5ZJ5+XXy5a6ZjlQgEgiEuEmlDaqmVVYzGBTzxrShXoUQxs/fnmEh1dYXetLq1eU+bc9+XYPHLU9RTZ3jv6ZZTDHPc+6L18GssLdko70Bm+wNOK4r4O3UFztPxhEeWtzt2EluggDA238e5PFutTNUr5v51xEeuTXriue3lv7jfJ2WhC9tpXjk8Yt5CgTx1ywci71K82qlcz64iJBAIEQBCg91rCloke5DIq0bqXb5EhkCQWim7ozu9cuz0sOKYU2qlGL3qXiP23VSV2CkZRJ9bJFcoRhlSWCIcQ09DdsYbHQMwiZrx8fFt7ZuzLDewXnKeHz97Pznx125/qD3RMzlZCzpnram/XEABYzLFAzcJdzLvIzhSrKV//ywi5dvb+R2wsB9czax+1T8DTVeIIFAiALUrFppVj3blfDQrNM6Jw9sSJ9GFXngs80uz721bjk+Hdkqw5hCxZJBnE1IynJs4yolcxUIwJFGe5G9nfP9z/bOgKa2OkV7wz7qq5NUURe437SM+03LSNDBHNGV2WqvQ6S9HpvzMBspP0HgqJspuRa7duZDSvPGHwdoVaMMzdIFYavd8+yv3205yaLdZygXEshLgxq5PMZu186/u9aaQ+evULeCYybR1uMXuaVcCbdZbtP7bedpbmuWuzUYeSWBQIgCVjOblcZBZiO31i3H3NFtqJEaKFY925WywQHsOR1P+1qhzvUL5UMC+eGRDlQPDeZcQhI/bTvFZ+uPEXM5maplijHltkbM35x9Jbb/Dm3Kv3/Y5UFrlWN8wXa9v7qW9TTdDdupqmJoZjjKCOMKHjI5glOiDiCWkhQnib32GrxkfYDD2jd93a5qN6e5lE0iwKEzN2R4/6uL1dJpbHbNoXOXqVY2mGSrHWPq3/7XHacoExzA+O61ndvS1Hr+emXejtNWcjo+iS8ebM2tdcsx5OMNNKpckkUTOme4x+uL9/Nw51pULJWxxsMv209JIBDCX3Wpe33GUVrQ6JhuOuSR1/tjUNfHGSqUDOLRrrfQq2EFer7zFw0qlSTIfH3my3P96vNGuhKW9SqEcHdENQ8DQVZHdWWO2q5/QBmx0d6wj9rqFLXUGUqqq5ThCs0Nh/k5YAq/2dqzTdchUQeywd6Qi3hnANZiy36gfeAH6/J9/Y1HY5mabrrrXa0cAe3SNQvvLj9IpdJBNKpckkaVS7k8/3S840nt2IWrdEr977f3dAKz1x5lTKeaKKXYdCyWOeuOcfDcZeaNaZvh/JgryXy8+gj3tqlG86nLeL5/fcZ28U1mVwkEQtxgMn8LTVO7fAlmDG/hDCR7X+7Dqn/OM7BpZcZ0qsm/f9zFT9tOUSKbnEgA0+9twYRcTgW1YWSdvQnryFjsp6o6z/OmbxhsXMtwdb2bJtJel7O6DAfs1TmqK5FAcS7rYhix08e4hYbqOBvtDZlv6+61oJEXmTO0fp8pdXdaIN34nPuqcy//to9Kpa7XvXh10X5a1ihDy+plnOMQrgr97IqOZ1d0PG8ucQTx1xcfkEAghMhZ+nQOxQNNzvcmo4Hhbarz07ZTBAc4nha+G9eeiiWDmLX2KPM2Huf25pUZ1Kwyh85dzlDvOK+idXkeszxFICmUV5eozEV6GrfSx7CFcMNZBho3ZTnHqg0c1ZWYaP6OR0y/scHekHIqnpJcxY6BnfoWVtuascHekFhcfxP3lqMXPEsL8vDcyByPeeSrrRnep1jt/LL9FKWCzQD8fSQ29w30IpVdvvWiKiIiQkdG5vyHF0JkZLdr3l1+kJHta2SoOZxstfHiL3t4tk895/bvI09Sq1zxDOUt0xgUeGP5QymuUEFdoiRXCVGJFCeJTfb6xFCGhiqKp00/Uk+dIJ7inNZhVFKx1FPRBCoLSdrMYV2FCuoiAVgJwsJBXYUfbLfyh62NV2Y0FYZa5Yq7HQQ/+np/l3muPKGU2qq1jnC5TwKBECI7rlJJT+hem+k5PDFM7FOPtYdi2HjUuwVmgkimoTrOPcbVVFQXiaE0V3UgAVhpYjhGY0MUAAk6GA0c05VYbmvJCV2eIJVCdXWeyzqYi4Rw1F6JQ7oqKZhIInf5owrLtMFNGNamep7OdRcIpGtICJGttOmpUdMGOIPC491rM6FHHZbsPcv4b1yPJzzerTaPd6ud55oE2Uki0JE+w1rXxV5Nc3WETobdVFQXsWOgteEAz5q/z/G6e+zhrLE3ZZe9Fnt0ONG6HL6uG50X5xJ8U7FOAoEQIlt/PtOFq6m5fZ7pVZeKpYKcuXg6174+u2nB2HZM+nEXBqX48+kuzu0mg8JaAGk0HBQ7dG122Gpn2FqcRCqpWGwYOanLEUwSISTSynCIcuoSZdVlOht287BxEWaTI/dGgg7mtA4lRF1jnz2cw7oyYcRTXsWRRAAlSCQZM3YUldRFkjHzl60Zi+1tOZSn6bKaQCwk436NwZn4xDxcO2fSNSSEyLMNR2JpVKUkJYPMLvcv2XOGR77almHbQ51qcvFqCj9tP+XyHICPRrTksa8znrdzSm+6vb3amQLC2wJJob46QTvDfoYY13BcVyQFI43Ucaqp88RSivO6NIFYuEYgxUjGiolzujQhKpEIw0EAVthasNbehLO6LKvszbN8uAeTRIThHyqqi7RQh6lnOEldFU0JlcRblrv50HaH23bmdcWydA0JIXyi/S2hbvf3SU2/nV6H2qF0r1+BVf+c51K6ojCdaoex7vAFAPo3yZrWulQxM6sndqXpS3mv6eBOMgHs1LXZaavNJ7bbMuwzYsOG+6yk4eoMdxjXc49xNT2Mji4zizYSR3ECsVJSXeOCLkkprmJWjiePOF2cA7o6P9i6Bc1l6QAAC2ZJREFUEKtLstHewN0tCDT5JgmgBAIhhM8opQgPDSYq9hqLJnRi4Y7T3FrXUTdhZPtwpq845Dx21v0RNJi8xPl+zgMRjPky49N/ySAzh1/rlyV19zcPtaVBpZIEmg00nLw0w76XBzViysK9+fo9cgoCAFG6Eu9Zh/KedQjliKe+4QRtDfupqc4Qo0tTSl0lmGSO6MpsttfnpC7HEV2Z3IxFpF8o6E0SCIQQPjV/bDtWHYihUeVSGVbhjutSyxkIJnSvTbHU9Q1d6znGHno0qMDfk7rTYdpKFqdLy2AyGtjzch/OxCXS6901/PhoB1rVcD1dtG6FEjzQIZzNxy6yaPcZX/2KTp3rhLH20AViKE2MvTRr7U29ev0gs2+eCAom2bgQwm9VKlWM4W2zTnksHmji1TsaA3B362oAHHujP5+Pau08pnLpYkRNG5AlHXSJQBN1KoQQNW1AtkEAYHz3OgB8OKIlE/vUo3iAEVPqPPy6FUo4jzvwSl+OvdGfd+9pluUaLwxoQNS0AR71zbfwcerp9CuUvUkGi4UQhUZrTbLV7tUuj8/WHaNcSCCNKpekVrkSWfZvOBLLG3/s54dHOmCza66mWDOklU4/5fXtu5oxtNX1WUBbj19iyMd/AzC+W21CgkxUKxvM8n3nmHpHY/4+fIGx8xyriP93VzP+9f3OfP8+x97oT83nHMnsdk7u7VyNnFsyWCyEKJKUUl7v9x7dqabb/e1vCWXh+E7O92ldUpm9MbhJhiAA0KCSI6X0gx3DebZPPef2tMHtHg0qOLfd3rwyb/yxn4QkKwdf7Qe4XqAHsPn5Hmw4GsuTC3Zk2F4zrHiGAjt5DQI5kUAghBDpdLgllGFtqjPIRQro4AATO6f0pkSg649Oo0Fl6ELa/HxPj+4ZWiKQ25tXYeNRR73ip3rWYcjHG2gT7qghvfyZWzkbn7XuhLdI15AQQhSQHSfjOJeQxLh5WykRaGLlv25F40glntmSPWfoWq+8156YCq1rSCnVF3gfMAKztdbTMu0PBOYCrYBY4B6tdZQv2ySEEIWlebXSaK15umddhkZUpbyLAJCmb+Osayl8xWezhpRSRuBDoB/QELhXKdUw02FjgEta69rAu8CbvmqPEEIUBUopnuxZhyqlfTMDKC98OX20DXBYa31Ua50CLABuz3TM7cCXqa9/AHqo9CMjQgghfM6XgaAKkL5oanTqNpfHaK2tQDyQZc26UmqsUipSKRUZExPjo+YKIYR/8mUgcPXNPvPItCfHoLX+VGsdobWOKFeunItThBBC5JUvA0E0UC3d+6rA6eyOUUqZgFKAdytZCCGEcMuXgWALUEcpVVMpFQAMAxZmOmYh8EDq66HASn2jzWcVQogbnM+mj2qtrUqp8cBSHNNHP9Na71VKTQX+v71zj5WrqsL470sL5SltKZomGNomiKJR2ghSQdIoViGEGBNDq4mNaFR8ojGklYTE/1ATgyZGID5jSkVBkDRoIUgl1tBSSltaa+lVamykDx+Aj2gQln/sNenp9M7tve3M3F3O90tOzj5r9tn7m9kzs87Z55y1NkbEfcB3gB9KGqGcCSwZlB5jjDGjM9DnCCLifuD+LttNjfJ/gPcNUoMxxpixcfRRY4xpOcddiAlJB4A/HuXus4C/9FHOIKhdY+36oH6NteuD+jXWrg/q03hORIx62+Vx5wiOBUkbe8XaqIXaNdauD+rXWLs+qF9j7frg+NDYwVNDxhjTcuwIjDGm5bTNEdw+2QLGQe0aa9cH9WusXR/Ur7F2fXB8aARado3AGGPM4bTtjMAYY0wXdgTGGNNyWuMIJL1b0k5JI5KWD7Hf70raL2lbwzZT0oOSduV6Rtol6RupcaukBY19lmX9XZKWjdbXMWh8taSHJe2QtF3SZ2vSKekkSRskbUl9X0r7XEnrs687M6YVkqbl9ki+PqfR1oq075T0rn7oa7Q9RdITklZXqm+3pCclbZa0MW1VjHG2O13SXZJ+l9/FhZXpOy8/u87yvKTra9J41ETEy36hxDr6PTAPOBHYApw/pL4vAxYA2xq2rwDLs7wc+HKWrwR+TgnPfTGwPu0zgT/kekaWZ/RR42xgQZZPB56iZJWrQmf2c1qWTwDWZ78/Bpak/Vbguix/Arg1y0uAO7N8fo79NGBufiem9PFz/DxwB7A6t2vTtxuY1WWrYoyz7R8AH8nyicD0mvR1aZ0C7AXOqVXjhN7PZHY+tDcJC4E1je0VwIoh9j+HQx3BTmB2lmcDO7N8G7C0ux6wFLitYT+k3gD0/gx4Z406gVOATcBbKE9tTu0eY0qgw4VZnpr11D3uzXp90HU28BDwdmB19leNvmxvN4c7girGGHgF8DR5A0tt+kbRuxhYV7PGiSxtmRoaT7a0YfKqiHgGINevTHsvnUPTn9MU8ylH3dXozGmXzcB+4EHK0fKzUTLbdffVK/PdID/HW4AbgJdy+8zK9EFJ+vSApMclfTRttYzxPOAA8L2cXvu2pFMr0tfNEmBVlmvVOG7a4gjGlQmtAnrpHIp+SacBdwPXR8TzY1XtoWdgOiPixYi4gHLkfRHwujH6Gqo+SVcB+yPi8aZ5jL4ma5wviYgFwBXAJyVdNkbdYWucSplC/VZEzAf+RZlm6cWk/VbyWs/VwE+OVLWHlur+j9riCMaTLW2Y7JM0GyDX+9PeS+fA9Us6geIEVkbET2vVGRHPAmspc67TVTLbdffVK/PdoPRdAlwtaTfwI8r00C0V6QMgIv6c6/3APRSHWssY7wH2RMT63L6L4hhq0dfkCmBTROzL7Ro1Toi2OILxZEsbJs3MbMsoc/Id+wfzboOLgefyVHMNsFjSjLwjYXHa+oIkUZIE7YiIr9WmU9JZkqZn+WTgcmAH8DAls91o+kbLfHcfsCTv2pkLnAtsOFZ9EbEiIs6OiDmU79YvI+IDtegDkHSqpNM7ZcrYbKOSMY6IvcCfJJ2XpncAv61FXxdLOTgt1NFSm8aJMZkXKIa5UK7gP0WZW75xiP2uAp4BXqAcCXyYMh/8ELAr1zOzroBvpsYngTc32rkWGMnlQ33WeCnl1HQrsDmXK2vRCbwReCL1bQNuSvs8yh/lCOU0fVraT8rtkXx9XqOtG1P3TuCKAYz3Ig7eNVSNvtSyJZftnd9ALWOc7V4AbMxxvpdyR001+rLtU4C/Amc0bFVpPJrFISaMMabltGVqyBhjTA/sCIwxpuXYERhjTMuxIzDGmJZjR2CMMS3HjsC0Fkn/zPUcSe/vc9tf7Nr+TT/bN6af2BEYU4ICTsgRSJpyhCqHOIKIeOsENRkzNOwIjIGbgbdljPnPZYC7r0p6LOPIfwxA0iKVvA13UB4QQtK9GcRteyeQm6SbgZOzvZVp65x9KNveppIb4JpG22t1MB7/ynzi25iBM/XIVYx52bMc+EJEXAWQf+jPRcSFkqYB6yQ9kHUvAt4QEU/n9rUR8bcMffGYpLsjYrmkT0UJktfNeylP0L4JmJX7PJKvzQdeT4k7s44Sw+jX/X+7xhyKzwiMOZzFlBgxmynhuM+kxP0B2NBwAgCfkbQFeJQSSOxcxuZSYFWUaKr7gF8BFzba3hMRL1HCfMzpy7sx5gj4jMCYwxHw6Yg4JBCYpEWU8MjN7cspyWP+LWktJY7QkdruxX8b5Rfx79MMCZ8RGAP/oKTo7LAGuC5DcyPpNRmxs5szgL+nE3gtJTR2hxc6+3fxCHBNXoc4i5LKtC8RRo05WnzEYUyJdvm/nOL5PvB1yrTMprxgewB4zyj7/QL4uKStlGihjzZeux3YKmlTlJDUHe6hpK3cQon4ekNE7E1HYsyk4OijxhjTcjw1ZIwxLceOwBhjWo4dgTHGtBw7AmOMaTl2BMYY03LsCIwxpuXYERhjTMv5P/9CKvbQ3WESAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(cost_list, label='Minibatch cost')\n",
    "plt.plot(np.convolve(cost_list, \n",
    "                     np.ones(200,)/200, mode='valid'), \n",
    "         label='Running average')\n",
    "\n",
    "plt.ylabel('Cross Entropy')\n",
    "plt.xlabel('Iteration')\n",
    "plt.legend()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEICAYAAACwDehOAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAgAElEQVR4nO3dd3xV9f348dc7OyGTLAIBw4awMQwFB+LCAUpxUCdqqbau+vWnaG3VTtpaR2u1RcWJIoIU3HXgQGWFEZbsACGBDDIgO7mf3x+fSwwQQuY9l+T9fDzu49577rnnvO+BnPc5nynGGJRSSikAH6cDUEop5T00KSillKqhSUEppVQNTQpKKaVqaFJQSilVQ5OCUkqpGq2WFERktohki8iGWss6isinIrLN/RzlXi4i8g8R2S4iaSIyvLXiUkopdWLSWv0URORs4DDwmjFmoHvZX4GDxpiZIjIDiDLGPCgilwB3AZcAo4BnjDGjTraPmJgYk5SU1CrxK6VUW5WampprjImt6zO/1tqpMeZrEUk6ZvEk4Fz361eBL4EH3ctfMzZDLRORSBFJMMZk1bePpKQkVq1a1ZJhK6VUmyciu0/0mafrFOKPnOjdz3Hu5V2AvbXWy3AvU0op5UHeUtEsdSyrs1xLRKaLyCoRWZWTk9PKYSmlVPvi6aRwQEQSANzP2e7lGUDXWuslApl1bcAYM8sYk2KMSYmNrbNITCmlVBO1Wp3CCSwGbgJmup8X1Vp+p4jMxVY0F56sPuFEKisrycjIoKysrCXiVUBQUBCJiYn4+/s7HYpSqpW1WlIQkbewlcoxIpIBPIpNBvNE5FZgD3CVe/UPsS2PtgMlwLSm7jcjI4OwsDCSkpIQqatUSjWGMYa8vDwyMjLo3r270+EopVpZa7Y+mnqCj8bXsa4BftkS+y0rK9OE0IJEhOjoaLT+Rqn2wVsqmluUJoSWpcdTqfajTSYFpZRqqwpKKvjrxz+wO6+4Vbbv6YrmNi8vL4/x420J2f79+/H19eVIK6kVK1YQEBBw0m1MmzaNGTNm0Ldv3xOu869//YvIyEiuu+66lglcKeXVisoqeembXcxeuotD5VUkRAZzQ3SHFt+PJoUWFh0dzdq1awF47LHHCA0N5f777z9qHWMMxhh8fOq+UXv55ZdPup9f/rJFqmCUUh5mjGHvwVIqXS66R3fAx6f+4tni8ipe+S6dWV/vpLC0kosGxPOrC/rQr1N4q8SnScFDtm/fzhVXXMHYsWNZvnw577//Po8//jirV6+mtLSUa665ht/+9rcAjB07lmeffZaBAwcSExPD7bffzkcffURISAiLFi0iLi6ORx55hJiYGO69917Gjh3L2LFj+eKLLygsLOTll1/mzDPPpLi4mBtvvJHt27eTnJzMtm3bePHFFxk6dKjDR0Op9qOiysXGzEJSd+ezKj2f1D355BwqByAi2J+hXSMZ3i2K4adFMqRrJOFBtul3WWU1r3+/m39/tYO84grO6xfHfRf0YWCXiFaNt00nhcff28imzKIW3WZy53AevXxAk767adMmXn75Zf79738DMHPmTDp27EhVVRXjxo1jypQpJCcnH/WdwsJCzjnnHGbOnMl9993H7NmzmTFjxnHbNsawYsUKFi9ezO9+9zs+/vhj/vnPf9KpUycWLFjAunXrGD5cB59VqrWVVlSzMv0gy3bmsWp3PmkZBZRVugDo2jGYsb1iOP20KAJ8fVizN5/Vuwt4+vOtGAMi0DsulIFdIli6LZfsQ+Wc1TuGX13Qh+HdojwSf5tOCt6mZ8+ejBgxoub9W2+9xUsvvURVVRWZmZls2rTpuKQQHBzMhAkTADj99NP55ptv6tz25MmTa9ZJT08HYOnSpTz44IMADBkyhAEDmpbMlDpVGWMa3HruUFkl6zMKWZtRwLq9BWzZf4hOEUEM6BzBwC7hDOgcQY+YDvj5Hl3sW1XtYl1GId9tz2Xp9lzW7CmgotqFn48woHM4Px15GilJUZx+WhTx4UFHfffqEXYgh6KyStbtLWDNngJW78nnyy059IkP5Z9ThzGqR3TLHIwGatNJoalX9K2lQ4cfK4W2bdvGM888w4oVK4iMjOT666+vsxd27YppX19fqqqq6tx2YGDgceu01rDoSnm7DfsKeXjhejZnFREbGkhseBBxYYHEhwcSF2Zfx4QGklVYytq9hazLKGBHzmGO/MkkRYfQPyGcrMIy3li2m/Iqe6Uf5O9Dv07hDOgcTufIYFbvzmf5roMcLq9CBJITwrl5TBJn9oxmZPeOhAQ07BQbHuTPWb1jOau380P3tOmk4M2KiooICwsjPDycrKwsPvnkEy6++OIW3cfYsWOZN28eZ511FuvXr2fTpk0tun2lvE1pRTVPfbaVF7/ZSXRoIDedkcTBkgpyDpWzJ6+EVekHyS+pPOo7MaEBDEmMZOKQzgzpGsngLhFEdfjxYqyq2sXO3GI2ZhayYV8RGzMLWbwuk0NlVSRFhzBxaGfG9IzhjJ7RdOxw8taF3k6TgkOGDx9OcnIyAwcOpEePHowZM6bF93HXXXdx4403MnjwYIYPH87AgQOJiGjdSiqlGqOwtJKi0kqqXYYql6G61qPK5cLf14d+ncKOK7Kpyzfbcnh44Xr2Hixl6shuzJjQj4jg48frKq+qJvdwBdlFZcSGBdIlMrjeIiY/Xx/6xIfRJz6MK4fZZcYYikqriAhpe+OBtdrMa56QkpJijp1kZ/PmzfTv39+hiLxLVVUVVVVVBAUFsW3bNi688EK2bduGn1/jrwX0uKqW9s6qvTy8cD2V1fWfg8KC/BjbK4Zz+sRyTt9YEiKCj/o8v7iCP3ywmQWrM+gR04E/TR7EaA+Xw59qRCTVGJNS12d6p9CGHT58mPHjx1NVVYUxhv/85z9NSghKtSRjDM99uYO/fbKFMb2iuWJoF/x8BV8fH/x8BB8R/HwEX1/hcFkV327P5autOXy0YT8AfePDOKdvLOf0iSX3cDm/e28ThaWV3DmuF3ee14sgf1+Hf+GpTc8QbVhkZCSpqalOh6FUjWqX4bHFG3l92W6uGNqZv04ZQoBf/UVDlw/pjDGGbdmH+XJLNl9tzeGVb21nLoAhiRG8cdso+ie0Tmeu9kaTglLKI8oqq7l37lo+3rif6Wf3YMbF/U7am/cIEakp159+dk+Ky6tYtjOP4opqLh2UgG8Dt6NOTpOCUqrVFZZU8rPXVrEi/SC/uSyZW8c2b26ODoF+jO8f30LRqdo0KSilGq2iysU323L4IC2L4ooqBidGMrRrJIMSI2qGaTgiq7CUm2avYFduMf+YOoyJQzo7FLVqCE0KSrVzDe31W1Xt4vudeby/LouPNmRRVFZFRLA/USH+fLLxAGCHaegZG8qQxEiGdo2gS1Qwv164gUNlVbw6bSRn9opp7Z+jmkmTQgs799xzeeihh7joootqlj399NNs3bqV5557rs7vhIaGcvjwYTIzM7n77ruZP39+ndt94oknSEmpsxVZzX6mT59OSEgIAJdccglvvvkmkZGRzfxVqi3ak1fC3z/dwofrs4gKCaBLVDCdI4NJjLTPXdzPh8oqeT8tiw/XZ5FXXEFooB8XJsdz+ZDOjOkVQ4CfDwUlFaRlFLJubwHrMgr4ams2C1ZnABAXFsi8n59BcmetCD4VaFJoYVOnTmXu3LlHJYW5c+fyt7/97aTf7dy5c50JoaGefvpprr/++pqk8OGHHzZ5W6rtyjlUzrNfbOPNFXvw9RGmnN6VqmoX+wpK2bivkE83HqCi2nXUd4L8fRjfL57LhyRwbt+445p9RoYEcHafWM7uY4dpMMaQWVjGpswihnSNIC7s6DF/lPfSpNDCpkyZwiOPPEJ5eTmBgYGkp6eTmZnJ0KFDGT9+PPn5+VRWVvKHP/yBSZMmHfXd9PR0LrvsMjZs2EBpaSnTpk1j06ZN9O/fn9LS0pr17rjjDlauXElpaSlTpkzh8ccf5x//+AeZmZmMGzeOmJgYlixZQlJSEqtWrSImJoYnn3yS2bNnA3Dbbbdx7733kp6ezoQJExg7dizfffcdXbp0YdGiRQQHH905SLUNh8oqeeHrnby4dBflVS6uGdGVe8b3Pm6QNpfLkFtczr78UjILyhCBs/vEEhrY8NOFiNDFfbehTi1tOyl8NAP2r2/ZbXYaBBNmnvDj6OhoRo4cyccff8ykSZOYO3cu11xzDcHBwSxcuJDw8HByc3MZPXo0EydOPGFZ7vPPP09ISAhpaWmkpaUdNez1H//4Rzp27Eh1dTXjx48nLS2Nu+++myeffJIlS5YQE3N0uW1qaiovv/wyy5cvxxjDqFGjOOecc4iKimLbtm289dZbvPDCC1x99dUsWLCA66+/vmWOlfIKZZXVvLFsN/9asp38kkouHZzA/Rf2pXtM3bN2+fiIe9C4IIZ183CwynGOJAURuQf4GSDAC8aYp0WkI/A2kASkA1cbY/KdiK+5jhQhHUkKs2fPxhjDww8/zNdff42Pjw/79u3jwIEDdOrUqc5tfP3119x9990ADB48mMGDB9d8Nm/ePGbNmkVVVRVZWVls2rTpqM+PtXTpUq688sqaUVonT57MN998w8SJE+nevXvNpDu1h91Wp6bC0kp25Bxme/ZhduQcZkd2MesyCshxj8v/wEX9GJSo41+pE/N4UhCRgdiEMBKoAD4WkQ/cyz43xswUkRnADODBZu2sniv61nTFFVdw33331cyqNnz4cF555RVycnJITU3F39+fpKSkOofKrq2uu4hdu3bxxBNPsHLlSqKiorj55ptPup36xrc6MuQ22GG3axdTKe9W7TKsSj/IJxsPsCmrkO3ZxeQeLq/53N9X6B7TgRFJUVw36jTGaMsf1QBO3Cn0B5YZY0oAROQr4EpgEnCue51XgS9pblJwSGhoKOeeey633HILU6dOBewManFxcfj7+7NkyRJ2795d7zbOPvts5syZw7hx49iwYQNpaWmAHXK7Q4cOREREcODAAT766CPOPfdcAMLCwjh06NBxxUdnn302N998MzNmzMAYw8KFC3n99ddb/oerVldV7WL5roN8uD6LTzbuJ/dwBYF+PgzoHM64vrH0jAulV2woPeNC6RoV3KDRRZWqzYmksAH4o4hEA6XAJcAqIN4YkwVgjMkSkbi6viwi04HpAN26eW+B59SpU5k8eTJz584F4LrrruPyyy8nJSWFoUOH0q9fv3q/f8cddzBt2jQGDx7M0KFDGTlyJGBnUBs2bBgDBgw4bsjt6dOnM2HCBBISEliyZEnN8uHDh3PzzTfXbOO2225j2LBhWlR0iqisdvHdjjw+cieC/JJKgv19Oa9fHBMGdWJc3zg6NKISWKn6ODJ0tojcCvwSOAxswiaHacaYyFrr5Btj6p2UVIfO9hw9rp5VWe3i2+257juCAxSWVhIa6Md5/eK4ZFAnzukTR3CAjgaqmsbrhs42xrwEvAQgIn8CMoADIpLgvktIALKdiE0pp1RWu/h+Rx4fpGXxyab9FJTYRHBBcjyXDErgrN4xOiy0anVOtT6KM8Zki0g3YDJwBtAduAmY6X5e5ERsSnlaem4x//5qR03RUGigH+f3j+PSwZ01ESiPc6ogcoG7TqES+KUxJl9EZgLz3EVLe4Crmrrxho7lohrmVJ6dz9tlF5Xx0xeWUVBayQXJ8Vw6KIGz+8RqIlCOcar46Kw6luUB45u77aCgIPLy8oiOjtbE0AKMMeTl5REUpMMUtLTSimp+9toq8ksqeef2MxjYRfsPKOe1uSYLiYmJZGRkkJOT43QobUZQUBCJiYlOh9GmuFyG++atJW1fIbNuSNGEoLxGm0sK/v7+dO/evAk8lGptT/xvCx9t2M8jl/bngmSdLEZ5D+3ZopSHvbNqL899uYOpI7s1ewYypVqaJgWlPGjZzjweXriesb1i+N2kAVrvpbyOJgWlPGRXbjG3v5FKt44h/Ou64fjrEBTKC7W5OgWlPC09t5hnl2wnKsSf/gnhJHcOp2ds6FEn/YKSCm55ZSUCzL55BBHB/ifeoFIO0qSgVDN8uukA981bS7XLUOUyVFTZGcsCfH3oHR9KckI4/RPC+WTjfvbllzLnZ6M4LbrueQyU8gaaFJRqgqpqF09+upXnvtzBoC4RPHfdcBIigtiVW8ymrCI2ZRaxKauIL37I5p1UO1fxU9cMYURSR4cjV6p+mhSUaqTcw+Xc/dYavtuRx9SR3Xj08uSaHsi948PoHR/GpKFdANv5L+dQOeVVLrp2DHEybKUaRJOCUo2wek8+v3hjNfklFfxtymCuSula7/oiQly49gZXpw5NCko1gDGG177fzR8+2ERCRDDv/uJMBnTWXsiq7dGkoNq9DfsK+fXC9Ww5cIhgf19CAvwIDvAlJMCXIH/7XFJezYr0g4zvF8eTVw8lIkRbD6m2SZOCarfKKqt56rOtvPjNLqI7BHD9qNMor3JRUlFNaWUVJRXVlFRUc7C4gvJKFw9c3Jfbz+6Jj492OFNtlyYF1S4t25nHQ++uZ1duMdeO6MpDl/TXvgNKoUlBtTNFZZXM/OgH3ly+h24dQ3jztlGc2SvG6bCU8hqaFFS78emmAzzy3/XkHCpn+tk9+NX5fXSeY6WOoUlBtXnGGP7wwWZeWrqLfp3CmHVDCkO6RjodllJeSZOCavOe+XwbLy3dxU1nnMavL00mwE8HolPqRDQpqDbtlW938fRn27jq9EQem6hDVSt1MnrJpNqs/67Zx2PvbeKC5Hj+PHmQJgSlGkCTgmqTlvyQzf3vrGN0j478c+ow/HTuAqUaxJG/FBH5lYhsFJENIvKWiASJSHcRWS4i20TkbREJcCI2depbmX6QO+ak0i8hjBduTKkZrE4pdXIeTwoi0gW4G0gxxgwEfIFrgb8ATxljegP5wK2ejk2d+jZnFXHLKyvpHBHMK9NGEhakHdKUagynKpr9gGARqQRCgCzgPOCn7s9fBR4DnnckOuV1isureOW7dPx9hdOiO5AU3YFuHUOO6mewO6+YG15aQWigH6/fNoqY0EAHI1bq1OTxpGCM2SciTwB7gFLgf0AqUGCMqXKvlgF0qev7IjIdmA7QrVu31g9YOc4YwwPz0/hgfdZxn8WHB7qTRAjf78yj2uVi7vQz6BIZ7ECkSp36PJ4URCQKmAR0BwqAd4AJdaxq6vq+MWYWMAsgJSWlznVU2/LCNzv5YH0WMyb0Y+qIbuw+WEx6Xgl78uzz7rxilmzJAeCVaSPpFRfmcMRKnbqcKD46H9hljMkBEJF3gTOBSBHxc98tJAKZDsSmvMx323OZ+dEPXDKoEz8/uwciwuCQSAYnHt8j2RijzU6VaiYnWh/tAUaLSIjYv+DxwCZgCTDFvc5NwCIHYlNeZF9BKXe+tYaesaH8dcqQk57wNSEo1XweTwrGmOXAfGA1sN4dwyzgQeA+EdkORAMveTo25T3KKqu5441UKqtc/PuG0wkN1M73SnmCI39pxphHgUePWbwTGOlAOMrLGGP47aINpGUUMuuG0+kZG+p0SEq1G9rNU3mdt1bsZd6qDO46rxcXDujkdDhKtSuaFJRXWb0nn0cXb+CcPrHce34fp8NRqt3RglrlMdlFZew+WEJUiD+RIQFEBvsfNSZRzqFy7ngjlYSIYJ65dii+OheyUh6nSUF5RFpGAde9uJxDZVVHLQ8L8iMqJICoEH/ySyopLK3k3TtGEhmiQ18p5QRNCqrVbdhXyPUvLici2J+nrh5KSWU1BSUVHCyuoKCkkvySCvJLKvHz9eGRS/uT3Dnc6ZCVarc0KahWtWFfIde9uJywIH/mTh9NYlSI0yEppeqhFc2q1WzKLOL6l5YTGuinCUGpU4QmBdUqNmcVcd2Lywjx9+Wtn42ma0dNCEqdCjQpqBa3Zf8hrntxOYF+vrw1fTTdojUhKHWq0KSgWtTWA4f46QvL8PcV5k4fzWnRHZwOSSnVCFrRrBrM5TKsyyigpKKaymoXVdWGKpeLSvdzWaWLv/9vC74+wls/G01SjCYEpU41mhRUgxhjeGBBGvNTM+pdLz48kDd/NpoeOl5Rw+Xvhn2rIKYPRPcG/yCnI/pRRTEcPgAde7T8trPSIPUV6HEO9J8IrTXKrasaKkvdj5Ifn6vKIDAMIhIhKLL19l+X4jz73CG68d8t2AuL74Txv4Uup7dsXGhSUA308rfpzE/N4Nax3bkwOR4/Xx/8fQU/Hx8C/Oyzn68QExpIkL/vyTfoTVzVsPk9qK6EvhMg0EMJzRhYNRv+94g9SQGIL0T3hLj+ENvfPsclQ1AElB+C8kIoK3K/LrKvKw5DQAcIja/1iLPfac6JbvN78OEDcCgTOg2CYTfAoKsgpGPzfvfhbPji97D6dRvfqpeg83A4/zGbIFpCyUH47y9gxxdQXX7y9f07QEQXmyDC3c8RXaH3hRAa2zIxARRlwtKnIPVV+9vH/RpG/wJ8G3gqXj8f3r8PTDUU7muVpCDGnLqTl6WkpJhVq1Y5HUabt3RbLje9vILx/eL49/Wn49NWhp8wBrZ+Ap//DrI32mX+HaD/ZTD4GuhxLvi0UoIryoRFd8KOz6HHODj3ISjKgOzNPz4O7uQEExA2jF+QTQ5hnSF5Egy7HoIa0DGwcB98+P9gywcQPwgGTYGNCyFrLfgGQN9LbILoOa5xx6eqHJY9D18/AVWlMPLncPb9sOUjWPIn+/t7ngfjH4XOQ5v+u3O2wFvX2ivqlFsgJBr8g495hIBfIJQV2t9bmGH3X7gPivbZuyOwvzf5Chg5HRJTmp5kCzNsMlj9GhgXDJkKJXmw5UPoPAwmPgudBp74+2WF8MH9sH4eJI6Eyf9p1t2biKQaY1Lq/EyTgqrP7rxiJj77LfHhgbz7izFtZ16DPcvhs0dhz/f2j+u8R+zJM22uPQGWFUJoJ3tCHHyNvVKufUKoKLEnjiOP0gJIHGGv7Os7cRgDGxbAB/dBVQVc+HsYcVvd36kogdytNkFUHLZX/oHhtsgjKPzH14Hh9vMjsRw6cHRsOVvsCT0gDIbfAKN+DlFJx+/PVQ0rXrBX8a5qGPeQ+yrW336+fz2smQNpb0PpQXu8hk6FPhOgY3d78q3rdxgDP7xv74jy06HPxXDhHyGm14/rVJbByhfhmyegNB8GTLb/JtE9G/Kv+aOtn8D8W20R3DVvQLfRjfv+EVXlkLsN1rwOa9+0d2UJQ2xyGPgTm1gaomAvLH0S1rxhk8HQ6+Cs++zxN8b+X/vw/0FZAYy9zyZJv8Cjt5H+LSz8ub2QOOdBOOv/Gn5ncQKaFFSTHC6vYvJz33KgqJzFd45pGy2JsjfbO4MtH9pilnMegOE3/XjiA3uC2vYJpM2zJxlXpS3KCYmGw/tt8Ud5Ud3bjzzNXkn3nQCnnXn0dksO2mSwcaFNIFf+p/Envabal2qv0jcutCenvpfYE/5pZ9oTeVYavHcPZK6GXufDpX+vO3GAPWFu/die6LZ/ZrcHEBBqv1P7ERpnE036NxDbDy76E/Qaf+I4ywrhu3/C9/+C6gp7RzJyOsQn1//7jIFvn4HPHoOEwXDtm7YIqCWUH7aJcMULkLMZgqPsXdfQ621yqCo7uq6issy+3r3UJlGw6591H0R2O377JQfh44fsBUlMX5j0LHQdaS8avvyzvcOISoLJL0DXES3ykzQpqEZzuQx3zEnl000HeO2WUYztHeN0SM1TsNcWUax7y15dj7kHRt9hy+LrU3IQNr4LG/9rr57D4o8uuz/y3j/Envh++BB2fmnLsQMjoPcFNkH4BsCH99vtjXsIzryn2Vd7TVKUaU9uqS/bK/KEIZAw1J7gQzrCxTPtlXBDi0mKsuxdSH768Y+qMrtOcJQtOz99WsN/86ED8PXfbEW0q9LWOQy73sYWfMz83JWlsPhuW7Qy4EqY9BwEtELfGGNg97ewYhZsft+W69fHN8AmtbG/gsiuJ9/+tk/hvXtt8dWIWyFjlT22w26Ai/9s/9+2EE0KqtGe/mwrT3+2jd9clsytY7s7HY5VfshejTamXLesEL550l4lA4z8mb39bm5laX0qimHHEltWvvVjKMm1y+OS7d1BwuDW23dDVZTYq99lz0PuFhh+I5z/eMsdF5cLirNtMo7pffyJvKGK8+zJfvXrtt7HLwj6X24TRNLZ9s5t7nX2Due8R+Cs+z3Tiqhwn71L8vH9sY7CPxj8atVZdIhp/O8uPwSfPQ4rX7DJ9PJ/QPLEFg9fk4JqlI837Of2N1L5yfBEnrhqMOLJpnp1qSiG//3GtlKJS4bTb4bBV9s/mhOprrQte776i63QG3ytPWk05IqtJbmq7RVffjoMuOL48mKnuVy2KKypJ21PMcZeNa+ZY5NEWSFEdLN3ZBXFMHkW9LvU6ShbzoGN0CGuZVs+1aJJQTXYD/uLmPzcd/SOD+Pt6aOdb166Z7mtZMtPt5V02ZvsVaFfkK2MTJlmy+ePJK4jlZqfPgoHd0DSWXDhH5rXmkV5l8oy+2+8do5tkjvxnyevc1BHqS8ptJGmJKqhNuwrZHdeCSUVVZRWVlNSYR+lFVWUVFTz5ZYcQgP9mHXD6c4mhKpyW8n27TO2wvDmDyBpjP0sc60ta17/Dqx7E+IG2LuHuH623mDP97ZS86fzbDtzp+90VMvyD7KtwgZNcTqSNumkdwoicicwxxiT3yI7FOkLvF1rUQ/gt8Br7uVJQDpw9cn2qXcKjbP3YAnnPvEl1a7j/82D/H0ICfAjukMAf50ymGHd6imaaW37N9i7gwMbbFn3RX+qu5Kt/DBsmG8TROYau6xDHIx72FbOOVGRq9QpoLl3Cp2AlSKyGpgNfGKaUeZkjNkCDHUH5gvsAxYCM4DPjTEzRWSG+/2DTd2POt4r36UjwDu3n0F8WBDBAb6EBPgS7O/rHR3SXNX2zmDJn2x9wdS3oe/FJ14/MNTeIZx+s717OLDBdtJqwVYaSrU3J00KxphHROQ3wIXANOBZEZkHvGSM2dHM/Y8HdhhjdovIJOBc9/JXgS/RpNBiDpVV8vbKvVwyKIERSU1sYXJgk23nXlcHm5MxBj5/HHZ+deJ1SvMhf5c9sV/6VOPGhek8VOsNlGoBDbq/NsYYEdkP7AeqgChgvjrk0d8AABlTSURBVIh8aox5oBn7vxZ4y/063hiT5d5flojE1fUFEZkOTAfo1q2OjiCqTvNTMzhcXsUtTW1emrcDXpsIxTl2LJyJzzaurP77f9lOOF1H2R64dekQa9uzD5qi9QBKOeSkSUFE7gZuAnKBF4H/Z4ypFBEfYBvQpKQgIgHAROChxnzPGDMLmAW2TqEp+25vql2Gl79NZ3i3SIZ2bULTw6IseP0KW7wz/EY7fkvCUNvmvyF2fgWf/taOhHn1a3rCV8qLNeROIQaYbIzZXXuhMcYlIpc1Y98TgNXGGPfIUxwQkQT3XUICkN2MbataPt98gD0HS3jw4n6N/3JpPrwx2fbEvWkxJAyzwzx8PMOO85M0tv7vF+yF+dNsB6YrntOEoJSXa8jMax8CB4+8EZEwERkFYIzZ3Ix9T+XHoiOAxdg7EtzPi5qxbVXL7G930TkiiIsGxDfuixUl8OY1kLcdrp1jh+n18bEdhaK6w7wb7Un/RCpL4e3rbUeya+ZoBbBSp4CGJIXngcO13he7lzWZiIQAFwDv1lo8E7hARLa5P5vZnH0oa2NmIct2HuSmM5Pw823E7KvVlfakn7ESfvKiHUb6iKAImPqWXWfuT23yOJYx8MH/2V6oV/7n6BExlVJeqyFnCandBNUY46KZnd6MMSXGmGhjTGGtZXnGmPHGmN7u54P1bUM1zOyl6YQE+HLtiEZUyrtcdoKS7Z/CZU/Z1kDHiultk8X+9fDe3TYJ1LbyRdvj9JwZ0O+S5v0IpZTHNCQp7BSRu0XE3/24B9jZ2oGp5ss+VMZ76zKZcnoiESH+J/8C2JP7Jw/b8WXG/9b2ATiRPhfBeb+2PYu/++ePy3d/b+sc+lxsx39XSp0yGnLFfzvwD+AR7DRQn+NuEqq82xvL9lBR7eLmM5PsgvJD8NEMO0BcSDSERNnn4I7u9x3t6J7Ln4fRv7STfpzMWffbu4XPHoX4AXbAundusvMKXPkfWwehlDplNKTzWja2P4HyAn//3xaW7czjqWuGkhh14jHjyyqrmbNsN+P7xdEjNvTHSuO9y+2EMfvTbHI4MuZ9bUOm2kHkGtJSSMSOX5+7HebfYicDKT8MNy7y/pE3lVLHaUg/hSDgVmAAEHRkuTHmllaMS9UhdXc+zy7ZjjEw6dlv+c8Np5Nygt7Ji9dlkldcYTurVZXbVkC7v4MpL9mJSo6oKLFTK5bk2YfL5Z6buBFX+IGhtnXSrHNtxfJVr9rmqkqpU05D/vJfx45/dBHwFZAIHGrNoNTxKqtdPPzuejqFB7H4zjGEB/sz9YVlzFt5fJNQYwyzl+6iX6cwzkwKt1fwOz63QwzXTghgZ6iKSLQzcPU8D3qf37SB5Dp2t/0YrnrVzhuglDolNSQp9DLG/AYoNsa8ClwKDGrdsNSxXvhmJ1sOHOL3kwYyODGS//5iDKO6R/PAgjR+//4mqqpdNet+vyOPH/Yf4tYzuyGLfmHHnp/wVztpe2tKGKIJQalTXEOSQqX7uUBEBgIR2OGtlYfszivmmc+2MWFgJ85Pth3QIkL8eWXaCG4+M4mXlu7illdXUVhq/6lmf7uLjiH+XJn5d9syaPyjMOrnTv4EpdQpoiFJYZaIRGFbHy0GNgF/adWoVA1jDL9euIEAXx8emzjgqM/83Mv+PHkQ323P5crnvmXJD9l8/sMBXohfgN/a12zroLMa0IpIKaU4SUWze9C7IvdkN19jJ8RRHvTftftYuj2X308aQHx4UJ3rTB3ZjR4xHbhjzmqmvbKS+/0XcHrWuzDqDjsvsVJKNVC9ScE96N2dwDwPxaNqyS+u4Pfvb2ZYt0iuG3Xajx+4qqG04KhWQ6NKDvLF6P2kpi5jfNlndjTTi/+sA9AppRqlIc1MPhWR+7FTZRYfWajDULS+P364maLSSv48edCPM6N9/BAsex7bj/BokcB43wCbEC57WhOCUqrRGpIUjvRH+GWtZQYtSmpV3+3IZX5qBr84tyf9OrknpVkzB5Y9BwMmQ7fR7p7IRx7unskBHTQZKKWarCE9mps4VZdqqrLKan69cAOnRYdw9/jeduGBjXbU0aSz7EB0Pr7OBqmUapMa0qP5xrqWG2Nea/lwFMC/lmxnV24xb9w6iiB/Xztm0bybICgcfvKSJgSlVKtpSPHRiFqvg4DxwGpAk0IrWLe3gH9/tYMrh3VhbO8YO2rpe/fAwR1w42IIa+REOUop1QgNKT66q/Z7EYnADn2hWpAxhte+380fP9xMdIdAHrnUPXbQqpdgwwI7jHX3s5wNUinV5jVlspwSoHdLB9KeHSyu4IH56/hsczbj+sbyxFVDiA4NhH2rbWuj3hfCmF85HaZSqh1oSJ3Ce/zY/tEHSEb7LbSY77bn8qt5a8kvruS3lyUzbUwSIgKl+XZegg5xOi+BUspjGnKn8ESt11XAbmNMRivF025UVrt4+rOtPPflDrrHdOClm0YwsEuE/dAY+O8voSgLpn1km5wqpZQHNCQp7AGyjDFlACISLCJJxpj0Vo2sDdt7sIS73lrD2r0FXJPSlUcnJhMSUOuf4vtnYcsHcPFM6DrixBtSSqkW1pCk8A5wZq331e5lTT5biUgk8CIwEFs0dQuwBdtrOglIB652j7nUpvywv4irnv8egH9OHcblQzrbD4rz7Kxoe76H7/8F/SfCqNsdjFQp1R41JCn4GWMqjrwxxlSISEAz9/sM8LExZop7WyHAw8DnxpiZIjIDmAG0qVnfq12GB+enEeArvP/TOBKKPoP/Loe9yyBvu13Jxx96nAOTntWeyUopj2tIUsgRkYnGmMUAIjIJyG3qDkUkHDgbuBlskgEq3Ns9173aq8CXtLGk8Mp36fjuW8m34c8S9Ib7EAZ3hK6jYNj19rnzMPAPdjZQpVS71ZCkcDswR0Sedb/PAOrs5dxAPYAc4GURGQKkAvcA8caYLABjTJaIxDVjH15n78ES5n+yhHnBTxLYIQYuftwmgeheekeglPIaDem8tgMYLSKhgBhjmjs/sx8wHLjLGLNcRJ7BFhU1iIhMB6YDdOvWrZmheIYxhpkLvuE/PjMJCfRHrl8AHXU8QaWU9zlp43cR+ZOIRBpjDhtjDolIlIj8oRn7zAAyjDHL3e/nY5PEARFJcO8zAciu68vGmFnGmBRjTEpsbGwzwvCc91ft4La9D9PZtwDfn76tCUEp5bUa0iNqgjGm4Mgbd4ugS5q6Q2PMfmCviPR1LxqPneJzMXCTe9lNwKKm7sOb5BWVEPLBHQzx2YFMeVGbmCqlvFpD6hR8RSTQGFMOtp8CENjM/d6FracIAHYC07AJap6I3IrtG3FVM/fhFTa8fBfjWUH2mMeJS57odDhKKVWvhiSFN4DPReRl9/tp2NZBTWaMWQuk1PHR+OZs19tsX/QXzsmfT2rnn3L6Bfc6HY5SSp1UQyqa/yoiacD5gAAfA6fV/y1VlraQHmv+zNd+ZzBq2j+cDkcppRqkoaOs7QdcwE+wV/ObWy2itmDvCnwXTmetqycdrp1NoL+/0xEppVSDnPBOQUT6ANcCU4E87BAUYowZ56HYTk2l+VTOuZaM6ig+Gfw0D/Xq7HRESinVYPUVH/0AfANcbozZDiAiOqj/SVR/8wy+ZQf5TcATPH/ZaKfDUUqpRqmv+Ogn2GKjJSLygoiMx9YpqBM5dADXsudZXH0GN0++nLAgLTZSSp1aTpgUjDELjTHXAP2w4xD9CogXkedF5EIPxXdKyfnwj1Bdyea+d3J+ss6lrJQ69Zy0otkYU2yMmWOMuQxIBNbSiGEp2ouy7F1Ebp7DB37j+eUUzZlKqVNTo+Z4NMYcNMb8xxhzXmsFdKraPPdhXEZInPQo4VpspJQ6RenEvy1gTeoyBud9RGr8FFIGD3Q6HKWUajJNCs10qKySgg8eo0yCGPbT3zkdjlJKNYsmhWZ6+Z13Gef6nsJhPyc4sk1NAaGUaoc0KTTDZ5sOMGTrs5T4RdD5ov9zOhyllGo2TQpNdLC4gnkL5nKObxoB5/4fBIU7HZJSSjWbJoUmMMbwm4XrmV45h8oOnfAbNd3pkJRSqkU0ZOhsdYzF6zIp2fQRKQFbYNzT4B/sdEhKKdUiNCk0kjGGv320mdeCF2DCuyPDrnc6JKWUajGaFBpp64HDDD30JT0CdsK4F8FXO6oppdoOrVNopK+2HOAuv4VURveDgT9xOhyllGpRmhQaafvGVfT1ycB/9M/ARw+fUqpt0bNaIxSXV5GY9T8MAv0udzocpZRqcZoUGuG7HXlcJMspih8JYTo0tlKq7XEkKYhIuoisF5G1IrLKvayjiHwqItvcz1FOxFafTWkr6OuTQYehk50ORSmlWoWTdwrjjDFDjTEp7vczgM+NMb2Bz/GyORuMMYRs/wAXgt+ASU6Ho5RSrcKbio8mAa+6X78KXOFgLMfZkVPM2Iql5EYNhfAEp8NRSqlW4VRSMMD/RCRVRI6MERFvjMkCcD971ZCja9espL/PXvwHXel0KEop1Wqc6rw2xhiTKSJxwKci8kNDv+hOItMBunXr1lrxHce1aREAUadP8dg+lVLK0xy5UzDGZLqfs4GFwEjggIgkALifs0/w3VnGmBRjTEpsbKxH4i2tqGZgwRL2dhgIEV08sk+llHKCx5OCiHQQkbAjr4ELgQ3AYuAm92o3AYs8HduJrFu3mmRJp6LvRKdDUUqpVuVE8VE8sFBEjuz/TWPMxyKyEpgnIrcCe4CrHIitTofXzAegy5nXOhyJUkq1Lo8nBWPMTmBIHcvzgPGejqchuu7/lB0B/egZc5rToSilVKvypiapXilj52b6unaQn3SJ06EopVSr06RwEvu/fxuATqOvdjgSpZRqfZoUTqLj7o/4wacXiT36Ox2KUkq1Ok0K9SjLTadHxQ/s7XSB06EopZRHaFKoR+Z3cwEIHaaT6Sil2gdNCvXw3/IeG00SQ4cMdzoUpZTyCE0KJ1KYQdfiDWyMHEdwgK/T0SillEdoUjiBgtQFAEiyVw3WqpRSrUqTwglUrF/IZlc3hg1LOfnKSinVRmhSqEtRFjH5a1kaMIaesR2cjkYppTxGk0IdqjYuwgfD4Z6X4R6jSSml2gWn5lPwakUbPibflcDAISOcDkUppTxK7xTqEJCdxjp6cUbPaKdDUUopj9KkcKxD+wmtzKMgYgChgXojpZRqXzQpHKMyYw0A/l2HOhyJUkp5niaFY+RsXY7LCJ36jHQ6FKWU8jhNCseo2LuGnSaBIT10LmalVPujSeEYEQUb2enXi7jwIKdDUUopj9OkUNvhHKKqcjgcPdDpSJRSyhGaFGo5uGMlAEFdhzkciVJKOUOTQi05W5cDkDhgtMORKKWUMxxLCiLiKyJrROR99/vuIrJcRLaJyNsiEuDpmEzmWtJNJ/qdlujpXSullFdw8k7hHmBzrfd/AZ4yxvQG8oFbPR1QVOFmMoL6EOCnN1BKqfbJkbOfiCQClwIvut8LcB4w373Kq4BHJzKoKMol3nWA8thBntytUkp5FacuiZ8GHgBc7vfRQIExpsr9PgOos6OAiEwXkVUisionJ6fFAtqzaRkAod11/gSlVPvl8aQgIpcB2caY1NqL61jV1PV9Y8wsY0yKMSYlNja2xeLK37ECgKSBZ7TYNpVS6lTjxIhvY4CJInIJEASEY+8cIkXEz323kAhkejIo3/1p7COeLvEJntytUkp5FY/fKRhjHjLGJBpjkoBrgS+MMdcBS4Ap7tVuAhZ5Mq7Ywz+QHdrXk7tUSimv403NbB4E7hOR7dg6hpc8teOc3Gy6miyq4gd7apdKKeWVHJ0wwBjzJfCl+/VOwJGhSXelfU8sENlTZ1pTSrVv3nSn4JhD6XZ4i24DtJJZKdW+aVIAArI3kOMTS2BEvNOhKKWUo9p9UqisdtGldAt5Yf2dDkUppRzX7pPC1t1ZJJEFnYc4HYpSSjmu3SeFPZuX4yOGGJ1+UymlNCmU7rYdq6N7aVJQSql2nxRC8jZS4BuNhHVyOhSllHJcu04KuYfL6V65jcLIZKdDUUopr9Cuk8K6nVn0kn34Jer0m0opBe08KWRtWYmvGGL7jHI6FKWU8grtOilUZKwGIEDvFJRSCmjHSaGq2kVE/iYO+0VBeGenw1FKKa/QbpPClgOH6M8uijsOAKlrjh+llGp/2m1SWLvrAL0lg6DTTnc6FKWU8hrtNilkb0/FX6oJ765JQSmljmi3ScFkrgVAEoY6HIlSSnmPdpkUDhZX0Kl4C2V+ERDZzelwlFLKa7TLpLB2bz6DfHZRFjtQK5mVUqqWdpkUNu7Jo6/spUOS1icopVRt7TIp3DmwggCpxr+LdlpTSqna2mVSkKx19kVnrWRWSqnaPJ4URCRIRFaIyDoR2Sgij7uXdxeR5SKyTUTeFpGAVguiQwz0vRSiurfaLpRS6lTkxJ1COXCeMWYIMBS4WERGA38BnjLG9AbygVtbLYJ+l8LUN7WSWSmljuHxpGCsw+63/u6HAc4D5ruXvwpc4enYlFKqvXOkTkFEfEVkLZANfArsAAqMMVXuVTKALif47nQRWSUiq3JycjwTsFJKtROOJAVjTLUxZiiQCIwE+te12gm+O8sYk2KMSYmNjW3NMJVSqt1xtPWRMaYA+BIYDUSKiJ/7o0Qg06m4lFKqvXKi9VGsiES6XwcD5wObgSXAFPdqNwGLPB2bUkq1d34nX6XFJQCviogvNinNM8a8LyKbgLki8gdgDfCSA7EppVS75vGkYIxJA47rSmyM2YmtX1BKKeWQdtmjWSmlVN3EmDob+ZwSRCQH2H2Cj2OAXA+G01jeHJ/G1jQaW9NobE3TnNhOM8bU2XzzlE4K9RGRVcaYFKfjOBFvjk9jaxqNrWk0tqZprdi0+EgppVQNTQpKKaVqtOWkMMvpAE7Cm+PT2JpGY2saja1pWiW2NlunoJRSqvHa8p2CUkqpRmqTSUFELhaRLSKyXURmOB1PbSKSLiLrRWStiKxyOJbZIpItIhtqLesoIp+6Jzv6VESivCi2x0Rkn/vYrRWRSxyKrauILBGRze6Jou5xL3f82NUTm+PHzism2Gp8bK+IyK5ax82x6Rrdo0uvEZH33e9b57gZY9rUA/DFDsXdAwgA1gHJTsdVK750IMbpONyxnA0MBzbUWvZXYIb79QzgL14U22PA/V5w3BKA4e7XYcBWINkbjl09sTl+7AABQt2v/YHl2MEw5wHXupf/G7jDi2J7BZji9P85d1z3AW8C77vft8pxa4t3CiOB7caYncaYCmAuMMnhmLySMeZr4OAxiydhJzkCByc7OkFsXsEYk2WMWe1+fQg7oGMXvODY1ROb44zllRNs1RObVxCRROBS4EX3e6GVjltbTApdgL213p9wwh6HGOB/IpIqItOdDqYO8caYLLAnGCDO4XiOdaeIpLmLlxwp2qpNRJKwY3ktx8uO3TGxgRccu+ZMsOXp2IwxR47bH93H7SkRCXQiNuBp4AHA5X4fTSsdt7aYFOqaeNlrMj4wxhgzHJgA/FJEznY6oFPI80BP7NzeWcDfnQxGREKBBcC9xpgiJ2M5Vh2xecWxM82YYKu1HRubiAwEHgL6ASOAjsCDno5LRC4Dso0xqbUX17Fqixy3tpgUMoCutd571YQ9xphM93M2sBDvGxn2gIgkALifsx2Op4Yx5oD7D9cFvICDx05E/LEn3TnGmHfdi73i2NUVmzcdO3c8XjvBVq3YLnYXxxljTDnwMs4ctzHARBFJxxaHn4e9c2iV49YWk8JKoLe7Zj4AuBZY7HBMAIhIBxEJO/IauBDYUP+3PG4xdpIj8LLJjo6ccN2uxKFj5y7PfQnYbIx5stZHjh+7E8XmDcdOvHiCrRPE9kOtJC/YMnuPHzdjzEPGmERjTBL2fPaFMeY6Wuu4OV2j3hoP4BJsq4sdwK+djqdWXD2wraHWARudjg14C1uUUIm9w7oVW1b5ObDN/dzRi2J7HVgPpGFPwAkOxTYWe6ueBqx1Py7xhmNXT2yOHztgMHYCrTTsyfW37uU9gBXAduAdINCLYvvCfdw2AG/gbqHk1AM4lx9bH7XKcdMezUoppWq0xeIjpZRSTaRJQSmlVA1NCkoppWpoUlBKKVVDk4JSSqkamhSUqoeIVNcaIXOttOCouyKSVHsUWKW8gd/JV1GqXSs1dugDpdoFvVNQqgnEzovxF/cY/CtEpJd7+Wki8rl7ALXPRaSbe3m8iCx0j9e/TkTOdG/KV0RecI/h/z93b1qlHKNJQan6BR9TfHRNrc+KjDEjgWexY9Hgfv2aMWYwMAf4h3v5P4CvjDFDsPNEbHQv7w38yxgzACgAftLKv0epemmPZqXqISKHjTGhdSxPB84zxux0D0C33xgTLSK52CEkKt3Ls4wxMSKSAyQaO7DakW0kYYdo7u1+/yDgb4z5Q+v/MqXqpncKSjWdOcHrE61Tl/Jar6vRej7lME0KSjXdNbWev3e//g47kiXAdcBS9+vPgTugZjKXcE8FqVRj6FWJUvULds/GdcTHxpgjzVIDRWQ59uJqqnvZ3cBsEfl/QA4wzb38HmCWiNyKvSO4AzsKrFJeResUlGoCd51CijEm1+lYlGpJWnyklFKqht4pKKWUqqF3CkoppWpoUlBKKVVDk4JSSqkamhSUUkrV0KSglFKqhiYFpZRSNf4/FB+Cjx6XqeMAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(np.arange(1, NUM_EPOCHS+1), train_acc_list, label='Training')\n",
    "plt.plot(np.arange(1, NUM_EPOCHS+1), valid_acc_list, label='Validation')\n",
    "\n",
    "plt.xlabel('Epoch')\n",
    "plt.ylabel('Accuracy')\n",
    "plt.legend()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation ACC: 74.55%\n",
      "Test ACC: 73.68%\n"
     ]
    }
   ],
   "source": [
    "with torch.set_grad_enabled(False):\n",
    "    test_acc = compute_acc(model=model,\n",
    "                           data_loader=test_loader,\n",
    "                           device=DEVICE)\n",
    "    \n",
    "    valid_acc = compute_acc(model=model,\n",
    "                            data_loader=valid_loader,\n",
    "                            device=DEVICE)\n",
    "    \n",
    "\n",
    "print(f'Validation ACC: {valid_acc:.2f}%')\n",
    "print(f'Test ACC: {test_acc:.2f}%')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "matplotlib  3.1.0\n",
      "pandas      0.24.2\n",
      "torch       1.1.0\n",
      "numpy       1.16.4\n",
      "PIL.Image   6.0.0\n",
      "torchvision 0.3.0\n",
      "\n"
     ]
    }
   ],
   "source": [
    "%watermark -iv"
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "default_view": {},
   "name": "convnet-vgg16.ipynb",
   "provenance": [],
   "version": "0.3.2",
   "views": {}
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  },
  "toc": {
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": true,
   "toc_position": {
    "height": "calc(100% - 180px)",
    "left": "10px",
    "top": "150px",
    "width": "371px"
   },
   "toc_section_display": true,
   "toc_window_display": true
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}