{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# CS 20 : TensorFlow for Deep Learning Research\n",
    "## Lecture 07 : ConvNet in TensorFlow\n",
    "same contents, but different style with [Lec07_ConvNet mnist by high-level.ipynb](https://nbviewer.jupyter.org/github/aisolab/CS20/blob/master/Lec07_ConvNet%20in%20Tensorflow/Lec07_ConvNet%20mnist%20by%20high-level.ipynb)\n",
    "\n",
    "### ConvNet mnist by high-level\n",
    "- Creating the **data pipeline** with `tf.data`\n",
    "- Using `tf.keras`, alias `keras` and `eager execution`\n",
    "- Creating the model as **Class** by subclassing `tf.keras.Model`\n",
    "- Training the model with **Drop out** technique by `tf.keras.layers.Dropout`\n",
    "- Using tensorboard"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Setup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1.12.0\n"
     ]
    }
   ],
   "source": [
    "from __future__ import absolute_import, division, print_function\n",
    "import os, sys\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import tensorflow as tf\n",
    "from tensorflow import keras\n",
    "%matplotlib inline\n",
    "\n",
    "print(tf.__version__)\n",
    "tf.enable_eager_execution()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load and Pre-process data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "(x_train, y_train), (x_tst, y_tst) = tf.keras.datasets.mnist.load_data()\n",
    "x_train = x_train  / 255\n",
    "x_train = x_train.reshape(-1, 28, 28, 1).astype(np.float32)\n",
    "x_tst = x_tst / 255\n",
    "x_tst = x_tst.reshape(-1, 28, 28, 1).astype(np.float32)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(55000, 28, 28, 1) (55000,)\n",
      "(5000, 28, 28, 1) (5000,)\n"
     ]
    }
   ],
   "source": [
    "tr_indices = np.random.choice(range(x_train.shape[0]), size = 55000, replace = False)\n",
    "\n",
    "x_tr = x_train[tr_indices]\n",
    "y_tr = y_train[tr_indices].astype(np.int32)\n",
    "\n",
    "x_val = np.delete(arr = x_train, obj = tr_indices, axis = 0)\n",
    "y_val = np.delete(arr = y_train, obj = tr_indices, axis = 0).astype(np.int32)\n",
    "\n",
    "print(x_tr.shape, y_tr.shape)\n",
    "print(x_val.shape, y_val.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Define SimpleCNN class by high-level api"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "class SimpleCNN(keras.Model):\n",
    "    def __init__(self, num_classes):\n",
    "        super(SimpleCNN, self).__init__()\n",
    "        self.__conv1 = keras.layers.Conv2D(filters=32, kernel_size=[5,5], padding='same',\n",
    "                                           kernel_initializer=keras.initializers.truncated_normal(),\n",
    "                                           bias_initializer=keras.initializers.truncated_normal(),\n",
    "                                           activation=tf.nn.relu)\n",
    "        self.__conv2 = keras.layers.Conv2D(filters=64, kernel_size=[5,5], padding='same',\n",
    "                                           kernel_initializer=keras.initializers.truncated_normal(),\n",
    "                                           bias_initializer=keras.initializers.truncated_normal(),\n",
    "                                           activation=tf.nn.relu)\n",
    "        self.__pool = keras.layers.MaxPooling2D()\n",
    "        self.__flatten = keras.layers.Flatten()\n",
    "        self.__dropout = keras.layers.Dropout(rate =.5)\n",
    "        self.__dense1 = keras.layers.Dense(units=1024, activation=tf.nn.relu, \n",
    "                                           kernel_initializer=keras.initializers.truncated_normal(),\n",
    "                                           bias_initializer=keras.initializers.truncated_normal())\n",
    "        self.__dense2 = keras.layers.Dense(units=num_classes,\n",
    "                                           kernel_initializer=keras.initializers.truncated_normal(),\n",
    "                                           bias_initializer=keras.initializers.truncated_normal())\n",
    "    \n",
    "    def call(self, inputs, training=False):\n",
    "        conv1 = self.__conv1(inputs)\n",
    "        pool1 = self.__pool(conv1)\n",
    "        conv2 = self.__conv2(pool1)\n",
    "        pool2 = self.__pool(conv2)\n",
    "        flattened = self.__flatten(pool2)\n",
    "        fc = self.__dense1(flattened)\n",
    "        if training:\n",
    "            fc = self.__dropout(fc, training=training)\n",
    "        score = self.__dense2(fc)\n",
    "        return score"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Preparing for training a model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "550\n"
     ]
    }
   ],
   "source": [
    "# hyper-parameter\n",
    "lr = .001\n",
    "epochs = 10\n",
    "batch_size = 100\n",
    "total_step = int(x_tr.shape[0] / batch_size)\n",
    "print(total_step)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<BatchDataset shapes: ((?, 28, 28, 1), (?,)), types: (tf.float32, tf.int32)>\n",
      "<BatchDataset shapes: ((?, 28, 28, 1), (?,)), types: (tf.float32, tf.int32)>\n"
     ]
    }
   ],
   "source": [
    "## create input pipeline with tf.data\n",
    "# for train\n",
    "tr_dataset = tf.data.Dataset.from_tensor_slices((x_tr, y_tr))\n",
    "tr_dataset = tr_dataset.shuffle(buffer_size = 10000)\n",
    "tr_dataset = tr_dataset.batch(batch_size = batch_size)\n",
    "print(tr_dataset)\n",
    "\n",
    "# for validation\n",
    "val_dataset = tf.data.Dataset.from_tensor_slices((x_val,y_val))\n",
    "val_dataset = val_dataset.batch(batch_size = batch_size)\n",
    "print(val_dataset)\n",
    "\n",
    "## create optimizer\n",
    "opt = tf.train.AdamOptimizer(learning_rate = lr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def loss_fn(model, x, y, training):\n",
    "    score = model(x, training)\n",
    "    return tf.losses.sparse_softmax_cross_entropy(labels=y, logits=score)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Training a model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "cnn = SimpleCNN(num_classes=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# creating check point (Object-based saving)\n",
    "checkpoint_dir = '../graphs/lecture07/convnet_mnist_high_kde/'\n",
    "checkpoint_prefix = os.path.join(checkpoint_dir, 'cnn')\n",
    "checkpoint = tf.train.Checkpoint(cnn=cnn)\n",
    "\n",
    "# create writer for tensorboard\n",
    "summary_writer = tf.contrib.summary.create_file_writer(logdir=checkpoint_dir)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch :   1, tr_loss : 0.159, val_loss : 0.052\n",
      "epoch :   2, tr_loss : 0.048, val_loss : 0.031\n",
      "epoch :   3, tr_loss : 0.034, val_loss : 0.028\n",
      "epoch :   4, tr_loss : 0.025, val_loss : 0.025\n",
      "epoch :   5, tr_loss : 0.021, val_loss : 0.031\n",
      "epoch :   6, tr_loss : 0.015, val_loss : 0.025\n",
      "epoch :   7, tr_loss : 0.014, val_loss : 0.033\n",
      "epoch :   8, tr_loss : 0.013, val_loss : 0.024\n",
      "epoch :   9, tr_loss : 0.011, val_loss : 0.034\n",
      "epoch :  10, tr_loss : 0.008, val_loss : 0.026\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'../graphs/lecture07/convnet_mnist_high_kde/cnn-1'"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tr_loss_hist = []\n",
    "val_loss_hist = []\n",
    "\n",
    "for epoch in range(epochs):\n",
    "\n",
    "    avg_tr_loss = 0\n",
    "    avg_val_loss = 0\n",
    "    tr_step = 0\n",
    "    val_step = 0\n",
    "\n",
    "\n",
    "    with summary_writer.as_default(), tf.contrib.summary.always_record_summaries(): # for tensorboard\n",
    "        # for training\n",
    "        for x_mb, y_mb in tr_dataset:\n",
    "            with tf.GradientTape() as tape:\n",
    "                tr_loss = loss_fn(cnn, x_mb, y_mb, training = True)     \n",
    "                grads = tape.gradient(target=tr_loss, sources=cnn.variables)\n",
    "            opt.apply_gradients(grads_and_vars=zip(grads, cnn.variables))\n",
    "            tf.contrib.summary.scalar(name='tr_loss', tensor=tr_loss)\n",
    "            avg_tr_loss += tr_loss\n",
    "            tr_step += 1\n",
    "        else:\n",
    "            avg_tr_loss /= tr_step\n",
    "            tr_loss_hist.append(avg_tr_loss)\n",
    "            \n",
    "        # for validation\n",
    "        for x_mb, y_mb in val_dataset:\n",
    "            val_loss = loss_fn(cnn, x_mb, y_mb, training = False)\n",
    "            tf.contrib.summary.scalar(name='val_loss', tensor=val_loss)\n",
    "            avg_val_loss += val_loss\n",
    "            val_step += 1\n",
    "        else:\n",
    "            avg_val_loss /= val_step\n",
    "            val_loss_hist.append(avg_val_loss)\n",
    "    print('epoch : {:3}, tr_loss : {:.3f}, val_loss : {:.3f}'.format(epoch + 1, avg_tr_loss, avg_val_loss))\n",
    "\n",
    "checkpoint.save(file_prefix=checkpoint_prefix)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.legend.Legend at 0x7fa84d199748>"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAD8CAYAAACb4nSYAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzt3Xl8XPV56P/Po32XRrJso3UMBu+rRgZiIBCzmMV2SQCbJE2gSbjlF5rmpqQ/N/3dLNzmXtKmhKYlC0lIsxtikmASEwLBhKWEWjbeFywbyZLlRZK178vz++OMpLEsW6NldKSZ5/166aWZM+eceWZsPd/zXY+oKsYYYyJDlNsBGGOMmTiW9I0xJoJY0jfGmAhiSd8YYyKIJX1jjIkglvSNMSaCWNI3xpgIYknfGGMiiCV9Y4yJIDFuBzDYtGnT1Ov1uh2GMcZMKTt27KhR1ezh9pt0Sd/r9VJSUuJ2GMYYM6WISHkw+1nzjjHGRBBL+sYYE0Es6RtjTAQJqk1fRFYD/wZEA99X1UcHvX4d8DiwGNigqpsDXisAvg/kAwrcpqpl4xK9MWbS6+rqorKykvb2drdDCQsJCQnk5eURGxs7quOHTfoiEg08AdwEVALbRWSLqh4I2O04cB/w8BCn+DHwVVV9SURSgN5RRWqMmZIqKytJTU3F6/UiIm6HM6WpKrW1tVRWVjJr1qxRnSOY5p0VQKmqHlPVTmATsG5QIGWquodBCV1E5gMxqvqSf79mVW0dVaTGmCmpvb2drKwsS/jjQETIysoaU60pmKSfC1QEPK/0bwvGFUC9iPxKRN4RkX/x1xyMMRHEEv74Get3GeqO3BjgWpxmn2LgUpxmoHOIyAMiUiIiJdXV1aN6o/rWTr75xyPsrWwYQ7jGGBPegkn6J3A6Yfvk+bcFoxLY5W8a6gZ+AywfvJOqPqmqPlX1ZWcPO6FsSNFRwuMvv8vLB0+P6nhjTHiqr6/nW9/61oiPu+2226ivrw9BRO4KJulvBy4XkVkiEgdsALYEef7tQIaI9GXyDwAHLrL/qKUmxDJnZho7yutCcXpjzBR1oaTf3d190eO2bt1KRkZGqMJyzbBJ33+F/hDwInAQeEZV94vIIyKyFkBEikWkErgb+K6I7Pcf24PTtPNHEdkLCPC90HwUKPZ62Hm8ju4eGyBkjHFs3LiRo0ePsnTpUoqLi7n22mtZu3Yt8+fPB+Av/uIvKCoqYsGCBTz55JP9x3m9XmpqaigrK2PevHl86lOfYsGCBdx88820tbW59XHGLKhx+qq6Fdg6aNsXAx5vx2n2GerYl3DG74dcUaGHH79VzqFTTSzMTZ+ItzTGjMBXnt/PgarGcT3n/Jw0vrRmwQVff/TRR9m3bx+7du3i1Vdf5fbbb2ffvn39Qx6feuopMjMzaWtro7i4mA996ENkZWWdc44jR47wi1/8gu9973vcc889PPvss3z0ox8d188xUcJqRm6xNxOA7WVnXY7EGDNZrVix4pwx7t/85jdZsmQJV111FRUVFRw5cuS8Y2bNmsXSpUsBKCoqoqysbKLCHXeTbpXNscjJSCQnPYGS8jruXzm6iQvGmNC52BX5RElOTu5//Oqrr/Lyyy/z1ltvkZSUxPXXXz/kGPj4+Pj+x9HR0VO6eSesrvQBfN5MSsrOoqpuh2KMmQRSU1Npamoa8rWGhgY8Hg9JSUkcOnSIP//5zxMc3cQLqyt9AJ/Xw5bdVVTWtZGfmeR2OMYYl2VlZbFy5UoWLlxIYmIiM2bM6H9t9erVfOc732HevHnMmTOHq666ysVIJ0b4Jf1Cp12/pPysJX1jDAA///nPh9weHx/PCy+8MORrfe3206ZNY9++ff3bH354qCXGpo6wa96ZMzOV1PgYSspsvL4xxgwWdkk/OkpYVuixpG+MMUMIu6QP4Cv08O6ZJhpau9wOxRhjJpXwTPpeD6qw87hd7RtjTKCwTPpL8zOIjhJKym2SljHGBArLpJ8UF8PCnDS2W7u+McacIyyTPkBRYSa7K+rp7LbF14wxwUtJSQGgqqqKu+66a8h9rr/+ekpKSi56nscff5zW1oEbBU6WpZrDNukXez10dPeyr8puqmKMGbmcnBw2b9486uMHJ/3JslRz2Cb9Iq8HgB3WxGNMRNu4cSNPPPFE//Mvf/nL/NM//ROrVq1i+fLlLFq0iOeee+6848rKyli4cCEAbW1tbNiwgXnz5nHnnXees/bOgw8+iM/nY8GCBXzpS18CnEXcqqqquOGGG7jhhhuAgaWaAR577DEWLlzIwoULefzxx/vfbyKWcA67Gbl9pqcmUJiVxPays3zqukvdDscYA/DCRji1d3zPOXMR3ProBV9ev349n/3sZ/n0pz8NwDPPPMOLL77IZz7zGdLS0qipqeGqq65i7dq1F7z/7Le//W2SkpI4ePAge/bsYfnygRsAfvWrXyUzM5Oenh5WrVrFnj17+MxnPsNjjz3Gtm3bmDZt2jnn2rFjBz/84Q95++23UVWuvPJK3v/+9+PxeCZkCeewvdIHZ0mGHeV1tviaMRFs2bJlnDlzhqqqKnbv3o3H42HmzJl84QtfYPHixdx4442cOHGC06cvfKvV1157rT/5Ll68mMWLB24R8swzz7B8+XKWLVvG/v37OXDg4jcHfOONN7jzzjtJTk4mJSWFD37wg7z++uvAxCzhHLZX+uCM1392ZyXv1bRwaXaK2+EYYy5yRR5Kd999N5s3b+bUqVOsX7+en/3sZ1RXV7Njxw5iY2Pxer1DLqk8nPfee4+vf/3rbN++HY/Hw3333Teq8/SZiCWcg7rSF5HVInJYREpFZOMQr18nIjtFpFtEzuvuFpE0EakUkf8Yj6CDVexv1y+x++YaE9HWr1/Ppk2b2Lx5M3fffTcNDQ1Mnz6d2NhYtm3bRnl5+UWPv+666/oXbdu3bx979uwBoLGxkeTkZNLT0zl9+vQ5i7ddaEnna6+9lt/85je0trbS0tLCr3/9a6699tpx/LQXN+yVvohEA08ANwGVwHYR2aKqgXWY48B9OPfDHcr/Bl4bW6gjd+m0FDKSYikpO8s9vvyJfntjzCSxYMECmpqayM3N5ZJLLuEjH/kIa9asYdGiRfh8PubOnXvR4x988EHuv/9+5s2bx7x58ygqKgJgyZIlLFu2jLlz55Kfn8/KlSv7j3nggQdYvXo1OTk5bNu2rX/78uXLue+++1ixYgUAn/zkJ1m2bNmE3Y1LhmvvFpGrgS+r6i3+5/8AoKr/d4h9/xP4rapuDthWBHwe+D3gU9WHLvZ+Pp9Phxv/OhKf/NF2jtW08MrfXT9u5zTGBO/gwYPMmzfP7TDCylDfqYjsUFXfcMcG07yTC1QEPK/0bxuWiEQB/8qFawAhV1SYybHqFmqbO9wKwRhjJo1Qj975f4Ctqlp5sZ1E5AERKRGRkurq6nENoK9df4e16xtjTFBJ/wQQ2CCe598WjKuBh0SkDPg68DEROa/7XlWfVFWfqvqys7ODPHVwFuamExcdZZ25xrjIhk2Pn7F+l8EM2dwOXC4is3CS/Qbgw8GcXFU/0vdYRO7DadM/b/RPKCXERrM4L52SMltx0xg3JCQkUFtbS1ZW1gUnP5ngqCq1tbUkJCSM+hzDJn1V7RaRh4AXgWjgKVXdLyKPACWqukVEioFfAx5gjYh8RVUXjDqqcVbk9fDUG+/R3tVDQmy02+EYE1Hy8vKorKxkvJtuI1VCQgJ5eXmjPn7Y0TsTbbxH7wC8fOA0n/xxCc/8j6tZMStzXM9tjDGTwXiO3pnyigqdztzt1sRjjIlwEZH0PclxzJ6eYiN4jDERLyKSPjg3Sy8pO0tv7+RqzjLGmIkUOUnfm0ljezel1c1uh2KMMa6JnKRv7frGGBM5Sb8wK4lpKfF2Jy1jTESLmKQvIvgKPWwvtyt9Y0zkipikD85NVSrOtnG6cfQ3OTDGmKkswpK+MzGrxJp4jDERKqKS/oKcNBJio6wz1xgTsSIq6cdGR7E0P8MmaRljIlZEJX2AYm8mB0420tLR7XYoxhgz4SIu6RcVeujpVXZV1LsdijHGTLiIS/rLCz2I2CQtY0xkirikn5YQy5wZqdaub4yJSBGX9MFp199ZXkd3T6/boRhjzISKyKTv83po6ezh0Kkmt0MxxpgJFaFJv2+SlrXrG2MiS1BJX0RWi8hhESkVkfNubC4i14nIThHpFpG7ArYvFZG3RGS/iOwRkfXjGfxo5WYkckl6AiXWrm+MiTDDJn0RiQaeAG4F5gP3isj8QbsdB+4Dfj5oeyvwMf9N0lcDj4tIxliDHg8+byYlZXVMtnsEG2NMKAVzpb8CKFXVY6raCWwC1gXuoKplqroH6B20/V1VPeJ/XAWcAbLHJfIx8hV6ONXYzon6NrdDMcaYCRNM0s8FKgKeV/q3jYiIrADigKNDvPaAiJSISEl1dfVITz0qPq9zUxVbfM0YE0kmpCNXRC4BfgLcr6rnjZNU1SdV1aeqvuzsiakIzJ2ZRkp8DCW2vr4xJoIEk/RPAPkBz/P824IiImnA74B/VNU/jyy80ImOEpYVZNiVvjEmogST9LcDl4vILBGJAzYAW4I5uX//XwM/VtXNow8zNHyFmRw+3URDW5fboRhjzIQYNumrajfwEPAicBB4RlX3i8gjIrIWQESKRaQSuBv4rojs9x9+D3AdcJ+I7PL/LA3JJxmFYq8HVdh53K72jTGRISaYnVR1K7B10LYvBjzejtPsM/i4nwI/HWOMIbO0IIPoKGFHWR03zJnudjjGGBNyETkjt09SXAwLctJsxU1jTMSI6KQPzvr6uyvr6ey2xdeMMeEv4pN+sTeT9q5e9lc1uB2KMcaEXMQnfV+hM0nL1tc3xkSCiE/609MSKMhMsnZ9Y0xEiPikD87V/o5yW3zNGBP+LOnjrLhZ09xJWW2r26EYY0xIWdIncPE1a+IxxoQ3S/rA7OwU0hNjbR0eY0zYs6QPREUJRYUeW3HTGBP2LOn7+bwejla3cLal0+1QjDEmZCzp+/kKnZul23h9Y0w4s6TvtzgvnbjoKOvMNcaENUv6fgmx0SzMTaPErvSNMWHMkn6AYm8meysbaO/qcTsUY4wJCUv6AXzeTDp7etl7whZfM8aEJ0v6AYr8i6/ZOjzGmHAVVNIXkdUiclhESkVk4xCvXyciO0WkW0TuGvTax0XkiP/n4+MVeChkJsdxWXYyO2ySljEmTA2b9EUkGngCuBWYD9wrIvMH7XYcuA/4+aBjM4EvAVcCK4AviYhn7GGHjq8wk5LyOnp7bfE1Y0z4CeZKfwVQqqrHVLUT2ASsC9xBVctUdQ8w+PZTtwAvqepZVa0DXgJWj0PcIePzemho6+JodbPboRhjzLgLJunnAhUBzyv924IxlmNd4fM6k7S2WxOPMSYMTYqOXBF5QERKRKSkurra1Vi8WUlMS4mzdXiMMWEpmKR/AsgPeJ7n3xaMoI5V1SdV1aeqvuzs7CBPHRoi/sXX7ErfGBOGgkn624HLRWSWiMQBG4AtQZ7/ReBmEfH4O3Bv9m+b1Iq9mRw/28qZxna3QzHGmHE1bNJX1W7gIZxkfRB4RlX3i8gjIrIWQESKRaQSuBv4rojs9x97FvjfOAXHduAR/7ZJrW+8vi3JYIwJNzHB7KSqW4Gtg7Z9MeDxdpymm6GOfQp4agwxTrgFOekkxEZRUlbHbYsucTscY4wZN5OiI3eyiYuJYklehnXmGmPCjiX9Cyj2ZrK/qpGWjm63QzHGmHFjSf8CirweenqV3RX1bodijDHjxpL+BSwv8CBik7SMMeHFkv4FpCfGMmdGqrXrG2PCiiX9i/B5PbxzvJ4eW3zNGBMmLOlfhK8wk+aObg6danQ7FGOMGReW9C/C5/VP0rJ2fWNMmLCkfxG5GYnMTEuwmbnGmLBhSf8iRASf10OJ3T7RGBMmLOkPw1fo4WRDOyfq29wOxRhjxsyS/jD6bqpiV/vGmHBgSX8Yc2emkhwXbZ25xpiwYEl/GDHRUSwv9LDdrvSNMWHAkn4Qigo9HD7dRGN7l9uhGGPMmFjSD0KxNxNV2GlDN40xU5wl/SAszc8gOkrYYUnfGDPFBZX0RWS1iBwWkVIR2TjE6/Ei8rT/9bdFxOvfHisiPxKRvSJyUET+YXzDnxjJ8THMvyTN2vWNMVPesElfRKKBJ4BbgfnAvSIyf9BunwDqVHU28A3ga/7tdwPxqroIKAL+R1+BMNUUFXrYVVFPV0+v26EYY8yoBXOlvwIoVdVjqtoJbALWDdpnHfAj/+PNwCoREUCBZBGJARKBTmBKrl5W7M2kvauX/VVTMnxjjAGCS/q5QEXA80r/tiH3UdVuoAHIwikAWoCTwHHg66o6JdtIBhZfm5LhG2MMEPqO3BVAD5ADzAL+TkQuHbyTiDwgIiUiUlJdXR3ikEZnRloC+ZmJNknLGDOlBZP0TwD5Ac/z/NuG3MfflJMO1AIfBn6vql2qegZ4E/ANfgNVfVJVfarqy87OHvmnmCC+wkxKyutQtZuqGGOmpmCS/nbgchGZJSJxwAZgy6B9tgAf9z++C3hFncx4HPgAgIgkA1cBh8YjcDf4vB5qmjsor211OxRjjBmVYZO+v43+IeBF4CDwjKruF5FHRGStf7cfAFkiUgp8Dugb1vkEkCIi+3EKjx+q6p7x/hATxVfoX3zNxusbY6aomGB2UtWtwNZB274Y8LgdZ3jm4OOah9o+VV0+PYW0hBhKys5yV1Ge2+EYY8yI2YzcEYiKEooKPXalb4yZsizpj5DPm0npmWbqWjrdDsUYY0bMkv4I+Qqd8fq2Do8xZiqypD9CS/IziI0WtpfbJC1jzNRjSX+EEmKjWZibzg6bpGWMmYIs6Y9CsTeTPZUNtHf1uB2KMcaMiCX9USgq9NDZ08u+Ew1uh2KMMSNiSX8U+jpzt1sTjzFmirGkPwpZKfFcOi2ZHdaZa4yZYizpj5LP60zS6u21xdeMMVOHJf1R8hVmUt/axbGaZrdDMcaYoFnSH6W+m6pYu74xZiqxpD9Ks6Ylk5UcZzdVMcZMKZb0R0mkb/E168w1xkwdlvTHwOf1UF7bypmmdrdDMcaYoFjSHwOf17mpii3JYIyZKizpj8HCnHTiY6JsfX1jzJQRVNIXkdUiclhESkVk4xCvx4vI0/7X3xYRb8Bri0XkLRHZLyJ7RSRh/MJ3V1xMFEvyMygps3Z9Y8zUMGzSF5FonHvd3grMB+4VkfmDdvsEUKeqs4FvAF/zHxsD/BT4a1VdAFwPdI1b9JNAsdfDvqpGWju73Q7FGGOGFcyV/gqgVFWPqWonsAlYN2ifdcCP/I83A6tERICbgT2quhtAVWtVNayWpvQVZtLTq+yqqHc7FGOMGVYwST8XqAh4XunfNuQ+qtoNNABZwBWAisiLIrJTRP5+7CFPLssLPIhg4/WNMVNCzASc/xqgGGgF/igiO1T1j4E7icgDwAMABQUFIQ5pfKUnxXLF9FTrzDXGTAnBXOmfAPIDnuf5tw25j78dPx2oxakVvKaqNaraCmwFlg9+A1V9UlV9qurLzs4e+adwmc/rYWd5HT22+JoxZpILJulvBy4XkVkiEgdsALYM2mcL8HH/47uAV1RVgReBRSKS5C8M3g8cGJ/QJw+f10NzRzeHTzW5HYoxxlzUsEnf30b/EE4CPwg8o6r7ReQREVnr3+0HQJaIlAKfAzb6j60DHsMpOHYBO1X1d+P/MdzlK3QmadmSDMaYyS6oNn1V3YrTNBO47YsBj9uBuy9w7E9xhm2GrTxPIjPS4ikpq+NjV3vdDscYYy7IZuSOAxHB5820SVrGmEnPkv448RV6qGpo50R9m9uhGGPMBVnSHyfF/sXX7GrfGDOZWdIfJ3NnppIUF80OG69vjJnEwifpd7bCT++C8v9y5e1joqNYXuCx2ycaYya18En6rTVQ9x78aC3s/LErIRQVejh8qpHG9rBaU84YE0bCJ+lnFMAnXwbvNbDlb+D3X4CeiV35stibSa/CO8dt8TVjzOQUPkkfINEDH9kMV/41/PkJ+Pk90DZxCXhpQQZRAjusM9cYM0mFV9IHiI6BW78GdzwO7/0Jvn8j1B6dkLdOiY9hfk6atesbYyat8Ev6fXz3w8eeg9Za+N4H4Oi2iXnbwkx2VdTT1dM7Ie9njDEjEb5JH5z2/Qe2Qeol8NMPwX9/DzS0K2EWezNp6+rhf/1mHzXNHSF9L2OMGanwTvoAHi984g9w+U2w9WH43eegJ3Sja25eMIP73ufllzsquf5fXuVbr5bS3hVWNwszxkxhoiG+8h0pn8+nJSUl43/i3h744yPw5uPgvRbu+TEkZY7/+/iVnmnm0RcO8vLBM+RmJPL3q+ewZnEOUVESsvc0xkQu/w2qfMPtF/5X+n2iouGmr8Cd34WKt+F7N8CZQyF7u9nTU/j+x4v5+aeuJCMplr/dtIs7v/Um221kjzHGRZGT9Pss2QD3bXVm8H7/Rnj3DyF9u/ddNo3nH7qGf717CacbO7j7O2/x1z/ZQVlNS0jf1xhjhhJ5SR8gv9jp4M2c5Yzlf/ObIe3gjYoSPlSUx7aHr+dzN13Ba0equekbf+KR5w9Q39oZsvc1xpjBIqdNfyidLfCbB+HAc7Dkw7DmcYiJD/nbnmls57GX3uWZkgpSE2L5mw/M5mNXe4mLicwy2BgzdsG26Ud20gfo7YXX/hle/b+QfyWs/ymkTJ+Qtz50qpGv/u4grx+poTAriY2r57J64UxErLPXGDMy49qRKyKrReSwiJSKyMYhXo8Xkaf9r78tIt5BrxeISLOIPBzsB5gwUVFw/Ua4+z/h5B548gbn9wSYOzONn3ziSv7z/mLiY6J48Gc7uee7b7G7wtbuMcaExrBJX0SigSeAW4H5wL0iMn/Qbp8A6lR1NvAN4GuDXn8MeGHs4YbQgjvhr34PKDx1CxzYMmFvff2c6Wz9zLX8nzsX8V5NC+ueeJO/3fQOlXWtExaDMSYyBHOlvwIoVdVjqtoJbALWDdpnHfAj/+PNwCrxt1GIyF8A7wH7xyfkEMpZCp96BabPh2f+Ev70zyGfwdsnJjqKD19ZwKufv4GHbpjN7/ed4gP/+icefeGQLdVsjBk3wST9XKAi4Hmlf9uQ+6hqN9AAZIlICvD/Al+52BuIyAMiUiIiJdXV1cHGHhqpM+G+38HiDbDtq7D5r5zhnRMkJT6Gh2+Zw7aHr+eORZfwnT8d5YZ/eZWf/LmcblvPxxgzRqEeLvJl4Buq2nyxnVT1SVX1qaovOzs7xCEFITYB7vwO3PgV2P9r+OGt0Fg1oSHkZCTy2PqlPP/QNcyensL/+s0+Vv/b67xy6DSTrfPdGDN1BJP0TwD5Ac/z/NuG3EdEYoB0oBa4EvhnESkDPgt8QUQeGmPME0MErvks3PsLqC11Ongrd0x4GIvy0tn0wFU8+ZdF9PQqf/WfJXz0B29zoKpxwmMxxkx9wST97cDlIjJLROKADcDgXs4twMf9j+8CXlHHtarqVVUv8Djwf1T1P8Yp9okx51b4xEsQE+dc8e/55YSHICLcvGAmf/if1/HlNfM5UNXI7f/+Op//5W5ON7ZPeDzGmKlr2KTvb6N/CHgROAg8o6r7ReQREVnr3+0HOG34pcDngPOGdU5pM+bDp16FPB/86pPOwm29E9++HhsdxX0rZ/Hq52/gU9deynO7qrj+X17lGy+9S2vnxN4a0hgzNdnkrJHo7oStf+fceH3O7fDBJyE+xbVwjte28rUXD/G7PSeZnhrPwzfP4UNFeUTbSp7GRBxbZTMUYuJgzTdh9dfg3Rec8fx15a6FU5CVxBMfXs6zD15NrieRv392D7d/83XeOFLjWkzGmMnNkv5IicBVf+3cgL2+wlmiufwtV0MqKszkVw++j3+/dxnNHd189Advc/8P/5t3Tze5GpcxU0LrWTi115UmWzdY885Y1ByBX2xwrvbv+AYs/0u3I6K9q4cf/VcZ/7GtlKb2bpYXZLB2SQ63L84hOzX0i8kZMyV0tTu19T3PwJE/QG83JGXBZaucu+xdtgqSs9yOckRswbWJ0lYHv7wfjm2Dqz4NNz0C0TFuR8XZlk42bT/Oll1VHDrVRJTAytnTWLMkh1sWzCQ9MdbtEM1E62yFo6/Aweeh/L/gksUwfx1ccQskpLsdXej19sLx/4Ldm5xlVjoaIGUmLL7bmYV/7FUo/SO01gACuUVOATD7JshZ5qzTNYlZ0p9IPd3wh3+Et78Ds2+Eu56aVH9ER043sWV3Fc/tquL42VbioqO4fk4265bmsmredBJio90O0YRKW71zJXtwi5PQulohIQO818CJHdB0EqJi4dLrYd4amHs7JE9zO+rxVX3YSfR7fwkNFRCbDPPXwuL1MOs65656fXp74eQ7cORlKH0JKksAnRK1AEv6bij5oXPz9YwCuPwW56bsfT8ZBRCX5Gp4qsruygae23WC3+45SXVTB8lx0dyyYCZrluZwzexpxEZP7quZEetqg4r/huNvQXScc3/knKUQHcY1neYzcOh3zhX9e69BbxekXuIk9HlroHCl8/l7e+FEiVMgHNgC9eUgUc7r89bCvDsgLcftTzM6Tadh37OwZxOc3O18rss+4CyvMvc2iEsO7jwttU7tqPSlQbWA5U4N4PK+WoD7F06W9N1S9ga8+I9Oe3/XoFsipsw8tyDwFA48Tpk5odXHnl7l7WO1bNldxda9J2ls7yYzOY7bFs1k7ZJcfIWeqXkT945m5x7I5W9C2ZvO1WxvFyCA//96bDIUXOlc7Xqvdf5op3ohUFcOh37rJPrjfwYUMi/1X72vcZoqLvb/S9XpzOwrAGoOO9vzip1zzFvr3GluMutscQq7PU/D0W2gPXDJUueKfuGHIHXG2M5/0VrAB+Dym12tBVjSd5sqtNZCXZn/5z3/73Lnd0Ml/UkIIDr+3ELgnFpCYUjnA3R09/DauzVs2V3FSwdO0d7VS056AncsyWHtkhwW5KRN3hu7tDc6Sa78DSfJn9zldMpJtJPMvSuh8Bonyfd0+QuDN5yfMweccwQWAoXXOMfFxLn7uYajCtWH4OBvnUR9yn8PiBmL/En6DqederT/btWHnfMefN65UgaYuciogeD1AAAPRElEQVRfA1gD2XNHf+7x1NsD7/0Jdj/txNrVAun5sOhuJ9lPnxu6924969QCjrwEpS+7XguwpD/ZdXc67Yv9hUHgTzl0DFpbJ2na0AWCx+tUwcfpP1ZLRzcvHzzNll1V/Ondarp7lUuzk1nrLwAuzXZvMhrgdJyXvzWQvE/tAe112qVzi/xJfqVzF7ThCsqWmoBC4E0441/9OzbJOT6wJjAZCgFVOLHTScaHfuusCQVOrPPWwNw7QnM1Xlc2ULhUvO1sy7rcec/5a52r6YksAFTh9D5/O/1maD4F8emwYJ2T6AveN/Gdrr29zgVH6ctOH0pfLSAxE2avcgqB2atC2l9iSX8qU3WS23mFQdlALUF7BvaPioWM/KFrCOn5kJQ5qj/KupZOXth3iud2neC/y86iCoty01m7JIc7llzCJemJY/6ow+pPzG86v0/vB9SpGeUVDyT5vOKx95m01A6qCQQWAisCCoHlE1cI9HQ7I04OPu80XTSegKgYJ46+jtfUmRMTC0DjyYFmpLI3nP+H6QUDBUDeitAl3IYTTmfsnqedWlpUjNOksng9XLHaWR13srhQLSBn2cCIoNzl41oLsKQfznq6nMR/oUKhfdDtFmMSIT3vAj/5kJY77B/MqYZ2frvHGQG090QDIrDCm8m6pbncunAmnuRxSoJNpweaasrfdJow+j5DX+ItXOlc1Yf6j7yvEOgrCE7vG4jlnD6BcS4Eutqd4YMHn4fDW6HtrPOes1c5yfWKWyDRM37vN1ottc5Y9wNbnCHLPZ2QMsOpccxb43w/Y+0raW90ahh7nob3XgfUKVgW3wMLPjgpR9Gc55xawEtO57n2jnstwJJ+JGurd0Zi1JU5V0cNlU5TUqP/cfPp849JmnZuQTC4cEie3n8Fd6y6med3n+S53Sc4Vt1CTJRw3RXZrFuaw43zZpAcP4J5Cg0nBpJq+ZsDTRZxKVBwlZPgvdc4TQhuN7G0nj23JhBYCOSvcAoA7zXOFVzMCCfCdTT5h1Y+7ySGzmanyeKKW5wEOntV8CNO3NDeODA09MhLztDQRA/Muc2J/9Ibgi+ke7qcq+Q9Tzu1m+528MyCJRuctvqsy0L7WUKtrxZQ+rLz01JNfy1g3hq49nOjOq0lfXNh3R0DBUBgodBQOfAzeORRVCyk5w7UDNLz0PQ8yrsz+cOJWJ4+3MvRRiEhNoob581g7ZIc3j8nm/iYgOqrqlMYlQVcPdf71y6KT4fCq/1JfiXMXDIpJrldVOtZZ5JTfyGw19keWCvxXuPUSoYqBFpqnSv5g88PXCknZw8MrfRe535BNxr9k8C2wOHfO5Og4lICCrCbzu9vUYWqnU6H7L5nneaQRI8z6mbxeqf5bjJ0HI+33l44tdspKI+85IwE+vCmUZ3Kkr4ZPVWniSiwEGioCCggKqGpyqmiBuiOS6M6Kpsj7RmUdWdyNiabGXmzWTIzjoKmd0g++TbSWOnsnOhxEnxfkp+xcFKMdR6T8wqBfYBCTMJATSB/hX9kzPNOwae9zhyOuWuchJi/Yup/D4G6O525Agefc67aW2ud7+OyVU4fwMxFcGirM56+ttTpq5mz2hlPP/vGqVnojUVvz6j//S3pm9Dq6XZmczaeX1PQhkq66yqI7Wzo371a09iu83kveQkN01eQmLuQ2TPSmD09hVnTksNzVnDrWWdSWNkbUPY6nPIXAgDZ8waGVs5cHJ5XsYP1dDvfx8HnnZ+mgFuQFq50rujnr4PEDPdinMIs6Rv3dTTRUXucIzVtHGifTmlNC6Vnmik900xFXSt9//WiBPIzk5idncLs6SlcNt35PXt6CmkJU3zSVKDWs85kMY8Xpl3udjTu6u11mnNO7XX6KzIK3I5oyrOkbya19q4ejlW3cLTaKQRKq5s5eqaZYzUtdHYPNBtlp8b3FwaBP9NT4yfvhDFjXBBs0g+qp0xEVgP/BkQD31fVRwe9Hg/8GCjCuSH6elUtE5GbgEeBOKAT+LyqvjKiT2LCUkJsNPNz0pifk3bO9p5epeJsa39B0Fcz+M07J2jqGLglZGp8TH+N4LKAQqEgM8nuHGbMRQyb9EUkGngCuAmoBLaLyBZVPRCw2yeAOlWdLSIbgK8B64EaYI2qVonIQpz77OaO94cw4SM6SvBOS8Y7LZkbGVgrRVWpbuo4rzB47d1qNu+o7N8vLjqKWdOSz20myk7h0uww7TcwZoSCudJfAZSq6jEAEdkErAMCk/464Mv+x5uB/xARUdV3AvbZDySKSLyqdow5chNRRITpaQlMT0vgfbPPncTS0NbFUX/zUF8z0f6qBl7Yd5Je7TseZk1LZmleBksLMliSl8HcS1LPHVJqTAQIJunnAhUBzyuBKy+0j6p2i0gDkIVzpd/nQ8BOS/hmvKUnxrK8wMPygnNnqbZ39VBWO9B5vL+qkddLa/jVOycAp1YwLyeNpXnp/QWBNyt5aq4uakyQJmT2i4gswGnyufkCrz8APABQUGC9+GZ8JMRGM3dmGnNnDvQbqConG9rZXVHPrsp6dlfUs3lHJT96y5kklpYQw5J8pwBYkp/Bkvx0pqdOojVdjBmjYJL+CSA/4Hmef9tQ+1SKSAyQjtOhi4jkAb8GPqaqR4d6A1V9EngSnNE7I/kAxoyEiJCTkUhORiK3LroEcDqPS880n1MQfPtPR+nxtw3lZiSyJD+9vyBYlJs+sqUmjJlEgvmfux24XERm4ST3DcCHB+2zBfg48BZwF/CKqqqIZAC/Azaq6pvjF7Yx4yc6SpgzM5U5M1O5p9i5vmnr7OHAyQbeOV7P7soGdlfUs3XvKcCZV3DFjNRzagNzZqQSE253HTNhKahx+iJyG/A4zpDNp1T1qyLyCFCiqltEJAH4CbAMOAtsUNVjIvL/Af8AHAk43c2qeuZC72Xj9M1kdbalk93+msCuCud3XWsXAAmxUSzMSfcXAhksy88gz5NocwnMhLHJWcaEmKpScbatv0loV0U9+0400OGfXJaZHMeSvIGCYEleBpnjtQS1MYOM6+QsY8z5RISCrCQKspJYu8S5gXhXTy+HTzX11wh2VzTw6rtH+pecKMhMIjs1nthoIS4mmrhoIS4mirjoKGKjo5zH/udxMQHboqOIjYkifvD2mChio4X4wftGRxEfc+6+MVFiNQ9jSd+Y8RQbHcXC3HQW5qbzkSsLAWju6GZvZQO7K+vZW9lAQ1sXnd29NPp/d/b00tXTS2e387uju7d/+3hWxEWc+PoKjrTEWDKSYslIjCUjKY70wc8DHmckxpKWGGuzncOAJX1jQiwlPoarL8vi6stGfpen7h5/odCtdPT0+AsGdQoFf8HQX3AMet63T1fgNv/jDn+h09DWRU1zJ6XVzdS3dtHU3n3ReNISYpxCICnWX0jE+QuG858725zCJC7GOrknC0v6xkxiMdFRzqigOIDQrzja3dNLY3s39a2d1Ld10dDaRX1bJ/WtXdS3OoVE32v1rV1U1rVR39pJQ1tX/+znoSTHRZ9bm/AXCBlJsWQmxeFJjsOTFIsnOc55nhRHakKMTZQLAUv6xph+MdFRZCbHjbjDubdXaeroPreQaOuioXXgsVNoOM/fPd3sL0g66b5AaREdJXiSnNqDUzDE4vEXEJn+2kZmcl+B4WyzgmJ4lvSNMWMWFSWkJzpNPAUkBX2cqtLc0U1dSxdnWzupa+mkrrWTs/7fda1d1LU4z8tqWtnZWk9dy8ULiozE2PMKhoykODL7Co2+gsNfu0hLiI2ogsKSvjHGNSJCakIsqQmxFGQFV1icV1D4C4uzLU4tIrDwKK9t5Z2KeupbO+nqGbqgiBLwJMWRnRrPtJR4/+/Bz53fnqS4Kd+ZbUnfGDOljLWgqGvtDCgYnJpEbUsnNc0dVDd1UFbbQnVTR/98i0BRAlkpTiHQVzBkDyoY+n5nJE7OGoQlfWNM2BtpQdFXSFQ3dVDTPFAgDP59rNopIDp7zi8gYqKErJS4cwqDwTWJvgIjPTF2wuZQWNI3xphBAguJS7Mvvq+q0tjefcGCoaa5k+qmDg6faqKmuWPIZqbYaGFaSjw+byb/fu+yEH0qhyV9Y4wZA5GBTuzLslMuuq+q+udGdHAmoECoae6gpqmD7NT4kMdrSd8YYyaIiPgnt8Uxe3qqKzHYNDljjIkglvSNMSaCWNI3xpgIYknfGGMiiCV9Y4yJIJb0jTEmgljSN8aYCGJJ3xhjIsikuzG6iFQD5WM4xTSgZpzCmersuziXfR/nsu9jQDh8F4WqOsyiEZMw6Y+ViJQEc0f4SGDfxbns+ziXfR8DIum7sOYdY4yJIJb0jTEmgoRj0n/S7QAmEfsuzmXfx7ns+xgQMd9F2LXpG2OMubBwvNI3xhhzAWGT9EVktYgcFpFSEdnodjxuEpF8EdkmIgdEZL+I/K3bMblNRKJF5B0R+a3bsbhNRDJEZLOIHBKRgyJytdsxuUlE/qf/72SfiPxCRBLcjimUwiLpi0g08ARwKzAfuFdE5rsblau6gb9T1fnAVcCnI/z7APhb4KDbQUwS/wb8XlXnAkuI4O9FRHKBzwA+VV0IRAMb3I0qtMIi6QMrgFJVPaaqncAmYJ3LMblGVU+q6k7/4yacP+pcd6Nyj4jkAbcD33c7FreJSDpwHfADAFXtVNV6d6NyXQyQKCIxQBJQ5XI8IRUuST8XqAh4XkkEJ7lAIuIFlgFvuxuJqx4H/h7odTuQSWAWUA380N/c9X0RSXY7KLeo6gng68Bx4CTQoKp/cDeq0AqXpG+GICIpwLPAZ1W10e143CAidwBnVHWH27FMEjHAcuDbqroMaAEitg9MRDw4rQKzgBwgWUQ+6m5UoRUuSf8EkB/wPM+/LWKJSCxOwv+Zqv7K7XhctBJYKyJlOM1+HxCRn7obkqsqgUpV7av5bcYpBCLVjcB7qlqtql3Ar4D3uRxTSIVL0t8OXC4is0QkDqcjZovLMblGRASnzfagqj7mdjxuUtV/UNU8VfXi/L94RVXD+kruYlT1FFAhInP8m1YBB1wMyW3HgatEJMn/d7OKMO/YjnE7gPGgqt0i8hDwIk7v+1Oqut/lsNy0EvhLYK+I7PJv+4KqbnUxJjN5/A3wM/8F0jHgfpfjcY2qvi0im4GdOKPe3iHMZ+fajFxjjIkg4dK8Y4wxJgiW9I0xJoJY0jfGmAhiSd8YYyKIJX1jjIkglvSNMSaCWNI3xpgIYknfGGMiyP8PBVbH4PYmJd0AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(tr_loss_hist, label = 'train')\n",
    "plt.plot(val_loss_hist, label = 'validation')\n",
    "plt.legend()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "test acc: 99.34%\n"
     ]
    }
   ],
   "source": [
    "yhat = np.argmax(cnn.predict(x_tst), axis=-1)\n",
    "print('test acc: {:.2%}'.format(np.mean(yhat == y_tst)))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}