{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# CS 20 : TensorFlow for Deep Learning Research\n",
    "## Lecture 07 : ConvNet in TensorFlow\n",
    "Specification of SimpleCNN is same that of [Lec7_ConvNet mnist by high-level.ipynb](https://nbviewer.jupyter.org/github/aisolab/CS20/blob/master/Lec07_ConvNet%20in%20Tensorflow/Lec07_ConvNet%20mnist%20by%20high-level.ipynb)  \n",
    "Different things are as below.\n",
    "- Initializing weights of model by He initialization\n",
    "- Applying Batch normalization instead of Drop out  \n",
    "\n",
    "### ConvNet mnist with Weight initialization and Batch norm\n",
    "- Creating the **data pipeline** with `tf.data`\n",
    "- Using `tf.contrib.slim`, alias `slim`\n",
    "- Creating the model as **Class** with `slim`\n",
    "- Initializaing weights of model with **He initialization** by `slim.variance_scaling_initializer`\n",
    "- Training the model with **Batch normalization** technique by `slim.batch_norm`\n",
    "- Using tensorboard"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Setup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1.12.0\n"
     ]
    }
   ],
   "source": [
    "from __future__ import absolute_import, division, print_function\n",
    "import os, sys\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import tensorflow as tf\n",
    "%matplotlib inline\n",
    "\n",
    "slim = tf.contrib.slim\n",
    "print(tf.__version__)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load and Pre-process data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "(x_train, y_train), (x_tst, y_tst) = tf.keras.datasets.mnist.load_data()\n",
    "x_train = x_train  / 255\n",
    "x_train = x_train.reshape(-1, 28, 28, 1).astype(np.float32)\n",
    "x_tst = x_tst / 255\n",
    "x_tst = x_tst.reshape(-1, 28, 28, 1).astype(np.float32)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(55000, 28, 28, 1) (55000,)\n",
      "(5000, 28, 28, 1) (5000,)\n"
     ]
    }
   ],
   "source": [
    "tr_indices = np.random.choice(range(x_train.shape[0]), size = 55000, replace = False)\n",
    "\n",
    "x_tr = x_train[tr_indices]\n",
    "y_tr = y_train[tr_indices].astype(np.int32)\n",
    "\n",
    "x_val = np.delete(arr = x_train, obj = tr_indices, axis = 0)\n",
    "y_val = np.delete(arr = y_train, obj = tr_indices, axis = 0).astype(np.int32)\n",
    "\n",
    "print(x_tr.shape, y_tr.shape)\n",
    "print(x_val.shape, y_val.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Define SimpleCNN class by high-level api (slim)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "class SimpleCNN:\n",
    "    def __init__(self, X, y, n_of_classes):\n",
    "        \n",
    "        self._X = X\n",
    "        self._y = y\n",
    "        self._is_training = tf.placeholder(dtype = tf.bool)\n",
    "        \n",
    "        with slim.arg_scope([slim.conv2d, slim.fully_connected], activation_fn = tf.nn.relu,\n",
    "                            weights_initializer = slim.variance_scaling_initializer(), # He initialization\n",
    "                            biases_initializer = tf.random_normal_initializer(),\n",
    "                            normalizer_fn = slim.batch_norm,\n",
    "                            normalizer_params = {'decay' : .9, 'is_training': self._is_training}):\n",
    "            with slim.arg_scope([slim.conv2d], kernel_size = [5, 5], stride = 1, padding = 'SAME'):\n",
    "                with slim.arg_scope([slim.max_pool2d], kernel_size = [2, 2], stride = 2, padding = 'SAME'):\n",
    "                    \n",
    "                    conv1 = slim.conv2d(inputs = self._X, num_outputs = 32, scope = 'conv1')\n",
    "                    pool1 = slim.max_pool2d(inputs = conv1, scope = 'pool1')\n",
    "                    conv2 = slim.conv2d(inputs = pool1, num_outputs = 64, scope = 'conv2')\n",
    "                    pool2 = slim.max_pool2d(inputs = conv2, scope = 'pool2')\n",
    "                    flattened = slim.flatten(inputs = pool2)\n",
    "                    fc = slim.fully_connected(inputs = flattened, num_outputs = 1024, scope = 'fc1')\n",
    "                    self._score = slim.fully_connected(inputs = fc, num_outputs = n_of_classes,\n",
    "                                                       activation_fn = None, scope = 'score',\n",
    "                                                       normalizer_fn = None, normalizer_params = None)\n",
    "                    self.ce_loss = self._loss(labels = self._y, logits = self._score, scope = 'ce_loss')\n",
    "                    \n",
    "                    self._update_ops= tf.get_collection(tf.GraphKeys.UPDATE_OPS)\n",
    "                    \n",
    "        with tf.variable_scope('prediction'):\n",
    "            self._prediction = tf.argmax(input = self._score, axis = -1)\n",
    "        \n",
    "    def _loss(self, labels, logits, scope):\n",
    "        with tf.variable_scope(scope):\n",
    "            ce_loss = tf.reduce_mean(tf.losses.sparse_softmax_cross_entropy(labels = labels, logits = logits))\n",
    "            return ce_loss\n",
    "        \n",
    "    def predict(self, sess, x_data, is_training = True):\n",
    "        feed_prediction = {self._X : x_data, self._is_training : is_training}\n",
    "        return sess.run(self._prediction, feed_dict = feed_prediction)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Create a model of SimpleCNN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "550\n"
     ]
    }
   ],
   "source": [
    "# hyper-parameter\n",
    "lr = .01\n",
    "epochs = 30\n",
    "batch_size = 100\n",
    "total_step = int(x_tr.shape[0] / batch_size)\n",
    "print(total_step)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<BatchDataset shapes: ((?, 28, 28, 1), (?,)), types: (tf.float32, tf.int32)>\n",
      "<BatchDataset shapes: ((?, 28, 28, 1), (?,)), types: (tf.float32, tf.int32)>\n"
     ]
    }
   ],
   "source": [
    "## create input pipeline with tf.data\n",
    "# for train\n",
    "tr_dataset = tf.data.Dataset.from_tensor_slices((x_tr, y_tr))\n",
    "tr_dataset = tr_dataset.shuffle(buffer_size = 10000)\n",
    "tr_dataset = tr_dataset.batch(batch_size = batch_size)\n",
    "tr_iterator = tr_dataset.make_initializable_iterator()\n",
    "print(tr_dataset)\n",
    "\n",
    "# for validation\n",
    "val_dataset = tf.data.Dataset.from_tensor_slices((x_val,y_val))\n",
    "val_dataset = val_dataset.batch(batch_size = batch_size)\n",
    "val_iterator = val_dataset.make_initializable_iterator()\n",
    "print(val_dataset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "## define Iterator\n",
    "# tf.data.Iterator.from_string_handle의 output_shapes는 default = None이지만 꼭 값을 넣는 게 좋음\n",
    "handle = tf.placeholder(dtype = tf.string)\n",
    "iterator = tf.data.Iterator.from_string_handle(string_handle = handle,\n",
    "                                               output_types = tr_iterator.output_types,\n",
    "                                               output_shapes = tr_iterator.output_shapes)\n",
    "\n",
    "x_data, y_data = iterator.get_next()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "## connecting data pipeline with model\n",
    "cnn = SimpleCNN(X = x_data, y = y_data, n_of_classes = 10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Create training op and train model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "## create training op\n",
    "with tf.control_dependencies(cnn._update_ops):\n",
    "    opt = tf.train.AdamOptimizer(learning_rate = lr)\n",
    "    # equal to 'var_list = None'\n",
    "    training_op = opt.minimize(loss = cnn.ce_loss)\n",
    "\n",
    "#for tensorboard\n",
    "loss_summ = tf.summary.scalar(name = 'loss', tensor = cnn.ce_loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "## for tensorboard\n",
    "tr_writer = tf.summary.FileWriter('../graphs/lecture07/convnet_mnist_batch_norm/train/', graph = tf.get_default_graph())\n",
    "val_writer = tf.summary.FileWriter('../graphs/lecture07/convnet_mnist_batch_norm/val/', graph = tf.get_default_graph())\n",
    "saver = tf.train.Saver()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch :   5, tr_loss : 0.028, val_loss : 0.040\n",
      "epoch :  10, tr_loss : 0.018, val_loss : 0.049\n",
      "epoch :  15, tr_loss : 0.012, val_loss : 0.098\n",
      "epoch :  20, tr_loss : 0.007, val_loss : 0.046\n",
      "epoch :  25, tr_loss : 0.010, val_loss : 0.071\n",
      "epoch :  30, tr_loss : 0.007, val_loss : 0.106\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'../graphs/lecture07/convnet_mnist_batch_norm/cnn/'"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sess_config = tf.ConfigProto(gpu_options=tf.GPUOptions(allow_growth=True))\n",
    "sess = tf.Session(config = sess_config)\n",
    "sess.run(tf.global_variables_initializer())\n",
    "tr_handle, val_handle = sess.run(fetches = [tr_iterator.string_handle(), val_iterator.string_handle()])\n",
    "\n",
    "tr_loss_hist = []\n",
    "val_loss_hist = []\n",
    "\n",
    "for epoch in range(epochs):\n",
    "\n",
    "    avg_tr_loss = 0\n",
    "    avg_val_loss = 0\n",
    "    tr_step = 0\n",
    "    val_step = 0\n",
    "\n",
    "    # for mini-batch training\n",
    "    sess.run(tr_iterator.initializer)    \n",
    "    try:\n",
    "        while True:\n",
    "            _, tr_loss, tr_loss_summ = sess.run(fetches = [training_op, cnn.ce_loss, loss_summ],\n",
    "                                               feed_dict = {handle : tr_handle, cnn._is_training : True})\n",
    "            avg_tr_loss += tr_loss\n",
    "            tr_step += 1\n",
    "            \n",
    "    except tf.errors.OutOfRangeError:\n",
    "        pass\n",
    "\n",
    "    # for validation\n",
    "    sess.run(val_iterator.initializer)\n",
    "    try:\n",
    "        while True:\n",
    "            val_loss, val_loss_summ = sess.run(fetches = [cnn.ce_loss, loss_summ],\n",
    "                                               feed_dict = {handle : val_handle, cnn._is_training : False})\n",
    "            avg_val_loss += val_loss\n",
    "            val_step += 1\n",
    "    \n",
    "    except tf.errors.OutOfRangeError:\n",
    "        pass\n",
    "\n",
    "    avg_tr_loss /= tr_step\n",
    "    avg_val_loss /= val_step\n",
    "    tr_writer.add_summary(summary = tr_loss_summ, global_step = epoch + 1)\n",
    "    val_writer.add_summary(summary = val_loss_summ, global_step = epoch + 1)\n",
    "    tr_loss_hist.append(avg_tr_loss)\n",
    "    val_loss_hist.append(avg_val_loss)\n",
    "    \n",
    "    if (epoch + 1) % 5 == 0:\n",
    "        print('epoch : {:3}, tr_loss : {:.3f}, val_loss : {:.3f}'.format(epoch + 1, avg_tr_loss, avg_val_loss))\n",
    "\n",
    "tr_writer.close()\n",
    "val_writer.close()\n",
    "saver.save(sess = sess, save_path = '../graphs/lecture07/convnet_mnist_batch_norm/cnn/')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.legend.Legend at 0x7fe7ac1395f8>"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAD8CAYAAACb4nSYAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzt3Xl4lNXZ+PHvnX0jewhbIKAsgbBJWKyAC2rRVtC64Vawtqit1S7+3tq+bbXaRd+3r7WL+143pFqXKoqiKK4Ispmwy04gkEDITrbz++PMhCFMkkkyk9nuz3XlymTmmWfOk8A9Z865z33EGINSSqnwEOHvBiillOo5GvSVUiqMaNBXSqkwokFfKaXCiAZ9pZQKIxr0lVIqjGjQV0qpMKJBXymlwogGfaWUCiNRnhwkIjOBvwKRwGPGmLtbPT4duA8YA8wxxrzU6vFkYD3wqjHmpvZeKzMz0+Tm5np8AUoppeDLL78sNcZkdXRch0FfRCKB+4FzgD3AChF53Riz3uWwXcA84NY2TnMXsKyj1wLIzc1l5cqVnhyqlFLKQUR2enKcJ8M7k4Ctxphtxph6YAEw2/UAY8wOY8w6oNlNQyYA2cA7njRIKaWU73gS9PsDu11+3uO4r0MiEgH8H21/AlBKKdWDfD2R+0NgkTFmT3sHich8EVkpIisPHjzo4yYppVT48mQidy+Q4/LzAMd9njgVmCYiPwSSgBgRqTLG3OZ6kDHmEeARgIKCAq31rFQIaWhoYM+ePdTV1fm7KSEhLi6OAQMGEB0d3aXnexL0VwBDRWQwNtjPAa705OTGmKuct0VkHlDQOuArpULbnj176NWrF7m5uYiIv5sT1IwxlJWVsWfPHgYPHtylc3Q4vGOMaQRuAhYDG4CFxpgiEblTRGYBiMhEEdkDXAo8LCJFXWqNUirk1NXVkZGRoQHfC0SEjIyMbn1q8ihP3xizCFjU6r7futxegR32ae8cTwFPdbqFSqmgpwHfe7r7uwyZFbkVdQ385d3NrNld7u+mKKVUwAqZoG+a4a/vbWHljkP+bopSKoCUl5fzwAMPdPp5559/PuXlodeJDJmgnxwfRVSEUFZd7++mKKUCSFtBv7Gxsd3nLVq0iNTUVF81y288GtMPBiJCRlIMZVVH/d0UpVQAue222/j6668ZN24c0dHRxMXFkZaWxsaNG9m8eTMXXnghu3fvpq6ujltuuYX58+cDx0rCVFVVcd555zF16lQ+/fRT+vfvz2uvvUZ8fLyfr6xrQiboA2QkxlJWpT19pQLV7/5TxPriCq+ec2S/ZG6/YFSbj999990UFhayZs0aPvjgA771rW9RWFjYkvL4xBNPkJ6eTm1tLRMnTuTiiy8mIyPjuHNs2bKFF154gUcffZTLLruMl19+mauvvtqr19FTQivoJ8VQqsM7Sql2TJo06bgc97/97W+88sorAOzevZstW7acEPQHDx7MuHHjAJgwYQI7duzosfZ6W0gF/cykWLaXVvu7GUqpNrTXI+8piYmJLbc/+OADlixZwmeffUZCQgJnnHGG2xz42NjYltuRkZHU1tb2SFt9IWQmcgEyEmN0eEcpdZxevXpRWVnp9rEjR46QlpZGQkICGzdu5PPPP+/h1vW8kOrpZyTFUtvQRE19IwkxIXVpSqkuysjI4LTTTiM/P5/4+Hiys7NbHps5cyYPPfQQeXl5DB8+nClTpvixpT0jpCJjRlIMAGVV9SSkh9SlKaW64fnnn3d7f2xsLG+99Zbbx5zj9pmZmRQWFrbcf+utwV0pPqSGdzKdQV8nc5VSyq2QCvrpiXayRXP1lVLKvZAK+hmJx4Z3lFJKnSi0gr5jeKe0Wnv6SinlTkgF/YSYKBJiIrWnr5RSbQipoA9o/R2llGpH6AX9xFjN3lFKdVlSUhIAxcXFXHLJJW6POeOMM1i5cmW757nvvvuoqalp+TlQSjWHXNDPTIqhVId3lFLd1K9fP1566aUuP7910A+UUs0hF/RtpU0d3lFKWbfddhv3339/y8933HEHv//975kxYwannHIKo0eP5rXXXjvheTt27CA/Px+A2tpa5syZQ15eHhdddNFxtXduvPFGCgoKGDVqFLfffjtgi7gVFxdz5plncuaZZwK2VHNpaSkA9957L/n5+eTn53Pfffe1vF5eXh4/+MEPGDVqFOeee65PavyE3LLVjKQYDlXX09xsiIjQfTmVCihv3Qb7v/LuOfuMhvPubvPhyy+/nJ/85Cf86Ec/AmDhwoUsXryYm2++meTkZEpLS5kyZQqzZs1qc//ZBx98kISEBDZs2MC6des45ZRTWh77wx/+QHp6Ok1NTcyYMYN169Zx8803c++997J06VIyMzOPO9eXX37Jk08+yfLlyzHGMHnyZE4//XTS0tJ6pIRz6PX0k2JpbDZU1DX4uylKqQAwfvx4Dhw4QHFxMWvXriUtLY0+ffrwq1/9ijFjxnD22Wezd+9eSkpK2jzHsmXLWoLvmDFjGDNmTMtjCxcu5JRTTmH8+PEUFRWxfv36dtvz8ccfc9FFF5GYmEhSUhLf+c53+Oijj4CeKeHsUU9fRGYCfwUigceMMXe3enw6cB8wBphjjHnJcf844EEgGWgC/mCMedF7zT+RsxRDaVU9qQkxvnwppVRntdMj96VLL72Ul156if3793P55Zfz3HPPcfDgQb788kuio6PJzc11W1K5I9u3b+fPf/4zK1asIC0tjXnz5nXpPE49UcK5w56+iEQC9wPnASOBK0RkZKvDdgHzgNZVjWqA7xpjRgEzgftExKczGRmOUgyHNINHKeVw+eWXs2DBAl566SUuvfRSjhw5Qu/evYmOjmbp0qXs3Lmz3edPnz69pWhbYWEh69atA6CiooLExERSUlIoKSk5rnhbWyWdp02bxquvvkpNTQ3V1dW88sorTJs2zYtX2z5PevqTgK3GmG0AIrIAmA20fIYxxuxwPNbs+kRjzGaX28UicgDIAnyWt3Ss0qZO5iqlrFGjRlFZWUn//v3p27cvV111FRdccAGjR4+moKCAESNGtPv8G2+8kWuvvZa8vDzy8vKYMGECAGPHjmX8+PGMGDGCnJwcTjvttJbnzJ8/n5kzZ9KvXz+WLl3acv8pp5zCvHnzmDRpEgDf//73GT9+fI/txiXGmPYPELkEmGmM+b7j52uAycaYm9wc+xTwhnN4p9Vjk4CngVHGmObWjzsVFBSYjvJf23Ogso5Jf3iPuy7M55opg7p8HqWUd2zYsIG8vDx/NyOkuPudisiXxpiCjp7bIxO5ItIXeAa41l3AF5H5IrJSRFYePHiwW6+VnqA9faWUaosnQX8vkOPy8wDHfR4RkWTgTeC/jTFu9yIzxjxijCkwxhRkZWV5emq3oiIjSEuI1vo7SinlhidBfwUwVEQGi0gMMAd43ZOTO45/BfinuyEfX0lPjKFMK20qFTA6GkZWnuvu77LDoG+MaQRuAhYDG4CFxpgiEblTRGYBiMhEEdkDXAo8LCJFjqdfBkwH5onIGsfXuG612AMZSbFaikGpABEXF0dZWZkGfi8wxlBWVkZcXFyXz+FRnr4xZhGwqNV9v3W5vQI77NP6ec8Cz3a5dV2UmRTDpv0npkoppXregAED2LNnD92dr1NWXFwcAwacEG49FnJlGMBZabPM381QSgHR0dEMHjzY381QDiFXhgFsrn55TQMNTW1mhiqlVFgK0aBvV+Ue1lW5Sil1nJAM+pmJx+rvKKWUOiYkg76zp69pm0opdbwQDfrOVbna01dKKVchGfQzHZU2S7UUg1JKHSckg35yfBRREaLllZVSqpWQDPoiQkZSjA7vKKVUKyEZ9MG5QEuHd5RSylXoBv2kGE3ZVEqpVkI26GcmaU9fKaVaC9mgn5GoY/pKKdVayAb99KQYauqbqKlv9HdTlFIqYIRs0Hfm6mtvXykVFLYssV8+FpKllcFlVW51PTnpCX5ujVJKdeDje6G5EYae7dOXCdmefkv9HV2Vq5QKdMZASSFkj/L5S4Vu0E/U+jtKqSBRsRfqjmjQ7w7n8E6ppm0qpQJdiWNb8ex8n79UyAb9hJgoEmIitaevlAp8JYX2e+88n79UyAZ9wFF/R3v6SqkAV1IEqQMhLsXnL+VR0BeRmSKySUS2ishtbh6fLiKrRKRRRC5p9dhcEdni+JrrrYZ7wtbf0Z6+UirAlRRBb9+P54MHQV9EIoH7gfOAkcAVIjKy1WG7gHnA862emw7cDkwGJgG3i0ha95vtmUyttKmUCnQNdVC6pUcmccGznv4kYKsxZpsxph5YAMx2PcAYs8MYsw5obvXcbwLvGmMOGWMOA+8CM73Qbo+0WWmzvgYq9vVUM5RSqm2lm8A0BVTQ7w/sdvl5j+M+T3Tnud3mrKlvjDn+gbd/AY+f21PNUEqptpWst997IHMHAmQiV0Tmi8hKEVl58OBBr503IymWxmZDRa1L/Z3Go1D0GhzZBfXVXnstpZTqkpJCiIqD9CE98nKeBP29QI7LzwMc93nCo+caYx4xxhQYYwqysrI8PHXHMt3l6n+9FI4esbePeHoZSinlIyVFkDUCInumKo4nQX8FMFREBotIDDAHeN3D8y8GzhWRNMcE7rmO+3pEhruia0WvHLt9ZFdPNUUFk9rD8OhZsG+dv1uiwkFJUY8N7YAHQd8Y0wjchA3WG4CFxpgiEblTRGYBiMhEEdkDXAo8LCJFjuceAu7CvnGsAO503Ncj0ltKMTh6+o1HYdMiGHKm/bl8dxvPVGFt1+ew90vY6vuKhyrMVR2A6gM9NokLHlbZNMYsAha1uu+3LrdXYIdu3D33CeCJbrSxy44N7zh6+l+/D0crYMqNsOMjOKJBX7mxd5X9XrrZv+1Qoa+l/ELPBf2AmMj1lbTWPf2iVyEu1fb0k/tpT1+5V7zafj+4yb/tUKFPg753RUdGkJoQbcf0G+rs0E7etyEqBlIGhlZPv6HWlmdV3WPMsaBfull/p8q3SoogKRsSM3vsJUM66INjr9zqo8eGdkZeZB9IzQmdnn7NIfifIbDxTX+3JPgd2Q01pdB7JNRX2ZK3SvlKD9XQdxX6QT8pltKqeljvHNo53T6QkgOVxdDU4N8GekPpZmiosfMUqnucvfyxc+x3HeJRvtLUaP99adD3rsykGCoqK2GjY2gnMto+kJoDphkqiv3bQG8od6Se7i/0bztCQfFqiIiGUd+xP+tkrvKVQ19D09EeTdeEMAj6GYmxDK9eAfWVMOqiYw+kONaMhcK4fvlO+73kKx2D7q69qyB7JKQMgPg07ekr33HW0NeevndlJMVwRuMnmPg0GHz6sQdSB9rvoTCu7+zp1x2BI3v825ZgZgwUr4F+40EEModrT1/5TkkRRERB5rAefdmQD/pZ8XB2xCrqTjrv2NAOQLKj7ltI9PR3QaRdfdzSe1Cdd2ibLdHR7xT7c9Yw7ekr3ykpsgE/KrZHXzbkg/6Iqi/oJbUcHHj+8Q9Ex9lUqfIQKMVQvgsGT7e3dVy/65yTuP3G2++Zw20mT3WZ/9oUyHZ+qhlj3VFS1ONDOxAGQX/Q/nc4ZJLYnVJw4oMpOcHf029utkNU2SMhLdeO66uuKV5tqx069ynNGm6/l2pv/wTGwGs32a/m1ttoqA7VltvYo0HfyxpqSd+zhMVNEymtdfMPMzUn+MfAq/ZDc4Odo8jO155+d+xdBX1GHxsGdI616hDPifZ+abNPag/BwQ3+bk3wOeCood9DWyS6Cu2gv/U9IhqqebN5is3Vby3FEfSDOePFOTyVOsgGrEPbdJ+Armhugn1rjw3tgP33EZ2gk7nurF1gJyEBtuv6kE7zQ/kFp9AO+kWvYOLTWcGoY/V3XKUOhMY6qPbexi09riXoO3r6mGM78SjPlW6Bhurjg35EBGQO1Z5+a431UPgS5F1gOxu6KLDzSorsYtHkfj3+0qEb9BtqYfPbSN4FpCbFu98gPcVRGDSY0zadOfopOdDHschDx/U7r9hRWdOZueOkaZsn2vqu3XNg7BUweBrs+FjH9TvLWUNfpMdfOnSD/tYltnbKqItIb2uD9JYFWkGcwVO+CxJ7Q0yC7XXFJuu4flcUr4boRNuzd5U1zE64Ha3yT7sC0doFkJAJJ50FudOhrlxThTujudmO6fthaAdCOegXvQoJGZA7jcykGPdj+qmOoB/UPf1dxxaaidjeg/4H7Lzi1dB3LEREHn9/pjODR3v7gO3hb34bRl9iJ7xzp9r7dYjHc+U7bYdUg74XNdTCprfsmGNk1LFKm63FpUBsSnCnbboGfbBDPCVF+nG7M5oaYP9X0P+UEx/L0qB/nKJXoakexlxuf07pbzf03vGxf9sVTFomcXu25o5TaAb9Le/aSbmRFwK20qbbMX0I7hLLzhx916CfnW97EeU7/NasoHNgg53Qd53EdUofYrNUdDLXWveiTWV1/V3lToUdn9gMKNWxkiJAoPcIv7x8aAb99ceGdsDW36mpb6KmvvHEY4N5gZZrjr6TczJXx/U913olrqvIaBv4tacPh3fArs9sL991AjJ3ui1fsV8TCDxSUmj/TcUk+uXlQy/oN9TCprchbxZE2jzizERb28Jtbz+Ye/quOfpOvUeCROi4fmcUr7LDfOlD3D+eNVx7+gDrFtrvYy47/n4d1++ckiK7gt5PQi/oO4d2Rl3YcleGY4P0suo2FmgdPWIrVAYb1xx9p+h4yDhZe/qdUbwa+o1rO30uc7hd9NbYxhBhODAG1r4Ag6Ye/+8NILmv/Teni7Q6Vl9j/y35aTwfPAz6IjJTRDaJyFYRuc3N47Ei8qLj8eUikuu4P1pEnhaRr0Rkg4j80rvNd6PoFZtONmhqy10ZSbanf8jdZG4wZ/A4c/Sd1+CUna+5+p5qqLOL2dxN4jplDQfTZMsOhKs9K22wcu4o1lruNDv00+RmCFUdc3ADYPyWuQMeBH0RiQTuB84DRgJXiEjrzybXAYeNMScDfwHucdx/KRBrjBkNTACud74h+ER9jU0nc2TtOGUk2p6++1IMjl5LMI7rO3P0o+OPv79Pvn0sGD+99LSSIjsv4m4830lr8MC6BbYY3cjZ7h/PnWr3oN6/tmfbFWz8WH7ByZOe/iRgqzFmmzGmHlgAtP7Lzwaedtx+CZghIgIYIFFEooB4oB6o8ErL3dn6rt0r1nWHLFyGd0ItV791uqZT9mj73fkPTLWtZSVue0HfsWArXCdzG+uh8GUYfj7EJbs/xpE0oUM8HSgpsosAU3P91gRPgn5/wDUi7nHc5/YYY0wjcATIwL4BVAP7gF3An40xh7rZ5rYVvQKJWTDotOPuToiJIiEm0n39ncQsuwFJsPb03QV9zeDxXPFqOxyYktP2MTGJ9hNhuPb0W8outDG0A9Ar234i0nz99jkncSP8N53q61eeBDQB/YDBwM9F5IQUCRGZLyIrRWTlwYNdLH5WXwObF58wtOOUkRTjfiJXxNbgCbag7y5H36lXX4hP13F9TxSvPrY9YnuyhoVvXX3XsgvtaRnXb+iZdgUbY2xWnR+HdsCzoL8XcO0GDXDc5/YYx1BOClAGXAm8bYxpMMYcAD4BTtjNxBjziDGmwBhTkJWV1fmrADueOHI2jL7M7cMZibGUuuvpQ3CmbbrL0XcSsb197em3r74aDm5sf2jHKWuErcQZbguQWpddaM/gaXZh4D4d13ercp/9ffoxcwc8C/orgKEiMlhEYoA5wOutjnkdmOu4fQnwvjHGYId0zgIQkURgCrDRGw0/Qa8+cNFDMOhUtw9nJsW0vSo3GBdoucvRd5U92q40Dbcg1Rn71oFpbj9zxylzmF21Gwrba3ZG67IL7XFmzG1f5ts2BasAmMQFD4K+Y4z+JmAxsAFYaIwpEpE7RWSW47DHgQwR2Qr8DHCmdd4PJIlIEfbN40ljzDpvX4QnMtqqtAm2t1xVYtP3gsVhZ7qmm54+2J5+Yy2UhXGaYUecK3H7juv42HCtwbN2wYllF9qSlAVZebpIqy3OBZPO7Tj95MTBbzeMMYuARa3u+63L7Tpsembr51W5u98fMhw9fWMM0nr81jmJV7EXMk7q+cZ1RUtPv40JyGyX2vpZw3qmTcGmeLWd/0ju2/Gxrmmbw77p23YFikPbYffnMOO3ntd9z50Ka56z4/odDQeFm5L1kDwA4tP82ozQW5HbhvTEGBqbDRW1bhaPtKRtBtFH9/Kd7nP0nbKG20JhOq7ftuJVJ26a0paEdJvpFU6Tuc6yC23Mk7k1eJpNm967yjdtCmYlRX4f2oEwCvqZjlW5pe1uphJE4/ptpWs6RcXa8gFag8e9uiNQttWzYQunzOFwMEyGd4yxC7Jyp7X9adKdQVqHx63Getth0KDfc9pdoJXczxYpC6YMno6CPmgGT3ucGSb9OxH0s4bb/7jG+KZNgcRZdsGTCVxXiRnQe5QG/dZKN0Nzowb9npTRUmnTTU8/Mhp69Quenn5zExzZ03HQz86HymKo8d16uKDlHH7o28mgX3fETvqHuo7KLrRn8DTYtRwa20icCEd+3jjFVdgE/UxHT7/U3QItCK5c/cp2cvRdtazM1UVaJyhebX9/iRmePydcavB4UnahPblTbeaYjusfU1IIkTG2GqmfhU3QT0t0Du+00ftIyQmeDdI7ytF3aqnBo0M8J+jMJK5TuKRtelJ2oT2DTgNEh3hclRTZBX5uqgX0tLAJ+tGREaQmRHOovZ5+RXFwLGZyV0ffnaQsSMrWcf3Wqsvs77Azk7hg0ztjeoV+T3/tC56VXWhLQrodxtBFWseUFAXE0A54mKcfKjIS21uVO8BOtFTus7cDWUc5+q60tv6J9rWzPWJ7REK/Bk/tYVvDquB73cuzHzwNVj5hFzxGx3mvfa5Kiux8VUSU4yvS5bbzZ8d9CZm+a0dHqsts2RQ/7pblKryCflI79XecdfXLdwdB0O8gR99Vn3z4/EFdLONqrzPoe7ASt7XM4fD1+95tTyDZ+p4tu5B/cffOkzsNPn8A9q48tp2iN61/HRZe4/nx8Wkw6XqYfL39JNKTDgRG+QWnsAr6mUkxbC6pcv+gs9d8ZE/PNairPEnXdMoebf8Tl24OmH90fle82k6oxaV0/rlZw2Ht81BbDvGp3m+bv21dYiu09p/QvfMMOhU7rv+x94N+fQ28/Uv7KXbm3fYTummyQ7PNjS5fjp+b6mHzO/Dh3fDp32DCPDj1Rz3XuQugzB0Is6CfkRhLWVWZ+wed/wCCYTK3M+PRrrX1Nehbxash97SOj3PHdTI3Z5L32hQImptt0D/pLDss0h3xadB3jN1U5YwTdljtno/vhYo9cPGjMOgbnj1nwjxbgPDj+2D5w/DFo3YNwtSfHNskx1dKCu1q7qTevn0dD4XNRC7YBVqHaxpobGo+8cGYREjICPy0TU9z9J0yhtpNYnRc36rcb9cudDZzxymU0zb3r4XqgzD0HO+cL3ca7PkCGmq9cz6wBQQ/+astDeFpwHfqnQffeRhuWQMF10LhS/CPifDiNb5NLw2Q8gtOYRb0HRuk1wRxiWVPc/SdIqOg9wjN4HEq7uIkrlNarn0TDcXJ3C1L7PeuZu20ljvNDq3sWeGd8wEs/pXNdz/nzq6fI3UgnP+/8JNCmPZz2PYhPHomPD0Ltn3g3RXXzU32E0aADO1AmAX9zMR2SjFAcCzQ8jRH31X2aM3VdypebUtu9B3TtedHRNr5gFCswbP1XVtm2lvDEINOtb9rb+2bu3mx3dDl9F94Vhm1I0lZMOM38NNCOPt3dkOdf86Gv+TbOYNdn9shr65orLdvJm/9wu7DEEA9/fAa009ylmJoq6c/0GYvGON5Kdme5mmOvqs++bDmWagssXuZhrO9q+wimZjErp8ja9ixTwyhouaQ7ZFP+7n3zhmXAn3HemeRVkOdDaCZw2DyDd0/n6u4ZDu2P/kGWP+q3ThmxWM2+yipj92CdeRsO5zU3lxH1UH7xrl5sc3wOlphP5WcfA6cfLZ329wNYRb0HT39NjdTybFlYWsOdW55fk/qTI6+k2tt/XAO+sbYYN3deviZw21gaKj1LG02GGxbancRO9lL4/lOudNsynB9DcQkdP08n/0DDm+Ha16BqBjvtc9VdJxdhTx2DtRV2OC94TVY/QyseNROxo74tn0DyJ3qKF2+zmYGbX4b9n4JGPtGMepCGPpNGHIGxCb5pr1dFF5B3zG8U9retolgM3gCNuh3IkffyTWDJ4B6HD3uyB6oKe36eL5T1nDA2D1zuzpMFGi2LIG4VBhwwhbW3ZM7zaZJ7l4OJ53ZtXOU74Zlf7Y9bm/NN3QkLhnGXGq/jlbZHvz61+weA18+abOTouJtUgDYFNczfmk7FH3GQETgjpyHVdBPjosmKkLarr/TspnK7u4HBl/pTI6+U3ya3bEn3Mf1ix0ZGl3N3HFyTdsMhaDvzVTN1gadChJp8/W7GvTf+TVg4Jt/9GrTPBabBKMusl8NtXYIeMPrtoro0HPsp6Mg+gQdVkE/IkJIb7cUQxBsptKVmjGgtfXBDu1ERHV/Ui3jZDtBGSppm/vXQfUB76VquortZf+9dnVcf9sHdpz9zP/ufGfHF6LjIe/b9itIBe5nEB/JSGpng/T4NIhODNwMns7m6LvKzrc902Da/N2bmpvt0v0BE7tfgyUq1qZuhkra5tZ37XdfDf3lTrXj3UfbWA3flqYGWPRf9nf9jZt90rRwFHZBPzMppu0xfRE7xBOoPf3O5ui76pNvl6of3Oj9dgWDbUvh0Ncw4VrvnC+Utk7cssRm2fhqxejgabYcwhs/7VyH6otH7BvrN//kv2JpIcijoC8iM0Vkk4hsFZET1lSLSKyIvOh4fLmI5Lo8NkZEPhORIhH5SkT8+tfLSIxpu7wy2CGeQN0gvSs5+k7O2vrhuqHKisdspcVRF3rnfFnD7B67TY3eOZ+/1B62q2a9nbXjasiZMOVHdpjm76fAov9nOzDtqSyBpX+y7Rp+nu/aFoY6DPoiEgncD5wHjASuEJHWNUKvAw4bY04G/gLc43huFPAscIMxZhRwBtDgtdZ3QUZSbNsTuRDYPf2u5Og7pQ+G6ATPJ3OLXoEHToWvl3b+tQJN+S6bUjdhrh2a8YbM4fZbMixvAAAgAElEQVRT1+Ht3jmfv2z7wKZq+mI83ykiEmb+EX68yqZDrngc/joO3vlN21t5Lrkdmo7CefcE7pqZIOVJT38SsNUYs80YUw8sAFpvnDkbeNpx+yVghogIcC6wzhizFsAYU2aM8esuJRlJMVTXN1Fb30YzUnJs76ez4489oSs5+k4RkdB7ZMeTuVUHYeF34V/z7PLxRf/Pjq0Gs5VP2O/eGtoBu8ALgn8yd8sSu4iqv5dTNd1JzYFZf4ebVtj0y0//DveNgaV/tHsPO+363G7kcupNkHGS79sVZjwJ+v0B167vHsd9bo8xxjQCR4AMYBhgRGSxiKwSkf/qfpO7J9O5QXqbC7QcvehALLHclRx9V30cG6q4qy1iDHz1Etw/CTa9BTNuh8ufgbIt8OVT3Wq2XzXUwap/2v1eu/Jm2RZnZcZgnsw15liqZk9u45dxkq2Q+cPPbBrnh/fY4P/RvXZR1KJbIbk/TL+159oURnw9kRsFTAWucny/SERmtD5IROaLyEoRWXnw4EGfNqhlVW4wpm12JUffVXa+7VG1fkOr3A8LroKXr4P0IXD9RzDtZ3b1Ye40+OBPx/fEgsn6V6GmDCZ+37vnjUuGXv2CezJ3/1d2Rydfjue3p3ee7VjM/9CWqX7vd/B/w227zv1990plqDZ5EvT3Aq5dpAGO+9we4xjHTwHKsJ8KlhljSo0xNcAi4ISVMcaYR4wxBcaYgqysrM5fRSe01N9prxQDBOZkbneDfp9WG6UbA2tesL37r9+Dc+6C696xVTnBjqWee5cNmh//pXtt95cvHrXlpYec4f1zB/vWib5O1fRUv3Fw1b/ge+9AzmQY9R27EEr5hCdBfwUwVEQGi0gMMAd4vdUxrwNzHbcvAd43xhhgMTBaRBIcbwanA+u90/Su6bAUQ1IfiIgOvJ5+d3L0nZyLkvYXwpG98Pxl8OoNkJUHN3wCp9184orMfuNhzBz47IHAXb/Qlr2r7HZ9E7/vm8lAZ9qmJ6V4966Ct39lf++BYssSWzIgUFaTDpwM330VLn1SJ299qMOg7xijvwkbwDcAC40xRSJyp4jMchz2OJAhIluBnwG3OZ57GLgX+8axBlhljHnT+5fhuQ6HdyIiIKV/4AW47uToO8X2sgtd1i2AB6bYkrcz74ZrF0HmyW0/76xf2/+E73Wjhrk/rHjMLrYbd4Vvzp81DBqq257/McaWH3jmIluv/fP74U0vVrHsjtpyWw/H37181eM8mr0xxizCDs243vdbl9t1wKVtPPdZbNpmQEiIiSIhJrL9tM1A3EzFOdyU1oUcfVd9RsOG/8CgqTD773YMvyOpOTDlh3abuik3Qv9u1q7pCTWHoPBlGHtF1/bC9USmswbPpuMniY2BLe/AR/9nA2tiFpx9h63b8uE9tnpjdyt9dte2D+xiPV+maqqAFHYrcsH29svaW6CVOjDwevrdWZjlasbtcOlTMPc/ngV8p6k/tYub3vmNd3cW8pXVz9jNKyb9wHev0ZK26ZjMbW6ybzQPTbNDZxXFcP6f4Sdf2d/ftFttPfi3fuH/chhb34XYFBgQYvv8qg6FZdBPT4yltKOefuU+u/tNoHAGfecG7l2VOdROknW29Gtcst3geufHNqXT1+oqOl612ZbmJrsAaOA3fLtjUWKmrddUUmjTQv8xEV76nl1UdOGDcPNq+6bjTLGNirGLjQ5vtznq/mKMrRR50hk9m6qpAkJYBv3M9iptgiOwGqgIoEm38p2QlO3fTTsmzLM91Xd/49sFWzs+thlF/5hoN5XurK1L7O9rkpfTNFsTsUM8a56D139sS/Be9gz8cDmMuxIio098zklnQd4sO/TjrwyxkiLbqfFXqqbyq/AM+kmx7K+oo7m5jWGK1ADM1S/f6f/SspHRdkPqsq2+WbDV3AQf3ANPX2BLRsQkwrMXd36o7YtH7RvkiAu838bWxl5ug+fVL9t885GzOv4U5awLv/hXvm+fO4GSqqn8IiyD/qknZXCoup7PtpW5P8C5QCuQxvW7m6PvLcNm+mbBVsU+uyn1B3+E0ZfC9R/aQFpfbQN/WzVaWju0zfb0J8zz3bZ6rgq+B1e/ZAOop2mGqTkw/ed2Qn3re75tnztbltgCfN7YXFwFnbAM+jPz+5ASH82CFW0Edee4eaD09L2Ro+8tvliwtWUJPOSouT77AbjoYZtemj0K5jxvx8BfuMJmv3RkxeN2rYE36+z4wjduthPpb/2iZ+eO6ipg9+cwVHv54Sosg35cdCQXje/P4sL97sssR8XaRVqB0tOv3GfrkQdC0AfvLdhqarDZQM9dbIdj5n8A4686vsc8eJp9E9i9HF7+vn0DbEt9Dax+1paPCPRebFQszLzH1jb6/IGee91tH9h/SzqeH7bCMugDzJmUQ31TM/9e1cbCmkAqsdydksq+0t0FW4d3wpPn2U2zC74HP3jv2N6zreV/B2b+CTa+Yat+tpUyWvgy1JX7Nk3Tm4adawvBffg/XVup29wM2z60b3ae2vouxCbbWjcqLIVt0B/RJ5lxOam8uGI3xl0QCaQFWt7K0fcm54KtrxbaEgOdsf51eHiaLUt86VPw7b90nJU05UY7JLLycfjozyc+bgyseNSWlBh0Wufa408z/2R73u/+pnPPK98N/5xlvx6YbBd8dcQYO5Q25Az3mUUqLIRt0AeYMzGHLQeqWLXr8IkPpubYcfTm5p5vWGveytH3Nk8XbNWW203JC1+G134EC6+B9JPg+mWdK6x19u9g9GXw/u/tMI6rPSth31qbphlMdVvScu3vsfBl2L6s4+ONgXUL4cHT7O/0zF9DVLxdDPbi1e1/YjiwHiqLdRVumAvrlRkXjO3HXW+sZ8EXu5kwKP34B1NyoKkeqg9Arz7+aaBTIOTou+NcsLXoVli7wAaww9ttBs2h7cdu17q8qUqE3Rxjxu2dz66JiIDZ90P1QXj9Zru3wLBz7WMrHoWYXjDmcq9dXo+Z+hNY+7zdBPyGj9ruhdccgjd/Znc1y5kCFz1kd0Q77Rb47O/w4f/C15PgjF/C5BtOXHi1RVM1VZgH/cTYKGaN68erq4v57QUj6RXn8p/NOX5evjsAgn6ApGu6M2Ge3cD61RuO3ScR9k0zfTCMvNBmqaQPhrTB9o0hNqnrrxcVY2uwP3k+/GsuzH3D/m6KXrFtie3VzQvyg+h4W/huwZX2d3nqj0485uv34dUf2je8Gb+F035yrCJqVAxM+znkX2zfON75b/sm/O17jx+737rE7qmQ3K9nrksFpLAO+gCXTxzIC1/s5rU1xVw9xWXMvGUzlV2QM9E/jXMq3wX9J/i3DW2JjLarUHd8ZIN6+mD7u/NljnxsL7jqJXj8HHj+Uput01Tv/Y1SetLw821GzdI/Qf4lx8odN9TCkjtg+UN29e8VC2z9eXfScuHKF+2E91u/sL+fU+baYm8RUbDrM/spS4W1sB7TBxg7IIURfXrxYuucfef4ub/TNgMpR78tvUfYjJmhZ9ut8HpiUVSvbLj63/b2qqdh8PS2s3+CgYity9N0FN51FLAtXgMPT7cBf/KNdsFaWwHf9Tx5F8CPvoBv/NjOffyjwPb+mxt1aEdp0BcR5kzM4au9Ryjc67LCNC7ZluT1dwZPoOXoB5LMk+HKhbaHOy1A6tR3R8ZJNlCvWwD/uQUemwFHK+GaV+C8uzs3pxObZLccvH4ZZJxsC8LF9IKBU3zXfhUUwj7oA1w0fgCxURFuevsBUGI5EHP0A8mAArhlrW+2Q/SHaT+H5AG2tlHeLLjxU1ukrav65MO1b9sFbrP+qqmaSsf0AVISojl/dF9eXbOXX52fR3yMY4IsNQcO7/Br2wIyR1/5TkwiXLXQpl4OPcc76acRETB2TvfPo0KC9vQdLp+YQ2VdI29+te/YnSk5tqfvz01DAjVHX/lO9iibihpM6w1U0NCg7zB5cDqDMxN5cYVLjfPUHKivtEv7/SVQc/SVUkFJg76DiHD5xBxW7DjM1gOV9s5AKLEcyDn6Sqmgo0HfxcWnDCAqQo5N6Do3U9n4Bmz/CPats0G47kjPlWfQoK+U8iKPJnJFZCbwVyASeMwYc3erx2OBfwITgDLgcmPMDpfHBwLrgTuMMW6qZQWGrF6xnJ2Xzcur9nLrN4cTm34SRMXBh/fYL1cSYasVxqfa1M64VDvuPug0Ww7YG4HamaPfmfo0SinVjg6DvohEAvcD5wB7gBUi8roxZr3LYdcBh40xJ4vIHOAewLUIyr1AD+ym3X1zJuXwdtF+lqw/wLfG9IWfrrd75daV2x5+reO7u583v233SwUb9HOnOb6mHvvU0Bmao6+U8jJPevqTgK3GmG0AIrIAmI3tuTvNBu5w3H4J+IeIiDHGiMiFwHag2mut9qFpQ7PonxrPghW7bNBPzLBfnmhuhoMb7cbeO5bBprdc3gQG2TeAwdPsp4GUAR1nZ2iOvlLKyzwJ+v0B15nMPcDkto4xxjSKyBEgQ0TqgF9gPyXc2v3m+l5khHBpwQDuW7KF3YdqyElP8PzJERGQPdJ+TZ7veBPYYOcDdnwEm96ENY6SwAkZ0Hsk9M5zfB9pyxnEpRw7n+boK6W8zNeLs+4A/mKMqZJ2erUiMh+YDzBwoP97tZcV5PDX97awcOVufn5uN+q5RETYnOvsUTDlBvsmcKAIdn4KJYVwYAOseR7qq449J3mA440gz5YlBs3RV0p5jSdBfy/gOiA9wHGfu2P2iEgUkIKd0J0MXCIi/wOkAs0iUmeM+Yfrk40xjwCPABQUFPhxJZTVLzWe04dlsXDlbm6ZMZSoSC8lOUVEQJ/R9supudnW9zmwwW5ycWCD/dr+oa0cmTpIc/SVUl7jSdBfAQwVkcHY4D4HuLLVMa8Dc4HPgEuA943dg3Ca8wARuQOoah3wA9WciQO54dkv+XDzQWbkZfvuhSIiIG2Q/Ro+89j9TQ22px+T6LvXVkqFnQ67sMaYRuAmYDGwAVhojCkSkTtFZJbjsMexY/hbgZ8Bt/mqwT1lRl5vMpNieeELPy3Mioy2pYJ1aEcp5UUejekbYxYBi1rd91uX23XApR2c444utM9voiMjuGTCAB79aBslFXVkJ8f5u0lKKdVtuiK3HXMm2qmMuU98caw0g1JKBTEN+u3IzUzksbkFHKg8ygV//4SFK3Zj/FlxUymlukmDfgfOHN6bt26ZxviBqfzXy+u4ZcEaKusa/N0spZTqEg36HshOjuOZ6yZz67nDePOrfXzrbx+zdrcfyy0rpVQXadD3UGSEcNNZQ3lx/hSamg0XP/gpjy7bRnOzDvcopYKHBv1OKshNZ9HN05iR15s/LNrA955eQWnVUX83SymlPKJBvwtSEqJ56OoJ3HVhPp9+Xcb5f/2IT7eW+rtZSinVIQ36XSQiXDNlEK/96DR6xUVx1ePL+f0b6zlQWefvpimlVJs06HdTXt9k/vPjqcyZmMPjn2xn6t1L+fnCtawvrvB305RS6gQSaHnnBQUFZuXKlf5uRpfsKK3myU+2868v91BT38Q3TsrguqmDOXN4byIiOqidr5RS3SAiXxpjCjo8ToO+9x2paeCFFbt4+tMd7DtSx5DMRK49LZeLJwwgIcbX1ayVUuFIg34AaGhq5q3C/Tz+8XbW7i4nJT6aKycPZO6pufRJ0Vo+Sinv0aAfQIwxrNp1mMc+2s7iov1EiPD7C/OZM8n/G8YopUKDp0Ffxxp6gIgwYVA6Ewals/tQDf/9aiG/fOUrEmKjmDW2n7+bp5QKI5q908Ny0hN45JoJTMxN52cvruH9jSX+bpJSKoxo0PeDuOhIHp9bQF7fZG58dhWfbyvzd5OUUmFCg76f9IqL5unvTWJgegLXPbVCC7gppXqEBn0/Sk+M4ZnrJpOeFMPcJ79gc4lu1KKU8i0N+n7WJyWO566bQkxkBFc/tpxdZTX+bpJSKoRp0A8AAzMSePb7k6lvauaqxz9n/xGt36OU8g0N+gFiWHYvnr52Eoeq6rnm8eUcqq73d5OUUiHIo6AvIjNFZJOIbBWR29w8HisiLzoeXy4iuY77zxGRL0XkK8f3s7zb/NAyNieVx+ZOZOehGuY+8UW72zLWNTSxZnc5zy3fyS///RWXPfwZd72xni93HtaNXZRSbepwRa6IRAKbgXOAPcAK4ApjzHqXY34IjDHG3CAic4CLjDGXi8h4oMQYUywi+cBiY0z/9l4vFFfkdtZ7G0q4/pkvOWVQGv/83iQamppZX1xBYXEFRcVHKNpbwdaDVTQ5gnuvuCiGZCayYV8l9U3N9E2J47z8vnxrTB/G56RpsTelwoDXyjCIyKnAHcaYbzp+/iWAMeZPLscsdhzzmYhEAfuBLONychERoAzoa4xpc6spDfrWa2v28pMX15ASH015zbEef1avWPL7JTOqXwr5/e33AWnxiAgVdQ28t6GEN9ftZ9nmg9Q3NdMnOY6Z+X341pi+TBiobwBKhSpvlmHoD+x2+XkPMLmtY4wxjSJyBMgAXLeTuhhY1V7AV8fMHtefCBHeWV/C8OwkRvVPYVS/ZHr3artQW3JcNBeNH8BF4wdQWdfAexsO8OZX+3j+i1089ekOspNjOS+/LzPyejO6fwqpCTE9eEVKqUDQI7V3RGQUcA9wbhuPzwfmAwwcqEXInC4Y248Lulibp1dcNBeO78+F4/tTWdfA+xsPsOirfbzgeAMAGJAWT77zE0P/FPL7pZDVK9aLV6CUCjSeBP29QI7LzwMc97k7Zo9jeCcFO5SDiAwAXgG+a4z52t0LGGMeAR4BO7zTmQtQHesVF83scf2ZPa4/VUcbWbOrnMLiIxTuPUJRcQVvF+1vOTY7OZb8fimM6p/CyL7JJMdHER0ZQXRkBFERYr9HCtERju+REURHCkmxUURFajKYUoHOk6C/AhgqIoOxwX0OcGWrY14H5gKfAZcA7xtjjIikAm8CtxljPvFes1VXJcVGMXVoJlOHZrbcV1nXcGyieO8RCouPsHTTATqTBDQgLZ7H5hYwok+yD1qtlPIWj+rpi8j5wH1AJPCEMeYPInInsNIY87qIxAHPAOOBQ8AcY8w2Efk18Etgi8vpzjXGHGjrtXQiNzDU1jex5UAl1UebaGhqprG5mYYmQ2OTcbndTEOz4WhDE49+tI3qo03848rxnDG8t7+br1TY0U1UVI/ad6SW655ayaaSSu6YNYprpgzyd5OUCiueBn0dhFVe0Tclnn/dcCpnDMviN68Wctcb61vWESilAofunKW8JjE2ike+W8Bdb6zn8Y+3s7Oshr/OGUdirOf/zIwxLNtSyqPLtlFcXsu0oZmcMaI3pw7JIC460oetDxxvrttHSUUd156Wi13eopT3aNBXXhUZIdwxaxSDMxP53X+KuOzhz3hi3kSyk9vfCL6xqZk3v9rHQx9uY8O+Cvokx5HXtxcLV+7h6c92EhcdwTdOyuTM4VmcOaI3A9ISeuiKetZjH23j929uAKC4vJb//laeBn7lVRr0lU/M/UYuA9MTuOn5Vcz+xyc8Pq+AUf1STjiupr6RhSt289jH29lzuJaTeyfxv5eMYfa4/sRERVDX0MTy7YdYuvEA7zu+eK2Iob2TOHNEb84c3puC3DSigzxd1BjD/72zmX8s3cq3RvclIymGxz7ejgF+rYFfeZFO5CqfWl9cwXVPr+BIbQP/uHI8Z43IBuBQdT1Pf7qDf362g8M1DRQMSuP6009ixojebZaKMMawrbSapRsPsHTTAb7YfoiGJkOv2CjOGZnNBWP7MXVoZtC9ATQ3G25/vYhnPt/JnIk5/OGi0UQI/O4/63nq0x1ce1ouv/32SA38ql2avaMCRklFHdc9vYL1xRX8/NzhHKio48WVu6lraObsvGxuOH0IBbnpnT5v1dFGPt5SynsbSlhctJ+KukZSE6I5L78PF4ztx+TBGUQGeK2hhqZmbv3XWl5bU8z104dw23kjWoK7MYY731jPk59o4Fcd06CvAkpNfSO3LFjDu+tLiI4ULhzXn/nThzA0u5dXzn+0sYmPNpfyn3XFvLu+hJr6JrJ6xfKt0X25YGw/ThmY6jZgNjcb9lXU8fWBKrYdrOLrg9VsK61i96Fa4qIjSI2PITUhmtSEaNISYkhxfE+NjyY1wT6Wk55AUicmq53qGpr44XOreH/jAf5r5nB+eMbJJxxjjOGuNzbwxCfbmfeNXG6/QAO/ck+Dvgo4Tc2GxUX7GT8wlb4p8T57ndr6Jt7feID/rC3m/U0HqG9spn9qPBeM7cfwPklsL61pCfDbS6uoa2hueW6v2CiG9E5iUHoC9Y3NlNfWU17TwOGaeg7XNFDf2HzC68VGRXDB2H5cNXkg43Lcv7m0VlHXwPefXsmKHYe4a3Y+V7ezrsEYwx/e3MBjH29n7qmDuGPWKA386gQa9JXClph4d30Jr68t5uMtpTQ2GyIEctITGJKZyJCsJE7KSmJIViJDshLJSoptN6DW1jdRXlvP4eqGljeET7aW8urqvVTXNzGybzJXTh7IheP7t9n7L6s6ytwnv2DjvkruvXwcszwoqmeM4Y+LNvDoR9v57qmD+J0GftWKBn2lWjlcXU9p1VEGZiQQG+XdnP+qo428tmYvz36+iw37KkiMiWT2+P5cNXngcVlLe8truebx5ew9XMtDV0/gzBGel6wwxvCntzbyyLJtXDNlEHfODt/A//7GEv66ZAuDMhKZPiyL6UMz6d1BWnCo06CvlB8YYxzbWO7iP2uLOdrYzLicVK6aPJCR/ZL5wdMrqaxr5PF5E5k0uPOT18YY7n5rIw8v28bVUwZy56x8v2yMU9fQxIZ9FRyoPEpzs6HZQLMxNBuDabmN42dDZEQE547KJjkuuluve7SxiXve2sQTn2xnUEYC1UebKK2yW3SM6NOL04dlMX1YFgW5aV5/Yw90GvSV8rMjNQ28vGoPzy3fydcHqwHISIzh6e9NIr//iWsWPGWM4Z63N/HQh19z5eSBXD99COmJMSTFRvmk51/hrMK694ijGusRvj5Y3ekyG5lJMfxi5gguPmVAl96otpdW8+MXVlG4t4K5pw7il+fnERMZwYb9FSzbXMqyzQdZudOm8cZHRzJlSLr9FDAsiyGZidQ1NFNT30hNfRPV9Y1UH22i1nG7xvFzszGcMzLbp3NOvqJBX6kAYYzhi+2HWFxUwtVTBjIkK8kr5/yfxZt48INjW1TEREaQlmizizKSYuz3xBjSEu33xNgoIkQQsSunI0SIEBA5djtChMZmw+aSSrsfc3EFO8tqWl4jOznWbtXZL5mRjq06neeKjDjxXM7XKi6v5Q9vbmDVrnLGD0zld7NGMWZAqsfX+8rqPfz6lUKiIiP430vGcO6oPm6Pqz7ayOfbyli2+SDLtpSyvdS+2YqAp6EuKkKYNbYfP5g+hLy+PVcqvPpoI/sr6jipi/8+NOgrFeKMMazYcZhdh2o4XF1PWXU9h6vrOVRTzyHH7bLqeo7UNnR8MjcGZSQwyrEfs/N7d3ZWa242vLJ6L396ayNl1UeZMzGHW88dTkZS2+esPtrIb14r5N+r9jIpN5375oyjX6rnvfDdh2r4cPNBSirqSIiJIjE2kvjoSBJjo0iIsd+dPyfGRFJd38Q/P9vBiyt2U1PfxPRhWcyfNoTTTs7w2fxJTX0j//xsJ48s20Z2chyLbp7apdfSoK+UAmxdo8M1DdTUNx43zt5sbBpt63F4AQZnJXZ7/L0tFXUN/G3JFp76dAcJMZH8/NzhXDV54Ak7rxXuPcKPX1jNzrJqbjprKDefdXKP7c52pKaBZ5fv5KlPd3Cw8iij+iUzf/oQzh/d12srvmvrm3j28508vOxrSqvqmT4si5+ePZTxA9O6dD4N+kqpgLalpJI7/lPEJ1vLGNGnF7+bNYrJQzIwxvDkJzu4+62NpCVGc9/l4zn1pAy/tPFoYxOvrS7mkY+2sfVAFf1T47n2tFzmTBrYpQV5YCfBn1u+iwc/+JrSqqNMPTmTn54zlAmDOj+x70qDvlIq4BljeLtwP79/cwN7y2uZNbYfNfWNLNlwgBkjevO/l44lPTHG382kudmwdNMBHl62jS+2HyI5LorzR/clr28yw/v0YkSfXqQmtN/OuoYmFnyxiwc++JoDlUf5xkkZ/PScYUzsQgkSdzToK6WCRm19Ew9+sJWHlm0DA788fwTzvhGY+wms2V3Oox9t4+MtpcfNl/TuFcvwPr0Ynt3L8UaQzMm9k4iIgIUr93D/+1vZX1HHpMHp/PTsYV7/9KJBXykVdIrLazna2MzgzER/N6VDxhhKKo6yqaSSTfsr2LS/ik0lFWwpqeKoo1yHCCTFRFF5tJGJuWktwd4Xb2aeBn2tp6+UChidyczxNxGhT0ocfVLiOH1YVsv9Tc2GnWXVbNpfyaaSSorLa23Z75MzA+KTiwZ9pZTyosgIYUhWEkOykjhvdF9/N+cEHuUeichMEdkkIltF5DY3j8eKyIuOx5eLSK7LY7903L9JRL7pvaYrpZTqrA6DvohEAvcD5wEjgStEZGSrw64DDhtjTgb+AtzjeO5IYA4wCpgJPOA4n1JKKT/wpKc/CdhqjNlmjKkHFgCzWx0zG3jacfslYIbYwavZwAJjzFFjzHZgq+N8Siml/MCToN8f2O3y8x7HfW6PMcY0AkeADA+fq5RSqocExA7SIjJfRFaKyMqDBw/6uzlKKRWyPAn6e4Ecl58HOO5ze4yIRAEpQJmHz8UY84gxpsAYU5CVldX6YaWUUl7iSdBfAQwVkcEiEoOdmH291TGvA3Mdty8B3jd21dfrwBxHds9gYCjwhXearpRSqrM6zNM3xjSKyE3AYiASeMIYUyQidwIrjTGvA48Dz4jIVuAQ9o0Bx3ELgfVAI/AjY0yTj65FKaVUBwKuDIOIHAR2duMUmUCpl5oTCELteiD0rinUrgdC75pC7XrgxGsaZIzpcHw84IJ+d4sl+L4AAAOKSURBVInISk/qTwSLULseCL1rCrXrgdC7plC7Huj6NQVE9o5SSqmeoUFfKaXCSCgG/Uf83QAvC7XrgdC7plC7Hgi9awq164EuXlPIjekrpZRqWyj29JVSSrUhZIJ+R+Wfg5GI7BCRr0RkjYgE3XZiIvKEiBwQkUKX+9JF5F0R2eL4nubPNnZWG9d0h4jsdfyd1ojI+f5sY2eISI6ILBWR9SJSJCK3OO4Pyr9TO9cTzH+jOBH5QkTWOq7pd477BztK2W91lLb3aDPhkBjecZRr3gycgy3qtgK4whiz3q8N6yYR2QEUGGOCMr9YRKYDVcA/jTH5jvv+BzhkjLnb8eacZoz5hT/b2RltXNMdQJUx5s/+bFtXiEhfoK8xZpWI9AK+BC4E5hGEf6d2rucygvdvJECiMaZKRKKBj4FbgJ8B/zbGLBCRh4C1xpgHOzpfqPT0PSn/rHqYMWYZdoW2K9cy3E9j/0MGjTauKWgZY/YZY1Y5blcCG7CVcIPy79TO9QQtY1U5fox2fBngLGwpe+jE3yhUgn6olnA2wDsi8qWIzPd3Y7wk2xizz3F7P5Dtz8Z40U0iss4x/BMUQyGtOXa8Gw8sJwT+Tq2uB4L4byQikSKyBjgAvAt8DZQ7StlDJ2JeqAT9UDXVGHMKdteyHzmGFkKGoyhf8I8vwoPAScA4YB/wf/5tTueJSBLwMvATY0yF62PB+Hdycz1B/TcyxjQZY8ZhKxVPAkZ09VyhEvQ9KuEcbIwxex3fDwCvEBq7jpU4xl2d468H/NyebjPGlDj+UzYDjxJkfyfHOPHLwHPGmH877g7av5O76wn2v5GTMaYcWAqcCqQ6StlDJ2JeqAR9T8o/BxURSXRMRCEiicC5QGH7zwoKrmW45wKv+bEtXuEMjg4XEUR/J8ck4ePABmPMvS4PBeXfqa3rCfK/UZaIpDpux2MTVjZgg/8ljsM8/huFRPYOgCMF6z6OlX/+g5+b1C0iMgTbuwdbAvv5YLsmEXkBOANbDbAEuB14FVgIDMRWU73MGBM0E6NtXNMZ2GEDA+wArncZDw9oIjIV+Aj4Cmh23P0r7Dh40P2d2rmeKwjev9EY7ERtJLajvtAYc6cjRiwA0oHVwNXGmKMdni9Ugr5SSqmOhcrwjlJKKQ9o0FdKqTCiQV8ppcKIBn2llAojGvSVUiqMaNBXSqkwokFfKaXCiAZ9pZQKI/8fVIM7HoEQU/wAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(tr_loss_hist, label = 'train')\n",
    "plt.plot(val_loss_hist, label = 'validation')\n",
    "plt.legend()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "test acc: 99.15%\n"
     ]
    }
   ],
   "source": [
    "yhat = cnn.predict(sess = sess, x_data = x_tst)\n",
    "print('test acc: {:.2%}'.format(np.mean(yhat == y_tst)))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}