{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# CS 20 : TensorFlow for Deep Learning Research\n",
    "## Lecture 07 : ConvNet in TensorFlow\n",
    "Specification of SimpleCNN is same that of [Lec7_ConvNet mnist by high-level.ipynb](https://nbviewer.jupyter.org/github/aisolab/CS20/blob/master/Lec07_ConvNet%20in%20Tensorflow/Lec07_ConvNet%20mnist%20by%20high-level.ipynb)  \n",
    "But only different thing is to initialize weights of model by He initialization\n",
    "### ConvNet mnist with Weight initialization and Drop out\n",
    "- Creating the **data pipeline** with `tf.data`\n",
    "- Using `tf.contrib.slim`, alias `slim`\n",
    "- Creating the model as **Class** with `slim`\n",
    "- Initializaing weights of model with **He initialization** by `slim.variance_scaling_initializer`\n",
    "- Training the model with **Drop out** technique by `slim.dropout`\n",
    "- Using tensorboard"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Setup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1.12.0\n"
     ]
    }
   ],
   "source": [
    "from __future__ import absolute_import, division, print_function\n",
    "import os, sys\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import tensorflow as tf\n",
    "%matplotlib inline\n",
    "\n",
    "slim = tf.contrib.slim\n",
    "print(tf.__version__)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load and Pre-process data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "(x_train, y_train), (x_tst, y_tst) = tf.keras.datasets.mnist.load_data()\n",
    "x_train = x_train  / 255\n",
    "x_train = x_train.reshape(-1, 28, 28, 1).astype(np.float32)\n",
    "x_tst = x_tst / 255\n",
    "x_tst = x_tst.reshape(-1, 28, 28, 1).astype(np.float32)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(55000, 28, 28, 1) (55000,)\n",
      "(5000, 28, 28, 1) (5000,)\n"
     ]
    }
   ],
   "source": [
    "tr_indices = np.random.choice(range(x_train.shape[0]), size = 55000, replace = False)\n",
    "\n",
    "x_tr = x_train[tr_indices]\n",
    "y_tr = y_train[tr_indices].astype(np.int32)\n",
    "\n",
    "x_val = np.delete(arr = x_train, obj = tr_indices, axis = 0)\n",
    "y_val = np.delete(arr = y_train, obj = tr_indices, axis = 0).astype(np.int32)\n",
    "\n",
    "print(x_tr.shape, y_tr.shape)\n",
    "print(x_val.shape, y_val.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Define SimpleCNN class by high-level api (slim)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "class SimpleCNN:\n",
    "    def __init__(self, X, y, n_of_classes):\n",
    "        \n",
    "        self._X = X\n",
    "        self._y = y\n",
    "        self._is_training = tf.placeholder(dtype = tf.bool)\n",
    "        \n",
    "        with slim.arg_scope([slim.conv2d, slim.fully_connected], activation_fn = tf.nn.relu,\n",
    "                            weights_initializer = slim.variance_scaling_initializer(), # He initialization\n",
    "                            biases_initializer = tf.truncated_normal_initializer()):\n",
    "            with slim.arg_scope([slim.conv2d], kernel_size = [5, 5], stride = 1, padding = 'SAME'):\n",
    "                with slim.arg_scope([slim.max_pool2d], kernel_size = [2, 2], stride = 2, padding = 'SAME'):\n",
    "                    \n",
    "                    conv1 = slim.conv2d(inputs = self._X, num_outputs = 32, scope = 'conv1')\n",
    "                    pool1 = slim.max_pool2d(inputs = conv1, scope = 'pool1')\n",
    "                    conv2 = slim.conv2d(inputs = pool1, num_outputs = 64, scope = 'conv2')\n",
    "                    pool2 = slim.max_pool2d(inputs = conv2, scope = 'pool2')\n",
    "                    flattened = slim.flatten(inputs = pool2)\n",
    "                    fc = slim.fully_connected(inputs = flattened, num_outputs = 1024, scope = 'fc1')\n",
    "                    dropped = slim.dropout(inputs = fc, keep_prob = .5, is_training = self._is_training)\n",
    "                    self._score = slim.fully_connected(inputs = dropped, num_outputs = n_of_classes,\n",
    "                                                       activation_fn = None, scope = 'score')\n",
    "                    self.ce_loss = self._loss(labels = self._y, logits = self._score, scope = 'ce_loss')\n",
    "        \n",
    "        with tf.variable_scope('prediction'):\n",
    "            self._prediction = tf.argmax(input = self._score, axis = -1)\n",
    "        \n",
    "    def _loss(self, labels, logits, scope):\n",
    "        with tf.variable_scope(scope):\n",
    "            ce_loss = tf.reduce_mean(tf.losses.sparse_softmax_cross_entropy(labels = labels, logits = logits))\n",
    "            return ce_loss\n",
    "        \n",
    "    def predict(self, sess, x_data, is_training = True):\n",
    "        feed_prediction = {self._X : x_data, self._is_training : is_training}\n",
    "        return sess.run(self._prediction, feed_dict = feed_prediction)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Create a model of SimpleCNN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "550\n"
     ]
    }
   ],
   "source": [
    "# hyper-parameter\n",
    "lr = .001\n",
    "epochs = 30\n",
    "batch_size = 100\n",
    "total_step = int(x_tr.shape[0] / batch_size)\n",
    "print(total_step)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<BatchDataset shapes: ((?, 28, 28, 1), (?,)), types: (tf.float32, tf.int32)>\n",
      "<BatchDataset shapes: ((?, 28, 28, 1), (?,)), types: (tf.float32, tf.int32)>\n"
     ]
    }
   ],
   "source": [
    "## create input pipeline with tf.data\n",
    "# for train\n",
    "tr_dataset = tf.data.Dataset.from_tensor_slices((x_tr, y_tr))\n",
    "tr_dataset = tr_dataset.shuffle(buffer_size = 10000)\n",
    "tr_dataset = tr_dataset.batch(batch_size = batch_size)\n",
    "tr_iterator = tr_dataset.make_initializable_iterator()\n",
    "print(tr_dataset)\n",
    "\n",
    "# for validation\n",
    "val_dataset = tf.data.Dataset.from_tensor_slices((x_val,y_val))\n",
    "val_dataset = val_dataset.batch(batch_size = batch_size)\n",
    "val_iterator = val_dataset.make_initializable_iterator()\n",
    "print(val_dataset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "## define Iterator\n",
    "# tf.data.Iterator.from_string_handle의 output_shapes는 default = None이지만 꼭 값을 넣는 게 좋음\n",
    "handle = tf.placeholder(dtype = tf.string)\n",
    "iterator = tf.data.Iterator.from_string_handle(string_handle = handle,\n",
    "                                               output_types = tr_iterator.output_types,\n",
    "                                               output_shapes = tr_iterator.output_shapes)\n",
    "\n",
    "x_data, y_data = iterator.get_next()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "## connecting data pipeline with model\n",
    "cnn = SimpleCNN(X = x_data, y = y_data, n_of_classes = 10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Create training op and train model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "## create training op\n",
    "opt = tf.train.AdamOptimizer(learning_rate = lr)\n",
    "\n",
    "# equal to 'var_list = None'\n",
    "training_op = opt.minimize(loss = cnn.ce_loss)\n",
    "\n",
    "#for tensorboard\n",
    "loss_summ = tf.summary.scalar(name = 'loss', tensor = cnn.ce_loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "## for tensorboard\n",
    "tr_writer = tf.summary.FileWriter('../graphs/lecture07/convnet_mnist_drop_out/train/', graph = tf.get_default_graph())\n",
    "val_writer = tf.summary.FileWriter('../graphs/lecture07/convnet_mnist_drop_out/val/', graph = tf.get_default_graph())\n",
    "saver = tf.train.Saver()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch :   5, tr_loss : 0.039, val_loss : 0.037\n",
      "epoch :  10, tr_loss : 0.021, val_loss : 0.035\n",
      "epoch :  15, tr_loss : 0.015, val_loss : 0.037\n",
      "epoch :  20, tr_loss : 0.013, val_loss : 0.053\n",
      "epoch :  25, tr_loss : 0.011, val_loss : 0.044\n",
      "epoch :  30, tr_loss : 0.007, val_loss : 0.050\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'../graphs/lecture07/convnet_mnist_drop_out/cnn/'"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sess_config = tf.ConfigProto(gpu_options=tf.GPUOptions(allow_growth=True))\n",
    "sess = tf.Session(config = sess_config)\n",
    "sess.run(tf.global_variables_initializer())\n",
    "tr_handle, val_handle = sess.run(fetches = [tr_iterator.string_handle(), val_iterator.string_handle()])\n",
    "\n",
    "tr_loss_hist = []\n",
    "val_loss_hist = []\n",
    "\n",
    "for epoch in range(epochs):\n",
    "\n",
    "    avg_tr_loss = 0\n",
    "    avg_val_loss = 0\n",
    "    tr_step = 0\n",
    "    val_step = 0\n",
    "\n",
    "    # for mini-batch training\n",
    "    sess.run(tr_iterator.initializer)    \n",
    "    try:\n",
    "        while True:\n",
    "            _, tr_loss, tr_loss_summ = sess.run(fetches = [training_op, cnn.ce_loss, loss_summ],\n",
    "                                               feed_dict = {handle : tr_handle, cnn._is_training : True})\n",
    "            avg_tr_loss += tr_loss\n",
    "            tr_step += 1\n",
    "            \n",
    "    except tf.errors.OutOfRangeError:\n",
    "        pass\n",
    "\n",
    "    # for validation\n",
    "    sess.run(val_iterator.initializer)\n",
    "    try:\n",
    "        while True:\n",
    "            val_loss, val_loss_summ = sess.run(fetches = [cnn.ce_loss, loss_summ],\n",
    "                                               feed_dict = {handle : val_handle, cnn._is_training : False})\n",
    "            avg_val_loss += val_loss\n",
    "            val_step += 1\n",
    "    \n",
    "    except tf.errors.OutOfRangeError:\n",
    "        pass\n",
    "\n",
    "    avg_tr_loss /= tr_step\n",
    "    avg_val_loss /= val_step\n",
    "    tr_writer.add_summary(summary = tr_loss_summ, global_step = epoch + 1)\n",
    "    val_writer.add_summary(summary = val_loss_summ, global_step = epoch + 1)\n",
    "    tr_loss_hist.append(avg_tr_loss)\n",
    "    val_loss_hist.append(avg_val_loss)\n",
    "    \n",
    "    if (epoch + 1) % 5 == 0:\n",
    "        print('epoch : {:3}, tr_loss : {:.3f}, val_loss : {:.3f}'.format(epoch + 1, avg_tr_loss, avg_val_loss))\n",
    "\n",
    "tr_writer.close()\n",
    "val_writer.close()\n",
    "saver.save(sess = sess, save_path = '../graphs/lecture07/convnet_mnist_drop_out/cnn/')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.legend.Legend at 0x7f23d80dba20>"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAD8CAYAAACb4nSYAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzt3Xl8XNV99/HPb0Yz0mgdSZZ3yzZg4x3bCENi1rAZKAbCnqWQzU0KJXmlaUvSJqSkeUralJI8D2FJ4jRJSRyHFOw0JgYSE7YAlsH7jhckr7Jkydqlmfk9f5yRPJZlaSSNtpnf+/Wa19y5c+/VuRrpe++cc+65oqoYY4xJDZ7BLoAxxpiBY6FvjDEpxELfGGNSiIW+McakEAt9Y4xJIRb6xhiTQiz0jTEmhVjoG2NMCrHQN8aYFJI22AXoaMSIETpp0qTBLoYxxgwr69atO6aqRd0tN+RCf9KkSZSWlg52MYwxZlgRkf3xLGfVO8YYk0Is9I0xJoVY6BtjTAoZcnX6xpjk0traSnl5OU1NTYNdlKSQkZHB+PHj8fl8vVrfQt8Y06/Ky8vJyclh0qRJiMhgF2dYU1UqKyspLy9n8uTJvdqGVe8YY/pVU1MThYWFFvgJICIUFhb26VuThb4xpt9Z4CdOX3+XSRP6J5pa+c+XdrK+rHqwi2KMMUNW0oR+JKJ87w+7WLf/+GAXxRgzhFRXV/ODH/ygx+tdf/31VFcn30lk0oR+ToZrya5paBnkkhhjhpIzhX4oFOpyvVWrVhEMBvurWIMmaXrveD1CbkYaNY2tg10UY8wQ8uCDD/L+++8zd+5cfD4fGRkZ5Ofns337dnbu3MnNN99MWVkZTU1NfPGLX2TJkiXAySFh6urquO6667j44ot58803GTduHCtWrCAQCAzynvVO0oQ+QF6mz0LfmCHsn3+7ha0HTyR0mzPG5vLQjTPP+P4jjzzC5s2bWb9+Pa+88go33HADmzdvbu/yuHTpUgoKCmhsbOSCCy7g1ltvpbCw8JRt7Nq1i1/+8pf88Ic/5I477uA3v/kNn/jEJxK6HwMluUI/4KPaQt8Y04UFCxac0sf9+9//Ps899xwAZWVl7Nq167TQnzx5MnPnzgXg/PPPZ9++fQNW3kRLqtAPBvx2pm/MENbVGflAycrKap9+5ZVXePnll/nzn/9MZmYml19+ead94NPT09unvV4vjY2NA1LW/hBXQ66ILBKRHSKyW0Qe7OT9z4vIJhFZLyKvi8iMmPe+Gl1vh4hcm8jCd5QXsOodY8ypcnJyqK2t7fS9mpoa8vPzyczMZPv27bz11lsDXLqB1+2Zvoh4gceBq4FyYK2IrFTVrTGL/UJVn4wuvxh4FFgUDf+7gJnAWOBlEZmqquEE7wcAuQEfNQ0W+saYkwoLC1m4cCGzZs0iEAgwatSo9vcWLVrEk08+yfTp0zn33HO56KKLBrGkAyOe6p0FwG5V3QMgIsuAm4D20FfV2JaZLECj0zcBy1S1GdgrIruj2/tzAsp+mmC0IVdV7QpAY0y7X/ziF53OT09P54UXXuj0vbZ6+xEjRrB58+b2+V/5ylcSXr6BFE/ojwPKYl6XAxd2XEhE7gO+DPiBj8SsG/t9qTw6r1/kBXyEIkp9S5js9KRqrjDGmIRI2MVZqvq4qp4N/APwTz1ZV0SWiEipiJRWVFT0ugzBQPQCLavXN8aYTsUT+geACTGvx0fnncky4OaerKuqT6tqiaqWFBV1e1/fM8prC32r1zfGmE7FE/prgSkiMllE/LiG2ZWxC4jIlJiXNwC7otMrgbtEJF1EJgNTgHf6XuzOtYV+daMNxWCMMZ3ptuJbVUMicj+wGvACS1V1i4g8DJSq6krgfhG5CmgFjgP3RNfdIiLLcY2+IeC+/uq5A+6KXIATVr1jjDGdiqu1U1VXAas6zPtGzPQXu1j328C3e1vAnmg/07fqHWOM6VTSjLIJEMz0A9aQa4zpvezsbAAOHjzIbbfd1ukyl19+OaWlpV1u57HHHqOhoaH99VAZqjmpQj/L78XrEQt9Y0yfjR07lmeffbbX63cM/aEyVHNShb6I2KBrxphTPPjggzz++OPtr7/5zW/yL//yL1x55ZXMnz+f2bNns2LFitPW27dvH7NmzQKgsbGRu+66i+nTp3PLLbecMvbOF77wBUpKSpg5cyYPPfQQ4AZxO3jwIFdccQVXXHEF4IZqPnbsGACPPvoos2bNYtasWTz22GPtP2/69Ol87nOfY+bMmVxzzTX9MsZP0l3BFLTxd4wZul54EA5vSuw2R8+G6x4549t33nknX/rSl7jvvvsAWL58OatXr+aBBx4gNzeXY8eOcdFFF7F48eIzXsn/xBNPkJmZybZt29i4cSPz589vf+/b3/42BQUFhMNhrrzySjZu3MgDDzzAo48+ypo1axgxYsQp21q3bh0/+clPePvtt1FVLrzwQi677DLy8/MHZAjnpDrTBzf+jvXeMca0mTdvHkePHuXgwYNs2LCB/Px8Ro8ezde+9jXmzJnDVVddxYEDBzhy5MgZt/Hqq6+2h++cOXOYM2dO+3vLly9n/vz5zJs3jy1btrB169YzbQaA119/nVtuuYWsrCyys7P56Ec/ymuvvQYMzBDOyXemn+mjss766RszJHVxRt6fbr/9dp599lkOHz7MnXfeyTPPPENFRQXr1q3D5/MxadKkTodU7s7evXv57ne/y9q1a8nPz+fee+/t1XbaDMQQzkl3pm/DKxtjOrrzzjtZtmwZzz77LLfffjs1NTWMHDkSn8/HmjVr2L9/f5frX3rppe2Dtm3evJmNGzcCcOLECbKyssjLy+PIkSOnDN52piGdL7nkEp5//nkaGhqor6/nueee45JLLkng3nYt6c708wI+qu3m6MaYGDNnzqS2tpZx48YxZswYPv7xj3PjjTcye/ZsSkpKmDZtWpfrf+ELX+BTn/oU06dPZ/r06Zx//vkAnHfeecybN49p06YxYcIEFi5c2L7OkiVLWLRoEWPHjmXNmjXt8+fPn8+9997LggULAPjsZz/LvHnzBuxuXKKq3S81gEpKSrS7/q9defTFHfzfNbt5/9vX4/HY8MrGDLZt27Yxffr0wS5GUunsdyoi61S1pLt1k656JzfgQxVqm0KDXRRjjBlyki70bdA1Y4w5s6QLfRuKwZihZ6hVIw9nff1dJl3o26BrxgwtGRkZVFZWWvAngKpSWVlJRkZGr7eRdL13gpl29yxjhpLx48dTXl5OX+6KZ07KyMhg/PjxvV4/6UI/z26ZaMyQ4vP5mDx58mAXw0QlbfWOhb4xxpwu6UI/w+clPc1joW+MMZ1IutCH6FAM1pBrjDGnScrQD2b6rJ++McZ0IilD3wZdM8aYziVt6Fs/fWOMOV2Shr7fbqRijDGdSNLQt+odY4zpTFyhLyKLRGSHiOwWkQc7ef/LIrJVRDaKyB9EZGLMe2ERWR99rExk4c8kmOmjviVMazgyED/OGGOGjW5DX0S8wOPAdcAM4G4RmdFhsfeAElWdAzwL/FvMe42qOjf6WJygcnfJLtAyxpjOxXOmvwDYrap7VLUFWAbcFLuAqq5R1Yboy7eA3g8MkQA26JoxxnQuntAfB5TFvC6PzjuTzwAvxLzOEJFSEXlLRG7uRRl7LM8GXTPGmE4ldMA1EfkEUAJcFjN7oqoeEJGzgD+KyCZVfb/DekuAJQDFxcV9Lkfbmb714DHGmFPFc6Z/AJgQ83p8dN4pROQq4B+Bxara3DZfVQ9En/cArwDzOq6rqk+raomqlhQVFfVoBzoTtLtnGWNMp+IJ/bXAFBGZLCJ+4C7glF44IjIPeAoX+Edj5ueLSHp0egSwENiaqMKfSXtDrtXpG2PMKbqt3lHVkIjcD6wGvMBSVd0iIg8Dpaq6Evh3IBv4tYgAfBDtqTMdeEpEIrgDzCOq2u+hn9vee8dujm6MMbHiqtNX1VXAqg7zvhEzfdUZ1nsTmN2XAvaGz+shOz3NqneMMaaDpLwiF+yqXGOM6UzShn6ujalvjDGnSdrQD9qZvjHGnCZpQ9+qd4wx5nRJG/ru7lkW+sYYEytpQ7/tTF9VB7soxhgzZCRt6OcGfLSEIjS12vDKxhjTJmlDP2iDrhljzGmSNvRtTH1jjDld0oZ+MOAHoLrBrso1xpg2SRv6dqZvjDGns9A3xpgUkryhbw25xhhzmqQN/Zz0NEQs9I0xJlbShr7HI+QFfHZzdGOMiZG0oQ82/o4xxnRkoW+MMSkk6UPfBl0zxpiTkj70T1joG2NMu6QPfbsi1xhjTkrq0A9m+jjRFLLhlY0xJiqpQz8v4CMcUeqaQ4NdFGOMGRKSOvRPDrpm9frGGANxhr6ILBKRHSKyW0Qe7OT9L4vIVhHZKCJ/EJGJMe/dIyK7oo97Eln47uTa+DvGGHOKbkNfRLzA48B1wAzgbhGZ0WGx94ASVZ0DPAv8W3TdAuAh4EJgAfCQiOQnrvhdaxt0zXrwGGOME8+Z/gJgt6ruUdUWYBlwU+wCqrpGVRuiL98CxkenrwVeUtUqVT0OvAQsSkzRu9d29yzrq2+MMU48oT8OKIt5XR6ddyafAV7o5boJZcMrG2PMqdISuTER+QRQAlzWw/WWAEsAiouLE1ae9jN9a8g1xhggvjP9A8CEmNfjo/NOISJXAf8ILFbV5p6sq6pPq2qJqpYUFRXFW/ZuBXxefF6xM31jjImKJ/TXAlNEZLKI+IG7gJWxC4jIPOApXOAfjXlrNXCNiORHG3Cvic4bECJig64ZY0yMbqt3VDUkIvfjwtoLLFXVLSLyMFCqqiuBfweygV+LCMAHqrpYVatE5Fu4AwfAw6pa1S97cgYu9G0oBmOMgTjr9FV1FbCqw7xvxExf1cW6S4GlvS1gX9mZvjHGnJTUV+QCBDP91pBrjDFRSR/6dqZvjDEnWegbY0wKSYnQr20KEY7Y8MrGGJMSoQ82/o4xxkAKhb5V8RhjTAqEvg26ZowxJyV96NuZvjHGnJT0oX9y0DW7KtcYY5I+9HOtIdcYY9olfehb9Y4xxpyU9KGfnuYl4PPaUAzGGEMKhD7YVbnGGNMmJUI/mOmzLpvGGEOKhH6unekbYwyQIqGfF/BZ7x1jjCFFQj8Y8FlDrjHGkCKhbw25xhjjpEToBzN9NLaGaQ6FB7soxhgzqFIi9O0CLWOMcVIi9G0oBmOMcVIi9IOZfgBrzDXGpLyUCH2r3jHGGCeu0BeRRSKyQ0R2i8iDnbx/qYi8KyIhEbmtw3thEVkffaxMVMF7wkLfGGOctO4WEBEv8DhwNVAOrBWRlaq6NWaxD4B7ga90solGVZ2bgLL2WjDQNqa+hb4xJrV1G/rAAmC3qu4BEJFlwE1Ae+ir6r7oe5F+KGOf5dqZvjHGAPFV74wDymJel0fnxStDREpF5C0RublHpUsQr0fIyUiz0DfGpLx4zvT7aqKqHhCRs4A/isgmVX0/dgERWQIsASguLu6XQthVucYYE9+Z/gFgQszr8dF5cVHVA9HnPcArwLxOlnlaVUtUtaSoqCjeTfeIhb4xxsQX+muBKSIyWUT8wF1AXL1wRCRfRNKj0yOAhcS0BQykYKbPbo5ujEl53Ya+qoaA+4HVwDZguapuEZGHRWQxgIhcICLlwO3AUyKyJbr6dKBURDYAa4BHOvT6GTB2pm+MMXHW6avqKmBVh3nfiJlei6v26bjem8DsPpYxIfICfmoaQ4NdDGOMGVQpcUUutJ3pt6Cqg10UY4wZNCkV+q1hpbHVhlc2xqSulAn9YKZdlWuMMSkT+jb+jjHGpFDoBy30jTEmdUI/1wZdM8aY1An9PLt7ljHGpE7otzfkNtpVucaY1JUyoZ+dnobXI1anb4xJaSkT+iJCrg2vbIxJcSkT+uBukG4NucaYVJZSoZ9rg64ZY1JcSoV+MOCz3jvGmJSWUqGfF/BRbaFvjElhKRf6Vr1jjEllKRX6wUwX+pGIDa9sjElNKRX6eQEfqlDbbDdTMcakppQLfbChGIwxqSslQ9/66htjUlVKhr415hpjUlVKhX4w0w/YoGvGmNSVUqFvZ/rGmFSXUqHfNryyhb4xJlXFFfoiskhEdojIbhF5sJP3LxWRd0UkJCK3dXjvHhHZFX3ck6iC90aGz4s/zUONNeQaY1JUt6EvIl7gceA6YAZwt4jM6LDYB8C9wC86rFsAPARcCCwAHhKR/L4Xu/fsqlxjTCqL50x/AbBbVfeoaguwDLgpdgFV3aeqG4FIh3WvBV5S1SpVPQ68BCxKQLl7LWihb4xJYfGE/jigLOZ1eXRePPqybr/IC/isn74xJmUNiYZcEVkiIqUiUlpRUdGvP8uqd4wxqSye0D8ATIh5PT46Lx5xrauqT6tqiaqWFBUVxbnp3snLtNA3xqSueEJ/LTBFRCaLiB+4C1gZ5/ZXA9eISH60Afea6LxBY2f6xphU1m3oq2oIuB8X1tuA5aq6RUQeFpHFACJygYiUA7cDT4nIlui6VcC3cAeOtcDD0XmDJhjwU9ccojXcsc3ZGGOSX1o8C6nqKmBVh3nfiJlei6u66WzdpcDSPpQxofICbpdPNLZSmJ0+yKUxxpiBNSQacgdSnl2Va4xJYSkX+sFA26BrFvrGmNSTcqGfa4OuGWNSWMqFftuga3b3LGNMKkq50Le7ZxljUlnKhr5V7xhjUlHKhb7P6yHL77XQN8akpJQLfbBB14wxqSs1Qz/Tb2f6xpiUlJqhH0ijxm6OboxJQSka+jbomjEmNaVk6Odn+jlU00RVvZ3tG2NSS0qG/l0LimkORVjys1KaWsODXRxjjBkwyRX6qnEtNndCkP+8Yy6l+4/zt8s3EInEt54xxgx3yRP6Jw7C0kWw/824Fr9hzhi+dv00frfpEN/5/fZ+LpwxxgwNyRP66TlQfxR+81loiO8+LZ+75Cw+edFEnnp1Dz9/a38/F9AYYwZfcoX+bUuh7iisuD+uqh4R4aEbZ3DV9JE8tGIzf9h2ZAAKaowxgyd5Qh9g7Dy4+p9hx+9g7Y/iWiXN6+H7d89j1rg87v/Fe2wqr+nnQhpjzOBJrtAHuOivYco1sPof4fCmuFbJ9Kfxo3tKKMjy8+mfrqWsqqGfC2mMMYMj+UJfBG5+AgL58OtPQUt9XKuNzMngp5++gObWMJ/6r7XU2Ng8xpgklHyhD5A1Am79IVTuhhf+Pu7VzhmZw9N/WcL+ynr+6r9LaQ5ZH35jTCf+8DD89EZ44/tQsTPu7uJDgegQK2xJSYmWlpYmZmN/+Ba89l249ccw+7a4V3v+vQN86VfruWXeOB694zxEJDHlMSYVNR53IVl/DIjmTWzutE9Hn8fNh4u/DB7vQJYyflueg1/fCzljoPaQm5c/GaYugqnXwsSFkOYf8GKJyDpVLeluubSBKMygufyrsO91+O2X3B9SwVlxrXbzvHGUH2/guy/uZHx+gL+95tx+LqgxSUoVVv4N7HgBCqe4ee0nUXLqNECkFXasggPvwq0/An/WQJe4a8f3w8ovwrgS+PTvofYw7FoNO1dD6VJ4+wnw58DZV7iDwJRrILtosEt9irjO9EVkEfA9wAv8SFUf6fB+OvAz4HygErhTVfeJyCRgG7Ajuuhbqvr5rn5WQs/0Aao/gCcvdoH/6RfjPgKrKl/9n00sW1vGvR+exJevmUpuhi9x5TImFaz9Efzub+Hqb8HCB+Jb5+2n4ff/AKNnw92/gtwx/VvGeIVb4SfXQcUO+PxrkD/p1Pdb6mHvq7Dz9+4gUHsIEBh3Ppx7HZx7PYycHnOgS6x4z/S7DX0R8QI7gauBcmAtcLeqbo1Z5q+BOar6eRG5C7hFVe+Mhv7/quqseAue8NAH2LoSln8SPvw3cM2/xL1aazjCt/53Kz9/az+FWen84w3TuHnuOKvuMcPLsV2w6VlobYBZt8KY8/oteE5xeBP88EqYfCl8bDl4etCEuHM1PPtpyMiDj/3KHQAG28vfhNf/E277Ccz6aNfLqrr937nafXM5+K6bnz/Jhf+510Pxh8CbuMqWRIb+h4Bvquq10ddfBVDVf41ZZnV0mT+LSBpwGCgCJjIUQh/gf78MpT+Gj/8GplzVo1U3ldfw9RWbWV9WzYJJBTx880ymjc5NfBmNSZS6o7D5N7DxV3DwPUDAk+aqT4qmwZw7Yc4dkDe+f35+Sz08dRk018LnX+9dFcfhTfDMHdB8wgXt1GsSX854vf9H+PktMP8eWPz9nq9/4pD7BrBjFez5E4SbISPo2gDOvQ7OucpdYNoHiQz924BFqvrZ6OtPAheq6v0xy2yOLlMeff0+cCGQDWzBfVM4AfyTqr7W1c/rt9BvbXRnHXVH4AtvQM7oMy/bUu/+UcrXuj+8nDFERp/H76tG8vVXG6luVu750CS+dPUUq/IxQ0dzHWz/nQv6Pa+AhmH0HBfws24FX4ZrhNzwKyh7CxCYdDGcdzfMWNzn0DnF8/fB+mfgL1fAWZf1fjsnDsEv73T/h4u+AxcuSVwZ41V3FJ5YCJkF8Lk14M/s2/aa69xBZMcqdyBoPA5eP0y6BGbcBOff06vNDpXQrwWyVbVSRM4HngdmquqJDj9jCbAEoLi4+Pz9+/tpHJyj2+Hpy2HCAvjk8+7rZiQCVe+7gG97HNnq/mEAcsdDfYU7MgPqy2S//xzW1Ixhf/oULrn0aj5y8ULEa+FvBkE4BHvWuKDf/jtXhZNXDHNuh9l3wMhpna9XtRc2LocNv4TjeyEtANNugPPugrOu6Fu1w8bl8D+fg0v/Dj7yT73fTpuWejem1o5VcOHn4dr/M3A9eyIReOZWN5Dj5/4Io2YmdvvhEJS97fZt++8gWAz3rOzVpoZE9Y522LiIvAJ8RVXPeCrfb2f6bdb9FH77AExf7P5Bykuhqdq958+B8efD+AvcY1wJZBW6BpyKHXBoQ/SxnsihjXhCjQC04Cc8ciaBc690jVUZef1XfpOaGquhak/0sffkdMV29/ebEXT1zLPvgAkXxl9/rupOdDYsc9VBTdWQPco1vM65o+d1/5Xvw1OXujr4e/43cXXWkTC8+HV463HXK+bWH0N6dmK23ZU3vgcvfQNueBQu+Ez//ixVVx2W0buq40SGfhqueuZK4ACuIfdjqrolZpn7gNkxDbkfVdU7RKQIqFLVsIicBbwWXe6Mw2D2e+irurOQTc+6lvTxJSdDfsTU+M8gImEiFbt4+80/8P6GN5gaeZ8Szw4iGQWkXf0QzPvE0O1nbIa28lLY/fLJYK98Hxo7/MvkjnM90grOcvXC51zd977hoWbY9aK74Kj8HZh1G9zwHxAIxr/+j6923Rq/8Eb/tBes/RGs+jt3xn33ryBvXOJ/RpvyUlh6rWt0veNnA9P43QcJC/3oxq4HHsN12Vyqqt8WkYeBUlVdKSIZwM+BeUAVcJeq7hGRW4GHgVYgAjykqr/t6mf1e+iD+8oWaup73VzU8foWvvviDjaXvso/ef+LCzw7qcydTsbi75J1zsUJ+RkmBRzZ4i4o3PkCIJA3AQomnwz39sdk8AX6rxzhkOul8sq/Qu5YuOUpmLSw+/VeeND1U7/rlzDt+v4r366X3MVR6TmwYIk78090V8imGnjyEtCI654ZyE/ctvtJQkN/IA1I6PeTqvoWnnu3nKNvPsO9DUsZI1W8m3slXP0w82bNtK6epnNVe13AblwO6bmuivDCv0psw2pvlJe6uvTj++CSL7uLHc/UdrV9FSy729W5X/ed/i/b4c3uoq+2rpB5xa53z9RFrkHUl9H7bau67qJbV8CnXoDiCxNT5n5moT+IVJXNew9Rsfo7LDz8DGE8/NJ/K60X3sfNF5zD6Lwu/iBrD0PZO65xp3wtREKunnbOHa73wMDthLsbWfaohPYlNjFqD8Or/+7amTxeF/QLvzSwn3N3muvg9w/Cez93Q5d/9Ecw4pxTl6k5AE8udN9MPvsypKUPXPlOHHRVUjtXux5LrQ3gy4SzLndXw0691n1b6Yl3f+YOKB/5Olz6lX4odP+w0B8imo7upfK5v2PcoZcoixTxr+GP03j29XzonBHMGBVgprec/Mr10ZB/x11BDOBNd/9koUbXeOz1w7S/gPmfhMmX9+xCl544us1169vyPBzb4Rqlz7nKnUGdc9XQCqSBFIm4KyzbG1P3uF5fVXtdI+Po2TBmjusiOXp217+nxmrXQPj2kxBugfl/CZf+/dC58rQzW1fAygdceRf9q+uvLuKqgn56o/sb/atXTz8gDKTWJjfsyq7Vritk2//S6DlQfJEL/5wxJx+5Y07/NnVKD7/nhlW7nIX+ULP3VVp++3f4q7azQabTGIY5sodMcV1BqzyFHM6bQ+uYC8g+58OMm34hGYFom8PhTfDef7tueY3H3RnV3I/D3I9B/sS+l+3oNhfyW55zQY+4QaOmXuN6Le160XVbFY/rGTL1WncQKJo2dBq3wq1uQK/6o1BXEX0+6spdd/Tk61CTG8/Fl+XadHyZ4M8+fdqbDicOnNpjJtpbC3AH4fxJJ8dzOrQRag+efD+v+NQDwZg5EChwQf/GY67OeNZtcMXXoPDsAf1V9VrNAXj+826ogWl/ATd+H955Cv70HbjlaTjvzsEu4UmqrmfTzui4OEc2u4u8OvJnRw8Co91B4cA6d1Du7lqeIchCfygKh2DdT+CN7xEKFHI0eB470qbzZvPZvFUZYOfROppDEQC8HuHsoizmTgjyF3PG8uGzC0mLtLi7gr333/D+GrfNsy6DeZ90/4Q9qcc8uh22RoO+YjvtQT/zZtedNWfUyWUjEXex2s7fu8fhjW5+sPjkyIITLnJhOhAHgUjYNXruf8M9yt5xF911xpcJWUWQPRKyRrrfUWsjtNRBS4PrA95af3L6lGBPP0ND6lmuZ0rHs8D6Y+6M9/BGdxA4vNH1vGkbPdKT5qrrplwLV359aAwt0FORiOs2+fI/u1499cfcxV23PDHYJetec52rUqs96J5PHOzw+pC7HufmJ+CcKwe7tD1moT8MhcIR9lU2sP3wCbYfqmXboRO8s7eK2uYQI7L93DB7DIvnjmN+cRCpKYP1v4D3noGaDyA9zwW1x+fq4D1pbtqTFn0dM31sN1Rso8ug70rNgVPrUdvXV2DZAAANQ0lEQVSC0ut3/cUDwZjnvA7z8l0I54xy7QVZRd1/hQ6HXJjuf91dJPPBn92ZMkBwIkz8sDvrjg337CL33NO+3JGwqxcONbuy9vXrfXOdO0Ad3ui+MUxfDBM/1LdtDgWHNrquz4irxx+IPvOmSxb6SaKpNcwrO46yYv1B/rD9KC2hCOPzA9x43lhumjuWaSOzYe+f3Bl7U407k4yEXHVH23TH15mF7nLv6Tf2/StsayPsfc19fW6qdl+Nm2pipmPm0cnfmnhOhnX2aHcgaDsgNNe6M/kP3nZn4wCF57gD1aSLXdj319gxpnuRiPt7GoSx483pLPSTUG1TKy9uOcKKDQd5Y/cxwhFl6qhsbpo7jhvnjGVCQWDodguNRKCl1rVJ1FVA3WFXJVN7xD3XHXFfseuOuum2YTBGznAhP/HD7jnebyPGpBgL/SR3rK6ZFzYdYsX6g5TuPw6AzyvkBXzkBnzkxTyC0ee2+WODAWaPzxu6g8VFIu4KVPGkbm8hY3rIQj+FlB9v4OWtRzhS20xNYys1ja2caGyluqH15Oum1lPuUCcC5xRlM684yLzifOYVB5kyMgevZ4h+UzDGdMlul5hCxudncu/CyV0uE4kotU0hahpb2VdZz/qyataXVfPS1iMsLy0HIMvv5bwJQeZOcAeCuROCFOUM4IU2xph+Z6GfIjweIS/TR16mj+LCTC6d6m5qoarsr2zgvbLjvPdBNe99UM3Tr+4hFHFfC84dlcNl5xZx2dQiSiblk542fC5WMcaczqp3zGkaW8JsPljDuv3HeW1XBWv3HqclHCHg8/Lhswu5dKo7CEwaMcRuWm1MCrM6fZMw9c0h3tpTyZ92VvDKjgo+qGoAYGJhJpdFDwDTx+TSHIrQ2BKmsTVMU2uYxpYwDa1hmqLzGlvDNLdGyA2kUZSTTlF2OiNy0inKSScnPW3o9jwyZhiw0Df9Zt+xev60s4I/7azgz+9X0tga7vM209M8jMh2B4CinHQ3ne2nMDudwmw/hVnpjIi+DgZ8eM7Q4ByOKEdONHGgupEDxxvdc8x0pt/LjXPGcuN5Y7se+M6YYcZC3wyI5lCYtXuPU3a8gYDPS4bPS8DvJeCLPvweNy863+/1UNPYSkVdM8dqW6ioa4o+N1NR28yx6HNFbTNVDS109ufpESiIHgQKsvwEM30cq2vhwPFGDp9oIhw5daWCLD/jggHGBQMcOtHEhrJqROBDZxVy87xxLJo1euh2XzUmThb6ZtgLR5TjDS1U1rVQWdfMsXr3XFXfwrHovMr6Fo43tFDYFuz5AcZGA358dDrTf2p/hb3H6nn+vQOsWH+AfZUN+NM8XD19FDfNHcvl547En9ZPI5ga048s9I3phqqyvqyaFesP8tsNB6msbyEv4OOGOWNYfN5Ypo7K6bIqqSdC4Qg1ja2EIko4+lCFsLZNa8w05AV8jM8fwldYmyHHQt+YHmgNR3h99zGef+8AL2450t5O4fUI+Zl+RmT7GZF9sl2hMOZ1msdDVX0LlfUtVNW7byJtj8roc01ja6dVVV0pyPJz3vg8zpsQdI/xQQqyej7OTSgc4UhtMw3NIXxeD2lewe/14PN68KV58HkFn8eTkIObGTx2cZYxPeDzerji3JFcce5I6ptDvLbrGIdqGqmsa+FYXbOrTqpvZv8H9VTWtdDQ0nnjddtBoiDLR0GWn+mjcynIcm0P+Zk+/GlePOKum/CK4PGARwSvR/CItE8fOdHExvJqNpTV8MrOXe0HjOKCzOgBII+5E4LMGJtLXVOIgzVNHKpubH8+VNPEwZpGDlU3cbS2iUgcBxyvR/B5hfQ0LxMLM5kyMoepo7KZOiqHKaOyGRe0bx7JwM70jemFhpZQ+wEhFFEKsvwUZvnJzUhMdVCsuuYQmw/UsL6smg3Rx8GapjMun57mYWwwwJi8DMbkBRgbdM85GWmEIhFaQ0prJEJrKEJrWGkJR2htfygNLSH2Hqtn55E6Kmqb27eb5fdyzqgcpo48eSAYGwy4bw1pHvxeT3TafZPweqTPB4kTTa0cON7IwQ69sA5UuwPy1FHZXDCpgJJJBcwel5fS7TFWvWNMEjt6ookN5TVsP3SC3ICPMXkZjA26huv8TF/CzsirG1rYdbSOnUdq2XXEPe88UsexuuZu1xVx36DSY6qR/G0HhzRvdDp2npvf0BxqD/baptAp2/R7PYwJZjAuGCA/y8/WgyfYe8wNu52e5mHuhCALJruDwPziIDln6JXV1Bqm/HgjZVUNfFDVwP5K93ywuhERTi1T9NnXXkY3Lz3N9UxzvdNOTruHp703W6bfy5hggOz0/q1YsdA3xvSb4/Ut7DxSS0Vds/uGEFKaw23fHiK0RJ+bo++1hMPRZ/dec8xyLTHLt4Qi+NM87T2vYntkjQ8GGJGdfto3qYraZkr3VbF233HW7qtiy8EaIuq69k4fk8sFkwoozPK7cK9qoKyqgcMnmk5pYwn4vBQXZDI2mIGIuLKHTi1XS3T/2srbFH0/XsFMHxPyMxmf73qWTShw0xPyMxmXf3ovs56y0DfGpKS65hDrP6jmnX1VlO6r4r0PqmlsDTMqN53igkwmFGQysSCL4sIAxQWZFBdkMSLb36tvR5GI0hyKuCvQ265Ebw3T1BqhOTpd1xziUE0TZVUN7tvFcffc8YAxItvPRWcV8v8+Nr9X+20NucaYlJSdnsbFU0Zw8ZQRgOuZFY4oGb7EDxbo8Yi7GNHvJb8H60UiyrH6ZsqqGimPHgTKjzf0qndWT8UV+iKyCPge4AV+pKqPdHg/HfgZcD5QCdypqvui730V+AwQBh5Q1dUJK70xxnTD5/XQD3nfJx6PMDIng5E5GZw/sSeHiwT87O4WEBEv8DhwHTADuFtEZnRY7DPAcVU9B/hP4DvRdWcAdwEzgUXAD6LbM8YYMwji6d+0ANitqntUtQVYBtzUYZmbgJ9Gp58FrhRXQXYTsExVm1V1L7A7uj1jjDGDIJ7QHweUxbwuj87rdBlVDQE1QGGc6yIiS0SkVERKKyoq4i+9McaYHhkSVzKo6tOqWqKqJUVFRYNdHGOMSVrxhP4BYELM6/HReZ0uIyJpQB6uQTeedY0xxgyQeEJ/LTBFRCaLiB/XMLuywzIrgXui07cBf1R3AcBK4C4RSReRycAU4J3EFN0YY0xPddtlU1VDInI/sBrXZXOpqm4RkYeBUlVdCfwY+LmI7AaqcAcGosstB7YCIeA+Ve37bZaMMcb0il2Ra4wxSWDYDsMgIhXA/j5sYgRwLEHFGQqSbX8g+fYp2fYHkm+fkm1/4PR9mqiq3faEGXKh31ciUhrP0W64SLb9geTbp2TbH0i+fUq2/YHe79OQ6LJpjDFmYFjoG2NMCknG0H96sAuQYMm2P5B8+5Rs+wPJt0/Jtj/Qy31Kujp9Y4wxZ5aMZ/rGGGPOIGlCX0QWicgOEdktIg8OdnkSQUT2icgmEVkvIsPu4gURWSoiR0Vkc8y8AhF5SUR2RZ8HdjDxPjrDPn1TRA5EP6f1InL9YJaxJ0RkgoisEZGtIrJFRL4YnT8sP6cu9mc4f0YZIvKOiGyI7tM/R+dPFpG3o5n3q+iICd1vLxmqd6Jj9O8ErsaN5LkWuFtVtw5qwfpIRPYBJao6LPsXi8ilQB3wM1WdFZ33b0CVqj4SPTjnq+o/DGY5e+IM+/RNoE5VvzuYZesNERkDjFHVd0UkB1gH3AzcyzD8nLrYnzsYvp+RAFmqWiciPuB14IvAl4H/UdVlIvIksEFVn+hue8lyph/PmP9mgKnqq7hhOWLF3nvhp7h/yGHjDPs0bKnqIVV9NzpdC2zDDX8+LD+nLvZn2FKnLvrSF30o8BHc/UugB59RsoR+XOP2D0MKvCgi60RkyWAXJkFGqeqh6PRhYNRgFiaB7heRjdHqn2FRFdKRiEwC5gFvkwSfU4f9gWH8GYmIV0TWA0eBl4D3gero/UugB5mXLKGfrC5W1fm4W1XeF61aSBrRkViHf/0iPAGcDcwFDgH/MbjF6TkRyQZ+A3xJVU/EvjccP6dO9mdYf0aqGlbVubjh6RcA03q7rWQJ/aQct19VD0SfjwLPkRy3mjwSrXdtq389Osjl6TNVPRL9p4wAP2SYfU7ReuLfAM+o6v9EZw/bz6mz/Rnun1EbVa0G1gAfAoLR+5dADzIvWUI/njH/hxURyYo2RCEiWcA1wOau1xoWYu+9cA+wYhDLkhBt4Rh1C8Poc4o2Ev4Y2Kaqj8a8NSw/pzPtzzD/jIpEJBidDuA6rGzDhf9t0cXi/oySovcOQLQL1mOcHPP/24NcpD4RkbNwZ/fg7nvwi+G2TyLyS+By3GiAR4CHgOeB5UAxbjTVO1R12DSMnmGfLsdVGyiwD/irmPrwIU1ELgZeAzYBkejsr+HqwYfd59TF/tzN8P2M5uAaar24E/XlqvpwNCOWAQXAe8AnVLW52+0lS+gbY4zpXrJU7xhjjImDhb4xxqQQC31jjEkhFvrGGJNCLPSNMSaFWOgbY0wKsdA3xpgUYqFvjDEp5P8DA6WREFPHXvYAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(tr_loss_hist, label = 'train')\n",
    "plt.plot(val_loss_hist, label = 'validation')\n",
    "plt.legend()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "test acc: 98.72%\n"
     ]
    }
   ],
   "source": [
    "yhat = cnn.predict(sess = sess, x_data = x_tst)\n",
    "print('test acc: {:.2%}'.format(np.mean(yhat == y_tst)))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}