{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# CS 20 : TensorFlow for Deep Learning Research\n",
    "## Lecture 05 : Variable sharing and managing experiments\n",
    "### Applied example with tf.data\n",
    "Ref : [Toward Best Practices of TensorFlow Code Patterns](https://wookayin.github.io/TensorFlowKR-2017-talk-bestpractice/ko/#1) by Jongwook Choi, Beomjun Shin  \n",
    "\n",
    "- Using **high-level api** `tf.keras.layers`\n",
    "- Creating the **input pipeline** with `tf.data`\n",
    "- Creating the model as **Class**\n",
    "- Training the model with **learning rate scheduling** by exponential decay learning rate\n",
    "- Saving the model and Restoring the model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Setup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1.12.0\n"
     ]
    }
   ],
   "source": [
    "from __future__ import absolute_import, division, print_function\n",
    "import os, sys\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import tensorflow as tf\n",
    "from tensorflow import keras\n",
    "%matplotlib inline\n",
    "\n",
    "print(tf.__version__)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load and Pre-process data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "(x_train, y_train), (x_tst, y_tst) = keras.datasets.mnist.load_data()\n",
    "x_train = x_train  / 255\n",
    "x_train = x_train.reshape(-1, 784)\n",
    "x_tst = x_tst / 255\n",
    "x_tst = x_tst.reshape(-1, 784)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(55000, 784) (55000,)\n",
      "(5000, 784) (5000,)\n"
     ]
    }
   ],
   "source": [
    "tr_indices = np.random.choice(range(x_train.shape[0]), size = 55000, replace = False)\n",
    "\n",
    "x_tr = x_train[tr_indices]\n",
    "y_tr = y_train[tr_indices]\n",
    "\n",
    "x_val = np.delete(arr = x_train, obj = tr_indices, axis = 0)\n",
    "y_val = np.delete(arr = y_train, obj = tr_indices, axis = 0)\n",
    "\n",
    "print(x_tr.shape, y_tr.shape)\n",
    "print(x_val.shape, y_val.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Define DNN Classifier with two hidden layer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "class DNNClassifier:\n",
    "    def __init__(self, X, y, n_of_classes, hidden_dims = [100, 50], name = 'DNN'):\n",
    "        \n",
    "        with tf.variable_scope(name):\n",
    "            with tf.variable_scope('input_layer'):\n",
    "                self.X = X\n",
    "                self.y = y\n",
    "        \n",
    "            h = self.X\n",
    "\n",
    "            for layer, h_dim in enumerate(hidden_dims):\n",
    "                with tf.variable_scope('hidden_layer_{}'.format(layer + 1)):\n",
    "                    h = keras.layers.Dense(units = h_dim, activation = keras.activations.tanh,\n",
    "                                           kernel_initializer = keras.initializers.VarianceScaling())(h)\n",
    "        \n",
    "            with tf.variable_scope('output_layer'):\n",
    "                score = keras.layers.Dense(units = n_of_classes)(h)\n",
    "        \n",
    "            with tf.variable_scope('ce_loss'):\n",
    "                self.loss = tf.losses.sparse_softmax_cross_entropy(labels = self.y,\n",
    "                                                                   logits = score)\n",
    "                \n",
    "            with tf.variable_scope('prediction'):\n",
    "                self.__prediction = tf.argmax(input = score, axis = 1)\n",
    "    \n",
    "    def predict(self, sess, X):\n",
    "        feed_predict = {self.X : X}\n",
    "        return sess.run(fetches = self.__prediction, feed_dict = feed_predict)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Create a model of DNN Classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "859\n"
     ]
    }
   ],
   "source": [
    "# hyper-parameter\n",
    "epochs = 15\n",
    "batch_size = 64\n",
    "learning_rate = .005\n",
    "total_step = int(x_tr.shape[0] / batch_size)\n",
    "print(total_step)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<BatchDataset shapes: ((?, 784), (?,)), types: (tf.float64, tf.uint8)>\n",
      "<BatchDataset shapes: ((?, 784), (?,)), types: (tf.float64, tf.uint8)>\n"
     ]
    }
   ],
   "source": [
    "## create input pipeline with tf.data\n",
    "# for train\n",
    "tr_dataset = tf.data.Dataset.from_tensor_slices((x_tr, y_tr))\n",
    "tr_dataset = tr_dataset.shuffle(buffer_size = 10000)\n",
    "tr_dataset = tr_dataset.batch(batch_size = batch_size)\n",
    "tr_iterator = tr_dataset.make_initializable_iterator()\n",
    "print(tr_dataset)\n",
    "\n",
    "# for validation\n",
    "val_dataset = tf.data.Dataset.from_tensor_slices((x_val,y_val))\n",
    "val_dataset = val_dataset.batch(batch_size = batch_size)\n",
    "val_iterator = val_dataset.make_initializable_iterator()\n",
    "print(val_dataset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "## define Iterator\n",
    "# tf.data.Iterator.from_string_handle의 output_shapes는 default = None이지만 꼭 값을 넣는 게 좋음\n",
    "handle = tf.placeholder(dtype = tf.string)\n",
    "iterator = tf.data.Iterator.from_string_handle(string_handle = handle,\n",
    "                                               output_types = tr_iterator.output_types,\n",
    "                                               output_shapes = tr_iterator.output_shapes)\n",
    "\n",
    "x_data, y_data = iterator.get_next()\n",
    "x_data = tf.cast(x_data, dtype = tf.float32)\n",
    "y_data = tf.cast(y_data, dtype = tf.int32)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "dnn = DNNClassifier(X = x_data, y = y_data, n_of_classes = 10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Create training op and train model\n",
    "Applying exponential decay learning rate to train DNN model  \n",
    "```python\n",
    "decayed_learning_rate = learning_rate * decay_rate ^ (global_step / decay_steps)\n",
    "\n",
    "```\n",
    "Ref : https://www.tensorflow.org/api_docs/python/tf/train/exponential_decay"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Applying exponential decay learning rate to train dnn model\n",
    "global_step = tf.Variable(initial_value = 0 , trainable = False)\n",
    "exp_decayed_lr = tf.train.exponential_decay(learning_rate = learning_rate,\n",
    "                                            global_step = global_step,\n",
    "                                            decay_steps = total_step * 5,\n",
    "                                            decay_rate = .9,\n",
    "                                            staircase = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# create training op\n",
    "opt = tf.train.AdamOptimizer(learning_rate = exp_decayed_lr)\n",
    "\n",
    "# equal to 'var_list = None'\n",
    "training_op = opt.minimize(loss = dnn.loss,\n",
    "                           var_list = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES),\n",
    "                           global_step = global_step) \n",
    "\n",
    "# create summary op for tensorboard\n",
    "loss_summ = tf.summary.scalar(name = 'loss', tensor = dnn.loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_writer = tf.summary.FileWriter(logdir = '../graphs/lecture05/applied_example_wd/train',\n",
    "                                     graph = tf.get_default_graph())\n",
    "val_writer = tf.summary.FileWriter(logdir = '../graphs/lecture05/applied_example_wd/val',\n",
    "                                     graph = tf.get_default_graph())\n",
    "saver = tf.train.Saver()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch :   5, tr_loss : 0.09, val_loss : 0.13\n",
      "epoch :  10, tr_loss : 0.06, val_loss : 0.12\n",
      "epoch :  15, tr_loss : 0.04, val_loss : 0.12\n"
     ]
    }
   ],
   "source": [
    "# epochs = 15\n",
    "# batch_size = 64\n",
    "# total_step = int(x_tr.shape[0] / batch_size)\n",
    "sess_config = tf.ConfigProto(gpu_options=tf.GPUOptions(allow_growth=True))\n",
    "sess = tf.Session(config = sess_config)\n",
    "sess.run(tf.global_variables_initializer())\n",
    "tr_handle, val_handle = sess.run(fetches = [tr_iterator.string_handle(), val_iterator.string_handle()])\n",
    "\n",
    "tr_loss_hist = []\n",
    "val_loss_hist = []\n",
    "\n",
    "for epoch in range(epochs):\n",
    "\n",
    "    avg_tr_loss = 0\n",
    "    avg_val_loss = 0\n",
    "    tr_step = 0\n",
    "    val_step = 0\n",
    "    \n",
    "    # for mini-batch training\n",
    "    sess.run(tr_iterator.initializer)\n",
    "    try:\n",
    "        while True:\n",
    "            _, tr_loss,tr_loss_summ = sess.run(fetches = [training_op, dnn.loss, loss_summ],\n",
    "                                               feed_dict = {handle : tr_handle})\n",
    "            avg_tr_loss += tr_loss\n",
    "            tr_step += 1\n",
    "            \n",
    "    except tf.errors.OutOfRangeError:\n",
    "        pass\n",
    "    \n",
    "    # for validation\n",
    "    sess.run(val_iterator.initializer)\n",
    "    try:\n",
    "        while True:\n",
    "            val_loss, val_loss_summ = sess.run(fetches = [dnn.loss, loss_summ],\n",
    "                                                          feed_dict = {handle : val_handle})\n",
    "            avg_val_loss += val_loss\n",
    "            val_step += 1\n",
    "            \n",
    "    except tf.errors.OutOfRangeError:\n",
    "        pass\n",
    "    \n",
    "    train_writer.add_summary(tr_loss_summ, global_step = (epoch + 1))\n",
    "    val_writer.add_summary(val_loss_summ, global_step = (epoch + 1))\n",
    "\n",
    "    avg_tr_loss /= tr_step\n",
    "    avg_val_loss /= val_step\n",
    "    tr_loss_hist.append(avg_tr_loss)\n",
    "    val_loss_hist.append(avg_val_loss)\n",
    "    \n",
    "    if (epoch + 1) % 5 == 0:\n",
    "        print('epoch : {:3}, tr_loss : {:.2f}, val_loss : {:.2f}'.format(epoch + 1, avg_tr_loss, avg_val_loss))\n",
    "        saver.save(sess = sess, save_path = '../graphs/lecture05/applied_example_wd/dnn', global_step = (epoch + 1))\n",
    "\n",
    "train_writer.close()\n",
    "val_writer.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.legend.Legend at 0x7f7c44d8f160>"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYQAAAD8CAYAAAB3u9PLAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzt3Xd8VfX9+PHXO5MMMgHJIASUGUBGAHFbHDiKoyqIVq2Dr6u29Wu/tbZftf5q67dDra2LWhyts1gVFcSFGxBQhARkr4QwAiSMJGS9f398TuASMi5ZN8l9Px+PPO69n3vOue8LyXmf85miqhhjjDEhgQ7AGGNM+2AJwRhjDGAJwRhjjMcSgjHGGMASgjHGGI8lBGOMMYAlBGOMMR5LCMYYYwBLCMYYYzxhgQ7gaHTr1k0zMzMDHYYxxnQoixcvLlTV7o1t16ESQmZmJosWLQp0GMYY06GIyEZ/trMqI2OMMYCfCUFEJojIShFZIyJ31fH+HSKyXESWisiHItLb570qEVni/cz0Ke8jIgu8Y74iIhEt85WMMcY0RaMJQURCgceAc4HBwBUiMrjWZt8A2ao6DJgB/MHnvVJVHe79TPQp/z/gYVU9DtgNXN+M72GMMaaZ/GlDGAOsUdV1ACLyMnAhsLxmA1Wd67P9fOCqhg4oIgJ8D5jiFT0H3Ac84W/gxpiOraKigry8PMrKygIdSqfRpUsX0tPTCQ8Pb9L+/iSENGCzz+s8YGwD218PzPZ53UVEFgGVwIOq+gaQDBSpaqXPMdP8jtoY0+Hl5eXRtWtXMjMzcdeIpjlUlZ07d5KXl0efPn2adIwW7WUkIlcB2cBpPsW9VTVfRPoCH4nIMqD4KI45FZgKkJGR0ZLhGmMCqKyszJJBCxIRkpOT2bFjR5OP4U+jcj7Qy+d1uldWO5gzgV8BE1X1QE25quZ7j+uAj4ERwE4gQURqElKdx/T2m6aq2aqa3b17o91ojTEdiCWDltXcf09/EsJCoJ/XKygCmAzM9N1AREYAT+GSwXaf8kQRifSedwNOAparW7dzLnCpt+k1wJvN+iYNeHNJPv+a71c3XGOMCVqNJgSvnv82YA6wAnhVVXNF5H4Rqek19EcgFvh3re6lg4BFIvItLgE8qKo1jdG/AO4QkTW4NoV/tNi3qmX2sq08/dm61jq8MaYDKioq4vHHHz/q/c477zyKiopaIaLA86sNQVVnAbNqld3j8/zMevb7Ehhaz3vrcD2YWl1Wahzv5m5lb1kFXbs0rfXdGNO51CSEW2655bDyyspKwsLqPzXOmjWr3vc6uqAYqZyVFgfAioK9AY7EGNNe3HXXXaxdu5bhw4czevRoTjnlFCZOnMjgwW6Y1UUXXcSoUaPIyspi2rRpB/fLzMyksLCQDRs2MGjQIG688UaysrI4++yzKS0tDdTXaREdai6jphqSGg9A7pZixvRJCnA0xpjafvNWLsu37GnRYw5OjePe72fV+/6DDz5ITk4OS5Ys4eOPP+b8888nJyfnYJfN6dOnk5SURGlpKaNHj+YHP/gBycnJhx1j9erVvPTSS/z973/n8ssv57XXXuOqqxochtWuBUVC6BHXhW6xkeS28C+cMabzGDNmzGH99x999FFef/11ADZv3szq1auPSAh9+vRh+PDhAIwaNYoNGza0WbytISgSArh2hJx8v4c/GGPaUENX8m0lJibm4POPP/6YDz74gHnz5hEdHc3pp59e54jqyMjIg89DQ0M7fJVRULQhgEsIa7bv40BlVaBDMca0A127dmXv3rrbFYuLi0lMTCQ6OprvvvuO+fPnt3F0gRFEdwjxVFYrq7buY2h6fKDDMcYEWHJyMieddBJDhgwhKiqKY4455uB7EyZM4Mknn2TQoEEMGDCAE044IYCRtp2gSQhDvJ5GuVuKLSEYYwB48cUX6yyPjIxk9uzZdb5X007QrVs3cnJyDpbfeeedLR5fWwuaKqNeidF0jQyzhmVjjKlH0CSEkBBhUGocOVusYdkYY+oSNAkBXMPydwV7qarWQIdijDHtTpAlhHhKK6pYX7gv0KEYY0y7E2QJoaZh2doRjDGmtqBKCMf1iCUiLMQSgjHG1CGoEkJ4aAgDe3a1EcvGmCaJjY0FYMuWLVx66aV1bnP66aezaNGiBo/zyCOPUFJScvB1e5lSO6gSArhqo9wte3Br9BhjzNFLTU1lxowZTd6/dkKYNWsWCQkJLRFas/iVEERkgoisFJE1InJXHe/fISLLRWSpiHwoIr298uEiMk9Ecr33Jvns86yIrPcW1FkiIsNb7mvVb3BqPMWlFeQXdew5R4wxzXfXXXfx2GOPHXx933338dvf/pbx48czcuRIhg4dyptvHrmY44YNGxgyZAgApaWlTJ48mUGDBnHxxRcfNp/RzTffTHZ2NllZWdx7772AmzRvy5YtnHHGGZxxxhnAoSm1AR566CGGDBnCkCFDeOSRRw5+XltMtd3oSGURCQUeA84C8oCFIjLTZ+UzgG+AbFUtEZGbgT8Ak4AS4GpVXS0iqcBiEZmjqjX3Rj9X1aan2SbwbVhOT4xuy482xtRn9l2wdVnLHrPnUDj3wQY3mTRpEj/96U+59dZbAXj11VeZM2cOt99+O3FxcRQWFnLCCScwceLEetcrfuKJJ4iOjmbFihUsXbqUkSNHHnzvgQceICkpiaqqKsaPH8/SpUu5/fbbeeihh5g7dy7dunU77FiLFy/mmWeeYcGCBagqY8eO5bTTTiMxMbFNptr25w5hDLBGVdepajnwMnCh7waqOldVa+5/5gPpXvkqVV3tPd8CbAe6t1TwTTGoZxwhYj2NjDEwYsQItm/fzpYtW/j2229JTEykZ8+e3H333QwbNowzzzyT/Px8tm3bVu8xPv3004Mn5mHDhjFs2LCD77366quMHDmSESNGkJuby/Lly+s7DACff/45F198MTExMcTGxnLJJZfw2WefAW0z1bY/cxmlAZt9XucBYxvY/nrgiElARGQMEAGs9Sl+QETuAT4E7lLVA37E0yxREaEc2z2WXGtYNqb9aORKvjVddtllzJgxg61btzJp0iReeOEFduzYweLFiwkPDyczM7POqa8bs379ev70pz+xcOFCEhMTufbaa5t0nBptMdV2izYqi8hVQDbwx1rlKcA/gR+parVX/EtgIDAaSAJ+Uc8xp4rIIhFZtGPHjhaJs6Zh2RhjJk2axMsvv8yMGTO47LLLKC4upkePHoSHhzN37lw2btzY4P6nnnrqwUnycnJyWLp0KQB79uwhJiaG+Ph4tm3bdthkefVNvX3KKafwxhtvUFJSwv79+3n99dc55ZRTWvDbNsyfhJAP9PJ5ne6VHUZEzgR+BUz0vdIXkTjgHeBXqnpwUnFVLVDnAPAMrmrqCKo6TVWzVTW7e/eWqW3KSo1n654ydu5r9RsSY0w7l5WVxd69e0lLSyMlJYUrr7ySRYsWMXToUJ5//nkGDhzY4P4333wz+/btY9CgQdxzzz2MGjUKgOOPP54RI0YwcOBApkyZwkknnXRwn6lTpzJhwoSDjco1Ro4cybXXXsuYMWMYO3YsN9xwAyNGjGj5L10Paaz7pYiEAauA8bhEsBCYoqq5PtuMAGYAE2raDLzyCFz10Vuq+kit46aoaoG4lpqHgTJVPaIHk6/s7GxtrH+vP75cU8iUpxfw/HVjOLV/QJs0jAlaK1asYNCgQYEOo9Op699VRBaranZj+zZ6h6CqlcBtwBxgBfCqquaKyP0iMtHb7I9ALPBvrwvpTK/8cuBU4No6upe+ICLLgGVAN+C3jX7TFpKV6tZDsGojY4w5xK8FclR1FjCrVtk9Ps/PrGe/fwH/que97/kfZsuKjw4nPTGKXJsK2xhjDgq6kco1rGHZmMCzGQNaVnP/PYM4IcSzvnA/+w5UBjoUY4JSly5d2LlzpyWFFqKq7Ny5ky5dujT5GEGzpnJtNSOWVxTsYXRmUoCjMSb4pKenk5eXR0t1Jzcuyaanpzd5/6BNCEPSvIbl/GJLCMYEQHh4OH369Al0GMZH0FYZ9egaSbfYCGtHMMYYT9AmBBFhcGo8OZYQjDEGCOKEAK4dYfW2vRyorAp0KMYYE3BBnxAqq5XV2/YFOhRjjAm4oE4IQw6OWLYBasYYE9QJISMpmtjIMGtYNsYYgjwhhIQIg1PiyLG1EYwxJrgTAsDg1DhWFOylqtpGSxpjglvQJ4Ss1DhKK6pYX7g/0KEYY0xAWUKwhmVjjAEsIdDvmFgiQkNYbg3LxpggF/QJITw0hAE9u5JjdwjGmCDnV0IQkQkislJE1ojIEctcisgdIrJcRJaKyIci0tvnvWtEZLX3c41P+SgRWeYd81FvKc2AqFkbwabhNcYEs0YTgoiEAo8B5wKDgStEZHCtzb4BslV1GG5t5T94+yYB9wJjgTHAvSKS6O3zBHAj0M/7mdDsb9NEWalxFJVUsKW4LFAhGGNMwPlzhzAGWKOq61S1HHgZuNB3A1Wdq6ol3sv5QM2E3OcA76vqLlXdDbwPTBCRFCBOVeeruyx/HrioBb5PkwxOPTQVtjHGBCt/EkIasNnndZ5XVp/rgdmN7JvmPW/0mCIyVUQWicii1lpIY1BKV0IEG7FsjAlqLdqoLCJXAdnAH1vqmKo6TVWzVTW7e/fuLXXYw0RHhNG3e6x1PTXGBDV/EkI+0MvndbpXdhgRORP4FTBRVQ80sm8+h6qV6j1mW6ppWDbGmGDlT0JYCPQTkT4iEgFMBmb6biAiI4CncMlgu89bc4CzRSTRa0w+G5ijqgXAHhE5wetddDXwZgt8nybLSo2joLiMXfvLAxmGMcYETKMJQVUrgdtwJ/cVwKuqmisi94vIRG+zPwKxwL9FZImIzPT23QX8P1xSWQjc75UB3AI8DawB1nKo3SEgbMSyMSbYhfmzkarOAmbVKrvH5/mZDew7HZheR/kiYIjfkbayrNQ4wDUsn9KvddoqjDGmPQv6kco1EqIjSEuIsqmwjTFByxKCj6zUOJvTyBgTtCwh+MhKjWf9zv3sP1AZ6FCMMabNWULwkZUahyqsKLC7BGNM8LGE4GNIWk1PI0sIxpjgYwnBxzFxkSTHRFjDsjEmKFlC8CEiDLYRy8aYIGUJoZas1HhWb99LeWV1oEMxxpg2ZQmhlqzUOCqqlFXb9gY6FGOMaVOWEGqpaVi28QjGmGBjCaGW3knRxEaG2ZxGxpigYwmhlpAQYVBKV3LsDsEYE2QsIdQhKzWeFQV7qKrWQIdijDFtxhJCHQanxlFSXsWGnfsDHYoxxrQZSwh1GJJqI5aNMcHHr4QgIhNEZKWIrBGRu+p4/1QR+VpEKkXkUp/yM7wFc2p+ykTkIu+9Z0Vkvc97w1vuazVPv2NiiQgNsYZlY0xQaXSBHBEJBR4DzgLygIUiMlNVl/tstgm4FrjTd19VnQsM946ThFsd7T2fTX6uqjOa8wVaQ3hoCP17xpKbb3cIxpjg4c8dwhhgjaquU9Vy4GXgQt8NVHWDqi4FGhreeykwW1VLmhxtG8pKiSd3SzGq1rBsjAkO/iSENGCzz+s8r+xoTQZeqlX2gIgsFZGHRSSyCcdsNVlpcewuqaCguCzQoRhjTJtok0ZlEUkBhgJzfIp/CQwERgNJwC/q2XeqiCwSkUU7duxo9Vhr+K6xbIwxwcCfhJAP9PJ5ne6VHY3LgddVtaKmQFUL1DkAPIOrmjqCqk5T1WxVze7evftRfmzTDUqJQwRrWDbGBA1/EsJCoJ+I9BGRCFzVz8yj/JwrqFVd5N01ICICXATkHOUxW1V0RBh9u8WQYw3Lxpgg0WhCUNVK4DZcdc8K4FVVzRWR+0VkIoCIjBaRPOAy4CkRya3ZX0QycXcYn9Q69AsisgxYBnQDftv8r9OyslLjWW53CMaYINFot1MAVZ0FzKpVdo/P84W4qqS69t1AHY3Qqvq9owk0ELJS45j57RZ27y8nMSYi0OEYY0yrspHKDciyEcvGmCBiCaEBh3oaWbWRMabzs4TQgMSYCNISomwqbGNMULCE0IjBqXF2h2CMCQqWEBqRlRrH+sL97D9QGehQjDGmVVlCaERWajyq8N1WqzYyxnRulhAaMSTNprAwxgQHSwiN6BnXhaSYCHLyrR3BGNO5WUJohIiQlRpndwjGmE7PEoIfBqfGsWrbXsorG1ruwRhjOjZLCH7ISo2nokpZvX1voEMxxphWYwnBD0NsbQRjTBCwhOCHzOQYYiJCybWGZWNMJ2YJwQ8hIcKgFGtYNsZ0bpYQ/JSVGseKgj1UV2ugQzHGmFbhV0IQkQkislJE1ojIXXW8f6qIfC0ilSJyaa33qkRkifcz06e8j4gs8I75ircaW7uVlRrP/vIqNmwvgsLV8N078OVfYefaQIdmjDEtotEFckQkFHgMOAvIAxaKyExVXe6z2SbgWuDOOg5RqqrD6yj/P+BhVX1ZRJ4ErgeeOMr4W09ZMRSugcJVULiKCXnLGRmxlMyntoP6zGv0zb9g6icQ3iVwsRpjTAvwZ8W0McAaVV0HICIvAxcCBxOCtyoaIuJXR31vHeXvAVO8oueA+2jrhFBdDXvyvZP+6oMnfwpXw76th7YLCaNr0rHMJ43tKWdx4phx0K0/7MmDV6+Gj38HZ93fpqEbY0xL8ychpAGbfV7nAWOP4jO6iMgioBJ4UFXfAJKBIm+95ppjHrHMZoupKINda+s+8VeUHNouMh6694fjxkO3fu6k320AJPZGQsP5y6OfkRQawYnDa77+KBh5tas6GngB9BrTal/BGGNam19rKjdTb1XNF5G+wEcisgzwu/+miEwFpgJkZGQ0LYIXL4P1nx56HZ/hTvgjT3QJoJv3E9MdROo9TFZqHB+s2I6qIjXbnf0ArJ0Lr98EN30OEdFNi9EYYwLMn4SQD/TyeZ3ulflFVfO9x3Ui8jEwAngNSBCRMO8uod5jquo0YBpAdnZ207r4nHArjLzGnfSTj2vySTsrNZ5XF+WxdU8ZKfFRrrBLHFz4GDw/ET68H859sEnHNsaYQPOnl9FCoJ/XKygCmAzMbGQfAEQkUUQivefdgJOA5aqqwFygpkfSNcCbRxu83wZMgKGXQsqwZl3BH1xjOb/WeIS+p8HoG2HBE7Dh8+ZEaowxAdNoQvCu4G8D5gArgFdVNVdE7heRiQAiMlpE8oDLgKdEJNfbfRCwSES+xSWAB316J/0CuENE1uDaFP7Rkl+sNQxKiUOknikszvoNJPaBN26BA/vaPjhjjGkmv9oQVHUWMKtW2T0+zxfiqn1q7/clMLSeY67D9WDqMGIiw+jTLYacutZYjoiBi56AZ86F9/8XLni47QM0xphmsJHKRykrNZ7l9U1h0XscjLsVFk2HtR+1bWDGGNNMlhCOUlZqHPlFpezeX173Bt/7tWu8fvM2N7jNGGM6CEsIR6mmYXl5QT13CeFRrupobwG8e3cbRmaMMc1jCeEoZaXGA5BbVztCjfRsOOmnsORfsPLdNorMGGOaxxLCUUqKiSA1vgs5tbue1nb6XdAjC966HUp2tU1wxhjTDJYQmmBwanzDdwgAYZFw8RNQshNm/0/bBGaMMc3QFlNXdDpZqXF8+N02SsoriY5o4J8w5Xg49efw8e9h0EQYPLHtgjRt55sX4OvnIbG3zxxY/SGpr7swMMHjwF6IiG1wCpz2zBJCE2SlxqEKKwr2Mqp3YsMbn/LfsHIWvP0z6H0ixHRrmyBN21jyIrx5i5sSpTgPlr5y6D0JgcRML0H4JIpu/SE6KWAhmxakCtty4LtZsPIdKPgW0kfDmb+BzJMCHd1Rs4TQBEPSXMPy8i3FjSeE0HC46EmYdppLCpc/32GvHkwtua/Dm7dC39PhilfcmhgH9sHONUfOqrt2LlQdOLRvdHIdiaIfJPSGkNBAfSPjj6pK2PSlWyRr5Swo2gSIm+34pJ/A0lfh2fOg/wQYfy8cMzjQEfvNEkITpMR3ITE6vPGG5RrHDIbTfwkf/gZyXnPzKpmObdUceO0GSB8Dk188tEBSZCykDnc/vqqr3ImjdqL4bhaUPH9ou9BISD72UKIYcB6kjWy772XqdmAvrPnQJYBVc6CsCMK6QN8zXLVw/wkQ28Nte9pd8NVT8NnD8MSJMHyK+/tP6NXwZ7QD4uaZ6xiys7N10aJFgQ4DgKueXkBRaTlv//gU/3aoqoTp57irx1sXQNeerRugaT3rPoEXLoMeg+CamdAlvnnHK9l1ZKIoXAW71wMCZ94HJ/44OO8sq6uhYj9Edm37z967FVbOdklg3cdQVQ5RSe7kP/A8OPZ7bsqa+pTsgs8fggXT3OuxU+HkOwJSXSgii1U1u9HtLCE0ze9nreCZLzaQe/85hIf62VmrcDU8ebJXxfBycP6Bd3SbFsA/L3YNyNe+07p/3GXFbsT7ipnuTuGixyGqkSrKzkIVVr3rppTfvhwi4yAhw/3E9zr0vOYnKrH5f0+qsGOlawv4bhbke+eaxEwYcD4MPB96jYXQo6xYKdrsOpYsedF9j1N+BmNvcoNY24glhFb25pJ8fvLyEmbdfgqDvdHLfpn3GMy5Gy58HEZc2XoBmpa3ZQk8N9F1DPjRbOh6TOt/pioseAre+zXEpcBlz3X+KqSNX8IH98HmBZB0LBw/GfbvcFVuRZuhaCOU15pROCK24YQRnVx3wqiugs1fHUoCu9a68tSR7i5gwPnuTrAlLt625boEt+pd6JoKZ/wSjp9y9AmmCSwhtLK1O/Yx/s+f8MdLh3FZ9lHUDVZXw7Pnu54Jt8yD+CMmiW15qq7+c+2HUFUBWuX+ELTKxaNVUF1ZR1nVoUff57X373acWw8i8+TOe9ezfQU8c5478Vw3u23+33zlLYJ/Xwv7tsE5v4PRN3S+f+uty9wJc/V7ENvTDe4ccZXrmOFLFUp3Q/FmL0ls8kkW3vMDtcYJhUfXSha9oHCNOzmXFEJIOPQ51UsC50Fcaut9zw1fwAf3Qt5Ct0Tvmfe6z2zF/09LCK2suloZ/cAHJMVEMOOmE4mPDm98pxq71sETJ0HGCXDVf1rvF6G6Cpa/AZ8/7P7Ywrq4n5Aw15NFQr3HkHrKQl157TIJ9dk+xF3Rle5yI7PHToWhl3eupUR3rnXJAOBHs1yjbyCU7ILX/8udMLMugYmPBqZuvaXtWg9zH4BlM9wKhCffAWOmNu93qLTIJ2HUPG50j8WbXUKJjId+Z7mqoOPOdJ/dVlThu7fhg9/AztWuKuqs+905oRW0aEIQkQnAX4BQ4GlVfbDW+6cCjwDDgMmqOsMrHw48AcQBVcADqvqK996zwGkcWl/5WlVd0lAc7SkhAHy5tpBrpy/k+F7x/PP6sXQJP4rugl/9HWbd6dZNyL6uZQOrKINvX4QvHnUNk8n94OSfuhN1WETLfhZARanrPTX/Sdi2DLokwKhr3FVsQhPXwW4vija7NS7K97tqoh4DAxtPdTV88Qh89P/cwLfLnoOeQwIbU1Pt3Qaf/hEWP+Ou0E+4yXXbbIt2kgN73cVR7buPtlZV6eY8m/t72LfV3SmMv7fFf89aLCGISCiwCjgLyMMtqXmFz8pniEgm7qR/JzDTJyH0B1RVV4tIKrAYGKSqRV5CeLtmW3+0t4QA8M7SAm576WvOGnQMj185kjB/G5irq+GfF7mqgFu+dA1XzVW2x63FMP9xV7WQOsJdbQ08v236tqvCpnmw4ElY8Tag7hd87E0dszpp7zaXDPYXut5EtbuSBtKGz2HG9a774/l/dlUrHUVZsbtYmf84VB5wFw+n/o9rIwlW5SVuCd7PH3HtI8OnwOl3Q3xaixy+JRPCOOA+VT3He/1LAFX9fR3bPksDJ3lvKc1LvQTR4LZ1aY8JAeDZL9Zz31vLuWJMBr+7eAji74mvaDM8Ps5NcXHNWxDSxKml9u1wv0xfPe3qTvucBqfc4R4DdRIuzoOF/4DFz3bM6qSSXa6aqGgT/PB1yBgb6IiOtG87vHY9rP8Uhl8J5/2pff/bVpTBwr/DZ392VTZZl7j1QwJVBdceleyCT//k/p0kBMb+F5z8s2bfNfmbEPw5A6UBm31e53llRxvQGCACWOtT/ICILBWRh0Wkw076cu1Jfbjl9GN56atN/OXD1f7vmNALJvwONn4OX007+g/evRHeuRMeGQKfPQR9T4MbP3JXs31PD+wVeXy6ayy7YzlM/Jv75X7rJ/DQIHj/Hm90ZztVVuy6lu5aB1e81D6TAbiBUD98w11dL3kRnh7vuja3N1WVbq6nv450vaVSR8LUT+CyZywZ1Bad5M4Jty2CwRe5O6m/DHePFWWt/vH+3CFcCkxQ1Ru81z8ExqrqbXVs+yx1XPWLSArwMXCNqs73KduKSxLTgLWqen8dx5wKTAXIyMgYtXHjxqP8im1DVfn5jKXMWJzH7y4eypSxftadq8KLl8P6z+Cmz12PncZsW+4ainNecyfa4ye59Re69Wvel2hNHaU6qXw//PMS1wd98ovQ/5xAR+SfNR/Af6a6KpiJj8KQHwQ6Ivd/vuIt195RuArSst1FQp9TAx1Zx7F1mWt4XvMB3PQZ9KxzifpGtZsqIxGJwyWD3zVQlXQ6cKeqXtBQLO21yqhGRVU1U59fxCerdvDEVaM4J8vP0ch7CuDxsa4L2nXv1l/fv2mBG/m46l0Ij4FR17o1nFuonrHNHFGdNNjdGge6OqmiDF6a5KpgLp0OWRcHLpamKM6HGT9y/fdH3+C6pwZqttV1n7ixBFu+dr/X4/8XBl7QfhJ/R7NjJXQf0OTdWzIhhOEalccD+bhG5SmqmlvHts/ikxBEJAKYDbylqo/U2jZFVQvEVbg/DJSp6l0NxdLeEwJASXklU/6+gBUFe/jXDWMZnennSNZvX4HXp7quZyf95FC5qrs6+OwhN6FWVKK7qh4ztePPmFlR6roaLnjqUO+kkVe7k1li77aNpaoCXvkhrJrtlkAdPqVtP7+lVFW4E/G8v0HKcLjsWUjq03afv+Ubd0W7bi7EpbvBV8Mmt8ngK1O/lu52eh6uW2koMF1VHxCR+4FFqjpTREYDrwOJQBmwVVWzROQq4BnAN3lcq6pLROQjoDsgwBLgJlWtNfzwcB0hIQDs2l/OpU98SeG+A8wQGLUBAAAXk0lEQVS4+UT6H+NHX3FVeOUqWP0+/Nenbjrl5W+4XgfblkFcmpvPZuTVDc+f0hHVV510/GTXFtLafe2rq9xEdbn/cQ2zY25s3c9rC9+9A6/f7J5f/ITradYayophxyrYscJduCx/0833c+qdkH39oUn/TEDZwLQA27yrhB888SWhIcJrN59IaoIf85bs2w6PjXVTI1SVw+4NbsbLk34KQy9rnTEE7U3RZlj0D1j8nKtOCo1wbQz9z3X1+S1951BdDTN/7PqC17476+h2rXejmwuWwLjb3CR5Te13X1bsqi22r3CPO7zHPfmHtonoCifc7C5c2nKQl2mUJYR2YPmWPUx6ah4947vw75vGkRDtxwk99w349zWuJ8Ypd7i5VJraHbUjq6qATfNde8mqd90ssQDdB8GACW7GyfTRzRtfoeqWN/1qGpz2Czjj7paJvT2pKIP3fgULn3ajYS99puE2p9Kiw0/4NQlg75ZD24RFQff+0H3goZ8eA20th3bMEkI7MW/tTq6Z/hXD0uP51w1+jmbet8PdJVgD3CGFa2D1HDcd8aZ5bu6lqCTod7a7czhu/NFNQ63q6tq/eMRdPZ/92879771shuv2GxYJl/wd0kbBju/cz/bvDj3fW3Bon4Mn/kGuQbPHIHfyT8iwE38HYwmhHakZzXzmoGN44mhGM5u6lRbB2o/cncPq99wgp5Awt0Rpf+/uobH+7Z/+ET76LYz6kZs+pDMngxqFq+HVq9100r7Co13VZA/vxF+TABJ6B+fdaSdkCaGdafJoZtOw6io3a+TK2W4lqx0rXHlyP3fnMOBcbw57n7rzeY/DnF/CsEluedNgOunVTJEgoYeqeuIzguvfIAhZQmiH/vDudzz+8Vp+Mr4fPzurf6DD6Zx2rXd3DavedYP9qitcVdJxZ7qG6dJdrt1g4AVuYjjrDmmCgL8Jwf4a2tDPzxnA9r0H+MuHq+kRF8mVY9u4r30wSOrjBrmN/S83o+Xaue7OYfUcN7IbXHK4dLolA2Nqsb+INiQi/P6Soezcd4D/fSOHbrGR/o9mNkcvsisMnuh+qqvdqNmCJW6VqkCN4DWmHbOKwzYWHhrCY1eOZFh6Aj9+6RsWbtgV6JCCQ0gIpGe7UdDteUZQYwLIEkIAREeEMf3a0aQnRnH9swtZuXVvoEMyxhhLCIGSFBPBcz8aQ5fwUK6Z/hVbikoDHZIxJshZQgigXknRPHfdGPYfqOTq6V9RVFIe6JCMMUHMEkKADUqJY9rV2WzaWcL1zy2irKIq0CEZY4KUJYR2YNyxyTw8aThfb9rNbS9+Q2VVdaBDMsYEIUsI7cT5w1K47/tZfLBiG79+I4fq6o4zYNAY0znYOIR25JoTM9m+t4zH5q5lwfpdXHdyH34wMo3oCPtvMsa0PrtDaGfuPHsAf5sygrguYfzvGzmM+/1H/OHd79i2p/UX2DbGBDe/EoKITBCRlSKyRkSOWOZSRE4Vka9FpFJELq313jUistr7ucanfJSILPOO+ajYbG+AG818wbBU3rj1JGbcNI5xfZN54pO1nPx/H/GzV5aQk18c6BCNMZ1Uo3URIhIKPAacBeQBC0Vkpqr6zqG7CbgWuLPWvknAvUA2oMBib9/dwBPAjcACYBYwAbf+ssElhuzMJLIzk9i4cz/PfLGBfy/azOvf5HNC3yRuOLkv3xvYg5AQy6PGmJbhzx3CGGCNqq5T1XLgZeBC3w1UdYOqLgVqd485B3hfVXd5SeB9YIKIpABxqjpf3XSrzwMXNffLdFa9k2O4b2IWX/5yPHefN5BNO0u44flFjH/oE/45bwMl5ZWBDtEY0wn4kxDSgM0+r/O8Mn/Ut2+a97zRY4rIVBFZJCKLduzY4efHdk7xUeFMPfVYPvmfM/jrFV47w5u5B9sZthZbO4MxpunaffcVVZ0GTAO3HkKAw2kXwkND+P7xqVwwLIXFG3fz9GfrefKTtUz7dB3fPz6V60/uw5C0o1hO0hhj8C8h5AO9fF6ne2X+yAdOr7Xvx155ehOPaTy+7QybdpbwzJfreXWhtTMYY5rGnyqjhUA/EekjIhHAZGCmn8efA5wtIokikgicDcxR1QJgj4ic4PUuuhp4swnxG09GcjT3ft/aGYwxTefXEpoich7wCBAKTFfVB0TkfmCRqs4UkdHA60AiUAZsVdUsb9/rgLu9Qz2gqs945dnAs0AUrnfRj7WRYDr6EpptqbKqmtk5W3n68/V8u7mI+KhwpozN4KoTepOWEBXo8IwxbcjWVDYAqCpfb3LtDHNytwJw1uBjuHpcJicem4wN/zCm87M1lQ3g2hlG9U5iVO8k8otKeWH+Rl5euJk5uds4rkcsV4/rzSUj04mNtF8FY4Kd3SEEobKKKt5ZWsDz8zbwbV4xsZFh/GBkGj8cl8lxPWIDHZ4xpoVZlZHxy5LNRTw/bwNvf1tAeVU1Jx2XzNXjMhk/sAdhoTbVlTGdgSUEc1QK9x3glYWbeWH+RrYUl5GWEMWVJ2QweXQGSTERgQ7PGNMMlhBMk1RWVfPBiu08P28DX67dSURYCN8flso1J/ZmWHpCoMMzxjSBJQTTbKu37eWf8zfy2uI89pdXcXyvBK4Z15vzh6UQGRYa6PCMMX6yhGBazN6yCv7zdT7PzdvAuh37SY6JYPKYXlw5tjepNqbBmHbPEoJpcarKF2t28ty8DXy4YhsAZw/uyQ2n9CE7MymwwRlj6mXjEEyLExFO7teNk/t1I293CS8s2MTLX23i3dytnD6gOz8/ZwBZqTapnjEdld0hmGYpLa/i+XkbePzjtRSXVvD941O546z+9OkWE+jQjDEeqzIybaq4tIK/f7qOf3y+nvKqai7P7sVPxvejZ3yXQIdmTNCzhGACYvveMh77aA0vfrWJEBGuPTGTm047lkQby2BMwFhCMAG1eVcJD3+wite/ySc2Ioypp/blupP7EGNzJhnT5iwhmHZh5da9/Om9lby/fBvdYiO49YzjmDI2w8YxGNOGLCGYduXrTbv5w7vfMX/dLtISovjZWf25eEQaobaamzGtzt+E4NfsZSIyQURWisgaEbmrjvcjReQV7/0FIpLplV8pIkt8fqpFZLj33sfeMWve63F0X9F0JCMzEnnpxhN4/roxJMVEcOe/v2XCI5/ybs5WOtJFiTGdWaN3CCISCqwCzgLycEtqXqGqy322uQUYpqo3ichk4GJVnVTrOEOBN1T1WO/1x8Cdqur3Jb/dIXQOqsrsnK386b2VrNuxn+N7JfCLcwZw4nHdAh2aMZ1SS94hjAHWqOo6VS0HXgYurLXNhcBz3vMZwHg5cimuK7x9TZATEc4bmsJ7Pz2V//vBULbvKWPK0wu46ukFfLu5KNDhGRO0/EkIacBmn9d5Xlmd26hqJVAMJNfaZhLwUq2yZ7zqov+tI4GYTi4sNIRJozOYe+fp/Pr8QeRuKebCx77g5n8tZs32vYEOz5ig0yZ9AEVkLFCiqjk+xVeqar6IdAVeA34IPF/HvlOBqQAZGRltEa5pY13CQ7nhlL5MGt2Lpz9bz9OfrWNO7lZGZiQysnei95hAj642yM2Y1uRPQsgHevm8TvfK6tomT0TCgHhgp8/7k6l1d6Cq+d7jXhF5EVc1dURCUNVpwDRwbQh+xGs6qK5dwvnZWf25elxvnvliA1+uLeTZLzYw7dN1AGQkRTMyI4FRvV2iGHBMV1vVzZgW5E9CWAj0E5E+uBP/ZGBKrW1mAtcA84BLgY/Ua60WkRDgcuCUmo29pJGgqoUiEg5cAHzQzO9iOonk2EjuPGcAMIADlVXk5O/h6427WbxxN1+s3ckbS7YAEB0RyvBeXoLISGRERgIJ0TYi2pimajQhqGqliNwGzAFCgemqmisi9wOLVHUm8A/gnyKyBtiFSxo1TgU2q+o6n7JIYI6XDEJxyeDvLfKNTKcSGRbKqN6JjOqdyI24Hkp5u0v5etNulyQ27ebxj9dSVe1uHo/rEXvwLmJU70T6doslxMY6GOMXG5hmOryS8kq+3VzM15vcXcTXm3ZTVFIBQFyXMEb2TmSU1x5xfK8EYm36DBNkbD0EEzSiI8IYd2wy4451HdtUlXWF+1m8cTffeEni45U7Dm4fExFKQnQECdHh3k8ECVHhJB4sc68PPo8OJyEq3NorTKdnCcF0OiLCsd1jObZ7LJdnu/4QxaUVLNlcRE5+Mbv2l7O7pJzikgp2l5RTULyHopIKikrKqW7ghrlrZBgJMeEkREXUSiThHBPfhePTExjY0xq6TcdlCcEEhfiocE7r353T+nevd5vqamXvgUqKSyooKi1nt5ckXLJwyaO41D0WlVSQt7v0YFlNzWt0RCjD0uNdV1mvoTs5NrKNvqUxzWMJwRhPSIgQHxVOfFQ4GUT7vV91tZJfdKih+5vNRUz7dB2V3u1GZnK0Sw69ExmZkWDdZU27ZQnBmGYKCRF6JUXTKymaC4e7Qfyl5VUsyy8+mCQ+XV3If75xw3fsLsK0V5YQjGkFURGhjOmTxJg+ScCR3WW/3mR3Eab9sYRgTBsQqfsuYmleEV9vKuKbTXXfRQxKiSMtIYr0xCjSEqJJS4wiMTocm/rLtAZLCMYESFREKGP7JjO276HusrXvIl5duJn95VWH7RcdEUpaQhRpiVGHPaYnRpGeGE332EgbjGeaxBKCMe1EXXcRqkpRSQX5RaXk7S4lv6iU/N2l5BeVkLe7lCWbiw4OwqsRERpCSkIXlywOSxjRpCdG0TO+C+FWHWXqYAnBmHZMREiMiSAxJoIhafF1brP/QOXBRJFX87i7hPyiUj5ZtYPtew8ctn2IQHZmEucN6cmEISn0jLdZZI1jU1cY08kdqKyioKjsYNJYV7ifud9tZ+U2t+bEyIwEzhuawoQhPUlP9L+7rek4/J26whKCMUFqzfZ9vJtTwOycreRu2QPA8enxnDs0hXOH9KR3ckyAIzQtxRKCMcZvG3fuZ3bOVmYvK+DbvGIABqfEcd7Qnpw7NIVju8cGOELTHJYQjDFNkre7hHdztjI7ZyuLN+4GYMAxXTl3aE/OG5pCvx6x1u21g7GEYIxptq3FZbybU8CsnK0s3LALVejbPYbzhqRw7tCeDE6Js+TQAbRoQhCRCcBfcIvZPK2qD9Z6PxK3/OUo3NKZk1R1g4hkAiuAld6m81X1Jm+fUcCzQBQwC/iJNhKMJQRjAmf73jLey93G7JwC5q3dSbVC7+Rozh3i2hyGpcdbcminWiwhiEgosAo4C8jDLal5haou99nmFmCYqt4kIpOBi1V1kpcQ3lbVIXUc9yvgdmABLiE8qqqzG4rFEoIx7cPOfQd4f/k2ZuVs5cs1hVRWK2kJUZw/LIULhqUwNM2SQ3vSkgvkjAHW1CyBKSIvAxcCy322uRC4z3s+A/ibNPDbICIpQJyqzvdePw9cBDSYEIwx7UNybCSTx2QweUwGRSXlfLBiO+8s3cL0z9cz7dN19E6O5vyhKVwwLJVBKV0tOXQQ/iSENGCzz+s8YGx923hrMBcDyd57fUTkG2AP8GtV/czbPq/WMdOOPnxjTKAlREdw6ah0Lh2VTlFJOe/lbuOtpVt46tN1PP7xWvp2j+GCYal8f1gK/Y7pGuhwTQNae6RyAZChqju9NoM3RCTraA4gIlOBqQAZGRmtEKIxpqUkREdw+eheXD66Fzv3HeDd3K28/W0Bf/1oNY9+uJoBx3TlgmEpXHB8Kn262TiH9safhJAP9PJ5ne6V1bVNnoiEAfHATq+R+ACAqi4WkbVAf2/79EaOibffNGAauDYEP+I1xrQDybGRXDm2N1eO7c32vWXMXraVt5du4c/vr+LP768iKzWOC4alcsGwFHol2Qjp9sCfRuUwXKPyeNxJeyEwRVVzfba5FRjq06h8iapeLiLdgV2qWiUifYHPvO121dGo/FdVndVQLNaobEzHV1BcyjtLC3h7aQFLNhcBboT0BcNSOX9YCqkJUQGOsPNp6W6n5wGP4LqdTlfVB0TkfmCRqs4UkS7AP4ERwC5gsqquE5EfAPcDFUA1cK+qvuUdM5tD3U5nAz+2bqfGBJfNu0p4Z1kBby/dQk6+mz5jVO9ELhiWwvlDU+gRZxPvtQQbmGaM6VA2FO7n7aVbeHtpAd9t3YsIjMlMYtyxyQhClSrV1Xro0fe5KlXVHP6+um2qvceqag4+r1al9qlPObzgiPfrOFXW3kdw63Inx0bQLTaSbt5jsvc8OTaSuC5hbd7ryhKCMabDWrN9L2971Uprtu87WB4iEBoihIgQGiKEihASIj5lHFZ28PnBskPvh9RxUq5dUnsTOWKLw3dSVYpLKyjcV87ukvI6k0hEaMjBhOH72N33dUwk3bpGkBQd0SJLqVpCMMZ0CuWV1d4Jnw41nqGyqppdJeXs3FdO4b4DFO47wM595ezwHgtrPZZXVR9xDBFIjI4gOSaCaVdnN7lnVksOTDPGmICJCOuYq7uFhYbQo2sXenRtvB1EVdlTVsnOfQco3FfuPbrnNQkjNrL1T9eWEIwxJsBEXNtDfFQ4fbsHLo6OmXqNMca0OEsIxhhjAEsIxhhjPJYQjDHGAJYQjDHGeCwhGGOMASwhGGOM8VhCMMYYA3SwqStEZAewsYm7dwMKWzCc1taR4rVYW09HircjxQodK97mxtpbVRsd8tahEkJziMgif+byaC86UrwWa+vpSPF2pFihY8XbVrFalZExxhjAEoIxxhhPMCWEaYEO4Ch1pHgt1tbTkeLtSLFCx4q3TWINmjYEY4wxDQumOwRjjDENCIqEICITRGSliKwRkbsCHU99RKSXiMwVkeUikisiPwl0TI0RkVAR+UZE3g50LI0RkQQRmSEi34nIChEZF+iY6iMiP/N+B3JE5CURaVerzYvIdBHZLiI5PmVJIvK+iKz2HhMDGaOveuL9o/e7sFREXheRhEDGWKOuWH3e+28RURHp1hqf3ekTgoiEAo8B5wKDgStEZHBgo6pXJfDfqjoYOAG4tR3HWuMnwIpAB+GnvwDvqupA4HjaadwikgbcDmSr6hAgFJgc2KiO8CwwoVbZXcCHqtoP+NB73V48y5Hxvg8MUdVhwCrgl20dVD2e5chYEZFewNnAptb64E6fEIAxwBpVXaeq5cDLwIUBjqlOqlqgql97z/fiTlhpgY2qfiKSDpwPPB3oWBojIvHAqcA/AFS1XFWLAhtVg8KAKBEJA6KBLQGO5zCq+imwq1bxhcBz3vPngIvaNKgG1BWvqr6nqpXey/lAepsHVod6/m0BHgb+B2i1ht9gSAhpwGaf13m045NsDRHJBEYACwIbSYMewf2CHrk6ePvTB9gBPONVcT0tIk1bsbyVqWo+8CfclWABUKyq7wU2Kr8co6oF3vOtwDGBDOYoXQfMDnQQ9RGRC4F8Vf22NT8nGBJChyMiscBrwE9VdU+g46mLiFwAbFfVxYGOxU9hwEjgCVUdAeynfVVpHOTVvV+IS2KpQIyIXBXYqI6Ouu6LHaILo4j8Cldd+0KgY6mLiEQDdwP3tPZnBUNCyAd6+bxO98raJREJxyWDF1T1P4GOpwEnARNFZAOuGu57IvKvwIbUoDwgT1Vr7rhm4BJEe3QmsF5Vd6hqBfAf4MQAx+SPbSKSAuA9bg9wPI0SkWuBC4Artf32wT8Wd3Hwrff3lg58LSI9W/qDgiEhLAT6iUgfEYnANc7NDHBMdRIRwdVxr1DVhwIdT0NU9Zeqmq6qmbh/049Utd1exarqVmCziAzwisYDywMYUkM2ASeISLT3OzGedtoAXstM4Brv+TXAmwGMpVEiMgFX5TlRVUsCHU99VHWZqvZQ1Uzv7y0PGOn9TreoTp8QvEaj24A5uD+qV1U1N7BR1esk4Ie4q+0l3s95gQ6qE/kx8IKILAWGA78LcDx18u5iZgBfA8twf6ftalStiLwEzAMGiEieiFwPPAicJSKrcXc5DwYyRl/1xPs3oCvwvve39mRAg/TUE2vbfHb7vUsyxhjTljr9HYIxxhj/WEIwxhgDWEIwxhjjsYRgjDEGsIRgjDHGYwnBGGMMYAnBGGOMxxKCMcYYAP4/oT2FUh3LDGMAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(tr_loss_hist, label = 'train')\n",
    "plt.plot(val_loss_hist, label = 'validation')\n",
    "plt.legend()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "test acc: 96.89%\n"
     ]
    }
   ],
   "source": [
    "yhat = dnn.predict(sess = sess, X = x_tst)\n",
    "print('test acc: {:.2%}'.format(np.mean(yhat == y_tst)))\n",
    "sess.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Restore model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Example 1\n",
    "Restore my model at epoch 15"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "tf.reset_default_graph()\n",
    "\n",
    "x_data = tf.placeholder(dtype = tf.float32, shape = [None, 784])\n",
    "y_data = tf.placeholder(dtype = tf.int32, shape = [None])\n",
    "\n",
    "dnn_restore = DNNClassifier(X = x_data, y = y_data, n_of_classes = 10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "model_checkpoint_path: \"../graphs/lecture05/applied_example_wd/dnn-15\"\n",
      "all_model_checkpoint_paths: \"../graphs/lecture05/applied_example_wd/dnn-5\"\n",
      "all_model_checkpoint_paths: \"../graphs/lecture05/applied_example_wd/dnn-10\"\n",
      "all_model_checkpoint_paths: \"../graphs/lecture05/applied_example_wd/dnn-15\"\n",
      "\n"
     ]
    }
   ],
   "source": [
    "ckpt_list = tf.train.get_checkpoint_state(checkpoint_dir = '../graphs/lecture05/applied_example_wd/')\n",
    "print(ckpt_list) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from ../graphs/lecture05/applied_example_wd/dnn-15\n"
     ]
    }
   ],
   "source": [
    "# restore my model at epoch 15\n",
    "sess = tf.Session()\n",
    "saver = tf.train.Saver()\n",
    "saver.restore(sess = sess, save_path = '../graphs/lecture05/applied_example_wd/dnn-15')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "test acc: 96.89%\n"
     ]
    }
   ],
   "source": [
    "yhat = dnn_restore.predict(sess = sess, X = x_tst)\n",
    "print('test acc: {:.2%}'.format(np.mean(yhat == y_tst)))\n",
    "sess.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Example 2\n",
    "Restore my model at epoch 10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "tf.reset_default_graph()\n",
    "\n",
    "x_data = tf.placeholder(dtype = tf.float32, shape = [None, 784])\n",
    "y_data = tf.placeholder(dtype = tf.int32, shape = [None])\n",
    "\n",
    "dnn_restore = DNNClassifier(X = x_data, y = y_data, n_of_classes = 10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "model_checkpoint_path: \"../graphs/lecture05/applied_example_wd/dnn-15\"\n",
      "all_model_checkpoint_paths: \"../graphs/lecture05/applied_example_wd/dnn-5\"\n",
      "all_model_checkpoint_paths: \"../graphs/lecture05/applied_example_wd/dnn-10\"\n",
      "all_model_checkpoint_paths: \"../graphs/lecture05/applied_example_wd/dnn-15\"\n",
      "\n"
     ]
    }
   ],
   "source": [
    "ckpt_list = tf.train.get_checkpoint_state(checkpoint_dir = '../graphs/lecture05/applied_example_wd/')\n",
    "print(ckpt_list) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from ../graphs/lecture05/applied_example_wd/dnn-10\n"
     ]
    }
   ],
   "source": [
    "# restore my model at epoch 10\n",
    "sess = tf.Session()\n",
    "saver = tf.train.Saver()\n",
    "saver.restore(sess = sess, save_path = '../graphs/lecture05/applied_example_wd/dnn-10')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "test acc: 96.77%\n"
     ]
    }
   ],
   "source": [
    "yhat = dnn_restore.predict(sess = sess, X = x_tst)\n",
    "print('test acc: {:.2%}'.format(np.mean(yhat == y_tst)))\n",
    "sess.close()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}