{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 乗算レイヤー"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class MulLayer:\n",
    "    def __init__(self):\n",
    "        self.x = None\n",
    "        self.y = None\n",
    "    \n",
    "    def forward(self, x, y):\n",
    "        self.x = x\n",
    "        self.y = y\n",
    "        out = x * y\n",
    "        return out\n",
    "    \n",
    "    def backward(self, dout):\n",
    "        dx = dout * self.y\n",
    "        dy = dout * self.x\n",
    "        \n",
    "        return dx, dy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "apple = 100\n",
    "apple_num = 2\n",
    "tax = 1.1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### layer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "mul_apple_layer = MulLayer()\n",
    "mul_tax_layer = MulLayer()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### forward"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "apple_price = mul_apple_layer.forward(apple, apple_num)\n",
    "price = mul_tax_layer.forward(apple_price, tax)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "220.00000000000003"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "price"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### backward"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "dprice = 1\n",
    "dapple_price, dtax = mul_tax_layer.backward(dprice)\n",
    "dapple, dapple_num = mul_apple_layer.backward(dapple_price)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(2.2, 110.00000000000001, 200)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(dapple, dapple_num, dtax)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 加算レイヤー"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class AddLayer:\n",
    "    def __init__(self):\n",
    "        pass\n",
    "    \n",
    "    def forward(self, x, y):\n",
    "        out = x + y\n",
    "        return out\n",
    "    \n",
    "    def backward(self, dout):\n",
    "        dx = dout * 1\n",
    "        dy = dout * 1\n",
    "        return dx, dy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "apple = 100\n",
    "apple_num = 2\n",
    "orange = 150\n",
    "orange_num = 3\n",
    "tax = 1.1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### layer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "mul_apple_layer = MulLayer()\n",
    "mul_orange_layer = MulLayer()\n",
    "add_apple_orange_layer = AddLayer()\n",
    "mul_tax_layer = MulLayer()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### forward"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "apple_price = mul_apple_layer.forward(apple, apple_num)\n",
    "orange_price = mul_orange_layer.forward(orange, orange_num)\n",
    "all_price = add_apple_orange_layer.forward(apple_price, orange_price)\n",
    "price = mul_tax_layer.forward(all_price, tax)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "715.0000000000001"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "price"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### backward"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "dprice = 1\n",
    "dall_price, dtax = mul_tax_layer.backward(dprice)\n",
    "dapple_price, dorange_price = add_apple_orange_layer.backward(dall_price)\n",
    "dorange, dorange_num = mul_orange_layer.backward(dorange_price)\n",
    "dapple, dapple_num = mul_apple_layer.backward(dapple_price)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(2.2, 110.00000000000001, 3.3000000000000003, 165.0, 650)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(dapple, dapple_num, dorange, dorange_num, dtax)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Relu"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class Relu:\n",
    "    def __init__(self):\n",
    "        self.mask = None\n",
    "    \n",
    "    def forward(self, x):\n",
    "        self.mask = (x <= 0)\n",
    "        out = x.copy()\n",
    "        out[self.mask] = 0\n",
    "        \n",
    "        return out\n",
    "    \n",
    "    def backward(self, dout):\n",
    "        dout[self.mask] = 0\n",
    "        dx = dout\n",
    "        \n",
    "        return dx"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[False,  True],\n",
       "       [ True, False]], dtype=bool)"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x = np.array([[1.0, -0.5], [-2.0, 3.0]])\n",
    "(x <= 0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Sigmoid"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class Sigmoid:\n",
    "    def __init__(self):\n",
    "        self.out = None\n",
    "    \n",
    "    def forward(self, x):\n",
    "        out = 1 / (1 + np.exp(-x))\n",
    "        self.out = out\n",
    "        \n",
    "        return out\n",
    "    \n",
    "    def backward(self, dout):\n",
    "        dx = dout * (1.0 - self.out) * self.out\n",
    "        \n",
    "        return dx"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X = np.random.rand(2)\n",
    "W = np.random.rand(2, 3)\n",
    "B = np.random.rand(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(2,)"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(2, 3)"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "W.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(3,)"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "B.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(3,)"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Y = np.dot(X, W) + B\n",
    "Y.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X_dot_W = np.array([[0, 0, 0], [10, 10, 10]])\n",
    "B = np.array([1, 2, 3])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 1,  2,  3],\n",
       "       [11, 12, 13]])"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_dot_W + B"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "dY = np.array([[1, 2, 3], [4, 5, 6]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([5, 7, 9])"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dB = np.sum(dY, axis=0)\n",
    "dB"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class Affine:\n",
    "    def __init__(self, W, b):\n",
    "        self.W = W\n",
    "        self.b = b\n",
    "        self.x = None\n",
    "        self.dW = None\n",
    "        self.db = None\n",
    "    \n",
    "    def forward(self, x):\n",
    "        self.x = x\n",
    "        out = np.dot(x, self.W) + self.b\n",
    "        \n",
    "        return out\n",
    "    \n",
    "    def backward(self, dout):\n",
    "        dx = np.dot(dout, self.W.T)\n",
    "        self.dW = np.dot(self.x.T, dout)\n",
    "        self.db = np.sum(dout, axis=0)\n",
    "        \n",
    "        return dx"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from common.functions import softmax, cross_entropy_error"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class SoftmaxWithLoss:\n",
    "    def __init__(self):\n",
    "        self.loss = None\n",
    "        self.y = None\n",
    "        self.t = None\n",
    "    \n",
    "    def forward(self, x, t):\n",
    "        self.t = t\n",
    "        self.y = softmax(x)\n",
    "        self.loss = cross_entropy_error(self.y, self.t)\n",
    "        \n",
    "        return self.loss\n",
    "    \n",
    "    def backward(self, dout=1):\n",
    "        batch_size = self.t.shape[0]\n",
    "        dx = (self.y - self.t) / batch_size\n",
    "        \n",
    "        return dx"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from common.layers import *\n",
    "from common.gradient import numerical_gradient\n",
    "from collections import OrderedDict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class TwoLayerNet:\n",
    "    def __init__(self, input_size, hidden_size, output_size, \n",
    "                weight_init_std=0.01):\n",
    "        self.params = {}\n",
    "        self.params['W1'] = weight_init_std * np.random.randn(input_size, hidden_size)\n",
    "        self.params['b1'] = np.zeros(hidden_size)\n",
    "        self.params['W2'] = weight_init_std * np.random.randn(hidden_size, output_size)\n",
    "        self.params['b2'] = np.zeros(output_size)\n",
    "        \n",
    "        self.layers = OrderedDict()\n",
    "        self.layers['Affine1'] = Affine(self.params['W1'], self.params['b1'])\n",
    "        self.layers['Relu1'] = Relu()\n",
    "        self.layers['Affine2'] = Affine(self.params['W2'], self.params['b2'])\n",
    "        \n",
    "        self.lastLayer = SoftmaxWithLoss()\n",
    "        \n",
    "    def predict(self, x):\n",
    "        for layer in self.layers.values():\n",
    "            x = layer.forward(x)\n",
    "            \n",
    "        return x\n",
    "    \n",
    "    def loss(self, x, t):\n",
    "        y = self.predict(x)\n",
    "        return self.lastLayer.forward(y, t)\n",
    "    \n",
    "    def accuracy(self, x, t):\n",
    "        y = self.predict(x)\n",
    "        y = np.argmax(y, axis=1)\n",
    "        if t.ndim != 1 : t = np.argmax(t, acis=1)\n",
    "        \n",
    "        accuracy = np.sum(y == t) / float(x.shape[0])\n",
    "        return accuracy\n",
    "    \n",
    "    def numerical_gradient(self, x, t):\n",
    "        loss_W = lambda W: self.loss(x, t)\n",
    "        \n",
    "        grads = {}\n",
    "        grads['W1'] = numerical_gradient(loss_W, self.params['W1'])\n",
    "        grads['b1'] = numerical_gradient(loss_W, self.params['b1'])\n",
    "        grads['W2'] = numerical_gradient(loss_W, self.params['W2'])\n",
    "        grads['b2'] = numerical_gradient(loss_W, self.params['b2'])\n",
    "        \n",
    "        return grads\n",
    "    \n",
    "    def gradient(self, x, t):\n",
    "        # forward\n",
    "        self.loss(x, t)\n",
    "        \n",
    "        # backward\n",
    "        dout = 1\n",
    "        dout = self.lastLayer.backward(dout)\n",
    "        \n",
    "        layers = list(self.layers.values())\n",
    "        layers.reverse()\n",
    "        for layer in layers:\n",
    "            dout = layer.backward(dout)\n",
    "        \n",
    "        grads = {}\n",
    "        grads['W1'] = self.layers['Affine1'].dW\n",
    "        grads['b1'] = self.layers['Affine1'].db\n",
    "        grads['W2'] = self.layers['Affine2'].dW\n",
    "        grads['b2'] = self.layers['Affine2'].db\n",
    "        \n",
    "        return grads"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "from dataset.mnist import load_mnist"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "(x_train, t_train), (x_test, t_test) = load_mnist(normalize=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "network = TwoLayerNet(input_size=784, hidden_size=50, output_size=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "x_batch = x_train[:3]\n",
    "t_batch = t_train[:3]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "grad_numerical = network.numerical_gradient(x_batch, t_batch)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "grad_backprop = network.gradient(x_batch, t_batch)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "W1:2.14890239345e-13\n",
      "b1:8.16454107013e-13\n",
      "W2:8.68228468992e-13\n",
      "b2:1.19237957008e-10\n"
     ]
    }
   ],
   "source": [
    "for key in grad_numerical.keys():\n",
    "    diff = np.average( np.abs(grad_backprop[key] - grad_numerical[key]) )\n",
    "    print(key + \":\" + str(diff))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "network = TwoLayerNet(input_size=784, hidden_size=50, output_size=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "iter_num = 10000\n",
    "train_size = x_train.shape[0]\n",
    "batch_size = 100\n",
    "learning_rate = 0.1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train_loss_list = []\n",
    "train_acc_list = []\n",
    "test_acc_list = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "iter_per_epoch = max(train_size // batch_size, 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.124316666667 0.1146\n",
      "0.903116666667 0.9053\n",
      "0.925433333333 0.9269\n",
      "0.936916666667 0.9373\n",
      "0.945333333333 0.9442\n",
      "0.9526 0.9487\n",
      "0.957616666667 0.9553\n",
      "0.961666666667 0.9584\n",
      "0.965816666667 0.9612\n",
      "0.966033333333 0.9596\n",
      "0.9708 0.9639\n",
      "0.97305 0.9664\n",
      "0.97405 0.9656\n",
      "0.973933333333 0.965\n",
      "0.9774 0.9678\n",
      "0.978283333333 0.9686\n",
      "0.97955 0.9702\n"
     ]
    }
   ],
   "source": [
    "for i in range(iter_num):\n",
    "    batch_mask = np.random.choice(train_size, batch_size)\n",
    "    x_batch = x_train[batch_mask]\n",
    "    t_batch = t_train[batch_mask]\n",
    "    \n",
    "    # backprop\n",
    "    grad = network.gradient(x_batch, t_batch)\n",
    "    \n",
    "    # update\n",
    "    for key in ('W1', 'b1', 'W2', 'b2'):\n",
    "        network.params[key] -= learning_rate * grad[key]\n",
    "    \n",
    "    loss = network.loss(x_batch, t_batch)\n",
    "    train_loss_list.append(loss)\n",
    "    \n",
    "    if i % iter_per_epoch == 0:\n",
    "        train_acc = network.accuracy(x_train, t_train)\n",
    "        test_acc = network.accuracy(x_test, t_test)\n",
    "        train_acc_list.append(train_acc)\n",
    "        test_acc_list.append(test_acc)\n",
    "        print(train_acc, test_acc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.981033333333 0.969\n"
     ]
    }
   ],
   "source": [
    "final_train_acc = network.accuracy(x_train, t_train)\n",
    "final_test_acc = network.accuracy(x_test, t_test)\n",
    "train_acc_list.append(final_train_acc)\n",
    "test_acc_list.append(final_test_acc)\n",
    "print(final_train_acc, final_test_acc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x7f7fb954b6d8>]"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD8CAYAAACMwORRAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xl4VNX9BvD3m4UdQSAosjSsKm4IqYJWSxFcwNa2an/a\nqtXWUlpttdpaqGvVFupWF5BFS1FUVASBsij7viYQQiCEBEhIQshCIPtkkpnz++PemcyamSSTzNyb\n9/M8eZh7586dcwK8c+fcs4hSCkREZC5R4S4AERGFHsOdiMiEGO5ERCbEcCciMiGGOxGRCTHciYhM\niOFORGRCDHciIhNiuBMRmVBMuN64V69eKj4+PlxvT0RkSElJScVKqbhAx4Ut3OPj45GYmBiutyci\nMiQRyQ7mODbLEBGZEMOdiMiEGO5ERCbEcCciMiGGOxGRCTHciYhMiOFORGRChgv3E0UVeH5ZKmpt\n9nAXhYgoYhku3I+eKcfC3dkY+uyacBeFiChiGS7cbx1+UbiLQEQU8QwX7jHR9UVm0wwRkW+GC3cA\nuHZAdwBAuaUuzCUhIopMhgz3q/t2AwAcKygPc0mIiCKTIcP9tisvDncRiIgimiHDvX2MVuwNaQVh\nLgkRUWQyZLgXldcAAD7YdjLMJSEiikyGDPfrB/YEADw05jthLgkRUWQyZLi3j9WK/fGuoBYkISJq\ncwwZ7tFREu4iEBFFNGOGuzDciYgaYsxw55U7EVGDDBnuwit3IqIGGTLciYioYYYO9+F9Lgh3EYiI\nIpKhw/1Iflm4i0BEFJEMHe5EROQbw52IyIQY7kREJsRwJyIyIcOG+6Sr+mBwXOdwF4OIKCIZNtyj\nowR2Fe5SEBFFpoDhLiL9RWSTiBwRkcMi8oSPY0RE3hWRTBFJEZGRLVPcetFRgjo7F8gmIvIlJohj\n6gA8rZTaLyJdASSJyDql1BGXY+4AMFT/uR7AbP3PFmOptSGnpLol34KIyLACXrkrpfKVUvv1x+UA\n0gD09TjsLgAfK81uAN1FpE/IS+tiTeoZAFrIExGRu0a1uYtIPIBrAezxeKovgByX7Vx4fwBARCaL\nSKKIJBYVFTWupH7Y2PBOROQl6HAXkS4AlgB4UinVpHH/Sql5SqkEpVRCXFxcU07hfc6QnIWIyFyC\nCncRiYUW7J8qpZb6OCQPQH+X7X76vhanFOOdiMhTML1lBMB/AKQppd7yc9gKAA/pvWZGAyhVSuWH\nsJx+ZRRWtMbbEBEZSjC9ZW4E8CCAQyKSrO/7G4ABAKCUmgNgNYCJADIBVAF4JPRF9e1cpbW13oqI\nyDAChrtSajuABpc+UlrbyGOhKhQRETWPYUeoOrDJnYjIm+HD3cZ0JyLyYvhwt7OfOxGRF8OG+1MT\nhgEABvfuEuaSEBFFHsOG++A4hjoRkT+GDfcovf+OnW3uREReDBvu2tgqgLP+EhF5M2y488qdiMg/\nw4a748qd2U5E5M2w4b7xaCEAYMn+3DCXhIgo8hg23E+VVAIAUvNKw1wSIqLIY9hwbxetFb2mjndU\niYg8GTbco/U7qlyJiYjIm2HDvWuHWADAkfwmLQpFRGRqhg33u0f2C3cRiIgilmHDvUOsYYtORNTi\nDJuQjjZ3IiLyZthwj4kybNGJiFqcYROS2U5E5J9hI5JX7kRE/hk2IQfHdQYAjL/8ojCXhIgo8hg2\n3GOioyACDO/TNdxFISKKOIYNdwAQABygSkTkzdDhHiUCBaY7EZEnw4c7r9yJiLwZOtxFuBITEZEv\nhg73KBGuxERE5IPBwx2ws12GiMiLwcOdbe5ERL4YOtzLa+qwPbMo3MUgIoo4hg53AIiNNnwViIhC\nztDJGN+zE4b07hLuYhARRRxDh3uUCNdQJSLywdjhHsWukEREvgQMdxGZLyKFIpLq5/mxIlIqIsn6\nzwuhL6ZvURzERETkU0wQxywAMBPAxw0cs00pdWdIStQIbJYhIvIt4JW7UmorgJJWKEujsZ87EZFv\noWpzv0FEUkRkjYhc4e8gEZksIokiklhU1Pz+6VFRbJYhIvIlFOG+H8AApdTVAN4DsMzfgUqpeUqp\nBKVUQlxcXLPfOFqE4U5E5EOzw10pVaaUqtAfrwYQKyK9ml2yIAjb3ImIfGp2uIvIxSIi+uPr9HOe\nbe55gxHNrpBERD4F7C0jIosAjAXQS0RyAbwIIBYAlFJzANwD4HciUgegGsB9SrVO5EYJeOVORORD\nwHBXSt0f4PmZ0LpKtrootrkTEflk7BGqDHciIp+MHe5RYD93IiIfjB3uvHInIvLJ+OHOS3ciIi+G\nDncAqKmzh7sIREQRJ5iJwyLWlmNcYo+IyBfDX7kTEZE3hjsRkQkx3ImITIjhTkRkQoYO95+O7Bvu\nIhARRSRDh3vPzu3QqV10uItBRBRxDB3uFTU2VFltaKVJKImIDMPQ4b5o7ykAwLaM4jCXhIgoshg6\n3B04SpWIyJ0pwp2IiNyZItzZ5k5E5M4c4R7uAhARRRhzhDuv3ImI3Jgi3CtqbOEuAhFRRDFFuL+3\nMSPcRSAiiiimCPcqK6/ciYhcmSLcbVxqj4jIjSnCvc7GQUxERK7MEe68cicicsNwJyIyIVOEO9vc\niYjcMdyJiEzIFOFORETuDB3uXIWJiMg3Q4d7lEi4i0BEFJEMHu7hLgERUWQydLgP7t0l3EUgIopI\nhg73V+66MtxFICKKSAHDXUTmi0ihiKT6eV5E5F0RyRSRFBEZGfpi+taRN1SJiHwK5sp9AYDbG3j+\nDgBD9Z/JAGY3v1jB6dWlfWu9FRGRoQQMd6XUVgAlDRxyF4CPlWY3gO4i0idUBWxIt46xrfE2RESG\nE4o2974Acly2c/V9XkRksogkikhiUVFRCN6aiIh8adUbqkqpeUqpBKVUQlxcXGu+NRFRmxKKcM8D\n0N9lu5++j4iIwiQU4b4CwEN6r5nRAEqVUvkhOC8RETVRTKADRGQRgLEAeolILoAXAcQCgFJqDoDV\nACYCyARQBeCRliosEREFJ2C4K6XuD/C8AvBYyEpERETNZugRqkRE5BvDnYjIhBjuREQmxHAnIjIh\n04S7dl+XiIgAE4X7wdzScBeBiChimCbcFyfmBD6IiKiNME24f7rnVLiLQEQUMUwT7kREVI/hTkRk\nQgx3IiITYrgTEZkQw52IyIQY7kREJsRwJyIyIYY7EZEJmSrcOb8MEZHGVOH+3sbMcBeBiCgimCrc\n31p3LNxFICKKCKYKdyIi0pg23J/4/ACWJ+f5fd5Sa8P/Dp5uxRIREbUew4f7/dcNcNsuLLMAAJYn\nn8YTnyf7fd0/V6fhD4sOYNfxsy1aPiKicDB8uA/v09Vt+9GPE4N63enz2odAuaU25GUiIgo3w4f7\nnVdf4rZdXF4TppIQEUUOw4d7lIjbtp1d3YmIjB/udo+BS57bgfCzgIjMyPDh7hnOheU1+NvXh8JS\nFiKiSGH8cPdxpf4Z11MlojbO8OHerWNso45/cXkq4qeuaqHSEBFFBsOHe0x046rw0a7sFioJEVHk\nMHy4B/LW2nRMX5Pmtd+jkw0RkamYPtzf3ZiJuVtOhLsYREStyvThTkTUFjHciYhMKKhwF5HbRSRd\nRDJFZKqP58eKSKmIJOs/L4S+qEREFKyYQAeISDSAWQAmAMgFsE9EViiljngcuk0pdWcLlNFU5m8/\niS7tY/Cz7/YPd1GIyMQChjuA6wBkKqVOAICIfA7gLgCe4R7xqqx1zsc1dfawlOHlldqvjeFORC0p\nmGaZvgByXLZz9X2ebhCRFBFZIyJXhKR0IXT6fDV+uzDJub31WJHzcU5JlVvwN0adzY631x9DZU3T\nXk9E1BJCdUN1P4ABSqmrAbwHYJmvg0RksogkikhiUVGRr0NazA0zNmJbRrHXfqWAm17bhIfn72vS\neZcln8bb6zPw5lqu30pEkSOYcM8D4NqG0E/f56SUKlNKVeiPVwOIFZFenidSSs1TSiUopRLi4uKa\nUWx3ndtFN/m1yTnnAQB7s0r8HrP1WBFOFlf6fK6mzgYAqK61NbkMREShFky47wMwVEQGikg7APcB\nWOF6gIhcLKKN+RSR6/Tzttr6dd/+6eYmv3bOluMBj3lo/l784I3NOFdpbeAo35MHZxSUw8LgJ6JW\nFjDclVJ1AB4H8C2ANABfKqUOi8gUEZmiH3YPgFQROQjgXQD3KV/TNbaQfhd2Csl5AhX5zve2B3Ue\nu10hp6QKZZZaTPj3Vjy9+GAoitck56usKCy3hO39iSg8gukt42hqWe2xb47L45kAZoa2aK3vg20n\nMPnmwX6fzztfHdR5Zm85jte/Tcdnj14PANhzwn+TT0sb8fI6AEDWjElhKwMRtT7TjFDt36Njs8/x\nTeoZAFo7erBXuwLvGch2n9BapPJLHedQOJhzHrYg1wBUSqHWFp6umkRkDqYJ95io5ldl/ynt5uqU\nhUm47h8bED91FXZmevewcZVd4vtGK1A/82RxhRV3zdqBmRszgyrHB9tOYOizawK08RMR+WeacA9V\nE3/uuSpsSq/vprn2SIHP46x1dkxZmOSccTKjoALHiyrcjim3uPd9T8svC6oMS5K0zkgFbCsnoiYy\nTbiHyvf+tclte8HOLK9jKmvqsPpQPr45fMa5LzH7HG55c4vbcS+uOOy2Hcwc8kop1Nmb1yRjqbXh\nyOngPkiIyJyCuqFqBK3VNScl9zx+NHNHk14bTLi/syEDx4v8N/UEY9rSQ/j6QB4SnxvfrPMQkXGZ\n5sr9X3df3SrvEyjYR72yzudIWADYdby+679j8BQAvLTiMNYcygfgvri3r5u1wUjM1nrncEoEorbL\nNOE+elBPPPq9geEuBs42cBP0XFWt8/GPZ2kfElXWOizYmYXffbo/ZGVw3H5o6ocDERmfacIdAJ67\nc3i4i9BoN7/m3sZfWF7jtn3kdJnPXjMZBeWIn7oK05ameN1MdoY7s52ozTJVuBtNTkkViiv8X+nP\n3JSJie9uw82vb8L0NWmo0/u+K6Uw4d9bAQCL9uagLsj+88EqLLNg/6lzjXpNtdWG+Kmr8MFWrldL\nFAkY7mF0k8dVu6f/HTwNQOtSOXfLCaxP07pl/v1/7lPpOy7QC8sseHF5qnMk7bGC8iaVa/xbW/DT\n93c26jVLD+QCAP6z/WST3pOIQovhHkGqrQ1PMOYYtOrZPdNx3f7cslR8tCvbuf/XHyU2qRxlev98\nR9PPQZebv/48+3VqUOfOLCyHNUwLpRC1JQz3CHL5C9806XV2pVBaVet3wFVTbThaCABYdSgfSinM\n2pSJgrKGB1adKbNg4a4sn88Vlddg/Ftb8cLy4D4IiKjpTBfuJ/45MdxFaFHLk/O89r2zPgPXvLy2\n0ec6fb4aH+/KCnicAMgorMDr36bj95/ux58XH8RLHgO0XH3s8u3BVZlF6y2052T4JlILlqXWhqlL\nUnC2oibwwa2g1mZHTklVuItBBmK6cI+KMm8Xkcc+248nPk/22v/htsDt3OerrCiptGLt4TNI1Bcm\n+fVHiXhh+WHklFT5nL7BdZdjIrMqqw1fJeViwc4sWGptWJWSH3T5HX8zjmkZkrLP4YTHlA2R4n8H\nT+PzfTmYvuZog8c5PrBa2qsrj+Cm1zahOEI+bCjymS7c2yJrEDNIjnh5HUa+sg6TFybhnjm7YK2z\no6xaC6abXtuEN9ame71Gwdlh3hn0rvPjTF+dhsc+2++cBbP+db7p67k4A+ru2TsxzmPKhuaw2RU+\n2pnlXB0rFOx+5iw6dbYKOzOLcfVLa7F0f27I3s8fx8C40urW+TABtN9nBQfCGRbDvY0a9twat2mN\nZ2067nX1bqnVPjT8DYbaqY+49dcrp8pah0O5pc6VqFzPkpQdfFfLlSmnET91FXLP+W+WKCqvwROf\nH8CLKw5jVpCzbzbk6BmtTmV+wvTm1zfh5x/uAQBsOeZ7PeCnvkzGa9/UX/lvTi9E/NRVOB3kugDh\n9sxXKbjyxW/DXQxqIlOGe8J3Lgx3EQyh1uYe5mtSz7gF/LsbMgBoSxHmnvMOpIxCrUnlheUeE6Tp\nf075ZD9+OHM7Lnv+GxSUWXDPnF3OY+6e7d3V0mZXeOzT/fhiX/0UDCeLK/H4ZwcAAGn55X6vXH/x\n4W6s1JuIyizNv9p0dOnceTzwapGOX1mZpdbtm83S/Xl4f3P9Mo6L9mr1Cqb3UUv7v7m7MGVhUoPH\nLGmFbyTUckwZ7q22vp/J/P7T/c4g9fTkF773+2OzK2x1uaK9/p8b/LYXv7g8FStTTmPSu9uw6lA+\n/rrkEGrqbLDU2vCDNzY7j5u8MBHX/H0tvj7gHTqBJlt7flkq4qeuci50vvFoAWZvPo4qa/M/CFYc\nPI0qax1+/sFu3PHONq/nPevtb+RwYZkFCa+uQ2Zh08YnNMaekyX45vAZHDh1DiWVVtz2761+F4Fv\nxRUzKYRMGe5PTxiG7p1iMXpQj3AXxXBWHfJ9g9TRRBOs2ZuDbxr5aFc2Hv/sgLMpBAAufe4bXPa8\ne9dQR8b86YuD2Hm8GCNfWYdVKflYnpzntspVlI/0XLhb68Gz+8RZZBSU41cLEvGvb45i+AvuzQ6L\nE3Pw/LL6rppVVhsW7Ah8w3rJ/jyk5tVftbsGYsKr63HqbH2T0uwtJ9xuJJ8pteDVlUew+lA+iius\nPqeZDkb22Up8uS/HrQwrU043OK7gJ+/vxG8+TkR6QTnm+lks3ozZvnBXltvvyoxMGe43DOmF5Bdu\nxeeTxyBrxiS8ee814S5Sm5JRWIE31h5r0ff4+Qd7UFJp9dmDyDXbrXV2tzbuBTuykOjR3u8axH/5\nKsX5QeDwksuI4KNnynD721sDlm/Uq+vdtl2ncziYcx7/N2+3c/vPiw/iw+0nkaSvBOZrNolg1u/9\n8awdeGZJinN7c3oRHv/sAP69vuG/C8ciMza78nmV/tD8vZi31XfwA8ALy1OxSR8TkZRdgrtn74z4\ngWrPLz/s9rtqDck557E5vbDV3s+U4e7p7lH9cHL6RPw6AmaNpJa3PFmbtkEphWHPrcENMzY6n0sv\nKPdaRKUxUya88e0xt28YvuSdr0aJx2Rvnl8mLC6jkR1B6OjFq5TCf7afxLSlWvhYam2o0Y/J83Hv\nAwDOVVrdZh0F4CzD4sRcxE9dhaJy381ijjxfnJSLP33h3dV2e2Yx/rnaf5fQj3dl45EF+wAAU5cc\nQlL2OWSdbfqaBHU2O9YePmO65qAfz9qBh/+7r9Xer02EO6B1w/vbxMvDXQxqBcUVNUg/U46B01b7\nfN7zqtIxz35plf9uhuWWWny47QROFgful3+jy4eJw/q0QvfmDZewd3S3XK+PMLbZFV5ZeQSL9mrN\nBq7NUw/N3+t23vNVVlRbbbj2lXXOfY5QdJzX0eafkuv7Rq7rTeplyadhqdUmgQuG6/oDDUnLL2uw\nt5OD3a4w5Nk1mLwwCevTtKvcjILyFu9hVG21BRwkVmapRVZxJXLPVQW92H04mWYlpmBERwmGXdQF\nxwoic+AMhc5tQTSdODhGzL7fwH2Cq15qeARwRYAeOo5J4BxcL+QdIVypX81/mVh/w9hfqP1qwT7s\nPF4MS60dg+M6ez1/+nw1/vKVe7NDsHm0zs80FoVlFvS+oINz+6216XjXT7dTz4tux43mrBmTGnzv\nuS6zijq66jpmQA302uaYvDAR2zKKne9xvsqK7p3auR1z9/s7nT3Epnx/MKbecVnA8yql8GViDiZd\nfUnoCx1Am7lyd+jUrv7zbEjvLmEsCUWKipo6ZBZWuAVLYzW2e6NrN9T9p/y/NjWv1Gtf/NRV2Hi0\n0HmT27OnkFJwa4qq36+959EzDa+v+4dFvntGTfboOukZ7EopZ/g5VFnr3D6g/I2JcExS96bLYLqM\nggqfNz2Tc87jx7N2OMdPNEedTbsn4xgktuloIb7cl4MRL6/DioPuN6Nd67Yj03u1tWqrzat+idnn\n8Nclh/DCstafT6lNXbkDwOwHRmLMdO0f/vqnvg8AmL/9JF5eeaShl5HJPfF547p6enJdLD0Y1bU2\nfLbnFP729aEGj3u+CZOsDfqb7+aoxOxzXgHdGMk55/HbhYmY+2CCz+fvfG+783H22UpcenFX/OLD\nPTjg8uF167+3ImvGJGSfrcSOzLO4bmAPjH+rfpSy69oE/noNvbg8FQdzS/HQf/biir4X4K+3Xwa7\n0kbT9u7aAeWWWszadBxP3zoMNXV2dG4X7Rwd7WnKJ/udU2kDcN47AIA/6h9yfxg3BPdfN8DtdY7R\n23a7wqJ9p3D9wJ546stkpOSWYsfUcejSPgbrjxQ4l7wscukO+9meU/jRiEvQpX3Lxq+E66ZFQkKC\nSkxs2pS0zeVoT3T9mtdQG+OPR1yCZfpNut5d23utlkTUlowe1AP9LuyEr5IaHuSUNWOSz/9XNw3t\nheSc8yi31OGpCcPw1rrgelbNeWAUUvNKseN4sdsHxthL47A5XRtTsf6p7zs/LJ64ZSje2ZCBqXdc\nhinfH+x2rmDvKfjTLjoKK/5wIxbuysanQdx3uGloL7e1la8f2ANf/HZMk95bRJKUUr4/YV20uSt3\nf24Y3NPvaMRHbhyIZcmnseyxGzGif3f8efHBgP+wicxq94kSAIFn9kz201TlGnLBBjsATPnE97cO\nR7ADcPsW8I4+wnpVSj5KKq1Yn1aAjU+Pxa8WNL/HitVmx+1vew9Y88fz3klrzIza5trcAWDeg6Pw\np/HD3PZNu0PrSfOX2y517vvjuCH47c2DcE3/7siaMQkj+ncHALzBfvNEATkWgQ+3Q3mlmLf1BE4U\nVWL25uPYeLT1+po7BBpB3RLaZLOMP1nFlfhOz07IKamG1WZv8IZral4pzpRa8HVyHn54dR8s3Z8X\n9GIZYwb1xK4TgecsISLzamrvn2CbZdrklbs/8b06Q0QwoGengD1pruzbDeOHX4RZPx+J26/sg3kP\nJSCua3u3Y7p3ivX52kWTR2PMoJ4hKzcRkSeGewjte3Y87h3Vz7n9/i9GYtszP8CCR77r/LDY8LTW\nQ2fR5NGNPr/ruYmIGsIbqiE2/adX4blJw1FYbsHQi7oCAPr36IRr+nVH1tlKDI6r/0aw6DejkXOu\nCs98lYKPfnUdfjl/L343djBm69PE9u3e0W1OkX/85Cos9riRe+D5CbiwszbY4mxFDVYfysfzy72X\nwPvjLUOdU/gSkfkx3EMsJjoK3TpFoZtHk8yFnds5Q9hhzOCeGIOe+FlCfwD1bXACoGNsNB4Y/R3M\n3nIcIsBPru2LdjFReOb2S7H+SAEG9OiEx8cNcTtnzy7t8eCYeFTX2lBttTsnjPpqyhgkxPfAj67p\ng+IKK0YP6ukc0PLT2TvdupURkTnwhqqJ/XrBPmw4Whjwxk1pVS1ue3srzpRZ8Ma91+DPiw/i5mFx\nbvOxn5w+EaNeXe81IdZfbrsUafllWJmSj+vie2BvVn0Xr7kPjsJvGxg0kzVjEg7lluKHM7f7PYbI\nrFr6hmpQV+4icjuAdwBEA/hQKTXD43nRn58IoArAw0qp/Y0uNYXUnAdHocoaeIh2t06x2DF1HNLy\ny3Bl3264x6Vtf0dmMQb06AQRwf7nJ2DrsSK8ue4YLFYb6ux2PPaDIai12fG9Ib1wb0J/pJ8px7oj\nBbAphduuuNjrvZ6aMAy/vCEe3Tpq32yu6tcN3xvSC9szi/HyXVfgheWH8eT4oXj4hnjn3B5jX9+E\nrLOBJ53yhT2TKBJ1iG35250Br9xFJBrAMQATAOQC2AfgfqXUEZdjJgL4A7Rwvx7AO0qp6xs6L6/c\n24YPt51ASaUVllo7qmvrMP2nVzfpPJmF5Th93gKbXeGRBfsw7rLeeP8XI5F1thLxPTujuKIGndvF\n4MLO7VBUXgOrzY6+3TsCcB+N+Oa91+DuUf1grbPjZHGl1wRjz026HLM3H8eU7w/GP1an4aUfDnfO\n535l3wucC3J88+RNyCquxJRPtGuY/j064slbhuHpxQe9yn7tgO7IPluFkkor/nX3Vbj04gswZWES\nxg/vjU92a6MbRV+EfP7DCejRuT26dYzFgVPn8NSXB/HwDfFNXsCDIlNstCDjHxOb9Npgr9yDCfcx\nAF5SSt2mb08DAKXUdJdj5gLYrJRapG+nAxirlPK9rA8Y7tR0qw/lY9xlvdEhNjro19jsCqXVtejh\ncd8jq7gSF13QAdW1NgjgvIdhsytsPFqI8Zf3RqXV5pyfxFJrQ5mlFr27arMjVlttUFBuE9IB2iRa\n/uYzcTh1tgo3v74Jd1x5MWY/MCpgHZYn5+GWyy9Cl/YxsNkV/rk6DY/eNBAXdIhFZ5d5SnJKqlBp\nrcOeEyUoq67Fm+uOYVCvzjhVUoWZPx+J7ZlFSMktRUquNinZT67ti63HitCtYyweuTEeB3LO495R\n/TFny3H8+dZL8VVSDj7apS1g8siN8dh0tBBLf38jfvNxIqJFsDerBNPuuAzT1xxF+5go9OrS3tkR\n4IpLtLlf6ux2fDe+B9aknsGX+3KQmH0Ob/3sGqSfKcfcrSdw59V9nGvgAto6yBd2bod1RwowelAP\nVFttOJhbinfuG4HP9+Zg14mzmDD8Ir8zWAJA1/YxmPvgKOdC5s3Rp1sH5JdaAh8YpCG9uzjntmqs\nUIb7PQBuV0o9qm8/COB6pdTjLsesBDBDKbVd394A4K9KKb/pzXAnCq9ySy3OV9Wif49OITmf5wda\nTZ0NsVFRiIry/yGnlEJKbimu0Ud/A1qvr24dYxETHYWzFTXo2aW939cHo9xSizWpZ3DPyH6wK4WY\n6ChUW23o2M774uBYQTn2nizBj0Zcggs6xKLWZodA6yhhqbXhzbXpeHzcUHTrGIu889UQaONZHB/u\nmYUVOFlcibGXxqGmzo4KSx0u7lY/TbJSCjM3ZuKuEX0xoGfTfu8RGe4iMhnAZAAYMGDAqOxs9+XM\niIioYaEcoZoHoL/Ldj99X2OPgVJqnlIqQSmVEBcXF8RbExFRUwQT7vsADBWRgSLSDsB9AFZ4HLMC\nwEOiGQ2gtKH2diIialkBu0IqpepE5HEA30LrCjlfKXVYRKboz88BsBpaT5lMaF0hH2m5IhMRUSBB\n9XNXSq30abpwAAAES0lEQVSGFuCu++a4PFYAHgtt0YiIqKk4cRgRkQkx3ImITIjhTkRkQgx3IiIT\nCtuskCJSBKCpo5h6ASgOeJS5sM5tA+vcNjSnzt9RSgUcKBS2cG8OEUkMZoSWmbDObQPr3Da0Rp3Z\nLENEZEIMdyIiEzJquM8LdwHCgHVuG1jntqHF62zINnciImqYUa/ciYioAYYLdxG5XUTSRSRTRKaG\nuzxNJSL9RWSTiBwRkcMi8oS+v4eIrBORDP3PC11eM02vd7qI3Oayf5SIHNKfe1cCLQEUZiISLSIH\n9HUATF9nEekuIl+JyFERSRORMW2gzn/S/12nisgiEelgtjqLyHwRKRSRVJd9IaujiLQXkS/0/XtE\nJL5RBVRKGeYH2qyUxwEMAtAOwEEAw8NdribWpQ+AkfrjrtDWqR0O4DUAU/X9UwH8S388XK9vewAD\n9d9DtP7cXgCjAQiANQDuCHf9AtT9KQCfAVipb5u6zgA+AvCo/rgdgO5mrjOAvgBOAuiob38J4GGz\n1RnAzQBGAkh12ReyOgL4PYA5+uP7AHzRqPKF+xfUyF/mGADfumxPAzAt3OUKUd2WQ1uEPB1AH31f\nHwDpvuoKbQrmMfoxR1323w9gbrjr00A9+wHYAGCcS7ibts4AuulBJx77zVznvgByAPSANvPsSgC3\nmrHOAOI9wj1kdXQcoz+OgTboSYItm9GaZRz/aBxy9X2Gpn/duhbAHgAXqfqFTs4AuEh/7K/uffXH\nnvsj1dsAngFgd9ln5joPBFAE4L96U9SHItIZJq6zUioPwBsATgHIh7Z4z1qYuM4uQllH52uUUnUA\nSgH0DLYgRgt30xGRLgCWAHhSKVXm+pzSPrJN051JRO4EUKiUSvJ3jNnqDO2KaySA2UqpawFUQvu6\n7mS2OuvtzHdB+2C7BEBnEXnA9Riz1dmXcNfRaOEe1FqtRiEisdCC/VOl1FJ9d4GI9NGf7wOgUN/v\nr+55+mPP/ZHoRgA/EpEsAJ8DGCcin8Dcdc4FkKuU2qNvfwUt7M1c5/EATiqlipRStQCWArgB5q6z\nQyjr6HyNiMRAa+I7G2xBjBbuwaznagj6HfH/AEhTSr3l8tQKAL/UH/8SWlu8Y/99+h30gQCGAtir\nfwUsE5HR+jkfcnlNRFFKTVNK9VNKxUP7u9uolHoA5q7zGQA5InKpvusWAEdg4jpDa44ZLSKd9LLe\nAiAN5q6zQyjr6Hque6D9fwn+m0C4b0g04QbGRGg9S44DeDbc5WlGPb4H7StbCoBk/WcitDa1DQAy\nAKwH0MPlNc/q9U6HS68BAAkAUvXnZqIRN13CWP+xqL+hauo6AxgBIFH/u14G4MI2UOe/Aziql3ch\ntF4ipqozgEXQ7inUQvuG9utQ1hFABwCLoa1NvRfAoMaUjyNUiYhMyGjNMkREFASGOxGRCTHciYhM\niOFORGRCDHciIhNiuBMRmRDDnYjIhBjuREQm9P/nsQ2W5tqo4AAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f7fad9eef28>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(list(range(len(train_loss_list))), train_loss_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.legend.Legend at 0x7f7fad22bf60>"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXkAAAD8CAYAAACSCdTiAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3XtwnHd97/H3dy/S6i5ZkhNfEmwg4MSJnYtzIZcmwaWx\nA4dQ5rRc2qHhlJowCU2nJSfuwIF2aGfShjIkNCQ1xZMmMCUFCqStaXzSJOUcaMCOjx1sJ9hOCFi+\nSKu7tJL2+jt/PI/k9VqyZWulvTyf18zOc/vt7teP1x8/++zze37mnENERKpTqNQFiIjI/FHIi4hU\nMYW8iEgVU8iLiFQxhbyISBVTyIuIVDGFvIhIFVPIi4hUMYW8iEgVi5TqjTs6OtyKFStK9fYiIhXp\npZde6nXOdc62fclCfsWKFezcubNUby8iUpHM7Jdn0/6Mp2vMbKuZ9ZjZ3hm2m5k9bGaHzOxlM7vy\nbAoQEZH5M5tz8o8DG06zfSNwkf/YBDw697JERKQYzhjyzrkfAv2naXIH8ITzvAi0mtmSYhUoIiLn\nrhhX1ywDDuctd/nrRESkxBb0Ekoz22RmO81sZzweX8i3FhEJpGKE/BHggrzl5f66Uzjntjjn1jnn\n1nV2zvoKIBEROUfFCPmngY/4V9lcBww5544V4XVFRGSOznidvJn9I3AL0GFmXcDngCiAc+4xYBtw\nO3AIGAM+Ol/FiogUm3OOTM6RzfnTrCOTy5EpWM7mHGl/Pp3122dzpCenhe3ytmWy3mtNrrt6RRs3\nXbQwZzPOGPLOuQ+dYbsD7i5aRSJSNZxzTKRzjKUyjKWy/iPDuD+fyJvPbzPub0tlcuScF6hZB7nc\n5Lwjl3Petrz1J9p6270pJwLcD+/J5Uw2R64Ew1x/4pa3lE/Ii0hx5XKO0VSGkYkMIxNpkukcqWyO\nVCZHMpP1p94jlclNLacyOVLZ7MnLk/NZ7whyMugy2ZODbnJbNndqCGazk228UJ5kll+1nbIuf/Pk\nevPXZnKO8VSGsXQWdxYhGg4Z9TVh/xGhNhIiZEY4ZIRCRti8NiEzavxthetPtLWpdZGQEQ7709Dk\nNDS1HA2fvBwJn2gXCYWIhHJEXJZIOEQ4HCUSCRONhKeeGwmFiISNaDjkrfOXI+G8+ampYSfv3Hml\nkJfAcM6RyubI5cDhcA5c3jYHJwLJeW3w1zlOBKADcs4xOuEF9fBE2puOp0+an9w2XLA8msycVfDl\nC4eMurCjKZKhMZylMZKhKZyhLpQlHApjoTBYiFAojAuFCYVCEPbWh8ybWiiEhaOYedstHMYsQjgS\nImST+yNvv3HquhNrT6zP3x4qCOv6qNEYTtMYztAUylAfTlFvKeotTczSxEhSS4pINollxiE9AZlx\nyKbB5SCXBZfNm3cFy5PzOW9auC2ThWQacmnvNbN587kMZFN582lveXI+59dQKBTxHhb258Mn1s24\n7K9b8wG4+vfP7UNwlhTyUvaSmSyJZJZE0gvV0WTGm/eno/660cn5lDdNJPPW+23T2fn/bh6yHOfV\nZjg/lmZxTZq3RZMsqk2xqHGCtlCSltAYTTZBI2PUuXGiLkXEpYjkkkRyKcK5JKFsilAuSSjrPSwz\nARl/6rJexmb8RzGZ95/E9A+bYX6adS53IqjT415ozqmukB+m4RPzFoJQaIZtdvJyKALhCISiEK6B\nSA2EGiAc9R6hgunUfMRrPzkP3n8cuUzeIzvDutMsh8Jz/7uaJYW8zIt0NsfweJqh8fRJAZxIZRj1\nAzuRt348OUF6YozMRIJccpRscgyXHoP0GNFckjpS1FmSGClqSREjTcxS1OFta7M0DSHvUR9KU29p\n6sxrW0uKmlCKmtgEkVwKZ2FyFvEeIX9q0RPzU9MoLm/Zhbx1OYvgLEzMJYnlEtRmE0SzCSLpUUKp\nESw16u2ECf8xk2gD1DRAtA4iMYjUetOaGETbTixPTfMfBdvCUf8I1j/CzWVPHNFOzedmOCouXO9O\nbn/Ksv/AzdwGg2gMInXen2/yz3jKtD6vXezEtmidty5c4wW5nDOFvEzLOcdIMjMV1MPjGW86kfZO\nS4x7pyGGxtOMjSVgrJfQeB81E/3UpgZoyg7SbsMsYphGG6eOFI0k6bQUMZLUk6RuMqQtRYTs9IWc\n5hPqLISLeAFhNXXYVFDUQbQlLzj8abTeC0WXg6z/FT2X9uanvsYXfE2fajfmzWcmv+JnvYCubYLG\ndqhd4c3XNvvT/Mc062oaTxwZiswjfcoCZDK44yNJekeSxEeT3rw/jY8k6RuZYHRkiFRikCY3MhXU\nHTbMIhumnWEutGHabZiO0AjtDNPI2MlvFPIeOYuQrGkjW9vsB3EjVlNPqKaeSG094doGLFoPNfX+\n0duZpnUnBbd3XnnhfsASqUQK+SqQzTn6RpMcGxynty/OcH8Po8MDjI/0kxodJDM+RG58GJLD1Lsx\nmhijycZoZJxLbYxmxmkJjdNo49S7MUI4qDn1fZyFydW1Q0MHocalWEMHNHRCfQc0TD5OLIdiLdQp\nhEVKSiFf5pKpJPHjxxiIdzHSd4yJwWOkh3uwRJzweC91qX6aMgO02xCrGKbWTvNLXBgyoRoy0SZc\nTSMWayFcdx6R+hasthlizSdOLcSaoa7tpNC2WCthnR8VqSgK+VKaGGbk2CF6u37O6PFDZAe6CI3F\niU70UZ/qpzk3SIsbYbk5lhc8NUWEkXArYzXtpJvPZ6xhDcnGxcRaz6ehtZOG5kWE61tOnBOOefOR\nSK3+0kUCRP/e51M2A8NdjPe8xkDXAca6XyM38AaxkcO0Jo/S7IZpApr85iOujgFrYSS8iN7YhRyv\nuxIaOok0n0dd2/k0LVpC23nLaGxbQk2shXYz2kv55xORsqeQn6v0BMRfId37OkNHDzLe8zo28AZ1\nicO0po4TJkcdUAekXZgu18HR8BJ+XncT6eY3Ee1YSdOSi1h84UUsPX8JF0YX7vpZEal+Cvmz5Rz0\nHmB03zOM7t/OovhPqXFJokAH0Oua6XKdHA+vJNFwE9mWN1HT8Waal17E+cvfworFTays0W4XkYWh\ntJmNsX6yh55n4Gf/Tu0vX6Ap1UMj0J1bwgvhdzK65Fpqz3sbi5ZdxPLzF7Oyo4HL66KlrlpERCE/\nrWwaunaSeOUZJl59lrbBvYRx1Lh6/m/uUn7Z+gFiq97FVWsv57eXNhMK6TJBESlPCvlJ/a+TPfgf\nDO99hvqjP6I2O0bMGa+6t/LtyG+TetPNvHntTdz4tiVsrNdRuohUhuCGfDYDB59hbP92sof+g6ax\nw4SBhOvg33Pv4Gj7O2he/etcv/rNbFrSrJ6VIlKRAhvyR1/4Kkv/z2acq+UnuUv4fzW/AW9+J5de\ndiW3X9RJi86pi0gVCGzIH/vFK7S7CF+/+T/5tYuXsf78Jh2ti0jVCWzIhxLH6bM2Pv7Oi0tdiojI\nvAnsjUhqJ+IMhheVugwRkXkV2JBvTPWSqOkodRkiIvMqsCHfmu0nVbe41GWIiMyrQIZ8LjlGM6Pk\nGs4rdSkiIvMqkCE/EO8CINSypMSViIjMr0CG/FDPYQBqW5eWuBIRkfkVyJAf6/OO5Bval5W4EhGR\n+RXIkE8OHAWg9bwLS1yJiMj8CmTI54aPk3Zh2jt1Tl5EqlsgQz6U6KbX2qiJBrbDr4gERCBDvnYi\nzpB6u4pIAAQy5BtTcfV2FZFACGTIt2b7Saq3q4gEQOBCPpuaoJURnHq7ikgABC7kB/yOUKHm80tc\niYjI/AtcyA/1eB2hatrUEUpEql/gQj7R5x3JN6q3q4gEQOBCPjlwDFBvVxEJhlmFvJltMLOfm9kh\nM9s8zfYWM/sXM9tjZvvM7KPFL7U4csPHyLgQixbr5mQiUv3OGPJmFgYeATYClwAfMrNLCprdDex3\nzq0FbgH+xsxqilxrUYQT3fRZK9GIeruKSPWbzZH8NcAh59zrzrkU8E3gjoI2DmgyMwMagX4gU9RK\ni8Tr7dpe6jJERBbEbEJ+GXA4b7nLX5fvb4GLgaPAz4B7nXO5olRYZF5vV4W8iARDsX54vQ3YDSwF\nLgf+1syaCxuZ2SYz22lmO+PxeJHe+uy0ZPtJ1qkjlIgEw2xC/ghwQd7ycn9dvo8C/+w8h4BfAKsK\nX8g5t8U5t845t66zs/Ncaz5nmdQEixjW2K4iEhizCfkdwEVmttL/MfWDwNMFbX4FrAcws/OAtwOv\nF7PQYujv8f5vUm9XEQmKM15i4pzLmNk9wDNAGNjqnNtnZnf52x8DPg88bmY/Awy43znXO491n5Oh\nnsMsBmrV21VEAmJW1xE657YB2wrWPZY3fxT4jeKWVnwJf2zXxg6FvIgEQ6B6vE6N7bpYvV1FJBgC\nFfJu+BhZZ7R1qreriARDoEI+nOim31qJRKOlLkVEZEEEKuRrNLariARMoEK+MdXLaM3CX58vIlIq\ngQr5lmw/qTqFvIgER2BCPp1OscgNqberiARKYEK+v6eLkDlCzUtKXYqIyIIJTMgPdns30qxt0+WT\nIhIcgQn5yd6uDR3LS1yJiMjCCUzIp6Z6u15whpYiItUjMCGfGz5Ozhlti3UkLyLBEZiQDye6GbAW\nwhH1dhWR4AhMyNeO96i3q4gETmBCviGt3q4iEjyBCflW9XYVkQAKRMinUmkWuUGy6u0qIgETiJDv\nix8hbI6weruKSMAEIuQHu38FqLeriARPIEJ+rFe9XUUkmAIR8snBY4B6u4pI8AQi5HPDfsh3Litx\nJSIiCysQIR9OdNNPM6FobalLERFZUIEIea+3a3upyxARWXCBCPmGdC+Jmo5SlyEisuACEfKt2X6S\ndYtLXYaIyIKr+pBPplK0u0FyDQp5EQmeqg/53p5jRCxHuFkdoUQkeKo+5Id6vN6uNW26pYGIBE/V\nh3yi9wgADR3qCCUiwVP1IZ8c9MZ2bVNvVxEJoKoPeef3dm1Rb1cRCaCqD/lQoptBmrBorNSliIgs\nuKoPeY3tKiJBVvUh35Dq09iuIhJYVR/yrbk+je0qIoFV1SE/kUrT4QbJaWxXEQmoWYW8mW0ws5+b\n2SEz2zxDm1vMbLeZ7TOz/yxumeemt+cYUcsS0tiuIhJQkTM1MLMw8AjwLqAL2GFmTzvn9ue1aQW+\nAmxwzv3KzMriRjGDPb9iORrbVUSCazZH8tcAh5xzrzvnUsA3gTsK2nwY+Gfn3K8AnHM9xS3z3Ki3\nq4gE3WxCfhlwOG+5y1+X721Am5m9YGYvmdlHpnshM9tkZjvNbGc8Hj+3is/CZG/X1k4N4C0iwVSs\nH14jwFXAu4HbgP9lZm8rbOSc2+KcW+ecW9fZOf9XvEz2dm1erJAXkWA64zl54AiQf75jub8uXxfQ\n55xLAAkz+yGwFjhQlCrPUSjRwzCNNEfrSlmGiEjJzOZIfgdwkZmtNLMa4IPA0wVtvg/caGYRM6sH\nrgVeKW6pZ0+9XUUk6M54JO+cy5jZPcAzQBjY6pzbZ2Z3+dsfc869Ymb/DrwM5IC/d87tnc/CZ6Mx\n1ctorTpCiUhwzeZ0Dc65bcC2gnWPFSw/CDxYvNLmrjXbR7xuZanLEBEpmart8TqWTNPOIFn1dhWR\nAKvakO/tOU6tZQirt6uIBFjVhvygP7areruKSJBVbcgn+ryOUA0dukZeRIKrakM+NeBdyt/aqVsa\niEhwVW3I50aOA9CosV1FJMCqNuRDiW5GqMdqGkpdiohIyVRtyMfGexgKt5e6DBGRkqrakG9I9ZKo\n6Sh1GSIiJVW1Id+a7SOpsV1FJOCqMuRHJ9J0Mkiu4fxSlyIiUlJVGfK98W5qLU2oRSEvIsFWlSE/\n2OMNZFXbqt6uIhJsVRnyY31dgMZ2FRGpypBPDfhju2rYPxEJuKoM+eyw19u1oV29XUUk2Koy5ENj\n3SSow2qbSl2KiEhJVWXIxzS2q4gIUKUh35DqZbRGHaFERKou5J1ztGb7Sam3q4hI9YW819t1gKx6\nu4qIVF/Ix3t7qbMU4WaFvIhI1YX8kMZ2FRGZUnUhP+r3dq3X2K4iItUX8ump3q66pYGISNWF/Ine\nrjqSFxGpupAPJboZJwbq7SoiUn0hH5voYVBju4qIAFUY8g2pPo3tKiLiq6qQ93q7amxXEZFJVRXy\nwxMZOhkg13BeqUsRESkLVRXy8d5eGixJqHlJqUsRESkLVRXyQ5Nju6q3q4gIUGUhn5ga21XXyIuI\nQJWF/OTYri2dCnkREaiykM8OHwOgXr1dRUSAWYa8mW0ws5+b2SEz23yadlebWcbM/nvxSpy9cKKb\nCWqgtrkUby8iUnbOGPJmFgYeATYClwAfMrNLZmj3V8D2Yhc5W7UTca+3q1mpShARKSuzOZK/Bjjk\nnHvdOZcCvgncMU27TwLfAXqKWN9ZaUj1qreriEie2YT8MuBw3nKXv26KmS0DfhN4tHilnZ3J3q6p\nusWlKkFEpOwU64fXLwH3O+dyp2tkZpvMbKeZ7YzH40V6a8/QeJpOBsmqt6uIyJTILNocAfJH4Fju\nr8u3DvimeefCO4DbzSzjnPtefiPn3BZgC8C6devcuRY9nZ6+ft5m4xrbVUQkz2xCfgdwkZmtxAv3\nDwIfzm/gnFs5OW9mjwP/Whjw822w2+/tumjZGVqKiATHGUPeOZcxs3uAZ4AwsNU5t8/M7vK3PzbP\nNc7KWL/35aJBIS8iMmU2R/I457YB2wrWTRvuzrk7517W2UsOeCHforFdRUSmVE2PV+f3do3pSF5E\nZErVhHxotJsUUYi1lroUEZGyUTUhr96uIiKnqpqQb0irt6uISKGqCHnnHG3ZPpIxje0qIpKvKkJ+\nYMzr7ZprVEcoEZF8VRHyPX0DNNuYeruKiBSoipAf7PkVADVtunxSRCRfVYT8WL837F9Du0JeRCRf\nVYR8Sr1dRUSmVRUhnx0+DkCtTteIiJykKkI+PHqcFBGoayt1KSIiZaUqQr52Is6QeruKiJyiKkJe\nY7uKiEyv4kM+l/N6u6bU21VE5BQVH/L9Yyk6bZBco8Z2FREpVPEh39M/SKslCDUvKXUpIiJlp+JD\nfijeBUBN29ISVyIiUn4qPuQTvf7YrurtKiJyiooP+RO9XS8scSUiIuWn4kN+srdrTatO14iIFKr4\nkA8nuskQhrpFpS5FRKTsVHzI1070MBReBKGK/6OIiBRdxSdjo3q7iojMqKJDPptztGb7ScYWl7oU\nEZGyVNEh35dI0mkD6u0qIjKDig75+MAwi2yUkMZ2FRGZVkWH/IneruoIJSIynYoO+dFeL+Qb1dtV\nRGRaFR3yyQFvAO9m9XYVEZlWpNQFzEXO7+0abdEdKEXKTTqdpquri4mJiVKXUpFisRjLly8nGo3O\n6XUqOuTDiW6yhAjX6zp5kXLT1dVFU1MTK1aswDQ051lxztHX10dXVxcrV66c02tV9Okab2xX9XYV\nKUcTExO0t7cr4M+BmdHe3l6Ub0EVnY6Nqbh6u4qUMQX8uSvWvqvYkM9kc+rtKiJyBhUb8n0Jje0q\nIjMbHBzkK1/5ylk/7/bbb2dwcHAeKiqNig35+MAIHTZMqEm9XUXkVDOFfCaTOe3ztm3bRmtr63yV\nteBmdXWNmW0AHgLCwN875x4o2P47wP2AASPAJ5xze4pc60kG4t6IULXq7SpS9v78X/ax/+hwUV/z\nkqXNfO6/rZ5x++bNm3nttde4/PLLiUajxGIx2traePXVVzlw4ADve9/7OHz4MBMTE9x7771s2rQJ\ngBUrVrBz505GR0fZuHEjN954Iz/+8Y9ZtmwZ3//+96mrq5v2/b761a+yZcsWUqkUb33rW3nyySep\nr6+nu7ubu+66i9dffx2ARx99lOuvv54nnniCL3zhC5gZa9as4cknnyzq/pl0xiN5MwsDjwAbgUuA\nD5nZJQXNfgHc7Jy7DPg8sKXYhRZK9B0GoLFDIS8ip3rggQd4y1vewu7du3nwwQfZtWsXDz30EAcO\nHABg69atvPTSS+zcuZOHH36Yvr6+U17j4MGD3H333ezbt4/W1la+853vzPh+73//+9mxYwd79uzh\n4osv5mtf+xoAf/iHf8jNN9/Mnj172LVrF6tXr2bfvn38xV/8Bc899xx79uzhoYcemp+dwOyO5K8B\nDjnnXgcws28CdwD7Jxs4536c1/5FYHkxi5xOcuAYAE2dF8z3W4nIHJ3uiHuhXHPNNSddc/7www/z\n3e9+F4DDhw9z8OBB2tvbT3rOypUrufzyywG46qqreOONN2Z8/b179/KZz3yGwcFBRkdHue222wB4\n7rnneOKJJwAIh8O0tLTwxBNP8Fu/9Vt0dHhXBy5aNH8j280m5JcBh/OWu4BrT9P+94EfTLfBzDYB\nmwAuvHButyLIDXshH1FvVxGZhYaGhqn5F154gWeffZb/+q//or6+nltuuWXaa9Jra2un5sPhMOPj\n4zO+/p133sn3vvc91q5dy+OPP84LL7xQ1PrPVVF/eDWzW/FC/v7ptjvntjjn1jnn1nV2ds7pvSJ+\nb1ca5vY6IlKdmpqaGBkZmXbb0NAQbW1t1NfX8+qrr/Liiy/O+f1GRkZYsmQJ6XSab3zjG1Pr169f\nz6OPPgpANptlaGiId77znXzrW9+aOkXU398/5/efyWxC/giQf05kub/uJGa2Bvh74A7n3Kknt4qs\nZiLOSLgNQuH5fisRqUDt7e3ccMMNXHrppdx3330nbduwYQOZTIaLL76YzZs3c9111835/T7/+c9z\n7bXXcsMNN7Bq1aqp9Q899BDPP/88l112GVdddRX79+9n9erVfPrTn+bmm29m7dq1/PEf//Gc338m\n5pw7fQOzCHAAWI8X7juADzvn9uW1uRB4DvhIwfn5Ga1bt87t3LnzXOvmR39+CytiCZbdv+OcX0NE\n5s8rr7zCxRdfXOoyKtp0+9DMXnLOrZvta5zxnLxzLmNm9wDP4F1CudU5t8/M7vK3PwZ8FmgHvuJ3\nxc2cTRFnKz3V21U/uoqInM6srpN3zm0DthWseyxv/mPAx4pb2sx6R5MstkGGGuft/xERkWndfffd\n/OhHPzpp3b333stHP/rRElV0ehV5q+GewQSXMcyIeruKyAJ75JFHSl3CWanI2xoMxo8QMqferiIi\nZ1CRIZ/o88Z2bVBvVxGR06rIkE/5Y7s2dWpsVxGR06nIkM/6vV3DzTonLyLTO9dbDQN86UtfYmxs\nrMgVlUZFhnw40U0OgwYNGCIi01PIeyoy5GvH44yEWyFckRcHicgCyL/V8H333ceDDz7I1VdfzZo1\na/jc5z4HQCKR4N3vfjdr167l0ksv5amnnuLhhx/m6NGj3Hrrrdx6660zvv4nPvEJ1q1bx+rVq6de\nD2DHjh1cf/31rF27lmuuuYaRkRGy2Syf+tSnuPTSS1mzZg1f/vKX5/3PP6kiU7IxFScR66Cl1IWI\nyOz8YDMc/1lxX/P8y2DjAzNufuCBB9i7dy+7d+9m+/btfPvb3+anP/0pzjne+9738sMf/pB4PM7S\npUv5t3/7N8C7p01LSwtf/OIXef7556fuEjmdv/zLv2TRokVks1nWr1/Pyy+/zKpVq/jABz7AU089\nxdVXX83w8DB1dXVs2bKFN954g927dxOJROb1XjWFKi7kU5kcrbl+knW6skZEZmf79u1s376dK664\nAoDR0VEOHjzITTfdxJ/8yZ9w//338573vIebbrpp1q/5T//0T2zZsoVMJsOxY8fYv38/ZsaSJUu4\n+uqrAWhubgbg2Wef5a677iIS8SJ3Pm8tXKjiQj7u93Ydbbiy1KWIyGyd5oh7ITjn+NM//VM+/vGP\nn7Jt165dbNu2jc985jOsX7+ez372s2d8vV/84hd84QtfYMeOHbS1tXHnnXdOe6viclBx5+R7BhN0\nMERI95EXkdPIv9XwbbfdxtatWxkdHQXgyJEj9PT0cPToUerr6/nd3/1d7rvvPnbt2nXKc6czPDxM\nQ0MDLS0tdHd384MfeENovP3tb+fYsWPs2OHdOHFkZIRMJsO73vUu/u7v/m5qfFmdrjmNwfhRwuao\nbV1a6lJEpIzl32p448aNfPjDH+Yd73gHAI2NjXz961/n0KFD3HfffYRCIaLR6NR93zdt2sSGDRtY\nunQpzz///CmvvXbtWq644gpWrVrFBRdcwA033ABATU0NTz31FJ/85CcZHx+nrq6OZ599lo997GMc\nOHCANWvWEI1G+YM/+APuueeeBdkPZ7zV8Hw511sNH973Yy741kYSv/kEDWvvmIfKRKQYdKvhuSvG\nrYYr7nTNBRFvxPeG9nkfRlZEpOJV3Oka6lph1XugRSEvIvPv2muvJZlMnrTuySef5LLLLitRRWen\n8kL+wuu8h4jIAvjJT35S6hLmpOJO14iIyOwp5EVk3pTqwo5qUKx9p5AXkXkRi8Xo6+tT0J8D5xx9\nfX3EYrE5v1blnZMXkYqwfPlyurq6iMfjpS6lIsViMZYvn/sFJgp5EZkX0WiUlStXlrqMwNPpGhGR\nKqaQFxGpYgp5EZEqVrJ715hZHPjlOT69A+gtYjkLQTUvjEqrudLqBdW8UGaq+U3Ouc7ZvkjJQn4u\nzGzn2dygpxyo5oVRaTVXWr2gmhdKsWrW6RoRkSqmkBcRqWKVGvJbSl3AOVDNC6PSaq60ekE1L5Si\n1FyR5+RFRGR2KvVIXkREZqGsQ97MNpjZz83skJltnma7mdnD/vaXzezKUtSZV88FZva8me03s31m\ndu80bW4xsyEz2+0/zjw0/DwzszfM7Gd+PaeMyViG+/nteftvt5kNm9kfFbQp6X42s61m1mNme/PW\nLTKz/21mB/1p2wzPPe3nfoFrftDMXvX/3r9rZq0zPPe0n6EFrvnPzOxI3t/97TM8t5z281N59b5h\nZrtneO7Z72fnXFk+gDDwGvBmoAbYA1xS0OZ24AeAAdcBPylxzUuAK/35JuDANDXfAvxrqfdvQU1v\nAB2n2V5W+3maz8lxvGuHy2Y/A78GXAnszVv318Bmf34z8Fcz/HlO+7lf4Jp/A4j48381Xc2z+Qwt\ncM1/BnxqFp+bstnPBdv/BvhssfZzOR/JXwMccs697pxLAd8ECkfuvgN4wnleBFrNbMlCFzrJOXfM\nObfLnx9h/0lSAAAC+klEQVQBXgGWlaqeIiqr/VxgPfCac+5cO9bNC+fcD4H+gtV3AP/gz/8D8L5p\nnjqbz/28mK5m59x251zGX3wRKKtxN2fYz7NRVvt5kpkZ8NvAPxbr/co55JcBh/OWuzg1MGfTpiTM\nbAVwBTDd2GHX+19/f2Bmqxe0sOk54Fkze8nMNk2zvWz3M/BBZv4HUW77+Tzn3DF//jhw3jRtynlf\n/w+8b3TTOdNnaKF90v+73zrDabFy3c83Ad3OuYMzbD/r/VzOIV+xzKwR+A7wR8654YLNu4ALnXNr\ngC8D31vo+qZxo3PucmAjcLeZ/VqpC5oNM6sB3gt8a5rN5bifpzjvu3fFXNpmZp8GMsA3ZmhSTp+h\nR/FOw1wOHMM7/VEpPsTpj+LPej+Xc8gfAS7IW17urzvbNgvKzKJ4Af8N59w/F253zg0750b9+W1A\n1Mw6FrjMwpqO+NMe4Lt4X2Xzld1+9m0Edjnnugs3lON+BronT3P5055p2pTdvjazO4H3AL/j/+d0\nill8hhaMc67bOZd1zuWAr85QSznu5wjwfuCpmdqcy34u55DfAVxkZiv9I7YPAk8XtHka+Ih/9cd1\nwFDe1+EF559P+xrwinPuizO0Od9vh5ldg/d30LdwVZ5ST4OZNU3O4/3QtregWVnt5zwzHvWU2372\nPQ38nj//e8D3p2kzm8/9gjGzDcD/BN7rnBuboc1sPkMLpuD3ot+coZay2s++Xwdedc51TbfxnPfz\nQvyaPIdfoW/Hu0LlNeDT/rq7gLv8eQMe8bf/DFhX4npvxPsK/jKw23/cXlDzPcA+vF/zXwSuL3HN\nb/Zr2ePXVfb72a+pAS+0W/LWlc1+xvvP5xiQxjvf+/tAO/AfwEHgWWCR33YpsC3vuad87ktY8yG8\nc9eTn+fHCmue6TNUwpqf9D+nL+MF95Jy38/++scnP795bee8n9XjVUSkipXz6RoREZkjhbyISBVT\nyIuIVDGFvIhIFVPIi4hUMYW8iEgVU8iLiFQxhbyISBX7/yxGajdSVFk5AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f7fb953acc0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "x1 = list(range(len(train_acc_list)))\n",
    "plt.plot(x1, train_acc_list, label=\"train_acc\")\n",
    "plt.plot(x1, test_acc_list, label=\"test_acc\")\n",
    "plt.legend()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}