{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Introduction to Deep and Transfer Learning\n",
    "\n",
    "### by Andreas Damianou,  6 March 2019\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "         **NOTE !!! This notebook will download around 6mb of files for the demos to work**"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Some background on the transfer learning discussion is contained in the following blog post:   \n",
    "    https://medium.com/apache-mxnet/xfer-an-open-source-library-for-neural-network-transfer-learning-cd5eac4accf0"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### First import necessary helper libraries for this tutorial and define helper functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from pylab import *\n",
    "%matplotlib inline\n",
    "\n",
    "import mxnet as mx\n",
    "# Install as follows: \n",
    "# $pip install mxnet\n",
    "\n",
    "import xfer\n",
    "# You can install it with:\n",
    "#    $pip install xfer-ml\n",
    "\n",
    "# Or clone and install from souce:\n",
    "#    https://github.com/amzn/xfer.git"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define helper function for later\n",
    "\n",
    "def plot_point(func, t, xmin, xmax):\n",
    "    N = 10\n",
    "    x = np.zeros((N,1))+t\n",
    "    x = x[:,0]\n",
    "    y = np.linspace(0,func(t),N)\n",
    "    plt.plot(x,y, 'k--', linewidth=1)\n",
    "    \n",
    "    x = np.linspace(xmin, t, N)\n",
    "    y = np.zeros((N,1))+func(t)\n",
    "    plt.plot(x,y, 'k--', linewidth=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Define the activation function and its derivative"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Sigmoid activation function\n",
    "def activation(x):\n",
    "    return 1/(1+np.exp(-x))\n",
    "\n",
    "# Derivative of activation wrt input x\n",
    "def activation_derivative(x):\n",
    "    # the activation of the sigmoid function conveniently can be written in terms of its outputs.\n",
    "    f = activation(x)\n",
    "    return f*(1-f)\n",
    "\n",
    "# Derivative of activation wrt input x but expressed when activation(x) is given as argument instead of x\n",
    "def activation_derivative_f(f):\n",
    "    return f*(1-f)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Plots to explore the activation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "First, plot the activation for input domain between -10 and 10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Text(0,0.5,'activation(x)')"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEKCAYAAAD9xUlFAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4zLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvIxREBQAAIABJREFUeJzt3XmcVOWd7/HPrxe6WbpZm30XVHBF\nG9wSNaIEMVGTccEsJiEZMs41MTfjTUxyX06iuXcycSYzyY1ZjONgTNyiJhJDAkrcjcgqCoi00NDN\n0iyNNNB009X1u3/U6bZsq7uL5fSpqv6+X6+izvKcql+dPtSvnuec8zzm7oiIiADkRR2AiIhkDiUF\nERFppaQgIiKtlBRERKSVkoKIiLRSUhARkVZKCiIi0kpJQUREWikpiIhIq4KoAzhSgwYN8rFjx0Yd\nhohIVlm+fPludy/rrFzWJYWxY8eybNmyqMMQEckqZrY5nXJqPhIRkVZKCiIi0kpJQUREWikpiIhI\nKyUFERFpFVpSMLP7zGynmb3Zznozs5+YWYWZrTazs8KKRURE0hNmTWEeMLOD9ZcDE4PHXODnIcYi\nIiJpCO0+BXd/wczGdlDkKuDXnhgP9FUz62dmw9x9e1gxiUhuiMedxlichqZmGmLNxJqdpuY4zXGn\nqdmJxePE4k6sZbrZg3WJMrG4E3fHHRwnHicxD3iwPO6pljlxf2/e8ffKJY1snDzM8fuWp1z2wbLv\nGyQ5qfD0SUM4Y1S/Y919HYry5rURQFXSfHWw7ANJwczmkqhNMHr06C4JTkTC4e68W9/Ezv2N7Nrf\nyJ6DjdQ1xKg71ERdQxN1h2LUNTSxP1jW0NQcPOIcCqYbY/GoP0aXMks8Dy4tzumkYCmWeYpluPs9\nwD0A5eXlKcuISGZojjtb9x5ic+1BttTWs2VPPVtq69n27iF27m9k94FGmppT/zfukZ9Hac9CSosL\nKAmeB5cUUVyYT3FhHsWF+fQszKcomO9ZmE9xYT6F+XkU5BkF+ZZ4zssjP98ozMsjP88ozLfgOY+C\nfCPfjLw8w4A8M/LMMCN4GHmWWG4k5s1ayoFhWB4f2JZgvkXS5Pu+7CxY8f5lH1wflSiTQjUwKml+\nJLAtolhE5Cg0xpp5c+s+3qjex1s79rNuex3ra/bT0PTeL/ke+XmMHNCTEf16MmFwCYNLiyjrU0RZ\nSRGDS4oY2KeI0p4FlBYXUlyYH+GnEYg2KcwHbjazh4FzgH06nyCS2Q7H4iyrrOXFit0sq6zl9ep9\nHA6acvr3KmTSsFI+NW0MJw3tw5iBvRk9oBdDS4vJy4v216+kL7SkYGYPARcDg8ysGvhnoBDA3X8B\nLABmARVAPfCFsGIRkaNX19DEojU1LF5Xw4sbdnOgMUZBnnHqiL587rwxnD1mAGeO6seQ0qLImz7k\n2IV59dENnax34H+E9f4icvSa485z63fyxMqtPLO2hsZYnKGlxXz8jGF85KTBXDBhEL2Lsq6TZUmD\n/qoi0qquoYlHl1Zx/98qqao9xIDePZg9dRRXTxnBmaP6qSbQDSgpiAh1DU3c++Im7ntpEwcaY0wd\n259vXT6JyyYPoTBfveF0J0oKIt1YY6yZeS9X8vPn3+Hd+iYuP3Uo/3jxBE4b2Tfq0CQiSgoi3dQr\nFbv53394k427D3LRiWXcOuMkJQNRUhDpbg40xvje/DX8bnk1owf04v4507joxE6H7pVuQklBpBtZ\nsWUvX3t4FdV76/nHi0/gq9Mn6oYxeR8lBZFuwN35zaub+d4f1zKktJhHvnweU8cOiDosyUBKCiI5\n7nAsznf/uIYHl2zhkpMH8x/Xn0nfnoVRhyUZSklBJIfVH47x5QeW8+KG3dx08QncOuMk8tXlhHRA\nSUEkR+071MSceUtZuWUvP7zmdK4rH9X5RtLtKSmI5KB99U3c8KtX2bBzPz/79FnMPHVY1CFJllBS\nEMkx9YdjfGHea1TsPMCvbizn4pMGRx2SZBHdvy6SQxpjzXz5geWsqnqXn9xwphKCHDHVFERyhLvz\n7Sfe5MUNu/nhNaeryUiOimoKIjniv17axOMrqrll+kSdVJajpqQgkgOef3sX/3fBOi4/dSi3TJ8Y\ndTiSxZQURLLctncP8dWHVnLikBL+7dozNPSlHBMlBZEs1hx3vvbIKmLNcX7xmbM1GpocMx1BIlns\n7mcreG1TLT+67gzGDuoddTiSA1RTEMlSK7fs5ceLN3D1mcP55Fkjow5HcoSSgkgWOhyL883HVzOk\npIg7rz416nAkh6j5SCQL/ey5Ct6uOcB9ny+npFg9nsrxo5qCSJbZULOfu5+t4MozhnPJyUOiDkdy\njJKCSBZxd779+zfoU1TAP398ctThSA5SUhDJIk+t3s7Syr18Y+bJDOxTFHU4koOUFESyRENTMz/4\n81tMHlaqbiwkNEoKIlninhc2svXdQ/zzxydr9DQJjZKCSBaoqWvg58+9w6zThnLO+IFRhyM5TElB\nJAv89K8VNDXHuW3mpKhDkRynpCCS4apq63l46RaunzqK0QN7RR2O5LhQk4KZzTSz9WZWYWa3pVg/\n2syeNbOVZrbazGaFGY9INvrJ4g2YGV+5RF1iS/hCSwpmlg/cDVwOTAZuMLO2F1b/b+BRd58CzAZ+\nFlY8Itlo464DPL6ims+eO4ahfYujDke6gTBrCtOACnff6O6HgYeBq9qUcaA0mO4LbAsxHpGs85PF\nGygqyOemi0+IOhTpJsLs+2gEUJU0Xw2c06bMd4FFZvYVoDdwaYjxiGSVqtp6/rh6O3MuGMsg3agm\nXSTMmkKqC6m9zfwNwDx3HwnMAh4wsw/EZGZzzWyZmS3btWtXCKGKZJ5fvbiRPIMvfmh81KFINxJm\nUqgGkm+7HMkHm4e+CDwK4O5/A4qBQW1fyN3vcfdydy8vKysLKVyRzLH7QCOPLK3ik1NG6lyCdKkw\nk8JSYKKZjTOzHiROJM9vU2YLMB3AzCaRSAqqCki3N+/lSg43x5l7kWoJ0rVCSwruHgNuBhYC60hc\nZbTGzO4wsyuDYv8E/L2ZvQ48BHze3ds2MYl0KwcbY/z6b5VcfupQTijrE3U40s2EOsiOuy8AFrRZ\ndnvS9FrggjBjEMk2T6yopq4hxpc+rFqCdD3d0SySQeJxZ94rlZwxqh9nje4fdTjSDSkpiGSQlyp2\n886ug3z+/DFRhyLdlJKCSAaZ90olg/oUMeu0YVGHIt2UkoJIhqjcfZBn1+/k0+eMpqggP+pwpJtS\nUhDJEL95dTP5Znz6nNFRhyLdmJKCSAZojDXzxMqtzDhlCINLdbOaREdJQSQDPL22htqDh7l+qmoJ\nEi0lBZEM8MjSKkb068mHJnyglxeRLqWkIBKxqtp6Xtywm+vKR5Gfl6ofSZGuo6QgErFHl1VhBteW\nj4w6FBElBZEoxZrj/G5ZNRedWMbwfj2jDkdESUEkSs+/vYsddQ3Mnjqq88IiXUBJQSRCjy2vZmDv\nHkyfNCTqUEQAJQWRyOw71MTidTv5+BnDKczXf0XJDDoSRSLy5ze2c7g5zifPGhF1KCKtlBREIvLE\nyq2ML+vNaSP6Rh2KSCslBZEIVO+t57VNtXzizBGY6d4EyRxKCiIReHLVNgCuOlNNR5JZlBREupi7\n8/uVWykf05/RA3tFHY7I+ygpiHSxNdvqqNh5gE/oBLNkICUFkS72h5VbKcw3rtDoapKBlBREulA8\n7ix4YzsXTiyjX68eUYcj8gFKCiJdaGXVu2zb18DHzlAtQTKTkoJIF/rT6u30KMjjUnVrIRlKSUGk\niyQ3HZUUF0YdjkhKaSUFM8szsylmdoWZXWJm+pkjcoRWVu1lR10DHztdTUeSuQo6WmlmJwDfBC4F\nNgC7gGLgRDOrB34J3O/u8bADFcl2TwVNR9MnDY46FJF2dZgUgO8DPwe+7O6evMLMBgOfAj4L3B9O\neCK5oaXp6OIT1XQkma3DpODuN3Swbifwn8c9IpEctGLLXmrqGrlCTUeS4dI9p3CnmRUkzZea2X+H\nF5ZIbnmv6Uin4ySzpXv1UQGwxMxON7MZwFJgeXhhieSOlqajj5xURp+izlpsRaKVVlJw92+ROOG8\nBJgHXOHuP+1sOzObaWbrzazCzG5rp8x1ZrbWzNaY2YNHELtIVli+ZS879zcyS91aSBZI62eLmV0I\n/Bi4AzgN+KmZzXH3bR1skw/cDVwGVANLzWy+u69NKjMR+BZwgbvvDU5ei+SURWt20CM/j0tO1uEt\nmS/duuy/Ade2fKGb2SeBvwInd7DNNKDC3TcG2zwMXAWsTSrz98Dd7r4XWk9ei+QMd2fR2hrOnzBQ\nVx1JVkj3nMJ5yb/w3f0J4IJOthkBVCXNVwfLkp1I4p6Hl83sVTObmeqFzGyumS0zs2W7du1KM2SR\n6L1dc4DNe+qZMXlo1KGIpKXDpGBmnzGzPHdvbrvO3feY2Qlm9qH2Nk+xzNvMFwATgYuBG4B7zaxf\nive6x93L3b28rKyso5BFMsqiNTswg0snq+lIskNnzUcDgZVmtpzE1UYtdzRPAC4CdgMpTyCTqBmM\nSpofCbQ9B1ENvOruTcAmM1tPIkksPZIPIZKpFq2tYcqofgwuKY46FJG0dFhTcPcfA2cBDwFlwPRg\nfivwWXf/O3ff0M7mS4GJZjbOzHoAs4H5bcr8AfgIgJkNItGctPEoP4tIRtn27iHe2LqPGaeo6Uiy\nR6cnmoOmo6eDR9rcPWZmNwMLgXzgPndfY2Z3AMvcfX6wboaZrQWagf/l7nuO9EOIZKKn19YAMGOy\nbliT7JHuJallJK4UGpu8jbvP6Wg7d18ALGiz7PakaQe+HjxEcsqitTuYMLgP48v6RB2KSNrSvST1\nSeBF4BkSv+hFpAP76pt4dWMtX75wfNShiByRdJNCL3f/ZqiRiOSQv66voTnuOp8gWSfd+xSeMrNZ\noUYikkMWralhSGkRp4/oG3UoIkck3aRwC4nE0GBm+4NHXZiBiWSrhqZmnn97F5dNHkJeXqrbdUQy\nV1rNR+5eEnYgIrni5Yrd1B9u1l3MkpXS7sfXzK4ELgxmn3P3p8IJSSS7LVpTQ0lRAeeOHxh1KCJH\nLN1Bdn5AoglpbfC4JVgmIkma484z62r4yMmD6VGQbuusSOZIt6YwCzjT3eMAZnY/sJL2u7gQ6ZZW\nbNnLnoOHmXGKbliT7HQkP2WSO6rTJRUiKbSMnXDRieq4UbJTujWFfyHRMd6zJHo/vZDE4DgiEtDY\nCZIL0r366CEzew6YSiIpfNPdd4QZmEi2aRk74csXnhB1KCJHrbPxFE4Ons8ChpHo6roKGB4sE5GA\nxk6QXNBZTeHrwFzg31Osc+CS4x6RSJbS2AmSCzpMCu4+N5i83N0bkteZmY58kUDL2Am3Xd7RsOUi\nmS/dq49eSXOZSLeksRMkV3RYUzCzocAIoKeZTeG9cZdLgV4hxyaSNTR2guSKzs4pfBT4PInxlX+U\ntHw/8O2QYhLJKho7QXJJZ+cU7gfuN7O/c/fHuygmkayisRMkl6R7n8LjZnYFcApQnLT8jrACE8kW\nGjtBckm6HeL9Arge+AqJ8wrXAmNCjEskKzQ0NfPc+l1cOkljJ0huSPfqo/Pd/UZgr7t/DzgPGBVe\nWCLZ4aUNuznU1MxH1XQkOSLdpHAoeK43s+FAEzAunJBEsseitTsoKdbYCZI70u0Q7ykz6wfcBawg\ncTfzr0KLSiQLxJrjPLNuJ5do7ATJIemeaL4zmHzczJ4Cit19X3hhiWS+5Zv3UnvwsIbdlJyS7onm\n183s22Z2grs3KiGIJPo66lGQx0UnaewEyR3p1nmvBGLAo2a21MxuNbPRIcYlktHcnYVrdvChCYPo\nU5T2UOciGS+tpODum939h+5+NvAp4HRgU6iRiWSwddv3U733kPo6kpyT9k8cMxsLXEfifoVm4Bvh\nhCSS+RatTYydMH2SkoLklrSSgpktAQqB3wHXuvvGUKMSyXAL19RQPqY/ZSVFUYciclylW1P4nLu/\nFWokIlmiqraeddvr+M6sSVGHInLcddZ19mfc/TfALDOb1Xa9u/8oxWbJ288EfgzkA/e6+w/aKXcN\niVrIVHdflm7wIlFY1DJ2wilqOpLc01lNoXfwXJJinXe0oZnlA3cDl5EY23mpmc1397VtypUAXwWW\npBWxSMQWrtnByUNLGDOwd+eFRbJMZ11n/zKYfMbdX05eZ2YXdPLa04CKlvMPZvYwcBWwtk25O4Ef\nAremG7RIVPYcaGRZZS03f2RC1KGIhCLd+xT+X5rLko0AqpLmq4NlrYLR3Ea5+1NpxiESqcXrdhJ3\nNHaC5KzOzimcB5wPlJnZ15NWlZI4T9Dh5imWtTY5mVke8B8kRnbr+IXM5gJzAUaP1j1zEp2Fa3Yw\nol9PThleGnUoIqHorKbQA+hDInmUJD3qgGs62baa93evPRLYljRfApwKPGdmlcC5wHwzK2/7Qu5+\nj7uXu3t5WZm6FJBo1DU08eKG3cw8dShmGjtBclNn5xSeB543s3nuvvkIX3spMNHMxgFbgdkk7oZu\nee19wKCWeTN7DrhVVx9Jplq8robDzXFmnTYs6lBEQpPuOYV7g66zATCz/ma2sKMN3D0G3AwsBNYB\nj7r7GjO7w8yuPOqIRSLyp9U7GFpazJRR/TovLJKl0r15bZC7v9sy4+57zWxwZxu5+wJgQZtlt7dT\n9uI0YxHpcvsbmnhhwy4+fc5oDbspOS3dmkI8uVfUoB+kDu9TEMkli9ft5HAszhVqOpIcl25N4TvA\nS2b2fDB/IcHVQCLdwYI3tjO0tJizRvePOhSRUKXbdfZfgHJgPfAI8E+8N26zSE470Bjjubd3MfPU\noWo6kpyXbi+pXwJuIXFZ6SoSl4/+DbgkvNBEMsPidTUcjumqI+ke0j2ncAswFdjs7h8BpgC7QotK\nJIMseGM7g0uKKB+jpiPJfekmhQZ3bwAws6KgG+2TwgtLJDMcbIzx3PpdXK6mI+km0j3RXB3cp/AH\n4Gkz28v7704WyUmL39pJo5qOpBtJKym4+yeCye+a2bNAX+AvoUUlkiGeXLmVoaXFTB07IOpQRLpE\n2mM0twi6vhDJebUHD/P827uY86FxajqSbiPdcwoi3c6f3thOLO5cdebwqEMR6TJKCiLteHLlViYO\n7sPkYeomW7oPJQWRFKpq61m2eS9XTxmhbrKlW1FSEElh/uuJi+uuPENNR9K9KCmItOHu/GHlVsrH\n9GfUgF5RhyPSpZQURNpYt30/G3Ye0Alm6ZaUFETaeHxFNYX5xhWnKylI96OkIJLkcCzO71du5dJJ\nQxjQu0fU4Yh0OSUFkSSL19VQe/Aw100dFXUoIpFQUhBJ8siyKoaWFnPhxLKoQxGJhJKCSGD7vkO8\n8PYurjl7JPnq1kK6KSUFkcATK7YSd7i2fGTUoYhERklBBIjHnUeXVXHu+AGMGdg76nBEIqOkIAIs\n2VTL5j31XK8TzNLNKSmIAL9dspnS4gJmnqLBdKR7U1KQbm9nXQN/eXMH15WPomeP/KjDEYmUkoJ0\new++toVY3PnMuWOiDkUkckoK0q01Ncd5cMkWLj6pjLGDdIJZRElBurWFa3awc38jN56nWoIIKClI\nN3f/K5WMHtCLi04cHHUoIhlBSUG6rVVV77K0ci83njdGdzCLBJQUpNu654V3KCkuYPa00VGHIpIx\nQk0KZjbTzNabWYWZ3ZZi/dfNbK2ZrTazxWamhl3pEpW7D/KXN3fwmXPH0KeoIOpwRDJGaEnBzPKB\nu4HLgcnADWY2uU2xlUC5u58OPAb8MKx4RJLd+9JGCvLy+ML5Y6MORSSjhFlTmAZUuPtGdz8MPAxc\nlVzA3Z919/pg9lVAPZFJ6PYcaOR3y6q5espwBpcWRx2OSEYJMymMAKqS5quDZe35IvDnVCvMbK6Z\nLTOzZbt27TqOIUp3dO9LmzjcHGfuheOjDkUk44SZFFJdzuEpC5p9BigH7kq13t3vcfdydy8vK9Pg\nJ3L09hxo5P5XKrnitGFMGFwSdTgiGSfMM2zVQHKXkyOBbW0LmdmlwHeAi9y9McR4RLjnxY0camrm\nlukTow5FJCOFWVNYCkw0s3Fm1gOYDcxPLmBmU4BfAle6+84QYxFh94FGfv3KZj5++nAmDlEtQSSV\n0JKCu8eAm4GFwDrgUXdfY2Z3mNmVQbG7gD7A78xslZnNb+flRI7ZPS9spDHWzFdVSxBpV6gXaLv7\nAmBBm2W3J01fGub7i7Soqq1n3iuVXH3mCCYM7hN1OCIZS3c0S7dw18L1GHDrR0+KOhSRjKakIDlv\nVdW7zH99G1/68DiG9+sZdTgiGU1JQXKau/P9p9YyqE8Pbrp4QtThiGQ8JQXJafNf38ayzXv5n5ed\nqD6ORNKgpCA5a199E3c+tZYzRvZl9lT1hCqSDv10kpz1g7+8xd76Ju6fM03jJYikSTUFyUnLKmt5\n6LUtzLlgLKcM7xt1OCJZQ0lBck794RjfeGw1I/r15GuXnhh1OCJZRc1HknP+z5/WsWnPQX77pXPo\nrZPLIkdENQXJKYvX1fDbJVuY++HxnH/CoKjDEck6SgqSM3bsa+Abj61m0rBSvj5DzUYiR0NJQXJC\nY6yZm367nENNzfxk9pkUFeRHHZJIVlKDq+SE7/1xLSu3vMvPP32WusUWOQaqKUjW+82rm3lwyRZu\nuvgELj9tWNThiGQ1JQXJagvX7OD2J9/kkpMHc+sM9YAqcqyUFCRrLa2s5asPreT0kf346aem6K5l\nkeNASUGy0oote5kzbykj+vfkvs9PpVcPnR4TOR6UFCTrvLapls/eu4QBvXvwwBfPYUDvHlGHJJIz\n9PNKsspz63dy029WMKxfMQ/9/bkMKS2OOiSRnKKagmSN+1+pZM68pYwd1JtH5p6nhCASAtUUJOM1\nxpr5/lPreODVzVw6aTA/nj1FfRqJhET/sySjbdp9kK88tII3t9Yx98LxfHPmybrKSCRESgqSkeJx\n58HXtvAvC9ZRWJDHr24s57LJQ6IOSyTnKSlIxtlQs59v//4Nllbu5YIJA7nrmjMY3q9n1GGJdAtK\nCpIxauoa+M9n3uaRpVWUFBdy1zWnc83ZIzFTc5FIV1FSkMjV1DXw3y9Xcv8rlcTicW48byxfuWQC\nA/sURR2aSLejpCCRWbNtH//9ciVPrtpKc9z52OnD+acZJzJmYO+oQxPptpQUpEvtPtDI/FXbeGx5\nNWu319GzMJ9PTRvNnA+NUzIQyQBKChIqd2fj7oM8s7aGZ9bVsHzzXuIOp43oy/euPIWrzhxOv17q\npkIkUygpyHHVHHc27jrA0sq9vLZpD69tqmXbvgYAThleylcumcis04Zx0lANhCOSiUJNCmY2E/gx\nkA/c6+4/aLO+CPg1cDawB7je3SvDjEmOD3dn5/5GKncf5J1dB1mzbR9rt9fx1vb9HGpqBqCspIhp\n4wZw0/iBTD95sC4rFckCoSUFM8sH7gYuA6qBpWY2393XJhX7IrDX3SeY2WzgX4Hrw4pJ0hNrjrPv\nUBM79zdSU9fAzv2N7Aqmt+9rYMueejbXHqShKd66TUlxAZOHlTJ72ihOGd6X8jH9GTOwly4nFcky\nYdYUpgEV7r4RwMweBq4CkpPCVcB3g+nHgJ+ambm7hxhX1ojHnVjcaY47sXg8eE7MNzW/fz7WHCyP\nx2mKxWmIxTl0uJnGWDOHDjfT0NTMoaY4DU3NrY8Djc3sO9REXUMTdYeCR0OMA42xlPH07VnIkNIi\nRg/ozYcnDmLMoN6MHdiLsQN7M7J/TyUAkRwQZlIYAVQlzVcD57RXxt1jZrYPGAjsPt7BPLq0il++\n8A6t2cbBE+/buswdHE88J6WlljIt698r21Ku7TJPWpf0Hk7Se73/NVvLBv/E4nHiIaXGooI8igvz\n6d0jn9KehfTtWcioAb3o27OQ0uLEfN+eBQwpLWZwaRGDS4opKymiuDA/nIBEJGOEmRRS/Wxs+zWX\nThnMbC4wF2D06NFHFUz/3j04eWgp2HtvamYYYCmWJcpZsIykckbLD2ILCr5/+/fKWPB6pFjX8uHN\n3v+ewStSmG/k5xkFeUZBfh4Fee/N5+clzecbBXl5763LNwrz8ujZI/HFX1yYT8+k56KCPPLUoZyI\ntCPMpFANjEqaHwlsa6dMtZkVAH2B2rYv5O73APcAlJeXH9Xv58smD1GHaiIinQhzkJ2lwEQzG2dm\nPYDZwPw2ZeYDnwumrwH+qvMJIiLRCa2mEJwjuBlYSOKS1PvcfY2Z3QEsc/f5wH8BD5hZBYkawuyw\n4hERkc6Fep+Cuy8AFrRZdnvSdANwbZgxiIhI+jRGs4iItFJSEBGRVkoKIiLSSklBRERaKSmIiEgr\ny7bbAsxsF7D5KDcfRAhdaBwHiuvIKK4jl6mxKa4jcyxxjXH3ss4KZV1SOBZmtszdy6OOoy3FdWQU\n15HL1NgU15HpirjUfCQiIq2UFEREpFV3Swr3RB1AOxTXkVFcRy5TY1NcRyb0uLrVOQUREelYd6sp\niIhIB3IuKZjZtWa2xsziZlbeZt23zKzCzNab2Ufb2X6cmS0xsw1m9kjQ7ffxjvERM1sVPCrNbFU7\n5SrN7I2g3LLjHUeK9/uumW1Nim1WO+VmBvuwwsxu64K47jKzt8xstZn93sz6tVOuS/ZXZ5/fzIqC\nv3FFcCyNDSuWpPccZWbPmtm64Pi/JUWZi81sX9Lf9/ZUrxVCbB3+XSzhJ8H+Wm1mZ3VBTCcl7YdV\nZlZnZl9rU6bL9peZ3WdmO83szaRlA8zs6eC76Gkz69/Otp8Lymwws8+lKnNE3D2nHsAk4CTgOaA8\naflk4HWgCBgHvAPkp9j+UWB2MP0L4KaQ4/134PZ21lUCg7pw330XuLWTMvnBvhsP9Aj26eSQ45oB\nFATT/wr8a1T7K53PD/wj8ItgejbwSBf87YYBZwXTJcDbKeK6GHiqq46ndP8uwCzgzyTGHjwXWNLF\n8eUDO0hcxx/J/gIuBM4C3kxa9kPgtmD6tlTHPTAA2Bg89w+m+x9LLDlXU3D3de6+PsWqq4CH3b3R\n3TcBFcC05AKWGDvzEuCxYNH9wNVhxRq833XAQ2G9RwimARXuvtHdDwMPk9i3oXH3Re4eC2ZfJTGK\nX1TS+fxXkTh2IHEsTbfkcVlTcyY4AAAEcElEQVRD4O7b3X1FML0fWEdiDPRscBXwa094FehnZsO6\n8P2nA++4+9HeFHvM3P0FPjjqZPJx1N530UeBp9291t33Ak8DM48llpxLCh0YAVQlzVfzwf80A4F3\nk76AUpU5nj4M1Lj7hnbWO7DIzJYH41R3hZuDKvx97VRX09mPYZpD4ldlKl2xv9L5/K1lgmNpH4lj\nq0sEzVVTgCUpVp9nZq+b2Z/N7JQuCqmzv0vUx9Rs2v9hFsX+ajHE3bdDIukDg1OUOe77LtRBdsJi\nZs8AQ1Os+o67P9neZimWtb30Kp0yaUkzxhvouJZwgbtvM7PBwNNm9lbwi+KodRQX8HPgThKf+U4S\nTVtz2r5Eim2P+RK2dPaXmX0HiAG/bedljvv+ShVqimWhHUdHysz6AI8DX3P3ujarV5BoIjkQnC/6\nAzCxC8Lq7O8S5f7qAVwJfCvF6qj215E47vsuK5OCu196FJtVA6OS5kcC29qU2U2i6loQ/MJLVea4\nxGhmBcAngbM7eI1twfNOM/s9iaaLY/qSS3ffmdmvgKdSrEpnPx73uIITaB8DpnvQmJriNY77/koh\nnc/fUqY6+Dv35YNNA8edmRWSSAi/dfcn2q5PThLuvsDMfmZmg9w91D5+0vi7hHJMpelyYIW717Rd\nEdX+SlJjZsPcfXvQnLYzRZlqEuc+WowkcT71qHWn5qP5wOzgypBxJDL+a8kFgi+bZ4FrgkWfA9qr\neRyrS4G33L061Uoz621mJS3TJE62vpmq7PHSph33E+2831JgoiWu0upBouo9P+S4ZgLfBK509/p2\nynTV/krn888ncexA4lj6a3uJ7HgJzln8F7DO3X/UTpmhLec2zGwaif//e0KOK52/y3zgxuAqpHOB\nfS3NJl2g3dp6FPurjeTjqL3vooXADDPrHzT3zgiWHb2uOLPelQ8SX2bVQCNQAyxMWvcdEleOrAcu\nT1q+ABgeTI8nkSwqgN8BRSHFOQ/4hzbLhgMLkuJ4PXisIdGMEva+ewB4A1gdHJDD2sYVzM8icXXL\nO10UVwWJdtNVweMXbePqyv2V6vMDd5BIWgDFwbFTERxL47tgH32IRLPB6qT9NAv4h5bjDLg52Dev\nkzhhf34XxJXy79ImLgPuDvbnGyRdNRhybL1IfMn3TVoWyf4ikZi2A03B99cXSZyHWgxsCJ4HBGXL\ngXuTtp0THGsVwBeONRbd0SwiIq26U/ORiIh0QklBRERaKSmIiEgrJQUREWmlpCAiIq2UFEREpJWS\ngoiItFJSEDlGZjY16ESwOLiDd42ZnRp1XCJHQzeviRwHZvZ9Ency9wSq3f1fIg5J5KgoKYgcB0E/\nSEuBBhLdITRHHJLIUVHzkcjxMQDoQ2LUs+KIYxE5aqopiBwHZjafxChs40h0JHhzxCGJHJWsHE9B\nJJOY2Y1AzN0fNLN84BUzu8Td/xp1bCJHSjUFERFppXMKIiLSSklBRERaKSmIiEgrJQUREWmlpCAi\nIq2UFEREpJWSgoiItFJSEBGRVv8fEsQCff7iZeQAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "x1 = np.linspace(-10, 10, 500)\n",
    "plt.plot(x1, activation(x1))\n",
    "plt.xlabel('x'); plt.ylabel('activation(x)')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's see what happens with low, intermediate and high values:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEKCAYAAAD9xUlFAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4zLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvIxREBQAAIABJREFUeJzt3Xl8VdW5//HPk4GEKSBzEkYVB0RR\niFqHqhWLaAE7abWT3lpi22trX21vtcMt1PZ3e297O/z6q23B6nW4dapDG5AWhzq1VSQBJ0AkRYYk\njIIMAiHD8/vjnMSTeJJsIOfss0++79frvHL22uvs85ydlfNk77X3WubuiIiIAOSEHYCIiGQOJQUR\nEWmlpCAiIq2UFEREpJWSgoiItFJSEBGRVkoKIiLSSklBRERaKSmIiEirvLADOFRDhgzxsWPHhh2G\niEikVFVVbXf3oV3Vi1xSGDt2LJWVlWGHISISKWa2Pkg9nT4SEZFWSgoiItJKSUFERFopKYiISCsl\nBRERaZWypGBmt5vZVjN7rYP1Zma/NLNqM3vFzCanKhYREQkmlUcKdwDTO1l/CTA+/igHfpPCWERE\nJICU3afg7s+a2dhOqlwG3OWx+UBfMLOBZlbs7ptSFZOIZIfmZqe+sZkDDU0caGyisclpaGqmqdlp\naHIam5tpbHYaW543eXxdrE5js9Psjjs4TnMzsWXA4+XNnqzMafZ3lx1/t17CzMaJ0xy3KU9a9t66\nbSZJTqg89cThTBo18Eh3X+diHy41D2As8FoH6xYC5yYsPwmUdVC3HKgEKkePHu2Ha86cOR7f3w54\nZWWlV1ZWtimbM2eOu7sXFxe3lk2ePNnd3WfPnt2mbm1trVdUVLQpmzdvnscTXetjxowZ7u4+Y8aM\nNuXu7vPmzWtTVlFR4bW1tW3KZs+e7e7ukydPbi0rLi7WZ9Jniuxn2rG33i//1DVt6v7wgb/5Z+bc\n0qZs0pX/5hf//Jk2ZX2PPcPH3LjQex9zepvyMTcu9EEXX9+mbOjH/t1Lv3Rnm7J+ky72MTcu9F7D\nj2kty+03yMfcuNAHnHNVm7ojrv6Fj7j6F23KBpxzlY+5caHn9hvUWtZr+DE+5saF3m/SxW3qln7p\nTh/6sX/3glETfcyNCw/7Mfam2OPu59cF+7JLAqj0AN/b5okpq5vFjxQWuvvEJOseBX7k7n+LLz8J\nfNPdqzrbZllZmeuOZpHM1dTs1O7cz/od77Bhxz42vLWPDTv2Uff2frbuqWf73noampJ/7/TKzaGo\ndz5FhXn0j//snZ9LYX4uhfk5FObn0js/l4L4csu6/Nwc8nKMvFyL/czJITfXyM/JITfHyM+1+M8c\n8nKNXDNycgwDcszIMcOM+MPIsVi5EVs2a6kHhmE5vOe1xJdbJDylV14uDY1N8fLYioTVbepa4kI3\nMrMqdy/rql6Yw1zUAKMSlkcCdSHFIiKHob6xiddqd/FqzS5e37yHVZt2s3rLHg40NLfW6ZWbw8hB\nvSkd2Jtjh/VnWFEBQ/sVMLR/AcP6FzC4XwFFvfMoKsynMD83xE+TOvPmzSMvNxoXe4aZFCqA683s\nPuBMYJerP0Ekox1sbKZy3Q6eq95O5bodvFyzi4ONsQRwVJ98Tiwu4pNnjOH4Ef0YM7gvowf1YURR\nITk5qfnvNyrKy8vDDiGwlCUFM7sXuAAYYmY1wBwgH8DdfwssAi4FqoF9wL+kKhYROXy7DzTw2Iot\nPLlqC8+t2c7e+kbycoyJpQO4+qwxTBkziFNHDWR4UUHKTn1EnZmRylP13SmVVx9d1cV6B/41Ve8v\nIoevqdl5evVWHl5eyxMrt1Df2MyIokJmTirmA8cP45xjh9C3IHKDLEsA+q2KSKvdBxp4YOlG7nx+\nHRt37GdQ315cefooPnxaKaeOGqgjgR5ASUFE2H2ggd899ya3/+1N9tY3cvrYo/jWJSfywQnDyY9I\nB2kmmzFjRtghBKakINKD1Tc2ccff1/GbZ/7J2/sauGTiCL50wbGcPHJA2KFllQULFoQdQmBKCiI9\n1D+qt/PdP77G2u3vcP5xQ/nGtOOVDFJk5syZkUkMSgoiPcze+ka+X7GCP1TVMHpQH+783Bmcf1yX\nU/fKEVi4cGHYIQSmpCDSgyzbsJOv3vcSNTv38aULjuErU8dn7Q1jcniUFER6AHfnf19Yz/cXrGR4\nUSH3X3cWp48dFHZYkoGUFESy3MHGZuYuWME9SzZw4QnD+PknTmVA7/yww+pRonLjGmjmNZGstu9g\nI9feuZR7lmzgixccw62fLVNCCMH8+fPDDiEwJQWRLLVrfwOfue1F/l69nR9//BRunH4CuT18DKKw\nXHfddWGHEJhOH4lkoV37Grjq1hdYs3UPv/7UZKZPLA47JIkIJQWRLLPvYCP/cseLVG/dy62fLeOC\n44eFHZJEiE4fiWSR+sYmrru7ipc2vs0vrzpVCSFDVFRUhB1CYDpSEMkS7s63H36N59bE+hB0yihz\nTJkyJewQAtORgkiWuO1vb/LQshpumDqeK8pGdf0CSZvS0tKwQwhMSUEkCzzzxjb+Y9EqLpk4ghum\njg87HIkwJQWRiKt7ez9fuXc5x48o4qdXTOrxU1/KkVFSEImwpmbnq/e/RGNTM7/51GT69FI3YSaa\nPXt22CEEphYkEmG3PFXNi2/u4GdXTGLskL5hhyMd0B3NIpJyyzbs5BdPvMGHTy3ho5NHhh2OdEJX\nH4lISh1sbObGB19hRFEhP/jwxLDDkS4sW7Ys7BAC0+kjkQj69dPVrNm6l9uvKaN/oQa4k+6jIwWR\niFmzZQ+3PFXNrEklXHjC8LDDkQCKi6NzI6GSgkiEuDvffuRV+hXkMWfmhLDDkYDq6urCDiEwJQWR\nCFn4yiaWrtvJN6efwOB+BWGHIwHNnTs37BACU1IQiYgDDU38559fZ0JxkYaxiJjvf//7YYcQmJKC\nSETMf3YttW/vZ87MCZosR1JGSUEkArbsPsBvnv4nl548gjOPHhx2OJLFlBREIuBXf62moamZm6af\nGHYochgqKyvDDiEwJQWRDLdxxz7uW7qBT5w+itGD+4QdjmS5lCYFM5tuZqvNrNrMbkqyfrSZPWVm\ny83sFTO7NJXxiETRL59cg5nx5Qs1JHZUlZWVhR1CYClLCmaWC9wCXAJMAK4ys/YXVn8XeMDdTwOu\nBH6dqnhEomjttr08tKyGz7xvDCMGFIYdjvQAqTxSOAOodve17n4QuA+4rF0dB4rizwcA0bnDQyQN\nfvnkGgrycvniBceEHYr0EKkc+6gU2JiwXAOc2a7OXOAxM/sy0Be4KIXxiETKxh37WPDKJj53zliG\n6Ea1SJszZ07YIQSWyiOFZBdSe7vlq4A73H0kcClwt5m9JyYzKzezSjOr3LZtWwpCFck8tz63lhyD\na889OuxQ5AjpjuaYGiDxtsuRvPf00LXAAwDu/jxQCAxpvyF3n+/uZe5eNnTo0BSFK5I5tu+t5/6l\nG/nY5JHqS8gCJSUlYYcQWCqTwlJgvJmNM7NexDqSK9rV2QBMBTCzE4klBR0KSI/3P39/k4NNzZSf\np6OEbLBp06awQwgsZUnB3RuB64HFwCpiVxmtMLObzWxWvNrXgdlm9jJwL3CNu7c/xSTSo+ytb+Su\n59dzycQRHD20X9jhSA+T0kl23H0RsKhd2fcSnq8EzkllDCJR8/CyGvYcaOTz79dRQraYPHly2CEE\npjuaRTJIc7Nz5z/WMWnUQCaPPirscKSbVFVVhR1CYEoKIhnkb9Xb+ee2d7jm7DFhhyLdqLy8POwQ\nAlNSEMkgd/xjHUP6FXDpydGZvlG6duutt4YdQmBKCiIZYt32d3hq9VY+deZoCvJyww5HeiglBZEM\n8b8vrCfXjE+dOTrsUKQHU1IQyQD1jU08vLyWaScNZ1iRblbLNrW1tWGHEJiSgkgGeHzlFna8c5BP\nnK6jhGykq49E5JDcv3QjpQN7c+6x7xnlRbLArFmzuq6UIZQUREK2ccc+nluznSvKRpGbk2wcSZH0\nUVIQCdkDlRsxg8vLRoYdioiSgkiYGpua+UNlDecfN5SSgb3DDkdSZN68eWGHEJiSgkiInnljG5t3\nH+DK00d1XVkiS3c0i0ggD1bVMLhvL6aeODzsUCSFzKLTV6SkIBKSXfsbeHLVVmZOKiE/V3+KkhnU\nEkVCsujVTRxsauajk0vDDkWklZKCSEgeWV7L0UP7cnLpgLBDkRSbMWNG2CEEpqQgEoKanft48c0d\nfOTU0kidb5bDs2DBgrBDCExJQSQEf3qpDoAPn6ZTRz3BzJkzww4hMCUFkTRzdx5ZXsvpY49i1KA+\nYYcjabBw4cKwQwhMSUEkzVbU7aZ6614dJUhGUlIQSbM/Lq8lP9f4kGZXkwykpCCSRs3NzqOvbuK8\n8UMZ2KdX2OFImrh72CEEpqQgkkbLN77Npl0HmDFJRwk9yfz588MOITAlBZE0evSVTfTKy+EiDWvR\no1x33XVhhxCYkoJImjQ3O4te3cT5xw2lf2F+2OGIJJUXpJKZ5QCTgBJgP7DC3bekMjCRbLNsw042\n7z7ATSefEHYoIh3qNCmY2THAjcBFwBpgG1AIHGdm+4B5wJ3u3pzqQEWi7tFXY6eOpp44LOxQJM0q\nKirCDiGwro4Ufgj8BrjO23Wfm9kw4JPAZ4A7UxOeSHZoOXV0gU4d9UhTpkwJO4TAOk0K7n5VJ+u2\nAr/o9ohEslDVhp1s2V3Ph07RVUc9UWlpaWQuSw3U0WxmPzCzvITlIjP7n9SFJZJdHn1lEwV5OZpM\nRzJe0KuP8oAlZnaKmU0DlgJVqQtLJHu0njo6fij9CgJd2yESmkBJwd2/RazDeQlwB/Ahd/9VV68z\ns+lmttrMqs3spg7qXGFmK81shZndcwixi0RC1YadbN1Tz6Ua1qLHmj17dtghBBb0ktTzgP8L3Ayc\nDPzKzD7n7nWdvCYXuAX4IFADLDWzCndfmVBnPPAt4Bx33xnvvBbJKotf20yvXJ066smy8Y7m/wYu\nd/cfufsngfnAX7t4zRlAtbuvdfeDwH3AZe3qzAZucfed0Np5LZI13J3HVm7hnGMH69RRDxalq4+C\nJoWzEv/Dd/eHgXO6eE0psDFhuSZelug4Yvc8/N3MXjCz6ck2ZGblZlZpZpXbtm0LGLJI+FZv2cOG\nHfuYdtKIsEOREC1btizsEALrNCmY2afNLMfdm9qvc/e3zOwYMzu3o5cnKWt/TVYeMB64ALgK+J2Z\nDUzyXvPdvczdy4YOHdpZyCIZ5bEVWzBDN6xJZHR1PDsYWG5mVcSuNmq5o/lY4HxgO5C0A5nYkcGo\nhOWRQPs+iBrgBXdvAN40s9XEksTSQ/kQIpnqsZWbmTz6KIb1Lww7FAlRcXF0LjLo9EjB3f8vMBm4\nFxgKTI0v1wKfcfePufuaDl6+FBhvZuPMrBdwJdD+Xu8/Ah8AMLMhxE4nrT3MzyKSUWrf3s9rtbuZ\nNkEdzD1dXV2H1+RknC57vuKnjh6PPwJz90Yzux5YDOQCt7v7CjO7Gah094r4umlmthJoAv7N3d86\n1A8hkokeX7EZQP0Jwty5c5k7d27YYQRiQW69NrOhxK4UGktCInH3z6Ussg6UlZV5ZWVlut9W5JB9\n8tYX2Lannse/dn7YoUjIzCz0YS7MrMrdy7qqF/QauT8BzwFPEPuPXkQ68fa+gyx5cwdfOP/osEMR\nOSRBk0Ifd78xpZGIZJG/vr6VpmZn2gSdOpJoCXqfwkIzuzSlkYhkkcdWbGFEUSEnlw4IOxTJAFE6\n5R00KdxALDEcMLM98cfuVAYmElUHGpp45o1tfHDCcHJykt2uI5K5Ap0+cvf+qQ5EJFv8bc129jc0\nMe0kXYoqMWVlZaF3NAcVeDAWM5sFnBdffNrdF6YmJJFoe2zlZvoX5nHmuMFhhyJyyIJOsvOfxE4h\nrYw/boiXiUiCpmbniVVbufCEYfTKC3p2ViRzBD1SuBQ41d2bAczsTmA5HQ9xIdIjVa3fyY53Duqq\nI2ljzpw5YYcQ2KH8K5M4UJ0uqRBJ4rEVsbkTzj9eAzfKu6JyNzMEP1L4EbGB8Z4iNvrpecQmxxGR\nOM2dIB0pKSmJzPhHQa8+utfMngZOJ5YUbnT3zakMTCRq3tiylw079vHFC44JOxTJMJs2bQo7hMC6\nmk/hhPjPyUAxsaGuNwIl8TIRiXtsxWbNnSCR19WRwteAcuCnSdY5cGG3RyQSUY+t3KK5EySpyZOj\n8z90V/MplMefXuLuH0h8ELsiKVLmzp2LmbU+qqqqqKqqalPW0iFUUlLSWtYyv2p5eXmbunV1dSxY\nsKBNWcsE3YllM2fOBGDmzJltyiE2oXdi2YIFC6irq2tTVl4e+zVMmTKltaykpESfKcM+09O3/VBz\nJ0hSVVVVYYcQWNChs5e5++SuytJBQ2dLJrrzH+uYU7GCs+oe5N67/ifscCTDlJeXt/4jEpagQ2d3\n1acwwsymAL3N7DQzmxx/XAD06aZYRSLvsZWbGT+sH/fdfUfYoUgGuvXWW8MOIbCu+hQuBq4hNr/y\nzxLK9wDfTlFMIpGya18DL6yNzZ3wRNjBiByhTpOCu98J3GlmH3P3h9IUk0ik/HX1lta5E74ZdjAi\nRyjofQoPmdmHgJOAwoTym1MVmEhULH7t3bkTamtrww5HMlCU2kXQAfF+C3wC+DKxm9cuB8akMC6R\nSGiZO+GiCcPIybFIXWUi6ROldhF07KOz3f2zwE53/z5wFjAqdWGJREPL3AkXnxQbAG/WrFkhRySZ\nKErtImhS2B//uc/MSoAGYFxqQhKJjsUrNHeCZJego3YtNLOBwE+AZcTuZo7ONVYiKdDY1MwTq7Yw\nVXMnSBYJ2tH8g/jTh8xsIVDo7rtSF5ZI5qtav5Od+xqYdtK7cyfMmzcvxIgkU0WpXQTtaH7ZzL5t\nZse4e70SgggsXrGFXnk5nH/cu3MntAzfIZIoSu0i6DHvLKAReMDMlprZN8xsdArjEslosbkTNvP+\nY4fQN2HuhJaxkkQSRaldBEoK7r7e3X/s7lOATwKnAG+mNDKRDLZq0x5qdu5n2kkaAE+yS+Dpocxs\nLHAFsfsVmkA3b0rPtbh17gQlBckugZKCmS0B8oE/AJe7+9qURiWS4R5buYWyMUcxpF9Bm/IZM2aE\nFJFksii1i6BHCle7++spjUQkIjbu2MeqTbv57odOfM+6BQsWhBCRZLootYuuhs7+dPzppWb2tfaP\nrjZuZtPNbLWZVZvZTZ3U+7iZuZl1Oda3SNgWr4hNT/7BJBPqtEzUI5IoSu2iqyOFvvGf/ZOs63R2\nHjPLBW4BPkhsbuelZlbh7ivb1esPfAVYEihikZA9tnILJ4zoz5jBfd+zbuHChSFEJJkuSu2iq6Gz\nW+64eMLd/564zszO6WLbZwDVLf0PZnYfcBmwsl29HwA/Br4RNGiRsLy1t57KdTu4/sLxYYcikhJB\n71P4fwHLEpUCGxOWa+JlrczsNGCUu0cnjUqP9uSqrTQ7motZslanRwpmdhZwNjC0XR9CEZDbxbaT\n3a3ResrJzHKAnxOb2a3zDZmVA+UAo0frnjkJz+IVmykd2JuTSoqSrg8y57n0PFFqF10dKfQC+hFL\nHv0THruBj3fx2hraDq89EqhLWO4PTASeNrN1wPuAimSdze4+393L3L1s6NCh7VeLpMXuAw08t2Y7\n0yeO6PAO1bAnZ5fMFKV2YUEymJmNcff1h7RhszzgDWAqUAssBT7p7is6qP808A13r+xsu2VlZV5Z\n2WkVkZR4eFkNX3vgZR764tlMGXNU0jpmFqn/CiU9MqFdmFmVu3d5hWfQPoXfxYfObtn4UWa2uLMX\nuHsjcD2wGFgFPODuK8zsZjOLzowTInGLXt1M8YBCThs1sOvKIhEV9Oa1Ie7+dsuCu+80s2Fdvcjd\nFwGL2pV9r4O6FwSMRSTt9hxo4Nk12/j0mWPIyYnO4GYihyrokUJz4qio8XGQdIwsPcaTq7ZysLGZ\nD50yotN6FRUVaYpIoiRK7SLokcJ3gL+Z2TPx5fOIXw0k0hM8+uomRhQVctqo5H0JLaZMmZKmiCRK\notQugg6d/RegDFgN3A98nXfnbRbJanvrG3nmjW1Mnziiy1NHpaWlna6XnilK7SLoKKmfB24gdlnp\nS8QuH30euDB1oYlkhidXbYmfOioOOxSRlAvap3ADcDqw3t0/AJwGbEtZVCIZZNGrmxheVMCU0Z2f\nOhLJBkGTwgF3PwBgZgXxYbSPT11YIpnhnfpGnl69jUsmFge66mj27NlpiEqiJkrtImhHc038PoU/\nAo+b2U7a3p0skpWefH0r9Y3NXDKx86uOWkTpzlVJnyi1i6AdzR9x97fdfS7w78BtwIdTGZhIJvjT\n8lqKBxRy+thBgepH6SoTSZ8otYvAczS3cPdnuq4lEn073jnIM29s49pzxwW+YW3ZsmUpjkqiKErt\nImifgkiP8+irm2hsdi47NTqXE4ocKSUFkQ78aXktxw3vx4nFySYeTK64WJetyntFqV0oKYgksXHH\nPirX7+SyU0s7HCY7mbo6XX8h7xWldqGkIJJExcuxP+JZk0oO6XVz585NQTQSdVFqF4HmU8gkmk9B\nUs3dmfbzZxnQO58Hv3j2Ib02E8bNl8yTCe2iu+dTEOkxVm7azZqte7nsNHUwS8+jpCDSzkNVteTn\nGh86OTqdgyLdRUlBJMHBxmYeWV7DBycMZ1DfXof8ep3alGSi1C6UFEQSPLFqCzv3NXB52aiwQxEJ\nhZKCSIIHKjdSPKCQ88YPPazXl5V12Y8nPVCU2oWSgkjcpl37efaNbXx8ykhyNQ+z9FBKCiJxD1XV\n0Oxw+RSdOpKeS0lBBGhudh6orOGsowczenCfw97OnDlzujEqyRZRahdKCiLAkjd3sGHHPq44feQR\nbSdKd65K+kSpXSgpiAC/X7KeosI8pp90ZPcmlJQc2rAY0jNEqV0oKUiPt2X3Af7y2mauKBtF7165\nR7StTZs2dVNUkk2i1C6UFKTHu2fJBprc+fT7xoQdikjolBSkRzvY2Mw9L27g/OOGMnZI3yPe3uTJ\nk7shKsk2UWoXSgrSoy1esZlte+q5+qyx3bK9qqqqbtmOZJcotQslBenR7np+HaMH9eH84w7vDub2\nysvLu2U7kl2i1C6UFKTHWr5hJ0vX7eSzZ40hp5vuYL711lu7ZTuSXaLULpQUpMea/+xa+hfmceUZ\no8MORSRjpDQpmNl0M1ttZtVmdlOS9V8zs5Vm9oqZPWlmuvxD0mLd9nf4y4rNfOZ9Y+hXkBd2OCIZ\nI2VJwcxygVuAS4AJwFVmNqFdteVAmbufAjwI/DhV8YgkuvW5teTn5HDN2WO7dbu1tbXduj3JDlFq\nF6k8UjgDqHb3te5+ELgPuCyxgrs/5e774osvAEc2xoBIANv31vNgVQ0fOa2UYUWF3brtKF1lIukT\npXaRyqRQCmxMWK6Jl3XkWuDPyVaYWbmZVZpZ5bZt27oxROmJfvfcmxxsamb2eeO6fduzZs3q9m1K\n9EWpXaQyKSS7nMOTVjT7NFAG/CTZenef7+5l7l42dGj3XDooPdNbe+u56/l1zDilhGOH9Q87HJGM\nk8oethogcWD6kUBd+0pmdhHwHeB8d69PYTwizH92Lfsbmrhh6rFhhyKSkVJ5pLAUGG9m48ysF3Al\nUJFYwcxOA+YBs9x9awpjEWH73nruen49syal7ihh3rx5KdmuRFuU2kXKkoK7NwLXA4uBVcAD7r7C\nzG42s5YTbD8B+gF/MLOXzKyig82JHLF5z/yT+sYmvjJ1fMreI0p3rkr6RKldpPQCbXdfBCxqV/a9\nhOcXpfL9RVps3LGPO/+xng+fVsoxQ/ul7H3MDPekXWfSg0WpXeiOZukRfrx4NWbwjWnHhx2KSEZT\nUpCst3zDTha8XMfs9x9NycDeYYcjktGUFCSruTs/fHQVQ/oV8IULjkn5+82YMSPl7yHRE6V2oaQg\nWe1PL9VRtX4nX592XFrGOFqwYEHK30OiJ0rtQklBstbb+w7yg4UrmTRqIFeUjer6Bd1g5syZaXkf\niZYotQsNDylZ67/+8jpv72/gro9MJLeb5kvoysKFC9PyPhItUWoXOlKQrLR03Q7ufXEj1547jpNK\nBoQdjkhkKClI1nmnvpF/+8PLlA7szVcvSt2NaiLZSKePJOv88NFVrN+xj3s+/z769EpvE4/KDUqS\nXlFqFzpSkKzy+Mot3PviBsrPO5qzjhmc9vefP39+2t9TMl+U2oVFKYMBlJWVeWVlZdhhSAbatGs/\nM375N4YXFfLIv55NQV5u2mOI0nAGkj6Z0C7MrMrdy7qqpyMFyQr1jU188X+XcaChiV9edWooCUEk\nG6hPQbLC3IoVvLTxbX776cmaPEfkCOhIQSLv7ufXce+LG/nXDxzD9InFocZSUaHR3+W9otQulBQk\n0v7y2mbmVKxg6gnD+NoHwx8BdcqUKWGHIBkoSu1CSUEia8nat/jKfcs5ddRAfvXJyWm7a7kzpaWl\nYYcgGShK7UJJQSKpav1OPn9nJaMH9eG2q0+ndy91LIt0ByUFiZwX1r7FZ25bwpD+Bdx97Rkc1bdX\n2CGJZA1dfSSR8tTrW/ni76sYeVQf7vn8mQwrKgw7pDZmz54ddgiSgaLULnTzmkTGHX9/k5sXruTE\n4iLu/NwZDOlXEHZIIpGhm9ckaxxoaOI7j7zK3AUrmXricB647qyMTQhRuspE0idK7UKnjySjrd22\nl+vvWc7KTbu57vyj+ebFJ2TEVUYdWbZsWdghSAaKUrtQUpCM1Nzs/H7Jen7059cpyMvhtqvLmHri\n8LDDEsl6SgqScd7YsodvPfwqVet38v7xQ/jxx0+heEDvsMMKpLg43DuqJTNFqV0oKUjG2LzrAD9/\n/A3+ULWRot75/PTySXx0cilmmXu6qL26urqwQ5AMFKV2oY5mCd3mXQf4j0WruOC/n+Lh5TVcc/Y4\n/vr1C/jYlJGRSggAc+fODTsEyUBRahe6JFVC81rtLm7/+5tUvFRHszuzJpXw9WnHM2pQn7BDO2yZ\nMG6+ZJ5MaBdBL0nV6SNJq2176vnTS7U8WFXD65v30KdXLp9+3xiuPXdcpJOBSLZQUpCUcnf+uW0v\nj6/cyhOrtrBsw07cYdLIAfx5J63PAAAKRklEQVTgspOYNamUAX3yww5TROKUFKRbNTU71Vv3snTd\nDl58M/bYvPsAACeXDuCrU4/j0pNHMH54dk6Eo1ObkkyU2kVKO5rNbLqZrTazajO7Kcn6AjO7P75+\niZmNTWU8c+fOxcxaH1VVVVRVVbUpa+kQKikpaS1ruRuxvLy8Td26ujoWLFjQpqxlgu7EspkzZwIw\nc+bMNuUQm9A7sWzBggXU1dW1KSsvLwdid0W2lJWUlIT6mWbMmEnd2/s598KL25SfNOcvnH3Nt/nM\nWWO572ff5fRxg/g/H5nI89+6kAVfPpcbLhqftQlBJBukrKPZzHKBN4APAjXAUuAqd1+ZUOdLwCnu\n/gUzuxL4iLt/orPtqqM59Q42NvP2/oNs3V3Ptj31bNl9gK176tm65wCbdx1g/Vv7WL9jHwcbm1tf\nM6B3PieVFDGhuIgJJUWUjRnEmCF9Q+9cS7dM6FCUzJMJ7SITOprPAKrdfW08oPuAy4CVCXUuA+bG\nnz8I/MrMzMPeexnA3Wlqdhqb3/3Z2NSctKz98sGmZuobmjnQ0MT+hiYONDTHfyY+mtlb38ju/Q3s\n2t/A7gMN7N7fyK79DexvaEoa01F98hleVMi4IX35wAnDGDO4D2MH92XckL4UDyiM3OWjIvJeqUwK\npcDGhOUa4MyO6rh7o5ntAgYD27s7mPuXbmDeM2tpyTbujgPu4PFS99ijZT3Qpk7sJ611eU9Zwjbd\nW8vxlu14m9e32WbL+8bLGptTlxd75+dSmJ9Dn155DOidz4De+Ywb0pcBvfMpKowtD+iTz7D+hQwr\nKmBY/wKG9i+gIE8T2Yhku1QmhWT/Nrb/pgtSBzMrB8oBRo8efVjBDO5bwISSoti579btxgJILMPA\nsNZ179aLl9m7Yb/7+tj69ttsWY69wtrUJ/F9E7bfsvW83Bzycoy8XCMvx8jNyYn/NPJzO1/Oy7X4\nF39um58F+TkU5OWk7T/6OXPmpOV9MklP/MzStSi1i1T2KZwFzHX3i+PL3wJw9x8l1Fkcr/O8meUB\nm4GhnZ0+Up+CiMihy4T5FJYC481snJn1Aq4EKtrVqQCujj//OPBX9SeIiIQnZaeP4n0E1wOLgVzg\ndndfYWY3A5XuXgHcBtxtZtXADmKJQ0REQpLSm9fcfRGwqF3Z9xKeHwAuT2UMIiISnEZJFRGRVkoK\nIiLSSklBRERaKSmIiEgrJQUREWkVuZnXzGwbsP4wXz6EFAyh0Q0U16FRXIcuU2NTXIfmSOIa4+5D\nu6oUuaRwJMysMsgdfemmuA6N4jp0mRqb4jo06YhLp49ERKSVkoKIiLTqaUlhftgBdEBxHRrFdegy\nNTbFdWhSHleP6lMQEZHO9bQjBRER6UTWJQUzu9zMVphZs5mVtVv3LTOrNrPVZnZxB68fZ2ZLzGyN\nmd0fH/a7u2O838xeij/WmdlLHdRbZ2avxuulfBIJM5trZrUJsV3aQb3p8X1YbWY3pSGun5jZ62b2\nipk9YmYDO6iXlv3V1ec3s4L477g63pbGpiqWhPccZWZPmdmqePu/IUmdC8xsV8Lv93vJtpWC2Dr9\nvVjML+P76xUzm5yGmI5P2A8vmdluM/tquzpp219mdruZbTWz1xLKBpnZ4/HvosfN7KgOXnt1vM4a\nM7s6WZ1D4u5Z9QBOBI4HngbKEsonAC8DBcA44J9AbpLXPwBcGX/+W+CLKY73p8D3Oli3DhiSxn03\nF/hGF3Vy4/vuaKBXfJ9OSHFc04C8+PP/Av4rrP0V5PMDXwJ+G39+JXB/Gn53xcDk+PP+wBtJ4roA\nWJiu9hT09wJcCvyZ2KSD7wOWpDm+XGITfI0Ja38B5wGTgdcSyn4M3BR/flOydg8MAtbGfx4Vf37U\nkcSSdUcK7r7K3VcnWXUZcJ+717v7m0A1cEZiBYvNU3kh8GC86E7gw6mKNf5+VwD3puo9UuAMoNrd\n17r7QeA+Yvs2Zdz9MXdvjC++AIxM5ft1Icjnv4xY24FYW5pqKZ4D1d03ufuy+PM9wCpic6BHwWXA\nXR7zAjDQzIrT+P5TgX+6++HeFHvE3P1ZYnPKJEpsRx19F10MPO7uO9x9J/A4MP1IYsm6pNCJUmBj\nwnIN7/2jGQy8nfAFlKxOd3o/sMXd13Sw3oHHzKzKYvNUp8P18UP42zs4XA2yH1Ppc8T+q0wmHfsr\nyOdvrRNvS7uIta20iJ+uOg1YkmT1WWb2spn92cxOSlNIXf1ewm5TV9LxP2Zh7K8Ww919E8SSPjAs\nSZ1u33cpnWQnVczsCWBEklXfcfc/dfSyJGXtL70KUieQgDFeRedHCee4e52ZDQMeN7PX4/9RHLbO\n4gJ+A/yA2Gf+AbFTW59rv4kkrz3iS9iC7C8z+w7QCPy+g810+/5KFmqSspS1o0NlZv2Ah4Cvuvvu\ndquXETtFsjfeX/RHYHwawurq9xLm/uoFzAK+lWR1WPvrUHT7votkUnD3iw7jZTXAqITlkUBduzrb\niR265sX/w0tWp1tiNLM84KPAlE62URf/udXMHiF26uKIvuSC7jszuxVYmGRVkP3Y7XHFO9BmAFM9\nfjI1yTa6fX8lEeTzt9Spif+eB/DeUwPdzszyiSWE37v7w+3XJyYJd19kZr82syHuntIxfgL8XlLS\npgK6BFjm7lvarwhrfyXYYmbF7r4pfjpta5I6NcT6PlqMJNafeth60umjCuDK+JUh44hl/BcTK8S/\nbJ4CPh4vuhro6MjjSF0EvO7uNclWmllfM+vf8pxYZ+tryep2l3bncT/SwfstBcZb7CqtXsQOvStS\nHNd04EZglrvv66BOuvZXkM9fQaztQKwt/bWjRNZd4n0WtwGr3P1nHdQZ0dK3YWZnEPv7fyvFcQX5\nvVQAn41fhfQ+YFfLaZM06PBoPYz91U5iO+rou2gxMM3Mjoqf7p0WLzt86ehZT+eD2JdZDVAPbAEW\nJ6z7DrErR1YDlySULwJK4s+PJpYsqoE/AAUpivMO4AvtykqARQlxvBx/rCB2GiXV++5u4FXglXiD\nLG4fV3z5UmJXt/wzTXFVEztv+lL88dv2caVzfyX7/MDNxJIWQGG87VTH29LRadhH5xI7bfBKwn66\nFPhCSzsDro/vm5eJddifnYa4kv5e2sVlwC3x/fkqCVcNpji2PsS+5AcklIWyv4glpk1AQ/z761pi\n/VBPAmviPwfF65YBv0t47efiba0a+JcjjUV3NIuISKuedPpIRES6oKQgIiKtlBRERKSVkoKIiLRS\nUhARkVZKCiIi0kpJQUREWikpiBwhMzs9PohgYfwO3hVmNjHsuEQOh25eE+kGZvZDYncy9wZq3P1H\nIYckcliUFES6QXwcpKXAAWLDITSFHJLIYdHpI5HuMQjoR2zWs8KQYxE5bDpSEOkGZlZBbBa2ccQG\nErw+5JBEDksk51MQySRm9lmg0d3vMbNc4B9mdqG7/zXs2EQOlY4URESklfoURESklZKCiIi0UlIQ\nEZFWSgoiItJKSUFERFopKYiISCslBRERaaWkICIirf4/hVEzJ7ogOq0AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "x1 = np.linspace(-10, 10, 500)\n",
    "plt.plot(x1, activation(x1))\n",
    "plt.xlabel('x'); plt.ylabel('activation(x)')\n",
    "plot_point(activation, x1[150], -10, 10)\n",
    "plot_point(activation, x1[250], -10, 10)\n",
    "plot_point(activation, x1[470], -10, 10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Repeat but for the space -20 to 20"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEKCAYAAAD9xUlFAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4zLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvIxREBQAAIABJREFUeJzt3Xl4XPV97/H3V5vlFWNbBkm28YIx\nGMJiKWBCSoGQhMU2TckGTULT1OLeW3LT5qYNKS0W0OehSbqkt00TmzQNTQmUJG0iOXAJJEBCExNL\nxiwy2EhmsS3Zko0XyYvW7/1jRuOR0DIadObM0Xxez6NHM+ecOfPRzyN/dc7v/H7H3B0RERGAvLAD\niIhI9lBREBGRBBUFERFJUFEQEZEEFQUREUlQURARkQQVBRERSVBREBGRBBUFERFJKAg7wFjNmTPH\nFy5cGHYMEZFIqa+v3+/uJaNtF7misHDhQurq6sKOISISKWb2Rirb6fSRiIgkqCiIiEiCioKIiCSo\nKIiISIKKgoiIJARWFMzs22bWamYvDbPezOz/mlmjmb1gZiuCyiIiIqkJ8kjhO8A1I6y/Flga/6oC\nvhFgFhERSUFg4xTc/RdmtnCETW4A/s1j9wPdZGYzzazU3VuCyiQiA7k7x7t7OdrZy4nuXo519XK8\nu5djXT2c6O6lu9fp7XN6+pzevj56Bjw/ubz/rr6e2C94/FnyHX/7b/8bW8+A9Y4P3I9uFfw27zvn\nNC6YPzPQ9whz8Fo5sCvp+e74srcVBTOrInY0wYIFC9J+w+rqau66667E8/5BcJWVlYll69ato7q6\nmrKyMlpaYlFWrFhBfX09VVVV3HfffYlt9+zZQ319PWvWrEksW79+PVVVVZhZYtmqVauora1l9erV\nbNy4MbHc3dmwYQO33nprYllNTQ0VFRWUl5cnlq1du5YNGzZQUVHBli1bACgtLaW5uVk/k36mEX+m\n666/nq9seIBP3/RhNv/iiZPL/+EXvPzkf/Laj76WWFZy419SdNqZ7PnnWxLLpl3wQWZf81lavvM5\nuvY1AZA/bRbz/ujfOPTMAxz+7wcT255+S2xfe+//YwAmzT+P02/+a96JpB9FgLkzigMvCuYBVuP4\nkcJGdz9viHU/Ae5192fiz38G/Jm714+0z8rKSteIZpGhnejuZdPOAzy9o42tuw7R0HyErp6+xPrZ\nU4soP3Uys6cWMWvqJGZPK2LmlEKmTSpgcmE+k4vymVKUz+TCAiYX5VOYbxTk5ZGfZxTkWex7fvx7\nfHl+ntH/f7cZ9D+bMqmA4109ieUQW3fyMYkCNuD1qgSBMLN6d68cbbswjxR2A/OTns8DmkPKIhJZ\n7s7m1w/ywLNv8NOGfRzv7qW4MI/zy2dyy6Vn8K55M1lSMpUFs6YwvbgwY7nWr19PcWF+xt5PxkeY\nRaEGuM3MHgIuAQ6rP0Ekde7OYw37+NoTO3hlbzvTiwv40Ipy3r/8NC5dPDv0/5CrqqpCfX9JT2BF\nwcweBK4A5pjZbmAdUAjg7t8EHgGuAxqBY8Cng8oiMtE0tnbwxR++QP0bB1lSMpWv3Hg+qy4oZUpR\n9sxxaWYEeXpaghHk1Uc3jbLegT8K6v1FJiJ35zu/ep17H32FKUX5fPnGd3HjinkU5GscqoyP7Pmz\nQkRG1NXTx50/fomHNu/i6nPmcu/vnk/J9Elhx5IJRkVBJAK6evr4n/9ez89eaeWzV53Jn1x9Fnl5\n2X2VzqpVq8KOIGlQURDJcj29ffzJf2zlZ6+0cs/vnMcnV54RdqSU1NbWhh1B0qATkSJZ7q8ffYWf\nvNjCX1x/TmQKAsDq1avDjiBpUFEQyWKPvtjCt555jU9degZ/+FuLw44zJsmjwiU6VBREstSut47x\npz94gQvmz+SO688JO47kCBUFkSzk7tz545dwd75+80VMKtDIYMkMFQWRLPToS3t5cnsbn//AMuad\nOiXsOGnRwLVoUlEQyTLHunqormng3LIZ3HJpdDqWB9uwYUPYESQNKgoiWebffv0Gre2d3LXm3EiP\nVE6ealyiI7qfOJEJqKOzh/VPN/HbZ5VQuXBW2HEkB6koiGSR+3/1OgePdfMn7z8r7CiSo1QURLLE\nie5evvXLnVy5rIQLA767VibU1NSEHUHSoKIgkiU2vtDCwWPdrI3YILXhVFRUhB1B0qCiIJIlvrvp\nDZaUTOXSJbPDjjIuku9fLdGhoiCSBV7YfYjndx3ikyvP0D2KJVQqCiJZ4IFNbzKlKJ/frZgXdhTJ\ncSoKIiE70d3LIy+2cN27SplRXBh2nHGzdu3asCNIGlQUREL21PZW2jt7uOHCsrCjjCuNaI4mFQWR\nkNU+38KcaUVcunhidDD309VH0aSiIBKijs4ennh5H9e9qzTSU1oMZcuWLWFHkDRMrE+hSMQ8sW0f\nnT19rL5gYp06kuhSURAJ0WMNezltxiQqFpwadpRxV1paGnYESYOKgkhIunr6+OWr+7nq7Lnk5U28\nsQnNzc1hR5A0qCiIhKTu9bfo6OzhymVzw44SiOrq6rAjSBpUFERC8uT2Vory87jszDlhRwnEXXfd\nFXYESYOKgkhIfv5KK5csnsXUSQVhRxFJUFEQCcGbB47R1HZ0wp46kuhSURAJwdOvtgFwxbKSkJME\np66uLuwIkgYVBZEQbNp5gNJTilk0Z2rYUUQGCLQomNk1ZrbdzBrN7PYh1i8wsyfN7Dkze8HMrgsy\nj0g2cHee3XmAlYtnT+hpsisrK8OOIGkIrCiYWT7wdeBaYDlwk5ktH7TZXwAPu/tFwMeBfw4qj0i2\naGrrYH9HF5csmhV2FJG3CfJI4WKg0d13unsX8BBww6BtHJgRf3wKoNEuMuFt2vkWACsn2AR4MjEE\neS1cObAr6flu4JJB21QDPzWzzwJTgasDzCOSFTbtPMDpM4o5Y/aUsKMEat26dWFHkDQEeaQw1MlS\nH/T8JuA77j4PuA74rpm9LZOZVZlZnZnVtbW1BRBVJDPcnU0732Ll4lkTuj8BNKI5qoIsCruB+UnP\n5/H200OfAR4GcPdfA8XA24Z3uvsGd69098qSkol7CZ9MfDv3H2V/RyeX5MCpo7IyzfwaRUEWhc3A\nUjNbZGZFxDqSawZt8ybwPgAzO4dYUdChgExYW944CMC7F068WVEHa2lpCTuCpCGwouDuPcBtwGPA\ny8SuMmows7vNbE18s/8DrDWz54EHgd9398GnmEQmjOd2HWJ6cQGL50wLO4rIkAKddMXdHwEeGbTs\nzqTH24DLgswgkk22vnmIC+fPnJBTZQ+2YsWKsCNIGjSiWSRDjnX18MreI1w4f2bYUTKivr4+7AiS\nBhUFkQx5cfdh+hwuWpAbRaGqqirsCJIGFQWRDHlu1yEALpiXG0XhvvvuCzuCpEFFQSRDnnvzIAtm\nTWH2tElhRxEZloqCSAa4O8+9eShnTh1JdKkoiGRAa3snre2dOXPqCGDPnj1hR5A0qCiIZMBLew4D\ncF75KSEnyRxdfRRNKgoiGdDQfASAc0qnh5wkc9asWTP6RpJ1VBREMqCh+TCL5kxlenFh2FFERqSi\nIJIBDc1HWF42Y/QNRUKmoiASsEPHuth98Djn5lhRWL9+fdgRJA0qCiIB2xbvTzivLHc6mUEjmqNK\nRUEkYP2dzLl2pDDRbyI0UakoiASsofkwp88o1khmiQQVBZGANTQfybmjBIkuFQWRAHX29LJz/1HO\nKc29orBq1aqwI0gaVBREAtTUepTePmfZ6bkzaK1fbW1t2BEkDSoKIgHasa8dICeLwurVq8OOIGlQ\nURAJ0PZ97RTmGwtnTw07SsZt3Lgx7AiSBhUFkQDt2NvO4jnTKCrQr5pEgz6pIgF6ZW87Z+XgqSOJ\nLhUFkYC0n+hmz6HjnJ2jRcHdw44gaVBREAnIq60dAJx1Wm4WhQ0bNoQdQdKgoiASkB1741ce5WhR\nuPXWW8OOIGlQURAJyPZ97UwpymfeqZPDjiKSsoJUNjKzPOACoAw4DjS4+74gg4lE3fa97Sw9bTp5\neZoYTqJjxKJgZkuALwJXA68CbUAxcJaZHQPWA/e7e1/QQUWiZse+dq46e27YMUJTU1MTdgRJw2hH\nCn8FfAO41QddSmBmc4GbgU8C9wcTTySa9nd0sr+jK2c7mQEqKirCjiBpGLEouPtNI6xrBb427olE\nJoBcnt6iX3l5uS5LjaCUOprN7B4zK0h6PsPM/jW4WCLR1hi/HHXp3NwtChJNqV59VAA8a2bnm9kH\ngM1AfXCxRKKtqbWDaZMKOG2Gbqwj0ZJSUXD3LxHrcH4W+A5wvbv/02ivM7NrzGy7mTWa2e3DbPNR\nM9tmZg1m9r0xZBfJWk1tR1lSMjWnb0m5du3asCNIGlI9fXQ58A/A3cBTwD+ZWdkor8kHvg5cCywH\nbjKz5YO2WQp8CbjM3c8F/nisP4BINmpq62BJybSwY4RKI5qjKdXTR38DfMTd73X3m4ENwM9Hec3F\nQKO773T3LuAh4IZB26wFvu7uByHReS0SaR2dPbQcPsGSubldFHT1UTSlWhQudfdt/U/c/T+By0Z5\nTTmwK+n57viyZGcRG/Pw32a2ycyuGWpHZlZlZnVmVtfW1pZiZJFwvNZ2FIAlJbl3D4VkW7ZsCTuC\npGHEomBmnzCzPHfvHbzO3Q+Y2RIze+9wLx9i2eDr0wqApcAVwE3At8xs5hDvtcHdK929sqSkZKTI\nIqFraotdeXRmjh8pSDSNNnhtNvCcmdUTu9qof0TzmcBvA/uBITuQiR0ZzE96Pg9oHmKbTe7eDbxm\nZtuJFYnNY/khRLJJY2sH+XnGglm5faRQWloadgRJw4hHCu7+D8AK4EGgBHhf/Pke4JPufqO7vzrM\nyzcDS81skZkVAR8HBo97/xFwJYCZzSF2Omlnmj+LSFZoauvgjFlTcv5ua83Ng/8GlCgYdUK8+Kmj\nx+NfKXP3HjO7DXgMyAe+7e4NZnY3UOfuNfF1HzCzbUAv8KfufmCsP4RINmlq62Bxjl95BFBdXU11\ndXXYMWSMLJVh6GZWQuxKoYUkFRJ3/4PAkg2jsrLS6+rqMv22Iinp6e1j+Z2P8en3LuRL154TdpxQ\nmZmmucgiZlbv7pWjbZfS1NnAj4FfAk8Q+4teRIaw++Bxunr7cn6MgkRXqkVhirt/MdAkIhNA/5VH\nKgoSVan2hG00s+sCTSIyASQuR1VRQKd5oynVovA5YoXhhJm1x7+OBBlMJIqaWo8yZ9okTplSGHYU\nkbSkdPrI3TX/r0gKGts6cn4kc7/Kykp1NEdQqn0KmNka4PL406fcfWMwkUSiyd1pbO3g+vM1aEui\nK9VZUv+a2CmkbfGvz8WXiUjcW0e7OHy8W53MEmmpHilcB1zo7n0AZnY/8BzDT3EhknOaNBHeAOvW\nrQs7gqRhLOPwkyeqO2W8g4hEnS5HHUijmaMp1SOFe4lNjPcksdlPLyd2cxwRiWtq7aC4MI/ymZPD\njpIVysrKNP9RBKV69dGDZvYU8G5iReGL7r43yGAiUdPU1sHiOdPIy8vdW3Ama2lpCTuCpGG0+ymc\nHf++AiglNtX1LqAsvkxE4hrbOnL+bmsSfaMdKXweqAL+doh1Dlw17olEIuhEdy+7Dx7nxhXzwo6S\nNVas0N+NUTTa/RSq4g+vdfcrk7+IXZEUKdXV1ZhZ4qu+vp76+voBy/o7x8rKyhLL+u81W1VVNWDb\n5uZmamtrByzrv1l58rLVq1cDsHr16gHLIXZz8+RltbW1NDc3D1hWVRX7Z6ioqEgsKysr08+URT/T\n5KIC9j/6j+pkTlJfXx92BElDqlNnb3H3FaMtywRNnS3ZaOMLzdz2ved49+7v8/1//07YcbJCVVVV\novhK+FKdOnu0PoXTzawCmGxmF5nZivjXFcCUccoqEnlNrUcxgx88cH/YUbLGfffdF3YEScNofQof\nBH6f2P2V/y5peTvw5wFlEomcprYO5p06mdfDDiLyDo1YFNz9fuB+M7vR3X+YoUwikdPU1sGSkmk8\nE3YQkXco1XEKPzSz64FzgeKk5XcHFUwkKvr6nJ1tR1m5eDZ79uwJO07WUFtEU6oT4n0T+BjwWWKD\n1z4CnBFgLpHIaD58nOPdvSwpmaYrbpKoLaIp1bmP3uPunwIOuvtdwKXA/OBiiURH8kR4a9asCTlN\n9lBbRFOqReF4/PsxMysDuoFFwUQSiZbG1vgtODWaWSaAVCfE22hmM4GvAluIjWbW9WYixDqZZ04p\nZNbUorCjiLxjqXY03xN/+EMz2wgUu/vh4GKJREdjawdnlkzDzFi/fn3YcbKG2iKaUu1oft7M/tzM\nlrh7pwqCyElNrR2J6S36p+8QtUVUpdqnsAboAR42s81m9gUzWxBgLpFIOHi0iwNHuxL9Cf1zJYna\nIqpSKgru/oa7f8XdK4CbgfOB1wJNJhIB/XdbUyezTBSpdjRjZguBjxIbr9AL/FkwkUSiQ7fglIkm\npaJgZs8ChcD3gY+4+85AU4lERGNrB5MK8ig/NXYLzlWrVoWcKHuoLaIp1SOFW9z9lUCTiERQY2sH\ni+ZMJT9+C87a2tqQE2UPtUU0jTZ19ifiD68zs88P/hpt52Z2jZltN7NGM7t9hO0+bGZuZqPO9S2S\nTZrajg7oT+i/UY+oLaJqtI7mqfHv04f4GvEkqpnlA18HrgWWAzeZ2fIhtpsO/G/g2TElFwnZie5e\ndh08NqA/YePGjSEmyi5qi2gabers/tEnT7j7fyevM7PLRtn3xUBjf/+DmT0E3ABsG7TdPcBXgC+k\nGlokG7y2/yjuuvJIJpZUxyn8Y4rLkpUDu5Ke744vSzCzi4D57q4/KSRyNOeRTEQjHimY2aXAe4CS\nQX0IM4D8UfY91MiVxA2hzSwP+Htid3YbeUdmVUAVwIIFGjMn2aGprQMzWDRnamJZKvc8zxVqi2ga\n7UihiFjfQQED+xOOAB8e5bW7GTi99jygOen5dOA84Ckzex1YCdQM1dns7hvcvdLdK0tKSkZ5W5HM\naGztYP6pUyguPPn3kW5Uf5LaIposlWpuZme4+xtj2rFZAbADeB+wB9gM3OzuDcNs/xTwBXevG2m/\nlZWVXlc34iYiGXHN135B6SnF/OunL04sMzP9hRyntsguZlbv7qNe4Zlqn8K34lNn9+/8VDN7bKQX\nuHsPcBvwGPAy8LC7N5jZ3Wamu29IpPX2Oa/tP6r+BJlwUh28NsfdD/U/cfeDZjZ3tBe5+yPAI4OW\n3TnMtlekmEUkdHsOHqezp0/TW8iEk+qRQl/yrKjxeZB0XCg569XWduDtVx7V1NSEEScrqS2iKdUj\nhTuAZ8zs6fjzy4lfDSSSi7bvixWFs06fPmB5RUVFGHGyktoimlK989r/i18VVAVsBX7Myfs2i+Sc\nHXvbKTulmBnFhQOWl5eXq3M1Tm0RTanOkvqHwOeIXVa6ldjlo78Grgoumkj22r6v421HCSITQap9\nCp8D3g284e5XAhcBbYGlEsliPb19NLV2sOw0FQWZeFItCifc/QSAmU2KT6O9LLhYItnr9QNH6ert\nY9kQRwpr164NIVF2UltEU6odzbvj4xR+BDxuZgcZODpZJGds3xub8+isIY4UNIr3JLVFNKV6j+YP\nufshd68G/hL4F+B3ggwmkq2272snz4aeCE9X3JyktoimlO/R3M/dnx59K5GJa8fedhbOnjpgzqN+\nW7ZsCSFRdlJbRFOqfQoiErd9X/uQ/QkiE4GKgsgYnOju5fUDR4fsTwAoLS3NcKLspbaIJhUFkTFo\nbO3AnWGPFJqbdf1FP7VFNKkoiIzB9r3x6S2GOVKorq7OYJrspraIJhUFkTHYvq+dooI8Fs6eMuT6\nu+66K8OJspfaIppUFETG4OWWI5xZMo2CfP3qyMSkT7ZIitydbc1HOLdsRthRRAKjoiCSon1HOjlw\ntGvEoqBbxZ6ktogmFQWRFDU0Hwbg3PJTQk4iEhwVBZEUNTQfwQzOKR3+SKGyctT7oucMtUU0qSiI\npKih+TCLZk9l2qQxzw4jEhkqCiIpemnPEZark1kmOBUFkRQcOtbFnkPHObds5P6EdevWZShR9lNb\nRJOKgkgKtjUfAeC88pGPFDSK9yS1RTSpKIikoCFeFEY7UigrK8tEnEhQW0STioJICrbuPkT5zMnM\nmlo04nYtLS0ZSpT91BbRpKIgkoKtbx7iogUzw44hEjgVBZFRtB45wZ5Dx7lw/uhFYcWKFRlIFA1q\ni2hSURAZxXO7DgFw0YJTR922vr4+6DiRobaIJhUFkVFs3XWIwnxLaSK8qqqqDCSKBrVFNKkoiIzi\nuTcPsrx0BsWF+aNue99992UgUTSoLaJJRUFkBL19zgu7D6fUnyAyEQRaFMzsGjPbbmaNZnb7EOs/\nb2bbzOwFM/uZmZ0RZB6Rsdqxr51jXb0p9SeITASBFQUzywe+DlwLLAduMrPlgzZ7Dqh09/OBHwBf\nCSqPSDrq3jgIkPLlqHv27AkyTqSoLaIpyCOFi4FGd9/p7l3AQ8ANyRu4+5Pufiz+dBMwL8A8ImP2\n7M4DnD6jmAWzhr4n82C64uYktUU0BVkUyoFdSc93x5cN5zPAo0OtMLMqM6szs7q2trZxjCgyPHdn\n0863WLl4FmaW0mvWrFkTcKroUFtEU5BFYajfIh9yQ7NPAJXAV4da7+4b3L3S3StLSkrGMaLI8Jra\njrK/o5OVi2eHHUUkY4K8W8huYH7S83lA8+CNzOxq4A7gt929M8A8ImPy7GsHALhERUFySJBHCpuB\npWa2yMyKgI8DNckbmNlFwHpgjbu3BphFZMw27XyL02ZMYuHs1PoTANavXx9gomhRW0RTYEXB3XuA\n24DHgJeBh929wczuNrP+k41fBaYB3zezrWZWM8zuRDIq1p9wgJWLZ6fcnwAaxZtMbRFNgd5s1t0f\nAR4ZtOzOpMdXB/n+IulqajtKW3snlywa26kjM8N9yK6znKO2iCaNaBYZwlPbY2czf2vpnJCTiGSW\nioLIEJ7c3srSudOYn+L4BJGJQkVBZJCOzh5+89pbXHX23DG/dtWqVQEkiia1RTSpKIgM8syr++nu\nda5MoyjU1tYGkCia1BbRpKIgMsiTr7QyvbiAijPGPgne6tWrA0gUTWqLaFJREEnS1+c8ub2Vy5eW\nUJg/9l+PjRs3BpAqmtQW0aSiIJKk/s2DtLZ3cvXysZ86EpkIVBREktQ+38ykgjzev/z0sKOIhEJF\nQSSup7ePR15s4X3nzGXapPTGdWqw1klqi2hSURCJ+/XOA+zv6GLNBWVp72PDhg3jmCja1BbRpKIg\nEleztZlpkwq4Yln6/Qm33nrrOCaKNrVFNKkoiABHO3t49KW9fPDc0ykuzA87jkhoVBREgJrnm+no\n7OHmS+aPvrHIBKaiIAJ879k3WXbadFYsGPuAtWQ1NZr9vZ/aIppUFCTn1b3+Fi/uOczvrVwwpnsn\nDKWiomKcUkWf2iKaVBQk563/xU5mTinkwxXz3vG+ysvLxyHRxKC2iCYVBclpja3tPPHyPj618gym\nFAV6zymRSFBRkJz294+/ypTCfG55z8Kwo4hkBRUFyVkv7TnMT15s4TPvXcTsaZPGZZ9r164dl/1M\nBGqLaLKoDUWvrKz0urq6sGNIxLk7H1u/iVdb23n6z65kRnFh2JFEAmVm9e5eOdp2OlKQnPTjrc38\n5vW3+OI1Z49rQdAVNyepLaJJRUFyzt7DJ6iubeDC+TP5aOX4DlbbsmXLuO4vytQW0aSiIDmlr8/5\n/MNb6ezu4+8+egF5ee9sXILIRKNr8CSn/OPPG/lV0wG+fOO7WFwybdz3X1paOu77jCq1RTTpSEFy\nxg/qd/P3T+zgQxeVj/tpo37Nzc2B7DeK1BbRpKIgOeGxhr3c/sMXuOzM2Xz5xvPf8XQWw6murg5k\nv1GktogmXZIqE96Dv3mTO/7rRd41bybf/czFgV5+ama641ic2iK7pHpJqvoUZMI63tXLPT/Zxvee\nfZMrlpXwz7+3QlNZiIxCvyEyIf3y1TaqaxpoajvKrZcv5gsfXEZhvs6WioxGRUEmDHfnN6+9xTee\nbuKp7W0smDWFf//MJbx36ZyMZdCpzZPUFtEU6J9OZnaNmW03s0Yzu32I9ZPM7D/i6581s4VB5qmu\nrsbMEl/19fXU19cPWNbfOVZWVpZY1j8ys6qqasC2zc3N1NbWDljWf7Py5GWrV68GYPXq1QOWQ+zm\n5snLamtraW5uHrCsqqoKiI0Q7V9WVlamn2nQzzRzwdl8bMMmnnnycW6/9mwe//zlGS0IIhNBYB3N\nZpYP7ADeD+wGNgM3ufu2pG3+F3C+u/8PM/s48CF3/9hI+1VHc2473tXLjn3tvLL3CM+9eYhf7zzA\nGweOAbBiwUxurJjHJ1YuDK2DU52rJ6ktsks2dDRfDDS6+854oIeAG4BtSdvcAFTHH/8A+CczM9cn\nacLq63O6evvo6u2juyf2vasn9nW0q5cjx7s5fLybIydi3w8d66bl8AmaDx2n5dBxWo6coP/TMb24\ngEsWzeZTly7kA8tPY/6sKQB8IsSfTyTqgiwK5cCupOe7gUuG28bde8zsMDAb2D/eYR7evIsNv9xJ\n/L0GrPNhngyuTMmv8wHLk18zaN8+9OPBUtn34P37iFkHviq116SYYZjtBu8w+WmfO929fXT3jq3e\nF+XnUTqzmLJTJrNyyWwWzJrC2afP4JzS6cw/dYqmqRAZZ0EWhaF+Wwf/j5DKNphZFVAFsGDBgrTC\nnDq1iGWnTR/2nZOfJg9sGhwweczTcK952+sGvCZp3yNmGPo1b1s33BuNmDXFfQ9YPvx/vqnsz4DC\ngjyK8vMoSv6e9HhKUT6nTC5kxuTC2PfiQooL88Y80GzdunVj2n48hfne2UZtEU1B9ilcClS7+wfj\nz78E4O73Jm3zWHybX5tZAbAXKBnp9JH6FERExi4b7qewGVhqZovMrAj4OFAzaJsa4Jb44w8DP1d/\ngohIeAI7fRTvI7gNeAzIB77t7g1mdjdQ5+41wL8A3zWzRuAtYoVDRERCEujgNXd/BHhk0LI7kx6f\nAD4SZAYREUmdxv2LiEiCioKIiCSoKIiISIKKgoiIJKgoiIhIQuTuvGZmbcAbab58DgFMoTEOlGts\nlGvssjWbco3NO8l1hruXjLbb9PtpAAAE1ElEQVRR5IrCO2FmdamM6Ms05Rob5Rq7bM2mXGOTiVw6\nfSQiIgkqCiIikpBrRWFD2AGGoVxjo1xjl63ZlGtsAs+VU30KIiIyslw7UhARkRHkRFEws6+a2Stm\n9oKZ/ZeZzUxa9yUzazSz7Wb2wQzn+oiZNZhZn5lVJi1faGbHzWxr/Oub2ZArvi609hqUo9rM9iS1\n0XVhZYnnuSbeJo1mdnuYWZKZ2etm9mK8jUK7EYmZfdvMWs3spaRls8zscTN7Nf791CzJFfpny8zm\nm9mTZvZy/Hfxc/HlwbeZu0/4L+ADQEH88ZeBL8cfLweeByYBi4AmID+Duc4BlgFPAZVJyxcCL4XY\nXsPlCrW9BmWsBr4Q9mcrniU/3haLgaJ4Gy0PO1c82+vAnCzIcTmwIvlzDXwFuD3++Pb+38ssyBX6\nZwsoBVbEH08HdsR//wJvs5w4UnD3n7p7T/zpJmBe/PENwEPu3unurwGNwMUZzPWyu2/P1PulaoRc\nobZXFrsYaHT3ne7eBTxErK0kzt1/QeyeKcluAO6PP74f+J2MhmLYXKFz9xZ33xJ/3A68TOye9oG3\nWU4UhUH+AHg0/rgc2JW0bnd8WTZYZGbPmdnTZvZbYYeJy7b2ui1+SvDbYZx6SJJt7ZLMgZ+aWX38\nXufZ5DR3b4HYf4LA3JDzJMuWzxZmthC4CHiWDLRZoDfZySQzewI4fYhVd7j7j+Pb3AH0AA/0v2yI\n7cf1cqxUcg2hBVjg7gfMrAL4kZmd6+5HQs4VeHsNeLMRMgLfAO6Jv/89wN8SK/hhyGi7jNFl7t5s\nZnOBx83slfhfxzK8rPlsmdk04IfAH7v7EbOhPmrja8IUBXe/eqT1ZnYLsAp4n8dPyBH7i25+0mbz\ngOZM5hrmNZ1AZ/xxvZk1AWcB49ZRmE4uMtBeyVLNaGb3ARuDypGCjLbLWLh7c/x7q5n9F7FTXdlS\nFPaZWam7t5hZKdAadiAAd9/X/zjMz5aZFRIrCA+4+3/GFwfeZjlx+sjMrgG+CKxx92NJq2qAj5vZ\nJDNbBCwFfhNGxmRmVmJm+fHHi4nl2hluKiCL2iv+C9HvQ8BLw22bAZuBpWa2yMyKiN1rvCbEPACY\n2VQzm97/mNgFF2G202A1wC3xx7cAwx2hZlQ2fLYsdkjwL8DL7v53SauCb7Mwe9gz2JPfSOyc79b4\n1zeT1t1B7MqR7cC1Gc71IWJ/ZXYC+4DH4stvBBqIXcWyBVidDbnCbq9BGb8LvAi8EP9FKQ35M3Yd\nsStEmoidggstS1KmxfHP0PPxz1NouYAHiZ0W7Y5/tj4DzAZ+Brwa/z4rS3KF/tkC3kvs9NULSf9v\nXZeJNtOIZhERSciJ00ciIpIaFQUREUlQURARkQQVBRERSVBREBGRBBUFERFJUFEQEZEEFQWRd8jM\n3h2fPK04Poq4wczOCzuXSDo0eE1kHJjZXwHFwGRgt7vfG3IkkbSoKIiMg/icR5uBE8B73L035Egi\nadHpI5HxMQuYRuwuWcUhZxFJm44URMaBmdUQu+PaImITqN0WciSRtEyY+ymIhMXMPgX0uPv34lOe\n/8rMrnL3n4edTWSsdKQgIiIJ6lMQEZEEFQUREUlQURARkQQVBRERSVBREBGRBBUFERFJUFEQEZEE\nFQUREUn4/03lN8S2KGKUAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "x2 = np.linspace(-20, 20, 500)\n",
    "plt.plot(x2, activation(x2))\n",
    "plt.xlabel('x'); plt.ylabel('activation(x)')\n",
    "plot_point(activation, x1[150], -20, 20)\n",
    "plot_point(activation, x1[250], -20, 20)\n",
    "plot_point(activation, x1[470], -20, 20)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Explore the derivative"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Text(0,0.5,'activation(x)')"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEKCAYAAAD9xUlFAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4zLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvIxREBQAAIABJREFUeJzs3Xd4VFX6wPHvmfROIAkllASkSUIo\noakUBQGxgEqzd0SWXX669sq6q651dS2LoIKuLqAiRUVEOioCoYmE0AMpEAIJSUjPzPn9cZMQQsok\nmZtJyPt5nnkmM/fec99AknfuPee8R2mtEUIIIQAszg5ACCFEwyFJQQghRClJCkIIIUpJUhBCCFFK\nkoIQQohSkhSEEEKUkqQghBCilCQFIYQQpSQpCCGEKOXq7ABqKigoSIeFhTk7DCGEaFS2bdt2Smsd\nXN1+jS4phIWFERMT4+wwhBCiUVFKHbVnP7l9JIQQopQkBSGEEKUkKQghhCjV6PoUKlJYWEhiYiJ5\neXnODkU0Ap6enrRt2xY3NzdnhyJEg3NRJIXExET8/PwICwtDKeXscEQDprXm9OnTJCYmEh4e7uxw\nhGhwTLt9pJT6RCl1Uin1RyXblVLq30qpg0qp35VSfWp7rry8PFq0aCEJQVRLKUWLFi3kqlKISpjZ\npzAPGF3F9muAzsWPKcB/6nIySQjCXvKzIkTlTLt9pLXeoJQKq2KXscBn2lgP9DelVDOlVGut9XGz\nYhLCFNZCiPseTh2ANr2h8whnR3TR01pTYLWRV2gjv9BKgdWG1aYptOriZxtFNo3VZjv/Pasufl9j\n08YDML62gS7+Gl38ngaN8Uzxa5vW6OLn0mM16OJ9jfgqj7v8dn3e9pL39AXvAQzvHkLPts3q8k9X\nLWf2KYQCCWVeJxa/d0FSUEpNwbiaoH379vUSnJnWrVuHu7s7l112GQCzZs3C29ubO++8s8ZtzZs3\nj5EjR9KmTRsA7r//fh555BEuvfRSh8YsKpF1Ar6YACd+N14Pe0qSQjW01mTmFnEyK4/UrHxOZxeQ\nmVdIZm4RmXmFZJ33dRG5BVbyiqzkF9rILbSSV/ywNcHl5YP9PC7qpFDRNXyF/81a69nAbIDo6OhG\n/6Owbt06fH19S5PC1KlTa93WvHnziIiIKE0KH330kUNiFHbITYdPRsHZVDaNeg7a9mNQ64HGtpQ9\n0KwDePg6N0YnsNk0SWdyOZaWw7G0HI6eziEhLYekM7mkZuWTmpVPgdVW4bGuFkWAlxt+nq74Fz8H\nervh6eZS/LDg6Wp87eXugoerBU83F9xdLLhYFK4uCrfir91cFC4WC24Whet57ylcLRZcLAAKiwKL\nUqjiZwCLxXhfFW+neJtFKRTF+ynOO1ZhPJc47+syf+5K3i/7B7DsLU1Vbr/y283mzKSQCLQr87ot\nkOykWBxi3LhxJCQkkJeXx4wZM5gyZQorVqzg6aefxmq1EhQUxMcff8ysWbNwcXHh888/591332X1\n6tX4+vpy7bXXctddd7FlyxYA4uPjueGGG/j999958cUX+fbbb8nNzeWyyy7jww8/ZNGiRcTExHDb\nbbfh5eXFpk2buOaaa3jjjTeIjo5m/vz5vPzyy2itufbaa3n11VcB8PX1ZcaMGXz33Xd4eXmxdOlS\nWrZs6cx/usbp2G9w9iTcsYTZsbMgYw+D2g2FnDT45Brodi3cWKeusgavoMjGnuQM/kjKIPZ4FnEn\nMtl3IoucAmvpPm4uiraB3oQ286JjkA/B/h4E+3oQ7OdBiJ8nQb7u+Hu54e/phqebRfp8nMyZSWEZ\nMF0ptQAYAGQ4oj/hb9/uITY5s87BlXVpG39euL5Htft98sknNG/enNzcXPr168fYsWN54IEH2LBh\nA+Hh4aSlpdG8eXOmTp2Kr68vjz76KACrV68GoHv37hQUFHD48GE6duzIwoULmThxIgDTp0/n+eef\nB+COO+7gu+++Y/z48bz33nulSaCs5ORknnjiCbZt20ZgYCAjR45kyZIljBs3juzsbAYOHMhLL73E\n448/zpw5c3j22Wcd+U/WNHS9Bv7vD/BpAbGzzr3v3Rz63Qc/vwV97oAOlzkvRgcrtNqIiU/n54Op\nxMSnszPhDPlFxqf+AC83urXyY2J0O7q09COshTftW3jTOsALF4v8oW8sTEsKSqn5wDAgSCmVCLwA\nuAForWcBy4ExwEEgB7jHrFjqy7///W8WL14MQEJCArNnz2bIkCGl4+GbN29ebRsTJ07kyy+/5Mkn\nn2ThwoUsXLgQgLVr1/Laa6+Rk5NDWloaPXr04Prrr6+0na1btzJs2DCCg42iiLfddhsbNmxg3Lhx\nuLu7c9111wHQt29ffvrppzp9302OzQrHNkGHy42EUJEhj8Lur+CHJ+DBDeffC2hksvIK+Sk2hdV7\nT7LhQCpZeUW4WhQ92vhz24AO9AsLJKpdM1oHeMqn/IuAmaOPbqlmuwb+5Ojz2vOJ3gzr1q1j1apV\nbNq0CW9vb4YNG0ZUVBT79u2rUTuTJk1iwoQJ3HTTTSil6Ny5M3l5eUybNo2YmBjatWvHzJkzqx1n\nrysb/gC4ubmV/vK6uLhQVFRUoxibvD8WwTcPwJ1LoeOwivdx94FhT8LSP8HB1Y2u89lq02zYn8o3\nO5JYuecE+UU2Qvw8GBPRmiu7hXBF5yB8PS6Kua+iHPlfdZCMjAwCAwPx9vYmLi6O3377jfz8fNav\nX8+RI0fOu33k5+dHZmbFt7g6deqEi4sLf//735k0aRJAaQIICgri7NmzfP3114wfPx4APz8/srKy\nLmhnwIABzJgxg1OnThEYGMj8+fP585//bNJ334RoDZveh6CuEDak6n0jJ8KG1yE1rtEkhay8Qr6K\nSWTer/EcS8sh0NuNidHtGNe7Db3bBWKR20AXPUkKDjJ69GhmzZpFz5496dq1KwMHDiQ4OJjZs2dz\n0003YbPZCAkJ4aeffuL6669n/PjxLF26lHffffeCtiZNmsRjjz3GkSNHAGjWrBkPPPAAkZGRhIWF\n0a9fv9J97777bqZOnVra0VyidevWvPLKK1x55ZVorRkzZgxjx441/x/iYpcYA8d3wpg3wHJu7ufz\ng56/cF9Xd/jTVuO5gcvKK+Tjn4/w8cYjZOUX0bdDII+P7srIS1vh7ip1M5sSVdVthoYoOjpal19k\nZ+/evXTv3t1JEYnGqNY/M99MgX0/wCN7azbcNPsU+ATV/Hwmyy+y8tmvR/lg3UHScwoZ1aMl04Zd\nQlQ7c8fCi/qnlNqmtY6ubj+5UhDCXjYrHP0VIm66ICGsS1gHwLB2wy48bvWLEPMJ/HUfuHqYH6ed\nNh06zbNLdnMoNZvBnYN4dGRXSQZCkoIQdrO4wJ+3Q8HZCzZ9uudToJKk0OFy2Pgm7P8RLr3B5CCr\nl51fxIvfxrIwJoF2zb2Ye08/ruwa4uywRAMhSUGImnB1B9fqhxafJ3woeLeA2CVOTwo7E87wfwt2\ncDQth6lDOzFjeGe83F2cGpNoWKQHSQh75KbD+wPgQC3mdLi4QvfrYd8KKMx1fGx2+vy3o4z/z68U\nFNlY8MBAnrymmyQEcQFJCkLY4+BqY2ipZy3vufe4EQqza5dU6qjQauPZJbt5dskfXNE5iB9mDGFA\nx0om3YkmT24fCWGPfcvBJxhC+9bu+A5XwE1zIOwKx8ZVjZyCIqZ+vp0N+1N5cGhHHh/VTUpOiCpJ\nUhCiOkUFcGCV0R9gqfji+pXBr1Tdhosr9JxoQnCVy8gt5L55W9l+LJ1/3hTJ5P6Nv+y8MJ/cPnKC\ndevW8euvv5a+njVrFp999lmt2po3bx7JyeeKy95///3ExsbWOcbK+PrWvBT0mDFjOHPmTI2PO3Pm\nDB988EHp6+Tk5NKZ3PXq2K+QnwFdx1S6SyufVrTyaVV1O7ln4LdZkLrfwQFeKCO3kNs++o1diWd4\n79Y+khCE3SQpOEH5pDB16tRaLbADFyaFjz76qMEssKO1xmazsXz5cpo1q/m9+PJJoU2bNnz99deO\nDNE+ngFGyYqOwyrdZcWRFaw4sqLqdmxFsOJJYxSSiXIKirh33lb2nchi9p3RjIlsber5xMXl4kwK\nc6+98LFljrGtIKfi7Tu+MLZnn75wm53GjRtH37596dGjB7NnzwZgxYoV9OnTh6ioKIYPH058fDyz\nZs3iX//6F7169WLjxo3MnDmTN954g71799K/f//S9uLj4+nZsycAL774Iv369SMiIoIpU6agtebr\nr78uXU+hV69e5ObmMmzYMEpmfM+fP5/IyEgiIiJ44oknStv19fXlmWeeISoqioEDB5KSklLp93Tk\nyBEGDRpEv379eO65587b9vrrr9OvXz969uzJCy+8UBpz9+7dmTZtGn369CEhIYGwsDBOnTrFE088\ncd4f+ZkzZ/Lmm29y9uxZhg8fTp8+fYiMjGTp0qUAPPnkkxw6dIhevXrx2GOPER8fT0REBGDUdtqz\nZ09pW8OGDWPbtm1kZ2dz77330q9fP3r37l3aVp206Q03zwF370p3WbhvIQv3Lay6HZ8gCO0DB1bW\nPaZKFBTZmPr5dnYcS+edyb1l/oGosYszKTjJJ598wrZt24iJieHf//43KSkpPPDAAyxatIhdu3bx\n1VdfERYWxtSpU3n44YfZuXMngwcPLj2+7HoKwAXrKWzdupU//viD3Nzc0vUUoqOj+eKLL9i5cyde\nXl6lbZWsp7BmzRp27tzJ1q1bWbLE+IRasp7Crl27GDJkCHPmzKn0e5oxYwYPPfQQW7dupVWrc7dH\nVq5cyYEDB9iyZQs7d+5k27ZtbNiwAYB9+/Zx5513smPHDjp06FB6zOTJk0tLgQN8+eWXTJgwAU9P\nTxYvXsz27dtZu3Ytf/3rX9Fa889//pNOnTqxc+dOXn/99fPimjx5Ml9++SUAx48fJzk5mb59+/LS\nSy9x1VVXsXXrVtauXctjjz1GdnZ2zf4jy8o9A6cPVb7obk11HmnUT8o+7Zj2ytBa88zi3WzYn8or\nN0XKFYKolYuzo/me7yvf5u5d9XafFlVvr8LFuJ7CL7/8wqJFiwBjcZ+SK46VK1eycuVKevfuDcDZ\ns2c5cOAA7du3p0OHDgwcOPCCtnr37s3JkydJTk4mNTWVwMBA2rdvT2FhIU8//TQbNmzAYrGQlJRU\n5dVLyb/T1Vdfzd/+9rfS5FIS17Jly3jjjTcAo8LssWPHal8bK+57WDoNHtoELR1wW+6Sq2HdK3B4\nLUQ6tn/kk1/i+WpbIn8Z3plJ/aQPQdTOxZkUnOBiXk+hooVTtNY89dRTPPjgg+e9Hx8fj4+PT6Vt\njR8/nq+//poTJ04wefJkAL744gtSU1PZtm0bbm5uhIWFVfv9hYaG0qJFC37//XcWLlzIhx9+WBrX\nokWL6Nq1a5XH2+3IemMoanA3x7TXOgq8AiH9iGPaK7ZhfyovfR/L6B6t+L/hnR3atmha5PaRg1S3\nngJAWloaUPkaCGD/egolqlpPYf369Zw6dQqr1cr8+fMZOnRojb+vyy+/nAULFgDGH+8So0aN4pNP\nPuHsWaMOUFJSEidPnqy2vcmTJ7NgwYLz1oTIyMggJCQENzc31q5dy9GjR6v83sq29dprr5GRkUFk\nZGRpXO+++25pUtyxY0eNv+dSWsPh9RA+pNKhqDXm4gqPxMGQxxzTHpB8Jpe/LNhBl5Z+vDkxStY8\nEHUiScFBRo8eTVFRET179uS55567YD2FqKio0j/y119/PYsXLy7taC5v0qRJfP7556X9CWXXUxg3\nblyF6ymUdDSXKLueQlRUFH369KnVegrvvPMO77//Pv369SMjI6P0/ZEjR3LrrbcyaNAgIiMjGT9+\nfJV/wEv06NGDrKwsQkNDad3auOd92223ERMTU9o/0q2b8am8RYsWXH755URERPDYYxf+ER0/fjwL\nFiwo/XcCeO655ygsLKRnz55ERERc0DleI6n74OyJKkcdlXhr2Fu8Newt+9p186x9TOVYbZqHF+6k\noMjGf27vi4+shibqSNZTEE2SXT8zmz+EHx6HGb9DYIeq962J7FPw9b0QfY9R/qIO3l19gDd/2s+b\nE6K4uW9bBwUoLkaynoIQdRVxMzTrYFdCWHLQGNk17pJx1bfrFWis3nZwdZ2Sws6EM7y9+gBje7Xh\npj6htW5HiLIkKQgAXnrpJb766qvz3pswYQLPPPOMkyJqAHyCoOtou3ZdetCYD2FXUrC4GLWQ4i+8\ndWivgiIbT3z9OyF+Hvx9XESFgwGEqA1JCgKAZ555pmkngPIykmDPN8ZMZr+Wjm8/fDDs+x4yEiGg\n5rd9/rPuEPtSsvj4rmj8Pd0cH59osqSjWYiKHF4HK5+F3DRz2m9fPI8jYXONDz2QksV7aw9wfVQb\nhnc3IWGJJk2SghAVObbJWDshyEHzHcprGWnMbvbwr9FhxqzlP/DxcOWF6xtGjStxcZHbR0JUJGGz\n8WneUfMTynNxhdu+qn6/cr7ffZwt8Wm8fGMkQb4eJgQmmjq5UjBBSYG72rjsssuq3F7bMtTl3X33\n3fVecbQ2ZbedIvsUnNoP7QbYfcgHIz7ggxEfVL9jeflZYC20a9e8QiuvLI+je2t/JvVrV/NzCWEH\nSQoNTNmS2hWpbRnqxqak7LZTnNwLFldoP8juQ7xcvfBy9ap+x7Lif4F/todjv9m1+5wNh0k6k8sL\n118qq6cJ00hScJCXXnqJrl27MmLEiPPqHR06dIjRo0fTt29fBg8eTFxcHAApKSnceOONREVFERUV\nVZoMSj5NHz9+nCFDhtCrVy8iIiJKZz6XlKEGeOutt4iIiCAiIoK3334bOFe6+oEHHqBHjx6MHDny\nvJnOZa1atYrBgwfTpUsXvvvuO8AoqXHPPfcQGRlJ7969Wbt2LWCs2zB9+vTSY6+77jrWrVtXGnNF\npbgrK7tdWans8mW3//73v/Pwww+XHjdnzhweeeSRGv/f1Fj4YHjyGLTtV/2+xRbELWBB3IKanSek\nO2ibXZ3NKZl5fLDuENdEtGKgrK8sTHRR9incs+KeC94bFTaKyd0mk1uUy7RV0y7YPvaSsYy7ZBzp\neek8su78PzxzR8+t8nzbtm1jwYIF7Nixg6KiIvr06UPfvsZavlOmTGHWrFl07tyZzZs3M23aNNas\nWcNf/vIXhg4dyuLFi7FaraU1hEr873//Y9SoUTzzzDNYrVZycnIuOOfcuXPZvHkzWmsGDBjA0KFD\nCQwM5MCBA8yfP585c+YwceJEFi1axO23335B3PHx8axfv55Dhw5x5ZVXcvDgQd5//30Adu/eTVxc\nHCNHjmT//qpXCispxf3SSy/x+OOPM2fOHJ599tnSstt33nlnabtAaalsf39/Tp06xcCBA7nhhhsA\no+z23Llz+eCDD8jOzqZnz5689tpruLm5MXfu3NLCd6Zzr7yoX0V+jP8RgMndJtt/kHdzo9BewpZq\nd31/7UEKrTaevMZBhfmEqIRcKTjAxo0bufHGG/H29sbf37/0D9zZs2f59ddfmTBhAr169eLBBx/k\n+PHjAKxZs4aHHnoIMCqVBgQEnNdmv379mDt3LjNnzmT37t34+fmdt/3nn3/mxhtvxMfHB19fX266\n6abSq4nw8HB69eoFGKWx4+PjK4x74sSJWCwWOnfuTMeOHYmLi+Pnn3/mjjvuAKBbt2506NCh2qRQ\nvhR3yfl++eUXbrnlFoDSNsG4NfT000/Ts2dPRowYcV6p7LJlt318fLjqqqv47rvviIuLo7CwsLTw\nnWkK8+DT6+HgKnPPU6Jdf0jcAlXcKktIy2H+lmNM7NeODi1qlqyEqClTrxSUUqOBdwAX4COt9T/L\nbW8PfAo0K97nSa318rqet6pP9l6uXlVuD/QMrPbKoCIVzSi12Ww0a9aMnTt31ri9IUOGsGHDBr7/\n/nvuuOMOHnvssfOW7KyqZpWHx7lRKS4uLpXePiofs1Kq0nZdXV3Pu8dftrR1VaW4K/p3qapUdvmy\n2/fffz8vv/wy3bp14557LrwCdLiUPXBkA/S73/xzgdGZvf0zOH0Agise/vrumgMopfjzVZfUT0yi\nSTPtSkEp5QK8D1wDXArcopQqP7D6WeBLrXVvYDJQi+EbzjdkyBAWL15Mbm4uWVlZfPvttwD4+/sT\nHh5eWj5Ca82uXbsAGD58OP/5z38AsFqtZGZmntfm0aNHCQkJ4YEHHuC+++5j+/btF5xzyZIl5OTk\nkJ2dzeLFi89bxc0eX331FTabjUOHDnH48GG6du3KkCFDSktk79+/n2PHjtG1a1fCwsLYuXMnNpuN\nhIQEtmyp/pZHZWW3KyuVXZEBAwaQkJDA//73v9KrDlMlF/87t+lj/rnAqMB6zWvgVfECTIdTz7Jo\nexK3D+hA64AadmQLUQtm3j7qDxzUWh/WWhcAC4DytZs1UDJ7JwBIphHq06cPkyZNolevXtx8883n\n/XH+4osv+Pjjj4mKiqJHjx6lnarvvPMOa9euJTIykr59+5633jAYi/b06tWL3r17s2jRImbMmHHB\nOe+++2769+/PgAEDuP/++0tXQbNX165dGTp0KNdccw2zZs3C09OTadOmYbVaiYyMZNKkScybNw8P\nDw8uv/xywsPDiYyM5NFHH6VPn+r/aFZWdruyUtmVmThxIpdffjmBgYE1+v5qJXmHsahOLUpP1EpA\nWxjwIPgGV7j53TUHcXexMO3KTvUTj2jyTCudrZQaD4zWWt9f/PoOYIDWenqZfVoDK4FAwAcYobXe\nVlW7Ujq76bnuuut4+OGHGT58uMParPRn5v2B0KxdrSaW1VpmsnHbqvPV572dkJbDsDfWce/lYTxz\nrcxeFnVjb+lsM68UKhpIXT4D3QLM01q3BcYA/1VKXRCTUmqKUipGKRWTmppqQqiiITpz5gxdunTB\ny8vLoQmhUjabkRDCa75CXZ3EzIX/TYSC7PPe/mjjYSwK7ruiY/3GI5o0MzuaE4Gy0y7bcuHtofuA\n0QBa601KKU8gCDhvXUet9WxgNhhXCmYFLBqWZs2aVTvyyaEsllpfIcz7Yx4Ad0fcXfODQ/sa8xWO\n74IOxoz2U2fzWbA1gZt6t6VVgONWahOiOmZeKWwFOiulwpVS7hgdycvK7XMMGA6glOoOeAK1uhRo\nbCvICeep9GelDjOo1yeuZ33i+todHFrcP5N0bjDBvF/iKbDamDJUrhJE/TItKWiti4DpwI/AXoxR\nRnuUUi8qpW4o3u2vwANKqV3AfOBuXYu/7p6enpw+fVoSg6iW1prTp0/j6VnBp+8lU+HTGy5832y+\nIRDQDpKM7rTs/CI+2xTP6B6t6BTcSOpFiYuGqfMUiuccLC/33vNlvo4FLq/redq2bUtiYiLS3yDs\n4enpSdu2FYwuSowxSk84Q2if0uGw32xPJDOviPsHy1WCqH8XRZkLNzc3wsPDnR2GaMxyz0DaIeh1\nq3POf9Vz4OKGzaaZ92s8UW0D6NP+4i98KBqeiyIpCFFnx41JhbSp2VyPEh6udVzbIKgzAD/vT+VQ\najb/mhQl6y4Lp5CkIASUmclcu6Qwa8Ssusew9SN2bT1LkG8EYyJb1709IWpBkoIQYCyPOfBPRuVS\nJynY9CGXpvpw65AxeLi6OC0O0bRJUhACoPMI41FLs3YZVwpTo6bWuo296hKiLD8T2V9WVRPOI6Wz\nhSjIgfR4qMOQ5s3HN7P5ePWL5VQmv8jK8rQ2BKlMQvSpWrcjRF1JUhDi6C/wTpTx7CSrYk+yKa+D\n8SKpyvJfQphKkoIQSdsBBa16Oi2EBVuPkenfFe3qaVy1COEk0qcgRPIOY0iop3/1+5ogIS2HjQdO\n8X8jOqOGxoObrJsgnEeSghDJO6Bj3SqjNvOo/USzr2ISUAomRLeThCCcTpKCaNoyk+HsiTqvtPav\nK/9Vq+OKrDa+jElkaJdgQpt5GZPoVv0NxrwOLWRhHVH/pE9BNG0efjBhHnQZ5ZTTbziQyonMPCb3\nKx6GanGDQ6uNOkxCOIEkBdG0efhBjxuhed1qZ7297W3e3vZ2jY/7KiaRFj7uXNWtpfFGUBdw8zZu\naQnhBHL7SDRtcd9Dsw7QKqJOzexK3VXjYzJyC1m99yS3DmiPu2vx5zMXV2MUVPL2qg8WwiRypSCa\nLq1h6XTY7IC6RbXww+7jFFht3Ng79PwNoX3g+O9gLXJKXKJpk6Qgmq4zRyE37dzKZ/Vs8Y4kOgb7\n0LNtwPkb2g+CttFGbELUM7l9JJqukvv2tayMWhdJZ3LZfCSNv17d5cIS2ZfeYDyEcAJJCqLpStoO\nLu4Q0qPOTbX0aVmj/ZfsSAJgbK/Qyney2cAiF/OifklSEE1X8g5oGQGu7nVu6p+D/2n3vlprFu9I\nIrpDIO1beFe807czICUW7v+pzrEJUROSFETTdct8OHuy3k+7JzmTgyfP8o9xVYx48gwwklZRPtR1\nVTchakCuTUXT5eHnsFnDr255lVe3vGrXvkt2JOHmoriuZxWrq7XpDbZCSNnjkPiEsJckBdE0HdkA\na/4B+Wcd0lxcWhxxaXHV7mezaZbvPs6QzsE0867itlVJ2Q2ZxCbqmSQF0TTFLYdf3wNXz3o97Y6E\nMyRn5HFtVVcJAM3ag3cLmcQm6p30KYimKXkHtI4yZhDXo+9/P467i4URl1YzWkkpGDQd/KsYnSSE\nCSQpiKbHWmRUI+17d72e1mbT/PDHcYZ0Ccbf0636AwY/Yn5QQpRjV1JQSlmAKKANkAvs0VqnmBmY\nEKY5tQ+Kch06aa2Df4dq99mRkM7xjDyeGN3N/oYzjxujj7yb1yE6IexXZVJQSnUCngBGAAeAVMAT\n6KKUygE+BD7VWtvMDlQIh8lIBHc/h5a3mHnZzGr3+e7347i7WhjePcS+RrNOwFvdYPSrMHBq3QIU\nwk7VXSn8A/gP8KDWWpfdoJQKAW4F7gA+NSc8IUzQZRQ8ecy4b19PSkYdDe0SjJ89t44A/FqBX2vp\nbBb1qsqkoLW+pYptJ4GaF5AXoiFwcPmImb/ONJ4ruWLYfiydlMz8qucmVKRNbxmWKuqVXb8ZSqm/\nK6Vcy7z2V0rNNS8sIUxSVABzroLYpQ5t9mjmUY5mHq10+7lbRzWrkUSb3nDqAORl1jFCIexj78cl\nV2CzUqqnUmoksBXYZl5YQpgl2BdUAAAgAElEQVTk5B5I2gb12A1Wcuvoyq7B+HrUcMBfmz6ANkZL\nCVEP7EoKWuunMDqcNwPzgGu11u9Vd5xSarRSap9S6qBS6slK9pmolIpVSu1RSv2vBrELUXNJxffn\n29TfGgrbjqVzMiufMZE1vHUExroKN86GkO6OD0yICtg7JHUI8A7wIhAJvKeUuldrnVzFMS7A+8DV\nQCKwVSm1TGsdW2afzsBTwOVa6/TizmshzJO8A7yaGzOG68nKPSdwd7FwVbda/Hh7NYOoSY4PSohK\n2Hst+wYwoeQPulLqJmANUNWA6/7AQa314eJjFgBjgdgy+zwAvK+1TofSzmshzJO8wxiK6uCRR92a\nV/yroLVmZWwKl13Swv5RR+WlHYbEGOg5sQ4RCmEfe5PCIK21teSF1vobpdT6ao4JBRLKvE4EBpTb\npwuAUuoXwAWYqbVeUb4hpdQUYApA+/b19wlPXGS0hpY9TLl19ET/Jyp8f3/KWY6ezuHBIXWoxhq7\nDFa9AJ2Gg0+L2rcjhB2q7FNQSt2ulLKUTQgltNanlVKdlFJXVHZ4Be/pcq9dgc7AMOAW4COlVLMK\nzjVbax2ttY4ODg6uKmQhKqcU3DS7XieCrdxzAqVgxKV1uDNaMsnuuAxNFear7kqhBbBDKbUNY7RR\nyYzmS4ChwCmgwg5kjCuDdmVetwXK90EkAr9prQuBI0qpfRhJYmtNvgkh7FKYB27mVEV9cqPxa1B+\nBbaVsSn0bteMEL86nLd1lPGcvAMuGVH7doSwQ5VXClrrd4A+wHwgGBhe/DoJuENrfbPW+kAlh28F\nOiulwpVS7sBkYFm5fZYAVwIopYIwbicdruX3IkTVlk2H2cNMaTolO4WU7PPLgSWfyWV3UgYje7Sq\nW+OeAdCiMyTJlYIwX7V9CsW3jn4qfthNa12klJoO/IjRX/CJ1nqPUupFIEZrvax420ilVCxgBR7T\nWp+u6TchhF2StkHIpfV2up9ijSQxsroy2fZo0xvif657O0JUw94hqcEYI4XCyh6jtb63quO01suB\n5eXee77M1xp4pPghhHly0oxRPL1vr7dTrow9wSUhvnQM9q17YyNeADfvurcjRDXsHX20FNgIrML4\nRC9E41JSPyi0b72cLiOnkN8Op/HgkI6OaTCgrWPaEaIa9iYFb611xWPuhGgMSiqNtu5lSvNRwVHn\nvV6zLwWrTde9P6GsjW9Csw4QOd5xbQpRjr1J4Tul1Jji20FCND7tBsKwp40Zwib4v77/d97rlXtS\naOnvQc/QAMedZPci8G8jSUGYyt6kMAN4WilVABQWv6e11v7mhCWEg4UPNh71IK/Qyvr9qdzUJxSL\nxYEzp9v0hv0/GJPw6nEtCNG02FsQz09rbdFaexZ/7ScJQTQaeRlGlVFrYfX71tLDax/m4bUPA/DL\nwVPkFFgZeakDbx0BtOkFOachI6H6fYWoJbtXGlFK3aCUeqP4cZ2ZQQnhUEc2wIdDIHmnaac4k3+G\nM/lnAOPWkZ+HKwM7OrgkRcnM5iRZiU2Yx95Fdv6JcQsptvgxo/g9IRq+pO1gcYVWkaafymrTrNqb\nwpXdQnB3dezqbrSMAI8AyE51bLtClGFvn8IYoJfWxsokSqlPgR1UXuJCiIYjaZtRCM+kEhdlbT+W\nzunsAkb2cMCEtfJcPeCJeIcvJSpEWTX56So7bMOBQyqEMJHNVlwuu37mJ5SsnTC0i0mFGyUhCJPZ\n+xP2CkZhvHnFVwnbgJfNC0sIB0k7BPmZpq+0NqD1AAa0HlD3tROqk7AF5gyH04fMaV80efaOPpoP\nDAS+KX4M0lovMDMwIRzCPxTuWAydrzb1NFOjpnJly9s4ejrH8aOOynLzhqQY6WwWpqluPYVuxc99\ngNYYpa4TgDbF7wnRsLl7Q6erwM/EP9TFHLJ2QnWCu4Gr57myHUI4WHUdzY9grHj2ZgXbNHCVwyMS\nwpG2fwYhPaCtuX0KU1dNZfvRdHq3+0vd1k6ojosrtOp5rmyHEA5WZVLQWk8p/vIarXVe2W1KKfOH\ncghRF4V58P1fYeBDpieFrLxcsvJzHVvrqDKhfYxkZ7OCxcX884kmxd6O5l/tfE+IhuP4LrAWQNv+\npp8qLacAcNDaCdUJG2zcEsvLMP9cosmp8kpBKdUKCAW8lFK9Obfusj8gxd1Fw5aw2XhuZ35SSM8u\nwMvdxTFrJ1Sn+3XGQwgTVNenMAq4G2N95bfKvJ8FPG1STEI4RuIWCAwDXxM7fjHWTsjMK6JNQD3f\nUS3IMTrShXCg6voUPgU+VUrdrLVeVE8xCeEYJ/6ol1tHa/alUJjVjSt7hpl+rlJLpxtzFqZvqb9z\niibBrjIXWutFSqlrgR6AZ5n3XzQrMCHqbHqMMXHNZCv3pNC86GqeGjTc9HOVCgyDHf81lhn1bl5/\n5xUXPXsL4s0CJgF/xuhXmAB0MDEuIerOxdX0P5h5hVbW7Uvl6ktbOnbthOq0G2A8J26tv3OKJsHe\n0UeXaa3vBNK11n8DBgHtzAtLiDr69T1Y9TfTT/PzgVPkFlqJ1a9yz4p7TD9fqdC+oFzOdaYL4SD2\nVknNLX7OUUq1AU4D4eaEJIQD7P4SPM2v27gy9gR+nq74m1XrqDLu3tC6JxyTpCAcqyZrNDcDXge2\nY8xmnmNaVELURUG20cl8xcOmnqbIamPV3pNc1S2ETGesjnnZn40qsEI4kL0dzX8v/nKRUuo7wFNr\nLTNnRMOUtB209dx9d5NsO5pOWnYBo3q04qskU09VsYibnXBScbGzt6N5l1LqaaVUJ611viQE0aAl\nFg/TbBtt6mlWxqbg7mphiFlrJ9jjZBykxDrv/OKiY+/toxswRh99qZSyAQuBL7XWx0yLTIjaUhaj\nFISJI4+01vy45wRXXBKEr4cro8JGmXauKn0x3uh0nvipc84vLjr2rqdwVGv9mta6L3Ar0BM4Ympk\nQtTWFQ/D3d+Zeoq9x7NITM9lVPGym5O7TWZyt8mmnrNC7QYYI5C0rv9zi4uS3Wv7KaXClFKPAwuA\nbsDjpkUlRG3VU8frj8VrJwzvbiSF3KJccotyqznKBO0GQNZxyEio/3OLi5K9fQqbMVZccwEmaK37\na60rWmNBCOeK+RjejjRm+ppoZWwK0R0CCfL1AGDaqmlMWzXN1HNWqH1xZ7oMTRUOYu+Vwl1a6z5a\n61e01odNjUiIujj6i7HOgFegaadISMth7/FMc5fdtFfLCPDwN75vIRygutLZt2utPwfGKKXGlN+u\ntX6rgsPKHj8aeAfjCuMjrfU/K9lvPPAV0E9rHWNv8EKcR2uI/wU6DgNl3sSBH/ecAGBkj3pYO6E6\nFhe4fRG0uMTZkYiLRHWjj3yKn/0q2FZlz5ZSygV4H7gaY23nrUqpZVrr2HL7+QF/AeT6V9TN6UOQ\nfRI6XGbqaVbGptCtlR8dWvhUv3N9qIf1IkTTUV3p7A+Lv1yltT7v+lQpdXk1bfcHDpbcblJKLQDG\nAuUHVf8deA141N6ghajQ0Z+N5w7V/WjW3umz+cTEpzH9ygb0ybwgB7bMhrb9IMy87100Dfb2Kbxr\n53tlhQJlh0QkFr9Xqng1t3Zaa3PHD4qmIbg7DJwGQZ1NO8XqvSexaS5Yi3nsJWMZe8lY085bJRd3\n2PA6/PG1c84vLirV9SkMAi4DgpVSj5TZ5I/RT1Dl4RW8V3rLSSllAf6FsbJb1Q0pNQWYAtC+ffvq\ndhdNVfsB50bjmOTHPScIbeZFjzb+570/7pJxpp63Si6u0H6Q0Z8iRB1Vd6XgDvhiJA+/Mo9MYHw1\nxyZyfnnttkBymdd+QASwTikVDwwElimlLqhNoLWerbWO1lpHBwc7saSAaLhy0yFlj6nzFDLzCtl4\n4BSjI1qhynVkp+elk56Xbtq5qxV2OZzaB2dPOi8GcVGork9hPbBeKTVPa320hm1vBTorpcKBJGAy\nxmzokrYzgKCS10qpdcCjMvpI1Ercclg6Dab+Aq0iTDnFqtgUCqw2xkS2vmDbI+uMC+m5o+eacu5q\nhQ02nuN/hoibnBODuCjY26fwUXHpbACUUoFKqR+rOkBrXQRMB34E9mLUStqjlHpRKXVDrSMWoiJH\n1oNPMIRcatoplu8+QesAT3q3a1b9zvWtdZQxNyPTGeVaxcXE3oJ4QVrrMyUvtNbpSqmQ6g7SWi8H\nlpd77/lK9h1mZyxCnE9rOLwOwoeCxe7KLTWSlVfIhgOp3D6gQ/0uu2kvFzd49IDxLEQd2PsbZFNK\nlfbwKqXCqGaeghD1JjUOzqYYk9ZMsnrvSQqKbIyJbACzmCsjCUE4gL1J4RngZ6XUf5VS/wXWA0+Z\nF5YQNXB4nfHccZhpp/h+93Fa+XvSp7155TPqLPs0fDwKdi10diSiEbN35bUVxaOCpgA7gaWcW7dZ\nCOfqdRsEdYFm7arftxbO5hexfn8qt/ZvX+mto0ldJ5ly7hrxCoS0w3DwJ4hqAPGIRsmupKCUuh+Y\ngTGsdCfG8NFNwFXmhSaEnTz94ZLhpjW/em8KBUU2ru154aijEqPDR5t2frtZLMbV0qE1xtBck/pX\nxMXN3p+aGUA/4KjW+kqgN5BqWlRC2OvEblj/uqmlspfvPk6Inwd9q7h1dCL7BCeyT5gWg906XQU5\npyDlD2dHIhope5NCntY6D0Ap5aG1jgO6mheWEHba+x2sfcm05rPzi1i3L5VrIlpVOeroqY1P8dTG\nBtDN1nGY8XxojTOjEI2YvUNSE4vnKSwBflJKpXP+7GQhnOPwWmjT27T1mFfHnSS/qOIJaw2Sf2vo\new807+jsSEQjZW9H843FX85USq0FAoAVpkUlhD1y0iBxKwx5zLRTLN2RROsAT/qFmZN0THH9286O\nQDRi9l4plCoufSGE8x1cDdoGnUeZ0nxadgHr96dy3xXhDXPCWlWyT4G1APzbODsS0cjI8ATReGUm\nQUA74/aRCb7ffZwim2Zsr9Dqd25IigrgnSjYWOXCiEJUqMZXCkI0GFf8HwyabtrQy6U7kugc4kv3\n1hUtPHi+u3rcZUoMteLqDuFDYP+PMOZ1U5cmFRcfuVIQjZMurrLiYs7nmoS0HGKOpjOud+gFZbIr\nMqzdMIa1G2ZKLLXSZRRkHIOT5Rc6FKJqkhRE47TmH/DxSLBZTWl+2S5jcN0NUfbdkz+ScYQjGUdM\niaVWuhRPptv3g3PjEI2OJAXROO1fARY3sFS3AGDNaa1ZsiOJ6A6BtGvubdcxL256kRc3vejwWGrN\nr5XR17JfBgmKmpGkIBqfjERjxm6XkaY0H3s8kwMnzzK2dyPrYC5vzJtw80fOjkI0MtLRLBqfuO+N\n565jTGl+0bYk3FwU1zaWCWuVadvX2RGIRkiuFETjE7sMgrtDUGeHN11QZGPxjkRGdG9Jcx93h7df\n7/Z+C7/829lRiEZErhRE49NzIrh6mtL06r0ppOcUMrGfOWW4693h9bDjvxB9L3j4Ojsa0QhIUhCN\nT1/z5gQsjEmglb8nQzoH1+i4KT2nmBRRHfW4EbbOgQM/QsTNzo5GNAJy+0g0LvtWwFlzqrYfz8hl\nw/5Uxvdti0sNy1oMajOIQW0GmRJXnbQfCL4tYc8SZ0ciGglJCqLxyEmDhbfBpvdMaX7RtkRsGiZE\nt63xsXFpccSlxZkQVR1ZXKD7DXBgJeSfdXY0ohGQpCAaj/0rwFYEl97g8KZtNs2XMYkM7NicDi18\nanz8q1te5dUtrzo8LofocSMEhhtDeYWohvQpiMbjj2+KC+D1cXjTm4+kcSwth4evdvyIJqfrcBn8\n6TdnRyEaCblSEI1DVoqxmljkBFMKvH2x+Sj+nq6M7tHI5yZUpOTfqzAPivKdG4to8CQpiMYhfiNo\nK0RNdnjTJzPzWPHHCSZGt8PL3fFlMxqE04fg9U6wZ7GzIxENnCQF0ThEjoeHYyHY8UuD/2/LMaxa\nc/vADg5vu8Fo3hF8gmDnF86ORDRw0qcgGo8Ax9ciKiiy8cXmYwztEkxYUM07mEvM6DPDgVGZQCno\ndRusfQnOHINm7Z0dkWig5EpBNHxrXoKFd4DN5vCmf9xzgtSsfO4aFFandnqF9KJXSC/HBGWWkltv\nuxY4Nw7RoElSEA2bzQo7PoeiPFNWWPtsUzztm3sztEvNZjCXt/PkTnae3OmYoMzSrL2xItvOL84t\nUiREOXL7SDRsB1dBVjKMftnhTe84ls7W+HSevbY7lhrOYC7vne3vADB39FxHhGae4S+ARX7tReXk\np0M0bFs/Nso0dL3W4U3P3nAYP09XJvdvQvfX20Y7OwLRwJl6+0gpNVoptU8pdVAp9WQF2x9RSsUq\npX5XSq1WSl3Ewz9EjaUfNcoz9LnTWIzegeJPZbNizwluH9gBX48m9tno9CFYMg2yTzs7EtEAmZYU\nlFIuwPvANcClwC1KqUvL7bYDiNZa9wS+Bl4zKx7RCLn7wtAnoI/jq6J+9PNh3CwW7rkszOFtN3jW\nAqNfYfunzo5ENEBmXin0Bw5qrQ9rrQuABcDYsjtorddqrXOKX/4G1LwSmbh4+bSAK5+CZo5d2+D0\n2Xy+iknkxt6hhPibsy5DgxbSHcIGQ8wnYC1ydjSigTEzKYQCCWVeJxa/V5n7gB8q2qCUmqKUilFK\nxaSmmlM2WTQwRzZA7FJj9JGDzdl4hAKrjQeGhDuszSf6P8ET/Z9wWHumG/AgZCQYRQaFKMPMm6kV\nDeeocBycUup2IBoYWtF2rfVsYDZAdHS0jKW72GkNK5+D/Ezodr1Dmz59Np/PNsVzXc82XBLi57B2\nuzXv5rC26kWXa8C/LWyZDd2vc3Y0ogExMykkAmWv+9sCyeV3UkqNAJ4BhmqtpVqXMK4Sju+E6952\n+NyE2RsOk1toZcbwSxza7qbkTQANc6Gdiri4wmV/hsxEY1KgCXNARONkZlLYCnRWSoUDScBk4Nay\nOyilegMfAqO11idNjEU0Jr+8DT4hEHWLQ5s9dTafzzYd5YYox14lAMz+fTbQiJICwMCpzo5ANECm\nfTzQWhcB04Efgb3Al1rrPUqpF5VSJaukvA74Al8ppXYqpZaZFY9oJI7vMkpkD3wI3BzbCfzh+kPk\nF1n5y/CLcM2E2tIaDq42hqkKgcmT17TWy4Hl5d57vszXI8w8v2iEck5Dq0iIvtehzSak5fDppqOM\n6x1Kp2Bfh7bdqOWmw4LbjNXZbvyPs6MRDYDcSBQNS6er4MGN4NXMoc2+/uM+FPDoSMeX3m7UvJtD\n37vh94XGZEHR5ElSEA3H3m+N1cEcvLLajmPpLNuVzAODO9KmmZdD274oXPZnsLjABpk7KiQpiIYi\nYQssvB22znFos1pr/vH9XoJ8PZg6rJND2y7r+UHP8/yg56vfsSEKCIV+D8DO/0FKrLOjEU4mSUE4\nn9aw6m/gEwx973Fo08t2JbPtaDp/HdnF1BpH4QHhhAc4bjJcvRvyKDTvBJlJzo5EOFkTqwQmGqSD\nq+Doz3DN6+DhuE7gMzkF/P27WKLaNWNitGNLZZS3LmEdAMPaDTP1PKbxbg5/2iLzFYRcKQgnK8yD\nHx43PqX2dWzhu1dXxJGeU8jLN0bgUsf1Eqrz6Z5P+XRPIy8wZ7GAtRB+/8qU8iKicZCkIJwr5xR4\nBcK1b4Crh8Oa3RqfxvwtCdx3RTg92gQ4rN2L3v4f4Zv7YetHzo5EOIkkBeFcAW3h/tXGUFQHyc4v\n4rGvdhHazIv/GyET1Wqk27XG/8XqFyFD+heaIkkKwjm0hl/fg+xTDh+C+o/v93I0LYc3JkTh7S7d\nZjWiFFz7lnH76IfHnR2NcAJJCsI5ts2Flc/AnsUObfan2BTmbznGlCEdGdSphUPbbjKah8OwJyDu\nO9izxNnRiHomH6NE/Tt1EH58BjpeCdH3OazZ4xm5PLnody5t7c8jV3dxWLv2eGXwK/V6PtMNmg7H\nf4cAc0dtiYZHkoKoX9YiWDwFXNxh3AcOGwKZX2Tloc+3k1do5d+39MLD1cUh7dqrlU+rej2f6Vzc\nYMLcc6+1dvhtPtEwye0jUb82vgFJ2+C6f4F/G4c1O3PZHnYmnOHNiVEOL4ttjxVHVrDiyEW4ipnN\nCssfgw2vOzsSUU/kSkHUr+h7wbsFRNzksCb/uyme+VsS+NOVnRgd0dph7dbEwn0LARgdPtop5zeN\nskBeBmyZAy0joNsYZ0ckTCZXCqJ+pB81bh35hkD/BxzW7Io/TvDCsj0M7xbCI1dLBVSHUwqufwfa\n9IJF98OJP5wdkTCZJAVhvrQj8PHVsOzPDm128+HT/GXBDqLaNeO9W/uYPmu5yXLzgsnzwdMf5k+G\nrBPOjkiYSJKCMFfmcfhsrFE+4fIZDmt229F07v80hnaBXnxyVz+83Ou3Y7nJ8W8Nk/8HhTlwar+z\noxEmkj4FYZ6zqfD5TcZqanctg5BuDmn2t8OnuXfeVkL8PPjvfQMI9HF3SLuiGqF9YMbv54oW2qzG\nOgzioiJJQZhDa1h4m3Hr6LYvIbSvQ5pdG3eSh77YRmgzL+Y/MJAQf8eu41xbbw17y9kh1I+ShLDj\nc6Pz+dYvwa+lc2MSDiVJQZhDKRj9inHbqP1AhzQ575cjvPhdLN1b+zPvnv4E+zmugF5dBXoGOjuE\n+uXbyriN9PHVcPs3EHSJsyMSDiJ9CsKx4pbD+uJlHUP7OiQh5BVaeWbxbmZ+G8vw7i358sFBDSoh\nACw5uIQlB5tQSYjOI+Du76Ag20gMh9c5OyLhIJIUhGNYi2Dty7DgFtj3g7FOggMcTj3LTR/8yheb\nj/Hg0I7Mur0vPiauoFZbSw8uZenBpc4Oo36F9oX7VhrDjP97I5w64OyIhAM0vN8u0ficPgSLp0Li\nFoi61Zit7Fa3e/02m+aLzUd55Yc4PFwtfHxXNMO7y73rBqdFJ6P0+b7lEFRcpjz/rENX0BP1S5KC\nqJvCPPh4JNgK4eaPIXJ8nZvcn5LFU9/sZtvRdAZ3DuK18T1pHeDlgGCFKTx8oedE4+vkncYQ5OHP\nGetty+ikRkeSgqg5rSExBtpGG1cE170FbfvVuZbRiYw8/vXTfr7aloC/lxtvTojipj6hKCnE1nh4\nNzdmP3//V4iZB6NfhvAhzo5K1IAkBWE/reHwWlj3KiT8BpO+gO7XwaVj69TsiYw8PvnlCJ9tisdq\n09x9WTjTr7qE5jL/oPFp1h7uWGKsk/HTC/Dp9dD9Bpj4mVRZbSQkKYjqWYvgj0WwZTYkxYB/KIx5\nAzpfXadm/0jK4JNfjrBsZzI2rbkhqg1/HdmVds29HRR4/flgxAfODqHhUMooeNh1DGz58FzZbZsN\n9q8wfm5c3JwdpaiEJAVRMa0hMxkCQo1f6DX/MH6Rr30Tet8BrrUbEpqalc/SnUl8vS2RuBNZeLu7\ncPvADtx3RXijTAYlvFylz+MCbp7nlzaJ32iMTvMJhl63Qs/JENJdriAaGKW1dnYMNRIdHa1jYmKc\nHcbFqagAEjbDwVUQuxTyM+Gv+8HFFc4kGFcINVwUR2vNodSz/BR7klV7U9h+LB2tIaptAOP7tuWG\nqFACvBv/p8YFcQsAmNxtspMjacBsVji4GrbNM64YtBWad4Q7FkNgmLOju+gppbZpraOr20+uFJqy\n/LPGJ34XN+MXdcXTUJgNygU6DoUeNxq/uLhCM/uWZbTaNAdPnmVrfBpbjhiPE5nGnIXI0AD+b3gX\nxkS2onPL+l8Ix0w/xv8ISFKoksUFuow0HlkpsO97OLQG/Nsa29f8A07shrAroP0gaNnDqNAq6pWp\nSUEpNRp4B3ABPtJa/7Pcdg/gM6AvcBqYpLWONzOmJiv3DBzbBKn7jPIEyTshdS/c9R2EXQ5BXY1L\n+k5XGb+Unv5VNmezaU5k5hF/OptDqdnEJmcSezyTfScyySu0AdDS34P+4S0Y2LE5V3ULkWGl4hy/\nlsaCS9H3nnvP1RNOHzSuIsD4cNLpKrj9a+N10jbwCoSA9sbVqzCFaf+ySikX4H3gaiAR2KqUWqa1\nji2z231Autb6EqXUZOBVYJJZMV10tDYuyV1coTAX4n+BzCTjkZEEmYnQ737ofr0xwWx+8adYnxBo\n3dMYOeRXvLZwh0HGAygosnEmK4+TmfmkZuWTkpnHyax8TmblcSIjj6OnczialkNBka00lAAvN3q0\n8ef2AR24tI0/0R2a0665lwwnFfYb8qjxyDwOiVvh+C4jUZT48m7IOAYWNwjsAIHhRqf1gAeN7YfX\ngVdz8AkykodcZdSKmem2P3BQa30YQCm1ABgLlE0KY4GZxV9/DbynlFK6sXR0aG08Su6zF+WDthkP\nmxVsRcZyhl7NjO2nDhh/vG2FRqE4a6HxAxzS3dj+xzdQcBYKstEF2ej8sxS17kNh5zEUFeThtfhu\nKMhG5aVjyU3DkptOWp8/cbLvI6izKXT74mYjLBQFXiHkerVi36GTHMo8RkGeN76953HCrS0Z2oe8\nQhtnTxaR+W0aGbkpZOYVkplbREZuIbmF1gq/3UBvN1r6exIe5MOV3ULo0MKbsBY+hAf50DrAUxKA\ncAz/1nDpDcajrJvnGL9DaYeMDznp8ZCRaGyz2eDzm43fuRKunjDwIRgx0xhB9+Ud4O4Dbt7nnjsO\ng/DBxiTMuO+MY1zcjVuqLu7QPNyYf1OUb6weWPK+i7vxYczNB1zdjfPbCo3f99JH4/x9MDMphAIJ\nZV4nAgMq20drXaSUygBaAKccHczCrccI+HEGva2/o9AoNBY0iaoV0z1fBuDVvH8QYYsr3abQ7FMd\necjtH2g0cwsfp4s+ggJcMD4lbyWCB11mArDE+ifak3LeedcQzQz9OBpYz/20UJnnbV9mu4JHbdNB\nwy7XB/FSBQAowKotfGG9mr8VuaCwscz9ADl4kqF9SNMtOYMfG3/15JefN2LBRm/1Aid0c1IIpCjP\nFdKBZIDdxWdzB07i5eaCp5sFb3dXArzcCPByIzzIhwAvN/w9jdcB3m6E+HkS4u9BiJ8HwX4eeLjK\n7FThRO0HVl1g8Z4VkM3FaOoAAAlXSURBVJUMOWmQm248Qov7VQuz4cyx4g9dOcZiQQXZRhIIHwzZ\nqbDovgvbHPUKDJpmJKD3+1+4/YZ3oc+dkLwDPrqq3EYF4z8xhuce2QBfTDSShcXFSBjKAuPnQqcr\n4cAqWPqn4kRSnEyUgon/hbZ9Ye+3RhtjXq/FP1zNmJkUKkqT5a8A7NkHpdQUYApA+/btaxVMCx8P\nMgK7cSTfHa2MP/mgyHRtQXRIcxRwMm0Quwo7GClBGZ/+z7i1ZEhQEArFgdPXk2I9XZwyLKAg3b0N\nY1q0QqHYeeoO4mzZgEIrC1blRrpnKOMD2qJQrMp4Fou2YrO4YVWu2CxuZLsFcY93BxSKz/M+p9Di\nQZHFC+3ug3Jxx9XVwtMWhYvFwnbLUlwsCjcXhavFQiuLYrJFcYeLsd3VMgAXi8LVRRX/4Xc579nD\nzYKHq0U+0Ztg7ui5zg6habNYoF2/yrd7BsBDv5z/ntbGVT0Yt1H/tAWK8oyrCmuB8Wje8dz2mz8u\nvsIv3mYtPJd0/FvD8OeL7xQU39bVNgguXljKP9RYm7zkTkLJ3YSS27e+wUYHfOlNEm38JfQMMF76\nBENQl7r+K9nFtCGpSqlBwEyt9aji108BaK1fKbPPj8X7bFJKuQIngOCqbh/JkFQhhKg5e4ekmlk6\neyvQWSkVrpRyByYDy8rtswy4q/jr8cCaRtOfIIQQFyHTbh8V9xFMB37EGJL6idZ6j1LqRSBGa70M\n+Bj4r1LqIJCGkTiEEEI4iamDfbXWy4Hl5d57vszXecAEM2MQQghhP1l5TQghRClJCkIIIUpJUhBC\nCFFKkoIQQohSkhSEEEKUanTrKSilUoGjtTw8CBNKaDiAxFUzElfNNdTYJK6aqUtcHbTWwdXt1OiS\nQl0opWLsmdFX3ySumpG4aq6hxiZx1Ux9xCW3j4QQQpSSpCCEEKJUU0sKs50dQCUkrpqRuGquocYm\ncdWM6XE1qT4FIYQQVWtqVwpCiP9v7/xC7aiuOPz9SGpiVTRRrNFIm4CI2pemUazaIkZiDGK0qMQX\ng+mLLYH6UDASEKkFUakPhbaBqtgWW9Nq1YtE4rUqPohRDLn5QxJzYgPeJiTYP9FSamu7fNj7DuNx\n5t6Tc8/MaeLvg+Hss/eamTVrr7PX2XvPzDZmEo67oCDpZkk7Jf1P0uKusrsldSTtkXRNzf4LJG2W\ntFfShvza70HruEHS1rztl7S1Rm6/pO1ZrvFFJCTdK+nPJd2W18gtyzbsSFrbgl4PSdotaZukZySd\nViPXir2mun5Js3Idd7IvfaUpXUrnPFfSK5J2Zf//foXMlZKOlOr3nqpjNaDbpPWixE+yvbZJWtSC\nTueX7LBV0geS7uySac1ekh6TdFjSjlLeXEmjuS0alTSnZt9VWWavpFVVMkdFRBxXG3ABcD7wKrC4\nlH8hMAbMAhYA+4AZFfv/DliZ0+uB7zas74+Be2rK9gNntGi7e4EfTCEzI9tuIWl9zzHgwob1WgrM\nzOkHgAeGZa9erh/4HrA+p1cCG1qou3nAopw+BXinQq8rgefb8qde6wVYDrxAWonxUmBzy/rNIC3w\n9eVh2Qv4FrAI2FHKexBYm9Nrq/wemAu8mz/n5PSc6ehy3PUUImJXROypKFoBPBkRH0XEn4AO8KlF\nV5XWqbwKeCpn/RK4oSld8/luAX7b1Dka4BKgExHvRsS/gSdJtm2MiHgxIiZWZH8DmN/k+aagl+tf\nQfIdSL60RA2vgRoRByNiS05/COwirYF+LLAC+FUk3gBOkzSvxfMvAfZFRL8PxU6biHiNtKZMmbIf\n1bVF1wCjEfHXiPgbMAosm44ux11QmIRzgPdK38f57I/mdODvpQaoSmaQfBM4FBF7a8oDeFHS23md\n6jZYk7vwj9V0V3uxY5OsJv2rrKINe/Vy/YVM9qUjJN9qhTxc9TVgc0XxNySNSXpB0kUtqTRVvQzb\np1ZS/8dsGPaa4EsRcRBS0AfOrJAZuO0aXWSnKSS9BJxVUbQuIp6r260ir/vWq15keqJHHW9l8l7C\n5RFxQNKZwKik3fkfRd9Mphfwc+A+0jXfRxraWt19iIp9p30LWy/2krQO+Bh4ouYwA7dXlaoVeY35\n0dEi6WTgaeDOiPigq3gLaYjkH3m+6FngvBbUmqpehmmvE4Drgbsriodlr6Nh4LY7JoNCRFzdx27j\nwLml7/OBA10y75O6rjPzP7wqmYHoKGkm8G3g65Mc40D+PCzpGdLQxbQauV5tJ+kXwPMVRb3YceB6\n5Qm064AlkQdTK44xcHtV0Mv1T8iM53o+lc8ODQwcSV8gBYQnIuIP3eXlIBERGyX9TNIZEdHoO356\nqJdGfKpHrgW2RMSh7oJh2avEIUnzIuJgHk47XCEzTpr7mGA+aT61bz5Pw0cjwMp8Z8gCUsR/syyQ\nG5tXgJty1iqgrucxXa4GdkfEeFWhpJMknTKRJk227qiSHRRd47g31pzvLeA8pbu0TiB1vUca1msZ\ncBdwfUT8s0amLXv1cv0jJN+B5Esv1wWyQZHnLB4FdkXEwzUyZ03MbUi6hPT7/0vDevVSLyPAbfku\npEuBIxPDJi1Q21sfhr26KPtRXVu0CVgqaU4e7l2a8/qnjZn1NjdSYzYOfAQcAjaVytaR7hzZA1xb\nyt8InJ3TC0nBogP8HpjVkJ6PA3d05Z0NbCzpMZa3naRhlKZt92tgO7AtO+S8br3y9+Wku1v2taRX\nhzRuujVv67v1atNeVdcP/JAUtABmZ9/pZF9a2IKNriANG2wr2Wk5cMeEnwFrsm3GSBP2l7WgV2W9\ndOkl4KfZntsp3TXYsG5fJDXyp5byhmIvUmA6CPwnt1/fIc1D/RHYmz/nZtnFwCOlfVdnX+sAt09X\nFz/RbIwxpuDzNHxkjDFmChwUjDHGFDgoGGOMKXBQMMYYU+CgYIwxpsBBwRhjTIGDgjHGmAIHBWOm\niaSL80sEZ+cneHdK+uqw9TKmH/zwmjEDQNKPSE8ynwiMR8T9Q1bJmL5wUDBmAOT3IL0F/Iv0OoT/\nDlklY/rCw0fGDIa5wMmkVc9mD1kXY/rGPQVjBoCkEdIqbAtILxJcM2SVjOmLY3I9BWP+n5B0G/Bx\nRPxG0gzgdUlXRcTLw9bNmKPFPQVjjDEFnlMwxhhT4KBgjDGmwEHBGGNMgYOCMcaYAgcFY4wxBQ4K\nxhhjChwUjDHGFDgoGGOMKfgENMQU0kDqgnoAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(x1, activation(x1))\n",
    "plt.plot(x1, 4*activation_derivative(x1), '--')\n",
    "plt.plot(np.zeros((5,1)), np.linspace(0,1,5), '--')\n",
    "plt.legend(['activation','activation_derivative','decision boundary'])\n",
    "plt.xlabel('x')\n",
    "plt.ylabel('activation(x)')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Logistic regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create some toy data\n",
    "X = np.array([[0,0,1],\n",
    "              [0,1,1],\n",
    "              [1,0,1],\n",
    "              [1,1,1]])\n",
    "\n",
    "# Linear case\n",
    "y = np.array([[0],\n",
    "              [0],\n",
    "              [1],\n",
    "              [1]])\n",
    "\n",
    "# Fix the seed\n",
    "np.random.seed(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Text(0,0.5,'error in training data')"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEKCAYAAAD9xUlFAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4zLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvIxREBQAAIABJREFUeJzt3XmcHWWd7/HPr/clvaTTnb2TTkIS\nDEkgkIVAUATBwCiLOsriDKAj6gyDwOgMM97rIM54x3XuiCiiAjp3lEUFIxMlDiIKkZAOCVkICdlI\nOmtn6SW9pLff/aOqm5PmdPfJcrq6+3zfr1e9TlWdqjq/rnP6/M5TTz3PY+6OiIgIQFrUAYiIyMCh\npCAiIl2UFEREpIuSgoiIdFFSEBGRLkoKIiLSRUlBRES6KCmIiEgXJQUREemSEXUAJ6q0tNQrKiqi\nDkNEZFBZtWrVQXcv62u7QZcUKioqqKysjDoMEZFBxczeTGQ7XT4SEZEuSgoiItJFSUFERLooKYiI\nSBclBRER6aKkICIiXZQURESkS8okhcodh/nKb15Hw4+KiPQsZZLC+t21fPf3W9lX1xx1KCIiA1bK\nJIXZ5cUAvLqrNuJIREQGrpRJCjPGFJKRZqzbXRN1KCIiA1ZSk4KZLTazTWa2xczu7mGbD5vZa2a2\nwcx+kqxYcjLTmTaqgLVVKimIiPQkaR3imVk6cD9wGVAFrDSzJe7+Wsw2U4F/BC509yNmNjJZ8QCc\nXV7E0nX7cHfMLJkvJSIyKCWzpDAf2OLu29y9BXgUuLrbNp8A7nf3IwDufiCJ8TB7fDG1Ta3sPNyY\nzJcRERm0kpkUxgG7YparwnWxpgHTzOxFM3vJzBYnMR5mjSsC4FVdQhIRiSuZSSHe9ZnujQQygKnA\nxcD1wA/MrPhtBzK71cwqzayyurr6pAOaPrqA7Iw01u5SZbOISDzJTApVQHnM8nhgT5xtfunure6+\nHdhEkCSO4+4Puvtcd59bVtbnwEE9ykxPY8bYQtbuVklBRCSeZCaFlcBUM5tkZlnAdcCSbts8Bbwb\nwMxKCS4nbUtiTMweV8T63bW0d6hls4hId0lLCu7eBtwGPANsBB539w1mdq+ZXRVu9gxwyMxeA54D\nPufuh5IVEwSVzY0t7WytPprMlxERGZSSOkazuy8FlnZb94WYeQfuCqd+cXZ5UNm8tqqWaaMK+utl\nRUQGhZRp0dxpUukw8rPSWVulymYRke5SLimkpxkzxxXptlQRkThSLikAzB5fxMa9dbS0dUQdiojI\ngJKiSaGYlrYONu+vjzoUEZEBJSWTwtnjw260Va8gInKclEwK5SW5FOdlsk71CiIix0nJpGBmzFJl\ns4jI26RkUoDgEtLm/fU0tbRHHYqIyICRsklh1vgi2juc1/bWRR2KiMiAkbJJobOyWY3YRETekrJJ\nYXRRDiMLsjU8p4hIjJRNChA0YlNJQUTkLSmeFIrZdrCB+ubWqEMRERkQUjwpFOEO6zTojogIkPJJ\nIahsViM2EZFASieFkvwsxg/PVWWziEgopZMCBLemqg8kEZFAyieFWeOLqDrSxOGGlqhDERGJXMon\nhc5GbGt2HYk4EhGR6CkplBeRnma88qYuIYmIpHxSyMvK4MzRBbyyUyUFEZGUTwoA504Yzqu7amjv\n8KhDERGJlJICcO7EYhpa2jU8p4ikPCUFYE75cABdQhKRlKekAEwckUdJfpYqm0Uk5SkpEAzPee6E\nYlarpCAiKS6pScHMFpvZJjPbYmZ3x3n+ZjOrNrM14fRXyYynN3MmDGfbwQaOqBGbiKSwpCUFM0sH\n7geuAGYA15vZjDibPubu54TTD5IVT1/OnRDUK6zZpUtIIpK6kllSmA9scfdt7t4CPApcncTXOyVd\njdh0CUlEUlgyk8I4YFfMclW4rrsPmtlaM/uZmZUnMZ5eqRGbiEhyk4LFWde9ddivgAp3nw38D/Cj\nuAcyu9XMKs2ssrq6+jSH+ZY5E4pZs1ON2EQkdSUzKVQBsb/8xwN7Yjdw90Pufixc/D5wXrwDufuD\n7j7X3eeWlZUlJVgI6hXUiE1EUlkyk8JKYKqZTTKzLOA6YEnsBmY2JmbxKmBjEuPp07yKEgBe3n44\nyjBERCKTtKTg7m3AbcAzBF/2j7v7BjO718yuCje73cw2mNmrwO3AzcmKJxHlJXmMK85lxfZDUYYh\nIhKZjGQe3N2XAku7rftCzPw/Av+YzBhO1IJJJTy/uRp3xyxetYiIyNClFs3dnD95BIcaWthy4GjU\noYiI9DslhW4WTA7qFV5SvYKIpKA+k4KZlZnZ181sqZn9rnPqj+CiMKEkj9GFOazYpnoFEUk9iZQU\n/ougongS8EVgB8GdRUOSmXH+5BJe2nYYd7VXEJHUkkhSGOHuPwRa3f15d/8YcH6S44rUgskjOHj0\nGNsONkQdiohIv0okKbSGj3vN7M/MbA5BQ7Qha8GkoF5hxTbVK4hIakkkKfyLmRUBfwd8FvgBcEdS\no4rYpNJ8ygqy1V5BRFJOIu0Ujrh7LVALvBvAzC5MalQRC+oVRvDStkNqryAiKSWRksJ9Ca4bUhZM\nKmF/3THePNQYdSgiIv2mx5KCmS0ELgDKzOyumKcKgfRkBxa1hVNGAPDCloNUlOZHHI2ISP/oraSQ\nBQwjSBwFMVMd8KHkhxatyaX5jB+ey/Obk9dVt4jIQNNjScHdnweeN7NH3P3NfoxpQDAz3jWtjKdW\n76alrYOsDDX+FpGhL5FvukYz+1qqtGiOdfH0kTS0tFP5pm5NFZHUkGiL5tdJkRbNsRZOGUFmuukS\nkoikDLVo7sWw7AzmTizh+U1KCiKSGtSiuQ8XTy/j9X317K9rjjoUEZGkO9kWzXcmNaoB5F3TgzGh\ndQlJRFJBny2a3f3pcLarRXMqmT6qgFGF2Ty/qZoPzy2POhwRkaTqrfHafUCPfUe7++1JiWiA6bw1\n9Tfr99HW3kFGum5NFZGhq7dvuEpgFZADnAu8EU7nAO3JD23guHj6SOqa21izqybqUEREkqq3xms/\nAjCzm4F3u3truPwAsKxfohsgFk0tJTPdWPbafuZWlEQdjohI0iRyLWQsQfcWnYaF61JGYU4mi84o\nZem6vRqNTUSGtESSwr8Bq83sETN7BHgF+HJSoxqArpg5hqojTWzYUxd1KCIiSdNnUnD3h4EFwJPh\ntLDz0lIquWzGKNLTjKXr9kYdiohI0iR0K42773P3X4bTvmQHNRANz89i4eQR/Hr9Pl1CEpEhS/dX\nnoArZo1m+8EGNu2vjzoUEZGkSGpSMLPFZrbJzLaY2d29bPchM3Mzm5vMeE7V5TNGYwZL16VkYUlE\nUkCfScHMSuJMmQnslw7cD1wBzACuN7MZcbYrAG4HVpx4+P2rrCCb+RUl/Fr1CiIyRCVSUngFqAY2\nEzReqwa2m9krZnZeL/vNB7a4+zZ3bwEeBa6Os92XgK8Cg6LHuStnjeGNA0fZckCXkERk6EkkKfwG\nuNLdS919BMEv/8eBvwa+08t+44BdMctV4bouYY+r5TH9K8VlZreaWaWZVVZXR9sx3eKZwSWkJWv2\nRBqHiEgyJJIU5rr7M50L7r4MeKe7vwRk97KfxVnXdduOmaUB/07Q+2qv3P1Bd5/r7nPLysoSCDl5\nRhXmsOiMUn7+ym46OnQXkogMLYkkhcNm9g9mNjGc/h44EtYZdPSyXxUQ263oeCD253UBMBP4vZnt\nIBi4Z8lAr2wG+PO55eyuaWL51kNRhyIiclolkhRuIPhCfwr4JTAhXJcOfLiX/VYCU81skpllAdcB\nSzqfdPfa8JJUhbtXAC8BV7l75Un9Jf3o8hmjKMzJ4IlVu/reWERkEElkPIWDwN/28PSWXvZrM7Pb\ngGcIEshD7r7BzO4FKt19SU/7DnQ5melcdc5YnqisoraplaLcPm/GEhEZFPpMCmY2jWDEtYrY7d39\nkr72dfelwNJu677Qw7YX93W8geTPzyvn/720k6fX7uHGBROjDkdE5LToMykATwAPEAzDmVLjKPRm\n9vgipo0axhOVVUoKIjJkJJIU2tz9u0mPZJAxM/78vHL+delGthyo54yRBX3vJCIywCVS0fwrM/tr\nMxsT26o56ZENAtfMGUdGmvGTFapwFpGhIZGkcBPwOWA5wfCcqwiG6kx5ZQXZ/NnsMTxeuYv65tao\nwxEROWWJjKcwKc40uT+CGwxuuXASR4+18bNVVVGHIiJyynqsUzCzS9z9d2b2gXjPu/svkhfW4HFO\neTHnTijmR8t3cNPCCtLS4jXkFhEZHHorKbwrfHx/nOl9SY5rULnlwknsONTIc5sORB2KiMgp6bGk\n4O7/HD7e0n/hDE6LZ45mTFEOD724nUvfMSrqcERETloijdeygQ/y9sZr9yYvrMElMz2Nv1xYwVd+\n8zqb9tUzfbRuTxWRwSmRu49+STAOQhvQEDNJjOvnl5Obmc73/rA16lBERE5aIo3Xxrv74qRHMsgV\n52Vx44IJPLx8B5+5dCoTR+RHHZKIyAlLpKSw3MxmJT2SIeDWd00mI8349u967CdQRGRASyQpLAJW\nmdkmM1trZuvMbG2yAxuMRhbkcMOCCfxi9W52HmqMOhwRkROWSFK4ApgKXM5bt6O+P5lBDWafetcU\n0tOM+59TaUFEBp8ek4KZFYaz9T1MEseowhyun1fOz1+pYtdhlRZEZHDpraTwk/Cxs6+jVajvo4R8\n6uIppJlx3+/eiDoUEZET0mNScPf3hY+T3H2y+j5K3JiiXP5y4USeWFXFxr11UYcjIpKwROoUMLPh\nZjbfzN7ZOSU7sMHutkvOoDAnky8v3Rh1KCIiCeszKZjZXwF/IBhr+Yvh4z3JDWvwK87L4m8vOYM/\nvnGQ5zdXRx2OiEhCEikpfAaYB7zp7u8G5gD6lkvAXyycyISSPL783xtp7/CowxER6VMiSaHZ3Zsh\n6AfJ3V8Hpic3rKEhOyOdf1h8Jpv21/NEpUZnE5GBL5GkUGVmxcBTwG/N7JfAnuSGNXRcOWs0cycO\n56vPbOJIQ0vU4YiI9CqRkdeudfcad78H+N/AD4Frkh3YUGFmfOmamdQ2tfLVZ16POhwRkV71mhTM\nLM3M1ncuu/vz7r7E3fWT9wS8Y0wht1xQwU9f3sWqNw9HHY6ISI96TQru3gG8amYT+imeIeuOy6Yx\npiiHzz+5nrb2jqjDERGJK5E6hTHABjN71syWdE6JHNzMFocd6W0xs7vjPP+psIO9NWb2gpnNONE/\nYLAYlp3BP79/Bq/vq+fhF3dEHY6ISFyJjKfwxZM5sJmlA/cDlwFVwEozW+Lur8Vs9hN3fyDc/irg\nm8CQHbvhvWeN5tIzR/KN327i0neMZHLZsKhDEhE5TiIlhSvDuoSuCbgygf3mA1vcfVtYB/EowQhu\nXdw9tg+IfGBI38xvZnz5A7PISk/jcz9bq7YLIjLgJJIULouz7ooE9hsHxN6cXxWuO46Z/Y2ZbQW+\nCtyewHEHtVGFOXzx6rNY9eYRHnphe9ThiIgcp7eusz9tZuuA6eHgOp3TdiCRQXYszrq3/TR29/vd\nfQrwD8D/6iGWW82s0swqq6sHf2Pqa84Zx2UzRvG1ZZvYcuBo1OGIiHTpq+vs9wNLwsfO6Tx3/2gC\nx64CymOWx9N7o7dH6aH9g7s/6O5z3X1uWVlZAi89sJkZX752FvlZ6dz1+Bpa2nQ3kogMDL11nV3r\n7jvc/Xp3fzNmSvRG+5XAVDObZGZZwHUECaaLmU2NWfwzIGUGICgryOb/fGAWa6tq+cayTVGHIyIC\nJNh19slw9zbgNoJeVTcCj7v7BjO7N7zTCOA2M9tgZmuAu4CbkhXPQLR45hhuXDCB7/1hm3pSFZEB\nwdwH1x0wc+fO9crKoTPwW3NrO1d/+0UONRxj6WcuYmRBTtQhicgQZGar3H1uX9slraQgicnJTOe+\nG+Zw9Fgbdz32qm5TFZFIJTLIzgfM7A0zqzWzOjOrNzONMXkaTRtVwL1XzeSFLQdVvyAikUqkRfNX\ngfe7u8aVTKIPzytn9a4jfOf3W5k9vpjFM0dHHZKIpKBELh/tV0LoH/dcdRZnlxfz2SdeVfsFEYlE\nIkmh0sweM7Prw0tJHzCzDyQ9shSUnZHOd288l+yMND75n5XUNrVGHZKIpJhEkkIh0AhczlsN2N6X\nzKBS2djiXO6/8Vx2Hm7ktp+8om62RaRf9Vmn4O639Ecg8pbzJ4/gX6+Zxd//fC33/GoDX7p6Jmbx\neg0RETm9ekwKZvb37v5VM7uP+H0WDfnO66L04XnlbK0+yvf+sI0pZcO45cJJUYckIimgt5JCZ+Xy\n0GkpNsj8/eIz2XawgXuffo1RhTlcOWtM1CGJyBDXY1Jw91+Fjz/qv3AkVnqa8a3r5vDRH67gjkfX\nUJybyQVnlEYdlogMYWrRPMDlZqXzw5vmUlGaxyd+XMm6qtqoQxKRIUxJYRAozsvixx9bQHFeFjc/\n/DLbDzZEHZKIDFG9JgUzSzezO/srGOnZ6KIcfvzx+TjwFz9cwf665qhDEpEhqNek4O7tdBtXWaIz\npWwYj9wyjyMNLVz//Zc4oMQgIqdZIpePXjSzb5vZRWZ2bueU9Mgkrtnji3nkY/PZV9vMdUoMInKa\n9Tmegpk9F2e1u/slyQmpd0NtPIWTtXLHYW566GVGF+Xw6CfOZ2ShxmEQkZ6dtvEU3P3dcaZIEoK8\nZV5FCT/62Hz2q8QgIqdRIuMpFJnZN82sMpy+YWZF/RGc9G5eRQmPdCaGB19iT01T1CGJyCCXSJ3C\nQ0A98OFwqgMeTmZQkrjOEkN1/TE++N3lbDlQH3VIIjKIJZIUprj7P7v7tnD6IjA52YFJ4uZWlPDY\nJxfS2u586IE/sXrnkahDEpFBKpGk0GRmizoXzOxCQNcpBpgZYwv5xacvoDAnkxu+v4LnN1dHHZKI\nDEKJJIVPAfeb2Q4z2wF8G/hkUqOSkzJhRB4/+/RCKkrz+fgjK3lq9e6oQxKRQaavFs1pwHR3PxuY\nDcx29znuvrZfopMTNrIgh8c+eT7nTRzOHY+t4Zu/3Uxftx2LiHTqq0VzB3BbOF/n7nX9EpWcksKc\nTH788fl86LzxfOvZN7jtp6tpbm2POiwRGQQSuXz0WzP7rJmVm1lJ55T0yOSUZGek87UPzebuK85k\n6bq9fORBtWUQkb4l0qJ5e5zV7u6R3IGkFs0n7pkN+4LxGPIy+d5fnMfs8cVRhyQi/ey0tGgO6xQ+\n6u6Tuk0JJQQzW2xmm8xsi5ndHef5u8zsNTNba2bPmtnERI4rJ+a9Z43mZ59eSJoZH/run/jJip2q\nZxCRuBKpU/j6yRzYzNKB+4ErgBnA9WY2o9tmq4G57j4b+Bnw1ZN5LenbWWOLePpvF3H+lBH805Pr\n+OwTa2lqUT2DiBwvkTqFZWb2QTOzEzz2fGBL2OCtBXiUbt1wu/tz7t4YLr4EjD/B15ATMDw/i4dv\nnsftl07l569Uce13XmSHBuwRkRiJJIW7gCeAFjOrM7N6M0vkLqRxwK6Y5apwXU8+Dvw63hNmdmtn\n30vV1WqUdSrS04y7LpvGwzfPY29tM++77wWeXF0VdVgiMkAk0ktqgbunuXumuxeGy4UJHDteySLu\nhWwz+ygwF/haDzE86O5z3X1uWVlZAi8tfXn3mSP579sXceboAu587FU+8+hq6ppbow5LRCKWSC+p\nZmYfNbP/HS6Xm9n8BI5dBZTHLI8H9sQ5/nuAzwNXufuxxMKW02H88DwevfV87nzPNJ5eu5cr/+OP\nrHrzcNRhiUiEErl89B1gIXBDuHyUoAK5LyuBqWY2ycyygOuAJbEbmNkc4HsECeFAwlHLaZORnsZn\n3jOVxz+5EDP48wf+xNeeeZ1jbaqEFklFiSSFBe7+N0AzgLsfAbL62snd2whaQz8DbAQed/cNZnav\nmV0VbvY1YBjwhJmtMbMlPRxOkuy8icNZevtFfODc8dz/3Fbef98LrNlVE3VYItLPEmm8tgK4AFjp\n7ueaWRmwzN3n9EeA3anxWvI9t+kA//SLdeyva+YTF03mzsumkZOZHnVYInIKTttwnMC3gCeBkWb2\nr8ALwJdPMT4ZwN49fSTP3PlOPjKvnO/9YRtX/scfWb7lYNRhiUg/6LOkAGBmZwKXEtxR9Ky7b0x2\nYD1RSaF//fGNaj7/5Hp2Hm7kmnPG8k9/9g5GFuREHZaInKBESwoJJYWBREmh/zW3tvOd57bwwPPb\nyM5M43Pvnc6NCyaSnnai7RlFJCqn8/KRpLiczHTuunw6v7njIs4eX8wXfrmB99/3Asu36pKSyFCj\npCAJm1w2jP/8+Hy+fcMcaptaueH7K/jEjyvZrq4yRIYMJQU5IWbG+2aP5dm/exefe+90lm85yOX/\n/jxfevo1jjS0RB2eiJwi1SnIKTlQ38w3ntnM46t2kZ+VwV9dNImPL5pEQU5m1KGJSAxVNEu/2rSv\nnm/+dhPPbNjP8LxMPvWuKfzlwgpys9S+QWQgUFKQSKytquHryzbzh83VjCzI5rZLzuAj88rJzlBy\nEImSkoJEasW2Q3xj2WZe3nGYUYXZ/NWiydywYAL52RlRhyaSkpQUJHLuzotbDnH/c1v407ZDFOVm\nctMFFdxyQQXD8/vsPktETiMlBRlQVu88wnd+v5Xfvraf3Mx0blgwgY8tmsS44tyoQxNJCUoKMiBt\n3l/PA7/fyi9f3YO7896zRnPLhZOYVzGcEx/xVUQSpaQgA9rumiZ+/KcdPPryLmqbWpkxppCbL6zg\nqrPHqkdWkSRQUpBBoamlnafW7ObhF7ezef9RSvKz+PDccq6bV05FaX7U4YkMGUoKMqi4O3/aeoiH\nl+/gd68foL3DWTh5BNfNL+e9Z41W6UHkFCWaFHR/oAwIZsYFZ5RywRml7K9r5onKXTxWuYvPPLqG\n4rxMrp0zjo/MK+fM0YVRhyoypKmkIANWR4ezfOshfrpyJ8s27KO13XnHmEKunTOWq84ex+gijesg\nkihdPpIh5dDRYzy9di9Prt7Nml01mMGFU0q5Zs44Fs8czTA1ihPplZKCDFnbqo/y1Jo9PLV6NzsP\nN5KVkca7ppVx5azRXPqOURSqMz6Rt1FSkCHP3Xll5xGeXruXX6/bx766ZrLS01g0tZQrZo7mshmj\nKM5Ty2kRUFKQFNPR4aypqmHp2r38ev0+dtc0kZEWVF5fPmMUl5w5krFqPS0pTElBUpa7s7aqlqXr\ngxLEzsONALxjTCGXnjmSS94xkrPHF2uMaUkpSgoiBAlia/VRnt14gGdfP8CqN4/Q3uGMyM/i4ukj\nueTMkVx4xghdZpIhT0lBJI7axlaef6Oa323cz3ObqqltasUMZo8r4sIzSlk0tZTzJg7X+A8y5AyI\npGBmi4H/ANKBH7j7v3V7/p3A/wVmA9e5+8/6OqaSgpwube0dvFpVwx/fOMgLbxxk9a4a2juc3Mx0\n5k8q4aKppSycMoIzRxfqUpMMepEnBTNLBzYDlwFVwErgend/LWabCqAQ+CywRElBolTf3MpL2w7z\n4paD/PGNarZWNwBQmJPBvIoS5k8KppnjishMT4s4WpETMxC6uZgPbHH3bWFAjwJXA11Jwd13hM91\nJDEOkYQU5GRy2YxRXDZjFAB7appYsf0QL28/zIrth3n29QMA5Gamc97E4V1J4pzyYvXNJENGMpPC\nOGBXzHIVsCCJrydyWo0tzuXaOeO5ds54AKrrj7Fyx2FWbDvEiu2H+ff/2Yw7ZKYbM8YUMmfCcM4p\nL2bOhGImlORpfAgZlJKZFOL9R5zUtSozuxW4FWDChAmnEpPISSsryObKWWO4ctYYAGoaW6jccYSV\nbx5mzc4aHlu5i0eW7wCgJD+LOeXFYZIYzuzyIrW0lkEhmUmhCiiPWR4P7DmZA7n7g8CDENQpnHpo\nIqeuOC+L98wYxXvCy01t7R1s3n+U1buOsGZnDat31XRdcgKYVJrPWWMLOWtsETPHBY8lGqtaBphk\nJoWVwFQzmwTsBq4Dbkji64lEKiM9jRljC5kxtpAbF0wEoLaplbVVNazZWcOGPXWs2VXD02v3du0z\nrjiXGWMLmRkmipnjihhZkK1LTxKZZN+SeiXBLafpwEPu/q9mdi9Q6e5LzGwe8CQwHGgG9rn7Wb0d\nU3cfyWBX09jChj11bNhTy/rddazfU8v2gw10/iuOyM9i2qgCpo8Ops559QQrpyLyW1KTRUlBhqKG\nY21s3FvHut21vL63ntf31/PG/noaW9q7thlXnMuZowuYNrogeBxVwKTSfN35JAkZCLekikiC8rMz\nmFtRwtyKkq51HR3O7pomXt9Xz+b99cHjvnqe31xNW0fwYy7NYPzwPCaX5TOlbNhxj2XDdBlKTpyS\ngsgAlZZmlJfkUV6S19V2AqClrYPtBxvYtL+erQeOsrX6KNuqG3hp2yGaW99q8lOQncHkkcOYUprP\nlJHDmFyaz6SyfCaU5JGXpX99iU+fDJFBJisjrau+IVZHh7O3rpmtB46yrfoo2w42sLX6KH/adohf\nrN593Lalw7KZOCKPiWHSmTgimCaU5FM6LEsljBSmpCAyRKSlGeOKcxlXnMs7p5Ud91zDsTa2H2xg\n+8EGdh5uZOehRt48HJQunlyzm9iqxbysdCaU5DGhpDNR5DG+JK/r2Pmq8B7S9O6KpID87Axmjiti\n5riitz3X3NrO7pqmIFEcauDNw43sOtzI9oMNPL+5mmNtx/dCU5SbGSSI4bldiWJszLJKGoObkoJI\nisvJTGdK2TCmlA1723MdHc6B+mPsrmmk6kgTe2qa2V3TyO4jQRL509ZDHD3Wdtw+WRlpYaLIYXRh\nLqOLshldmMOowhxGFwWPpcOy1fPsAKWkICI9SkszRhcFX+bnTXz78+5OXVMbu2ua2F3TxJ7wcfeR\n4HH51oMcqD9Ge8fxt76npxllw7IZVZTD6MIwaRTlMLowp2t+ZEE2w7IzVOroZ0oKInLSzIyivEyK\n8jKZMbYw7jbtHc6ho8fYV9fMvtpm9tc1h/PH2F/XzNbqBpZvOUR9txIHQE5mGmUF2ZQOy6ZsWPZb\n8wVvzY8MH3Oz1F7jdFBSEJGkSk8zRhbmMLIwh9nje96u4Vgb++qa2V8bJI3q+mNU1x/j4NFjVB89\nxo5DDVS+eYTDDS1x9x+WnRFRNP5iAAAKhUlEQVQki2HZlBZkUTYsm5L8bEqGZVGSl0VJfhYjhmUx\nPC+L4XmZZGhMjLiUFERkQMjPzuixbiNWa3sHhxtaupJG9dGY5BE+btpXzwv1B6lrfnvpo1NRbiYj\n8rMYnh8kjJK8rOMSSOz88Pws8rPSU+JSlpKCiAwqmelpjAorrvvS2t7BkcYWDje8NR1paOFQw/Hr\ndh1u5NVdNRxpbKG1PX7XPxlpRnFeJkW5wVScl0VxbnDprDg3i6LcDIrzssLl4Pmi3EwKczIGValE\nSUFEhqzM9DRGFuQwsqDvBAJBxXn9sTYOH23hcGNL8NjQQk1TCzWNrdQ0tVLb2EptUysH6pvZvL+e\n2sbWuPUhsQpyMijuSh6ZXYmjMDeTwpxMCnMzKMgJEkjsusKczH7v20pJQUQkZGbBF3JOJhXkJ7xf\nW3sHdc1t1DS2dCWOmqaW8LGVmjCR1DS2UNvUyp7aJmobW6lrbu2xZNIpKyMtSBY5mdxx2TSuOnvs\nqf6ZvVJSEBE5RRnpaUE9xAkOmuTuHGvroK4pSBC1TW3UNbeGy23UN7dSF7NueF7yR+9TUhARiYiZ\nkZOZTk5mOiMTqCPpD4On9kNERJJOSUFERLooKYiISBclBRER6aKkICIiXZQURESki5KCiIh0UVIQ\nEZEu5t57E+uBxsyqgTdPcvdS4OBpDOd0GqixKa4To7hO3ECNbajFNdHdy/raaNAlhVNhZpXuPjfq\nOOIZqLEprhOjuE7cQI0tVePS5SMREemipCAiIl1SLSk8GHUAvRiosSmuE6O4TtxAjS0l40qpOgUR\nEeldqpUURESkFymTFMxssZltMrMtZnZ3hHGUm9lzZrbRzDaY2WfC9feY2W4zWxNOV0YQ2w4zWxe+\nfmW4rsTMfmtmb4SPw/s5pukx52SNmdWZ2R1RnS8ze8jMDpjZ+ph1cc+RBb4VfubWmtm5/RzX18zs\n9fC1nzSz4nB9hZk1xZy7B/o5rh7fOzP7x/B8bTKz9yYrrl5ieywmrh1mtiZc3y/nrJfvh/77jLn7\nkJ+AdGArMBnIAl4FZkQUyxjg3HC+ANgMzADuAT4b8XnaAZR2W/dV4O5w/m7gKxG/j/uAiVGdL+Cd\nwLnA+r7OEXAl8GvAgPOBFf0c1+VARjj/lZi4KmK3i+B8xX3vwv+DV4FsYFL4P5ven7F1e/4bwBf6\n85z18v3Qb5+xVCkpzAe2uPs2d28BHgWujiIQd9/r7q+E8/XARmBcFLEk6GrgR+H8j4BrIozlUmCr\nu59s48VT5u5/AA53W93TOboa+LEHXgKKzWxMf8Xl7svcvXNE+ZeA8cl47RONqxdXA4+6+zF33w5s\nIfjf7ffYzMyADwM/Tdbr9xBTT98P/fYZS5WkMA7YFbNcxQD4IjazCmAOsCJcdVtYBHyovy/ThBxY\nZmarzOzWcN0od98LwQcWGBlBXJ2u4/h/0qjPV6eeztFA+tx9jOAXZadJZrbazJ43s4siiCfeezeQ\nztdFwH53fyNmXb+es27fD/32GUuVpGBx1kV625WZDQN+Dtzh7nXAd4EpwDnAXoKia3+70N3PBa4A\n/sbM3hlBDHGZWRZwFfBEuGognK++DIjPnZl9HmgD/itctReY4O5zgLuAn5hZYT+G1NN7NyDOV+h6\njv8B0q/nLM73Q4+bxll3SucsVZJCFVAeszwe2BNRLJhZJsEb/l/u/gsAd9/v7u3u3gF8nyQWm3vi\n7nvCxwPAk2EM+zuLo+Hjgf6OK3QF8Iq77w9jjPx8xejpHEX+uTOzm4D3ATd6eBE6vDxzKJxfRXDt\nflp/xdTLexf5+QIwswzgA8Bjnev685zF+36gHz9jqZIUVgJTzWxS+IvzOmBJFIGE1yp/CGx092/G\nrI+9DngtsL77vkmOK9/MCjrnCSop1xOcp5vCzW4CftmfccU47pdb1Oerm57O0RLgL8M7RM4Hajsv\nAfQHM1sM/ANwlbs3xqwvM7P0cH4yMBXY1o9x9fTeLQGuM7NsM5sUxvVyf8UV4z3A6+5e1bmiv85Z\nT98P9OdnLNm16QNlIqil30yQ4T8fYRyLCIp3a4E14XQl8J/AunD9EmBMP8c1meDOj1eBDZ3nCBgB\nPAu8ET6WRHDO8oBDQFHMukjOF0Fi2gu0EvxK+3hP54igaH9/+JlbB8zt57i2EFxv7vycPRBu+8Hw\nPX4VeAV4fz/H1eN7B3w+PF+bgCv6+70M1z8CfKrbtv1yznr5fui3z5haNIuISJdUuXwkIiIJUFIQ\nEZEuSgoiItJFSUFERLooKYiISBclBRmyzOz3Zpb0MXbN7PawV8v/6rZ+rpl9K5y/2MwuOI2vWWFm\nN8R7LZFTkRF1ACIDkZll+FudyfXlrwnuqd8eu9LdK4HKcPFi4Ciw/DTFUAHcAPwkzmuJnDSVFCRS\n4S/ejWb2/bD/+GVmlhs+1/VL38xKzWxHOH+zmT1lZr8ys+1mdpuZ3RV2VvaSmZXEvMRHzWy5ma03\ns/nh/vlhR2wrw32ujjnuE2b2K2BZnFjvCo+z3szuCNc9QNDwb4mZ3dlt+4vN7OmwY7NPAXda0Bf/\nRWEL2Z+HMaw0swvDfe4xswfNbBnw4/D8/NHMXgmnztLGvwEXhce7s/O1wmOUhOdnbXg+Zscc+6Hw\nvG4zs9tjzsd/m9mr4d/2kVN7V2VQS2aLQU2a+poIfvG2AeeEy48DHw3nf0/YQhMoBXaE8zcTtNYt\nAMqAWsIWqMC/E3Qi1rn/98P5dxL2hw98OeY1iglauueHx60iTqtt4DyCFqP5wDCC1q1zwud20G0c\ninD9xcDT4fw9xIwhQPALf1E4P4GgW4PO7VYBueFyHpATzk8FKrsfO85r3Qf8czh/CbAm5tjLCcYr\nKCVoJZ5J0Fr3+zHHKur+t2hKnUmXj2Qg2O7ua8L5VQSJoi/PedDffL2Z1QK/CtevA2bHbPdTCPrO\nN7NCC0Yfuxy4ysw+G26TQ/DFDPBbd4/Xx/4i4El3bwAws18QdK+8OpE/MI73ADOCrm4AKOzsewpY\n4u5N4Xwm8G0zOwdoJ7FO2BYRfNHj7r8zsxFmVhQ+99/ufgw4ZmYHgFEE5+zrZvYVgsTyx5P8m2QI\nUFKQgeBYzHw7kBvOt/HWJc6cXvbpiFnu4PjPdfd+XJygv5gPuvum2CfMbAHQ0EOM8booPhVpwMKY\nL//OGOgWw53AfuDscJ/mBI7dW3fK3c91hrtvNrPzCPrY+T9mtszd703or5AhR3UKMpDtILhsA/Ch\nkzzGRwDMbBFBD5K1wDPA34Y9UmJmcxI4zh+Aa8wsz4JeZK8FTuQXdT3B5a5Oy4DbOhfCkkA8RcBe\nD7qZ/guCIUnjHa97rDeGx70YOOi99MlvZmOBRnf/f8DXCYaolBSlpCAD2deBT5vZcoJr4CfjSLj/\nAwQ9dAJ8ieCyzFoLBm3/Ul8H8WCIxEcIunJeAfzA3U/k0tGvgGs7K5qB24G5YWXwawQV0fF8B7jJ\nzF4iuHTUWYpYC7SFlcN3dtvnns5jE1RI30TvZgEvWzBI/eeBfzmBv0uGGPWSKiIiXVRSEBGRLkoK\nIiLSRUlBRES6KCmIiEgXJQUREemipCAiIl2UFEREpIuSgoiIdPn/8WZaqJXHyPsAAAAASUVORK5C\nYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEKCAYAAAD9xUlFAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4zLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvIxREBQAAHI9JREFUeJzt3X+8XHV95/HXe2b4oZCEQmIXSUIC\nDT42tSp4F1AQEagbWCUqriXIFtQtVUQESmv6oIuI3bagrfuQomxUCrgoYAsabCjhURFcMZgbfkR+\nBW4hLFcoRMEE5WeSz/5xzgzDcO69517mzMzJeT8fj3nMOd85c+Zzz8ydz3zP98dRRGBmZgZQ63cA\nZmY2OJwUzMysxUnBzMxanBTMzKzFScHMzFqcFMzMrMVJwczMWpwUzMysxUnBzMxaGv0OYLJmzpwZ\n8+bN63cYZmalsmbNml9ExKyJtitdUpg3bx7Dw8P9DsPMrFQkPZxnO58+MjOzFicFMzNrcVIwM7MW\nJwUzM2txUjAzs5bCkoKkiyU9IemuMR6XpC9LGpG0VtJ+RcViZmb5FFlTuARYNM7jRwIL0ttJwFcL\njMXMzHIobJxCRNwsad44mywGLovkeqCrJO0iafeIeKyIeFavf5If3b8h9/aH7DOLoXm7FhGKmdnA\n6ufgtT2AR9rWR9OyVyQFSSeR1CaYO3fulF7stoef4oIbR3JtGwE/Xf8kV5z0tim9lplZWfUzKSij\nLLI2jIhlwDKAoaGhzG0m8sfv3Js/fufeubb98NdX8dyLW6fyMmZmpdbP3kejwJy29dnAo32K5WXq\ntRpbtk4p95iZlVo/k8Jy4A/TXkgHAhuLak+YrEZNTgpmVkmFnT6S9G3gUGCmpFHgs8B2ABFxEbAC\nOAoYAZ4BPlJULJNVr4nNTgpmVkFF9j5aMsHjAXyyqNd/NZKagtsUzKx6PKI5g2sKZlZVTgoZ3KZg\nZlXlpJChXquxeYuTgplVj5NCBtcUzKyqnBQy1OtuUzCzanJSyODeR2ZWVU4KGdz7yMyqykkhg9sU\nzKyqnBQy1Gs11xTMrJKcFDK4pmBmVeWkkKGeJoVkJg4zs+pwUsjQqCWXenBtwcyqxkkhQ72eJAW3\nK5hZ1TgpZHBNwcyqykkhQ72WHBbXFMysapwUMrimYGZV5aSQoV5rtil4qgszqxYnhQyuKZhZVTkp\nZKg1awq+poKZVYyTQoZmTWGrB6+ZWcU4KWR4qU3BScHMqsVJIUMj7ZLqNgUzqxonhQx1tymYWUU5\nKWRw7yMzqyonhQwvzX3kcQpmVi1OChlcUzCzqnJSyODeR2ZWVU4KGdz7yMyqykkhg2sKZlZVTgoZ\nXmpTcEOzmVVLoUlB0iJJ6ySNSFqa8fhcSTdKul3SWklHFRlPXh6nYGZVVVhSkFQHLgSOBBYCSyQt\n7NjsL4CrImJf4FjgK0XFMxmNunsfmVk1FVlT2B8YiYgHI+IF4Apgccc2AUxPl2cAjxYYT24NtymY\nWUU1Ctz3HsAjbeujwAEd25wDrJT0KWAn4IgC48mt7t5HZlZRRdYUlFHW+S27BLgkImYDRwHflPSK\nmCSdJGlY0vCGDRsKCPXlXFMws6oqMimMAnPa1mfzytNDHwOuAoiInwA7AjM7dxQRyyJiKCKGZs2a\nVVC4L6m795GZVVSRSWE1sEDSfEnbkzQkL+/Y5v8BhwNI+o8kSaH4qsAEXFMws6oqLClExGbgFOB6\n4F6SXkZ3SzpX0tHpZn8C/JGkO4FvAydG9P9yZ3XPfWRmFVVkQzMRsQJY0VF2dtvyPcBBRcYwFc1p\nLjxOwcyqxiOaM9Q9TsHMKspJIUNdblMws2pyUsjQbFPY2v/mDTOznnJSyNDw3EdmVlETNjRLmgV8\nhmT+oh2b5RFxWIFx9VWtJiSPUzCz6slTU7icpEvpfOBzwHqSMQjbtEZNblMws8rJkxR2i4hvAC9G\nxE0R8VHgwILj6rt6Te59ZGaVk2ecwovp/WOS/gvJVBWziwtpMDRqNdcUzKxy8iSFv5Q0g2T08QUk\nU12fVmhUA8A1BTOrojxJ4amI2AhsBN4FIGngRiF3W9Km4IZmM6uWPG0KF+Qs26a4pmBmVTRmTUHS\n24C3A7MkndH20HSgXnRg/daoyeMUzKxyxjt9tD2wc7rNtLbyTcAHiwxqENTrrimYWfWMmRQi4ibg\nJkmXRMTDPYxpILj3kZlVUZ6G5mckfQH4XSoyohncpmBm1ZR3RPN9VHJEs3sfmVm1eETzGFxTMLMq\n8ojmMXjuIzOroqmOaD690KgGgGsKZlZFEyaFiPh+utga0VwFjVrN4xTMrHLGG7x2ATDmt2JEnFpI\nRAPCNQUzq6LxGpqHgTUk3VD3Ax5Ib28BthQfWn816u59ZGbVM97gtUsBJJ0IvCsiXkzXLwJW9iS6\nPnJNwcyqKE+X1Nfz8mkudk7Ltml1ufeRmVVPnt5HfwPcLunGdP2dwDmFRTQgXFMwsyrK0/voHyRd\nBxyQFi2NiH8vNqz+a3hCPDOroDw1BdIk8L2CYxko9VrNScHMKidPm0IleUSzmVWRk8IY3KZgZlU0\n4ekjSbtmFD/d7KK6rfIsqWZWRXlqCrcBG4D7SQavbQAeknSbpLcWGVw/uaZgZlWUJyn8C3BURMyM\niN2AI4GrgJOBr4z3REmLJK2TNCJp6RjbfEjSPZLulvStyf4BRXGbgplVUZ6kMBQR1zdXImIlcEhE\nrAJ2GOtJkurAhSRJZCGwRNLCjm0WAH8OHBQRvwucNvk/oRj1Wo0tnhDPzComT1J4UtJnJO2Z3v4M\neCr90h/vpPv+wEhEPBgRLwBXAIs7tvkj4MKIeAogIp6Ywt9QiGTuIycFM6uWPEnhOJKL6nyXZKzC\n3LSsDnxonOftATzStj6alrXbB9hH0o8lrZK0KGtHkk6SNCxpeMOGDTlCfvXcpmBmVZRnRPMvgE+N\n8fDIOE9V1u4yXn8BcChJ4vmRpDdGxK86YlgGLAMYGhrqyTe1ex+ZWRXl6ZK6D3AmMK99+4g4bIKn\njgJz2tZnk1zKs3ObVWn31ockrSNJEqsnjLxg9ZrYGrB1a1CrZeU3M7NtT55pLr4DXAR8ncldR2E1\nsEDSfODnwLEkp53afRdYAlwiaSbJ6aQHJ/EahWmkiWBLBLXMSo+Z2bYnT1LYHBFfneyOI2KzpFOA\n60naHy6OiLslnQsMR8Ty9LF3S7qHJOH8aUT8crKvVYR6LWlu2bI12K7e52DMzHokT1K4VtLJwDXA\n883CiHhyoidGxApgRUfZ2W3LAZyR3gZKs6bgHkhmViV5ksIJ6f2ftpUFsFf3wxkc9ebpI49VMLMK\nydP7aH4vAhk0jXqzpuAeSGZWHWMmBUmHRcQPJH0g6/GIuLq4sPqvVVPw6SMzq5DxagrvBH4AvDfj\nsQC26aTgNgUzq6Ixk0JEfDa9/0jvwhkc7b2PzMyqIs/gtR2AY3jl4LVziwur/1xTMLMqytP76HvA\nRmANbV1St3W1VpuCG5rNrDryJIXZEZE5Ud22rDWi2TnBzCokzyypt0j6vcIjGTD1mrukmln15Kkp\nHAycKOkhktNHIhmM/KZCI+uzhrukmlkF5UkKRxYexQCqu6HZzCpovMFr0yNiE/B0D+MZGA13STWz\nChqvpvAt4D0kvY6Cl180pzJzH2323EdmViHjDV57T3pf6bmPvnTD/Vz2k+1zPacm8YlD9+aNe8wo\nMjQzs8LkaVNA0m+RXBFtx2ZZRNxcVFCDYO9ZO7Pf3F341bMv8KtnX8j1nPsf/zV77vZaJwUzK608\nI5r/O/Bpkstp3gEcCPwEmOhynKW2607bc/XJB03qOW/4i+vYEj7dZGbllWecwqeB/wQ8HBHvAvYF\nNhQaVUk1avL1F8ys1PIkheci4jlI5kGKiPuANxQbVjnVa3IXVjMrtTxtCqOSdgG+C9wg6Sng0WLD\nKqdGveYurGZWanmuvPb+dPEcSTcCM4B/KTSqknJNwczKbtykIKkGrI2INwJExE09iaqkGjV5VlUz\nK7Vx2xQiYitwp6S5PYqn1FxTMLOyy9OmsDtwt6SfAr9pFkbE0YVFVVJJTcFJwczKK09S+FzhUWwj\nXFMws7LLkxSOiojPtBdIOg9w+0KHRq3mcQpmVmp5xin8fkZZJafTnohrCmZWduNNnf0J4GRgL0lr\n2x6aBvy46MDKqFF37yMzK7eJps6+DvhrYGlb+dMR8WShUZWUawpmVnbjTZ29EdgILOldOOXm3kdm\nVnZ52hQsJ9cUzKzsCk0KkhZJWidpRNLScbb7oKSQNFRkPEWru6ZgZiVXWFKQVAcuJOmptBBYImlh\nxnbTgFOBW4uKpVfqNU+IZ2blNmFSkPQBSQ9I2ihpk6SnJW3Kse/9gZGIeDAiXgCuABZnbPd54Hzg\nuUlFPoDcpmBmZZenpnA+cHREzIiI6RExLSKm53jeHsAjbeujaVmLpH2BORHx/dwRDzC3KZhZ2eVJ\nCo9HxL1T2LcyylrfmOkMrF8C/mTCHUknSRqWNLxhw+Be9M2zpJpZ2eWZ5mJY0pUkF9l5vlkYEVdP\n8LxRYE7b+mxefnGeacAbgR9KAvgPwHJJR0fEcPuOImIZsAxgaGhoYH+Ku6ZgZmWXJylMB54B3t1W\nFsBESWE1sEDSfODnwLHAca0dJOMgZjbXJf0QOLMzIZSJ2xTMrOzyXHntI1PZcURslnQKcD1QBy6O\niLslnQsMR8Tyqex3kNVrNTZ7QjwzK7Hx5j76s4g4X9IFtLUFNEXEqRPtPCJWACs6ys4eY9tDJ4x2\nwLmmYGZlN15Nodm4XNrTOb1Wr7tNwczKbby5j65N7y/tXTjl5t5HZlZ2nvuoi9z7yMzKzkmhi9ym\nYGZlN25SkFSXdHqvgim7eq3mmoKZldq4SSEitpA9X5FlcE3BzMouz+C1H0v6e+BK4DfNwoi4rbCo\nSqo5dXZEkI7SNjMrlTxJ4e3p/bltZQEc1v1wyq1RSxLBlq1Bo+6kYGblk2dE87t6Eci2oJ4mgs1b\ng0a9z8GYmU1BnuspzJD0d81ZSiX9raQZvQiubNprCmZmZZSnS+rFwNPAh9LbJuAfigyqrOq15HC6\nB5KZlVWeNoW9I+KYtvXPSbqjqIDKzDUFMyu7PDWFZyUd3FyRdBDwbHEhlVe91mxT8FQXZlZOeWoK\nHwcua2tHeAo4obiQyqvumoKZldy4SSG9ZOYbIuLNkqYDRMSmnkRWQk4KZlZ2E41o3gqcki5vckIY\nn9sUzKzs8rQp3CDpTElzJO3avBUeWQm91KbgpGBm5ZSnTeGj6f0n28oC2Kv74ZRbI+2S6pqCmZVV\nnjaF4yPixz2Kp9RaNQVfp9nMSipPm8IXexRL6blNwczKLk+bwkpJx8jTfk7opbmPPE7BzMopT5vC\nGcBOwBZJzwICIiKmFxpZCbmmYGZll2eW1Gm9CGRb4N5HZlZ2eWZJlaTjJf2PdH2OpP2LD6183PvI\nzMouT5vCV4C3Acel678GLiwsohJzTcHMyi5Pm8IBEbGfpNsBIuIpSdsXHFcpvdSm4IZmMyunPDWF\nFyXVSQasIWkW4G+9DB6nYGZllycpfBm4BnidpP8J/F/grwqNqqSa12V2m4KZlVWe3keXS1oDHE7S\nHfV9EXFv4ZGVUMNtCmZWcnnaFIiI+4D7Co6l9OrufWRmJZfn9JHl5JqCmZVdoUlB0iJJ6ySNSFqa\n8fgZku6RtFbSv0ras8h4ilZ37yMzK7nCkkLaY+lC4EhgIbBE0sKOzW4HhiLiTcA/AucXFU8vuKZg\nZmVXZE1hf2AkIh6MiBeAK4DF7RtExI0R8Uy6ugqYXWA8hfPlOM2s7IpMCnsAj7Stj6ZlY/kYcF3W\nA5JOkjQsaXjDhg1dDLG7mtNceJyCmZVVkUkha6rtzG9LSccDQ8AXsh6PiGURMRQRQ7NmzepiiN2V\n5gTXFMystHJ1SZ2iUWBO2/ps4NHOjSQdAZwFvDMini8wnsK1JsQLJwUzK6ciawqrgQWS5qdzJR0L\nLG/fQNK+wP8Gjo6IJwqMpSfcpmBmZVdYUoiIzcApwPXAvcBVEXG3pHMlHZ1u9gVgZ+A7ku6QtHyM\n3ZVCw3MfmVnJFXn6iIhYAazoKDu7bfmIIl+/12o1IXmcgpmVl0c0d1mjJo9TMLPSclLosnpNblMw\ns9JyUuiyRq3mmoKZlZaTQpe5pmBmZeak0GVJm4Ibms2snJwUusw1BTMrMyeFLmvU5HEKZlZaTgpd\nVq+7pmBm5eWk0GXufWRmZeak0GVuUzCzMnNS6DL3PjKzMnNS6DLXFMyszJwUusxzH5lZmTkpdJlr\nCmZWZk4KXdao1TxOwcxKy0mhy1xTMLMyc1LoskbdvY/MrLycFLqsJtcUzKy8nBS6zL2PzKzMnBS6\nzG0KZlZmTgpd1vCEeGZWYk4KXVav1ZwUzKy0nBS6zG0KZlZmTgpd5jYFMyszJ4Uu8yypZlZmTgpd\n5pqCmZWZk0KXuU3BzMrMSaHL6rUaWzwhnpmVlJNClyVzHzkpmFk5OSl0mdsUzKzMCk0KkhZJWidp\nRNLSjMd3kHRl+vitkuYVGU8vuPeRmZVZYUlBUh24EDgSWAgskbSwY7OPAU9FxO8AXwLOKyqeXqnX\nxNaAra4tmFkJFVlT2B8YiYgHI+IF4Apgccc2i4FL0+V/BA6XpAJjKlyjloS/JZwUzKx8GgXuew/g\nkbb1UeCAsbaJiM2SNgK7Ab8oMK5C1WtJnl30v26mVu78ZmYD5tTDF/DeN7++0NcoMilkfSN2/nzO\nsw2STgJOApg7d+6rj6xAv7/wddzz2Ca2uF3BzLpsxmu2K/w1ikwKo8CctvXZwKNjbDMqqQHMAJ7s\n3FFELAOWAQwNDQ30eZnfed00Lliyb7/DMDObkiLbFFYDCyTNl7Q9cCywvGOb5cAJ6fIHgR9E+GS8\nmVm/FFZTSNsITgGuB+rAxRFxt6RzgeGIWA58A/impBGSGsKxRcVjZmYTK/L0ERGxAljRUXZ22/Jz\nwH8tMgYzM8vPI5rNzKzFScHMzFqcFMzMrMVJwczMWpwUzMysRWUbFiBpA/DwFJ8+k8GdQmNQY3Nc\nk+O4Jm9QY9vW4tozImZNtFHpksKrIWk4Iob6HUeWQY3NcU2O45q8QY2tqnH59JGZmbU4KZiZWUvV\nksKyfgcwjkGNzXFNjuOavEGNrZJxVapNwczMxle1moKZmY2jMklB0iJJ6ySNSFraxzjmSLpR0r2S\n7pb06bT8HEk/l3RHejuqD7Gtl/Sz9PWH07JdJd0g6YH0/rd6HNMb2o7JHZI2STqtX8dL0sWSnpB0\nV1tZ5jFS4svpZ26tpP16HNcXJN2XvvY1knZJy+dJerbt2F3U47jGfO8k/Xl6vNZJ+s9FxTVObFe2\nxbVe0h1peU+O2TjfD737jEXENn8jmbr734C9gO2BO4GFfYpld2C/dHkacD+wEDgHOLPPx2k9MLOj\n7Hxgabq8FDivz+/jvwN79ut4AYcA+wF3TXSMgKOA60iuMHggcGuP43o30EiXz2uLa177dn04Xpnv\nXfp/cCewAzA//Z+t9zK2jsf/Fji7l8dsnO+Hnn3GqlJT2B8YiYgHI+IF4ApgcT8CiYjHIuK2dPlp\n4F6Sa1UPqsXApenypcD7+hjL4cC/RcRUBy++ahFxM6+8OuBYx2gxcFkkVgG7SNq9V3FFxMqI2Jyu\nriK5+mFPjXG8xrIYuCIino+Ih4ARkv/dnscmScCHgG8X9fpjxDTW90PPPmNVSQp7AI+0rY8yAF/E\nkuYB+wK3pkWnpFXAi3t9miYVwEpJa5RcFxvgtyPiMUg+sMDr+hBX07G8/J+038eraaxjNEifu4+S\n/KJsmi/pdkk3SXpHH+LJeu8G6Xi9A3g8Ih5oK+vpMev4fujZZ6wqSUEZZX3tdiVpZ+CfgNMiYhPw\nVWBv4C3AYyRV1147KCL2A44EPinpkD7EkEnJJV2PBr6TFg3C8ZrIQHzuJJ0FbAYuT4seA+ZGxL7A\nGcC3JE3vYUhjvXcDcbxSS3j5D5CeHrOM74cxN80oe1XHrCpJYRSY07Y+G3i0T7EgaTuSN/zyiLga\nICIej4gtEbEV+BoFVpvHEhGPpvdPANekMTzerI6m90/0Oq7UkcBtEfF4GmPfj1ebsY5R3z93kk4A\n3gN8ONKT0OnpmV+my2tIzt3v06uYxnnv+n68ACQ1gA8AVzbLennMsr4f6OFnrCpJYTWwQNL89Bfn\nscDyfgSSnqv8BnBvRPxdW3n7ecD3A3d1PrfguHaSNK25TNJIeRfJcToh3ewE4Hu9jKvNy3659ft4\ndRjrGC0H/jDtIXIgsLF5CqAXJC0CPgMcHRHPtJXPklRPl/cCFgAP9jCusd675cCxknaQND+N66e9\niqvNEcB9ETHaLOjVMRvr+4FefsaKbk0flBtJK/39JBn+rD7GcTBJ9W4tcEd6Owr4JvCztHw5sHuP\n49qLpOfHncDdzWME7Ab8K/BAer9rH47Za4FfAjPayvpyvEgS02PAiyS/0j421jEiqdpfmH7mfgYM\n9TiuEZLzzc3P2UXptsek7/GdwG3Ae3sc15jvHXBWerzWAUf2+r1Myy8BPt6xbU+O2TjfDz37jHlE\ns5mZtVTl9JGZmeXgpGBmZi1OCmZm1uKkYGZmLU4KZmbW4qRg2yxJP5RU+DV2JZ2azmp5eUf5kKQv\np8uHSnp7F19znqTjsl7L7NVo9DsAs0EkqREvTSY3kZNJ+tQ/1F4YEcPAcLp6KPBr4JYuxTAPOA74\nVsZrmU2ZawrWV+kv3nslfS2dP36lpNekj7V+6UuaKWl9unyipO9KulbSQ5JOkXRGOlnZKkm7tr3E\n8ZJukXSXpP3T5++UTsS2On3O4rb9fkfStcDKjFjPSPdzl6TT0rKLSAb+LZd0esf2h0r6fjqx2ceB\n05XMxf+OdITsP6UxrJZ0UPqccyQtk7QSuCw9Pj+SdFt6a9Y2/gZ4R7q/05uvle5j1/T4rE2Px5va\n9n1xelwflHRq2/H4Z0l3pn/bH7y6d9VKrcgRg775NtGN5BfvZuAt6fpVwPHp8g9JR2gCM4H16fKJ\nJKN1pwGzgI2kI1CBL5FMItZ8/tfS5UNI58MH/qrtNXYhGem+U7rfUTJGbQNvJRkxuhOwM8no1n3T\nx9bTcR2KtPxQ4Pvp8jm0XUOA5Bf+wenyXJJpDZrbrQFek66/FtgxXV4ADHfuO+O1LgA+my4fBtzR\ntu9bSK5XMJNklPh2JKN1v9a2rxmdf4tv1bn59JENgoci4o50eQ1JopjIjZHMN/+0pI3AtWn5z4A3\ntW33bUjmzpc0XcnVx94NHC3pzHSbHUm+mAFuiIisOfYPBq6JiN8ASLqaZHrl2/P8gRmOABYmU90A\nML059xSwPCKeTZe3A/5e0luALeSbhO1gki96IuIHknaTNCN97J8j4nngeUlPAL9Ncsy+KOk8ksTy\noyn+TbYNcFKwQfB82/IW4DXp8mZeOsW54zjP2dq2vpWXf64753EJkvlijomIde0PSDoA+M0YMWZN\nUfxq1IC3tX35N2OgI4bTgceBN6fPeS7HvsebTrnzWDci4n5JbyWZY+evJa2MiHNz/RW2zXGbgg2y\n9SSnbQA+OMV9/AGApINJZpDcCFwPfCqdkRJJ++bYz83A+yS9Vskssu8HJvOL+mmS011NK4FTmitp\nTSDLDOCxSKaZ/m8klyTN2l9nrB9O93so8IsYZ05+Sa8HnomI/wN8keQSlVZRTgo2yL4IfELSLSTn\nwKfiqfT5F5HM0AnweZLTMmuVXLT98xPtJJJLJF5CMpXzrcDXI2Iyp46uBd7fbGgGTgWG0sbge0ga\norN8BThB0iqSU0fNWsRaYHPaOHx6x3POae6bpEH6BMb3e8BPlVyk/izgLyfxd9k2xrOkmplZi2sK\nZmbW4qRgZmYtTgpmZtbipGBmZi1OCmZm1uKkYGZmLU4KZmbW4qRgZmYt/x9zy1i7Peq6PwAAAABJ\nRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Initialize weight\n",
    "W0 = 2*np.random.random((3,1)) - 1\n",
    "error = []\n",
    "error_of_rounded = []\n",
    "fs = []\n",
    "\n",
    "# Optimize for 2000 iterations\n",
    "for i in range(200):\n",
    "    # FORWARD PASS: x -> f1 \n",
    "    f0 = X\n",
    "    f1 = activation(np.dot(f0, W0))\n",
    "    loss = (0.5 * (y-f1)**2 ).sum()\n",
    "    loss_of_rounded = (0.5 * (y-np.round(f1))**2 ).sum()\n",
    "    \n",
    "    # BACKWARDS PASS (derivatives of loss wrt W0)\n",
    "    e1 = y - f1   # Error contribution\n",
    "    delta_1 = e1 * activation_derivative_f(f1)  # The gradient contribution from activation\n",
    "    gradient_1 = np.dot(f0.T,delta_1)\n",
    "    \n",
    "    learning_rate = 0.2\n",
    "    \n",
    "    # Update the weights. This is simply adding the gradient multiplied by a learning rate\n",
    "    W0 += learning_rate * gradient_1\n",
    "\n",
    "    # Keep track of error\n",
    "    error.append(loss)\n",
    "    error_of_rounded.append(loss_of_rounded)\n",
    "    fs.append(f1)\n",
    "\n",
    "plt.plot(error)\n",
    "plt.xlabel('number of iterations')\n",
    "plt.ylabel('error in training data')\n",
    "\n",
    "plt.figure()\n",
    "plt.plot(error_of_rounded)\n",
    "plt.xlabel('number of iterations')\n",
    "plt.ylabel('error in training data')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "## Deep neural network"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Create some toy data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = np.array([[0,0,1],\n",
    "              [0,1,1],\n",
    "              [1,0,1],\n",
    "              [1,1,1]])\n",
    "\n",
    "# NON-linear case\n",
    "y = np.array([[0],\n",
    "               [1],\n",
    "               [1],\n",
    "               [0]])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Define parameters and initialize them."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Fix the seed\n",
    "np.random.seed(1)\n",
    "\n",
    "# randomly initialize our weights with mean 0\n",
    "W0 = 2*np.random.random((3,4)) - 1\n",
    "W1 = 2*np.random.random((4,1)) - 1\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Optimize!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "error = []\n",
    "error_of_rounded = []\n",
    "\n",
    "# Optimize\n",
    "for i in range(1000):\n",
    "    # FORWARD PASS: x -> f1 -> f2\n",
    "    f0 = X\n",
    "    f1 = activation(np.dot(f0, W0))\n",
    "    f2 = activation(np.dot(f1, W1))\n",
    "    # Normally we'd use the cross-entropy loss, but here we'll use sq. error to make derivatives easier.\n",
    "    loss = (0.5 * (y-f2)**2 ).sum()\n",
    "    loss_of_rounded = (0.5 * (y-np.round(f2))**2 ).sum()\n",
    "    \n",
    "    # BACKWARDS PASS\n",
    "    e2 = y - f2\n",
    "    delta_2 = e2 * activation_derivative_f(f2)\n",
    "\n",
    "    e1 = np.dot(delta_2, W1.T)\n",
    "    delta_1 = e1 * activation_derivative_f(f1)\n",
    "\n",
    "    gradient_0 = f0.T.dot(delta_1)\n",
    "    gradient_1 = f1.T.dot(delta_2)\n",
    "        \n",
    "    # Update the weights\n",
    "    learning_rate = 0.8\n",
    "    W0 += learning_rate * gradient_0\n",
    "    W1 += learning_rate * gradient_1\n",
    "\n",
    "    # Keep track of error\n",
    "    error.append(loss)\n",
    "    error_of_rounded.append(loss_of_rounded)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Plot the error per iteration of the optimization."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Text(0,0.5,'error in training data')"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEKCAYAAAD9xUlFAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4zLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvIxREBQAAIABJREFUeJzt3Xd0HOXZ/vHvvaq2kWzLlnuRDMbG\nFYMwGBNiAwnGFNPBlFCSF/ICgUAIIcn7S4A0QkvDCS0QUuhJwHRT44RqGRdwxRV3y12uavfvjx0J\nocjSWtZqtLvX55w5OzM7O3vNjo9vTXsec3dEREQAImEHEBGR1kNFQUREaqgoiIhIDRUFERGpoaIg\nIiI1VBRERKSGioKIiNRQURARkRoqCiIiUiM97AD7qnPnzl5QUBB2DBGRhDJ9+vQN7p7f2HIJVxQK\nCgooLi4OO4aISEIxs+WxLKfTRyIiUkNFQUREaqgoiIhIjbgWBTMbZ2YLzGyRmd1cz/uXmlmJmc0M\nhm/EM4+IiDQsbheazSwNmAR8BVgJTDOzye4+t86iT7r7NfHKISIisYvnkcJIYJG7L3H3MuAJYEIc\nv09ERPZTPItCT2BFremVwby6zjKz2Wb2jJn1rm9FZnaFmRWbWXFJSUk8soqICPEtClbPvLp9fz4P\nFLj7MOB14NH6VuTuD7h7kbsX5ec3+uxFvWau2MK9b37K3NXbUBekIiL1i+fDayuB2n/59wJW117A\n3TfWmnwQ+GW8wnywZCN3TVnIXVMW0i03mzED8hkzoAujD+pETnZGvL5WRCShxLMoTAP6m1khsAo4\nH7ig9gJm1t3d1wSTpwHz4hXmyi8fyBmH9eTtBSW8vWA9L85ewxPTVpAeMY4oyOMrg7pyyvDudMnJ\njlcEEZFWz+J5KsXMxgO/BtKAh939Z2Z2G1Ds7pPN7BdEi0EFsAn4X3ef39A6i4qKvDmauSivrGL6\n8s28vaCEt+avZ8G6UiIGow/qzJmH9eSkId3Jzkjb7+8REWkNzGy6uxc1ulyinV9vrqJQ16L1pTw7\nYzXPzlzFys27yGuXyQUj+3DRUX3p1l5HDyKS2FQUmsjdeXfxRv707jJen7eONDNOH9GT647vT++8\ntnH7XhGReIq1KCRcK6nxZmaMPqgzow/qzGcbd/LwO0t57MPPeHbGKs4p6s11x/fXkYOIJC0dKcRg\n3bbdTHprEU98uIKMNOPa4/tz2ehCMtPVdJSIJIZYjxT0v1oMuuZmc9uEIbx+w5cZdWAnfvHyfE76\nzVQ+WLKx8Q+LiCQQFYV90KdTWx665AgevrSIssoqzn/wfX724lx2l1eGHU1EpFmoKDTBcQO78sp1\nx3LByD48+O+lnHbvf1i4rjTsWCIi+01FoYnaZaXzszOG8shlR7BpRzmnT3qH52etbvyDIiKtmIrC\nfho7oAsvXnsMh3TP5VuPz+C25+dSWZVYF+9FRKqpKDSDrrnZPP4/R3Hp0QU8/M5SrvzLdHaV6TqD\niCQeFYVmkpke4ZbTBnPbhMG8MX8dEx98n43b94QdS0Rkn6goNLOvjSrgvosOZ96abZxz33us3bo7\n7EgiIjFTUYiDEwd342/fOJL1pXs474H3WL1lV9iRRERioqIQJ0UFefz56yPZtL2M8x54j5Wbd4Yd\nSUSkUSoKcXRYn4789RtHsmVnORf/8UM26BqDiLRyKgpxNrx3Bx659AhWb9nFZY9MY/ueirAjiYjs\nlYpCCygqyOMPFx3G3DXbuPIvxeyp0O2qItI6qSi0kOMGduWOs4bxzqKN3PTMbBKtdVoRSQ3qT6EF\nnXV4L9Zu282dry7g4K45XD32oLAjiYh8gYpCC7tqzIEsXFfKna8u4MD8Axg3pFvYkUREauj0UQsz\nM3551jCG9+7ADU/NZO7qbWFHEhGpoaIQguyMNB68+HByszP45l+ns3VXediRREQAFYXQdMnNZtKF\nI1i9ZRfffXqWLjyLSKugohCiw/vm8f3xhzBl7joe+vfSsOOIiKgohO3y0QWcNKQbt78yn2nLNoUd\nR0RSnIpCyMyMX549jN4d23DNYx+xeUdZ2JFEJIWpKLQCudkZ3HvBYWzaUcbN/9CDbSISHhWFVmJI\nz/Z898QBvDpnHU9OWxF2HBFJUSoKrcg3junH6IM6cevzc1lcsj3sOCKSglQUWpFIxLj7nEPJyojw\n7SdmUlZRFXYkEUkxKgqtTLf22dx+5jA+XrWVe15bGHYcEUkxKgqt0Lgh3Zg4sjf3T11MsW5TFZEW\nFNeiYGbjzGyBmS0ys5sbWO5sM3MzK4pnnkTyw5MH0atjG77z9Cx2lqljHhFpGXErCmaWBkwCTgIG\nARPNbFA9y+UA1wIfxCtLIjogK507zx7O8o07+eXL88OOIyIpIp5HCiOBRe6+xN3LgCeACfUs9xPg\nDmB3HLMkpKP6deLy0YU8+t5y3lm0Iew4IpIC4lkUegK1b7hfGcyrYWYjgN7u/kIccyS0m8YNoF9+\nO256Zjalu9WaqojEVzyLgtUzr+ZRXTOLAL8CvtPoisyuMLNiMysuKSlpxoitX3ZGGnefM5w1W3fx\n0xfmhR1HRJJcPIvCSqB3relewOpa0znAEOBtM1sGHAVMru9is7s/4O5F7l6Un58fx8it04g+Hfnm\nlw/kyeIVvDl/XdhxRCSJxbMoTAP6m1mhmWUC5wOTq990963u3tndC9y9AHgfOM3di+OYKWFdd0J/\nBnbL4ea/f8yWnWo0T0TiI25Fwd0rgGuAV4F5wFPuPsfMbjOz0+L1vckqKz2Nu84ZzqYdZfzouTlh\nxxGRJJUez5W7+0vAS3Xm/Wgvy46JZ5ZkMKRne649vj/3vLaQcUO6MX5o97AjiUiS0RPNCeZ/xxzI\n0J7t+b9nP2HD9j1hxxGRJKOikGAy0iLcfe5wtu+p4Af/+Fh9L4hIs1JRSEAHd83hxq8ezJS563h2\n5qqw44hIElFRSFBfP6YfRX078qPn5rBm666w44hIklBRSFBpEeOuc4ZTUel87+86jSQizaPRomBm\n+WZ2l5m9ZGZvVg8tEU4aVtC5HT8YP5CpC0t4/EN14Ski+y+WI4W/EX3OoBC4FVhG9ME0aQUuPLIv\now/qxE9fnMtnG3eGHUdEElwsRaGTu/8RKHf3f7n75USbpJBWIBIx7jh7OBEzbnxmFlVVOo0kIk0X\nS1GobppzjZmdHLRs2iuOmWQf9ezQhh+dOogPl27ikXeXhR1HRBJYLEXhp2bWnmhrpjcCDwHfjmsq\n2WfnHN6L4wd24Y5X5rNo/faw44hIgoqlKGwOGq/7xN3HuvvhgDoObmXMjF+cOZQ2mWl85+lZVFRW\nhR1JRBJQLEXhdzHOk5B1yc3mJxOGMGvFFu6fuiTsOCKSgPbaIJ6ZjQKOBvLN7IZab+UCafEOJk1z\n6vAevDJnLb9+fSFjB3RhUI/csCOJSAJp6EghEziAaOHIqTVsA86OfzRpqp9MGEL7Npl8+8kZ7C6v\nDDuOiCSQvR4puPu/gH+Z2Z/cfXkLZpL9lNcuk7vPHc4lD3/IL16ax60ThoQdSUQSRCz9Kew0szuB\nwUB29Ux3Py5uqWS/ffngfC4fXcjD7yxlzIAujB3YJexIIpIAYn2ieT56ojnh3DRuAAO75fDdZ2ZR\nUqq+F0SkcXqiOYllZ6Tx24kjKN1dwXefmaVG80SkUXqiOckd3DWHH4w/hLcXlPConnYWkUY09Ynm\n6+OaSprV10b15biBXfj5y/NZsLY07Dgi0oo1WhTc/YW6TzS7++SWCCfNw8y44+xh5Ganc+3juk1V\nRPauoYfXfgfs9SS0u18bl0QSF50PyOLOc4Zz2SPTdJuqiOxVQ0cKxcB0orehHgZ8GgyHAvpTMwGN\nHdCFrx9TyKPvLeeVT9aEHUdEWqGGHl57FMDMLgXGunt5MH0fMKVF0kmz+964gRQv38x3n5nN4B7t\n6Z3XNuxIItKKxHKhuQfR5i2qHRDMkwSUmR7h3okjALjmsY8oq1BrqiLyuViKwu3ADDP7k5n9CfgI\n+HlcU0lc9c5ry51nD2fWyq3c/vL8sOOISCsSy91HjwBHAv8MhlHVp5YkcY0b0o1Ljy7g4XeWMmXO\n2rDjiEgrEcuRAu6+1t2fCwb9D5Ikvj9+IEN7tufGp2excvPOsOOISCsQU1GQ5JSVnsa9F4zAHb71\n+AzK1VubSMpTUUhxfTu14/azhjHjsy3c+eqCsOOISMgabTrbzPLqmV1afYuqJL6Th3Xn/SV9eWDq\nEo4szOP4Q7qGHUlEQhLLkcJHQAmwkOjDayXAUjP7yMwOb+iDZjbOzBaY2SIzu7me979pZh+b2Uwz\n+4+ZDWrKRsj+++HJhzC4Ry43PKXrCyKpLJai8Aow3t07u3sn4CTgKeAq4Pd7+5CZpQGTguUHARPr\n+U//MXcf6u6HAncA9zRhG6QZZGek8fsLD6OqyrnmsRl6fkEkRcVSFIrc/dXqCXefAhzr7u8DWQ18\nbiSwyN2XuHsZ8AQwofYC7r6t1mQ7GmhrSeKvb6d23HH2MGau2MIvX9HzCyKpKJaisMnMvmdmfYPh\nJmBzcCTQ0J+TPYEVtaZXBvO+wMyuNrPFRI8U1MheyE4a2p1Ljy7gj/9Zyqt6fkEk5cRSFC4g2qnO\ns8BzQJ9gXhpwbgOfs3rm/deRgLtPcvcDge8B/1fvisyuMLNiMysuKSmJIbLsj++PH8jwXtHnF1Zs\n0vUFkVQSyxPNG9z9W+4+wt0Pdfdr3L3E3cvcfVEDH10J9K413QtY3cDyTwCn7yXDA+5e5O5F+fn5\njUWW/RR9fuEwDLj6sY/YU6FGcUVSRaNFwcwONrMHzGyKmb1ZPcSw7mlAfzMrNLNM4HzgC53zmFn/\nWpMnE727SVqB3nltufOc4cxeuZVfvKTrCyKpotHnFICngfuIdsMZ85+M7l5hZtcArxI91fSwu88x\ns9uA4qD3tmvM7ASi/UBvBi7Z1w2Q+DlxcDe+fkwhf/zPUkYW5jF+aPewI4lInJl7wzf8mNl0d2/w\neYSWVFRU5MXFxWHHSBllFVWce/97LF6/nReuPYa+ndqFHUlEmiD4v7yoseViudD8vJldZWbdzSyv\nemiGjJIAMtMj3HvBCCIR46q/faT+nUWSXCxF4RLgu8C7RLvnnE60q05JEb06tuWec4czZ/U2fvri\n3LDjiEgcNXpNwd0LWyKItG7HH9KVK4/tx/1Tl3BkYSdOHa7O90SS0V6Lgpkd5+5vmtmZ9b3v7v+I\nXyxpjW48cQDFyzdz899nM7hHLv3yDwg7kog0s4ZOH305eD21nuGUOOeSVigjLXp9ITM9ousLIkmq\n0buPWhvdfRS+txas57JHpjFxZG9+ceawsOOISAxivfsolv4UsoCzgILay7v7bfsTUBLX2AFduGrM\ngfz+7cUcWdiJ00f8V5NWIpKgYrn76DmirZtWADtqDZLCbvjKwYwszOMH//yYxSXbw44jIs0kliea\ne7n7uLgnkYSSnhbhdxNHcOKvp3Lj07N45ptHkxaprw1EEUkksRwpvGtmQ+OeRBJO19xsbj1tMDM+\n28JD/14SdhwRaQaxFIVjgOlBt5qzg+4zZ8c7mCSG04b34MTBXbn7tYUsWl8adhwR2U+xFIWTgP7A\nV/n8dtRT4xlKEoeZ8dPTh9IuM43vPD2bikp14ymSyPZaFMwsNxgt3csgAkB+Tha3ThjCrBVb+NO7\ny8KOIyL7oaEjhceC1+q2jqajto9kL04d1p2xA/L59eufsn7b7rDjiEgT7bUouPspwWuhu/cLXquH\nfi0XURKBmfHjUwdTVlHFz1+aF3YcEWmiWK4pYGYdzWykmR1bPcQ7mCSegs7tuPLL/Xh25mreX7Ix\n7Dgi0gSxdMf5DWAq0R7Ubg1eb4lvLElUV405iJ4d2nDL5DlUViVWEyoiEtuRwnXAEcBydx8LjABK\n4ppKElabzDS+d9JA5q8tZfKsVWHHEZF9FEtR2O3uuyHaDpK7zwcGxDeWJLJThnZncI9c7p6ykD0V\naklVJJHEUhRWmlkH4FngNTN7Dlgd31iSyCIR46ZxA1m5eRePf/BZ2HFEZB/E0vPaGcHoLWb2FtAe\neCWuqSThHdu/M6P6deLetxZx3hF9aJOZFnYkEYlBg0cKZhYxs0+qp939X+4+2d3L4h9NEpmZcf1X\nDmbD9jKenKajBZFE0WBRcPcqYJaZ9WmhPJJERhbmUdS3Iw9MXUJZhZq/EEkEsVxT6A7MMbM3zGxy\n9RDvYJIcrh57EKu37ua5mboTSSQRxNKfwq1xTyFJa8yAfA7pnssf/rWYsw7rRUR9Loi0arEcKYwP\nriXUDMD4eAeT5GBmfPPL/VhSsoO3F64PO46INCKWovCVeuad1NxBJHmNH9qdrrlZPPLOsrCjiEgj\nGmo6+3/N7GNgQNC5TvWwFFAnOxKzjLQIFx3Zl39/uoFF69Wfs0hr1ljT2acCk4PX6uFwd7+oBbJJ\nEpl4ZB8y0yL8+b1lYUcRkQY01HT2Vndf5u4T3X15rWFTSwaU5ND5gCxOHd6DZ6avZNvu8rDjiMhe\nxNR0tkhzuPToAnaWVfJM8cqwo4jIXqgoSIsZ2qs9h/XpwF/fX06VmtUWaZXiWhTMbJyZLTCzRWZ2\ncz3v32Bmc4ML2G+YWd945pHwXTyqL0s27ODdxeqER6Q1iqWTnTPN7FMz22pm28ys1My2xfC5NGAS\n0dtXBwETzWxQncVmAEXuPgx4Brhj3zdBEsn4od3Ja5epC84irVQsRwp3AKe5e3t3z3X3HHfPjeFz\nI4FF7r4kaEDvCWBC7QXc/S133xlMvg/02pfwkniy0tM474jevD5vHau37Ao7jojUEUtRWOfuTemJ\nvSewotb0ymDe3nwdeLkJ3yMJ5sIj++DAY+prQaTViaUoFJvZk2Y2MTiVdKaZnRnD5+pr5Kbeq4tm\ndhFQBNy5l/evMLNiMysuKVFPoImuV8e2HD+wC09M+0ytp4q0MrEUhVxgJ/BVPn+A7ZQYPrcS6F1r\nuhf19NhmZicAPyR6impPfSty9wfcvcjdi/Lz82P4amntLh5VwIbtZbz8yZqwo4hILbH0vHZZE9c9\nDehvZoXAKuB84ILaC5jZCOB+YJy7q7W0FPKlgzpT0Kktf31/ORMObeisooi0pL0WBTO7yd3vMLPf\nUc9pH3e/tqEVu3uFmV0DvAqkAQ+7+xwzuw0odvfJRE8XHQA8bWYAn7n7aU3fHEkUkYhx0VF9+emL\n85i3ZhuHdI/l3gURibeGjhSqLy4XN3Xl7v4S8FKdeT+qNX5CU9ctie/sw3tx56sL+Mv7y/n5GUPD\njiMiNFAU3P354PXRlosjqaRD20wmHNqDZ2es4uaTBpKbnRF2JJGUp2YuJFQXHxVtD+kf09Uekkhr\noKIgoRraqz2H9u7AX95fjrvaQxIJW4NFwczSzOz6lgojqenio/qyuGQH76k9JJHQNVgU3L2SOk1T\niDS3k4d1p2PbDP783vKwo4ikvFhOH71jZvea2ZfM7LDqIe7JJGVkZ6Rx7hG9eW3eOtZsVXtIImGK\npSgcDQwGbgPuDoa74hlKUs9FR/alyp3H1R6SSKhieaJ5bEsEkdTWO68tYwd04W8ffMZVYw8iOyMt\n7EgiKSmW/hTam9k91Q3SmdndZta+JcJJarny2H5s3FHGU8UrGl9YROIiltNHDwOlwLnBsA14JJ6h\nJDWNLMzj8L4duf9fSyivVOupImGIpSgc6O4/DjrLWeLutwL94h1MUo+ZcfXYA1m1ZReTZ/5Xg7oi\n0gJiKQq7zOyY6gkzGw3oFhGJi7EDujCwWw5/+Ndiqqr0MJtIS4ulKHwTmGRmy8xsGXAvcGVcU0nK\nMjOuGnsQi9ZvZ8rctWHHEUk5jT3RHAEGuPtwYBgwzN1HuPvsFkknKenkod0p7NyOX7/+qY4WRFpY\nY080VwHXBOPb3H1bi6SSlJYWMb59Qn/mry3l+dm6tiDSkmI5ffSamd1oZr3NLK96iHsySWmnDuvB\nwG453PPaQt2JJNKCYikKlwNXA1OB6cHQ5I53RGIRiRjfPXEAyzfu5OliNast0lJiuaZwkbsX1hl0\nS6rE3XEDu3BYnw785o2F7C6vDDuOSEqI5ZqC2jmSUJgZN40byLpte/jjf5aGHUckJcRy+miKmZ1l\nZhb3NCJ1HNWvE18d1JVJby1i7dbdYccRSXqxFIUbgKeBMjPbZmalZqa7kKTF/N/Jg6iocm5/eV7Y\nUUSSXqNFwd1z3D3i7hnunhtM57ZEOBGAPp3a8j9fKuTZmauZvnxT2HFEklosraSamV1kZv8vmO5t\nZiPjH03kc1eNOYiuuVncMnkulXqgTSRuYjl99HtgFHBBML0dmBS3RCL1aJeVzg/GH8LHq7by6LvL\nwo4jkrRiKQpHuvvVwG4Ad98MZMY1lUg9ThvegzED8rlrygJWbNoZdhyRpBRLUSg3szTAAcwsH9Aj\nptLizIyfnj4EgB8++wnuOo0k0txiKQq/Bf4JdDGznwH/AX4e11Qie9GrY1tuOnEAUxeW8M8Zq8KO\nI5J0Yumj+W9mNh04HjDgdHfXvYESmotHFTB51mpumTyHUQd2onv7NmFHEkkasRwp4O7z3X2Su9+r\ngiBhS4sYd597KBVVzg1PzlLz2iLNKKaiINLaFHZuxy2nDua9JRt58N9Lwo4jkjRUFCRhnVPUi3GD\nu3HXlAV8smpr2HFEkkJci4KZjTOzBWa2yMxuruf9Y83sIzOrMLOz45lFko+Z8Yszh9KpXRZX/e0j\ntu4sDzuSSMKLW1EIbmOdBJwEDAImmtmgOot9BlwKPBavHJLcOrbLZNKFh7Fm6y6uf2qmri+I7Kd4\nHimMBBa5+xJ3LwOeACbUXsDdlwX9Peu5B2myw/t25P+dMog3569n0luLwo4jktDiWRR6AitqTa8M\n5ok0u4uP6ssZI3pyz+sLeX3uurDjiCSseBaF+vpfaNKxvZldYWbFZlZcUlKyn7EkGZkZPz9jKEN7\ntudbj8/g45W68CzSFPEsCiuB3rWmewGrm7Iid3/A3YvcvSg/P79ZwknyaZOZxkOXFJHXLpPLH53G\nqi27wo4kknDiWRSmAf3NrNDMMoHzgclx/D4RuuRk88hlR7C7vJLLH5nG1l26I0lkX8StKLh7BXAN\n8CowD3jK3eeY2W1mdhqAmR1hZiuBc4D7zWxOvPJI6ji4aw73XXQ4i0u2c/mfprFjT0XYkUQShiVa\nS5NFRUVeXFwcdgxJAC9/vIarH/uIkYV5/OmykWRnpIUdSSQ0Zjbd3YsaW05PNEvSOmlod+4591A+\nWLqJK/4ynT0VlWFHEmn1VBQkqZ0+oie3nzmUqQtL+Majxews06kkkYaoKEjSO++IPtxx9jDeWbSB\nix76QM1hiDRARUFSwrlFvfn9hYfxyaptnPfAe6zftjvsSCKtkoqCpIxxQ7rzx0uL+GzTTk6f9A5z\nVusBN5G6VBQkpXypfz5PXTkKB8657z2mzFkbdiSRVkVFQVLOkJ7tee7q0fTvmsOVf53Ob17/lEq1\nrioCqChIiuqSm82TVxzFGYf25FevL+RrD39ASemesGOJhE5FQVJWdkYad587nDvOGkbxss2c9Jt/\n886iDWHHEgmVioKkNDPj3CN689w1o2nfJp0LH/qAHz33iZrGkJSloiACDOyWywvf+hKXjS7gL+8v\nZ9xvpvLe4o1hxxJpcSoKIoE2mWn8+NTBPHnFKNLMmPjg+9zw1EzWl+qZBkkdKgoidYwszOPl647l\nqjEH8sKsNRx31794cOoSyivVa6wkPxUFkXq0yUzjpnEDefX6YxlZmMfPXprHib+ayvOzVlOl21cl\niakoiDSgsHM7Hr70CB6+tIiMtAjfenwGJ//uP7w+dx2J1uy8SCzUn4JIjCqrnBdmr+ae1xayfONO\nBvfI5Ypj+zF+aHcy0vT3lbRusfanoKIgso/KK6v450eruH/qYhaX7KBnhzZcfkwh5xb1Iic7I+x4\nIvVSURCJs6oq583563lg6hI+XLaJNhlpnDa8BxOP7MPwXu0xs7AjitSItSikt0QYkWQUiRgnDOrK\nCYO6MmvFFh7/8DMmz1rNk8UrGNgth3OLenPKsO50yc0OO6pIzHSkINKMSneXM3nWap74cAUfr9qK\nGRxV2InTDu3BuMHd6NguM+yIkqJ0+kgkZIvWlzJ51hqen7WapRt2kB4xjijI4/hDunDcwC70yz8g\n7IiSQlQURFoJd2fO6m28+PEa3py3ngXrSoHo7a5jB3Rh9EGdOKIwj1xdpJY4UlEQaaVWbNrJWwvW\n88a89by3ZCNlFVVELNrPw6h+nTjqwE4c3rejioQ0KxUFkQSwu7ySGZ9t4b0lG3l/8UZmrNhMeaVj\nBgfmH8DwXh04tE8HRvTuwIBuOXoeQppMRUEkAe0qq2T68s3M+GwzM1dsYeaKLWzcUQZAVnqEgd1z\nOaRbDgO65TCwWy4Du+Xo4rXEREVBJAm4Oys376opEHNXb2P+2m1s3lles0zX3CwGdMvloPwDKOzc\nlsLOB1DQuS092rchEtGzEhKl5xREkoCZ0TuvLb3z2nLq8B5AtFCUlO5h3tpSFqzdxvw1pcxfW8q0\npZvYVV5Z89ms9Ah9O7WlsHM7Cjq1o2fHNvTs0IYeHdrQs2MbXbOQeqkoiCQYM6NLbjZdcrP58sH5\nNfPdnXXb9rB0ww6WbtjBso3R18UlO3hrQQllFV9s+jsnK72mQPTokE233Gy65GSTn5NFfk4WXXKy\nyGuXSbquY6QUFQWRJGFmdGufTbf22Yw6sNMX3quqcjbs2MPqLbtZtXkXq7fsYlX1sHkX05dvZuuu\n8v9aZ8Qgr93nRSI/KBQd22bSsW0GHYLXvHaZdGibSYe2GboYnuBUFERSQCRidMmJHgkc2rtDvcvs\nLq+kpHQP60v3UFK6h5LS3dHX7XtYvy36umBtKZt2lv3XUUdtOdnpNUWjfdtMcrLTyc1OJyc7g5ys\ndHKqx7/w+vm4ikq4VBREBIDsjLSa6xcNcXd2lVeyeWc5m3eUsXlnGZt3lrNlZxmbdpSxZWc5m2vG\ny1i5aSfbdldQurucPQ0Uk89zRGiXmU6bzLTPX7PSaJORTts6422z0mibkUbbzPToeGb0vTaZaWSl\nR8hKj5CdEYxnpJGdHtHpsEZDwY5sAAAKh0lEQVSoKIjIPjGz6H/Cmen07NBmnz5bVlFF6e5ySndX\nBEN5TcHYvufzeTvKKtlVVsnOsgp2llWys6ySTTt21UzvKqtkR1kFTbl5Mi1iZAdFom7R+K9Ckp5G\ndkaEjLQImekRMtKMjLRgOi2YTq89/fm8L0zXfD5CesRqxmveS4u0mjvF4loUzGwc8BsgDXjI3W+v\n834W8GfgcGAjcJ67L4tnJhEJT2Z6hE4HZNHpgKz9Xpe7s6eiip1llezYU8Gu8mjx2Lmngj0VVewu\nr/zC656KSnaXR1/3lFexu+a1ij21li3dXcGG7WVfmFdeWUV5pVNeWUVFnLpjjRikRyKkpxnpkWix\nSAteq+ddd8LBnBbchRYvcSsKZpYGTAK+AqwEppnZZHefW2uxrwOb3f0gMzsf+CVwXrwyiUjyMDOy\nM9LIzkgjrwUf4KuqcsqrgiJRUUV5ZRVltYpGWTCvZrqyKliu1nStedXTFZVORZVTERSe8soqKquc\n8kqnoir6foc28b+NOJ5HCiOBRe6+BMDMngAmALWLwgTglmD8GeBeMzNPtCfqRCRlRCJGViSNrHRg\n/w94Wp14XnHpCayoNb0ymFfvMu5eAWwFOtVZBjO7wsyKzay4pKQkTnFFRCSeRaG+qyZ1jwBiWQZ3\nf8Ddi9y9KD8/v56PiIhIc4hnUVgJ9K413QtYvbdlzCwdaA9simMmERFpQDyLwjSgv5kVmlkmcD4w\nuc4yk4FLgvGzgTd1PUFEJDxxu9Ds7hVmdg3wKtFbUh929zlmdhtQ7O6TgT8CfzGzRUSPEM6PVx4R\nEWlcXJ9TcPeXgJfqzPtRrfHdwDnxzCAiIrHT894iIlJDRUFERGokXM9rZlYCLG/ixzsDG5oxTiLQ\nNqcGbXNq2J9t7uvujd7Tn3BFYX+YWXEs3dElE21zatA2p4aW2GadPhIRkRoqCiIiUiPVisIDYQcI\ngbY5NWibU0PctzmlrimIiEjDUu1IQUREGpAyRcHMxpnZAjNbZGY3h52nuZhZbzN7y8zmmdkcM7su\nmJ9nZq+Z2afBa8dgvpnZb4PfYbaZHRbuFjSNmaWZ2QwzeyGYLjSzD4LtfTJobwszywqmFwXvF4SZ\nu6nMrIOZPWNm84N9PSoF9vH1wb/pT8zscTPLTsb9bGYPm9l6M/uk1rx93rdmdkmw/Kdmdkl93xWL\nlCgKtXqBOwkYBEw0s0Hhpmo2FcB33P0Q4Cjg6mDbbgbecPf+wBvBNER/g/7BcAXwh5aP3CyuA+bV\nmv4l8KtgezcT7dUPavXuB/wqWC4R/QZ4xd0HAsOJbnvS7mMz6wlcCxS5+xCi7adV986YbPv5T8C4\nOvP2ad+aWR7wY+BIoh2c/bi6kOwzd0/6ARgFvFpr+vvA98POFadtfY5oF6gLgO7BvO7AgmD8fmBi\nreVrlkuUgWgz7G8AxwEvEO2XYwOQXnd/E22QcVQwnh4sZ2Fvwz5uby6wtG7uJN/H1R1w5QX77QXg\nxGTdz0AB8ElT9y0wEbi/1vwvLLcvQ0ocKRBbL3AJLzhkHgF8AHR19zUAwWuXYLFk+C1+DdwEVAXT\nnYAtHu29D764TTH17tfK9QNKgEeCU2YPmVk7kngfu/sq4C7gM2AN0f02neTez7Xt675ttn2eKkUh\nph7eEpmZHQD8Hfi2u29raNF65iXMb2FmpwDr3X167dn1LOoxvJco0oHDgD+4+whgB5+fTqhPwm9z\ncOpjAlAI9ADaET11Ulcy7edY7G07m237U6UoxNILXMIyswyiBeFv7v6PYPY6M+sevN8dWB/MT/Tf\nYjRwmpktA54gegrp10CHoPc++OI2JUPvfiuBle7+QTD9DNEikaz7GOAEYKm7l7h7OfAP4GiSez/X\ntq/7ttn2eaoUhVh6gUtIZmZEOyua5+731Hqrdq92lxC91lA9/2vBXQxHAVurD1MTgbt/3917uXsB\n0f34prtfCLxFtPc++O/tTeje/dx9LbDCzAYEs44H5pKk+zjwGXCUmbUN/o1Xb3PS7uc69nXfvgp8\n1cw6BkdZXw3m7buwL7C04IWc8cBCYDHww7DzNON2HUP0MHE2MDMYxhM9n/oG8Gnwmhcsb0TvxFoM\nfEz07o7Qt6OJ2z4GeCEY7wd8CCwCngaygvnZwfSi4P1+Yedu4rYeChQH+/lZoGOy72PgVmA+8Anw\nFyArGfcz8DjR6yblRP/i/3pT9i1webD9i4DLmppHTzSLiEiNVDl9JCIiMVBREBGRGioKIiJSQ0VB\nRERqqCiIiEgNFQVJWmb2tpnFvQ9fM7s2aLn0b3XmF5nZb4PxMWZ2dDN+Z4GZXVDfd4nsj/TGFxFJ\nPWaW7p+3sdOYq4CT3H1p7ZnuXkz02QKIPlOxHXi3mTIUABcAj9XzXSJNpiMFCVXwF+88M3swaDt/\nipm1Cd6r+UvfzDoHTVtgZpea2bNm9ryZLTWza8zshqCxuPeDZoSrXWRm7wZt8o8MPt8uaMN+WvCZ\nCbXW+7SZPQ9MqSfrDcF6PjGzbwfz7iP6QNVkM7u+zvJjzOyFoKHCbwLXm9lMM/uSmeWb2d+DDNPM\nbHTwmVvM7AEzmwL8Ofh9/m1mHwVD9dHG7cCXgvVdX/1dwTrygt9ndvB7DKu17oeD33WJmV1b6/d4\n0cxmBdt23v7tVUloYT/NpyG1B6J/8VYAhwbTTwEXBeNvEzyxCXQGlgXjlxJ9ajMHyCfaIuY3g/d+\nRbRRwOrPPxiMH0vQNDHw81rf0YHok+7tgvWuJHh6tE7Ow4k+QdoOOACYA4wI3lsGdK7nM2P4/Inr\nW4Aba733GHBMMN6HaDMl1ctNB9oE022B7GC8P1Bcd931fNfvgB8H48cBM2ut+12iTwZ3BjYCGcBZ\n1b9TsFz7sP9daAhv0OkjaQ2WuvvMYHw60ULRmLfcvRQoNbOtwPPB/I+BYbWWexzA3aeaWa6ZdSDa\nLsxpZnZjsEw20f+YAV5z9/oaUjsG+Ke77wAws38AXwJmxLKB9TgBGBRt1geAXDPLCcYnu/uuYDwD\nuNfMDgUqgYNjWPcxRP+jx93fNLNOZtY+eO9Fd98D7DGz9UBXor/ZXWb2S6KF5d9N3CZJAioK0hrs\nqTVeCbQJxiv4/BRndgOfqao1XcUX/13Xbcelupnhs9x9Qe03zOxIos1S16e+pon3R4RopzC7as8M\nikTtDNcD64j2thYBdsew7oaaUa77W6e7+0IzO5xom1m/MLMp7n5bTFshSUfXFKQ1W0b0tA183jLm\nvjoPwMyOIdqi5FairUd+K2h9EzMbEcN6pgKnB612tgPOAPblL+pSoqe7qk0BrqmeCI4E6tMeWOPu\nVcDFRLulrG99dbNeGKx3DLDBG+hjw8x6ADvd/a9EO7ZJyD6dpXmoKEhrdhfwv2b2LtFz4E2xOfj8\nfXzen+9PiJ6WmW3RztJ/0thK3P0jon3pfki0Z7uH3H1fTh09D5xRfaGZoP/h4GLwXKIXouvze+AS\nM3uf6Kmj6qOI2UBFcHH4+jqfuaV63UQvSDfWiftQ4EMzmwn8EPjpPmyXJBm1kioiIjV0pCAiIjVU\nFEREpIaKgoiI1FBREBGRGioKIiJSQ0VBRERqqCiIiEgNFQUREanx/wFb0BtUCwYrFwAAAABJRU5E\nrkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEKCAYAAAD9xUlFAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4zLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvIxREBQAAG1xJREFUeJzt3X28XFV97/HP14QHG0gQElskiQk2\n2EaKgucGECwIFIFaUoVrCXILyi31AVEQNb68BcS2VqWllaKYWkStgmgRA40SXoDilYI5AUwJEDjy\nIEdQDpIG5Dnk1z/2mskwTObsc+bsmdn7fN+v17zOfpo9vz07Ob+z1tprLUUEZmZmAC/pdQBmZtY/\nnBTMzKzOScHMzOqcFMzMrM5JwczM6pwUzMyszknBzMzqnBTMzKzOScHMzOqm9jqAsZo5c2bMmzev\n12GYmZXK6tWrH4mIWaMdV7qkMG/ePAYHB3sdhplZqUi6P89xrj4yM7M6JwUzM6tzUjAzszonBTMz\nq3NSMDOzusKSgqQLJT0s6bYt7Jekz0kakrRG0l5FxWJmZvkUWVK4CDiszf7DgQXpdRLwhQJjMTOz\nHArrpxAR10ua1+aQxcBXI5sP9EZJO0jaOSIeKiKeVfc9yo/uGini1Bz6mt9h911mFHJuM7Nu6mXn\ntV2ABxrWh9O2FyUFSSeRlSaYO3fuuD7s5vvXc951Q+N6bzsR8LORJzj/Ha79MrPy62VSUItt0erA\niFgGLAMYGBhoecxo/vKAV/GXB7xqPG9t683nXs/GTZsm/LxmZr3Qy6ePhoE5DeuzgQd7FEtHYlxp\nysys//QyKSwH/jw9hbQPsKGo9oQiqVV5x8yspAqrPpJ0MXAgMFPSMHAmsBVARFwArACOAIaAJ4F3\nFhVL0VxQMLOqKPLpoyWj7A/gfUV9frdIcvWRmVWGezRPCGcFM6sGJ4UOCTc0m1l1OCl0yA3NZlYl\nTgodklx5ZGbV4aTQISHC9UdmVhFOChPAKcHMqsJJoUOSG5rNrDqcFDrkdmYzqxInhU5Jrj4ys8pw\nUuhQ1k/BacHMqsFJwczM6pwUOuTOa2ZWJU4KHfIwF2ZWJU4KHZJEuKnZzCrCSaFDLimYWZU4KUwA\nJwUzqwonhQ65odnMqsRJoUPCbQpmVh1OCp3y2EdmViFOCh0SHiXVzKrDSWEiOCuYWUU4KXTIDc1m\nViVOCh1yQ7OZVYmTQoc8yY6ZVYmTQockNymYWXU4KZiZWZ2TQoeEPMmOmVWGk0KHXH1kZlXipDAB\nXFAws6pwUuhQNp+CmVk1FJoUJB0maZ2kIUlLW+yfK+k6SbdIWiPpiCLjMTOz9gpLCpKmAOcDhwML\ngSWSFjYd9v+ASyNiT+AY4PNFxVMUgeuPzKwyiiwpLAKGIuKeiHgWuARY3HRMANPT8gzgwQLjKYQb\nms2sSqYWeO5dgAca1oeBvZuOOQtYKen9wDTgkALjKYSn4zSzKimypNBqqLjmX59LgIsiYjZwBPA1\nSS+KSdJJkgYlDY6MjBQQ6vjJI+KZWYUUmRSGgTkN67N5cfXQicClABHxn8C2wMzmE0XEsogYiIiB\nWbNmFRTu+HlAPDOriiKTwipggaT5krYma0he3nTMz4GDAST9PllS6K+iwChcfWRmVVJYUoiIjcDJ\nwFXAHWRPGa2VdLakI9NhHwL+QtJPgYuBE6JkY0Z4lFQzq5IiG5qJiBXAiqZtZzQs3w7sV2QMxXPn\nNTOrDvdoNjOzOieFDmXVRy4rmFk1OCl0yA+kmlmVOCl0yA3NZlYlozY0S5oFfJRs/KJta9sj4qAC\n4yoNIfdTMLPKyFNS+DrZI6XzgU8A95H1QTAzs4rJkxR2ioh/BZ6LiB9GxLuAfQqOqzRcfWRmVZKn\nn8Jz6edDkv6YbKiK2cWFVC4eJdXMqiRPUvhrSTPIeh+fRzbU9QcLjapEhPxIqplVRp6ksD4iNgAb\ngDcBSCp5L+QJ5GdSzaxC8rQpnJdz26TlcoKZVcUWSwqS9gXeAMySdFrDrunAlKIDK4tsOs5eR2Fm\nNjHaVR9tDWyXjtm+YftjwNFFBlUmkgfEM7Pq2GJSiIgfAj+UdFFE3N/FmEolm0/BacHMqiFPQ/OT\nkj4LvAb3aH4Rz8ZpZlWSt0fznbhH8xa5nGBmVeEezR3ydJxmViXu0dyhrKHZWcHMqmG8PZpPLTSq\nEnFJwcyqZNSkEBFXpsV6j2Zr4IZmM6uQdp3XzqNNG2pEnFJIRCXkkoKZVUW7huZBYDXZY6h7AXen\n1+uA54sPrRzkooKZVUi7zmtfAZB0AvCmiHgurV8ArOxKdCWQzafgooKZVUOeR1JfwQuHudgubTPc\npGBm1ZLn6aO/A26RdF1aPwA4q7CISsaT7JhZleR5+ujLkr4H7J02LY2IXxYbVrm49sjMqiJPSYGU\nBL5bcCylJNx5zcyqI0+bgrWRNTT3Ogozs4nhpNAhj5JqZlUyavWRpB1bbH689oiqeZIdM6uOPCWF\nm4ER4C6yzmsjwL2Sbpb0+iKDKwtXH5lZVeRJCt8HjoiImRGxE3A4cCnwXuDz7d4o6TBJ6yQNSVq6\nhWPeLul2SWslfWOsF9Br8iTNZlYheZLCQERcVVuJiJXAH0bEjcA2W3qTpCnA+WRJZCGwRNLCpmMW\nAB8D9ouI1wAfHPsl9JZHSTWzKsmTFB6V9FFJr0yvjwDr0y/9TW3etwgYioh7IuJZ4BJgcdMxfwGc\nHxHrASLi4XFcQ0+5odnMqiRPUjiWbFKdy8n6KsxN26YAb2/zvl2ABxrWh9O2RrsBu0n6saQbJR3W\n6kSSTpI0KGlwZGQkR8jdIzc0m1mF5OnR/Ajw/i3sHmrz1lZ/Qzf//pwKLAAOJEs8P5K0e0T8d1MM\ny4BlAAMDA333O9gD4plZVeR5JHU34HRgXuPxEXHQKG8dBuY0rM8mm8qz+Zgb0+Ot90paR5YkVo0a\neZ/w2EdmViV5hrn4FnAB8CXGNo/CKmCBpPnAL4BjyKqdGl0OLAEukjSTrDrpnjF8Rs+5ScHMqiRP\nUtgYEV8Y64kjYqOkk4GryNofLoyItZLOBgYjYnnad6ik28kSzocj4tdj/axekuSnj8ysMvIkhSsk\nvRf4DvBMbWNEPDraGyNiBbCiadsZDcsBnJZepeU2BTOrijxJ4fj088MN2wLYdeLDKSenBDOrijxP\nH83vRiBlJeGsYGaVscWkIOmgiLhW0tta7Y+Iy4oLqzzkpmYzq5B2JYUDgGuBP2mxLwAnBfxIqplV\nyxaTQkScmX6+s3vhlJMbms2sKvJ0XtsGOIoXd147u7iwysNNCmZWJXmePvousAFYTcMjqZbxdJxm\nViV5ksLsiGg5UJ1lndfMzKoizyipN0j6g8IjKams+shFBTOrhjwlhf2BEyTdS1Z9lM0rE7FHoZGV\niKuPzKwq8iSFwwuPosz8SKqZVUi7zmvTI+Ix4PEuxlM67rxmZlXSrqTwDeAtZE8dBS8cJdpjHyUe\n5sLMqqRd57W3pJ8e+6gNAc9t2sR7/m11r0Ppe7vOmsaH3/x7vQ7DzNrI06aApJeRzYi2bW1bRFxf\nVFBlsveuO3HtnQ/zs5Hf9DqUvvbr3zzL9277JaceshtTp+R56M3MeiFPj+b/C3yAbDrNW4F9gP8E\nRpuOc1I4YLdZHLDbrF6H0ffOu+Zu/v7qu1zTZtbn8vzJ9gHgfwH3R8SbgD2BkUKjssry47tm/S1P\nUng6Ip6GbBykiLgTeHWxYVnV1Dp+u6OfWX/L06YwLGkH4HLgaknrgQeLDcuqxsOBmJVDnpnX3poW\nz5J0HTAD+H6hUVllufrIrL+1TQqSXgKsiYjdASLih12JyirHBQWzcmjbphARm4CfSprbpXis4lxS\nMOtvedoUdgbWSvoJ8ERtY0QcWVhUVjm14UDc0GzW3/IkhU8UHoVVnquPzMohT1I4IiI+2rhB0qcB\nty9YbrWc4Oojs/6Wp5/CH7XY5uG0bUw291Mws37Wbujs9wDvBXaVtKZh1/bAj4sOzKopXFQw62uj\nDZ39PeBTwNKG7Y9HxKOFRmWV43knzMqh3dDZG4ANwJLuhWNV5eojs3LwGMbWVa49MutvhSYFSYdJ\nWidpSNLSNscdLSkkDRQZj/WOXFQwK4XCkoKkKcD5ZE8qLQSWSFrY4rjtgVOAm4qKxfqHO6+Z9bdR\nk4Kkt0m6W9IGSY9JelzSYznOvQgYioh7IuJZ4BJgcYvjPgl8Bnh6TJFbqbiZ2awc8pQUPgMcGREz\nImJ6RGwfEdNzvG8X4IGG9eG0rU7SnsCciLgyd8RWSvXaIxcUzPpanqTwq4i4YxznbvXHYf1XQhqB\n9VzgQ6OeSDpJ0qCkwZERT/pWRvUezT2NwsxGk2eYi0FJ3ySbZOeZ2saIuGyU9w0DcxrWZ/PCyXm2\nB3YHfpAaIX8HWC7pyIgYbDxRRCwDlgEMDAz490oJ1Rqa3XnNrL/lSQrTgSeBQxu2BTBaUlgFLJA0\nH/gFcAxwbP0EWT+ImbV1ST8ATm9OCGZm1j15Zl5753hOHBEbJZ0MXAVMAS6MiLWSzgYGI2L5eM5r\n5eQnUs3Kod3YRx+JiM9IOo8W/5cj4pTRTh4RK4AVTdvO2MKxB44arZWWR0k1K4d2JYVa47Krc6xz\n8iQ7ZmXQbuyjK9LPr3QvHKuq+qNozglmfc1jH5mZWZ2TgnWFG5rNyqFtUpA0RdKp3QrGqqs2n4Ib\nms36W9ukEBHP03q8IrMx2VxScFYw62d5Oq/9WNI/A98EnqhtjIibC4vKKsslBbP+licpvCH9PLth\nWwAHTXw4VlUeJdWsHPL0aH5TNwKxanNDs1k55JlPYYakf6iNUirp7yXN6EZwVh2bG5qdFsz6WZ5H\nUi8EHgfenl6PAV8uMiirIM+nYFYKedoUXhURRzWsf0LSrUUFZGZmvZOnpPCUpP1rK5L2A54qLiSr\nIjc0m5VDnpLCu4GvNrQjrAeOLy4kq6LNk+z0OBAza6ttUkhTZr46Il4raTpARDzWlcisUjZPx+ms\nYNbPRuvRvAk4OS0/5oRg4yU3NJuVQp42haslnS5pjqQda6/CIzMzs67L06bwrvTzfQ3bAth14sOx\nqnLnNbNyyNOmcFxE/LhL8VhFufOaWTnkaVM4p0uxWIW5pGBWDnnaFFZKOkq1ZwrNOuCCgll/y9Om\ncBowDXhe0lNkTxdGREwvNDIzM+u6PKOkbt+NQKzaNhc0XVQw62d5RkmVpOMk/VVanyNpUfGhWZXU\nU4Jzgllfy9Om8HlgX+DYtP4b4PzCIrJKckOzWTnkaVPYOyL2knQLQESsl7R1wXFZxchD4pmVQp6S\nwnOSppD+yJM0C9hUaFRWWa4+MutveZLC54DvAC+X9DfA/wf+ttCorHI2Vx85K5j1szxPH31d0mrg\nYLL2wj+NiDsKj8wqxQ3NZuWQp02BiLgTuLPgWKzCPEqqWTnkqT4ymwBuaDYrg0KTgqTDJK2TNCRp\naYv9p0m6XdIaSddIemWR8VjvuU3BrL8VlhTSE0vnA4cDC4ElkhY2HXYLMBARewDfBj5TVDzWW64+\nMiuHIksKi4ChiLgnIp4FLgEWNx4QEddFxJNp9UZgdoHxWA+58sisHIpMCrsADzSsD6dtW3Ii8L1W\nOySdJGlQ0uDIyMgEhmjdUhv7yCUFs/5WZFJo9cdhy18Jko4DBoDPttofEcsiYiAiBmbNmjWBIVq3\nuKRgVg65Hkkdp2FgTsP6bODB5oMkHQJ8HDggIp4pMB7rA25oNutvRZYUVgELJM1PYyUdAyxvPEDS\nnsAXgSMj4uECY7Eec0OzWTkUlhQiYiNwMnAVcAdwaUSslXS2pCPTYZ8FtgO+JelWScu3cDorOY+S\nalYORVYfERErgBVN285oWD6kyM+3/uFRUs3KwT2arTvq1UcuK5j1MycF6yqnBLP+5qRgXeFRUs3K\nwUnBuqLWec1lBbP+5qRgXeFmZrNycFKwrnA/BbNycFKwrnJOMOtvTgrWFbV+Ci4pmPU3JwXrCrmf\nglkpOClYV7ih2awcnBSsq1xOMOtvTgrWHX76yKwUnBSsK+oNzS4rmPU1JwXrCrlRwawUnBSsK+o5\nwQUFs77mpGBd5Zxg1t+cFKwragPiuaHZrL85KVhXbJ6O01nBrJ85KVhXuJ3ZrBycFKwrPEqqWTk4\nKVhXOSeY9TcnBeuSWkOz04JZP3NSsK7Y3NBsZv3MScG6wg3NZuXgpGBdIRcVzErBScG6yv0UzPqb\nk4J1Ra36yO3MZv3NScG6wqOkmpWDk4J1RX0+BZcUzPqak4J1hduZzcqh0KQg6TBJ6yQNSVraYv82\nkr6Z9t8kaV6R8VjvufOaWX8rLClImgKcDxwOLASWSFrYdNiJwPqI+F3gXODTRcVj/cEpway/FVlS\nWAQMRcQ9EfEscAmwuOmYxcBX0vK3gYMlN0lWke+qWTlMLfDcuwAPNKwPA3tv6ZiI2ChpA7AT8EiB\ncVkP1Bqaz1q+lnOuWtfjaMzK6ZSDF/Anr31FoZ9RZFJo9bdhc+1BnmOQdBJwEsDcuXM7j8y67lUv\nn8aSRXPY8NRzvQ7FrLRmvHSrwj+jyKQwDMxpWJ8NPLiFY4YlTQVmAI82nygilgHLAAYGBlwtXULb\nTJ3Cp962R6/DMLNRFNmmsApYIGm+pK2BY4DlTccsB45Py0cD14YfTzEz65nCSgqpjeBk4CpgCnBh\nRKyVdDYwGBHLgX8FviZpiKyEcExR8ZiZ2eiKrD4iIlYAK5q2ndGw/DTwv4uMwczM8nOPZjMzq3NS\nMDOzOicFMzOrc1IwM7M6JwUzM6tT2boFSBoB7h/n22cy+YbQ8DVPDr7myaGTa35lRMwa7aDSJYVO\nSBqMiIFex9FNvubJwdc8OXTjml19ZGZmdU4KZmZWN9mSwrJeB9ADvubJwdc8ORR+zZOqTcHMzNqb\nbCUFMzNrY9IkBUmHSVonaUjS0l7HM1EkzZF0naQ7JK2V9IG0fUdJV0u6O/18WdouSZ9L38MaSXv1\n9grGR9IUSbdIujKtz5d0U7reb6bh2pG0TVofSvvn9TLu8ZK0g6RvS7oz3et9J8E9PjX9m75N0sWS\ntq3ifZZ0oaSHJd3WsG3M91bS8en4uyUd3+qz8pgUSUHSFOB84HBgIbBE0sLeRjVhNgIfiojfB/YB\n3peubSlwTUQsAK5J65B9BwvS6yTgC90PeUJ8ALijYf3TwLnpetcDJ6btJwLrI+J3gXPTcWX0T8D3\nI+L3gNeSXXtl77GkXYBTgIGI2J1s+P1jqOZ9vgg4rGnbmO6tpB2BM8mmPF4EnFlLJGMWEZV/AfsC\nVzWsfwz4WK/jKuhavwv8EbAO2Dlt2xlYl5a/CCxpOL5+XFleZLP4XQMcBFxJNq3rI8DU5vtNNp/H\nvml5ajpOvb6GMV7vdODe5rgrfo9r87fvmO7blcCbq3qfgXnAbeO9t8AS4IsN219w3Fhek6KkwOZ/\nYDXDaVulpCLznsBNwG9HxEMA6efL02FV+C7+EfgIsCmt7wT8d0RsTOuN11S/3rR/Qzq+THYFRoAv\npyqzL0maRoXvcUT8AjgH+DnwENl9W02173Ojsd7bCbvnkyUpqMW2Sj12JWk74N+BD0bEY+0ObbGt\nNN+FpLcAD0fE6sbNLQ6NHPvKYiqwF/CFiNgTeILN1QmtlP6aU9XHYmA+8ApgGlnVSbMq3ec8tnSd\nE3b9kyUpDANzGtZnAw/2KJYJJ2krsoTw9Yi4LG3+laSd0/6dgYfT9rJ/F/sBR0q6D7iErArpH4Ed\nJNVmEmy8pvr1pv0zyKZ+LZNhYDgibkrr3yZLElW9xwCHAPdGxEhEPAdcBryBat/nRmO9txN2zydL\nUlgFLEhPLmxN1mC1vMcxTQhJIpvr+o6I+IeGXcuB2hMIx5O1NdS2/3l6imEfYEOtmFoGEfGxiJgd\nEfPI7uO1EfEO4Drg6HRY8/XWvoej0/Gl+gsyIn4JPCDp1WnTwcDtVPQeJz8H9pH0W+nfeO2aK3uf\nm4z13l4FHCrpZamUdWjaNna9bmDpYkPOEcBdwM+Aj/c6ngm8rv3JiolrgFvT6wiy+tRrgLvTzx3T\n8SJ7EutnwH+RPd3R8+sY57UfCFyZlncFfgIMAd8Ctknbt03rQ2n/rr2Oe5zX+jpgMN3ny4GXVf0e\nA58A7gRuA74GbFPF+wxcTNZu8hzZX/wnjufeAu9K1z8EvHO88bhHs5mZ1U2W6iMzM8vBScHMzOqc\nFMzMrM5JwczM6pwUzMyszknBKkvSDyQVPoevpFPSyKVfb9o+IOlzaflASW+YwM+cJ+nYVp9l1omp\nox9iNvlImhqbx9gZzXuBwyPi3saNETFI1rcAsj4VvwFumKAY5gHHAt9o8Vlm4+aSgvVU+ov3Dkn/\nksbOXynppWlf/S99STPT0BZIOkHS5ZKukHSvpJMlnZYGi7sxDSNcc5ykG9KY/IvS+6elMexXpfcs\nbjjvtyRdAaxsEetp6Ty3Sfpg2nYBWYeq5ZJObTr+QElXpoEK3w2cKulWSW+UNEvSv6cYVknaL73n\nLEnLJK0Evpq+nx9Jujm9aqWNvwPemM53au2z0jl2TN/PmvR97NFw7gvT93qPpFMavo//kPTTdG1/\n1tldtVLrdW8+vyb3i+wv3o3A69L6pcBxafkHpB6bwEzgvrR8Almvze2BWWQjYr477TuXbFDA2vv/\nJS3/IWloYuBvGz5jB7Ke7tPSeYdJvUeb4nw9WQ/SacB2wFpgz7TvPmBmi/ccyOYe12cBpzfs+waw\nf1qeSzZMSe241cBL0/pvAdum5QXAYPO5W3zWecCZafkg4NaGc99A1jN4JvBrYCvgqNr3lI6b0et/\nF3717uXqI+sH90bErWl5NVmiGM11EfE48LikDcAVaft/AXs0HHcxQERcL2m6pB3IxoU5UtLp6Zht\nyX4xA1wdEa0GUtsf+E5EPAEg6TLgjcAteS6whUOAhdmwPgBMl7R9Wl4eEU+l5a2Af5b0OuB5YLcc\n596f7Bc9EXGtpJ0kzUj7/iMingGekfQw8Ntk39k5kj5Nllh+NM5rsgpwUrB+8EzD8vPAS9PyRjZX\ncW7b5j2bGtY38cJ/183juNSGGT4qItY17pC0N9mw1K20Gpq4Ey8hmxTmqcaNKUk0xnAq8Cuy2dZe\nAjyd49zthlFu/q6nRsRdkl5PNmbWpyStjIizc12FVY7bFKyf3UdWbQObR8Ycqz8DkLQ/2YiSG8hG\nj3x/Gn0TSXvmOM/1wJ+mUTunAW8FxvIX9eNk1V01K4GTayupJNDKDOChiNgE/B+yaSlbna851nek\n8x4IPBJt5tiQ9ArgyYj4N7KJbUo5p7NNDCcF62fnAO+RdANZHfh4rE/vv4DN8/l+kqxaZo2yydI/\nOdpJIuJmsrl0f0I2s92XImIsVUdXAG+tNTST5h9OjcG3kzVEt/J54HhJN5JVHdVKEWuAjalx+NSm\n95xVOzdZg/Rok7j/AfATSbcCHwf+egzXZRXjUVLNzKzOJQUzM6tzUjAzszonBTMzq3NSMDOzOicF\nMzOrc1IwM7M6JwUzM6tzUjAzs7r/AdaD7IwLhFQNAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(error)\n",
    "plt.xlabel('number of iterations')\n",
    "plt.ylabel('error in training data')\n",
    "\n",
    "plt.figure()\n",
    "plt.plot(error_of_rounded)\n",
    "plt.xlabel('number of iterations')\n",
    "plt.ylabel('error in training data')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "### With Automatic differentiation\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "from __future__ import print_function\n",
    "import mxnet as mx\n",
    "from mxnet import nd, autograd, gluon\n",
    "\n",
    "np.random.seed(1)\n",
    "mx.random.seed(1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Define the parameters."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "W0 = 2*nd.random.randn(3,4, ctx=mx.cpu()) - 1\n",
    "W1 = 2*nd.random.randn(4,1, ctx=mx.cpu()) - 1\n",
    "params = [W0, W1]\n",
    "for param in params:\n",
    "    param.attach_grad()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Define the neural network structure."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "def activation_mx(x):\n",
    "    return 1/(1+nd.exp(-x))\n",
    "\n",
    "def net(X):\n",
    "    f1 = activation_mx(nd.dot(X, W0))\n",
    "    f2 = activation_mx(nd.dot(f1, W1))\n",
    "    return f2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Do optimization!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "error = []\n",
    "error_of_rounded = []\n",
    "\n",
    "for i in range(2000):\n",
    "    # We do forward pass while recording the symbolic graph so as to be able to do automatic differentiation\n",
    "    with autograd.record():\n",
    "        # FORWARD PASS\n",
    "        f2 = net(nd.array(X))\n",
    "        loss = nd.sum(0.5 * (nd.array(y)-f2)**2 )\n",
    "    loss_of_rounded = nd.sum(0.5 * (nd.array(y)-np.round(f2))**2 )\n",
    "\n",
    "    # BACKWARDS PASS\n",
    "    loss.backward()\n",
    "\n",
    "    # PARAMETER UPDATE\n",
    "    for param in params:\n",
    "        param[:] = param - 0.8 * param.grad\n",
    "\n",
    "    # Keep track of error\n",
    "    error.append(loss.asscalar())\n",
    "    error_of_rounded.append(loss_of_rounded.asscalar())\n",
    "\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Plot the error per iteration of the optimization:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Text(0,0.5,'error in training data')"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEKCAYAAAD9xUlFAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4zLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvIxREBQAAIABJREFUeJzt3XmcHHWd//HXp3uuzJnMlTuZEBIg\n3GS4lFO51XAoCi4Kq/vj566Iiv48Hu4qsKvrtevvp4KKii6KIipqUBRQIYgQyCQEQggJuSbXJJlc\nk2My9+f3R9U0nWGOnmR6qmf6/Xw86tHV1dXV76k5PlPfqvp+zd0REREBiEUdQEREMoeKgoiIJKgo\niIhIgoqCiIgkqCiIiEiCioKIiCSoKIiISIKKgoiIJKgoiIhIQk7UAQarsrLSa2pqoo4hIjKiLF68\neIe7Vw203ogrCjU1NdTV1UUdQ0RkRDGz+lTWU/ORiIgkqCiIiEiCioKIiCSoKIiISIKKgoiIJKgo\niIhIgoqCiIgkZE1RqFu/i6/86VU0/KiISN+ypigs29zEd55cw479bVFHERHJWFlTFGoqiwBYv/NA\nxElERDJX1hSFGRVBUVi3Q0VBRKQvWVMUpowbQ07MWK+iICLSp6wpCjnxGFPLC9V8JCLSj6wpCgA1\nFYWs29EcdQwRkYyVXUWhsoj6nQd0WaqISB+yqijMqCyiua2Txn2tUUcREclIWVUUanQFkohIv7Kq\nKMzQvQoiIv3KqqIwsayA3LjpZLOISB+yqigkLktV85GISK+yqihAcGezmo9ERHqXdUWhpjIoCl1d\nuixVRKSnrCwKLe1dbNvXEnUUEZGMk3VFQR3jiYj0LeuKQk1lIQDrdQWSiMgbZF1RmFQ2hrycGOt2\n7I86iohIxsm6ohCLmTrGExHpQ9YVBQi6u9BlqSIib5SVRWFGVREbdjbTqctSRUQOkdaiYGaXmdlK\nM1ttZp/pZ713mZmbWW0683SbUVFEW2cXW/YcHI6PExEZMdJWFMwsDtwFXA7MAa43szm9rFcC3Ao8\nl64sPXV3jKfLUkVEDpXOI4UzgNXuvtbd24AHgCt7We/fga8Cw3Y3mYqCiEjv0lkUJgMbk55vCpcl\nmNmpwFR3/31/GzKzm82szszqGhsbjzhYVUk+RXlxFQURkR7SWRSsl2WJM7tmFgO+AXxioA25+z3u\nXuvutVVVVUcezIyayiIVBRGRHtJZFDYBU5OeTwG2JD0vAU4AnjSz9cBZwPxhO9lcqctSRUR6SmdR\nWATMMrMZZpYHXAfM737R3ZvcvdLda9y9BlgIzHP3ujRmSphRWcTGXc20dXQNx8eJiIwIaSsK7t4B\n3AI8CqwAHnT35WZ2p5nNS9fnpmpGZRFdDht3685mEZFuOencuLs/AjzSY9nn+1j3gnRm6an7CqQ1\n2/czs6p4OD9aRCRjZeUdzQCzxpcAsHLrvoiTiIhkjqwtCsX5OUyvKGTF1r1RRxERyRhZWxQAjp1Q\nwooGHSmIiHTL6qJw3MRS1u88QHNbR9RRREQyQtYXBXedVxAR6ZbVRWHOxFIAXt7cFHESEZHMkNVF\nYcq4MVSV5LNkw56oo4iIZISsLgpmRu30cdTV74o6iohIRsjqogAwd/o4Nu46yPa9w9Zzt4hIxlJR\nmD4OgMX1uyNOIiISvawvCsdPKqMoL84za3ZGHUVEJHJZXxTycmKcPbOCJ1dtx90HfoOIyCiW9UUB\n4LzZVWzcdZD1O9VjqohkNxUF4PzZwWhuT6068qE+RURGMhUFYHpFETUVhfzl1e1RRxERiZSKQujS\nEybwzOodNDW3Rx1FRCQyKgqhK06YSEeX8/iKbVFHERGJjIpC6KQpZUweO4ZHljVEHUVEJDIqCiEz\n44oTJ/C31xppOqgmJBHJTgMWBTOrMrOvm9kjZvbX7mk4wg23t500ifZO5486WhCRLJXKkcL9wApg\nBnAHsB5YlMZMkTl5ShlHVxfzq8Wboo4iIhKJVIpChbv/EGh39wXu/gHgrDTnioSZce3cKdTV72Zt\n4/6o44iIDLtUikJ3A3uDmb3NzE4FpqQxU6SuPnUyMYNfL9HRgohkn1SKwn+YWRnwCeCTwA+Aj6U1\nVYSqSws4f3YVDy3ZTGeX+kISkeySSlHY7e5N7v6yu1/o7nOBUT0qzbW1U2loamHBKt3hLCLZJZWi\n8K0Ul40aF88ZT3VJPvc9Wx91FBGRYZXT1wtmdjbwJqDKzG5LeqkUiKc7WJRy4zGuP2Ma3/zra9Tv\nPMD0iqKoI4mIDIv+jhTygGKCwlGSNO0F3pX+aNF675nTiJvx04U6WhCR7NHnkYK7LwAWmNmP3T3r\n/jKOLy3g0hMm8ItFG7nt4mMYkzeqD45ERIDUzik0m9nXsuGO5p7ef9Z09rZ0MP/FzVFHEREZFqne\n0fwqWXBHc09nzCjn2Akl3PdsvYbqFJGsoDua+2Fm3HDWdJZv2csLG/dEHUdEJO10R/MArj51MsX5\nOfxEl6eKSBY43DuaP57WVBmkKD+Hd542mT+81MCO/a1RxxERSasBi4K7/77nHc3uPj+VjZvZZWa2\n0sxWm9lnenn9Q2a2zMyWmtnTZjbncL6IdHvf2dNp6+ziwbqNUUcREUmr/m5e+xbQ59lVd7+1vw2b\nWRy4C7gY2AQsMrP57v5K0mo/c/fvhuvPA/4buCz1+MPj6OoSzj6qgvsXbuB/nzeTeMyijiQikhb9\nHSnUAYuBAuA04LVwOgXoTGHbZwCr3X2tu7cBDwBXJq/g7nuTnhbRTxGK2vvPns7mPQf566vqD0lE\nRq/+bl77HwAzuwm40N3bw+ffBR5LYduTgeT2lk3AmT1XMrMPA7cR3EH9llSDD7eL5oxnfGk+P11Y\nz8VzxkcdR0QkLVI50TyJoHuLbsXhsoH01sbyhiMBd7/L3WcCnwb+tdcNmd1sZnVmVtfY2JjCRw+9\n3HiMd82dwt9ea2T7vpZIMoiIpFsqReHLwAtm9mMz+zGwBPhSCu/bBExNej4F2NLP+g8AV/X2grvf\n4+617l5bVVWVwkenx1WnTKbL4Q8vaQxnERmdUrn66EcEzT6/Caezu5uWBrAImGVmM8wsD7gOOOSq\nJTOblfT0bQTnLDLWrPElHDexlN8t7a+2iYiMXKkcKeDuW939d+G0NcX3dAC3AI8CK4AH3X25md0Z\nXmkEcIuZLTezpQTnFW48jK9hWF11yiSWbtzD+h0Hoo4iIjLkUioKh8vdH3H32e4+092/GC77fPd9\nDu7+UXc/3t1PCe+BWJ7OPEPh7ScHp1MeeyWl2igiMqKktSiMRpPHjuG4iaX8ZYUuTRWR0WfAomBm\n5b1MucMRLlO99dhq6up309TcPvDKIiIjSCpHCkuARmAVwYngRmCdmS0xs7npDJep3nJcNZ1dzoLX\nork8VkQkXVIpCn8CrnD3SnevAC4HHgT+Bbg7neEy1clTxlJRlMdfVmyLOoqIyJBKpSjUuvuj3U/c\n/THgPHdfCOSnLVkGi8eM82dX8dSqRjq7MrZnDhGRQUulKOwys0+b2fRw+hSwO+zwrivN+TLW+cdU\nsbu5nZc3N0UdRURkyKRSFN5LcDfyb4HfAdPCZXHg3emLltnOOboSM1iwSucVRGT06LNDvG7uvgP4\nSB8vrx7aOCNHRXE+J00uY8GqRm5966yB3yAiMgIMWBTMbDbBiGs1yeu7e8b2aDpczp9dxbefWE1T\ncztlhVl9la6IjBKpNB/9EniBoAfT/5M0Zb3zj6miy+Hp1TuijiIiMiQGPFIAOtz9O2lPMgKdPGUs\nZWNyWbBqO287aWLUcUREjlgqRwoPm9m/mNnE5Lua055sBMiJxzhnViULVjXirktTRWTkS6Uo3EjQ\nXPQMwfCciwmG6hSC8wrb9rby6tZ9UUcRETliqVx9NGM4goxU588OBv1ZsKqR4yaWRpxGROTI9FkU\nzOwt7v5XM7umt9fd/aH0xRo5xpcWcOyEEhasbORD58+MOo6IyBHp70jhfOCvwDt6ec0BFYXQ+cdU\nce/T69jf2kFxfirn7kVEMlOff8Hc/Qvh4z8OX5yR6fzZVXxvwVqefq2Ry07QVUgiMnKlcvNaPvBO\n3njz2p3pizWynFFTTlVJPr9esllFQURGtFSuPvodcCXQARxImiSUE49xzamTeeLV7TTua406jojI\nYUulAXyKu1+W9iQj3LW1U/jeU2v57Qub+V/nHRV1HBGRw5LKkcIzZnZi2pOMcEdXl3DatLHc/1y9\nxlgQkRErlaJwDrDYzFaa2UtmtszMXkp3sJHoA+fMYP3OZv6sEdlEZIRKpfno8rSnGCUuO34CU8aN\n4ftPreXS4ydEHUdEZND6PFIws+7bc/f1MUkPOfEYH3jzDOrqd7No/a6o44iIDFp/zUc/Cx+7+zpa\njPo+GtB1Z0ylsjif/3pspTrJE5ERp8+i4O5vDx9nuPtR4WP3pMtr+lCYl8OHL5zJwrW7+PvqnVHH\nEREZlFRONGNm48zsDDM7r3tKd7CR7L1nTmNSWQFf09GCiIwwAxYFM/sn4CngUeCO8PH29MYa2fJz\n4nzsotm8uHEP81/cEnUcEZGUpXKk8FHgdKDe3S8ETgUa05pqFHjn3CmcOLmMLz2yggOtHVHHERFJ\nSSpFocXdWyDoB8ndXwWOSW+skS8eM26fdzzb9rZy1xOro44jIpKSVIrCJjMbC/wWeNzMfgeoTSQF\nc6eP45rTJvODv61j9fb9UccRERnQgEXB3a929z3ufjvwb8APgavSHWy0+Ozlx1GYH+dTv3pR3V+I\nSMbrtyiYWczMXu5+7u4L3H2+u7elP9roUFWSzx3zjmfJhj3c+/S6qOOIiPSr36Lg7l3Ai2Y2bZjy\njErzTp7ExXPG8/XHVqoZSUQyWirnFCYCy83sL2Y2v3tKd7DRxMz44tUnUJgX5yM/f4GW9s6oI4mI\n9CqVonAH8HbgTuC/kqYBmdllYe+qq83sM728fpuZvRL2vvoXM5s+mPAjSXVJAf/97lNY0bCXO3//\nStRxRER6lUpRuCI8l5CYgCsGepOZxYG7CHpZnQNcb2Zzeqz2AlDr7icBvwK+Orj4I8uFx1bzofNn\n8rPnNvC7pZujjiMi8gapFIWLe1mWSnfaZwCr3X1teGL6AYJhPRPc/Ql3bw6fLgSmpLDdEe0Tl8zm\n9JpxfPrXL/Hixj1RxxEROUR/XWf/s5ktA44Jm3e6p3VAKoPsTAY2Jj3fFC7ryweBP6YSeiTLjcf4\nzg1zqSzO55/uq2PznoNRRxIRSRio6+x3APPDx+5prrvfkMK2rZdlvV6ob2Y3ALXA1/p4/WYzqzOz\nusbGkd/DRmVxPj+66XRa2jr54I8X0XSwPepIIiJA/11nN7n7ene/3t3rk6ZUR4/ZBExNej6FXu6E\nNrOLgM8B89y9tY8s97h7rbvXVlVVpfjxmW3W+BK+c8Nc1jTu56YfPc9+9Y8kIhkgpa6zD9MiYJaZ\nzTCzPOA6gqOOBDM7FfgeQUHYnsYsGemcWZV86/rTeGlTEx/40SKa21QYRCRaaSsK7t4B3ELQ1fYK\n4EF3X25md5rZvHC1rwHFwC/NbGk23v9w2QkT+L/vOYW6+l3c9CM1JYlItGykDQJTW1vrdXWjbzTQ\n+S9u4RMPLmVmVTH3feAMqksLoo4kIqOImS1299qB1ktlkJ1rzOw1M2sys71mts/M9g5NTOk27+RJ\n3HvT6WzY1cw133mGlVv3RR1JRLJQKs1HXyVo8y9z91J3L3H30nQHy0bnzqrigZvPorWji6vv/jt/\neKkh6kgikmVSKQrb3H1F2pMIACdNGcvvP3IOx04o4cM/W8KXHllBW0dX1LFEJEukUhTqzOwXZnZ9\n2JR0jZldk/ZkWWx8aQEP3Hw2N5w1jXueWsvVd/+dVdvUnCQi6ZdKUSgFmoFLeP0GtrenM5RAXk6M\n/7jqRO5531y2NrXw9m89zT1PraGjU0cNIpI+uvpoBNixv5XPPrSMx1/ZxjHjS7jzyuM586iKqGOJ\nyAiS6tVHfRYFM/uUu3/VzL5FL91TuPutRx5z8LKxKAC4O4+/so07Hn6FzXsOMu/kSXziktlMryiK\nOpqIjACpFoWcfl7rPrmcfX+BM5CZccnxEzh3VhV3P7ma7/9tLY8sa+A9p0/lI2+ZxYQy3dcgIkdO\nzUcj1Pa9LXz7idX8/PkNmBnvPG0K/3TuDGZWFUcdTUQy0BE3H2UqFYVDbdzVzN1PruHXSzbR1tHF\nW4+t5h/fPIM3zawgFuuto1oRyUYqCllmx/5WfvJsPT9ZWM+uA21MLR/DtXOn8q65U5g0dkzU8UQk\nYkNSFMIhNW91928MZbgjoaLQv5b2Th5dvpUH6zby99U7MYM3zazgihMncunxE6gszo86oohEYMiO\nFMzsSXe/YKiCHSkVhdRt2NnMLxdv5OEXt7B+ZzMxg7OOquCyEyZwwexqplUURh1RRIbJUBaFLwJl\nwC+AA93L3X3JkYY8HCoKg+furGjYxx9fbuAPyxpY2xh8G2dUFnH+7CrOn13FmUeVU5jX38VoIjKS\nDWVReKKXxe7ubznccEdCReHIuDvrdhxgwapGFqxqZOHanbS0dxGPGSdMKuX0mnJOn1HO6TXllBfl\nRR1XRIaITjRLSlraO1m0fhcL1+5k0brdLN20J9EB31FVRZw0uYwTwun4SaWUFORGnFhEDsdQ3LzW\nvaEy4AvAeeGiBcCd7t50ZBElExTkxjl3VhXnzgrGvm5p72TZ5iaeX7eLFzbs5tm1O/nt0mBobTOY\nUVHE8ZPLOGZ8MUdXlzB7fDHTygvJiadzZFcRGS6pNCLfC7wMvDt8/j7gR4B6Sh2FCnLjQRNSTXli\n2fZ9LSzfvJdlm5tYtrmJJfW7efjFLYnX8+IxjqoqYtb4Eo6uKqamspBp5YVMryhiXGEuZrpfQmSk\nSOWcwlJ3P2WgZcNFzUeZYX9rB2u27+e17ft5bfs+XtsWPG7cdfCQ9Uryc5haXsj0ikKmVRQyvbyI\niWMLmFQ2holjCyhVc5TIsBiy5iPgoJmd4+5Phxt+M3BwgPfIKFecn8PJU8dy8tSxhyxvae9k465m\n6nc2U7+rmQ07D1C/q5mVW/fx5xXbaO/0N2xnYlkBE8eOYVJZARPKgoIxoayAqpJ8qkvyGVeYp7uz\nRYZJKkXhQ8B94bkFgN3AjemLJCNZQW6cWeNLmDW+5A2vdXY5DU0H2drUwpamFhr2HKShqYWGpuDx\nlS172bG/9Q3vi8eMyuI8qkryqSrODx4T8wWvPy/JpygvruYqkSPQb1EwsxhwjLufbGalAO6+d1iS\nyagTjxlTxhUyZVzfN821dnSyramVrXtb2LG/lcZ9SVP4fEXDPnbsb6Wj641NnwW5MapK8qksDopG\nZcmhj1UleVQVF1BZkqf7MkR60e9vhbt3mdktwIMqBjIc8nPiTAvPP/Snq8vZ3dyWKBSN+1oTRWTH\n/jYa97WyYVczi+t3s6u5jd5OnRXlxV8vGsX5VCYVjEMLST4FufE0fcUimSWVf5UeN7NP8sY7mnel\nLZXIAGIxo6I4n4rifI6d0P+6HZ1d7DrwegHpLhqvF5FW1jTu57l1rexubu91GyX5OVSX5ieuqqqp\nKGR6ZRE1FUVMKy8krnMeMkqkUhQ+ED5+OGmZA0cNfRyRoZcTj1FdWkB16cADEbV3drFzf9vrTVdJ\nhWNrUwv1O5t5ft0uDrR1Jt5TmBfn+EmlnDC5jFOmjuXNR1eq40EZsVI5p3CDu/99mPKIRCo3HmNC\neBVUX9ydHfvbqN95gLU7DvDKluAejgee38iP/r4eMzhxchmXHj+Ba+dOSakYiWSKVO5TeNbdzx6m\nPAPSfQqSqTo6u3ilYS8LVjbyxMrtLNmwh3jMeMdJE7nt4mPUK61Eaig7xLsDeAl4yDOgoyQVBRkp\n1jbu5+fPb+AnC+vp7HI+cckx3HzuUbrnQiIxlEVhH1AEdBLctGYEvaSWDkXQwVJRkJFma1MLdzy8\nnD++vJVLjx/PN68/lfwcXc0kwyvVojBgL2buXuLuMXfPdffS8HkkBUFkJJpQVsDd/3Aa//b2OTy6\nfBsf/flSunq5x0IkEwxYFCxwg5n9W/h8qpmdkf5oIqOHmfHBc2bwr287jj8t38r3nlobdSSRXqXS\n3/HdwNnAe8Pn+4G70pZIZBT74DkzuPyECXzjz6vYsLM56jgib5BKUTjT3T8MtAC4+25AQ3KJHAYz\n4/Z5x5MTM77y6KtRxxF5g1SKQruZxQluWMPMqoCutKYSGcXGlxZw45tq+OOyBjbu0tGCZJZUisI3\ngd8A1Wb2ReBp4EtpTSUyyr3/7OmYGT9dWB91FJFDpHL10f3Ap4D/BBqAq9z9l6ls3MwuM7OVZrba\nzD7Ty+vnmdkSM+sws3cNNrzISDWxbAwXHlPF/Be36EokySgpDazr7q+6+13u/m13X5HKe8Imp7uA\ny4E5wPVmNqfHahuAm4CfpR5ZZHS44sSJNDS18MLGPVFHEUlI52jrZwCr3X2tu7cBDwBXJq/g7uvd\n/SV0jkKy0FuPG088Zjy5cnvUUUQS0lkUJgMbk55vCpcNmpndbGZ1ZlbX2Ng4JOFEolY2JpcTJ5fx\n7JqdUUcRSUhnUeitg5fDajx193vcvdbda6uqqo4wlkjmOHtmBUs37uFAa0fUUUSA9BaFTcDUpOdT\ngC1p/DyREeeMGeV0dDnLNjdFHUUESG9RWATMMrMZZpYHXAfMT+PniYw4J04uA+BlFQXJEGkrCu7e\nAdwCPAqsIBjnebmZ3Wlm8wDM7HQz2wRcC3zPzJanK49IJqoszmdiWYGOFCRjpDIc52Fz90eAR3os\n+3zS/CKCZiWRrHXC5DIVBckY6Ww+EpEUnDi5jHU7DrBfJ5slA6goiERs9vgS3IOR2kSipqIgErGj\nq4sBWL1dRUGip6IgErHpFYXkxExFQTKCioJIxHLjMaZXFLJGzUeSAVQURDLAzKpiHSlIRlBREMkA\nR1cXU7+zmfZO9Q0p0VJREMkAR1cX09Hl1GvcZomYioJIBphZpSuQJDOoKIhkgJnhZak62SxRU1EQ\nyQDF+TlMKivQkYJETkVBJEPMrNYVSBI9FQWRDHF0dTFrGvfT1XVYY1GJDAkVBZEMcXR1Mc1tnWxp\nOhh1FMliKgoiGeJoXYEkGUBFQSRDzBpfAqgoSLRUFEQyRHlRHuVFeSoKEikVBZEMctzEEl7eolHY\nJDoqCiIZ5NSp41jRsI+DbZ1RR5EspaIgkkFOmTqWzi7X0YJERkVBJIOcMm0sAC9s2B1xEslWKgoi\nGaSyOJ+jKot4evXOqKNIllJREMkwbz2umoVrdrK/tSPqKJKFVBREMsxFx42nrbOLv6zYFnUUyUIq\nCiIZpramnKnlY7j/uQ1RR5EspKIgkmHiMeN9Z03n+XW7eH7drqjjSJZRURDJQO87q4ZJZQV8Yf5y\nWjt0z4IMHxUFkQw0Ji/OHVeewIqGvXz2oWXqTluGjYqCSIa6eM54PnbRLB5asplbfr6EpoPtUUeS\nLJATdQAR6dvHLppNUV4OX/7TqyyuX8BtF8/m6lOnkJej/+ckPcx9ZB2W1tbWel1dXdQxRIbVS5v2\n8LnfvMyyzU2ML83nylMmM+/kScyZWEosZlHHkxHAzBa7e+2A66koiIwM7s6Tqxr56bP1LFjVSEeX\nU16Ux9lHVXB6zTjmTCrj2IkllBbkRh1VMlCqRUHNRyIjhJlx4THVXHhMNbsPtPHnFdt4du1Onl2z\nkz8sa0isN7V8DDMqi5leXsi08kKmVRQydVwh40vzGVeYpyML6ZeKgsgINK4oj2trp3Jt7VTcnW17\nW1nRsJdXGvayomEv9TubWbphN3tbDu0qIydmVBbnU12aT3VJPlUlBZQX5TJ2TB5lY3IpK8xlbOIx\nj7GFuRTkxiP6KiUKaS0KZnYZ8P+AOPADd/9yj9fzgfuAucBO4D3uvj6dmURGGzNjQlkBE8oKuPDY\n6kNea2pup37XATbtPsj2vS1s39dK475Wtu9rZfOeFpZu3MPu5nY6+7nkNS8eoyg/TmFeDsX5ORTm\nxynKy6Eo8RgsK87LoTA/hzG5cfJzYhQkP+bGKMjp/TE/J4aZjl4yRdqKgpnFgbuAi4FNwCIzm+/u\nrySt9kFgt7sfbWbXAV8B3pOuTCLZpqwwl5MKx3LSlLF9ruPuHGjrZE9zG3ua29l7sJ09B9vZ09xO\n08Fgam7rYH9rB82tnRxo6+BAawc79rcGy9o62d/aQVtH12HnzMuJUZATIz83Tm7MyM2JkRMzcuMx\n8vqYz82JBesOMJ8TM+IxIydmxMLHeCxGPEbwaMHr8cRrvUzhOjlxO2T95Cmne1vhOrEYxCyYN2PE\nFL50HimcAax297UAZvYAcCWQXBSuBG4P538FfNvMzEfa2W+REczMKM4PjgKmjDv87bR3dtHc2klL\nRyet7V2HPLa097asi9akx9bwsb3T6ejsor3TaevsOmT+QGsH7Z1Oe2dXOHmPxy46wnUzjRlBsUgq\nGMEEsVh38TDiya/1KCwfu2g27zh5UlpzprMoTAY2Jj3fBJzZ1zru3mFmTUAFsCONuUQkDXLjMcoK\nY5QR/dVP7k5HlycKRFeX0+lOZ9frU0fXoc87E+sEhaXn+snv63LvdZ2OLqcraZ2uLqfLCebDqbMr\nyNfZz2uHrOfhel3O2ML079t0FoXejpV6HgGksg5mdjNwM8C0adOOPJmIjGpmRm7cyI3DGHSifDDS\neVvkJmBq0vMpwJa+1jGzHKAMeEO3kO5+j7vXunttVVVVmuKKiEg6i8IiYJaZzTCzPOA6YH6PdeYD\nN4bz7wL+qvMJIiLRSVvzUXiO4BbgUYJLUu919+VmdidQ5+7zgR8CPzGz1QRHCNelK4+IiAwsrfcp\nuPsjwCM9ln0+ab4FuDadGUREJHXqalFERBJUFEREJEFFQUREElQUREQkYcSNp2BmjUD9Yb69ksy8\nW1q5BidTc0HmZlOuwRmNuaa7+4A3eo24onAkzKwulUEmhptyDU6m5oLMzaZcg5PNudR8JCIiCSoK\nIiKSkG1F4Z6oA/RBuQYnU3NB5mZTrsHJ2lxZdU5BRET6l21HCiIi0o+sKQpmdpmZrTSz1Wb2mWH+\n7Klm9oSZrTCz5Wb20XD57WYXx516AAAIi0lEQVS22cyWhtMVSe/5bJh1pZldmsZs681sWfj5deGy\ncjN73MxeCx/HhcvNzL4Z5nrJzE5LU6ZjkvbJUjPba2Yfi2J/mdm9ZrbdzF5OWjbo/WNmN4brv2Zm\nN/b2WUOQ62tm9mr42b8xs7Hh8hozO5i0376b9J654fd/dZj9iMaM7CPXoL9vQ/372keuXyRlWm9m\nS8Plw7m/+vrbEN3PmLuP+omgl9Y1wFFAHvAiMGcYP38icFo4XwKsAuYQDEX6yV7WnxNmzAdmhNnj\nacq2HqjsseyrwGfC+c8AXwnnrwD+SDA40lnAc8P0vdsKTI9ifwHnAacBLx/u/gHKgbXh47hwflwa\ncl0C5ITzX0nKVZO8Xo/tPA+cHWb+I3B5GnIN6vuWjt/X3nL1eP2/gM9HsL/6+tsQ2c9YthwpJMaL\ndvc2oHu86GHh7g3uviSc3wesIBiKtC9XAg+4e6u7rwNWE3wNw+VK4H/C+f8Brkpafp8HFgJjzWxi\nmrO8FVjj7v3dsJi2/eXuT/HGgZ8Gu38uBR53913uvht4HLhsqHO5+2Pu3hE+XUgwsFWfwmyl7v6s\nB39Z7kv6WoYsVz/6+r4N+e9rf7nC//bfDfy8v22kaX/19bchsp+xbCkKvY0X3d8f5bQxsxrgVOC5\ncNEt4WHgvd2HiAxvXgceM7PFFgx7CjDe3Rsg+KEFqiPI1e06Dv1ljXp/weD3TxT77QME/1F2m2Fm\nL5jZAjM7N1w2OcwyHLkG830b7v11LrDN3V9LWjbs+6vH34bIfsaypSikNBZ02kOYFQO/Bj7m7nuB\n7wAzgVOABoJDWBjevG9299OAy4EPm9l5/aw7rPvRghH75gG/DBdlwv7qT185hnu/fQ7oAO4PFzUA\n09z9VOA24GdmVjqMuQb7fRvu7+f1HPqPx7Dvr17+NvS5ah8ZhixbthSFVMaLTiszyyX4pt/v7g8B\nuPs2d+909y7g+7ze5DFsed19S/i4HfhNmGFbd7NQ+Lh9uHOFLgeWuPu2MGPk+ys02P0zbPnCE4xv\nB/4hbOIgbJ7ZGc4vJmivnx3mSm5iSkuuw/i+Def+ygGuAX6RlHdY91dvfxuI8GcsW4pCKuNFp03Y\nZvlDYIW7/3fS8uT2+KuB7isj5gPXmVm+mc0AZhGc4BrqXEVmVtI9T3Ci8mUOHTv7RuB3SbneH14B\ncRbQ1H2ImyaH/AcX9f5KMtj98yhwiZmNC5tOLgmXDSkzuwz4NDDP3ZuTlleZWTycP4pg/6wNs+0z\ns7PCn9H3J30tQ5lrsN+34fx9vQh41d0TzULDub/6+ttAlD9jR3LmfCRNBGftVxFU/c8N82efQ3Ao\n9xKwNJyuAH4CLAuXzwcmJr3nc2HWlRzhFQ795DqK4MqOF4Hl3fsFqAD+ArwWPpaHyw24K8y1DKhN\n4z4rBHYCZUnLhn1/ERSlBqCd4L+xDx7O/iFo418dTv+YplyrCdqVu3/Gvhuu+87w+/sisAR4R9J2\nagn+SK8Bvk14Q+sQ5xr0922of197yxUu/zHwoR7rDuf+6utvQ2Q/Y7qjWUREErKl+UhERFKgoiAi\nIgkqCiIikqCiICIiCSoKIiKSoKIgo5aZPWlmaR9n18xuDXu5vL/H8loz+2Y4f4GZvWkIP7PGzN7b\n22eJHImcqAOIZCIzy/HXO5cbyL8QXGO/Lnmhu9cBdeHTC4D9wDNDlKEGeC/ws14+S+Sw6UhBIhX+\nx7vCzL5vQX/yj5nZmPC1xH/6ZlZpZuvD+ZvM7Ldm9rCZrTOzW8zstrADs4VmVp70ETeY2TNm9rKZ\nnRG+vyjsmG1R+J4rk7b7SzN7GHisl6y3hdt52cw+Fi77LsFNgPPN7OM91r/AzH4fdnT2IeDjFvTP\nf2541+yvwwyLzOzN4XtuN7N7zOwx4L5w//zNzJaEU/fRxpeBc8Ptfbz7s8JtlIf756Vwf5yUtO17\nw/261sxuTdoffzCzF8Ov7T1H9l2VEW2o7vzUpOlwJoL/eDuAU8LnDwI3hPNPEt6xCVQC68P5mwju\n2iwBqoAmwrtSgW8QdCrW/f7vh/PnEfaRD3wp6TPGEtw5WxRudxPh3aM9cs4luIO0CCgmuOP11PC1\n9fQYkyJcfgHw+3D+dpLGFCD4D/+ccH4aQTcH3estBsaEzwuBgnB+FlDXc9u9fNa3gC+E828BliZt\n+xmC8QsqCe4YzyW4g/f7Sdsq6/m1aMqeSc1HkgnWufvScH4xQaEYyBMe9D+/z8yagIfD5cuAk5LW\n+zkE/embWakFo5FdAswzs0+G6xQQ/GGGsE/6Xj7vHOA37n4AwMweIuhy+YVUvsBeXATMsdcH7iq1\nsB8qYL67Hwznc4Fvm9kpQCdBx2wDOYfgDz3u/lczqzCzsvC1P7h7K9BqZtuB8QT77Otm9hWCwvK3\nw/yaZBRQUZBM0Jo03wmMCec7eL2Js6Cf93QlPe/i0J/rnv24dHcz/E53X5n8gpmdCRzoI+MRDbvY\nixhwdtIf/+4M9MjwcWAbcHL4npYUtt1fN8o993WOu68ys7kEfe78p5k95u53pvRVyKijcwqSydYT\nNNsAvOswt/EeADM7h6BHySaC3iM/EvZQiZmdmsJ2ngKuMrNCC3qUvRoYzH/U+wiau7o9BtzS/SQ8\nEuhNGdDgQbfT7yMYqrK37fXM+g/hdi8Adng/ffSb2SSg2d1/CnydYNhKyVIqCpLJvg78s5k9Q9AG\nfjh2h+//LkGPnQD/TtAs85IFA7n/+0Ab8WDIxB8TdO38HPADdx9M09HDwNXdJ5qBW4Ha8GTwKwQn\nontzN3CjmS0kaDrqPop4CegITw5/vMd7bu/eNsEJ6YEGcT8ReN6Cges/B/zHIL4uGWXUS6qIiCTo\nSEFERBJUFEREJEFFQUREElQUREQkQUVBREQSVBRERCRBRUFERBJUFEREJOH/A8XLZlZFn+wOAAAA\nAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEKCAYAAAD9xUlFAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4zLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvIxREBQAAHS5JREFUeJzt3Xu0HGWZ7/Hvz3BTIBFI5ChJTHAC\nx+io4B5EQeWi3I4SFccBhxHUGY4XRGGYkVk6iDhnRtAZ1wJRThwZwEFBZwSDE4awRrwcEcwOBOQW\n2UI4RCIEyUlQroHn/FFvd2p3untX73R17+76fdbqtauqq6ueru5dT7/vW+9bigjMzMwAntfvAMzM\nbOpwUjAzszonBTMzq3NSMDOzOicFMzOrc1IwM7M6JwUzM6tzUjAzszonBTMzq9um3wF0aubMmTFv\n3rx+h2FmNlBWrFjxSETMmmi9gUsK8+bNY3R0tN9hmJkNFEn3F1nP1UdmZlbnpGBmZnVOCmZmVuek\nYGZmdU4KZmZWV1pSkHSRpIcl3d7ieUk6T9KYpNsk7VtWLGZmVkyZJYWLgSPaPH8ksCA9TgK+WmIs\nZmZWQGn9FCLix5LmtVllEXBpZPcDvVHSCyW9OCLWlhHP8tWP8pNfrmP/PXfjDX8wE4Bfrfsd31v5\nILS4Jakk3v3a2czZ9QVlhGRmNuX0s/PaHsADufk1adkWSUHSSWSlCebOnTupnd18/3rO+8EY169a\nx9UfOxCAS25YzaU/ux+p+Wsi4NnngtMP33tS+zQzGzT9TArNTsVNf7JHxGJgMcDIyEjzn/UT+J9v\nfhnLVz/K2g1P1pc9+1wwc6ftGP30W5u+Zq9PX8OzLUoRZmbDqJ9XH60B5uTmZwMPlr3T/Dm+yOne\nOcHMqqSfSWEJ8L50FdL+wIay2hM2a1FPZGZmQInVR5K+BRwEzJS0BvgMsC1ARFwILAWOAsaAx4H3\nlxXLBJFO4hkzs+FU5tVHx03wfAAfLWv/Lfc7LoYi67v+yMyqo1I9mltdZWRmZplKJQWAGFc8iLaJ\nQqJYa7SZ2ZCoVFJwQcHMrL1KJYVm2iUKOY2YWcVUOikUa2g2M6uOSiUFNzSbmbVXqaQAW5YOJmpo\nDndpNrMKqVRSaGwj8PnezGy8SiWFZto1Jru2ycyqpnJJId9DuUhvZZcmzKxKKpUU3NBsZtZepZIC\ndNrQLF+SamaVUqmk0JgAXDVkZjZepZJCM65RMjPbrHJJIVpMNyNcmjCzaqlUUvBYRmZm7VUqKcCW\nPZTVtqXZN9kxs2qpXFLIc9WQmdl41UoKrj0yM2urWkmBzobCdkOzmVVNpZJCY0HB7QVmZuNVKikA\nWxQVJurRbGZWJdVLCnkuKJiZjVOppNDsl78LA2Zmm1UqKUCHDc2+85qZVUylksKWDc1mZpZXqaQA\nTXo0T3DnNScOM6uSyiWFPFcNmZmNV6mk0KxR2Q3NZmabVSopQKcNzXKPZjOrlFKTgqQjJK2SNCbp\njCbPz5V0vaRbJN0m6ahS42mY9/nezGy80pKCpGnABcCRwELgOEkLG1b7NPDtiNgHOBb4Slnx1Gxx\nj+aJ1nfqMLMKKbOksB8wFhH3RsTTwOXAooZ1ApiepmcAD5YYzxYmqhpyc4OZVc02JW57D+CB3Pwa\n4HUN65wFLJP0MWBH4C0lxtOiR7NP/WZmNWWWFJqdbRt/mx8HXBwRs4GjgG9I2iImSSdJGpU0um7d\nuq0KqpPqoKxH81btzsxsoJSZFNYAc3Lzs9myeuiDwLcBIuJnwA7AzMYNRcTiiBiJiJFZs2ZNOiA3\nNJuZtVdmUlgOLJA0X9J2ZA3JSxrW+b/AoQCSXk6WFLauKDCBzhuazcyqo7SkEBGbgJOBa4G7yK4y\nukPS2ZKOTqv9JfAXkm4FvgWcGFOqm7HbG8ysWspsaCYilgJLG5admZu+EzigzBjGaTjHR4TP+2Zm\nOdXr0dxBOcQNzWZWNZVKCo0jovp8b2Y2XqWSQjOuPTIz26zySaGdLGG4PGFm1TFhQ7OkWcAnycYv\n2qG2PCIOKTGuUmzReTnco9nMLK9ISeEysktK5wOfBVaT9UEYSJ1e8eqGZjOrkiJJYbeI+DrwTET8\nKCI+AOxfclyl2LKg0P6M70KEmVVNkX4Kz6S/ayX9D7KhKmaXF1Jv+bxvZrZZkaTwd5JmkPU+Pp9s\nqOtPlBpViTq68xq+85qZVUuRpLA+IjYAG4CDAST1rhdyFzVWB0W4isjMLK9Im8L5BZcNhE5/+fvO\na2ZWJS1LCpJeD7wBmCXptNxT04FpZQdWhsYezROu71KEmVVMu+qj7YCd0jo755ZvBN5dZlC9ko2H\n5zO/mVlNy6QQET8CfiTp4oi4v4cxlaqjO6/hfgpmVi1FGpofl/QF4BUMWY/mIFxFZGaWU7RH890M\nTY/mDtcvJwwzsympUj2aO+VxkcysairVo7lZPwUzM9tssj2aTy01qhJ1mgecOMysSiZMChHx/TRZ\n79E8uLa885qriMzMNmvXee182vywjohTSomoZO7RbGbWWruG5lFgBdllqPsC96THa4Bnyw+t/1yI\nMLOqadd57RIASScCB0fEM2n+QmBZT6LrsqYD4vUnFDOzKanIJakvYfwwFzulZQPKHRXMzFopcvXR\n54FbJF2f5t8MnFVaRCXaslTQvkezq4/MrGqKXH30L5KuAV6XFp0REb8pN6zyuEezmVlrRUoKpCTw\nvZJjmXI8gqqZVU2RNoWh4TuvmZm1V6mkAJPp0ewKJDOrjgmrjyTt2mTxY7VLVAdJs+qgdlVELkWY\nWdUUKSncDKwDfknWeW0dcJ+kmyW9tszgypD/5V+kDOBygplVSZGk8J/AURExMyJ2A44Evg18BPhK\nuxdKOkLSKkljks5osc57JN0p6Q5J3+z0DZTJBQUzq5oiSWEkIq6tzUTEMuBNEXEjsH2rF0maBlxA\nlkQWAsdJWtiwzgLgb4ADIuIVwCc6fwvFbdnQ7DuvmZnlFUkKj0r6pKSXpsdfA+vTSf+5Nq/bDxiL\niHsj4mngcmBRwzp/AVwQEesBIuLhSbyHjnjobDOz1ookhfeS3VTnKrK+CnPTsmnAe9q8bg/ggdz8\nmrQsby9gL0k/lXSjpCOabUjSSZJGJY2uW7euQMjNNSsUtCsoeFhtM6uaIj2aHwE+1uLpsTYvbXZG\nbfzdvQ2wADiILPH8RNIrI+L/NcSwGFgMMDIyslW/3fO//N3QbGY2XpFLUvcCTgfm5dePiEMmeOka\nYE5ufjbZrTwb17kxXd56n6RVZEli+YSRm5lZ1xUZ5uI7wIXAP9PZfRSWAwskzQd+DRxLVu2UdxVw\nHHCxpJlk1Un3drCPjjRWB0V267XW65cViJnZFFUkKWyKiK92uuGI2CTpZOBasvaHiyLiDklnA6MR\nsSQ9d5ikO8kSzl9FxG873VeHcZW6vpnZICuSFK6W9BHgSuCp2sKIeHSiF0bEUmBpw7Izc9MBnJYe\nfdG2NOCigplVTJGkcEL6+1e5ZQHs2f1wyhctpousb2Y27IpcfTS/F4GYmVn/tUwKkg6JiB9Ielez\n5yPiu+WFVY5mbcpt77xWXihmZlNSu5LCm4EfAG9v8lwAA5cUgHH1QYUakV1/ZGYV0jIpRMRn0t/3\n9y6ccnV6JzX3aDazqinSeW174Bi27Lx2dnlhlafxh/9Ep/1wUcHMKqTI1UffAzYAK8hdkmpmZsOn\nSFKYHRFNB6obNM0bmt2j2cyspsgoqTdI+sPSI+mRcXdeK9LO7NojM6uQIiWFA4ETJd1HVn0kss7I\nryo1sinA7cxmVjVFksKRpUfRI6KxR3NM3NDskoKZVUi7zmvTI2Ij8FgP4zEzsz5qV1L4JvA2squO\ngvHtrgM59lHnPZpdf2Rm1dKu89rb0t+hGvvoyWee5cP/ugKAu9c+xp6zdmy7/i0PrK+vb9bOLjtu\nx2ePfgXbTity/YbZ1FSkTQFJu5DdEW2H2rKI+HFZQZVl/z134yf3PMKv1v0OgN122o6D//uLWq7/\n1oW7s+zO39TXN2tl4xOb+M3GJ/nzA+ez56yd+h2O2aQV6dH858DHyW6nuRLYH/gZMNHtOKecQ1++\nO4e+fPfC659++N6cfvjeJUZkw+J7K3/Nxy9f2e8wzLZakXLux4E/Au6PiIOBfYB1pUZlNqB8sZoN\nuiJJ4cmIeBKycZAi4m7AP5/NzIZQkTaFNZJeCFwFXCdpPfBguWGZmVk/FLnz2jvT5FmSrgdmAP9Z\nalRmA8qdHW3QtU0Kkp4H3BYRrwSIiB/1JCqzAeN7b9iwaNumEBHPAbdKmtujeMwGnIsKNtiKtCm8\nGLhD0s+B39cWRsTRpUVlZmZ9USQpfLb0KMwGnCuPbFgUSQpHRcQn8wsknQO4fcGsgRuabdAV6afw\n1ibLhmY4bbNucDuzDYt2Q2d/GPgIsKek23JP7Qz8tOzAzAaRCwo26CYaOvsa4B+AM3LLH4uIR0uN\nyszM+qLd0NkbgA3Acb0Lx2ww+d4bNiw88LtZF7mh2QZdqUlB0hGSVkkak3RGm/XeLSkkjZQZj1lZ\n3NBsw6K0pCBpGnAB2ZVKC4HjJC1sst7OwCnATWXFYtYr4aZmG3ATJgVJ75J0j6QNkjZKekzSxgLb\n3g8Yi4h7I+Jp4HJgUZP1PgecCzzZUeRmZtZ1RUoK5wJHR8SMiJgeETtHxPQCr9sDeCA3vyYtq5O0\nDzAnIr5fOGKzKci1RzYsiiSFhyLirklsu9n/Sb1snUZg/RLwlxNuSDpJ0qik0XXrfNM3m7rc0GyD\nrsgwF6OSriC7yc5TtYUR8d0JXrcGmJObn834m/PsDLwS+GEadvi/AUskHR0Ro/kNRcRiYDHAyMiI\n/+1synFDsw2LIklhOvA4cFhuWQATJYXlwAJJ84FfA8cC761vIOsHMbM2L+mHwOmNCcFskLikYIOu\nyJ3X3j+ZDUfEJkknA9cC04CLIuIOSWcDoxGxZDLbNTOz8rQb++ivI+JcSefTZEiXiDhloo1HxFJg\nacOyM1use9CE0ZpNWa4/suHQrqRQa1x2dY5ZQe6nYIOu3dhHV6e/l/QuHLPB5IZmGxYe+8isi9zQ\nbIPOScHMzOraJgVJ0ySd2qtgzAaVa49sWLRNChHxLM3HKzIzsyFUpPPaTyV9GbgC+H1tYUTcXFpU\nZmbWF0WSwhvS37NzywI4pPvhmA2mNFSLG5pt4BXp0XxwLwIxM7P+K3I/hRmS/qk2Sqmkf5Q0oxfB\nmQ0KNzTbsChySepFwGPAe9JjI/AvZQZlNqjco9kGXZE2hZdFxDG5+c9KWllWQGZm1j9FSgpPSDqw\nNiPpAOCJ8kIyGzy1YS7c0GyDrkhJ4UPApbl2hPXACeWFZGZm/dI2KaRbZu4dEa+WNB0gIjb2JDKz\nAeIB8WxYTNSj+Tng5DS90QnBrD3XHtmgK9KmcJ2k0yXNkbRr7VF6ZGZm1nNF2hQ+kP5+NLcsgD27\nH47ZYBK1Hs0uK9hgK9KmcHxE/LRH8ZiZWR8VaVP4Yo9iMRtcbmi2IVGkTWGZpGMkX19hNhFXHtmg\nK9KmcBqwI/CspCfIfhNFREwvNTIzM+u5IqOk7tyLQMwGWa0Y7XZmG3RFRkmVpOMl/W2anyNpv/JD\nMzOzXivSpvAV4PXAe9P874ALSovIbAC5yc2GRZE2hddFxL6SbgGIiPWStis5LrMB5fojG2xFSgrP\nSJpG+rZLmgU8V2pUZmbWF0WSwnnAlcCLJP0v4P8Af19qVGYDxg3NNiyKXH10maQVwKFk3/13RMRd\npUdmZmY9V6RNgYi4G7i75FjMBpbbmW1YFKk+MrOCXHtkg67UpCDpCEmrJI1JOqPJ86dJulPSbZL+\nS9JLy4zHzMzaKy0ppCuWLgCOBBYCx0la2LDaLcBIRLwK+Dfg3LLiMSvT5qGz+xyI2VYqs6SwHzAW\nEfdGxNPA5cCi/AoRcX1EPJ5mbwRmlxiPmZlNoMyksAfwQG5+TVrWygeBa5o9IekkSaOSRtetW9fF\nEM26ww3NNizKTArN/k2aFq4lHQ+MAF9o9nxELI6IkYgYmTVrVhdDNOsu33nNBl2hS1InaQ0wJzc/\nG3iwcSVJbwE+Bbw5Ip4qMR4zM5tAmSWF5cACSfPTWEnHAkvyK0jaB/jfwNER8XCJsZiVqt6jua9R\nmG290pJCRGwCTgauBe4Cvh0Rd0g6W9LRabUvADsB35G0UtKSFpszM7MeKLP6iIhYCixtWHZmbvot\nZe7frGfc0GxDwj2azbrI7cw26JwUzMyszknBrAvqPZrd1GwDzknBzMzqnBTMusA9mm1YOCmYdZNr\nj2zAOSmYmVmdk4JZF7j2yIaFk4JZF7n2yAadk4JZF8gtzTYknBTMusg9mm3QOSmYmVmdk4JZF7j2\nyIaFk4JZF3mYCxt0TgpmXeCCgg0LJwWzLnJDsw06JwUzM6tzUjDrAjc027BwUjDrItce2aBzUjDr\nChcVbDg4KZh1Ubil2Qack4KZmdU5KZh1gRuabVg4KZh1kSuPbNA5KZh1gQsKNiycFMy6yUUFG3BO\nCmZmVuekYNYFvvOaDQsnBbMu8tDZNuicFMy6wOUEGxalJgVJR0haJWlM0hlNnt9e0hXp+ZskzSsz\nHrOyuUOzDbrSkoKkacAFwJHAQuA4SQsbVvsgsD4i/gD4EnBOWfGYmdnEyiwp7AeMRcS9EfE0cDmw\nqGGdRcAlafrfgEPlFjsbQP7W2rDYpsRt7wE8kJtfA7yu1ToRsUnSBmA34JES4zIrzd9edTufv+bu\nfodhQ+qUQxfw9le/pNR9lJkUmv12aqxxLbIOkk4CTgKYO3fu1kdm1mV77b4zx/7RHDY++Uy/Q7Eh\nNuP525a+jzKTwhpgTm5+NvBgi3XWSNoGmAE82rihiFgMLAYYGRlxU55NOTtsO43PH/OqfodhttXK\nbFNYDiyQNF/SdsCxwJKGdZYAJ6TpdwM/CA9Ib2bWN6WVFFIbwcnAtcA04KKIuEPS2cBoRCwBvg58\nQ9IYWQnh2LLiMTOziZVZfURELAWWNiw7Mzf9JPDHZcZgZmbFuUezmZnVOSmYmVmdk4KZmdU5KZiZ\nWZ2TgpmZ1WnQugVIWgfcP8mXz2RqDqHhuDozVeOCqRub4+rMMMb10oiYNdFKA5cUtoak0YgY6Xcc\njRxXZ6ZqXDB1Y3NcnalyXK4+MjOzOicFMzOrq1pSWNzvAFpwXJ2ZqnHB1I3NcXWmsnFVqk3BzMza\nq1pJwczM2qhMUpB0hKRVksYkndHjfc+RdL2kuyTdIenjaflZkn4taWV6HJV7zd+kWFdJOrzE2FZL\n+kXa/2hatquk6yTdk/7ukpZL0nkprtsk7VtSTHvnjslKSRslfaIfx0vSRZIelnR7blnHx0fSCWn9\neySd0GxfXYjrC5LuTvu+UtIL0/J5kp7IHbcLc695bfr8x1LsW3Vj0RZxdfy5dfv/tUVcV+RiWi1p\nZVrey+PV6tzQv+9YRAz9g2zo7l8BewLbAbcCC3u4/xcD+6bpnYFfAguBs4DTm6y/MMW4PTA/xT6t\npNhWAzMblp0LnJGmzwDOSdNHAdeQ3TFvf+CmHn12vwFe2o/jBbwJ2Be4fbLHB9gVuDf93SVN71JC\nXIcB26Tpc3Jxzcuv17CdnwOvTzFfAxxZQlwdfW5l/L82i6vh+X8EzuzD8Wp1bujbd6wqJYX9gLGI\nuDcingYuBxb1aucRsTYibk7TjwF3kd2fupVFwOUR8VRE3AeMkb2HXlkEXJKmLwHekVt+aWRuBF4o\n6cUlx3Io8KuIaNdhsbTjFRE/Zsu7AXZ6fA4HrouIRyNiPXAdcES344qIZRGxKc3eSHa3w5ZSbNMj\n4meRnVkuzb2XrsXVRqvPrev/r+3iSr/23wN8q902Sjperc4NffuOVSUp7AE8kJtfQ/uTcmkkzQP2\nAW5Ki05OxcCLakVEehtvAMskrVB2L2yA3SNiLWRfWuBFfYir5ljG/7P2+3hB58enH8ftA2S/KGvm\nS7pF0o8kvTEt2yPF0ou4Ovncen283gg8FBH35Jb1/Hg1nBv69h2rSlJoVu/X88uuJO0E/DvwiYjY\nCHwVeBnwGmAtWREWehvvARGxL3Ak8FFJb2qzbk+Po7LbuB4NfCctmgrHq51WcfT6uH0K2ARclhat\nBeZGxD7AacA3JU3vYVydfm69/jyPY/wPj54frybnhpartoiha7FVJSmsAebk5mcDD/YyAEnbkn3o\nl0XEdwEi4qGIeDYingO+xuYqj57FGxEPpr8PA1emGB6qVQulvw/3Oq7kSODmiHgoxdj345V0enx6\nFl9qYHwb8KepioNUPfPbNL2CrL5+rxRXvoqplLgm8bn18nhtA7wLuCIXb0+PV7NzA338jlUlKSwH\nFkian359Hgss6dXOU53l14G7IuKfcsvz9fHvBGpXRiwBjpW0vaT5wAKyBq5ux7WjpJ1r02QNlben\n/deuXjgB+F4urvelKyD2BzbUirglGfcLrt/HK6fT43MtcJikXVLVyWFpWVdJOgL4JHB0RDyeWz5L\n0rQ0vSfZ8bk3xfaYpP3Td/R9uffSzbg6/dx6+f/6FuDuiKhXC/XyeLU6N9DP79jWtJwP0oOs1f6X\nZFn/Uz3e94FkRbnbgJXpcRTwDeAXafkS4MW513wqxbqKrbzCoU1ce5Jd2XErcEftuAC7Af8F3JP+\n7pqWC7ggxfULYKTEY/YC4LfAjNyynh8vsqS0FniG7NfYBydzfMjq+MfS4/0lxTVGVq9c+45dmNY9\nJn2+twI3A2/PbWeE7CT9K+DLpA6tXY6r48+t2/+vzeJKyy8GPtSwbi+PV6tzQ9++Y+7RbGZmdVWp\nPjIzswKcFMzMrM5JwczM6pwUzMyszknBzMzqnBRsaEn6oaTS77Mr6ZQ0yuVlDctHJJ2Xpg+S9IYu\n7nOepPc225fZ1tim3wGYTUWStonNg8tN5CNk19jfl18YEaPAaJo9CPgdcEOXYpgHvBf4ZpN9mU2a\nSwrWV+kX712SvqZsPPllkp6fnqv/0pc0U9LqNH2ipKskXS3pPkknSzotDWB2o6Rdc7s4XtINkm6X\ntF96/Y5pYLbl6TWLctv9jqSrgWVNYj0tbed2SZ9Iyy4k6wS4RNKpDesfJOn7aaCzDwGnKhuf/42p\n1+y/pxiWSzogveYsSYslLQMuTcfnJ5JuTo9aaePzwBvT9k6t7SttY9d0fG5Lx+NVuW1flI7rvZJO\nyR2P/5B0a3pvf7J1n6oNtG71/PTDj8k8yH7xbgJek+a/DRyfpn9I6rEJzARWp+kTyXpt7gzMAjaQ\neqUCXyIbVKz2+q+l6TeRxsgH/j63jxeS9ZzdMW13Dan3aEOcryXrQbojsBNZj9d90nOrabgnRVp+\nEPD9NH0WuXsKkP3CPzBNzyUb5qC23grg+Wn+BcAOaXoBMNq47Sb7Oh/4TJo+BFiZ2/YNZPcvmEnW\nY3xbsh68X8tta0bje/GjOg9XH9lUcF9ErEzTK8gSxUSuj2z8+cckbQCuTst/Abwqt963IBtPX9J0\nZXcjOww4WtLpaZ0dyE7MkMakb7K/A4ErI+L3AJK+Szbk8i1F3mATbwEWavONu6YrjUMFLImIJ9L0\ntsCXJb0GeJZsYLaJHEh2oicifiBpN0kz0nP/ERFPAU9JehjYneyYfVHSOWSJ5SeTfE82BJwUbCp4\nKjf9LPD8NL2JzVWcO7R5zXO5+ecY/71uHMelNszwMRGxKv+EpNcBv28R41bddrGJ5wGvz538azHQ\nEMOpwEPAq9Nrniyw7XbDKDce620i4peSXks25s4/SFoWEWcXehc2dNymYFPZarJqG4B3T3IbfwIg\n6UCyESU3kI0e+bE0QiWS9imwnR8D75D0AmUjyr4T6OQX9WNk1V01y4CTazOpJNDMDGBtZMNO/xnZ\nrSqbba8x1j9N2z0IeCTajNEv6SXA4xHxr8AXyW5baRXlpGBT2ReBD0u6gawOfDLWp9dfSDZiJ8Dn\nyKplblN2I/fPTbSRyG6ZeDHZ0M43Af8cEZ1UHV0NvLPW0AycAoykxuA7yRqim/kKcIKkG8mqjmql\niNuATalx+NSG15xV2zZZg/REN3H/Q+Dnym5c/yng7zp4XzZkPEqqmZnVuaRgZmZ1TgpmZlbnpGBm\nZnVOCmZmVuekYGZmdU4KZmZW56RgZmZ1TgpmZlb3/wE4ORS5A61ciAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(error)\n",
    "plt.xlabel('number of iterations')\n",
    "plt.ylabel('error in training data')\n",
    "\n",
    "plt.figure()\n",
    "plt.plot(error_of_rounded)\n",
    "plt.xlabel('number of iterations')\n",
    "plt.ylabel('error in training data')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Reusing deep neural networks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "import xfer\n",
    "# You can install it with:\n",
    "#    $pip install xfer-ml\n",
    "\n",
    "# Or clone and install from souce:\n",
    "#    https://github.com/amzn/xfer.git"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['squeezenet_v1.1-0000.params', 'squeezenet_v1.1-symbol.json']"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "path = 'http://data.mxnet.io/models/imagenet/'\n",
    "[mx.test_utils.download(path+'squeezenet/squeezenet_v1.1-0000.params'),\n",
    "mx.test_utils.download(path+'squeezenet/squeezenet_v1.1-symbol.json')]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Just utility functions to handle data."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json, os\n",
    "def get_iterators_from_folder(data_dir, train_size=0.6, batchsize=10, label_name='softmax_label', data_name='data', random_state=1):\n",
    "    import os,glob\n",
    "    from sklearn.model_selection import StratifiedShuffleSplit\n",
    "    \"\"\"\n",
    "    Method to create iterators from data stored in a folder with the following structure:\n",
    "    /data_dir\n",
    "        /class1\n",
    "            class1_img1\n",
    "            class1_img2\n",
    "            ...\n",
    "            class1_imgN\n",
    "        /class2\n",
    "            class2_img1\n",
    "            class2_img2\n",
    "            ...\n",
    "            class2_imgN\n",
    "        ...\n",
    "        /classN\n",
    "    \"\"\"\n",
    "    # assert dir exists\n",
    "    if not os.path.isdir(data_dir):\n",
    "        raise ValueError('Directory not found: {}'.format(data_dir))\n",
    "    # get class names\n",
    "    classes = [x.split('/')[-1] for x in glob.glob(data_dir+'/*')]\n",
    "    classes.sort()\n",
    "    fnames = []\n",
    "    labels = []\n",
    "    for c in classes:\n",
    "            # get all the image filenames and labels\n",
    "            images = glob.glob(data_dir+'/'+c+'/*')\n",
    "            images.sort()\n",
    "            fnames += images\n",
    "            labels += [c]*len(images)\n",
    "    # create label2id mapping\n",
    "    id2label = dict(enumerate(set(labels)))\n",
    "    label2id = dict((v,k) for k, v in id2label.items())\n",
    "\n",
    "    # get indices of train and test\n",
    "    sss = StratifiedShuffleSplit(n_splits=2, test_size=None, train_size=train_size, random_state=random_state)\n",
    "    train_indices, test_indices = next(sss.split(labels, labels))\n",
    "\n",
    "    train_img_list = []\n",
    "    test_img_list = []\n",
    "    train_labels = []\n",
    "    test_labels = []\n",
    "    # create imglist for training and test\n",
    "    for idx in train_indices:\n",
    "        train_img_list.append([label2id[labels[idx]], fnames[idx]])\n",
    "        train_labels.append(label2id[labels[idx]])\n",
    "    for idx in test_indices:\n",
    "        test_img_list.append([label2id[labels[idx]], fnames[idx]])\n",
    "        test_labels.append(label2id[labels[idx]])\n",
    "\n",
    "    # make iterators\n",
    "    train_iterator = mx.image.ImageIter(batchsize, (3,224,224), imglist=train_img_list, label_name=label_name, data_name=data_name,\n",
    "                                        path_root='')\n",
    "    test_iterator = mx.image.ImageIter(batchsize, (3,224,224), imglist=test_img_list, label_name=label_name, data_name=data_name,\n",
    "                                      path_root='')\n",
    "\n",
    "    return train_iterator, test_iterator, train_labels, test_labels, id2label, label2id\n",
    "\n",
    "\n",
    "def get_images(iterator):\n",
    "    \"\"\"\n",
    "    Returns list of image arrays from iterator\n",
    "    \"\"\"\n",
    "    iterator.reset()\n",
    "    images = []\n",
    "    while True:\n",
    "        try:\n",
    "            batch = iterator.next().data[0]\n",
    "            for n in range(batch.shape[0]):\n",
    "                images.append(batch[n])\n",
    "        except StopIteration:\n",
    "            break\n",
    "    return images\n",
    "\n",
    "\n",
    "def show_predictions(predictions, images, id2label, uncertainty=None, figsize=(9,1.2), fontsize=12, n=8):\n",
    "    \"\"\"\n",
    "    Plots images with predictions as labels. If uncertainty is given then this is plotted below as a\n",
    "    series of horizontalbar charts.\n",
    "    \"\"\"\n",
    "    num_rows = 1 if uncertainty is None else 2\n",
    "\n",
    "    plt.figure(figsize=figsize)\n",
    "    for cc in range(n):\n",
    "        plt.subplot(num_rows,n,1+cc)\n",
    "        plt.tick_params(\n",
    "                        axis='both',          # changes apply to the x-axis\n",
    "                        which='both',      # both major and minor ticks are affected\n",
    "                        bottom=False,      # ticks along the bottom edge are off\n",
    "                        top=False,         # ticks along the top edge are off\n",
    "                        left=False,\n",
    "                        labelleft=False,\n",
    "                        labelbottom=False) # labels along the bottom edge are off\n",
    "        plt.imshow(np.uint8(images[cc].asnumpy().transpose((1,2,0))))\n",
    "        plt.title(id2label[predictions[cc]].split(',')[0], fontsize=fontsize)\n",
    "        plt.axis\n",
    "        \n",
    "if not os.path.isfile('imagenet1000-class-to-human.json'):\n",
    "    import urllib.request\n",
    "    urllib.request.urlretrieve('https://raw.githubusercontent.com/amzn/xfer/master/docs/demos/imagenet1000-class-to-human.json', 'imagenet1000-class-to-human.json')\n",
    "\n",
    "# This utility just allows us to translate image-id's of the imagenet dataset to human-readable labels\n",
    "with open('imagenet1000-class-to-human.json', 'r') as fp:\n",
    "    imagenet_class_to_human = json.load(fp)\n",
    "\n",
    "imagenet_class_to_human = {int(k): v for k, v in imagenet_class_to_human.items()}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load train and test data  \n",
    "if not os.path.isdir('test_sketches'):\n",
    "    import urllib.request, zipfile\n",
    "    urllib.request.urlretrieve('http://adamian.github.io/talks/test_sketches.zip', 'test_sketches.zip')\n",
    "    zip_ref = zipfile.ZipFile('test_sketches.zip', 'r')\n",
    "    zip_ref.extractall('.')\n",
    "    zip_ref.close()\n",
    "train_iterator,test_iterator, train_labels, test_labels, id2label, label2id = get_iterators_from_folder('test_sketches', 0.6, 4, label_name='prob_label', random_state=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Download a small pre-trained neural network."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "source_model = mx.module.Module.load('squeezenet_v1.1', 0, label_names=['prob_label'])\n",
    "mh = xfer.model_handler.ModelHandler(source_model) # Attach the source model to xfer"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's try to predict with the pre-trained network."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA1IAAABzCAYAAACSPjMJAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4zLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvIxREBQAAIABJREFUeJzt3XmYFNW9//H3dxYQQWUREQQEQUFJ\nUARXXACN4hL3DfnhrlcgEQ2uV1FjEDdQMNHoRQ3RKKgoXpWgIAIKN4LgQiICKiLIIoIgOzJQvz9q\nobun15lepz+v55lnqruqq07X6ao63zqnzjHHcRAREREREZHkleQ6ASIiIiIiIoVGgZSIiIiIiEiK\nFEiJiIiIiIikSIGUiIiIiIhIihRIiYiIiIiIpEiBlIiIiIiISIryMpAys25m9n2WtznKzAZXcx33\nmtk/0pWmQmFmi83s5DSvc6qZXZPOdUpqzOwpMxtUxc86ZtY23WkqRvH2pZn1NrOJVVxv2o/bYqe8\nKgyh12oza+XlW1mK67jCzKZXMx1ZL+sUknSUy4pRus8XZrbRzA5I1/qqK9Fxk+3yR14GUiKSe47j\nXO84zp9AF/x85TjOi47jnJJoORVIck95JcUk1RvL6QhMJTMcx6nnOM6iTK2/0CshFEiJiIiIiEgg\n1VramibZ71+tQMrMbjezb8xsg5nNM7NzQ+ZdYWbTzWyoma01s2/N7LSQ+Vea2ZfeZxeZ2X/F2c7B\nXlOvdWb2hZmdFTKvkZm9ZWbrzexjMxsc766GmR1nZv/nrWupmV0RY7lrzexrM/vJzN40s2Yh8zqY\n2SRv3g9m9t9RPl9uZqPN7DUzqxV3R9YMh5nZXDP72cxeNrPdIOF+PNbLs5+9/8dGW7GZNfXWfXO2\nvkxNYWa3mdky7zhbYGZnmNkWM9vbm3+XmVWY2Z7e68FmNtybHuW9rgtMAJp5VfwbzayZmZWa2X+H\nnAPmmFmLkM2fbGZfecf/E2ZmWd8Becw7B74V8vprM3sl5PVSMzvMexl1X4bexTXXY2a2yjum5prZ\nr8zsOqA3cKuXd28RwcxKQs7na8zsFTNrmNEdUECUV/khlXwwsxHe6/Xeuen4Km6zhZm9bmY/evv7\nLzGWi3k9M7OGZvY3M1vu/S7eiLGOG8wtSzWvSlpzJdp1Bvhv4GLvd/y5t9xeZvasma3wlh/sXUcO\nBp4CjvGWXxey+gZmNt5b90wzaxOy3Xj7/Apzy5YbzC1/9g6Zd63tKn/OM7PDvffjlTVHmdvcfZL3\nuWlmtn/I/Pa2q1y4wMwuysCuTkXUMhkkLJc5ZtbfzL4Cvgp5r6251/2NIX+bzczxlikxtzzxnXde\ne97M9vLm+c1nLzezJWa22szu9Ob1JPpv5UpLMkaI4XTvc6vN7BEzK/HW28bM3veO5dVm9qKZ1Q/5\n/ou93/NcYJMlE0w5jlPlP+BCoBluQHYxsAlo6s27AtgOXAuUAn2B5YB5888A2gAGnAhsBg735nUD\nvvemy4GvvR1dC+gBbADaefPHeH+7A4cAS4HpMdLb0vtsL2+9jYDDvHmjgMHedA9gNXA4UBv4M/CB\nN28PYAUwENjNe32UN+9e4B9AHWC8t87S6uzjQvgDFgOzvN9CQ+BL4PoE+7EhsBboA5R5ebIWaOTN\nnwpcA7QCFgLX5fp7Ftof0M47Hpp5r1t5x9wHwPneexOBb4DTvNcfAOd606HHRHBMhqz/FuDf3nYM\nODQk/xzgbaC+d9z9CPTM9T7Jpz/gAGAd7vmzKfAdsCxk3lpvXsx9iXuene5NnwrM8ZYz4GB2nY+D\nvAzZ/mLgZG/6RuAjoLl3rD4NjM71PsqXP+VVfvwlmw/e6/+He40vw71erwR28+bdC/zDm27l5VtZ\nlO2VAp8DjwF1ca/5x0XJz0TXs/HAy0AD3LLHid773dhV1hkEfAI0zvV+TjFPYl1ngn0csuwb3u+1\nLrAPbrnhvyL3Z8jyo4CfgCO9/foiMCbRPvfWv55d5cSmQAdv+kJgGXCEd+y1BfYncVlzlPf6BO+4\nGxGS/3W9fXCll5bDccs+HXKUJ4uJUibz5sUsl3nzHWCS97k6Ie+1jbKdF/HOPcBV3v47AKgHvA68\nEHGMjcQtHx8KbAMOjjweQ9adVIwQ4/s7wBTvO7TELUNe481rC/zG++6Nccs8wyP23WdAC//7J9zf\nac68z4CzQw6Kr0Pm7e59uX1jfPYNYEDkTgKOxz0BloQsO9rb8aW4wVq7kHmDiR1I3QGMizFvFLsK\njc8CD4fMq+dtpxXuwfppjHXcC7wJTAMexwsaa/qf98P7fyGvH8a9uxRvP/YBZkWs51/AFd70VOBR\nb929cv0dC/HPO2GsAk4GykPe/5P3+yzzjq0BwIO4hYQtwN7ecqHHRKUTF7DAP96jbNvBK3B4r18B\nbs/1Psm3P9yL7+HAJcD/4F782uNekN9MtC8JL8z1wL1gHB16vozMy5D3FrOrcP4lcFLIvKbesVqp\ncFmsf8qr/PhLJh9ifG4tcKg3fS/JBVLH4AbD0eaF5mfM65mXPzuBBlHW0Q23UP8oMB3YK9f7twr5\nEes6E+xj73UT3MJznZD3egFTIvdnyPxRwDMhr08H5iexz+viBtznE1EYBt7FK2tGvB+zrBmSljEh\n8+oBO3AL3BcDH0as72ngnhzlyWKilMm86ZjlMu+1A/SIWF+lQAq4DfdmkB9sTQb6hcxv5623LOQY\nax4yfxZwSbTfSozvFDVGiLGsQ8iNW6AfMDnGsucQUqb39t1Vqezv6jbtu8zMPvOqQdcBvwL2Dllk\npT/hOM5mb7Ke99nTzOwjr2pxHe4BEvpZXzNgqeM4O0Pe+w7YDzeaLMM9sfpCpyO1wL37nkgzbxt+\n2jcCa7xtJlrH0UBH4EHHy5UisTJkejNuPsfbj2HzPH6++nrjXmTGZiC9NZ7jOF/j3r2+F1hlZmO8\nKvxpuCeiw3FrlCbh3vE5Gvfmx+okN5HoWIj2m5Bwfl6c4E1Pxc2LE73XvoT70nGc94G/AE8AP5jZ\n/5jXZDMJ+wPjQs7lX+IWEpqk8mVqOOVVfkgqH8xsoNc06GdvP+1F9DJGPC2A7xzHqUiwXLzrWQvg\nJ8dx1sb4bH3gOuABx3F+TjF9ORfnOhPJr/VZEfLbfRq3ZiqeWMdTzH3uOM4m3ODmem97482svbdM\nrOtWvLKmLyhfeuWZn7zP7Q8c5X8v77v1BvZN8N0yKan9FlEu88UrR2PuYzoDgHMcx9kSbb3edBnh\n56WkywQpxAixhH6H77z0YWb7eL/RZWa2HrcVWeR6437/SFUOpLy2oSOB3+FWX9cH/oNbDZfos7WB\n14ChQBPvs/+M8dnlQAu/faOnJW4B+0egAreJgy/0GY1IS3GrChNZjntg+Omti1tdvCyJdUwEHgAm\nm1mxXNhiibcfw+Z5/Hz13YtbBf2SmZVmNKU1lOM4LzmOcxzuvnaAh4D/w71bdC4wzXGcebj7/gzC\nC4Rhq4ryXrLHk8TmFwqP96anEb1wnhTHcR53HKcz0AE4CLf5JUTPv1BLcZt31g/5281xnGUJPldM\nlFf5IWE+mPs81G3ARbg1QfWBn0mifBJhKdAyieck4l3PlgINQ5/DiLAWOBP4m5l1TTF9eSHGdSby\nd7wUt0Zq75Df7Z6O43TwV5PiZuOWIRzHeddxnN/g1gjOxy2v+umIdt2KV9b0BeVLM6uH23RsubfO\naRHHZD3Hcfqm+J2yIV65zBczL8ysHfB34CLHcUIDjsj8aIlbPv8hiTSFbS/FGCGW0FigpZc+cMvn\nDtDRcZw9cZsAR643pd9idWqk6nob+xHcB8Nwa6SSUQu3feKPQIUX3cbqFnYm7rNXt5rbgUM34Le4\nVaw7cNth3mtmu3t3HC6Ls90XcR8EvsjMysztqOKwKMu9BFxpZod5GToEmOk4zmLc9u/7mtmNZlbb\nzPYws6NCP+w4zsPeOiab91B/kYq3H/8JHGRml3p5cTHuM25vh3x+O2575rrACxEnOEnAzNqZWQ9v\n32/Fbba3w6sdngP0Z1cB8P+A/yJ2gfAHoJH/8KjnGeBPZnaguTqaWaOMfJmaaxrQHbd5xPfAh0BP\n3Avbp6msyMyOMLOjzKwc95y5FbemAtz8izcOyFPA/d4NMsyssZmdndI3qfmUV/khmXzYA7cQ9yNQ\nZmZ3A8nW+IWahftM9INmVtfMdosR7MS8njmOswK3s54nzayBV445IfTDjuNMxa3BGBdZnsh3sa4z\nuL/jVv5129sPE4FhZranuZ0TtDGzE71V/QA0t+Q754q5z82siZmd5QUJ24CN7Dq+ngFuNrPO3nWr\nrXcsxSxrhmzzdHM7LKuF20R+phdMvO2lpY/32XLvGD84xd2ZDfHKZXGZW2v+v8BdjuNEduo2GrjJ\nzFp7QeYQ4OUkanMh4rdCajFCLLd4x1sL3Nqzl73398D9Pawzs/3YdQOryqpcMPXuYg/DbZP6A/Br\nYEaSn90A3IDbfnwtcCnus0XRlv0FOAs4Dbd24kngMsdx5nuL/A63yn4l8AJuZm6Lsa4luNWDA3Gr\nZD/DfegtcrnJuA9+voZ7Em2D2x7bT/tvcA+wlbi9mnSPso4/4bbpfM+KpEelSAn24xrcu3ADcauV\nbwXOjGxW5uX/ebjV/88pmEpJbdxnn1bj/lb3wX2QFtzCSDluQcF/vQfug5eVeMfbaGCRuU0XmuG2\n638F9+K4HrftdZ2MfJMaynGchbgn9Q+91+uBRcAM70ZRKvbEveu6FrcpwxrcO3rg5s0hXt5F6zFs\nBO45eKKZbcDtzKCgCnSZprzKD0nmw7u4wctC3P27lRSb63jr3oF7rW8LLAG+x20yFrlcoutZH9wb\ng/Nxnye6Mco6JuE952VmnVNNaw7Fus686s1fY2afeNOX4RaS5+H+9sfi1hgBvA98Aaw0s4TNyxPs\n8xLv/eW4Zb0TcZ+TwXGcV4H7cQOKDbjltIZJlDXxPnOPt87OuMGvXy48Bbd8s9zbDw95+yavxCuX\nJeFw3NYsj1pI733evOdwy+AfAN/iHnO/T3K9Yb+VVGKEOP4X94bxZ7idvTzrvf9H73v87L3/eorr\nrcTvQa/GMLOHcDu0uDzXaRERERGRwmZmo3A7OLgr12mR/FLwd/fN7bu/o1dFeyRwNTAu1+kSERER\nEZGaqyaMWrwHbpOjZrhV5sNwq/REREREREQyosY17RMREREREcm0gm/aJyIiIiIikm0KpERERERE\nRFKU0jNSe++9t9OqVasMJaU4LF68mNWrV6c6KGDKlFfVl428Uj6lx5w5c1Y7jtM4k9tQXqWH8qpw\nKK8Kg8oVhUN5VTiSzauUAqlWrVoxe/bsqqdK6NKlS1a2o7yqvmzklfIpPczsu0xvQ3mVHsqrwqG8\nKgwqVxSOYs4rMzcmKZS+GZLNKzXtExERERGRjPCDqMjpmqAmdH8uIiIiIiIFIDSYKpQaqlgUSImI\niIiISNpFa9IXrYaqUAMqNe0TEREREZG0itWMz3GcSoFToTb5UyAFVFRU8Pbbb/P73/+eTp06Ubt2\nbcws6l+TJk0YM2ZMrpMsIiIiIpKXkmm+FxlQ+WXtQpJ3TfvyfQeuWrWKXr160atXr4KthgyVzP6u\nCd9TRERECpfjOPTu3ZvRo0fHXe6NN97g7LPPzlKqJJpUn4Hyl/E/Z2YFU/bMqxqpI488MmfbvvDC\nC/n73//OvHnz2LBhQxAlR/4tXLgwZ2nMlXwPbiWzlP8iIpIrV111FWZGSUlJwiAK4JxzztF1K0+k\nGgzFeo4qn+VNjZS/w2rVqsW2bduC96+99lpGjhyZq2SFCc3U7t275zAl6Rfrx14oP2RJL+W7iIhk\n28KFC+nWrRsrVqyIOn/48OEMGDAg7jpCazU2btxI3bp1055Oia26nUeE1k4VQkcUeVEjFVpo84Mo\nfwc+88wzwfSWLVtykr7XX3+9UsHy/fffz0laRDJNQZRI9f3973+nrKws5vO2/t9uu+3Gd99lfNxb\nkbzw5Zdfcsstt1BSUhL1eGjXrl0QRJkZU6dODWsVlCiIArfQvX37dgDq1avHeeedl9HvJLuks/xQ\nKM9O5TSQGjp0aLBj5syZEzXiXLVqFU888QQAu+++e7W29/DDD4cFaF999RUVFRUxl58/fz5mxvnn\nnw/kd0Qskin63WdXv379Eha+dYc1f1RUVITd8PP/rrjiCnbs2BEs17lzZ3r27EnPnj056aSTgve3\nbdtGq1atgs89/vjjufgakgWJjut8LShW1VtvvUVpaWnY9zvkkEMYOnRopevKvvvuyx/+8AcqKiqC\noGnnzp2ceOKJVdp2WVlZsI1x48ZhZmzatKna30liC609Sle5oRA6o8hZILV69WpuueUWwG0md/jh\nh0ddbp999knL9oYPH85tt90WvL722ms56KCDeOutt6Iuv3DhQg4++ODgtQqTUqxq6kU+H5WVlfHX\nv/414XKbN2/OWH4899xzQX7PmzcvI9sodNOmTePQQw/FzCgvL+faa6+ttEyvXr1YsWJFUBCYPXs2\nEyZMYMKECbz33nvB+5988klYXg4YMKDaNw2lcNWU82xpaSlnnXUWO3fuDHu/Xbt2DB8+nI0bN4bV\nNK1YsYJhw4ZRWlqa1nSElt3q1auX1nVL9uRzV+k5CaQ+++wzGjduDMDYsWOjNpNzHIdBgwYB0L9/\nfwAaNGhQpe0NHjyYm266KViv4ziUlbmPhx1xxBGVlt9rr71o164dAL/88ktRBFGJCsrFdBet2IVe\n3CQ7Hn/8ccyMHTt20KxZs5id3fh///jHPwD3uPSbsKSDmXH11VcHrzt06KDjG/jpp59o3bp1cL7r\n1q0bc+fODeZfcMEFbNu2LSyPXnrpJfbdd9+E6+7UqRM7d+7EcRyWLl1KaWkpW7ZsCbY1ePDgTH41\nyZLIu/Wx/vxlC/W481v+7Ny5k5KSEmbMmBH2/ebPn8+AAQOyWqvuOA7Tp08HoHbt2lnbbrHI1rNM\n+Vo7lfVAatasWXTq1Alwd4rfbC6a++67L+wA/Omnn6q0zcmTJwfT/o73m/Q1b948mDdp0iTMjPXr\n19OyZUscx6G8vLxK26wpEp3wffnwY5b0U0CVeUOGDAna/W/YsIFly5Yl/Ezv3r2DfKlVqxaXXnpp\ntdNx4IEHBtOR+X7aaadVe/2F6Oeff8bMaNSoEYsXLwbcZ58iz4WvvvoqtWrVqvb2mjdvHjRtatmy\nJQCDBg0Krlt/+tOfqr0Nyb5UCpmF2GuZz8yClj+O47Bjxw6OPfbYHKfK1bVrV8rLy/nll194++23\nc52cGiEykMlWWSFWQJWr4yWrgVTv3r056qijgOw2lZsyZQoXXHBB2HsdOnQIS8O7777LKaecAsC8\nefOK8uHfVHvuUyG7OBRCrzmFav369dx5552Au39TbXri58no0aO5+eabq5yO1atX8/XXX1NeXh6W\nz/70O++8U+V1FyrHcahfvz4Ao0aNCs53l112WVa2/9133wXb7Nq1KwB33303ZsaVV16ZlTRI9VS1\ncJdPhcRktW7dOpjO12uF34nFOeeck+OUFLZoAVQu8jxfbupnNZB66aWXACp1a1lRUcHgwYNp2bJl\nUg9jhv6VlJRw3HHHMXDgQL788suY23711VfD7iD+5z//CZvfs2dPALZv3x48G/Xaa68VxAksHdJx\nEBTDfio2ytPM2muvvQC3CXFVPf300wAMGzas0vMIyfKbWkdLx8qVKwGCGpJi4Tc9+vrrr7n88stz\nmpbp06ezevVq9t57b8AN7AYOHJjTNEnyqlLQzNdgJBa/xraq56BsaNSoEUBYJzBSuGKVz7Ndbsla\nIPXhhx8G07fddltYMFReXs6gQYNYunRpyut1HIcZM2bw6KOPcsghh0QNtnbffXeuvPLKSsGT76OP\nPgLcDib8Z6eASrVYNU0yP7Zklim0E74klqsq+2LStGlTwG02V50mxNddd12QP6Wlpfzzn/9M6fP+\nOe/JJ5+MOr9JkybUrVuXpUuX0r59+yqns5D06dOHLVu2sOeee9KmTZtcJwdwC4E//vhjkNePPvqo\nbnTksXg1+ck+Y1woz6v++9//BtybDvpNFp9MPztflfVn83eYtUDqhBNOCKaff/55wG2TP2jQINas\nWZPwAcxk/+bMmcOAAQPYf//9g+1t2bKFUaNG8etf/zpqZpx++ulBenw1/WSgQrLEEvnb1+8jM/ya\nnlQDn1j8fDrjjDMwM+644464yw8bNizo4KJbt2707ds35rIbN24EYMGCBWlJaz4zs6Azj59//jnH\nqYnOcRwOOeQQoOZfqwpRsnlSU86t5557LkDe3HRIRqwemyWxyOA+3mMh6fhLlI54z+9nQ1ab9h1/\n/PFMmjQp+LILFy7kvvvuo2HDhmnbxuGHH87w4cNZvHhx1B08YcIEbrzxxrDMWbt2bdq2X2hqyolc\n0q8Q7oQWquuuuw6gymOkxOI4Dtdffz0ADz74YNyLk/9M1dixY5kyZUrCdfvNn0eMGJHWNOerfO+C\n/IsvvgiaYhZbs8t8lsxNytBza7xzbKEEyd98802uk5A0f3+fddZZur6lUboCmlQqTvJFVgOp9u3b\nc/LJJ2dzk5X07NmTxx57LOhuNtU7rKFN/wpVqifnRD/YQjnZF6NCeVC52IwcORJwewpNt7/+9a84\njkOvXr3iLtelSxccJ37PqaHuu+8+YNezrjWV//zE5s2b8/p5DyBoEpqu8RalaqKdZ/OpoCfhHn30\nUQBKSkro27cvX3/9dY5TVDNVpVVZOmWr3JPVQCofTywHHXRQSsv/7ne/y1BKciNRNWoqQVQ+5m8x\nU/CU39q3b5/R4RVeeumluBesjz/+OKX1+WPuzZo1KxPJzRslJSW8++67AGkfHDTSiBEjgvPu8uXL\nU/68P8bU7Nmz0500SVKy59mqPBSva2pm3HTTTfzrX/8C4KmnnuLAAw+MWha6+uqrgybYUjiyfdxk\nNZBatWpVNjeXEdFGsC808aL/VH6ACqJqNgVimTFjxgzAbZpVSPyx94rhd3HKKaewefNmIHPf18y4\n8cYbg9f77bdfSp93nF2D1ktuhHYoEfkX+rsphmOm0Bx99NFh+TVy5Migx2bfc889R9OmTcOCq9/8\n5jc5SrHkq6wGUlXplS9X2rVrF0yfd955wbT/gG+hFYKSUZ02rZJfqnPhVn5mll/4LSnJ+njo1TJ9\n+nQAunXrltuEZEmdOnWC6XR3l7x169ZgetGiRYwdOxYgCN6SceihhwK7nl2T/BMZaEVK9lyrQCzz\nrrnmGubNmxcWXD3zzDOVaqXfe+89zIwzzjgjRymVfJPVK/mnn34ad/5nn31WqWq1WbNmGU9XtJPU\n/PnzATjzzDN57bXXKs2vCSPMR6tR8t+L91xN6DwVuvNPZL5G5m0y8u1hzpokmY4d8lH37t0Bgh7t\nioF/DJSVlQU1cpHmzp1L3759U3rg3g/SHMehdevWwXNqfnCUDL/L6QkTJiT9GcmeVM6hCpTy09VX\nX01FRUVYcPXEE08Abm+rev5YIMuBVDxmRqdOnSq9v2LFioz/UG+99VYA1q1bF/a+4zgxu8h8+eWX\nM5qmTIvXLC9WW249TJv/EuVPZN5mcuwHqZmycXMrn/gBS7Tn2b744gsOPfRQnnrqKdq2bZvU8fPm\nm2/GnJfsQ+/+dmrCDb1ip3NuYenXr1+lIFl5WNzyIpDyH+w1s0rtjF988cVgXqbccMMNQOVAqhgk\n05lEtA4oFETln3hBVLSTfjI1jpIZhx12WK6TALhN1h599FE6duzIySefHPMc6AcR6e6uvRD86le/\nCqYvvPDCsHlXX301AJ9//nnQBNxvzRDNzp07Ofvss6uVHr+ZYceOHbnrrruqtS7JjshzaqxWA7E+\n6y+Xr/bYY49cJyEnHMcJhiDQNbN45UUgdf/99wPRB6a89NJLg+nbb789I9uvX78+EP8CGKpPnz4Z\nSUcupFIboQCqMMQbu8QXKzhW/mZH69atc50Ehg0bRllZGQMHDuTf//43kydPpkGDBtSqVStsuXHj\nxgXN2qZOnZqDlOaeH2D6zzL5/P1SXl4eNAEfPXp0zPWUlJSEHWO33XYbQFAYS4Y/2Pznn3+e9Gck\n/9Skgneuh7XJpfLy8qCTivXr1+c4NZILeRFIffjhh0Dsh2b98UuWLVuWke37Y7lcfvnlSS3//PPP\nAxRNj0kqYOe/ZO9aVrenRkmPdA5Cniq/RzF/QN5I27dvx8zYsGEDZhbUtBTz72SvvfZiw4YNQPRO\nQq644orgGEzmuuDvy4cffhgzo3bt2kBynVpk6jooVZfMMCKhx08y19RCahngDzBerObNmwe45wkp\nPlkLpPxR4qNF7Pvuuy9A0K9/pLvvvhuAF154ISNp8wddTLV7dn8Mj0KUaDA01VDUXGrfXbzKysqC\nQKBBgwaVjv8xY8YEy+65556A2wxR5wGoV68eJ554YhCIduvWjTlz5gC7xtZyHCfpQdsdx2HTpk1M\nmTKFJUuW4DhOwp4cC6GZl7jS9dxpIVyH/Zvg8Z7/S+Txxx/n1FNPpby8nJKSEk455RSeeuqpdCUx\n4379618DMHTo0BynREJlo3yTtUDq2GOPBeA///lPpXl+t+jHHnssjzzySNi86l447rjjjozcwevV\nqxcAd955Z9rXLZIKFa4KT6we4DLJzIIaj27duvHTTz9VWubiiy/GcRxWr14dvKfBXncJbdoYei3z\nu01O1e677063bt1o0aJFwmWPPvpoADp06JDydiSz0jU2o69Qz+mpPv/XpEmTINgcMGAAEydODHrJ\nmzRpEn379sXMaN68eZXSM2TIkKi1hWbGhRdeGNxET4e5c+cCcMstt6RtnVJ12Tx2shZI+c8ERBtL\nKvQO3q233lrpTk7btm1T2lbo5x988EEmTpxYjZRH53eBOWTIkLSvWyTbVCuVXYsXL87q9iLPu4m6\nYG/UqFFQYA8dU09gy5YtAKxZs4ZNmzbhOE6lgTwzYebMmUD0m5GSG1VpSp1IsZyLzSysFdCNN97I\n+++/z+bNm9m+fTuTJ08OBr9dtmwZBx10UErrX7duXdwb3WPHjqW0tJQTTjih4AJWyS9ZC6T8YOjb\nb7+NOt+/m9O4ceOw93v27MlGrchMAAAP5klEQVRXX32V1Da++uqrqCchv2elZCTb4USDBg2SXqdI\npqTjoluVcaakeqZNm5a1bW3dupWWLVsGr9esWZPU5/wCeyrjIxWD3XbbLThm6tatm5VtRrbUkPyR\nTOc+scR7tqrQCvevv/46kHig8RdeeCH4nn369AnKfo899hjdu3enTp06lJWV0aNHDyZOnIjjONSt\nWzdm+S4Wv4wW6zGGZ599FnCf0S8pKeGVV16pytcOc+CBBwLEHDZHaqasBVIHHHAAkPiivGrVqrAf\ne7KDDVZUVAR3LCKr2ZN5qNM3cuTIpLYnkk/SddFVMJV52b4J4w/+CrBy5cqcdnRRk4TegFi0aFFG\nt+WPdVhohetiFqtJWaygKdbzyoXi3HPPBajUNDiUmXHZZZcFy/kddyWycePGlNJyzz33AOG9Pke6\n6qqrcByH7du3A26z5uryA6ibbrqp2uuSwpG1QMrvYjxau/x08Mf6+P7776NuN1ZHFhA+BoLfg6BI\nvktn0FOIF+5C5d+5zcU+b9KkSda3WZOdeuqpALRp0yZj2/DHWRw+fHjGtiG5U5POvf44c5EtiyJl\n+jv7PT2PGDEi4bJlZWWccMIJadmu3wxatfjFJWuB1D777ANkruvWBQsWALDffvuFvf/ee+8BMGDA\ngJif7dixYzD98ccfZyB1IumVziYghdTNbk1w5JFHAgQ9vmVSaBPCBx54IOPbKzbvvPNOMJ2pZn5+\nj2jxrmGSP5KpXQp9vyade6dOnRq0Ior8Xueffz6Q3Waqe++9d1LLaUgBqY6sB1LROptIh9CTUqNG\njTj99NNp2LAhXbp0ASoHWKES3T0RyVdVDaJq0sW7kPTt2zcocPvnpky64oorgulUBzT3fyN+DZpE\n5zgOBx54IJs3b+a5555L67r79u0LELOplBSWQm22l4qePXsGnemE3qTzzyOxxq/LpXTUIPnj7sGu\nzsikOGQtkPKbz61duzZj2/Dbuv70009MmDAh2NbAgQMZN25cxrYrkk3p7ho31t1TBVvps23bNsws\nGBclW4WpqvYO6Lf1v+CCC4JnHyS2hQsXAok7NurXr1/C52V8EydODH4vjRo1Sn+iRTJk//33z9m1\n5P333wfgmGOOSWp5v9OJQw45JOVtff7557Rp0wYzC8bd22+//ejXr1/K65LCldzIgWmUzn77I5WV\nleE4DkuXLmXJkiW0bt2aZs2aZWx7ItmWrgtSvIK8P+CopM9uu+0WTBfCHemzzjoLgFdffTXHKSkc\nS5cupUWLFpSVlUUdJ+y8885L6Yae//zV+PHj05ZGkWxyHIfx48dz5plnBu8tX748o+Uy/3jp06dP\n3OWWL1/Ot99+yzXXXAO4HVM888wzSW/n2muvrfTeH//4R+6+++4UUis1QdYDqWxo0aJFUgMcihSq\nQiiMi1tD4Tf3mjdvXlbGGwrlF+qTLbisXbs26NUvsuMeia958+Z06tSJTz/9FDOrdIz6QdS2bduo\nVatW8H60mxahA/+efvrpmUmwSBacccYZYTfn4j1mkYxkb/L169cvpZqhu+66q6pJCtxzzz0KpIpQ\n1pr2SWEohjbchSqbo93XxAehs2nBggWYWRBEZWvQ1lDbt28PakaWL19O06ZN4y7/zTffBEHUrFmz\nql3gKUaffPJJMB2tB9jatWtHDaJGjx4dtlz37t0B3TCRmmPz5s0527aZ0bVrVx544AFmzZrF6tWr\nw46tefPmxRxvKtU/KT4KpEQKQDaDKKmehg0b0r59ewCefPLJnOTZ+PHjgwL7+vXrMTNWrlwZPI/z\n0UcfhS0/bty4YND0BQsWcMQRR2Q9zTWF33w9skvlZs2aBc/KPfjgg2E3KS655JJg2n8/k83gRbKt\nTp063HDDDcHrnj17Jh2clJaWAvCXv/wl4bKzZ88OtjFmzBimT5/Ohx9+yEMPPcTxxx/PEUccQaNG\njYLjbOPGjVm/ySU1S9YCqbIytxVhtLbjIpJ/FLSl7ogjjgg6udm6dWvQ61o27b///sEzCZMnT2aP\nPfZg586dDBo0KFjmmGOOCevo4Lzzzgvm+QObS9WYGUOHDgXg5ZdfDt4P7WL5jjvuCKZ37NgRTF90\n0UVh6xGpSUaMGBEMGfDOO+9QUlISdA4Rj3+M9O/fP+GynTt3DqYvueQSjjvuuLA/gJNPPhlwe5PO\n1LAFUjyyFkjVrl0bCL9oJCPVEa2r66STTsrq9kQSCS1QxevlS3LLzJg9e3bQ6Y1/zsuWAQMGYGYs\nWbIEcAPhHj16BPPvu+8+HMcJejcNVatWLbZs2aLgOU0GDhwIuAW50CE//LvmQ4YMYc6cOTiOQ0mJ\nexlet25d0LmH8kFqqlNPPRXHcdhnn31wHIeTTjop7vWsKq0xHMdh5cqV9OnTh549e4b9mRmTJ0+m\npKSEH374odrfRyQvO5uYOXMm27ZtY+zYsfz5z38Gsndh+e1vfxtz3qWXXsro0aNp2rQpy5cvz0p6\nRGL1opetYErNChPz99ENN9zAiBEjqr2+kpKSqPu7S5cuYYOGr1u3jgYNGoQt88svv1BeXh5z3X6g\nJ5m1adMm6tatS8uWLSvt79AaKZ+fj5kaa1Ekn4QGMXfeeWel+SNHjuS6664Ddg0vkIomTZrw/PPP\nh70Xes1M9aa+RKdrSZ4FUjNmzAiqXn1vvvkmZ511FhdddBGvvPJKxtPQoUOHSu89/fTTXH/99cHr\nFStWcOutt2Y8LSI+P5jKxUlLtV+xbdq0iXr16gHpuaCce+65vPHGGzHnz549O2Z+bN++PWhCLbm3\n++67B9MzZsyga9euMZf183THjh1BDZVIMZgwYQKlpaXcfffdOI7D3LlzefPNN4P5q1evTus4aiNG\njAh7VkuqJ1flknySV1ddP4jq3r170G7VryH64osvspKGyELKvHnzgiDq9ttvZ/DgwZSVlfHII49k\nJT0ikr/8IOqoo46q9rq+//77sCBq69atQfPAeMHskiVLNNxDnnrjjTc455xzOO6442IWNkKfG1YQ\nVbOp0FnZ+++/HzxTGOmdd95JaxClm02SCXnzi/JreB5++GFuueWWSvNzNbK7X0OVqLmMSL5QU7zs\nCN2/kb3gpeqjjz7imGOOCV5PmDAh7BmraN3Rjx07lvPPP79a25XMOvvss+nevTtTpkyhoqKiUiFu\ny5YtQc2VjteaS4Ocx3b22WfTqlUrwL2R0KJFi7ABfNNJQZRkQt7c/rrgggsAN6CaN29e8L4/Lkc6\nnjuojo8//phHHnkkOBnOnDkzp+kRiZRqJxT+sqEdWKgji+T5tQfTpk2r9rpCgyhwuwaOxn84G2Df\nffet9nYl8/xeyaLdiPODqK+++iqraZLc0Xk23HHHHUf//v3p378/ffv2zVgQJZIpeROeH3nkkZxy\nyilMnDgx6nNKnTp1ykGq4IADDmDRokWV2rf36tUrJ+kRCVXVi3FoECXVEzleUHUlGgh38uTJABx9\n9NFp3a5k15NPPhlM+2N4Sc2lWimRmilvaqQA3n33XbZu3Vrp/RUrVuQgNa5vvvmG9evXA+4YMStW\nrOD3v/89ixYtomHDhmqOIXmlur/HWJ/X7zy6YcOGVXsdt99+e9jr77//PqnP+YNUSv7zjx+/adHO\nnTuDMXF0bBWPWIPIikj6RWsSnwl5UyPlq127dt6dWPbYY48gTTfffHPQJfuaNWtymSwpcqHPQlX1\nIeZYJxo9FB3f3LlzAfjDH/5Q7XU99NBDwXRoLUU0fhNnPa9ZeJo1a8by5cvp1KkTn332GaAgSkSk\n0OVdIJVr9evXj/r+L7/8EvXhb5Fcq04zvWgBlCT21ltvZWS9ffv2jTv/xhtvBNzzkRSWZcuWYWZB\nELVt27Ycp0hERKorr5r25YPOnTtXem/16tUKokQksHjx4rSsJ5Vzid/s+fLLL0/LtiX7/A5KGjdu\nTK1atXKcGhGRmi0b5XUFUnFs2rQJM6Nx48YALFiwQEGU5K2qtLeP/Ix+38lZu3ZtWtZz6qmnBtN1\n6tSJu6w/f9SoUWnZtmTfjh07cByHVatW5TopIiJFI5OtbRRIxdC0adNgsM0uXbrgOA4HHXRQjlMl\nUpmCn+xL1z6fNGlSMN2nT5+oy1RUVKjJpYiISB5SIBWhbdu2mBkrV66kbt26OI7Dxx9/nOtkicQU\nOR5Uqp8N/YwK7NnzyiuvANCyZUvAra3wvfPOO7Rv3x4zC+tYQkGziIhI/lAgBbz00kvB9DfffAO4\nPSxt3LgxV0kSyahEBXIV2ONLR8B58cUXA/C3v/0NgGeffTYIbE877TQWLFhQ7W2IiIhI5m4U5ySQ\nCh3Zuzp/vXr1ijruVKp69+4dTF9++eU4jsOyZcuqvV6RTIo1Jkmyz0pV57PFrl27dtX6fKtWrYLp\nHj16MHDgwLD57du3Z/z48TiOw+mnnw7A0KFDq7VNERGRYpPpMk1Wuz/3v8yWLVtS+ty2bdtYsmQJ\n8+fP5/HHH2fGjBkAjBkzhjFjxlRa/oQTTqBLly60adMmGAAx0o4dO5g8eTKvvfYaAJdeeikvvvhi\nSukSkeJ0xx13MGTIEKZMmUL37t2T+szMmTP57W9/y48//ghA3bp1g1rvoUOHxgyUxo8fn55Ei4iI\nSFrlZBypRL1TRVu+fv36dOzYkYsuuihs3ooVK+jfvz/jxo0L3vvggw/44IMPkl5/eXm5giipceJV\nY6vWqXr8jmh69OhRaZ6/b6dOncr999/Pe++9V2mZiooKSktLM5tIERGRLCjm56sLfkDepk2b8vrr\nr4e9N2PGDL788kuWLFnC9u3bo36urKyMzp07c84552QjmSJSwyxdupQWLVpUej/WBaWkpISXX36Z\nCy64INNJExERyYpiDqKgBgRS0XTt2pWuXbvmOhkieSGy9qnYT3rp0rx582DfXnjhhYwdOzZsfqdO\nnbj11lu55JJLcpE8ERGRrCnWli41MpASEcmmV199NddJEBERyZpiDZwiqftzERERERGRFCmQEiki\natYnIiIikh4KpERqOH/cNRERERFJHwVSIkVAbZlFRERE0kuBlEgBqE6NkoIoERERkfRTr30iBURN\n9ERERETygwIpkRouWvClgExERESkehRIiRSAdDXPUwAlIiIikh4KpESKiJ6XEhEREUkPdTYhIiIi\nIiKSIgVSIiIiIiIiKVIgJSIiIiIikiIFUiIiIiIiIilSICUiIiIiIpIiBVIiIiIiIiIpUiAlIiIi\nIiKSIgVSIiIiIiIiKbJUBug0sx+B7zKXnKKwv+M4jTO9EeVVWmQ8r5RPaaO8KhzKq8KhvCoMKlcU\nDuVV4Ugqr1IKpERERERERERN+0RERERERFKmQEpERERERCRFCqRERERERERSpEBKREREREQkRQqk\nREREREREUqRASkREREREJEUKpERERERERFKkQEpERERERCRFCqRERERERERS9P8BRFtnL0/bxlcA\nAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 1080x108 with 8 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Get pre-trained model without modifications\n",
    "model = mh.get_module(iterator=test_iterator)\n",
    "# Predict on our test data\n",
    "predictions = np.argmax(model.predict(test_iterator), axis=1).asnumpy().astype(int)\n",
    "\n",
    "# Plot all test images along with the predicted labels\n",
    "images = get_images(test_iterator)\n",
    "show_predictions(predictions, images, imagenet_class_to_human, None, (15, 1.5))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "That clearly doesn't work. The images we have are different in nature that the ones that the pre-trained (source) model was trained on. Furthermore, the pre-trained images of the source model might not contain some of the labels we have here at all."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "So far, we have agreed that:\n",
    "(a) Training a neural network from scratch won't work in this case, since we have very few data.\n",
    "(b) Re-using a pre-trained neural netwrok as is doesn't work either.\n",
    "\n",
    "\n",
    "Solution is Repurposing: take a pre-trained neural network and repurpose it for the new task through transfer learning. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/damianou/anaconda/lib/python3.5/site-packages/sklearn/linear_model/sag.py:334: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  \"the coef_ did not converge\", ConvergenceWarning)\n"
     ]
    }
   ],
   "source": [
    "repLR = xfer.LrRepurposer(source_model=source_model, feature_layer_names=['pool10'])\n",
    "repLR.repurpose(train_iterator)\n",
    "predictionsLR = repLR.predict_label(test_iterator)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA1IAAABzCAYAAACSPjMJAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4zLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvIxREBQAAIABJREFUeJzt3Xl0FFX6N/Dv0+mwBZVFZN9EloER\njYAbiAEdiKiACGrgh6CIx8AoOAjIq+A7DKIiIDCj4os6LiOgoHhUBmRX4TeyuaBGdpEIYTAssoWQ\nhPv+0V1ld6fXdHUt3d/POX1OdVV11e2+XVX3qbqLKKVARERERERE0XNZnQAiIiIiIiKnYSBFRERE\nREQUIwZSREREREREMWIgRUREREREFCMGUkRERERERDFiIEVERERERBQjWwZSIjJURNZbnQ4ipxOR\nfSJys9XpIEpGvFYRUaphucKfLQMpciYeXERE5AS8XhGRERhIkSlExG11GohSHY9Dosh4nCQv5i0Z\nzfJASkQai8gHIvKriBwRkX/4LJsuIsdE5CcRucVn/kUi8pqIFIjIARGZIiJpPsvvF5EfvZ/9VESa\neueLiLwgIodF5DcR2SYif/Quq+zd334R+a+IzBWRqmb+Fk4mIm8DaALgYxE5JSLjRESJyDAR2Q9g\njXe9a0Xkf0XkuIh8KyJZPtsIm69UYVd6/+u/ici7IlIFAERkuIjsFpGjIvKRiDTwzm/mzTv9giMi\n60TkAe/0ZSLymXd7hSLyrs96bURkpXebO0TkLrO/bDILdr4UkRYissb7vlBE3hGRGj6f2Sci40Vk\nG4DTLEhUDK9VyYPXK+fiOdA2WK7wsjSQ8p50PgHwM4BmABoCWOhdfA2AHQAuBjANwGsiIt5lbwIo\nBXAZgEwAPQBomdEXwP8B0A9AHQBfAFjg/VwPAF0BtAJQA8DdAI54lz3nnX+ld7sNAUwy9hsnL6XU\nYAD7AdyulKoO4D3vohsB/AFATxFpCGApgCkAagF4DMD7IlLHu27IfKW43AUgG0BzAO0BDBWR7gCe\n8S6rD88xuDDkFvz9DcAKADUBNALwdwAQkQwAKwHMB3AJgBwAL4lIO8O+SQoLc74UePKyATzHWmMA\n/zfg4zkAbgVQQylVak6KkwevVcmF1ytn4jnQVliu0CilLHsBuA7ArwDcAfOHAtjt874aAAWgHoC6\nAIoBVPVZngNgrXd6GYBhPstcAM4AaAqgO4CdAK4F4PJZRwCcBtAiIG0/Wfn7OO0FYB+Am73Tzbx5\ndqnP8vEA3g74zKcAhkTKV77iypP/8Xk/DcBcAK8BmOYzvzqAEm++aXnn9lm+DsAD3um3APw/AI0C\n9nU3gC8C5r0C4Cmrf4dkeIU6XwZZry+ArwP+A/dbnX4nv3itSr4Xr1fOe/EcaI8XyxX+L6sfbzYG\n8LMKfnfgkDahlDrjvcFXHZ47Q+kACn6/6QcXgHzvdFMAs0Vkhs+2BEBDpdQab3WMFwE0EZEl8Nxl\nqgLPBXCrzzYFAB/Txy/fZ7opgAEicrvPvHQAa73LwuUrVdwhn+kz8Ny1qw3gK22mUuqUiByB5w7f\ngQjbGwfP3aNNInIMwAyl1Ovw5OE1InLcZ103gLfj/wqEEOdLEbkEwBwANwC4AJ7j5ljAZ3kcxYfX\nqtTA65W98RxoHyxXeFkdSOXDc5Fwh7hAhfpMMYCLQ3wmH8DTSql3gn1YKTUHwBzvgfcegLEAngJQ\nBKCdUipSZlNoKsK8fHju8A0PXElE6iN8vpKxDsJzggKgPz6vDc/J7rR3djUAJ7zT9bR1lVKHAAz3\nfq4LgFUi8jk8+fuZUupPCU99agp1vnwGnuOsvVLqiLfK2D8CPhvs2KTo8VqVfHi9ch6eA+0tJcsV\nVnc2sQlAAYBnRSRDRKqISOdwH1BKFcBTj3KGiFwoIi5vQ8MbvavMBTBBqz/pbRA6wDvdSUSuEZF0\neDL1LIAypdR5APMAvOC9aEFEGopIzwR852T2XwCXhln+LwC3i0hPEUnz5neWiDSKIl/JWPMB3Cci\nV4pIZQBTAWxUSu1TSv0Kz4nvf7z5dD+AFtoHRWSAiDTyvj0GzwWqDJ66661EZLCIpHtfnUTkD6Z+\ns+QV6nx5AYBTAI5723WMtTKRSYrXquTD65Xz8BxobylZrrA0kFJKlQG4HZ7GmvsB/AJPfchI7gVQ\nCUAePD/4YngatkEptQSexrgLReQEgO8BaL0oXQjPRegYPI3gjgCY7l02HsBuAF96P7cKQOv4vmHK\neQbAk95HsP0DFyql8gH0gaeB9a/w3GkYi9//hyHzlYyllFoNYCKA9+G5MLUAcI/PKsPhyZsjANoB\n+F+fZZ0AbBSRUwA+AjBKKfWTUuokPA2u74HnztQheI7Fyon9NqkhzPnyrwCuAvAbPI3jP7AqjcmK\n16qkxOuVw/AcaG+pWq4Qb8MtIiIiIiIiipLVVfuIiIiIiIgch4EUERERERFRjBhIERERERERxYiB\nFBERERERUYwYSBEREREREcUopgF5L774YtWsWbMEJSU17Nu3D4WFhRJ5zfgwr+JnRl4xn4yxdevW\nQqVUnUTug3llDOaVczCvnIHlCudgXjlHtHkVUyDVrFkzbNmypeKpInTs2NGU/TCv4mdGXjGfjCEi\nPyd6H8wrYzCvnIN55QwsVzhHKueViCcmccqwS9HmFav2ERERERFRQmhBVOB0MojpiRQREREREVFF\n+QZTTnlCFQoDKSIiIiIiMlywKn3BnlA5NaBi1T4iIiIiIjJUqGp8SqlygZNTq/wxkAJQWlqKTz75\nBA8//DAyMzNRuXJliEjQV926dbFw4UKrk0xEREREZEvRVN8LDKi0sraT2K5qn91/wMOHDyMnJwc5\nOTmOfQzpK5rfOxm+JxERETmXUgqDBg3CggULwq734Ycfok+fPialioKJtQ2Uto72ORFxTNnTVk+k\nrr76asv2PWDAALz55pvIy8vDyZMn9Sg58LVz507L0mgVuwe3lFjMfyIissr9998PEYHL5YoYRAFA\n3759ed2yiViDoVDtqOzMNk+ktB+sUqVKKC4u1ucPHz4c8+bNsypZfnwztVu3bhamxHih/uxO+SOT\nsZjvRERktp07dyIrKwsFBQVBl8+aNQujRo0Kuw3fpxqnTp1CRkaG4emk0OLtPML36ZQTOqKwxRMp\n30KbFkRpP+Crr76qTxcVFVmSvg8++KBcwXLNmjWWpIUo0RhEEcXvzTffhNvtDtneVntVqVIFP/+c\n8HFviWzhxx9/xNixY+FyuYIeD61bt9aDKBHBunXr/GoFRQqiAE+hu6SkBABQvXp19OvXL6HfiX5n\nZPnBKW2nLA2kpk+frv8wW7duDRpxHj58GC+++CIAoFq1anHtb9q0aX4B2q5du1BaWhpy/e3bt0NE\ncOeddwKwd0RMlCj835trxIgREQvfvMNqH6WlpX43/LTX0KFDUVZWpq/XoUMHZGdnIzs7GzfddJM+\nv7i4GM2aNdM/N2fOHCu+Bpkg0nFt14JiRX388cdIS0vz+35t27bF9OnTy11X6tWrh7/85S8oLS3V\ng6bz58/jxhtvrNC+3W63vo8lS5ZARHD69Om4vxOF5vv0yKhygxM6o7AskCosLMTYsWMBeKrJXXXV\nVUHXu+SSSwzZ36xZszB+/Hj9/fDhw9GqVSt8/PHHQdffuXMn/vCHP+jvWZikVJWsF3k7crvdePnl\nlyOud+bMmYTlx+uvv67nd15eXkL24XSfffYZrrjiCogI0tPTMXz48HLr5OTkoKCgQC8IbNmyBcuW\nLcOyZcuwatUqff5XX33ll5ejRo2K+6YhOVeynGfT0tLQu3dvnD9/3m9+69atMWvWLJw6dcrvSVNB\nQQFmzJiBtLQ0Q9PhW3arXr26odsm89i5q3RLAqlvvvkGderUAQAsXrw4aDU5pRQmTpwIABg5ciQA\noGbNmhXa35QpU/Doo4/q21VKwe32NA/r1KlTufUvuugitG7dGgBw7ty5lAiiIhWUU+kuWqrzvbiR\nOebMmQMRQVlZGRo0aBCysxvt9a9//QuA57jUqrAYQUQwbNgw/X27du14fAM4evQomjdvrp/vsrKy\nsG3bNn15//79UVxc7JdH8+fPR7169SJuOzMzE+fPn4dSCvn5+UhLS0NRUZG+rylTpiTyq5FJAu/W\nh3pp6zr1uNNq/pw/fx4ulwsbNmzw+37bt2/HqFGjTH2qrpTC+vXrAQCVK1c2bb+pwqy2THZ9OmV6\nILVp0yZkZmYC8PwoWrW5YCZPnux3AB49erRC+1y9erU+rf3wWpW+Ro0a6ctWrlwJEcGJEyfQpEkT\nKKWQnp5eoX0mi0gnfI0d/sxkPAZUiTd16lS93v/Jkydx4MCBiJ8ZNGiQni+VKlXCwIED405Hy5Yt\n9enAfL/lllvi3r4T/fbbbxAR1K5dG/v27QPgafsUeC5ctGgRKlWqFPf+GjVqpFdtatKkCQBg4sSJ\n+nXrb3/7W9z7IPPFUsh0Yq9lGhHRa/4opVBWVobrr7/e4lR5dO7cGenp6Th37hw++eQTq5OTFAID\nGbPKCqECKquOF1MDqUGDBuGaa64BYG5VubVr16J///5+89q1a+eXhk8//RQ9evQAAOTl5aVk499Y\ne+5jITs1OKHXHKc6ceIEnnjiCQCe3zfWqidanixYsACPPfZYhdNRWFiI3bt3Iz093S+ftenly5dX\neNtOpZRCjRo1AABvvPGGfr679957Tdn/zz//rO+zc+fOAIBJkyZBRHDfffeZkgaKT0ULd3YqJEar\nefPm+rRdrxVaJxZ9+/a1OCXOFiyAsiLP7XJT39RAav78+QBQrlvL0tJSTJkyBU2aNImqMabvy+Vy\noUuXLhgzZgx+/PHHkPtetGiR3x3E77//3m95dnY2AKCkpERvG/X+++874gRmBCMOglT4nVIN8zSx\nLrroIgCeKsQV9corrwAAZsyYUa49QrS0qtbB0nHo0CEA0J+QpAqt6tHu3bsxZMgQS9Oyfv16FBYW\n4uKLLwbgCezGjBljaZooehUpaNo1GAlFe2Jb0XOQGWrXrg0Afp3AkHOFKp+bXW4xLZD64osv9Onx\n48f7BUPp6emYOHEi8vPzY96uUgobNmzAzJkz0bZt26DBVrVq1XDfffeVC540X375JQBPBxNa2ykA\n5Z5iJZto/mzRrOO0Ez5FZtUj+1RSv359AJ5qc/FUIX7wwQf1/ElLS8O///3vmD6vnfNeeumloMvr\n1q2LjIwM5Ofno02bNhVOp5MMHjwYRUVFuPDCC9GiRQurkwPAUwj89ddf9byeOXMmb3TYWLgn+dG2\nMXZKe9XvvvsOgOemA/+TqSfRbecrsn0z/4emBVJdu3bVp9966y0Anjr5EydOxJEjRyI2wIz2tXXr\nVowaNQpNmzbV91dUVIQ33ngDl19+edDM6NWrl54eTbKfDFhIplAC//v8fySG9qQn1sAnFC2fbr31\nVogIJkyYEHb9GTNm6B1cZGVlITc3N+S6p06dAgDs2LHDkLTamYjonXn89ttvFqcmOKUU2rZtCyD5\nr1VOFG2eJMu59Y477gAA29x0iEaoHpspssDgPlyzECNekdIRrv2+GUyt2nfDDTdg5cqV+pfduXMn\nJk+ejFq1ahm2j6uuugqzZs3Cvn37gv7Ay5Ytw+jRo/0y59ixY4bt32mS5UROxnPCnVCnevDBBwGg\nwmOkhKKUwkMPPQQAePbZZ8NenLQ2VYsXL8batWsjblur/jx79mxD02xXdu+C/IcfftCrYqZatUs7\ni+Ympe+5Ndw51ilB8p49e6xOQtS037t37968vhnIqIAmlgcndmFqINWmTRvcfPPNZu6ynOzsbLzw\nwgt6d7Ox3mH1rfrnVLGenCP9YZ1ysk9FTmmonGrmzZsHwNNTqNFefvllKKWQk5MTdr2OHTtCqfA9\np/qaPHkygN/buiYrrf3EmTNnbN3eA4BeJdSo8RapYoKdZ+1U0CN/M2fOBAC4XC7k5uZi9+7dFqco\nOVWkVpmRzCr3mBpI2fHE0qpVq5jW//Of/5yglFgj0mPUWIIoO+ZvKmPwZG9t2rRJ6PAK8+fPD3vB\n2rx5c0zb08bc27RpUyKSaxsulwuffvopABg+OGig2bNn6+fdgwcPxvx5bYypLVu2GJ00ilK059mK\nNIrnNTUxHn30UfznP/8BAMydOxctW7YMWhYaNmyYXgWbnMPs48bUQOrw4cNm7i4hgo1g7zThov9Y\n/oAMopIbA7HE2LBhAwBP1Swn0cbeS4X/RY8ePXDmzBkAifu+IoLRo0fr7xs2bBjT55X6fdB6soZv\nhxKBL9//TSocM05z7bXX+uXXvHnz9B6bNa+//jrq16/vF1z96U9/sijFZFemBlIV6ZXPKq1bt9an\n+/Xrp09rDXydVgiKRjx1Wsle4rlwMz8TSyv8ulymj4cel/Xr1wMAsrKyrE2ISapWrapPG91d8tmz\nZ/XpvXv3YvHixQCgB2/RuOKKKwD83naN7Ccw0AoU7bmWgVjiPfDAA8jLy/MLrl599dVyT6VXrVoF\nEcGtt95qUUrJbky9kn/99ddhl3/zzTflHq02aNAg4ekKdpLavn07AOC2227D+++/X255MowwH+yJ\nkjYvXLsa32UsdNtPYL4G5m007NaYM5lE07GDHXXr1g0A9B7tUoF2DLjdbv2JXKBt27YhNzc3pgb3\nWpCmlELz5s31dmpacBQNrcvpZcuWRf0ZMk8s51AGSvY0bNgwlJaW+gVXL774IgBPb6tsf0yAyYFU\nOCKCzMzMcvMLCgoS/kcdN24cAOD48eN+85VSIbvIfPfddxOapkQLVy0vVF1uNqa1v0j5E5i3iRz7\ngZKTGTe37EQLWIK1Z/vhhx9wxRVXYO7cubjsssuiOn4++uijkMuibfSu7ScZbuilOp5znWXEiBHl\ngmTmYWqzRSClNewVkXL1jN955x19WaI88sgjAMoHUqkgms4kgnVAwSDKfsIFUcFO+tE8caTEuPLK\nK61OAgBPlbWZM2eiffv2uPnmm0OeA7Ugwuju2p3gj3/8oz49YMAAv2XDhg0DAHz77bd6FXCtNkMw\n58+fR58+feJKj1bNsH379njyySfj2haZI/CcGqrWQKjPauvZ1QUXXGB1EiyhlNKHIOA1M3XZIpB6\n+umnAQQfmHLgwIH69OOPP56Q/deoUQNA+Augr8GDByckHVaI5WkEAyhnCDd2iSZUcMz8NUfz5s2t\nTgJmzJgBt9uNMWPG4LvvvsPq1atRs2ZNVKpUyW+9JUuW6NXa1q1bZ0FKracFmFpbJo32u6Snp+tV\nwBcsWBByOy6Xy+8YGz9+PADohbFoaIPNf/vtt1F/huwnmQreVg9rY6X09HS9k4oTJ05YnBqygi0C\nqS+++AJA6Eaz2vglBw4cSMj+tbFchgwZEtX6b731FgCkTI9JLGDbX7R3LePtqZGMYeQg5LHSehTT\nBuQNVFJSAhHByZMnISL6k5ZU/p9cdNFFOHnyJIDgnYQMHTpUPwajuS5ov+W0adMgIqhcuTKA6Dq1\nSNR1kCoummFEfI+faK6pTqoZoA0wnqry8vIAeM4TlHpMC6S0UeKDRez16tUDAL1f/0CTJk0CALz9\n9tsJSZs26GKs3bNrY3g4UaTB0PiEInmxfnfqcrvdeiBQs2bNcsf/woUL9XUvvPBCAJ5qiDwPANWr\nV8eNN96oB6JZWVnYunUrgN/H1lJKRT1ou1IKp0+fxtq1a7F//34opSL25OiEal7kYVS7Uydch7Wb\n4OHa/0UyZ84c9OzZE+np6XC5XOjRowfmzp1rVBIT7vLLLwcATJ8+3eKUkC8zyjemBVLXX389AOD7\n778vt0zrFv3666/H888/77cs3gvHhAkTEnIHLycnBwDwxBNPGL5toliwcOU8oXqASyQR0Z94ZGVl\n4ejRo+XWufvuu6GUQmFhoT6Pg73+zrdqo++1TOs2OVbVqlVDVlYWGjduHHHda6+9FgDQrl27mPdD\niWXU2Iwap57TY23/V7duXT3YHDVqFFasWKH3krdy5Urk5uZCRNCoUaMKpWfq1KlBnxaKCAYMGKDf\nRDfCtm3bAABjx441bJtUcWYeO6YFUlqbgGBjSfnewRs3bly5OzmXXXZZTPvy/fyzzz6LFStWxJHy\n4LQuMKdOnWr4tonMxqdS5tq3b5+p+ws870bqgr127dp6gd13TD0CioqKAABHjhzB6dOnoZQqN5Bn\nImzcuBFA8JuRZI2KVKWOJFXOxSLiVwto9OjRWLNmDc6cOYOSkhKsXr1aH/z2wIEDaNWqVUzbP378\neNgb3YsXL0ZaWhq6du3quICV7MW0QEoLhn766aegy7W7OXXq1PGbn52djV27dkW1j127dgU9CWk9\nK0Uj2g4natasGfU2iRLFiItuRcaZovh89tlnpu3r7NmzaNKkif7+yJEjUX1OK7DHMj5SKqhSpYp+\nzGRkZJiyz8CaGmQf0XTuE0q4tlVOK9x/8MEHACIPNP7222/r33Pw4MF62e+FF15At27dULVqVbjd\nbnTv3h0rVqyAUgoZGRkhy3ehaGW0UM0YXnvtNQCeNvoulwvvvfdeRb62n5YtWwJAyGFzKDmZFkhd\neumlACJflA8fPuz3Z492sMHS0lL9jkXgY/ZoGnVq5s2bF9X+iOzEqIsug6nEM/smjDb4KwAcOnTI\n0o4ukonvDYi9e/cmdF/aWIdOK1ynslBVykIFTaHaKzvFHXfcAQDlqgb7EhHce++9+npax12RnDp1\nKqa0PPXUUwD8e30OdP/990MphZKSEgCeas3x0gKoRx99NO5tkXOYFkhpXYwHq5dvBG2sj19++SXo\nfkN1ZAH4j4Gg9SBIZHdGBj1OvHA7lXbn1orfvG7duqbvM5n17NkTANCiRYuE7UMbZ3HWrFkJ2wdZ\nJ5nOvdo4c4E1iwIl+jtrPT3Pnj074rputxtdu3Y1ZL9aNWg+xU8tpgVSl1xyCYDEdd26Y8cOAEDD\nhg395q9atQoAMGrUqJCfbd++vT69efPmBKSOyFhGVgFxUje7yeDqq68GAL3Ht0TyrUL4zDPPJHx/\nqWb58uX6dKKq+Wk9ooW7hpF9RPN0yXd+Mp17161bp9ciCvxed955JwBzq6lefPHFUa3HIQUoHqYH\nUsE6mzCC70mpdu3a6NWrF2rVqoWOHTsCKB9g+Yp094TIrioaRCXTxdtJcnNz9QK3dm5KpKFDh+rT\nsQ5orv1HtCdoFJxSCi1btsSZM2fw+uuvG7rt3NxcAAhZVYqcxanV9mKRnZ2td6bje5NOO4+EGr/O\nSkY8QdLG3QN+74yMUoNpgZRWfe7YsWMJ24dW1/Xo0aNYtmyZvq8xY8ZgyZIlCdsvkZmM7ho31N1T\nBlvGKS4uhojo46KYVZiqaO+AWl3//v37620fKLSdO3cCiNyx0YgRIyK2l9GsWLFC/7/Url3b+EQT\nJUjTpk0tu5asWbMGAHDddddFtb7W6UTbtm1j3te3336LFi1aQET0cfcaNmyIESNGxLwtcq7oRg40\nkJH99gdyu91QSiE/Px/79+9H8+bN0aBBg4Ttj8hsRl2QwhXktQFHyThVqlTRp51wR7p3794AgEWL\nFlmcEufIz89H48aN4Xa7g44T1q9fv5hu6Gntr5YuXWpYGonMpJTC0qVLcdttt+nzDh48mNBymXa8\nDB48OOx6Bw8exE8//YQHHngAgKdjildffTXq/QwfPrzcvL/+9a+YNGlSDKmlZGB6IGWGxo0bRzXA\nIZFTOaEwTp4nFFp1r7y8PFPGG/KlFeqjLbgcO3ZM79UvsOMeCq9Ro0bIzMzE119/DREpd4xqQVRx\ncTEqVaqkzw9208J34N9evXolJsFEJrj11lv9bs6Fa2YRjWhv8o0YMSKmJ0NPPvlkRZOke+qppxhI\npSDTqvaRM6RCHW6nMnO0+2RsCG2mHTt2QET0IMqsQVt9lZSU6E9GDh48iPr164ddf8+ePXoQtWnT\nprgLPKnoq6++0qeD9QBbuXLloEHUggUL/Nbr1q0bAN4woeRx5swZy/YtIujcuTOeeeYZbNq0CYWF\nhX7HVl5eXsjxpmJ9UephIEXkAGYGURSfWrVqoU2bNgCAl156yZI8W7p0qV5gP3HiBEQEhw4d0tvj\nfPnll37rL1myRB80fceOHejUqZPpaU4WWvX1wC6VGzRooLeVe/bZZ/1uUtxzzz36tDY/kdXgicxW\ntWpVPPLII/r77OzsqIOTtLQ0AMA//vGPiOtu2bJF38fChQuxfv16fPHFF3juuedwww03oFOnTqhd\nu7Z+nJ06dcr0m1yUXEwLpNxuTy3CYHXHich+GLTFrlOnTnonN2fPntV7XTNT06ZN9TYJq1evxgUX\nXIDz589j4sSJ+jrXXXedX0cH/fr105dpA5tTxYgIpk+fDgB499139fm+XSxPmDBBny4rK9On77rr\nLr/tECWT2bNn60MGLF++HC6XS+8cIhztGBk5cmTEdTt06KBP33PPPejSpYvfCwBuvvlmAJ7epBM1\nbAGlDtMCqcqVKwPwv2hEI9YRreN10003mbo/okh8C1Thevkia4kItmzZond6o53zzDJq1CiICPbv\n3w/AEwh3795dXz558mQopfTeTX1VqlQJRUVFDJ4NMmbMGACegpzvkB/aXfOpU6di69atUErB5fJc\nho8fP6537sF8oGTVs2dPKKVwySWXQCmFm266Kez1rCK1MZRSOHToEAYPHozs7Gy/l4hg9erVcLlc\n+O9//xv39yGyZWcTGzduRHFxMRYvXoy///3vAMy7sNx+++0hlw0cOBALFixA/fr1cfDgQVPSQxSq\nFz2zgilWK4xM+40eeeQRzJ49O+7tuVyuoL93x44d/QYNP378OGrWrOm3zrlz55Cenh5y21qgR4l1\n+vRpZGRkoEmTJuV+b98nUhotHxM11iKRnfgGMU888US55fPmzcODDz4I4PfhBWJRt25dvPXWW37z\nfK+Zsd7Up+B4LbFZILVhwwb90avmo48+Qu/evXHXXXfhvffeS3ga2rVrV27eK6+8goceekh/X1BQ\ngHHjxiU8LUQaLZiy4qTFp1+hnT59GtWrVwdgzAXljjvuwIcffhhy+ZYtW0LmR0lJiV6FmqxXrVo1\nfXrDhg3o3LlzyHW1PC0rK9MNIE2nAAAHnklEQVSfUBGlgmXLliEtLQ2TJk2CUgrbtm3DRx99pC8v\nLCw0dBy12bNn+7XVovhYVS6xE1tddbUgqlu3bnq9Ve0J0Q8//GBKGgILKXl5eXoQ9fjjj2PKlClw\nu914/vnnTUkPEdmXFkRdc801cW/rl19+8Quizp49q1cPDBfM7t+/n8M92NSHH36Ivn37okuXLiEL\nG77thhlEJTcWOstbs2aN3qYw0PLlyw0NoniziRLBNv8o7QnPtGnTMHbs2HLLrRrZXXtCFam6DJFd\nsCqeOXx/38Be8GL15Zdf4rrrrtPfL1u2zK+NVbDu6BcvXow777wzrv1SYvXp0wfdunXD2rVrUVpa\nWq4QV1RUpD+54vGavDjIeWh9+vRBs2bNAHhuJDRu3NhvAF8jMYiiRLDN7a/+/fsD8ARUeXl5+nxt\nXA4j2h3EY/PmzXj++ef1k+HGjRstTQ9RoFg7odDW9e3Agh1ZRE97evDZZ5/FvS3fIArwdA0cjNY4\nGwDq1asX934p8bReyYLdiNOCqF27dpmaJrIOz7P+unTpgpEjR2LkyJHIzc1NWBBFlCi2Cc+vvvpq\n9OjRAytWrAjaTikzM9OCVAGXXnop9u7dW65+e05OjiXpIfJV0YuxbxBF8QkcLyhekQbCXb16NQDg\n2muvNXS/ZK6XXnpJn9bG8KLkxadSRMnJNk+kAODTTz/F2bNny80vKCiwIDUee/bswYkTJwB4xogp\nKCjAww8/jL1796JWrVqsjkG2Eu//MdTn+T8PbsaMGXFv4/HHH/d7/8svv0T1OW2QSrI/7fjRqhad\nP39eHxOHx1bqCDWILBEZL1iV+ESwzRMpTeXKlW13Yrngggv0ND322GN6l+xHjhyxMlmU4nzbQlW0\nEXOoEw0bRYe3bds2AMBf/vKXuLf13HPP6dO+TymC0ao4s72m8zRo0AAHDx5EZmYmvvnmGwAMooiI\nnM52gZTVatSoEXT+uXPngjb+JrJaPNX0ggVQFNnHH3+ckO3m5uaGXT569GgAnvMROcuBAwcgInoQ\nVVxcbHGKiIgoXraq2mcHHTp0KDevsLCQQRQR6fbt22fIdmI5l2jVnocMGWLIvsl8WgclderUQaVK\nlSxODRFRcjOjvM5AKozTp09DRFCnTh0AwI4dOxhEkW1VpL594Gf4/47OsWPHDNlOz5499emqVauG\nXVdb/sYbbxiybzJfWVkZlFI4fPiw1UkhIkoZiaxtw0AqhPr16+uDbXbs2BFKKbRq1criVBGVx+DH\nfEb95itXrtSnBw8eHHSd0tJSVrkkIiKyIQZSAS677DKICA4dOoSMjAwopbB582ark0UUUuB4ULF+\n1vczLLCb57333gMANGnSBIDnaYVm+fLlaNOmDUTEr2MJBs1ERET2wUAKwPz58/XpPXv2APD0sHTq\n1CmrkkSUUJEK5Cywh2dEwHn33XcDAP75z38CAF577TU9sL3llluwY8eOuPdBREREibtRbEkg5Tuy\ndzyvnJycoONOxWrQoEH69JAhQ6CUwoEDB+LeLlEihRqTJNq2UvF8NtW1bt06rs83a9ZMn+7evTvG\njBnjt7xNmzZYunQplFLo1asXAGD69Olx7ZOIiCjVJLpMY2r359qXKSoqiulzxcXF2L9/P7Zv3445\nc+Zgw4YNAICFCxdi4cKF5dbv2rUrOnbsiBYtWugDIAYqKyvD6tWr8f777wMABg4ciHfeeSemdBFR\napowYQKmTp2KtWvXolu3blF9ZuPGjbj99tvx66+/AgAyMjL0p97Tp08PGSgtXbrUmEQTERGRoSwZ\nRypS71TB1q9Rowbat2+Pu+66y29ZQUEBRo4ciSVLlujzPv/8c3z++edRbz89PZ1BFCWdcI+x+dQp\nPlpHNN27dy+3TPtt161bh6effhqrVq0qt05paSnS0tISm0giIiITpHL7ascPyFu/fn188MEHfvM2\nbNiAH3/8Efv370dJSUnQz7ndbnTo0AF9+/Y1I5lElGTy8/PRuHHjcvNDXVBcLhfeffdd9O/fP9FJ\nIyIiMkUqB1FAEgRSwXTu3BmdO3e2OhlEthD49CnVT3pGadSokf7bDhgwAIsXL/ZbnpmZiXHjxuGe\ne+6xInlERESmSdWaLkkZSBERmWnRokVWJ4GIiMg0qRo4BWL350RERERERDFiIEWUQlitj4iIiMgY\nDKSIkpw27hoRERERGYeBFFEKYF1mIiIiImMxkCJygHieKDGIIiIiIjIee+0jchBW0SMiIiKyBwZS\nREkuWPDFgIyIiIgoPgykiBzAqOp5DKCIiIiIjMFAiiiFsL0UERERkTHY2QQREREREVGMGEgRERER\nERHFiIEUERERERFRjBhIERERERERxYiBFBERERERUYwYSBEREREREcWIgRQREREREVGMGEgRERER\nERHFSGIZoFNEfgXwc+KSkxKaKqXqJHonzCtDJDyvmE+GYV45B/PKOZhXzsByhXMwr5wjqryKKZAi\nIiIiIiIiVu0jIiIiIiKKGQMpIiIiIiKiGDGQIiIiIiIiihEDKSIiIiIiohgxkCIiIiIiIooRAyki\nIiIiIqIYMZAiIiIiIiKKEQMpIiIiIiKiGDGQIiIiIiIiitH/B/1tX8vyI0sYAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 1080x108 with 8 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "show_predictions(predictionsLR, images, id2label, None, (15,1.5))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Much better!"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Other utilities: refining existing architectures really easily! "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Bottom layer name before addition: conv1\n",
      "Bottom layer name after addition: new_conv_layer\n"
     ]
    }
   ],
   "source": [
    "print(\"Bottom layer name before addition: \" + mh.layer_names[0])\n",
    "conv1 = mx.sym.Convolution(name='new_conv_layer', kernel=(20,20), num_filter=64)\n",
    "mh.add_layer_bottom([conv1])\n",
    "print(\"Bottom layer name after addition: \" + mh.layer_names[0])\n",
    "\n",
    "# Dropping layers is also supported. The commented code below drops two top layers\n",
    "# mh.drop_layer_top(2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The following two figures demonstrate two ways of performing transfer learning: "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# from IPython.display import HTML\n",
    "# HTML('<img src=\"https://cdn-images-1.medium.com/max/1600/1*YeDK4bgLMeikGNbSySoNUg.gif\">')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# HTML('<img src=\"https://cdn-images-1.medium.com/max/1600/1*w8c3SbLq8b6wzsDgxPknzA.gif\">')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "To learn more, see tutorials here:\n",
    "https://github.com/amzn/xfer   "
   ]
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [Root]",
   "language": "python",
   "name": "Python [Root]"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.5"
  },
  "widgets": {
   "state": {
    "107f6ac042bf4b4a9aed5eb5fbf94039": {
     "views": [
      {
       "cell_index": 29
      }
     ]
    },
    "281ccc0819a2424faaff438c97584f74": {
     "views": [
      {
       "cell_index": 27
      }
     ]
    },
    "3cfafbca74cf40c988f786c31ea29f67": {
     "views": [
      {
       "cell_index": 27
      }
     ]
    },
    "550cdeedc6904b15b9f335213f49fb9e": {
     "views": [
      {
       "cell_index": 27
      }
     ]
    },
    "624b5a7a3ada4524acad9149ee7e99ee": {
     "views": [
      {
       "cell_index": 27
      }
     ]
    },
    "9d764dc3036e4b62ac55a4b386ad3e35": {
     "views": [
      {
       "cell_index": 27
      }
     ]
    },
    "ac16bb7740ff4502bb78b1aea079cf32": {
     "views": [
      {
       "cell_index": 36
      }
     ]
    },
    "af86f07d633b463c8fb5dae984ae3c83": {
     "views": [
      {
       "cell_index": 34
      }
     ]
    },
    "cb443bc196b443eaaf248dd9e37c0f87": {
     "views": [
      {
       "cell_index": 36
      }
     ]
    },
    "ccf1816ea3d24df78c8444ecbc75c9d1": {
     "views": [
      {
       "cell_index": 29
      }
     ]
    }
   },
   "version": "1.2.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}