{ "cells": [ { "cell_type": "code", "execution_count": 158, "metadata": { "collapsed": true }, "outputs": [], "source": [ "import numpy as np\n", "import matplotlib.pyplot as plt\n", "from IPython.display import display\n", "%matplotlib inline" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# 1 Data & Architecture" ] }, { "cell_type": "code", "execution_count": 279, "metadata": { "collapsed": false }, "outputs": [ { "data": { "text/plain": [ "(3, 2)" ] }, "execution_count": 279, "metadata": {}, "output_type": "execute_result" } ], "source": [ "X = np.array(([3,5],[5,1],[10,2]), dtype=float) # hours sleep, hours study - input matrix: 3 samples, 2 features\n", "y = np.array(([75],[82],[95]), dtype=float) # test score\n", "#normalize data:\n", "X = X / np.amax(X, axis=0)\n", "y = y / 100\n", "X.shape" ] }, { "cell_type": "code", "execution_count": 69, "metadata": { "collapsed": false }, "outputs": [ { "data": { "text/latex": [ "\\[\n", " \\textsf{Neurons sum inputs from synapses}\n", "\\]\n", "\\[\n", " z = x1 + x2 + x3 = \\sum xi\n", "\\]\n", "\\[\n", " \\textsf{and apply a sigmoid activation function}\n", "\\]\n", "\\[\n", " a = \\frac{1}{1 + e^{-z}}\n", "\\]" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "%%latex\n", "\\[\n", " \\textsf{Neurons sum inputs from synapses}\n", "\\]\n", "\\[\n", " z = x1 + x2 + x3 = \\sum xi\n", "\\]\n", "\\[\n", " \\textsf{and apply a sigmoid activation function}\n", "\\]\n", "\\[\n", " a = \\frac{1}{1 + e^{-z}}\n", "\\]" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# 2 Forward Propogation" ] }, { "cell_type": "code", "execution_count": 255, "metadata": { "collapsed": false }, "outputs": [], "source": [ "class Neural_Network(object):\n", " def __init__(self):\n", " # define hyperparams (constants that establish structure & behaviour of network;not updated while training):\n", " self.inputLayerSize = 2 # corespond to number of features\n", " self.hiddenLayerSize = 3\n", " self.outputLayerSize = 1\n", " # define params: weights on synapses\n", " self.W1 = np.random.randn(self.inputLayerSize , self.hiddenLayerSize)\n", " self.W2 = np.random.randn(self.hiddenLayerSize , self.outputLayerSize)\n", " \n", " #2\n", " def forward(self,X):\n", " # propogate weights through network\n", " # use matrices to pass multiple inputs at once\n", " self.z2 = np.dot(X, self.W1)\n", " self.a2 = self.sigmoid(self.z2)\n", " self.z3 = np.dot(self.a2, self.W2)\n", " yHat = self.sigmoid(self.z3)\n", " return yHat\n", " \n", " def sigmoid(self,z):\n", " # apply sigmoind to scalar / vector / matrix\n", " return 1 / (1 + np.exp(-z))\n", " \n", " #3\n", " def sigmoidPrime(self,z):\n", " # backpropagation \n", " return np.exp(-z) / ((1 + np.exp(-z))**2)\n", " \n", " #4\n", " def costFunction(self, X, y):\n", " self.yHat = self.forward(X)\n", " J = 0.5 * sum((y - self.yHat)**2)\n", " return J\n", " \n", " def costFunctionPrime(self, X, y):\n", " # batch gradient descent\n", " # compute derivative with respect to W1 and W2\n", " # ie which way is uphill in our 9D optimization space\n", " self.yHat = self.forward(X)\n", " #backpropagation error: delta\n", " delta3 = np.multiply(-(y - yHat), self.sigmoidPrime(self.z3))\n", " dJdW2 = np.dot(self.a2.T, delta3) \n", " delta2 = np.dot(delta3, self.W2.T) * self.sigmoidPrime(self.z2)\n", " dJdW1 = np.dot(X.T, delta2)\n", " return dJdW1, dJdW2\n", " \n", " #5\n", " def getParams(self):\n", " # get W1 and W2 rolled into 1 vector\n", " params = np.concatenate((self.W1.ravel(), self.W2.ravel()))\n", " return params\n", " def setParams(self, params):\n", " # Set W1 and W2 using single param vector\n", " W1_start = 0\n", " W1_end = self.hiddenLayerSize * self.inputLayerSize\n", " self.W1 = np.reshape(params[W1_start:W1_end], (self.inputLayerSize, self.hiddenLayerSize))\n", " W2_end = W1_end + self.hiddenLayerSize * self.outputLayerSize\n", " self.W2 = np.reshape(params[W1_end:W2_end], (self.hiddenLayerSize, self.outputLayerSize))\n", " \n", " def computeGradients(self, X, y):\n", " dJdW1, dJdW2 = self.costFunctionPrime(X,y)\n", " return np.concatenate((dJdW1.ravel(),dJdW2.ravel()))\n", " \n", " \n", "NN = Neural_Network() " ] }, { "cell_type": "code", "execution_count": 92, "metadata": { "collapsed": false }, "outputs": [ { "data": { "text/latex": [ "\\[\n", "\\begin{bmatrix}\n", " 3 & 5 \\\\ \n", " 5 & 1 \\\\ \n", " 10 & 2\n", "\\end{bmatrix}\n", "\\begin{bmatrix}\n", " W_{11}^{(1)} & W_{12}^{(1)} & W_{13}^{(1)}\\\\ \n", " W_{21}^{(1)} & W_{22}^{(1)} & W_{23}^{(1)}\n", "\\end{bmatrix}\n", "=\n", "\\begin{bmatrix}\n", " 3W_{11}^{(1)} + 5W_{21}^{(1)} & 3W_{12}^{(1)} + 5W_{22}^{(1)} & 3W_{13}^{(1)} + 5W_{23}^{(1)} \\\\ \n", " 5W_{11}^{(1)} + 1W_{21}^{(1)} & 5W_{12}^{(1)} + 1W_{22}^{(1)} & 5W_{13}^{(1)} + 1W_{23}^{(1)} \\\\\n", " 10W_{11}^{(1)} + 2W_{21}^{(1)} & 10W_{12}^{(1)} + 2W_{22}^{(1)} & 10W_{13}^{(1)} + 2W_{23}^{(1)}\n", "\\end{bmatrix}\n", "\\]\n", "\\[\n", " X W^{(1)} = Z^{(2)}\n", "\\]" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "%%latex\n", "\\[\n", "\\begin{bmatrix}\n", " 3 & 5 \\\\ \n", " 5 & 1 \\\\ \n", " 10 & 2\n", "\\end{bmatrix}\n", "\\begin{bmatrix}\n", " W_{11}^{(1)} & W_{12}^{(1)} & W_{13}^{(1)}\\\\ \n", " W_{21}^{(1)} & W_{22}^{(1)} & W_{23}^{(1)}\n", "\\end{bmatrix}\n", "=\n", "\\begin{bmatrix}\n", " 3W_{11}^{(1)} + 5W_{21}^{(1)} & 3W_{12}^{(1)} + 5W_{22}^{(1)} & 3W_{13}^{(1)} + 5W_{23}^{(1)} \\\\ \n", " 5W_{11}^{(1)} + 1W_{21}^{(1)} & 5W_{12}^{(1)} + 1W_{22}^{(1)} & 5W_{13}^{(1)} + 1W_{23}^{(1)} \\\\\n", " 10W_{11}^{(1)} + 2W_{21}^{(1)} & 10W_{12}^{(1)} + 2W_{22}^{(1)} & 10W_{13}^{(1)} + 2W_{23}^{(1)}\n", "\\end{bmatrix}\n", "\\]\n", "\\[\n", " X W^{(1)} = Z^{(2)}\n", "\\]" ] }, { "cell_type": "code", "execution_count": 205, "metadata": { "collapsed": false }, "outputs": [ { "data": { "text/plain": [ "" ] }, "execution_count": 205, "metadata": {}, "output_type": "execute_result" }, { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAAgUAAAFkCAYAAACw3EhvAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAAPYQAAD2EBqD+naQAAIABJREFUeJzs3Xd4FGXXx/HvnVBCKKFJhxBAmkgHCdhBEMQVFKRICyqg\nAQUxvIogTcEgFiSAoEiCPAbxASIqPhQ70iShqBSRqkgLRUroud8/hiQsSSCzJDs7k/O5rlzM3Mzs\nnP0xSQ6zU5TWGiGEEEIIP6sLEEIIIYRvkKZACCGEEIA0BUIIIYS4QpoCIYQQQgDSFAghhBDiCmkK\nhBBCCAFIUyCEEEKIK6QpEEIIIQQgTYEQQgghrpCmQAghhBCAB02BUuoupdRipdR+pVSyUsqVhXXu\nVUrFK6XOKaX+UEr19qxcIYQQQuQUT44UFAQ2As8CN3xwglKqMvAl8A1QD5gMfKiUesCDbQshhBAi\nh6ibeSCSUioZ6KC1XnydZSKBtlrruleNxQJBWut2Hm9cCCGEENnKG+cUNANWXDO2FAj1wraFEEII\nkUV5vLCNMsCha8YOAUWUUvm11uevXUEpVQJoA+wBzuV4hUIIIYRzBACVgaVa66NmVvRGU+CJNsB/\nrC5CCCGEsLEngE/MrOCNpuAgUPqasdLAyYyOElyxB2Du3LnUqlUrB0tzliFDhvDOO+9YXYbtSG7m\nSWaekdzM80Zmly/DqVNw8iT8+2/an6dOpc2fPg1nzkBSkvHn1dNJScZr+I6tQA+48rvUDG80BauB\ntteMtb4ynplzALVq1aJhw4Y5VZfjBAUFSV4ekNzMk8w8I7mZ50lmWsPx43Do0PW/EhPh2DHjF7+v\nCQiAfPnSvvLnz/r8yZOwaBHgwcfvppsCpVRBoBqgrgxVUUrVA45prf9SSk0AymmtU+5F8D4QfuUq\nhI+AlkAnQK48yGYHDx60ugRbktzMk8w8I7mZd21mWhu/yP/6K+1r3z73+X/+gQsXcrauwEAoXNj4\nKlIkbfrq+cBAKFAg7c9rpzObz58flLpxDZlJSEhtCkzz5EhBY+A7jHsUaOCtK+MxQF+MEwsrpiys\ntd6jlHoIeAd4DvgbeFJrfe0VCeIm7d+/3+oSbElyM08y84zkljWXL8Pff8Off8Iff+wnIsKY/vNP\n2LXLOFx/s5SC4sUz/ypWzH26aNG0X/iFCoG//83X4ItMNwVa6x+4zqWMWuuwDMZ+BBqZ3ZYwp1Ej\nidgTkpt5kplnJDd3Fy/CH3/A778bX7/9Blu2GL/40/6n34hJk7L+msWKQYUKUKYMlC5tfF09nfJV\nsiTk8dVT7S0kkThIt27drC7BliQ38yQzz+Tm3I4ehfh442vjRqMJ2L4dLl260ZppmeXNCyEhEBwM\nFSumfVWqZPxZoYLxv3jhuZu6o2FOUUo1BOLj4+PlpBwhhLCZkydh7VpYvz6tEdizJ2vrBgTArbdC\n1apQrZr7V4UKzj1sn50SEhJSjko10lonmFlXjhQIIXKFffv2kZiYaHUZjnTkCGzYYBwB2LjR+Ejg\nRv/fzJMHKlc2fvlXrQpVqhh/li+f8S/+o0eNLwElS5akUqVKOfLa0hQ4SFhYGLNnz7a6DNuR3Myz\nW2b79u2jVq1aJGXHGWoiW1y6lHbyoDAnMDCQrVu35khjIE2Bg7Ru3drqEmxJcjPPbpklJiaSlJQk\nN0QTtrd161Z69OhBYmKiNAXi+nLzSUw3Q3Izz66ZyQ3RhLg+bzwlUQghhBA2IE2BEEIIIQBpChxl\n5cqVVpdgS5KbeZKZEM4kTYGDTJw40eoSbElyM08yE8KZpClwkHnz5lldgi1JbuZJZvbSp08fQkJC\nrC7juvbu3Yufnx9z5sy54bJ2eD92JU2BgwQGBlpdgi1JbuZJZvailMLPz/d/3KssPhpQKZXlZYU5\nckmiEEI43IcffkhycrLVZVxXcHAwZ8+eJW/evFaXkqtJUyCEEA7n7++Pvw0eGpAvXz6rS8j1fP94\nksiyiIgIq0uwJcnNPMnMt5w+fZrBgwcTEhJCQEAApUuXpnXr1mzcuBHI+DP4Y8eO0bNnT4KCgihW\nrBhhYWFs3rw53ef6ffr0oXDhwvz111+0b9+ewoULU6FCBaZNmwbAr7/+SsuWLSlUqBCVK1cmNjY2\nXX27d++mc+fOlChRgoIFCxIaGsqSJUvclsnsnIK4uDjq1KlDgQIFqFu3LnFxcdmSmciYNAUOklMP\nyHA6yc08ycy39O/fnxkzZtC5c2emT59ORERE6v3xIf1n8Fpr2rdvz6effkpYWBjjx4/nwIED9O7d\nO91n9UopkpOTadu2LcHBwbz55puEhIQwaNAgYmJiaNu2LU2aNGHixIkUKVKE3r17s3fv3tT1Dx8+\nTGhoKMuXL2fgwIGMHz+e8+fP43K5+Pzzz6/7vpYtW0anTp3IkycPb7zxBh06dCAsLIz169dnY3rC\njdba576AhoCOj4/XQghxs+Lj47WTf6YULVpUDxo0KNO/79Onjw4JCUmdX7BggVZK6SlTprgt17Jl\nS+3n56djYmLc1vXz89ORkZGpYydOnNCBgYHa399ff/bZZ6nj27dv10opPWbMmNSxwYMHaz8/P71q\n1arUsdOnT+sqVaroKlWqpI7t2bNHK6Xctl2/fn1dvnx5ferUqdSxFStWaKWU2/vJTbKyL6csAzTU\nJn//yjkFQghxjcaN4eDBnN1GmTKQXf/hLVq0KGvXruXAgQOULVv2hssvXbqUfPny8dRTT7mNh4eH\n8+2332a4zpNPPpk6HRQURI0aNdi5cyedOnVKHa9evTpFixZl165dqWNff/01TZs2JTQ0NHWsYMGC\n9OvXj+HDh7NlyxZq166dbnsHDx5k06ZNDB8+nEKFCqWOt2zZktq1a8sTL3OINAVCCHGNgwdh/36r\nq8i6iRMn0qdPHypWrEijRo1o164dvXr1yvRa/r1791K2bFkCAgLcxqtVq5bh8gEBAZQoUcJtLCgo\niAoVKqRbNigoiOPHj7ttq1mzZumWS3la5d69ezNsClI+gsiopho1arBhw4YMaxU3R5oCB9m2bRs1\na9a0ugzbkdzMc3pmZcrYaxudO3fm7rvvZtGiRSxbtoxJkyYRGRnJokWLaNOmzU2/fmZXLmQ2ro2P\ngYUNSVPgIMOGDWPx4sVWl2E7kpt5Ts/MjuexlS5dmgEDBjBgwAASExNp0KABr7/+eoZNQXBwMN9/\n/z3nzp1zO1qwY8eObK8rODiY7du3pxtPOQkyODg40/Uyqymj1xPZQ64+cJCoqCirS7Alyc08ycx3\nJCcnc/LkSbexkiVLUq5cOc6fP5/hOm3atOHChQt88MEHqWNaa6ZOnZrtdwps164d69atY+3atalj\nZ86cYebMmYSEhGT40QFAmTJlqF+/PjExMZw6dSp1fPny5WzZsiVbaxRp5EiBg8hlYp6R3MyTzHzH\nqVOnqFChAp06daJevXoUKlSI5cuXs379et5+++0M1+nQoQNNmzZl6NCh7Nixg5o1a7J48WJOnDgB\nZP12w1nx0ksvERsby4MPPshzzz1H8eLFiY6OZu/evSxcuPC6606YMIH27dvTokUL+vbty9GjR4mK\niqJOnTqcPn0622oUaeRIgRBC2FhgYCDh4eFs2rSJ0aNH88ILL7Bjxw6mT5/O888/n7rc1b/o/fz8\nWLJkCV26dGHOnDmMGDGCsmXL8t5776G1TncCYmZNQkbj194ToVSpUqxevZrWrVsTFRXF8OHDCQgI\n4Msvv8Tlcl339dq0acNnn31GcnIyw4cPJy4ujujoaBo1aiTPPsghyhdPCFFKNQTi4+PjadiwodXl\nCCFsLiEhgUaNGiE/U64vLi6Oxx57jJUrV7pdQih8R1b25ZRlgEZa6wQzry9HChwkMjLS6hJsSXIz\nTzKzv3PnzrnNJycnM2XKFIoUKSKNUy4m5xQ4iNzMwzOSm3mSmf0NGjSIs2fPEhoayvnz51mwYAFr\n1qxhwoQJ5M+f3+ryhEWkKXCQMWPGWF2CLUlu5klm9nf//ffz9ttv89VXX3Hu3DmqVatGVFQUzzzz\njNWlCQtJUyCEELlQt27d6Natm9VlCB8j5xQIIYQQApCmwFESExOtLsGWJDfzJDMhnEmaAgfp27ev\n1SXYkuRmnmQmhDNJU+Ago0ePtroEW5LczJPMhHAmaQocRK4t9ozkZp5kJoQzSVMghBBCCECaAiGE\nEEJcIU2Bg8yaNcvqEmxJcjNPMrOfPn36EBISYnUZ17V37178/PyYM2fODZf15vuJjo7Gz8+Pffv2\neWV7VpKmwEESEkw990JcIbmZJ5nZj1IKPz/f/5Gf1acfXvs0xpzkzW1ZTe5o6CBTp061ugRbktzM\nk8zs58MPPyQ5OdnqMq4rODiYs2fPkjdvXqtLcdOrVy+6detGvnz5rC4lx/l+2yiEEOKm+fv7+9wv\n24zky5fPZ/5XnvLgL6VUrmgIQJoCIYSwvdOnTzN48GBCQkIICAigdOnStG7dmo0bN6Yuk9Fn8MeO\nHaNnz54EBQVRrFgxwsLC2Lx5c7rP9fv06UPhwoX566+/aN++PYULF6ZChQpMmzYNgF9//ZWWLVtS\nqFAhKleuTGxsbLoad+/eTefOnSlRogQFCxYkNDSUJUuWuC2T2TkFcXFx1KlThwIFClC3bl3i4uKy\nnE3lypVxuVwsX76cBg0aUKBAAW677TYWLVrktlxMTAx+fn78+OOPPPvss5QuXZqKFSsCGZ9TkPK6\nP/zwA02aNCEwMJC6devyww8/ALBw4ULq1q1LgQIFaNy4sdu/RYrt27fTqVMnSpQoQYECBWjSpAlf\nfPFFlt9bTpCmQAghbK5///7MmDGDzp07M336dCIiIggMDGTr1q2py1z7ubjWmvbt2/Ppp58SFhbG\n+PHjOXDgAL179073P3WlFMnJybRt25bg4GDefPNNQkJCGDRoEDExMbRt25YmTZowceJEihQpQu/e\nvdm7d2/q+ocPHyY0NJTly5czcOBAxo8fz/nz53G5XHz++efXfW/Lli2jU6dO5MmThzfeeIMOHToQ\nFhbG+vXrs5SNUoo//viDrl270q5dO9544w3y5s1L586d+eabb9It/+yzz7Jt2zZGjRrFSy+9lGF2\nKWM7duzgiSeewOVy8cYbb3D8+HFcLheffPIJQ4cOpVevXowdO5adO3fSpUsXt/V///13mjVrxvbt\n23n55Zd5++23KVSoEB06dLhhJjlKa+1zX0BDQMfHx2uRdQ8//LDVJdiS5Gae3TKLj4/XTv6ZUrRo\nUT1o0KDrLtOnTx8dEhKSOr9gwQKtlNJTpkxxW65ly5baz89Px8TEuK3r5+enIyMjU8dOnDihAwMD\ntb+/v/7ss89Sx7dv366VUnrMmDGpY4MHD9Z+fn561apVqWOnT5/WVapU0VWqVEkd27Nnj1ZKuW27\nfv36unz58vrUqVOpYytWrNBKKbf3k5nKlStrPz8/HRcXlzp28uRJXa5cOd2oUaPUsejoaK2U0vfc\nc49OTk52e43o6Gjt5+en9+7dm+51165dmzq2bNkyrZTSBQsW1H///Xfq+MyZM7Wfn5/+4YcfUsda\ntmyp69evry9evOi2rRYtWugaNWpk+n6ysi+nLAM01CZ//8qJhg4ycOBAq0uwJcnNPKdn1nhmYw6e\nPpij2yhTqAzr+2Xtf7s3UrRoUdauXcuBAwcoW7ZsltZZunQp+fLl46mnnnIbDw8P59tvv81wnSef\nfDJ1OigoiBo1arBz5046deqUOl69enWKFi3Krl27Use+/vprmjZtSmhoaOpYwYIF6devH8OHD2fL\nli3Url073fYOHjzIpk2bGD58OIUKFUodb9myJbVr1079zP9GypUrxyOPPJI6X7hwYXr16sXEiRM5\nfPgwpUqVAoz//T/99NNZPqehdu3aNG3aNHX+jjvuSK2vfPnybuNaa3bt2sXdd9/N8ePH+e677xg3\nbhz//vuv22u2bt2aMWPGmPq3zE7SFDhI69atrS7BliQ385ye2cHTB9l/ar/VZWTZxIkT6dOnDxUr\nVqRRo0a0a9eOXr16Xfc6/r1791K2bFkCAgLcxqtVq5bh8gEBAZQoUcJtLCgoiAoVKqRbNigoiOPH\nj7ttq1mzZumWq1WrVurfZ9QUpHwEkVFNNWrUYMOGDRnWeq2M1q9evToAe/bsSW0KwDhXIKsqVark\nNl+kSBGAdJkEBQUBpGby559/orVm5MiRjBgxIt3rKqU4fPiwNAVCCOELyhQqY6ttdO7cmbvvvptF\nixaxbNkyJk2aRGRkJIsWLaJNmzbZsg1/f39T49r4KNh2ChQokOVlPc0k5dLQF198MdN/n8yas5wm\nTYEQQlwjuw7re1Pp0qUZMGAAAwYMIDExkQYNGvD6669n+ksnODiY77//nnPnzrkdLdixY0e21xYc\nHMz27dvTjaecCBkcHJzpepnVlNHrZebPP//MdH0zRwayS5UqVQDImzcv999/v9e3fz1y9YGDmLlM\nR6SR3MyTzHxHcnIyJ0+edBsrWbIk5cqV4/z585mu16ZNGy5cuMAHH3yQOqa1ZurUqdl+n4B27dqx\nbt061q5dmzp25swZZs6cSUhISIYfHQCUKVOG+vXrExMTw6lTp1LHly9fzpYtW7K8/X/++cftEsST\nJ0/y8ccf06BBA7ePDrzllltu4d5772XGjBkcPJj+3JXExESv15RCjhQ4SGxsLB06dLC6DNuR3MyT\nzHzHqVOnqFChAp06daJevXoUKlSI5cuXs379et5+++1M1+vQoQNNmzZl6NCh7Nixg5o1a7J48WJO\nnDgBZP12w1nx0ksvERsby4MPPshzzz1H8eLFiY6OZu/evSxcuPC6606YMIH27dvTokUL+vbty9Gj\nR4mKiqJOnTqcPn06S9uvXr06Tz31FL/88gulS5dm1qxZHD58mJiYGLflvPmRx9SpU7nrrru4/fbb\nefrpp6lSpQqHDh1i9erV7N+/P8vnS2Q3aQoc5NNPP7W6BFuS3MyTzHxHYGAg4eHhLFu2jEWLFpGc\nnEy1atWYPn06/fr1c1v26l/0fn5+LFmyhOeff545c+bg5+fHI488wsiRI7nrrrvSnYCYWZOQ0fi1\n1/WXKlWK1atX83//939ERUVx7tw56taty5dffsmDDz543ddr06YNn332GSNGjGD48OFUrVqV6Oho\n4uLi+PHHH7OU0a233sqUKVN48cUX+eOPPwgJCWH+/Pm0atUqS+8xI5k9DyGr47Vq1WL9+vWMGTOG\nmJgYjh49SqlSpWjQoAGjRo3Kch3ZTfniySBKqYZAfHx8PA0bNrS6HCGEzSUkJNCoUSPkZ8qNxcXF\n8dhjj7Fy5Uq3SwjtKiQkhNtvv53FixdbXUq2yMq+nLIM0EhrberpZR6dU6CUCldK7VZKnVVKrVFK\nNbnB8k8opTYqpc4opf5RSs1SShX3ZNtCCCGyx7lz59zmk5OTmTJlCkWKFJHmKZcy/fGBUqoL8BbQ\nD1gHDAGWKqWqa63TnR2hlGoBxADPA18C5YEZwEyg07XLCyGE8I5BgwZx9uxZQkNDOX/+PAsWLGDN\nmjVMmDCB/PnzW12esIAnRwqGADO01nO01tuAAUAS0DeT5ZsBu7XWU7XWe7XWqzCagqaZLC88FBYW\nZnUJtiS5mSeZOcP999/P9u3bGTFiBK+88gonT54kKiqKYcOGWV1atsnsM36RMVNHCpRSeYFGwPiU\nMa21VkqtADL78Gk18LpSqq3W+mulVGmgM/CVhzWLTDj9LnM5RXIzTzJzhm7dutGtWzery8hRV99u\nWdyY2SMFJQF/4NA144eADG/PdeXIQA/gU6XUBeAAcBxw9s3TLeD0b+6cIrmZJ5kJ4Uw5fvMipVRt\nYDIwGuPph22AEIyPEK6rXbt2uFwut6/Q0NB0N05ZtmwZLpcr3frh4eHMmjXLbSwhIQGXy5Xu5hCj\nRo0iMjLSbWzfvn24XC62bdvmNj5lyhQiIiLcxpKSknC5XKxcudJtPDY2NsNDrV26dJH3Ie9D3oeX\n34cQTvHdd98BxvdQyu/GMmXK4HK5GDJkiMeva+qSxCsfHyQBj2mtF181Hg0Eaa07ZrDOHCBAa/34\nVWMtgJ+Aslrra486yCWJQohsJZckCqfwqUsStdYXgXigZcqYMs7gaAmsymS1QODSNWPJGM96lrM/\nstG1/wsTWSO5mSeZCeFMntzR8G0gWikVT9oliYFANIBSagJQTmvd+8ryXwAzlVIDgKVAOeAdYK3W\nOmcfWJ7LTJw4kTvvvNPqMmxHcjPPrpmlPIBHCLvK6X3YdFOgtZ6vlCoJjAVKAxuBNlrrI1cWKQNU\nvGr5GKVUISAcmAScAL4BXrrJ2sU15s2bZ3UJtiS5mWe3zEqWLElgYCA9evSwuhQhblpgYCAlS5bM\nkdf26NkHWutpwLRM/i7dWUNa66nAVE+2JbIuMDDQ6hJsSXIzz26ZVapUia1bt5KYmIjWMHIkfP21\n8Xc1a8Ls2ZAvn7U1CpFVJUuWpFKlSjny2vJAJCFErlCpUiUqVapEdHRaQ1C4MHzxBVSrZmlpQviM\nHL8kUQghfMUff0B4eNr8zJnSEAhxNWkKHOTaa8NF1khu5tkxs0uXoGdPSEoy5p96Crp29W4NdszN\napKZd0lT4CA59RmT00lu5tkxs/HjYd06Y/rWW+Hdd71fgx1zs5pk5l2mbl7kLXLzIiFEdvrlFwgN\nhcuXwd8ffv4Z7rjD6qqEyBleu3mREELYTVIS9OhhNAQAI0ZIQyBEZqQpEEI42rBhxgmGAE2awCuv\nWFuPEL5MmgIHufbBNCJrJDfz7JLZ0qUw9codUgoUgI8/hrx5ravHLrn5EsnMu6QpcJBhw4ZZXYIt\nSW7m2SGzf/+FJ59Mm580CWrUsK4esEduvkYy8y5pChwkKirK6hJsSXIzzw6ZRUTA/v3GdOvW8Mwz\n1tYD9sjN10hm3iVNgYPIpTuekdzM8/XMvvkGPvjAmC5UyJhWPvBMVl/PzRdJZt4lTYEQwlHOnIGn\nn06bnzgR5PeKEFkjTYEQwlFeeQV27zam774b+ve3th4h7ESaAgeJjIy0ugRbktzM89XMfv4Z3nvP\nmA4IgA8/BD8f+innq7n5MsnMu3zo20XcrKSUm7oLUyQ383wxs3PnjKsNUm7S+tprxu2MfYkv5ubr\nJDPvktscCyEc4dVXYdw4Y7ppU1i1yrilsRC5jdzmWAiRq23bBm+8YUznzQuzZklDIIQnpCkQQtia\n1sY9CC5eNOYjIqBOHWtrEsKupClwkMTERKtLsCXJzTxfymzuXPj+e2M6JMS3n23gS7nZhWTmXdIU\nOEjfvn2tLsGWJDfzfCWzY8dg6NC0+agoCAy0rp4b8ZXc7EQy8y5pChxk9OjRVpdgS5Kbeb6S2csv\nw5EjxnSnTtCunbX13Iiv5GYnkpl3ydUHQghbWrUKWrQwpgsVMk42LF/e2pqE8AVy9YEQIle5eBEG\nDEibf+01aQiEyA7SFAghbCcqCn791Zhu2BDCw62tRwinkKbAQWbNmmV1CbYkuZlnZWaHDkHKx8xK\nwfvvQ548lpVjiuxr5klm3iVNgYMkJJj66EhcIbmZZ2Vmw4fDyZPG9JNPQpMmlpVimuxr5klm3iUn\nGgohbGPdOrjjDmM6KAj++ANKlbK2JiF8jZxoKIRwvORkeO65tPkxY6QhECK7SVMghLCFjz+GtWuN\n6dq14dlnra1HCCeSpkAI4fNOnoT/+7+0+cmTjQcfCSGylzQFDuJyuawuwZYkN/O8ndm4ccZVBwAd\nO0KrVl7dfLaRfc08ycy7pClwkIEDB1pdgi1JbuZ5M7Pt2+Hdd43p/Pnhrbe8tulsJ/uaeZKZd0lT\n4CCtW7e2ugRbktzM82ZmERFw6ZIxPWyY8SREu5J9zTzJzLukKRBC+KzvvoMvvjCmy5VzP69ACJH9\npCkQQvik5GT3xyK//joULGhdPULkBtIUOEhcXJzVJdiS5GaeNzKbOxc2bDCm69eHXr1yfJM5TvY1\n8yQz75KmwEFiY2OtLsGWJDfzcjqzpCTjdsYp3noL/Bzw00r2NfMkM++S2xwLIXzOa6/ByJHGdPv2\naecVCCFuTG5zLIRwjIMH4Y03jGl/f5g40dp6hMhNpCkQQviUUaPgzBljul8/qFXL2nqEyE2kKRBC\n+IzffoMPPzSmCxeG0aMtLUeIXEeaAgcJCwuzugRbktzMy6nMIiKMSxHBONHQaU9BlH3NPMnMu6Qp\ncBC585dnJDfzciKzFSvgf/8zpitVguefz/ZNWE72NfMkM++Sqw+EEJbTGpo0gfh4Y/7jj6FHD2tr\nEsKu5OoDIYSt/fe/aQ1BvXrQvbu19QiRW0lTIISw1MWL8MorafMTJjjjRkVC2JF86znIypUrrS7B\nliQ387Izs9mzYccOY/qee+DBB7PtpX2O7GvmSWbeJU2Bg0yUu7x4RHIzL7syS0qCMWPS5t94A5TK\nlpf2SbKvmSeZeZc0BQ4yb948q0uwJcnNvOzKLCoK/vnHmO7QAZo1y5aX9Vmyr5knmXmXNAUOEhgY\naHUJtiS5mZcdmR0/bpw/AMY5BK+9dtMv6fNkXzNPMvMuaQqEEJaYOBFOnDCme/WC226zth4hhIdN\ngVIqXCm1Wyl1Vim1RinV5AbL51NKva6U2qOUOqeU2qWU6uNRxUII2/vnH5g82ZjOl8/9vAIhhHVM\nNwVKqS7AW8AooAGwCViqlCp5ndU+A+4DwoDqQDdgu+lqxXVFRERYXYItSW7m3WxmY8fC2bPGdHi4\ncQfD3ED2NfMkM+/K48E6Q4AZWus5AEqpAcBDQF8g3WmiSqkHgbuAKlrrKwcL2edZueJ6KuWWn6zZ\nTHIz72Yy27HD/aFHw4dnU1E2IPuaeZKZd5m6zbFSKi+QBDymtV581Xg0EKS17pjBOlOBW4F4oCdw\nBlgMjNRan8tkO3KbYyEcqmtX+PRTY3rsWBg50tp6hHCam7nNsdkjBSUBf+DQNeOHgBqZrFMF40jB\nOaDDldeYDhQHnjS5fSGEjW3enNYQlCoFQ4ZYW48Qwp03rj7wA5KB7lrr9Vrr/wEvAL2VUvmvt2K7\ndu1wuVwMFs3DAAAgAElEQVRuX6GhocTFxbktt2zZMlwuV7r1w8PDmTVrlttYQkICLpeLxMREt/FR\no0YRGRnpNrZv3z5cLhfbtm1zG58yZUq6z7mSkpJwuVzp7r4VGxub4aM/u3TpIu9D3keuex+DBqW9\nj5dfhi++sOf7cMq/h7wP+7+P2NjY1N+NZcqUweVyMeQmum1vfHwQDTTXWle/aqwm8DtQXWu9M4N1\n5OMDD2zbto2aNWtaXYbtSG7meZLZxo3QoIExXbYs7NwJBQrkQHE+TPY18yQz87z2lESt9UWMcwNa\npowppdSV+VWZrPYzUE4pdfUdKGpgHD3428z2xfUNGzbM6hJsSXIzz5PMRo9Om3755dzXEIDsa56Q\nzLzL1JECAKXU40A0MABYh3E1Qiegptb6iFJqAlBOa937yvIFgS3AGmA0cAvwAfCd1npAJtuQIwUe\n2Ldvn5yp6wHJzTyzmcXHQ+PGxnT58vDnnxAQkEPF+TDZ18yTzMzz5omGaK3nX7knwVigNLARaKO1\nPnJlkTJAxauWP6OUegCYAvwCHAU+BeSc42wm3ziekdzMM5vZ1UcJhg/PnQ0ByL7mCcnMuzy5TwFa\n62nAtEz+Lt1ZEVrrP4A2nmxLCGFvv/wCX35pTFesCE/KNUdC+Cx59oEQIkeNGpU2/corkP+61xwJ\nIawkTYGDXHtJjMgayc28rGa2Zg18/bUxHRwMGVxdlavIvmaeZOZd0hQ4SFJSktUl2JLkZl5WM7v6\nKMGIEcbDj3Iz2dfMk8y8y/TVB94gVx8IYX8//wx33mlMh4TA9u2QN6+1NQmRG3jtPgVCCJFVVx8l\nGDlSGgIh7ECaAiFEtvvpJ/jmG2O6alXo2dPaeoQQWSNNgYNcez9ukTWSm3k3yuzqowSvvgp5PLr4\n2XlkXzNPMvMuaQocpG/fvlaXYEuSm3nXy+z77+G774zpW2+F7t29U5MdyL5mnmTmXdIUOMjoq28b\nJ7JMcjMvs8y0lqME1yP7mnmSmXfJ1QdCiGzzzTfQqpUxXaMG/P47+PtbW5MQuY1cfSCEsNy1RwlG\njZKGQAi7kaZACJEtli837k0AULs2PP64tfUIIcyTpsBBZs2aZXUJtiS5mXdtZnKUIGtkXzNPMvMu\naQocJCHB1EdH4grJzbxrM1u61HjOAUCdOtCpkwVF2YDsa+ZJZt4lJxoKIW6K1nDHHcYjkgH++194\n7DFraxIiN5MTDYUQllmyJK0hqFsXOna0th4hhOekKRBCeOzacwnGjAE/+akihG3Jt68QwmNffAHx\n8cZ0gwbwyCPW1iOEuDnSFDiIy+WyugRbktzMc7lcGR4lUMq6muxA9jXzJDPvkqbAQQYOHGh1CbYk\nuZk3cOBA4uJg40ZjvnFjaN/e2prsQPY18yQz75KrD4QQpiUnGx8XbN5szH/1FbRrZ21NQgiDXH0g\nhPCqhQvTGoI77oC2ba2tRwiRPaQpEEKYkpwMVz+4bvRoOZdACKeQpsBB4uLirC7BliQ3cz77DH7/\n3cgsNBTatLG4IBuRfc08ycy7pClwkNjYWKtLsCXJLesuXzauMgAjM7niwBzZ18yTzLxLTjQUQmTZ\nJ5/AE08Y03feCT/+KE2BEL5GTjQUQuS4y5dh7Ni0eTlKIITzSFMghMiS2FjYvt2YvuceuO8+a+sR\nQmQ/aQqEEDd06VLKuQQGOUoghDNJU+AgYWFhVpdgS5Lbjf3nP/Dnn8b0ffdBdLRk5gnZ18yTzLwr\nj9UFiOzTunVrq0uwJcnt+i5eTH8uwd9/S2aekH3NPMnMu+TqAyHEdX30ETz5pDHdqhUsX25tPUKI\n65OrD4QQOeLCBRg3Lm3+6vMKhBDOI02BECJTMTGwZ48x3aYNNG9uaTlCiBwmTYGDrFy50uoSbEly\ny9iFC/Daa2nzVx8lkMw8I7mZJ5l5lzQFDjJx4kSrS7AlyS1jH30E+/YZ0+3aGU9DTCGZeUZyM08y\n8y450dBBkpKSCAwMtLoM25Hc0jt/HqpVg7//NubXrYMmTdL+XjLzjORmnmRmnpxoKADkG8dDklt6\nH36Y1hA8/LB7QwCSmackN/MkM++SpkAI4ebsWRg/Pm1+9GjLShFCeJk0BUIINzNmwD//GNMdO4J8\ngidE7iFNgYNERERYXYItSW5pzpyBCRPS5jM7SiCZeUZyM08y8y5pChykUqVKVpdgS5JbmmnT4PBh\nY7pzZ6hbN+PlJDPPSG7mSWbeJVcfCCEAOHUKQkLg6FHjCYi//Qa1a1tdlRDCLLn6QAhx06KijIYA\noFs3aQiEyI2kKRBC8O+/8OabxrSfH4waZW09QghrSFPgINu2bbO6BFuS3GDyZDh+3Jju2ROqV7/+\n8pKZZyQ38yQz75KmwEGGDRtmdQm2lNtzO34c3n7bmPb3h5Ejb7xObs/MU5KbeZKZd0lT4CBRUVFW\nl2BLuT23t982Pj4ACAuDqlVvvE5uz8xTkpt5kpl3SVPgIHLpjmdyc25Hj8K77xrTefPCK69kbb3c\nnNnNkNzMk8y8S5oCIXKxN9+E06eN6SefhMqVLS1HCGExaQqEyKUOH4YpU4zpfPmyfpRACOFcHjUF\nSqlwpdRupdRZpdQapVSTG68FSqkWSqmLSilTN1MQWRMZGWl1CbaUW3OLjISkJGO6f3+oUMHMurkz\ns5sluZknmXmX6aZAKdUFeAsYBTQANgFLlVIlb7BeEBADrPCgTpEFSSk/4YUpuTG3AweMWxoDBATA\nyy+bWz83ZpYdJDfzJDPvMn2bY6XUGmCt1vr5K/MK+At4T2s98TrrxQJ/AMnAI1rrTO9fLLc5FiJn\nDRpk3MEQ4IUX4K23rK1HCJF9vHabY6VUXqAR8E3KmDa6ihVA6HXWCwNCgDFmtieEyH67dxuPRwYo\nWBDkMnAhRIo8JpcvCfgDh64ZPwTUyGgFpdStwHjgTq11snFgQQhhldGj4eJFY3rwYChd2tJyhBA+\nJEevPlBK+QH/AUZprXemDGd1/Xbt2uFyudy+QkNDiYuLc1tu2bJluFyudOuHh4cza9Yst7GEhARc\nLheJiYlu46NGjUp3Qsu+fftwuVzpbrM5ZcqUdM/4TkpKwuVysXLlSrfx2NhYwsLC0tXWpUuXbH8f\niYmJjngf4N1/j8TEREe8D7jxv8fvv8PHHxvj+fOHU7q0Z+8jMTFR9isP3kdiYqIj3gd4798jZR27\nv48U2f0+YmNjU383lilTBpfLxZAhQ9Ktk2Va6yx/AXmBi4DrmvFoYFEGywdhnENw4cp6F4HLV43d\nm8l2GgI6Pj5ei6x7+OGHrS7BlnJTbh07ag3G1xtveP46uSmz7CS5mSeZmRcfH68BDTTUJn7Ha63N\nHSnQWl8E4oGWKWNXTjRsCazKYJWTQB2gPlDvytf7wLYr02vNbF9c3+jRo60uwZZyS27r1sGiRcZ0\n2bLGyYaeyi2ZZTfJzTzJzLvMnlMA8DYQrZSKB9YBQ4BAjKMFKKUmAOW01r211hrYcvXKSqnDwDmt\n9dabKVykJ1dqeCa35DZ8eNr0yJEQGOj5a+WWzLKb5GaeZOZdppsCrfX8K/ckGAuUBjYCbbTWR64s\nUgaomH0lCiFu1jffGF8AVaoYtzQWQohreXKkAK31NGBaJn+X/qwI978fg1yaKITXaO1+lGDsWOO2\nxkIIcS159oGDXHumrMgap+f2+efG+QQAdepA1643/5pOzyynSG7mSWbeJU2BgyQkyCMlPOHk3C5f\ndn/Q0euvg7//zb+ukzPLSZKbeZKZd5m+zbE3yG2Ohcgec+ZA797GdGgo/PwzyP3DhHA2r93mWAhh\nH+fPw6hRafPjx0tDIIS4PmkKhHCo6dNhzx5j+oEH4N57raxGCGEH0hQI4UAnTsC4cWnzb7xhXS1C\nCPuQpsBBMrp/t7gxJ+Y2fjwcO2ZM9+gB2X1qjhMz8wbJzTzJzLukKXCQgQMHWl2CLTktt7174b33\njOn8+eG117J/G07LzFskN/MkM++SpsBBWrdubXUJtuS03EaMME4yBHjuOQgOzv5tOC0zb5HczJPM\nvEuaAiEcZMMGmDvXmC5e3P1OhkIIcSPSFAjhEFrD1Y9pHzkSiha1rh4hhP1IU+AgcXFxVpdgS07J\nbenStIcehYTAM8/k3Lackpm3SW7mSWbeJU2Bg8TGxlpdgi05IbfLl92PEkyYYJxkmFOckJkVJDfz\nJDPvktscC+EAs2dD377GdJMmsHat3L1QiNxKbnMsRC52+rRxxUGKSZOkIRBCeEaaAiFsLjIS/vnH\nmHa54O67ra1HCGFf0hQIYWN79xpHBgDy5k2bFkIIT0hT4CBhYWFWl2BLds5t2DA4d86Yfu45uPVW\n72zXzplZSXIzTzLzLmkKHETu/OUZu+b2008wf74xfcstxn0JvMWumVlNcjNPMvMuufpACBtKTjau\nMki4cl7xzJnw9NPW1iSE8A1y9YEQuUx0dFpDUK9e2uWIQghxM6QpEMJmTp50f6bB5Mng729dPUII\n55CmwEFWrlxpdQm2ZLfcxo+HQ4eM6cceg3vu8X4NdsvMV0hu5klm3iVNgYNMnDjR6hJsyU65bd8O\nb79tTOfPD2++aU0ddsrMl0hu5klm3iVNgYPMmzfP6hJsyS65aQ2DBsHFi8b80KHGg4+sYJfMfI3k\nZp5k5l3SFDhIYGCg1SXYkl1y++9/YflyYzo4GF55xbpa7JKZr5HczJPMvEuaAiFs4NQpGDIkbX7y\nZJCflUKI7CZNgRA2MHYs7N9vTD/0kPGMAyGEyG7SFDhIRESE1SXYkq/n9vvv8O67xnT+/MZRAquf\ngujrmfkqyc08ycy7pClwkEqVKlldgi35cm5aw7PPwqVLxvzLL0PVqtbWBL6dmS+T3MyTzLxLbnMs\nhA+bOxd69jSmq1aF336DgABraxJC+Da5zbEQDnT8OLz4Ytr8lCnSEAghcpY0BUL4qIiItDsXduwI\nbdtaW48QwvmkKXCQbdu2WV2CLflibt99B7NmGdOFCxtHCXyJL2ZmB5KbeZKZd0lT4CDDhg2zugRb\n8rXczp6Ffv3S5iMjoXx56+rJiK9lZheSm3mSmXdJU+AgUVFRVpdgS76W27hx8OefxnSLFtC/v7X1\nZMTXMrMLyc08ycy7pClwELl0xzO+lNumTZDy/Jd8+WDmTPDzwe9SX8rMTiQ38yQz7/LBHzdC5E6X\nL8PTTxt/AgwfDrVrW1uTECJ3kaZACB8xeTL88osxXbs2vPSStfUIIXIfaQocJDIy0uoSbMkXctu+\nPe2ph0rBBx8YtzS22vlL59lyZAvf7v6WBVsW8EH8B0Sti6LDwA7MWD+DOZvm8L8//8eGAxs4fOYw\nvngzNF/iC/ua3Uhm3pXH6gJE9klKSrK6BFuyOrdLl6B3bzh3zph/7jlo3tz7dZw6f4p1+9ex6q9V\nrNm/hq1HtrLnxB40Gfyi/w0+/+rzdMPFCxTntltu4/ZSt9OiUgvuCb6H8kV87NIJC1m9r9mRZOZd\ncptjISw2YYJx/gBA9eqwYYP3Hov8x9E/+GL7F3zxxxes3LeSy/pytm+jarGqtK/eno41O3JnpTvx\n9/PP9m0IIdLczG2O5UiBEBb69VcYNcqY9vODmJicbwiOJh0l9rdYojdGE38gPtPliuQvQo0SNahe\nojoVilSgRIESFCtQjAJ5CnBZXyZZJ3P6wmkOnT7EoTOH2HNiD78f+Z1/Tv3j9jo7j+9k8trJTF47\nmZKBJel6W1eebPgk9cvUz9k3KoQwTZoCISxy4QL06gUXLxrzw4ZBs2Y5t701f6/hnTXvsGjrIi4m\nX0z399WKV6NVSCtaVGpB84rNCSkagvLgGc3Hzx4n/kA8P+z5gR/2/sCav9ekbi8xKZGoX6KI+iWK\nhmUb8mzjZ3mi7hME5JGHOgjhC+TjAwdJTEykZMmSVpdhO1blNmoUjB1rTNepA+vXZ//Jhck6mbht\ncby1+i1W/bUq3d83LteYx2s/zsM1HqZGiRpZbgLMZHbi3AmW7FjCwq0L+WrHV5y7dM7t70sXLM1z\ndzzHM42foViBYll6TbuS71HzJDPz5CmJAoC+fftaXYItWZHbunXw+uvGdJ48xscG2dkQaK2J2xZH\nvffr8dj8x9waglsCb2Fo6FA2D9jML0//QkSLCGqWrGnqqICZzIoGFKX77d357+P/5eDQg0x/aDqN\nyjZK/ftDZw7xyrevUOndSoz8diT/nvs3y69tN/I9ap5k5l1ypMBBEhISJC8PeDu3U6egQQPYudOY\nHzUKRo/OntfWWrN813JGfDuCX/75xe3vbrvlNl4IfYHut3e/6cP12ZHZ2r/XMmn1JBZuXUiyTk4d\nL16gOC+1eImBTQdSIG+Bm9qGr5HvUfMkM/Nu5kiBNAVCeFnv3jBnjjF9xx3w00+QN+/Nv+72xO0M\nXjqY//35P7fxO8rfwah7RvFgtQc9Okcgp+08tpNJqybx4YYPuZR8KXW8fOHyvPnAm3St09Un6xbC\nV8nHB0LYxCefpDUEhQsb8zfbEJw8f5KIZRHUmV7HrSGoV7oei7suZvWTq2l7a1uf/cVatXhVpref\nzvaB2+lZtycKo879p/bTfWF37o25l82HNltcpRC5gzQFQnjJ7t3wzDNp89OnQ5Uqnr+e1ppPfv2E\n6lOqM2n1pNT/ZVcsUpHYx2JJ6J/AwzUe9tlm4FpVilVhTsc5bH5mMw/d+lDq+I97f6TBjAY89/Vz\nnDp/ysIKhXA+aQocZNasWVaXYEveyO3SJXjiCTh50pjv0cOY99TfJ//m4diHeWLhExw6cwiA/P75\nefXuV9k2cBtd63TFT+Xct3dOZlanVB2+7P4lX3X/imrFqwHGVRRT1k3htmm38fWOr3Ns2zlNvkfN\nk8y8y6OfGkqpcKXUbqXUWaXUGqVUk+ss21EptUwpdVgp9a9SapVSqrXnJYvMJCSY+uhIXOGN3F59\nFVavNqarVIGpUz17nWSdzIz1M6g9tTZf7fgqdbxDzQ5sDd/KmPvGEJg352+H6I3M2t3ajt+e+Y0J\nLSekvqe/Tv5Fu0/a0WtRL44mHc3xGrKbfI+aJ5l5l+kTDZVSXYAYoB+wDhgCdAaqa60TM1j+HWA/\n8B1wAugLvAg01VpvymQbcqKhcIwvv4SHHzam/f3h55+NEwzN2nNiD2Gfh/H9nu9Tx8oUKsP0h6bT\noWaH7CnWR+0+vpt+X/Zjxa4VqWO3BN7C++3f59Faj1pYmRC+x9snGg4BZmit52ittwEDgCSMX/bp\naK2HaK0naa3jtdY7tdavADuAhz3YthC2sns39OyZNj9xovmGQGvNx5s+pu70um4NQd/6fdny7BbH\nNwQAIcVCWNZjGbNcswjKHwTAkaQjPDb/Mfp+3lfONRAim5hqCpRSeYFGwDcpY9o41LACCM3iayig\nMHDMzLaFsJtz56BzZzhxwpjv2BGGDDH3GsfPHqfrgq70iuvFqQvGL77goGCW91zOrEdmOf4OgFdT\nStG3QV+2hLs3QrM3zqb+jPoZ3rFRCGGO2SMFJQF/4NA144eAMll8jQigIDDf5LaFsJUhQyD+yvOG\nqlaF2bPBzIUA3+7+lrrv12X+72nfKr3r9WbzM5tpVaVVNldrH+UKl2Ph4wuJfiSaQvkKAbDr+C7u\nmn0Xr373Khcvp3+ugxAia7x69YFSqjswEuic0fkH12rXrh0ul8vtKzQ0lLi4OLflli1bhsvlSrd+\neHh4ujNXExIScLlcJCa6b37UqFFERka6je3btw+Xy8W2bdvcxqdMmUJERITbWFJSEi6Xi5UrV7qN\nx8bGEhYWlq62Ll26ZPv7cLlcjngf4N1/D5fLle3vY+5ceP99gFHkyRPJggUQFJS193Hh8gUilkXQ\nak4r/k78Gz6BwgcLM7/TfKI7RFMkfxHL/z1cLpel+5VSit71e7NpwCbK/FAGEoyTMMf9OI47Z9/J\n4u8XW75fZfQ+XC6X7b4/Mnof4L3v85Rt2P19pMju9xEbG5v6u7FMmTK4XC6GmD0keTWtdZa/gLzA\nRcB1zXg0sOgG63YFTgMPZmE7DQEdHx+vRdYtXbrU6hJsKbtz++UXrQMCtAbj66OPsr7u3hN7dbMP\nm2lGk/p1f8z9+q9//8rWGm+WL+1rFy9f1ON+GKf9x/inZhY0IUgv2LLA6tLS8aXc7EIyMy8+Pl4D\nGmioTfyO11p7dPXBGmCt1vr5K/MK2Ae8p7V+M5N1ugEfAl201l9mYRty9YGwpQMHoEkT2L/fmH/y\nSfjww6ytu2THEnou6smxs8bpNvn88zH+/vEMCR2So/cccIp1+9fRfUF3dh7fmTo2sMlAJrWeRP48\n2fz4SSF8mLevPngbeFop1UspVRN4HwjEOFqAUmqCUiomZeErHxnEAEOBX5RSpa98FfFg20L4rPPn\n4dFH0xqC5s2zdj+CS8mXGP7NcB765KHUhqBy0cr83PdnhjYfKg1BFjUt35SE/gk8ftvjqWNRv0TR\n/KPm7Dy28zprCiFSmP5po7Wej3GfgbHABqAu0EZrfeTKImWAilet8jTGyYlTgX+u+nrX87KF8C1a\nw4ABsGaNMV+hAixceOPHIR84dYBWc1oxYeWE1LFHajxCQr8EGpdrnIMVO1OR/EWY99g8pj80nfz+\nRvgJBxJoOLMhn/3+mcXVCeH7PPoviNZ6mta6sta6gNY6VGu9/qq/C9Na33/V/H1aa/8MvuQh2dns\n2hNoRNZkR27vvgvR0cZ0gQLw+edQuvT11/l297c0mNGAH/b+AIC/8mfSA5NY1GWRz19q6Mv7mlKK\nAY0HsOapNdxa/FbAeGjU4/99nGe/epZzl85ZVpsv5+arJDPvkuOSDhIbG2t1CbZ0s7l9/jm8+GLa\nfHQ0XO9UmGSdzLgfxvHAxw+kPregfOHy/NDnB4Y2H2qLBxjZYV+rX6Y+8f3i6X5799Sx6eun03yW\ndR8n2CE3XyOZeZfpEw29QU40FHaxZg3cfz+cPWvMjxgB48ZlvvyRM0fouagnS3cuTR1rU7UNH3f8\nmFsK3pLD1eZOWmtmbZjFoK8HpR4lKJK/CLMfmS23SBaO5O0TDYUQwJ9/Gs80SGkInngCxo7NfPlV\nf62iwYwGqQ2Bn/Jj3H3jWPLEEmkIcpBSiqcaPsXap9a6fZzw2PzHGPy/wVy4fMHiCoXwHdIUCOGB\nI0fgwQch5R4m990HH32U8R0Ltda8teot7om+h/2njEsTShUsxfKeyxlx9wi5usBL6pauS3y/eLrW\n6Zo6NnntZO6afRd7T+y1sDIhfIf8NBLCpDNnjCMEO698LH3bbcaVBvnypV/2+NnjdPy0Iy8uf5FL\nyZcAuCf4Hjb238j9IfenX0HkqML5C/PJo58wrd008vkb/2Dr9q+jwYwGfLH9C4urE8J60hQ4SEa3\nwxQ3Zia38+eNBxutXWvMlysHX38NRYumXzb+n3gazWzE59s/Tx0bfudwVvRaQdnCZW+2bEvZeV9T\nSvFMk2dY1XcVVYpVAeD4ueO45rkYtnxYjj47wc65WUUy8y5pChykdevWVpdgS1nN7dIl6NoVli83\n5osUgSVLoGJF9+W01kz7ZRrNP2rO7hO7ASheoDhfdf+K11u+Th6/PNlZviWcsK81KteI+H7xdKzZ\nMXXszVVvcl/Mffx98u8c2aYTcvM2ycy75OoDIbIgORl69zYedAQQGAjLlkGLFu7LnTx/kn5f9OPT\n3z9NHbuj/B3M7zyfSkGVvFixyCqtNZPXTiZieUTqRzwlA0syt+Nc2lRrY3F1QpgnVx8IkYO0hvDw\ntIYgXz7j3gTXNgSbDm6i8czGbg3B4DsG82PYj9IQ+DClFIObDeansJ9S/50SkxJp+5+2jPx2JJeT\nL1tcoRDeI02BENehNbzwQspjkMHfH+bPh1atrl5GMythFs1mNWPHsR0ABOUPYuHjC3nnwXdST2gT\nvq1ZhWZs6L+B9tXbA6DRvPbTa7T6uBUHTh2wuDohvEOaAge59lncImsyyy05GQYONG5hDMblhjEx\n8MgjacucuXCGPp/34akvnkq9MU7Dsg2Nz6prdczgVZ3Bqfta8QLF+bzr50S2isRf+QPw/Z7vaTCj\nAd/u/vamX9+pueUkycy7pClwkIkTJ1pdgi1llFtysvGAo2nTjHmlYNYs4wZFKbYc2ULTD5syZ9Oc\n1LFnGz/Lz31/pmrxqjldtqWcvK/5KT+GtRjG932+p1zhcgAcOnOIBz5+gHE/jLupjxOcnFtOkcy8\nS040dJCkpCQCAwOtLsN2rs3t8mV46qm0Bxz5+RlHCHr0MOa11sRsiiF8SThJF5MAKJSvEB88/IHb\njXGcLLfsa0fOHKHHoh4s27ksdeyBKg8w99G5lCpYyvTr5ZbcspNkZp6caCgA5BvHQ1fnduGC8cs/\npSHw94dPPklrCE6cO0G3Bd0I+zwstSG4vdTtrH96fa5pCCD37Gu3FLyFr5/4mnH3jUu98+TyXctp\nMKMBP+39yfTr5ZbcspNk5l3SFAhxxalT8NBDMG+eMZ8nj3FSYZcuxvyqv1ZR//36blcXPNXgKdY8\ntYYaJWtYULHwBj/lx4i7R7Ci5wpKFzSeh/3PqX+4L+Y+IldGkqyTLa5QiOwjTYEQwKFDxvMLVqww\n5gMCYNEiePRRuJx8mXE/jOPu2Xez91/jHvlB+YOY32k+H7g+IDCv/E8mN7gv5D42DtjIvZXvBeCy\nvsxL37xEu/+0459T/1hbnBDZRJoCB4mIiLC6BFt6+ukIWrSA+Hhjvlgx+OYbaN8e9v27j/ti7uPV\n71/lsjZOMGtRsQWbBmyi822dLazaWrl1XytTqAwreq5g5N0jURhPv1q6cym3T7+dBVsW3HD93Jrb\nzZDMvEuaAgepVElukGPWzz/DvHmVUh9uVLGiMRYaqpm7eS713q/HT/uMz479lB+j7xnN932+J7ho\nsIVVWy8372v+fv6MvW8sS3sspWwh4xkWx84eo9NnnegT14eT509mum5uzs1Tkpl3ydUHIteKjoZ+\n/cm9uu8AABcSSURBVODileff3HYb/O9/kK/YYZ756hkWbl2YumyloEr859H/cGelO60pVviko0lH\n6f9lfxZsTTtKEBwUzMcdP+au4LssrEzkZnL1gRAmXL4MEREQFpbWELRsCT/9BL+cWkSdaXXcGoLu\nt3dnY/+N0hCIdEoEluCzzp8R0yGGwvkKA7D3373cE30PL614ifOXzltcoRDmSFMgcpUTJ4w7Ek6a\nlDYWHg6xi07w3Pc9eXT+oxxJOgJAiQLGD/z/PPofihUoZlHFwtcppehVrxebn9nMXZWMowMaTeTP\nkTSY0YA1f6+xuEIhsk6aAgfZtm2b1SX4tIQEaNQIvvrKmPf3N+5YWPuhaTT4oA5zN89NXfaRGo/w\n+7O/06l2J4uq9W2yr6VXuWhlvuv9HZGtIsnrlxeArYlbaT6rOS8sfYGki0mSmwckM++SpsBBhg0b\nZnUJPklrmDkTmjeHXbuMsWLF4JMvDvBNiU6EDw5n/6n9gHGpYUyHGBZ1WUTpQqUtrNq3yb6WMX8/\nf4a1GEZ8v3gal2sMGEcN3lnzDrdPv52+A/taXKH9yL7mXdIUOEhUVJTVJficM2egd2/o3x/OX/l4\nt3GTZCLmzaTfplrGCWLtjPE2Vdvw6zO/0qteL5RS1hVtA7KvXd/tpW9n9ZOrmdhqIgF5AgDYdXwX\nq29fTf8v+nPs7DGLK7QP2de8S5oCB5FLd9ytXQsNGsDHH6eNPfH8NvL3v5fhq/vz7/l/Abil3C38\n59H/8PUTX1MxqKJF1dqL7Gs3lscvDxEtItg0YFPquQYUhZkJM6kRVYPZG2bL3RCzQPY175KmQDjO\npUswejS0aAE7dhhjBYuf5OH3hjG/RF1+/jvtnvV96vdha/hWut/eXY4OiBxRvUR1vu/zPVPbTaVQ\nvkIAJCYl0ndxX+6afRebDm6yuEIh0khTIBzljz/gzjthzBjj0kNUMlU6fkzgsBp8cexNLiYb1yBW\nLVaVb3p9w+xHZlMisIS1RQvH81N+PNvkWbaFb6PLbV1Sx1f9tYqGMxsy+H+D+ffcvxZWKIRBmgIH\niYyMtLoEy1y8CBMmQN26xscGAH7l46kw6k521evFkXMHAcjvn5+Rd4/k12d+5f6Q+4HcnZunJDPP\nzJ0+l3md5rG853Kql6gOQLJOZvLayVSPqs7769/nUvIli6v0LbKveZc0BQ6SlJRkdQmWWLcOGjeG\n4cOvnEwYtI/CvXqjn27C36xOXa5DzQ5sCd/C2PvGUiBvgdTx3JrbzZDMPJOSW6sqrdg8YDOv3/86\nBfIY++LhM8adNOu9X48lO5bgi3ebtYLsa94ltzkWtnXihHHuwJQpkJwMFDgGd03AP3QKl1XaneRq\nlKjBe23fo3XV1pbVKkRm9pzYw7Dlw/hsy2du462qtGLSA5OoV6aeRZUJu5LbHItc5fJl+OADqF4d\nJk+GZL+z0GIi/kOqQvNJqQ1BsYBiTHpgEpuf2SwNgfBZlYtWZn7n+fzc92eaVWiWOr5i1woazGhA\n9wXd2Z643cIKRW4iTYGwlZUroUkT40FGR46fhTsmw/NV4YH/43K+E4Bx3sCw5sPY+dxOhjYfSj7/\nfBZXLcSNNa/YnFV9VzHvsXlULloZMG58FPtbLLWn1aZPXB92HttpbZHC8aQpcJDExESrS8gxW7dC\np05w112w4bckaPYOPF8F2g6GwgcA4wzvsPph7Bi0g8gHIrP8vAIn55ZTJDPP3Cg3pRRd6nRha/hW\nJj0wiZKBJQHjZMSYTTHUnFqTpxc/zZ4Te7xQrW+Qfc27pClwkL59nXcL1T17oE8fqFMHFnx5EppP\nMpqBB1+AwgdTl+tYsyObBmzio0c+Mn0DIifmltMkM89kNbeAPAEMbT6UXc/t4vX7X6dYgNHgXkq+\nxIcbPqTae9XosbAHmw9tzslyfYLsa94lJxo6SEJCgmPy2rcPJk40nllwscBfcMd70GgmBJx0W+6x\nWo/x6j2vUrd0XY+35aTcvEUy84ynuf177l/eWfMO76x5h5Pn3b8H2t3ajv9r8X/cVekuR96AS/Y1\n827mRENpCoRP2bYNIiNh7ly4VHIDNH8LbvsU/NOu3VYoOt/WmZF3j6ROqToWViuEdx07e4yodVG8\nt/Y9jp496vZ3d5S/g0FNB9H5ts5yHk0uJ02BsL21a40jAwsXn4daC6Dx+xD8k9sy+f3z07NuT14I\nfYFat9SyqFIhrHfmwhk+2vARb61+i73/7nX7u9IFS9OvUT/6N+pP+SLlLapQWEmaAmFL587B/PkQ\nFQW/7PzT+Hig/mwo6H5iUYkCJQhvEs6zTZ6VxxkLcZWLly/y6e+f8uaqN9OdX+Cv/OlYqyNPN3ya\nliEt8ffzt6hK4W1ynwIBwKxZs6wuIUv27YNXXoGKVU/T++05/FKrFTx36/+3d+/hUdVnAse/b5LJ\nhUlIQhJIQiAxEIIRkUtXFlbQeikSKuttqfXS1a6tdGtb3fXxWZ+t7dpn6VrW7cXuqrt1xbrWC9hd\nH8VltYqAQhAMAcSAUAMJgXDJdcJMMjOZ+e0f5yQkQC4zkswMvB+e8yRzcs6Zd37P4Zx3fud3gT/7\n5z4JQVlOGU8vfpq6B+t47MuPDVtCECvlFk20zMJzrsvNEe/gzul3suO+HWy8eyNLL1lKvFg3/4AJ\n8Fr1ayx8cSFFvyrih+t+yB+b/3hO338k6Lk2sjQpOI9s3x5SQjiiOjrgpZfg2usCFF79Dj/dcxeN\nd4+Dm/4Sit/r2S4xPpHbL72djXdvZPd3drPsS8sY5Rg1rLFFc7lFKy2z8AxXuYkI8wvn8+qtr1L7\nQC0/WvAjxjlPJdH1rnqWf7Cckl+XsGDlAp6reo7WztZhieVc03NtZOnjAzVsgkGoqIDfvhDkdxs3\n4yn8PUx7tWdcgd4mZU7ivtn3cfeMu8lx5kQgWqXOL76AjzX71rByx0rW7l9LwAT6/N0R52Dh5IUs\nLVvKktIlpCenRyhSda5pmwIVNboTgVdW+3l58waaxv4epr7eZ0yBbhlJmdw27WvcddldzC2Ye152\np1IqGjS0N/DirhdZuWMlexr3nPH3xPhErp98PbdefCvlJeU6nXiM06RARZTPBxs3wqo1jbxW9TYt\n2Wth8loY1XzGtgniYPGUxXzjsrtYXLKYpISkCESs1IXJGMO2I9t4+ZOXWV29msPth8/YJk7imFsw\nlxum3MBXp3yVspwyTdhjjCYFasQdOQJr3grw0oZtbD6xFv/E/4Px20DOPJ8ckszCSdezdNrN3FB6\nAxnJGRGIWCnVW9AEqThUwapPV7G6ejUNJ898rAfWhE3lk8u5tvhariq6asjDh6vI0aRAAbBkyRLe\neOONYTl2Wxu8v6GLVR/sYMPB9RxxbLDGEUhuO+v2yZLGopJyvj79FhaVLCI1MXVY4joXhrPczlda\nZuGJ1nILmiCb6jbx5r43WbNvzVkfMYA1cNisvFlcc9E1XH3R1Vwx8Qqcic5hjS1ayyyafZGkIGF4\nQlKRcP/995+zYzU1wfpNblZvquTDg1s47NgAEz+A1HboZxDBiUmXcuO0Rdw0bRHzJsyLmVHVzmW5\nXSi0zMITreUWJ3HML5zP/ML5rLhuBZ83f85b+99izb41rD+4Hn/QD1izNlY2VFLZUMmKzStwxDmY\nmTeTeQXzmDthLvMmzKNgdME5jS1ay+x8pTUFCp8PKqu6eKOimvf3b2VP20e40rbC2N0QF+x3v1Em\nmzm5V3Lbl66nfMr15/xioJSKvHZvOxtqN7DuwDreO/DeoJMwFYwuYN6EecwZP4dZebOYkTtDHxmO\nMH18oIasrQ02Vbbwzs5P2Fa3i/2uXTTG78LkfAKJngH3TQmOZWbmldw44yrKy67UBkhKXYBOuE/w\n/sH3WXdgHRtqN7C3ce+g+1yUcRGz8mYxM3cmM/Nmctm4y8hPy9frxzDRpECdoa3NUPHJcTbt2UfV\noc/Y27iPI75qOkbvgvRDgx/AxJETvJTLsi+nfMblLCq7gtKsUv1PrJTqo7mjmS31W6g4VMHm+s18\nVP8Rbr970P1GJ42mLKeMi7MvpiynrOf3woxC4kTH1fsitE3BBaqlNci2vQ1U1dTySV0tWyrepD0/\nnhbZhz/9s1ONAOOBQUYITvUXU+KczYJJc1gy+3LmTJg17A2IosXrr7/OjTfeGOkwYoqWWXjOx3Ib\nkzKG8pJyykvKAegKdrH7+G4qj1RSdbSK7Q3b2XlsJx5/35pIl9fFlvotbKnf0md9SkIKk8dMpjiz\nmEmZk2jf1c7NN91McWYxRRlFMdNWKVZpTUGU8noN++pa2V3bwN4jR9jfcJia5loaPLU0Bw/iSawl\nmHoIEnyndnoWuHfg48b7R5NjplOaMZ05hdP5yozpXF44jbSktGH9PNFs7ty5VFRURDqMmKJlFp4L\ntdwCwQD7m/ezvWE7VQ1VfHriU6pPVJ8xw+NZ9bquCcKE9AkUZRRRMLqAgrQC62evZaxz7AU/+dOI\n1xSIyHeBh4BcYCfwPWPMtgG2vwr4F+ASoA5Yboz5bTjvHcvcngA1DS3UHG2k9ngTh1uaONTcyOHW\nBo51NNDib6CdBrwJDQSdDZDg7XuAUfbSn+6/GSGps5BsmUJR6hQuyS1lXukUFpSVUpQ5UR8BnCYn\nR4dVDpWWWXgu1HKLj4tnavZUpmZP5fZLb+9Z7/a52du4lz2Ne6g+UU31iWr2NO7hQMuBnh4Pva95\nBkNdWx11bXX9vldCXAL5afnkpuYy1jmWcc5xjHWO7bN0r8salUVCnFaY9xZyaYjI17Bu8N8GtgIP\nAm+LyBRjTONZti8C1gBPAbcD1wLPisgRY8wfwg995AUChhOtHRxudHG0xcXR1jaOt7lobG+j2e2i\nxeOitbONNm8rrb4mTgaa6KARb3wTXYlNkNxy1sF9cNhLCMQ3GqeviMy4QvJHFVE8ppAd6a/w7B0r\nmVk4iRRHyjn5zEopNVyciU5m589mdv7sPusDwQCH2w9T01LDA+seYPEVi6lpreHz5s+paamhqaOp\n32N2BbsGTRy6CUJGcgaZKZlkJmf2/ByTMqbP6+6fGckZpCamkpaURmpiKqmJqedd+4dwUqQHgX83\nxrwAICLLgMXAN4EVZ9n+O0CNMeZh+/VnInKFfZwvnBQYAz5/AJfHy8kOHyc7vJzs9OHp9HGy04vH\n68PV0Umb201bh4f2Tms56fXg9nnw+Dx4/B46Ah46Ax68QQ++oAcfHvy48Yubrrh2go42TKIL4rsG\nDigOSLGXMMV1ZpHkzyONPDIdeYxNyaNwTD5l4wuZXVzE7MmFZDnP7OKz5A8bmDe5n0EElFIqRsTH\nxTMxfWLPsvya5X3+7vF7OOw6TL2rvu/SXs+htkPUu+pp9DRiGPjxuMHQ0tlCS2dL2LE6Hc4+iUJa\nYtqp1w4rcUhxpJCSkEJyQnLP7ykO+7X9e3/rkhKScMQ5RqyGN6SkQEQcwGzgp93rjDFGRN4F5vaz\n258C75627m3gF4O934In7oVxDgL4CIqXoPgwcV6CcdZP4n0Q7x2wL/2g4oBhHn5fvOkk+LNIDmQz\nSrJIi88iIymbrJQsctOzKMrOZUpeHpdMzKN0fC7JDp0PQCml+jPKMYqSrBJKskr63aYr2EWTp4nj\n7uMcdx/nmPtYz++9Xzd3NNPSYSUGQRP6vcTtd+P2uznmPvZFPtKgHHEOEuMTe5akhKQ+r3svHXUd\nYb9PqDUF2Vht2U//9MeA0n72ye1n+9EikmSM8Z5ln2QAt7cKBu46P/wCDqTLSXwglYRgKg6cJOEk\nOT6V5PhUnIlOUh2pjE5yMjollfRRTrLSUskfk8H4rHTyx6TjSBhaMQeam6hu7r9abDBbt27VucfD\noOUWOi2z8Gi5he5clFmm/W9q4lRIBM4yfUPQBHH73LT72nF1unD5XLR723F5Xbi8Ltp97bh9VgLg\n8Xt6FrfPft1l1Tx3BQepTQ6T3/7nZvDunpx6kJ8c6vtEawuLIgD+O7JBWPwYWumilS6gE2iPdEgD\nsFucqhBpuYVOyyw8Wm6h0zILWxGwOZQdQk0KGoEAZ/Z6Hwcc7Wefo/1s7+qnlgCsxwt3AAex7sNK\nKaWUGppkrITg7VB3DCkpMMb4RaQSuAZ4A0Cs1g/XAE/2s1sFsOi0dV+x1/f3Pk3AS6HEppRSSqke\nIdUQdAunL8XPgW+JyDdEZCrwDFZP0ucBROSfRKT3GATPAMUi8jMRKRWRvwZutY+jlFJKqSgRcpsC\nY8wqEckGfoL1GGAHsNAYc8LeJBeY0Gv7gyKyGKu3wfeBeuCvjDGn90hQSimlVARF5TDHSimllBp5\n59dQTEoppZQKmyYFSimllAJiJCkQkcUiskVEPCLSLCJRMYJBtBORRBHZISJBEZke6XiimYgUisiz\nIlJjn2f7ReQf7FE8VS8i8l0ROSAiHfb/yz+JdEzRSkQeEZGtIuISkWMi8j8iMiXSccUSEfk7+xqm\njdMHISL5IvJfItJoX8d22rMOD1nUJwUicgvwAvCfwKXAPLS74lCtwGrYqQ1HBjcVEOBbQBnW3BzL\ngOUD7XSh6TUh2o+BmVizpL5tNz5WZ5oP/BqYgzUZnAN4R0R0xrIhsBPOb2OdZ2oAIpIBbAK8wELg\nYuBvgZAmdojqhoYiEo81gNGjxpjnIxtNbBGRRcATwC1ANTDDGLMrslHFFhF5CFhmjJkc6ViihYhs\nAT4yxvzAfi3AIeBJY8zZJkRTvdjJ03FggTHmw0jHE81EJBWoxJpU71GgyhjzN5GNKnqJyOPAXGPM\nlV/kONFeUzALyAcQke0ickRE/ldELolwXFFNRMYB/wHcCYQ/M4bKAJojHUS06DUh2nvd64z1rWKg\nCdFUXxlYNXd6Xg3u34A3jTHrIh1IjLgB+FhEVtmPqraLyL2hHiTak4JirCrdH2ONi7AYqypkvV1V\nos5uJfCUMaYq0oHEKhGZDNyPNfiWsgw0IVruyIcTW+xalV8CHxpjqiMdTzQTkduAGcAjkY4lhhRj\n1ap8hjVq8NPAkyJyVygHiUhSYI96GBxgCdiNcbrj+0djzOv2Te4erEz7LyIRe6QMtcxE5PtAKvCz\n7l0jGHbEhXCu9d5nPLAWeNUY81xkIlfnoaew2qvcFulAopmIFGAlT3cYY/yRjieGxAGVxphHjTE7\njTG/AX6D1TZqyCI1S+ITWN9mB1KD/egA2NO90hjjE5EaYOIwxRathlJmB4AvY1Xleq0vJj0+FpHf\nGWPuGab4otVQzzXAar0LrMP6NnffcAYWg8KZEE0BIvKvQDkw3xjTEOl4otxsIAfYLqcuYvHAAhG5\nH0gy0dwYLnIa6HWvtO0Bbg7lIBFJCuwJj5oG286efMkLlGJP7mA/1ywCaocxxKgTQpl9D/j7Xqvy\nsWbKWgpsHZ7ootdQyw16agjWAduAbw5nXLEozAnRLnh2QvDnwJXGmLpIxxMD3sXqadbb81g3uMc1\nIejXJqx7ZW+lhHivjFRNwZAYY9pF5BngMRGpx/pwD2M9Plgd0eCilDGmvvdrEXFjPUKoMcYciUxU\n0c+uIViPVdvyMDC2+0uKMeb0Z+gXsp8Dz9vJwVasrps9E6KpvkTkKeDrwBLAbTcCBmgzxui08Gdh\njHFj9ZjqYV/Hmowxp38TVqf8AtgkIo8Aq7C6wd6L1c16yKI6KbA9BPixxipIAT4CrjbGtEU0qtii\nmfXgrsNqqFOM1cUOrGTKYFVdKoY0IZrqaxnWObT+tPX3YF3T1NDoNWwQxpiPReQm4HGsLpwHgB8Y\nY14J5ThRPU6BUkoppUZOtHdJVEoppdQI0aRAKaWUUoAmBUoppZSyaVKglFJKKUCTAqWUUkrZNClQ\nSimlFKBJgVJKKaVsmhQopZRSCtCkQCmllFI2TQqUUkopBWhSoJRSSinb/wMmrQ8QIx9CSgAAAABJ\nRU5ErkJggg==\n", "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "\n", "testInput = np.arange(-6, 6, 0.01)\n", "plt.plot(testInput, NN.sigmoid(testInput), linewidth=2)\n", "plt.plot(testInput, NN.sigmoidPrime(testInput), linewidth=2) # largest where sigmoid is steepest\n", "plt.grid()\n", "plt.legend(['sigmoid', 'sigmoid prime'])" ] }, { "cell_type": "code", "execution_count": 119, "metadata": { "collapsed": false }, "outputs": [ { "data": { "text/latex": [ "\\[\n", " z^{(2)} = X W^{(1)}\n", "\\]\n", "\\[\n", " a^{(2)} = f(z^{(2)})\n", "\\]\n", "\\[\n", " z^{(3)} = a^{(2)}W^{(2)}\n", "\\]\n", "\\[\\]\n", "\\[\n", "W^{(2)} = \n", "\\begin{bmatrix}\n", " W^{(2)}_{(11)} \\\\ \n", " W^{(2)}_{(21)} \\\\ \n", " W^{(2)}_{(31)} \n", "\\end{bmatrix} \n", "\\]\n", "\\[\n", " \\hat{y} = f(z^{(3)})\n", "\\]" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "%%latex\n", "\\[\n", " z^{(2)} = X W^{(1)}\n", "\\]\n", "\\[\n", " a^{(2)} = f(z^{(2)})\n", "\\]\n", "\\[\n", " z^{(3)} = a^{(2)}W^{(2)}\n", "\\]\n", "\\[\\]\n", "\\[\n", "W^{(2)} = \n", "\\begin{bmatrix}\n", " W^{(2)}_{(11)} \\\\ \n", " W^{(2)}_{(21)} \\\\ \n", " W^{(2)}_{(31)} \n", "\\end{bmatrix} \n", "\\]\n", "\\[\n", " \\hat{y} = f(z^{(3)})\n", "\\]" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": true }, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": 135, "metadata": { "collapsed": false }, "outputs": [ { "data": { "text/plain": [ "array([[ 0.53030683],\n", " [ 0.54199931],\n", " [ 0.55222292]])" ] }, "execution_count": 135, "metadata": {}, "output_type": "execute_result" } ], "source": [ "yHat = NN.forward(X)\n", "yHat" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# 3 Gradient Descent" ] }, { "cell_type": "code", "execution_count": 157, "metadata": { "collapsed": false }, "outputs": [ { "data": { "text/latex": [ "\\[\n", " \\textsf{cost function, assuming convexity}\n", "\\]\n", "\\[\n", " J = argmin(W) \\sum_{i=0} \\frac{1}{2} e_{i}^{2}\n", "\\]\n", "\\[\n", " J = argmin(W) \\sum_{i=0} \\frac{1}{2} (y_{i}- \\hat{y_{i}})^{2}\n", "\\]\n", "\\[\n", " W^{(1)} \\textsf{is a 3x3 matrix --> curse of dimensionality}\n", "\\]" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "%%latex\n", "\\[\n", " \\textsf{cost function, assuming convexity}\n", "\\]\n", "\\[\n", " J = argmin(W) \\sum_{i=0} \\frac{1}{2} e_{i}^{2}\n", "\\]\n", "\\[\n", " J = argmin(W) \\sum_{i=0} \\frac{1}{2} (y_{i}- \\hat{y_{i}})^{2}\n", "\\]\n", "\\[\n", " W^{(1)} \\textsf{is a 3x3 matrix --> curse of dimensionality}\n", "\\]" ] }, { "cell_type": "code", "execution_count": 156, "metadata": { "collapsed": false }, "outputs": [ { "data": { "text/latex": [ "\\[\n", " \\textsf{combined equation}\n", "\\]\n", "\\[\n", " J = argmin(W) \\sum \\frac{1}{2} (y - f(f(XW^{(1)})W^{(2)}))^2\n", "\\]\n", "\\[\n", " \\textsf{minimize cost J by choosing weights W - examine rate of change (gradient): stop when}\n", "\\]\n", "\\[\n", " \\frac{\\partial J}{\\partial W}=0\n", "\\]" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "%%latex\n", "\\[\n", " \\textsf{combined equation}\n", "\\]\n", "\\[\n", " J = argmin(W) \\sum \\frac{1}{2} (y - f(f(XW^{(1)})W^{(2)}))^2\n", "\\]\n", "\\[\n", " \\textsf{minimize cost J by choosing weights W - examine rate of change (gradient): stop when}\n", "\\]\n", "\\[\n", " \\frac{\\partial J}{\\partial W}=0\n", "\\]" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# 4 Backpropogation\n", "* (a long chain rule)\n", "* weights that contribute more to overall error will be changed more by SGD" ] }, { "cell_type": "code", "execution_count": 160, "metadata": { "collapsed": false }, "outputs": [ { "data": { "text/plain": [ "array([[ 0.22224692, -0.27898779, 1.71877036],\n", " [ 0.23482195, -2.14545752, -0.10447487]])" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/plain": [ "(2, 3)" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/plain": [ "array([[-0.16169648],\n", " [ 0.02736267],\n", " [ 0.35196909]])" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/plain": [ "(3, 1)" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "display(NN.W1, NN.W1.shape)\n", "display(NN.W2, NN.W2.shape)" ] }, { "cell_type": "code", "execution_count": 177, "metadata": { "collapsed": false }, "outputs": [ { "data": { "text/latex": [ "\\[\n", "\\frac{\\partial J}{\\partial W^{(1)}}=\n", "\\begin{bmatrix}\n", " \\frac{ \\partial J}{ \\partial{W_{11}^{(1)}}} & \n", " \\frac{ \\partial J}{ \\partial{W_{12}^{(1)}}} & \n", " \\frac{ \\partial J}{ \\partial{W_{13}^{(1)}}}\\\\ \n", " \\frac{ \\partial J}{ \\partial{W_{21}^{(1)}}} & \n", " \\frac{ \\partial J}{ \\partial{W_{22}^{(1)}}} & \n", " \\frac{ \\partial J}{ \\partial{W_{23}^{(1)}}}\n", "\\end{bmatrix}\n", "\\]\n", "\\[\n", " \n", "\\]\n", "\\[\n", "\\frac{\\partial J}{\\partial W^{(2)}}=\n", "\\begin{bmatrix}\n", " \\frac{ \\partial J}{ \\partial{W_{11}^{(2)}}} \\\\ \n", " \\frac{ \\partial J}{ \\partial{W_{21}^{(2)}}} \\\\\n", " \\frac{ \\partial J}{ \\partial{W_{31}^{(2)}}}\n", "\\end{bmatrix}\n", "\\]" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "%%latex\n", "\\[\n", "\\frac{\\partial J}{\\partial W^{(1)}}=\n", "\\begin{bmatrix}\n", " \\frac{ \\partial J}{ \\partial{W_{11}^{(1)}}} & \n", " \\frac{ \\partial J}{ \\partial{W_{12}^{(1)}}} & \n", " \\frac{ \\partial J}{ \\partial{W_{13}^{(1)}}}\\\\ \n", " \\frac{ \\partial J}{ \\partial{W_{21}^{(1)}}} & \n", " \\frac{ \\partial J}{ \\partial{W_{22}^{(1)}}} & \n", " \\frac{ \\partial J}{ \\partial{W_{23}^{(1)}}}\n", "\\end{bmatrix}\n", "\\]\n", "\\[\n", " \n", "\\]\n", "\\[\n", "\\frac{\\partial J}{\\partial W^{(2)}}=\n", "\\begin{bmatrix}\n", " \\frac{ \\partial J}{ \\partial{W_{11}^{(2)}}} \\\\ \n", " \\frac{ \\partial J}{ \\partial{W_{21}^{(2)}}} \\\\\n", " \\frac{ \\partial J}{ \\partial{W_{31}^{(2)}}}\n", "\\end{bmatrix}\n", "\\]\n" ] }, { "cell_type": "code", "execution_count": 207, "metadata": { "collapsed": false }, "outputs": [ { "data": { "text/latex": [ "\\[\n", " \\textsf{sum rule in differentiation}\n", "\\]\n", "\\[\n", " \\frac{d(u + v)}{dx} = \\frac{du}{dx} + \\frac{dv}{dx}\n", "\\]\n", "\\[\\]\n", "\\[\n", " \\textsf{chain rule in differentiation}\n", "\\]\n", "\\[\n", " (f \\circ g)' = (f' \\circ g) \\cdot g'\n", "\\]\n", "\\[\\]\n", "\\[\n", " \\frac{dz}{dx} = \\frac{dz}{dy} \\cdot \\frac{dy}{dx}\n", "\\]" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "%%latex\n", "\\[\n", " \\textsf{sum rule in differentiation}\n", "\\]\n", "\\[\n", " \\frac{d(u + v)}{dx} = \\frac{du}{dx} + \\frac{dv}{dx}\n", "\\]\n", "\\[\\]\n", "\\[\n", " \\textsf{chain rule in differentiation}\n", "\\]\n", "\\[\n", " (f \\circ g)' = (f' \\circ g) \\cdot g'\n", "\\]\n", "\\[\\]\n", "\\[\n", " \\frac{dz}{dx} = \\frac{dz}{dy} \\cdot \\frac{dy}{dx}\n", "\\]" ] }, { "cell_type": "code", "execution_count": 229, "metadata": { "collapsed": false }, "outputs": [ { "data": { "text/latex": [ "\\[\\]\n", "\\[\n", "\\frac{\\partial J}{\\partial W^{(2)}} = \n", "\\frac{\\partial \\sum{\\frac{1}{2}(y - \\hat{y})^2}}{\\partial W^{(2)}} = \n", "\\sum{\\frac{\\partial \\frac{1}{2}(y - \\hat{y})^2}{\\partial W^{(2)}}}\n", "\\]\n", "\\[\\]\n", "\\[\n", " \\textsf{power rule and chain rule} \n", "\\]\n", "\\[\n", "\\begin{bmatrix}\n", " -y_{1} - \\hat{y_{1}} \\\\ \n", " -y_{2} - \\hat{y_{2}} \\\\\n", " -y_{3} - \\hat{y_{3}}\n", "\\end{bmatrix}\n", "\\begin{bmatrix}\n", " f'(z_{1}^{(3)}) \\\\ \n", " f'(z_{2}^{(3)}) \\\\\n", " f'(z_{3}^{(3)})\n", "\\end{bmatrix}\n", "=\n", "\\begin{bmatrix}\n", " \\delta _{1}^{(3)} \\\\ \n", " \\delta _{2}^{(3)} \\\\\n", " \\delta _{3}^{(3)} \n", "\\end{bmatrix} \n", " = \\delta ^{(3)}\n", "\\]\n", "\\[\n", " (a^{(2)})^{T} \\delta ^{(3)} = \n", "\\begin{bmatrix}\n", " a_{11}^{(2)} & a_{21}^{(2)} & a_{31}^{(2)} \\\\ \n", " a_{12}^{(2)} & a_{22}^{(2)} & a_{32}^{(2)} \\\\\n", " a_{13}^{(2)} & a_{23}^{(2)} & a_{33}^{(2)} \n", "\\end{bmatrix} \n", "\\begin{bmatrix}\n", " \\delta _{1}^{(3)} \\\\ \n", " \\delta _{2}^{(3)} \\\\\n", " \\delta _{3}^{(3)} \n", "\\end{bmatrix} \n", " =\n", "\\begin{bmatrix}\n", " a_{11}^{(2)} \\delta _{1}^{(3)} + a_{21}^{(2)} \\delta _{2}^{(3)} + a_{31}^{(2)} \\delta _{3}^{(3)} \\\\ \n", " a_{12}^{(2)} \\delta _{1}^{(3)} + a_{22}^{(2)} \\delta _{2}^{(3)} + a_{32}^{(2)} \\delta _{3}^{(3)} \\\\\n", " a_{13}^{(2)} \\delta _{1}^{(3)} + a_{23}^{(2)} \\delta _{2}^{(3)} + a_{33}^{(2)} \\delta _{3}^{(3)} \n", "\\end{bmatrix} \n", "\\]" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "%%latex\n", "\\[\\]\n", "\\[\n", "\\frac{\\partial J}{\\partial W^{(2)}} = \n", "\\frac{\\partial \\sum{\\frac{1}{2}(y - \\hat{y})^2}}{\\partial W^{(2)}} = \n", "\\sum{\\frac{\\partial \\frac{1}{2}(y - \\hat{y})^2}{\\partial W^{(2)}}}\n", "\\]\n", "\\[\\]\n", "\\[\n", " \\textsf{power rule and chain rule} \n", "\\]\n", "\\[\n", "\\begin{bmatrix}\n", " -y_{1} - \\hat{y_{1}} \\\\ \n", " -y_{2} - \\hat{y_{2}} \\\\\n", " -y_{3} - \\hat{y_{3}}\n", "\\end{bmatrix}\n", "\\begin{bmatrix}\n", " f'(z_{1}^{(3)}) \\\\ \n", " f'(z_{2}^{(3)}) \\\\\n", " f'(z_{3}^{(3)})\n", "\\end{bmatrix}\n", "=\n", "\\begin{bmatrix}\n", " \\delta _{1}^{(3)} \\\\ \n", " \\delta _{2}^{(3)} \\\\\n", " \\delta _{3}^{(3)} \n", "\\end{bmatrix} \n", " = \\delta ^{(3)}\n", "\\]\n", "\\[\n", " (a^{(2)})^{T} \\delta ^{(3)} = \n", "\\begin{bmatrix}\n", " a_{11}^{(2)} & a_{21}^{(2)} & a_{31}^{(2)} \\\\ \n", " a_{12}^{(2)} & a_{22}^{(2)} & a_{32}^{(2)} \\\\\n", " a_{13}^{(2)} & a_{23}^{(2)} & a_{33}^{(2)} \n", "\\end{bmatrix} \n", "\\begin{bmatrix}\n", " \\delta _{1}^{(3)} \\\\ \n", " \\delta _{2}^{(3)} \\\\\n", " \\delta _{3}^{(3)} \n", "\\end{bmatrix} \n", " =\n", "\\begin{bmatrix}\n", " a_{11}^{(2)} \\delta _{1}^{(3)} + a_{21}^{(2)} \\delta _{2}^{(3)} + a_{31}^{(2)} \\delta _{3}^{(3)} \\\\ \n", " a_{12}^{(2)} \\delta _{1}^{(3)} + a_{22}^{(2)} \\delta _{2}^{(3)} + a_{32}^{(2)} \\delta _{3}^{(3)} \\\\\n", " a_{13}^{(2)} \\delta _{1}^{(3)} + a_{23}^{(2)} \\delta _{2}^{(3)} + a_{33}^{(2)} \\delta _{3}^{(3)} \n", "\\end{bmatrix} \n", "\\]" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "simple linear relationship for:\n", "* each hidden-layer to output-layer synapse: dW2/dZ3 = a2 \n", "* each hidden-layer to output-layer synapse: dZ3/da2 = W2\n", "* each input-layer to hidden-layer synapse: dZ2/dW1 = X" ] }, { "cell_type": "code", "execution_count": 234, "metadata": { "collapsed": false }, "outputs": [ { "data": { "text/latex": [ "\\[\n", " \\frac{\\partial J}{\\partial W^{(2)}} = (a^{(2)})^{T} \\delta ^{(3)}\n", "\\]\n", "\\[\\]\n", "\\[\n", " \\delta ^{(3)} = -(y - \\hat{y})f'(z^{(3)})\n", "\\]\n", "\\[\\]\n", "\\[\n", " \\frac{\\partial J}{\\partial W^{(1)}} = X^{T} \\delta ^{(2)}\n", "\\]\n", "\\[\\]\n", "\\[\n", " \\delta ^{(2)} = \\delta ^{(3)} (W^{(1)})^{T} f'(z^{(2)})\n", "\\]" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "%%latex\n", "\\[\n", " \\frac{\\partial J}{\\partial W^{(2)}} = (a^{(2)})^{T} \\delta ^{(3)}\n", "\\]\n", "\\[\\]\n", "\\[\n", " \\delta ^{(3)} = -(y - \\hat{y})f'(z^{(3)})\n", "\\]\n", "\\[\\]\n", "\\[\n", " \\frac{\\partial J}{\\partial W^{(1)}} = X^{T} \\delta ^{(2)}\n", "\\]\n", "\\[\\]\n", "\\[\n", " \\delta ^{(2)} = \\delta ^{(3)} (W^{(1)})^{T} f'(z^{(2)})\n", "\\]" ] }, { "cell_type": "code", "execution_count": 242, "metadata": { "collapsed": false }, "outputs": [ { "data": { "text/plain": [ "(array([[ 0.02356812, -0.02571859, 0.05172803],\n", " [ 0.02094194, -0.02520551, 0.03866296]]), array([[-0.03604346],\n", " [-0.05906128],\n", " [-0.09162464]]))" ] }, "execution_count": 242, "metadata": {}, "output_type": "execute_result" } ], "source": [ "dJdW1, dJdW2 = NN.costFunctionPrime(X,y)\n", "dJdW1, dJdW2" ] }, { "cell_type": "code", "execution_count": 244, "metadata": { "collapsed": false }, "outputs": [ { "data": { "text/plain": [ "array([ 0.48228807])" ] }, "execution_count": 244, "metadata": {}, "output_type": "execute_result" } ], "source": [ "scalar = 3\n", "NN.W1 = NN.W1 + scalar * dJdW1\n", "NN.W2 = NN.W2 + scalar * dJdW2\n", "NN.costFunction(X,y)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# 5 Numerical Gradient" ] }, { "cell_type": "code", "execution_count": 245, "metadata": { "collapsed": false }, "outputs": [ { "data": { "text/plain": [ "" ] }, "execution_count": 245, "metadata": {}, "output_type": "execute_result" }, { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAAgMAAAFkCAYAAAC9wjgoAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAAPYQAAD2EBqD+naQAAIABJREFUeJzt3Xl4VNX9x/H3EbCKFv1ZFdzijlpFFKqCFVxQVCTj0ioo\nLkBdAasoiXUDRFQSFREQVypuBLcaVwQ3lLihxF1QFBXrgiCuhArC+f1xgpIQIPdm5p5773xezzNP\ny2Rm7nc+GZNv7j2LsdYiIiIi+Wst3wWIiIiIX2oGRERE8pyaARERkTynZkBERCTPqRkQERHJc2oG\nRERE8pyaARERkTynZkBERCTPqRkQERHJc2oGRERE8lzoZsAYc6Yx5i1jzA/Vt5eMMYfVeswQY8yX\nxpgqY8xTxpgdGl6yiIiIZFNDzgx8DlwAtAHaAs8CDxtjdgEwxlwA9ANOB/YGFgKTjDFrN6hiERER\nySqTzY2KjDHfAgOstbcbY74ErrbWXlf9tWbAXOAUa+19WTuoiIiINEhWxgwYY9YyxnQHmgIvGWO2\nBVoAzyx/jLX2R+BVoH02jikiIiLZ0bghTzbG7Aa8DKwD/AQcba39wBjTHrC4MwErmotrElb1en8C\nDgU+Bf7XkNpERETyzDrANsAka+23QZ7YoGYAmAm0BjYA/g7caYzp2IDXOxS4p4E1iYiI5LMewPgg\nT2hQM2Ct/RWYXf3PN4wxewPnAKWAAZpT8+xAc+CN1bzkpwB33303u+yyS0NKyzv9+/fnuuuu811G\noiizcJRbcMosHOUWzIwZMzjxxBOh+ndpEA09M1DbWsAfrLWfGGO+BjoBb8NvAwj3AW5YzfP/B7DL\nLrvQpk2bLJeWbhtssIEyC0iZhaPcglNm4Si30AJfZg/dDBhjrgQmAnOAP+JOS+wPdK5+yAjgEmPM\nR7gu5XLgv8DDYY8pq/b111/7LiFxlFk4yi04ZRaOcotOQ84MbArcAWwG/IA7A9DZWvssgLW21BjT\nFLgZ2BCYChxurV3csJKlLl988YXvEhJHmYWj3IJTZuEot+iEbgastafW4zGDgcFhjyH117ZtW98l\nJI4yC0e5BafMwlFu0dHeBClx/PHH+y4hcZRZOMotOGUWjnKLTlZXIGwoY0wbYPr06dM1aERERCSA\nysrK5WdT2lprK4M8V2cGRERE8pyagZTo1auX7xISR5mFo9yCU2bhKLfoqBlIic6dO6/5QVKDMgtH\nuQWnzMJRbtHRmAEREZEU0JgBERERCU3NgIiISJ5TM5ASFRUVvktIHGUWjnILTpmFo9yio2YgJUpL\nS32XkDjKLBzlFpwyC0e5RUcDCFOiqqqKpk2b+i4jUZRZOMotOGUWjnILRgMIRf/BhKDMwlFuwSmz\ncJRbdNQMiIiI5Dk1AyIiInlOzUBKFBUV+S4hcZRZOMotOGUWjnKLjpqBlCgoKPBdQuIos3CUW3DK\nLBzlFh3NJhAREUkBzSYQERGR0NQMiIiI5Dk1Aykxc+ZM3yUkjjILR7kFp8zCUW7RUTOQEsXFxb5L\nSBxlFo5yC06ZhaPcoqNmICVGjx7tu4TEUWbhKLfglFk4yi06agZSQlNwglNm4Si34JRZOMotOmoG\nRERE8pyaARERkTynZiAlSkpKfJeQOMosHOUWnDILR7lFR81ASlRVVfkuIXGUWTjKLThlFo5yi46W\nIxYREUkBLUcsIiIioakZEBERyXNqBlJi/vz5vktIHGUWjnILTpmFo9yio2YgJXr37u27hMRRZuEo\nt+CUWTjKLTpqBlJi8ODBvktIHGUWjnILTpmFo9yiE7oZMMZcaIyZZoz50Rgz1xjzkDGmZa3H3G6M\nWVbr9kTDy5baNPsiOGUWjnILTpmFo9yi05AzAx2AUcA+wMFAE2CyMWbdWo+bCDQHWlTfjm/AMUVE\nRCTLGod9orW2y4r/Nsb0BL4B2gIVK3zpF2vtvLDHERERkdzK5piBDQELLKh1/wHVlxFmGmPGGGM2\nyuIxpdrYsWN9l5A4yiwc5RacMgtHuUUnK82AMcYAI4AKa+37K3xpInAycBBQDOwPPFH9+ERZsgQW\nLfJdxapVVgZabEpQZmEpt+CUWTjKLTpZWY7YGHMjcCjwV2vtV6t53LbAx0Ana+1zdXw9lssRL1kC\nu+4Kp54KxcW+qxEREV8+/xw22wwah77InjtelyM2xowGugAHrK4RALDWfgLMB3ZY3eO6dOlCJpOp\ncWvfvj3l5eU1Hjd58mQymcxKz+/bt+9Kp5cqKyvJZDIrLWIxaNCglXbGmjNnDplMhpkzZwLQpAns\ntx9ceeUozj+/qMZjq6qqyGQyVFRU1Li/rKyMXr16rVRbt27dvL2P5UaNGkVRkd6H3ofeh96H3keQ\n9zFv3nyOOQaWH8Ln+ygrK/vtd2OLFi3IZDL0799/pefUV4PODFQ3AkcC+1trZ9fj8VsCnwFHWmsf\nq+PrsTwzAPDOO7D77nDPPXDCCb6rERGRqFVUQIcO8MQTcPjhvqtZmZczA8aYMUAP4ARgoTGmefVt\nneqvr2eMKTXG7GOM2doY0wkoBz4EJoU9ri+tWkHnznDttRCjjR5FRCQiI0bATjvBoYf6riT7GnKZ\n4EygGTAF+HKF23HVX18K7A48DHwA3Aq8BnS01i5pwHG9Oe88qKyEqVN9V7Kyuk53yeops3CUW3DK\nLJw45fbpp/DQQ3DuubBWCtfubcg6A6uNw1r7P+CwsK8fR507w5//DMOHQ8eOvqupqV+/fr5LSBxl\nFo5yC06ZhROn3K6/HjbcEE46yXcluZGV2QTZEucxA8uNHQunnQYffAA77ui7GhERybXvv4ettoJz\nzoGhQ31Xs2peZxPkmx49YOON3bUjERFJv1tvhcWLIUYnKrJOzUBA66wDffvC7bfDgtprLYqISKos\nXuwuEfToAS1a+K4md9QMhHDWWbBsGdx8s+9Kfld7Dq2smTILR7kFp8zCiUNu990HX3zhBpCnmZqB\nEDbd1A0iGTXKdY1xUFZW5ruExFFm4Si34JRZOL5zs9ZNJz/0UNhtN6+l5JwGEIb03nvuw3Hnnekd\nXSoiks+efRY6dYLJk+GQQ3xXs2YaQOjBrrvCYYe5aYYx6qdERCRLrr3WrTx78MG+K8k9NQMNcP75\n8OabMGWK70pERCSbZsxwyw6ffz4kb5/d4NQMNECnTm6Z4uHDfVciIiLZNHw4bL45dO/uu5JoqBlo\nAGPcCNPHHnOLEPlU1w5XsnrKLBzlFpwyC8dXbnPnwl13wdlnw9preykhcmoGGuj4493c02uv9VtH\n586d/RaQQMosHOUWnDILx1duN9wAjRvDGWd4ObwXmk2QBVddBZdd5jaySPOiFCIiaVdVBQUFbqv6\nkSN9VxOMZhN4dtZZ0KRJ8j44IiJS0513wnffud0J84magSzYcEN3OmnMGPjpJ9/ViIhIGEuXwnXX\nwTHHwHbb+a4mWmoGsuTcc2HhQrehhQ8VFRV+Dpxgyiwc5RacMgsn6twefhg+/BAGDIj0sLGgZiBL\nttzSbWRx3XV+liguLS2N/qAJp8zCUW7BKbNwoszNWigpgQMOgH32ieywsaEBhFm0fInicePglFOi\nPXZVVRVNmzaN9qAJp8zCUW7BKbNwosztuefgoINg4kS3umwSaQBhTOy6K3TtCldf7XY1jJJ+0ASn\nzMJRbsEps3CizK2kBFq3dpsS5SM1A1lWXOzOEEyc6LsSERGpjzffhEmT4IIL8mPp4bqoGciy/faD\ndu1AlwhFRJKhtBS23RaOPdZ3Jf6oGcgyY9zZgRdegFdeie64RUVF0R0sJZRZOMotOGUWThS5zZ4N\n997rNiRq3Djnh4stNQM5cOSR0LJltGcHCgoKojtYSiizcJRbcMosnChyu/Za2GgjyPftIzSbIEdu\nuw1OP91tg7nTTr6rERGR2r75BrbeGi6+GC65xHc1DafZBDF04onQvLn/DYxERKRuo0ZBo0bQp4/v\nSvxTM5Aj66wD55wDd9wBX37puxoREVnRTz/B6NHuDO5GG/muxj81Azl01lmuKRg+PPfHmjlzZu4P\nkjLKLBzlFpwyCyeXud16K/z8M/Tvn7NDJIqagRzaYAPo1w9uugm+/Ta3xyouLs7tAVJImYWj3IJT\nZuHkKrfFi90faSeeCFttlZNDJI6agRw791y35nWutzcePXp0bg+QQsosHOUWnDILJ1e5jR8PX3zh\npoGLo2YgxzbZxF2TGjkSfvwxd8fR1KXglFk4yi04ZRZOLnJbuhSuuspNAd9ll6y/fGKpGYjAgAFu\ne+Mbb/RdiYhIfnvgAbdNcRqmEmaTmoEIbLEF9OzprlEtWuS7GhGR/LRsGQwd6jYj+stffFcTL2oG\nInLBBTB/Powdm5vXLykpyc0Lp5gyC0e5BafMwsl2bo8+Cu++q7MCdVEzEJHtt4fjj3dLFC9enP3X\nr6qqyv6LppwyC0e5BafMwslmbtbCFVfA/vu7DeWkJi1HHKH33oPddoN//1vrYIuIRGnyZHd54Kmn\n4OCDfVeTG16WIzbGXGiMmWaM+dEYM9cY85AxpmUdjxtijPnSGFNljHnKGLND2GMm3a67wlFHuZGs\nS5f6rkZEJH8MHQp77w2dOvmuJJ4acpmgAzAK2Ac4GGgCTDbGrLv8AcaYC4B+wOnA3sBCYJIxZu0G\nHDfRLr4YZs1yI1pFRCT3XngBpk51YwWM8V1NPIVuBqy1Xay1d1lrZ1hr3wF6AgVA2xUedg5wubX2\nMWvtu8DJwObAUQ2oOdH+8hfo3BmuvNJdw8qW+fPnZ+/F8oQyC0e5BafMwslWbldcAa1bQ9euWXm5\nVMrmAMINAQssADDGbAu0AJ5Z/gBr7Y/Aq0D7LB43cS66CN5+Gx5/PHuv2bt37+y9WJ5QZuEot+CU\nWTjZyG3aNDde4OKLdVZgdbLSDBhjDDACqLDWvl99dwtcczC31sPnVn8tb3XsCH/9K1x+efbODgwe\nPDg7L5RHlFk4yi04ZRZONnK74grYeWc45piG15Nm2TozMAb4M9A9Gy/WpUsXMplMjVv79u0pLy+v\n8bjJkyeTyWRWen7fvn0ZW2tCf2VlJZlMZqXTToMGDVppLuucOXPIZDIr7Zg1atQoioqKatxXVVVF\nJpOhoqKixv1lZWX0qmPKQLdu3Xj44XIGDXId66RJ2XkfDz/8cOTvI+nfjzZt2qTifUC034+NN944\nFe8jyu9HmzZtUvE+INrvR5s2bRr0Pt5+Gx55BJo168ajj6brc1VWVvbb78YWLVqQyWTo34AtGBs8\ntdAYMxooBDpYa+escP+2wMfAHtbat1e4fwrwhrV2parTPrVwRda6swPWwksv6fSViEi2de/u/uj6\n4ANo0sR3NbnnZWoh/NYIHAkcuGIjAGCt/QT4Gui0wuOb4WYfvNSQ46aBMTBoELzyirueJSIi2TNz\nJtx3n1v9NR8agYZqyDoDY4AewAnAQmNM8+rbOis8bARwiTGm0BjTCrgT+C/wcEOKTovOnaFdO7js\nsoaPHah9WkvWTJmFo9yCU2bhNCS3IUNgyy21wFt9NeTMwJlAM2AK8OUKt+OWP8BaW4pbi+Bm3CyC\ndYHDrbU5WJA3eZafHXj5ZbcqVkNUVgY6IyQos7CUW3DKLJywub3/PkyY4GYQrJ23q9oEo+WIPbPW\nnR1o3BgqKjR2QESkoY4/3o3FmjUrv5oBb2MGpOGMgcGD3Qf36ad9VyMikmzvvQf33quzAkGpGYiB\nww6DvfbKztgBEZF8NmQIFBRAz56+K0kWNQMxsPzswIsvwjPPrPHhIiJSh3ffhfvv11mBMNQMxMTh\nh7t9C8KeHahr8QxZPWUWjnILTpmFEzS3IUNg663hlFNyVFCKqRmIieUzCyoq4Nlngz+/X79+2S8q\n5ZRZOMotOGUWTpDc3nlHZwUaQrMJYsRaN3agaVN4/nnNLBARqa9jj4XXX4cPP8zfRYY0myAllp8d\nmDpVYwdEROrr7bfhgQfgkkvytxFoKDUDMdO1qzs7cOmlmlkgIlIfl10G224LJ5/su5LkUjMQM8bA\n0KFuz4LHH6//82rvLCZrpszCUW7BKbNw6pPbW2/Bf/6jswINpWYghg45BDp0cGcHli2r33PKyspy\nW1QKKbNwlFtwyiyc+uQ2eDBstx2cdFLu60kzDSCMqRdegP33d7tuHXus72pEROLntddg773hjjt0\niQA0gDCVOnZ0uxoOHAhLl/quRkQkfi6+GHbZBXr08F1J8qkZiLGhQ92e3Pfc47sSEZF4mTLF7fY6\ndCg0auS7muRTMxBje+0FRx3lroktWeK7GhGReLDWnRVo2xaOPtp3NemgZiDmhgyBTz+Ff/979Y/r\n1atXJPWkiTILR7kFp8zCWVVuTzzhdnq94gotzpYtagZirlUr6N4dLr8c/ve/VT+uc+fO0RWVEsos\nHOUWnDILp67cli1zZwWWj6uS7NBsggT48EP485/hmmvg3HN9VyMi4s9990G3bm6l1v32811NvGg2\nQcq1bOl24brqKvj5Z9/ViIj48euvbv2Vww9XI5BtagYS4tJL4bvvYNQo35WIiPhx553uTOnQob4r\nSR81AwmxzTZw+ulQWgoLFqz89YqKishrSjplFo5yC06ZhbNibr/84vYg+PvfQVeRs0/NQIJccomb\nYjhs2MpfKy0tjb6ghFNm4Si34JRZOCvmdsst8N//uhlWkn0aQJgwgwZBSQnMmgVbbfX7/VVVVTRt\n2tRfYQmkzMJRbsEps3CW5/bTT7DDDm6swLhxvquKLw0gzCPnnw/NmrmFiFakHzTBKbNwlFtwyiyc\n5bkNHw7ff+8uE0huqBlImGbN3GDCcePgvfd8VyMikltz58LVV8PZZ8PWW/uuJr3UDCTQGWe4/ygu\nush3JSIiuTVkCDRpop93uaZmIIHWXtstw/nII7B8sG1RUZHfohJImYWj3IJTZuGcemoRt9wCF14I\nG23ku5p0UzOQUN26wZ57wgUXuE07CgoKfJeUOMosHOUWnDILZ/r0Alq0cJcIJLc0myDBnnrKrc1d\nXg5HHum7GhGR7Hn1VWjXDm6/HXr29F1NMmg2QZ465BA4+GB3Cu3XX31XIyKSHdZCcbHbqO2kk3xX\nkx/UDCTcsGEwY4ZbplNEJA2eeAJeeMH9fGvUyHc1+UHNQMK1beu2OL7wwplUVfmuJllmzpzpu4RE\nUm7BKbP6W7oU/vUvOOAA2HZb5RYVNQMpcMUVMG9eMcOH+64kWYqLi32XkEjKLThlVn933gnvvuv2\nYbngAuUWFQ0gTInTT5/D+PEFzJoFm23mu5pkmDNnjkZ5h6DcglNm9bNokduyfd994d57lVtQXgYQ\nGmM6GGMeMcZ8YYxZZozJ1Pr67dX3r3h7IuzxZPVKSwtYZx0YONB3JcmhHzLhKLfglFn9XHutW3Hw\niivcv5VbdBpymWA94E2gD7Cq0wsTgeZAi+rb8Q04nqzGhhu6TYzGjoW33/ZdjYhIMF9+6QYM/vOf\nblMiiVboZsBa+6S1dqC19mHArOJhv1hr51lrv6m+/RD2eLJmZ54JO+7oNjOK0dUfEZE1uvRSWGcd\nt1W7RC/XAwgPMMbMNcbMNMaMMcZoQckcKSkpoUkTuOYaePppmDjRd0XxV1JS4ruERFJuwSmz1Xvj\nDbe40GWXubOcyym36OSyGZgInAwcBBQD+wNPGGNWdRZBGqCqel5h165w4IEwYIAWIlqTKs3FDEW5\nBafMVs1aOO882HlntwnbipRbhKy1Db4By4DMGh6zbfXjDlzNY9oAtnnz5rawsLDGrV27dvahhx6y\nK5o0aZItLCy0tfXp08fedtttNe6bPn26LSwstPPmzatx/8CBA+2wYcNq3PfZZ5/ZwsJCO2PGjBr3\njxw50g4YMKDGfQsXLrSFhYV26tSpNe4fP3687dmz50q1HXfccTl/H5WV1hpj7dChyX4fyyX9+6H3\nofeh97Hq9/HQQ9aCtePGJft9LBfV92P8+PG//W5c/juzY8eOFjeGr40N+Hs8K1MLjTHLgKOstY+s\n4XHfABdba29dxdc1tTBLevWCxx6Djz6CDTbwXY2IyMoWL4Zdd4XttoMnnwSdN26YROxNYIzZEvgT\n8FVUx8xnQ4dCVRVceaXvSkRE6nbDDTB7tptSqEbAr4asM7CeMaa1MWaP6ru2q/73VtVfKzXG7GOM\n2doY0wkoBz4EJmWjcKlp/vz5Nf69xRZQVAQjRsAnn3gqKuZqZyb1o9yCU2Yr+/ZbGDIETj8ddtut\n7scot+g05MzAX4A3gOm4axTXApXAZcBSYHfgYeAD4FbgNaCjtXZJQwqWuvXu3Xul+4qKYNNN3WBC\nWVldmcmaKbfglNnKBg+GZcvcDIJVUW7RaRz2idba51l9M3FY2NeW4AYPHrzSfeut59b3PuEEN93w\n4IOjryvO6spM1ky5BafMapoxA2680V3G3HTTVT9OuUVHexOknLXQsSMsWABvvglNmviuSETymbVw\n6KHw8cfw3ntuoSHJjkQMIBQ/jIFRo2DmTBgzxnc1IpLvysvhqafceCY1AvGhZiAP7LEHnHaa27tg\n3jzf1YhIvlq0yC0wdPjhboE0iQ81AykxduzY1X596FB3luDiiyMqKAHWlJnUTbkFp8yc0lL44gt3\nVqA+UwmVW3TUDKREZeXqLw9tvDFcfjncdhtMnx5RUTG3psykbsotOGUGn33mdiU87zxo2bJ+z1Fu\n0dEAwjzy66+w557wxz/Ciy9qkQ8Ric7f/w4vvwwffADrr++7mnTSAEKpl8aN4frr3X+Q99zjuxoR\nyRfPPAMPPghXX61GIK7UDOSZgw5yHXpxMfz0k+9qRCTtliyBs8+GDh3g+ON9VyOromYgD11zDXz3\nnRtDICKSS6NHu0sDo0bp0mScqRlIiUwmU+/Hbr21m1Vw3XXw7rs5LCrmgmQmv1NuweVrZnPnumWH\nzzwTWrcO/vx8zc0HNQMp0a9fv0CPLypy24b26eNWBMtHQTMTR7kFl6+ZFRe7sUpDhoR7fr7m5oNm\nE+Sxp5+GQw6BcePglFN8VyMiaTJlChx4oJvO/I9/+K4mP2g2gYRy8MHQvbs7S7Bgge9qRCQtFi+G\ns86Cv/4VevXyXY3Uh5qBPDd8OPzvf3DRRb4rEZG0uOYamDXL7Uy4ln7LJIK+TSlRXl4e6nmbbeaW\nKr7lFnj11SwXFXNhM8t3yi24fMps9mw3U+m886BVq4a9Vj7l5puagZQoKysL/dw+fdxmRmedBUuX\nZrGomGtIZvlMuQWXL5lZC/36wSabwMCBDX+9fMktDjSAUAB3VqB9e7dC4dln+65GRJLowQfdombl\n5XDkkb6ryT8aQCgNts8+bpvjSy6Br77yXY2IJM1PP8E550Amo0YgidQMyG+uugrWXhv69/ddiYgk\nzaBBbmXTkSN9VyJhqBmQ32y0kVuV8N574fHHfVcjIknx5puuCRg0yK1wKsmjZiAlemVpMm+PHnDo\noW4wYdo3MspWZvlGuQWX5sx+/dUtKrTLLtk/q5jm3OJGzUBKdO7cOSuvY4ybG/ztt27/gjTLVmb5\nRrkFl+bMRoxwZwbGjoUmTbL72mnOLW40m0DqNHw4DBgAL70E7dr5rkZE4ujjj91aAmee6X5miF+a\nTSBZ989/Qtu2cOqpbmlREZEVWQunnw7Nm2s79DRQMyB1atzYbTAycyaUlvquRkTi5vbb4dln4eab\nYb31fFcjDaVmICUqKiqy/pqtW7tNjC6/3DUFaZOLzPKBcgsubZl9/TWcfz6cfDLk8rJ+2nKLMzUD\nKVGaoz/fBw6EggJ3OnDZspwcwptcZZZ2yi24tGV29tlusGCuxwmkLbc4UzOQEhMmTMjJ6667rtvE\naOpUdzowTXKVWdopt+DSlFl5OTzwgFtX4E9/yu2x0pRb3KkZSImmTZvm7LUPPNAtVVxcDJ9+mrPD\nRC6XmaWZcgsuLZl9/z307Qtdu0K3brk/XlpySwI1A1Iv11wD//d/bnGRtF0uEJH66d8ffv4Zxoxx\na5JIeqgZkHpp1szNLlg+elhE8stjj8G4cW6Roa228l2NZJuagZQoKirK+TE6d3aXC4qK4JNPcn64\nnIsiszRSbsElPbMFC9wg4i5doGfP6I6b9NySRM1AShQUFERynGuucYOG0nC5IKrM0ka5BZf0zM45\nBxYtcoOJo7w8kPTckiT0csTGmA5AEdAW2Aw4ylr7SK3HDAFOBTYEXgTOstZ+tJrX1HLECfDUU+4s\nwQ03QJ8+vqsRkVwqL4ejj4Y774STTvJdjayOr+WI1wPeBPoAK3UUxpgLgH7A6cDewEJgkjFm7QYc\nU2LgkEPcKcPi4nRcLhCRus2fD2ecAZkMnHii72okl0I3A9baJ621A621DwN1nTg6B7jcWvuYtfZd\n4GRgc+CosMeU+Lj6ane5oHfv5F8uEJG6nX2226L45ps1eyDtcjJmwBizLdACeGb5fdbaH4FXgfa5\nOGa+mxnxesHNmrktS6dMgdGjIz101kSdWVoot+CSmNkDD8CECe6/7xYt/NSQxNySKlcDCFvgLh3M\nrXX/3OqvSZYVFxdHfsyDD3Z/OVxwAbz/fuSHbzAfmaWBcgsuaZl9/TWcdRYccwx07+6vjqTllmSa\nTZASoz39eV5SAttuCz16JG+rY1+ZJZ1yCy5JmVnrLv81agQ33eT38kCScku6XDUDX+PGETSvdX/z\n6q+tVpcuXchkMjVu7du3p7y8vMbjJk+eTCaTWen5ffv2ZezYsTXuq6ysJJPJMH/+/Br3Dxo0iJKS\nkhr3zZkzh0wms9IpqlGjRq0077WqqopMJrPS7lplZWX06tVrpdq6deuWk/cxduxYL+9j3XXh7rvh\n3XfhpJOS9f0oKCjI2fcjLZ+rut4HkIr3EeX3o6CgIDHv48YbYeJEt0Vxv35+vx8FBQX6XK3ifZSV\nlf32u7FFixZkMhn69++/0nPqK/TUwhovYswyak0tNMZ8CVxtrb2u+t/NcJcJTrbW3r+K19HUwoQa\nNgwuusiNIejY0Xc1IhLGjBnQpo1bR0R/lCePl6mFxpj1jDGtjTF7VN+1XfW/ly9UOQK4xBhTaIxp\nBdwJ/BcxYdiYAAAczElEQVR4OOwxJb6KimC//dz+5j/84LsaEQlq8WJ3uW+bbUA7B+efhlwm+Avw\nBjAdN1jwWqASuAzAWlsKjAJuxs0iWBc43FqbsCvLyVDXqdwoNWrkFiVZsMANKkwC35kllXILLgmZ\nDRrkLvfdcw/EZbPAJOSWFo3DPtFa+zxraCastYOBwWGPIfVXVVXluwS22QZGjXJrlxcWwrHH+q5o\n9eKQWRIpt+DintkLL7jBwFdd5S4TxEXcc0uTrIwZyBaNGUg+a+G44+CZZ+Ctt7S7mUjcff89tG7t\nmvlnn3Vn+SSZfC1HLLISY9xqZeut564//vqr74pEZFWsdfuLfP+9u8ynRiB/qRmQrNtoIxg/Hl58\nES6/3Hc1IrIqt98OZWVuPYGtt/ZdjfikZiAlas9/9a1DB7jsMtcMPPec72rqFrfMkkK5BRfHzN5/\nH/r1g1NPheOP911N3eKYW1qpGUiJ3r17+y5hJRdeCAcc4HY7mzfPdzUri2NmSaDcgotbZlVVbmzP\ndtvB9df7rmbV4pZbmqkZSInBgwf7LmEljRq51QkXL3YzDOK2u2EcM0sC5RZc3DI791yYPRvuvTc+\n0wjrErfc0kzNQErEdfbF5pvDHXfAE0/AiBG+q6kprpnFnXILLk6Z3Xsv3HorjBwJu+7qu5rVi1Nu\naadmQHKuSxc4/3z417/gtdd8VyOSvz7+GE47ze1E+I9/+K5G4kTNgETiyithjz3cD6Hvv/ddjUj+\n+eUX6NYNNt3UTf/1uRuhxI+agZSovctW3Ky9tjs9uWCB278gDuMH4p5ZXCm34OKQWf/+8M477r/D\nZs18V1M/ccgtX6gZSInKykCLTXmx7bZuQOGjj8ZjI5QkZBZHyi0435nddZfbmnjUKHAL1CWD79zy\niZYjlshdeqm7bPDUU3DQQb6rEUm3t9+Gdu3cVMLbb9flgTTTcsSSKIMHuyage3f44gvf1Yik1w8/\nwN/+Bi1bwpgxagRk1dQMSOQaNXLLFf/hD25nw8Xa1Fok66x163vMmwcPPhjv9QTEPzUD4sUmm8D9\n98Prr0Nxse9qRNLn6quhvNyNF9h+e9/VSNypGUiJTCbju4TA2rWD665zy6GWlUV//CRmFgfKLbio\nM3vuObcc+EUXQWFhpIfOKn3WoqNmICX69evnu4RQ+vSBk06C3r1h+vRoj53UzHxTbsFFmdknn7jL\nbwcdBEOGRHbYnNBnLTqaTSDeLVoE++8PX33lLhs0b+67IpFk+vln2HdfWLgQpk2DP/3Jd0USJc0m\nkERbd1146CH49Vc38lkDCkWCW7YMTjnFnRl45BE1AhKMmgGJhS22gP/8x+1d0K+fGwktIvV3+eXu\nv6G7747/BkQSP2oGUqK8vNx3CQ3Wvj3cdJPbUe2mm3J/vDRk5oNyCy7Xmf3nP279jssvhyOPzOmh\nIqXPWnTUDKREmY/h+DnQqxf885/u9vzzuT1WWjKLmnILLpeZvfOO2+/j73+Hiy/O2WG80GctOhpA\nKLGzZAkcdhi89Ra8/DLsuKPvikTi6ZtvYJ99YIMN4MUXYb31fFckPmkAoaRKkyZuQaKNN4YjjoBv\nv/VdkUj8LFrkLgksWgQPP6xGQBpGzYDE0kYbweOPw3ffwdFHu73YRcRZtsxdGnjrLbcL6NZb+65I\nkk7NgMTW9tu7v3imTYNTT9UMA5HlLrrI7TcwfjzstZfvaiQN1AykRK9evXyXkBP77gvjxrnpUtle\nTS2tmeWacgsum5ndeiuUlMC118JRR2XtZWNJn7XoNPZdgGRH586dfZeQM927w8cfwyWXuLMFJ56Y\nnddNc2a5pNyCy1ZmkyfDWWe5ZbzPPTcrLxlr+qxFR7MJJBGsdfsX3HMPPPmkW3ddJJ+88w789a/Q\noYO7fNZYf8pJLZpNIKlnDNx8Mxx4oDs1+sYbvisSic6nn8Khh7ozYxMmqBGQ7FMzIImx9trwwAOw\n005w+OHu0oFI2n3zDXTu7PbwmDgR/vhH3xVJGqkZSImKigrfJUTij3+EJ56AZs3cX0pz54Z/rXzJ\nLNuUW3BhM/vpJ+jSBX780Y0XaNEiy4XFnD5r0VEzkBKlpaW+S4jMJpu4H4xVVe4MwY8/hnudfMos\nm5RbcGEy++UXt8bGrFlunMz22+egsJjTZy06OW0GjDGDjDHLat3ez+Ux89WECRN8lxCpbbZxPyBn\nzw6/KFG+ZZYtyi24oJktXeoWFaqocIMF99gjR4XFnD5r0YnizMC7QHOgRfVtvwiOmXeaNm3qu4TI\n7b6727f9xRfhhBPg11+DPT8fM8sG5RZckMysdRt1PfAAlJXBAQfkrq6402ctOlE0A79aa+dZa7+p\nvi2I4JiSJzp2dD80H3kETjnF/UUlklTWwgUXwJgxbhvvo4/2XZHkiyiagR2NMV8YYz42xtxtjNkq\ngmNKHuna1S3LOmECnHGGW7ddJIkGD4arr4brr4fTTvNdjeSTXDcDrwA9gUOBM4FtgReMMdpfK8uK\niop8l+DVscfCHXfAv//tTrHWZy2tfM8sLOUWXH0yGzbMLbk9bJj7DIs+a1HK6dIV1tpJK/zzXWPM\nNOAz4Djg9lweO98UFBT4LsG7E09027mefrqbk11a6hYrWhVlFo5yC25NmY0YARdeCIMGucsE4uiz\nFiFrbaQ3YBpwxSq+1gawzZs3t4WFhTVu7dq1sw899JBd0aRJk2xhYaGtrU+fPva2226rcd/06dNt\nYWGhnTdvXo37Bw4caIcNG1bjvs8++8wWFhbaGTNm1Lh/5MiRdsCAATXuW7hwoS0sLLRTp06tcf/4\n8eNtz549V6rtuOOO0/vI8fvYbbdCC1PtpZcm+32k5fuh97H699GxYx8Lt9niYmuXLUvu+0jL9yMp\n72P8+PG//W5c/juzY8eOFrBAGxvwd3OkexMYY9YH5gADrbWj6/i69iaQrCgtdX9hXXKJO/W6ujME\nIr6MGQN9+7rLAiNG6HMqDdOQvQlyepnAGHM18Cju0sAWwGXAEqAsl8cVKS52P1iLi2HxYncdVj9o\nJU5GjID+/d3ug8OH6/MpfuV6AOGWwHhgJjABmAe0s9Z+m+Pj5p2ZM2f6LiF2iorcD9zSUvdDt/ZJ\nMGUWjnILrnZmyz+TxcVqBFZHn7Xo5LQZsNYeb63d0lq7rrW2wFp7grX2k1weM18VFxf7LiGWzjnH\nnYq9/np3OnbFaYfKLBzlFtyKmQ0d6i5hXXqpzlitiT5r0dFGmCkxevRKQzCk2llnuR0PTzsNlixx\ni7k0aqTMwlJuwY0ePRpr3WyByy9341guvdR3VfGnz1p01AykhKbgrN4//uEagp494Ycf4K67lFlY\nyi24Lbcs4Oyz4YYboKTEXR6QNdNnLTpqBiRvnHSS2/q4Wzc44gh46CHtDS+5t3ix23To/vvhllu0\nsqDEk7Ywlrxy5JEwaRK89hocdBDMm+e7Ikmzn3+GwkLXeN5/vxoBiS81AylRUlLiu4TE2H9/mDIF\nZswooUMHmDPHd0XJos9a/Xz7LRx8MLz0Epx8cgnHHOO7ouTRZy06agZSoqqqyncJibLnntCrVxWL\nF8O++8Jbb/muKDn0WVuzTz6B/faD2bNd47n55sosDH3WohPpCoRrohUIJWpffeV2PfzwQ7j3XujS\nxXdFknSvvAKZjBuf8sQT0LKl74okXzRkBUKdGZC8ttlm8MILbvxAYaEb7S0S1v33w4EHugbglVfU\nCEhyqBmQvLfeevCf/7gFivr1c8vDLl3quypJEmvhqqvguOPgmGPg6adh4419VyVSf2oGUmL+/Pm+\nS0icFTNr1MgtC3vDDTBqFBx9NPz0k8fiYkyftZp++QVOPRUuuggGDoS774Z11qn5GGUWjnKLjpqB\nlOjdu7fvEhKnrsz69IHHHnODvvbZx40lkJr0Wfvdl1/CAQe4BuCOO+Cyy+peXliZhaPcoqNmICUG\nDx7su4TEWVVmhx8O06a5U7977eWaA/mdPmvOiy9C27bw+ecwdapbWGhVlFk4yi06agZSQrMvgltd\nZjvvDK++6gaDFRa6teRX3OQon+X7Z81at7/FgQfCDjvA66/D3nuv/jn5nllYyi06agZEVqFZMzew\ncMgQGDzYjSP4/nvfVYlPixa5VQTPOgvOOAOeeQZatPBdlUjDqRkQWY211nK7yz3yCDz/vFus6NVX\nfVclPsyc6caR3HMP3H67G2i69tq+qxLJDjUDKTF27FjfJSROkMy6doU333R/Be63H1x9df5eNsiH\nz9qHH37IxIkTmTVrFuAGB7Zt67bAnjbN7X4ZRD5klgvKLTpqBlKisjLQYlNC8My22cYtUHTeeW4L\n2q5d83OjozR/1hYsWMBhhx3BTjvtRJcuXWjZck+22OJpevZ0u12+/jq0ahX8ddOcWS4pt+hoOWKR\nEJ580o0eb9wY/v1vOOww3xVJNhx22BE8/fQrLF06EjgYWBdYi1atbuLttwd4rk5k9bQcsUjEDjvM\nXTZo1cpNRTzzTC1SlHQffvghkyY9wdKlo4EeQHOgGfAM77xT9NslA5E0UjMgEtLmm7szBDfe6Bad\nad3aXUaQZPr444+BVsDfan3FnaX86KOPoi5JJDJqBkQawBh3VuCtt2CLLdxqdP37w88/+65Mgvjl\nF5g4cS/gdaD2trnPA7DDDjtEXZZIZNQMpEQmk/FdQuJkM7Ptt3dLGF99Ndx8M+y6a3pXLkzbZ23q\nVNhjD7jxxo3ZbruHWGutXYC7gc+Bu2nU6BwOPbQLO+64Y+hjpC2zqCi36KgZSIl+/fr5LiFxsp1Z\no0Zw/vnw7ruwyy5u5cK//92tX58mafmsffedWzioY0f4v/+DN96A11/vzCGHtAFOAgqAkzj44HaU\nld3doGOlJbOoKbfoaDaBSA5YC/fe67ZDrqpyqxj26aNFauJg2TIYN87tMrhoEQwb5pqCtVb402jW\nrFl89NFH7LDDDg06IyASJc0mEIkZY6B7d7dq3YknujMGrVrB44+7RkH8qKhwm0/94x/QqRO8/75b\nWnitWj8Jd9xxRw4//HA1ApI31AyI5NCGG8KYMe4U9JZbuoWKDjsM3nvPd2X55bPP4PjjoUMHdznn\npZfcssJbbOG7MpF4UDOQEuXl5b5LSJwoM9t9d3j6aSgvh48/dv/u1Qs+/TSyErImSZ+1r7+Gf/4T\nWrZ0AzzHjYNXXoH27aOtI0mZxYlyi46agZQoKyvzXULiRJ2ZMXDkke6swHXXwcSJ7pdUnz7wxReR\nltIgSfisLVgA//qXm+Vx110waBDMmgWnnLLyJYEoJCGzOFJu0dEAQhFPFi6EG26AkhL3/884w40t\nKCjwXVlyzZ0L11/vcl261A3gHDDAXa4RSTsNIBRJoPXWcxseffKJG9l+992w3XZw0knw9tu+q0uW\njz92AwG33tptLXzaaTB7NgwdqkZApD7UDIh41qwZDBzoBrkNH+6WNG7d2u158OST+btV8ppY67Lq\n1s1dbnnwQbj0UpgzB665Bjbd1HeFIsmhZkAkJtZf3w12++gjd5bgq69cQ7DDDu5Swjff+K4wHn78\n0V0GaNUK9t/fbRg1cqRrpi6+2C0gJCLBqBlIiV69evkuIXHimlmTJtCjh5uO+NJLbjrcoEFuamL3\n7m6tgiVL/NXnI7elS+HZZ6F3b7dB1DnnwE47uRkaM2dC376w7rqRl1Vvcf2sxZ1yi46agZTo3Lmz\n7xISJ+6ZGeOmwN1xh1vSuKQE3nnHrVWw2WZuFkJFRfSXEaLKzVrXEA0Y4AZVduoEzz8PRUXuLMCD\nD7r7jImknAaJ+2ctrpRbdHI+m8AY0xcYALQA3gLOtta+torHajaByGpY6wYXjh8PZWXw+efuL+Wu\nXd1eCJ06xfsv5DVZssSNA3jkEXf79FPYZBM3LqBHD9hnn2T88hfxoSGzCRrnpiTHGNMNuBY4HZgG\n9AcmGWNaWmvn5/LYImlkjBtc2Lo1XHWVOzNQXg6PPgq33OIagQMPhIMOctsp77GHW3EvrpYtc+su\nTJkCzz3nLgX88ANstZVrbjIZ916aNPFdqUi65fTMgDHmFeBVa+051f82uH1BR1prS+t4vM4MiIRg\nLXzwgds2eeJEePlltwlPs2ZuzMHee0PbttCmjbvE4Mu338L06fD66/Daa66ZmT/fbeDUrp07s5HJ\nuGZHZwBEgonlmQFjTBOgLXDl8vustdYY8zQQ8WKg6VdRUcF+++3nu4xESVNmxsDOO7vbgAGweLH7\nZTtlijvtfv31blU+cM3Abru56Xg77eT+d9tt3f1//OOaj7Wm3BYudGMcPv/cNSjLbzNn/r78crNm\n8Je/uIWWDjzQjY1o2rTBMcRWmj5rUVJu0cnlZYKNgUbA3Fr3zwV2yuFx81Jpaan+owkozZmtvTb8\n9a/udvHF7szBZ59BZaX7y3zGDNco3HYb/PLL789bbz3XFDRv7n5hr7/+77e11nKv8+ijpXTtuh//\n+5+b5vfTT+5/v/3WNQE//PD76zVu7JYE3mknOPZY2HNP1wRsv72fZYF9SfNnLZeUW3Ri+Z9jly5d\nyGQyNW7t27dfadOKyZMnk8lkVnp+3759GTt2bI37KisryWQyzJ9fc6jCoEGDKCkpqXHfnDlzyGQy\nzJw5s8b9o0aNoqioqMZ9VVVVZDIZKioqatxfVlZW57SYbt265eR97Lbbbql4H1F+PyZMmJCK9wFr\n/n4YA9tsA8ccA99/35cjjhjLu++6v+I/+QRuvrmSNm0yFBfP58gj3UqI66wDr78+iKefLuG559w0\nvmeegcaNr2bcuAwvvzzzt1P8220Hm246itati7j7bnft/8MPYf78Klq2zFBUVEFpqds5cMcd4d57\n0/u5qut9TJgwIRXvA6L9fkyYMCEV7wOy//0oKyv77XdjixYtyGQy9O/ff6Xn1FfOxgxUXyaoAv5m\nrX1khfvHARtYa4+u4zkaMyAiIhJCLPcmsNYuAaYDnZbfVz2AsBPwUq6OKyIiIsHkdGohMBwYZ4yZ\nzu9TC5sC43J8XBEREamnnI4ZsNbeh1twaAjwBrA7cKi1dl4uj5uPal+LkjVTZuEot+CUWTjKLTq5\nPjOAtXYMMCbXx8l3BQUFvktIHGUWjnILTpmFo9yik/PliIPQAEIREZFwYjmAUERERJJBzYCIiEie\nUzOQErUXuJA1U2bhKLfglFk4yi06agZSori42HcJiaPMwlFuwSmzcJRbdNQMpMTo0aN9l5A4yiwc\n5RacMgtHuUVHzUBKaApOcMosHOUWnDILR7lFR82AiIhInlMzICIikufUDKRE7W00Zc2UWTjKLThl\nFo5yi46agZSoqqryXULiKLNwlFtwyiwc5RYdLUcsIiKSAlqOWEREREJTMyAiIpLn1AykxPz5832X\nkDjKLBzlFpwyC0e5RUfNQEr07t3bdwmJo8zCUW7BKbNwlFt01AykxODBg32XkDjKLBzlFpwyC0e5\nRUfNQEpo9kVwyiwc5RacMgtHuUVHzYCIiEieUzMgIiKS59QMpMTYsWN9l5A4yiwc5RacMgtHuUVH\nzUBKVFYGWmxKUGZhKbfglFk4yi06Wo5YREQkBbQcsYiIiISmZkBERCTPqRkQERHJc2oGUiKTyfgu\nIXGUWTjKLThlFo5yi46agZTo16+f7xISR5mFo9yCU2bhKLfoaDaBiIhICmg2gYiIiISmZkBERCTP\nqRlIifLyct8lJI4yC0e5BafMwlFu0VEzkBIlJSW+S0gcZRaOcgtOmYWj3KKTs2bAGPOpMWbZCrel\nxpjiXB0v322yySa+S0gcZRaOcgtOmYWj3KLTOIevbYFLgFsBU33fTzk8noiIiISQy2YA4Gdr7bwc\nH0NEREQaINdjBv5ljJlvjKk0xgwwxjTK8fFEREQkoFyeGbgeqAQWAPsCw4AWwIDVPGcdgBkzZuSw\nrHSaNm2a9v4OSJmFo9yCU2bhKLdgVvjduU7Q5wZagdAYcxVwwWoeYoFdrLUf1vHcnsDNwPrW2iWr\neP0TgHvqXZCIiIjU1sNaOz7IE4I2A38C/rSGh8221v5ax3P/DLwD7GytnbWa1z8U+BT4X70LExER\nkXWAbYBJ1tpvgzwxsr0JjDE9gHHAxtbaHyI5qIiIiKxRTsYMGGPaAfsAz+GmE+4LDAfuUiMgIiIS\nLzk5M2CM2RMYA+wE/AH4BLgTuG5V4wVERETEj1htYSwiIiLR094EIiIieU7NgIiISJ6LdTNgjDnC\nGPOKMabKGLPAGPMf3zUlhTFmbWPMm9WbRO3uu564MsZsbYy5zRgzu/pzNssYM9gY08R3bXFjjOlr\njPnEGLOo+r/LvXzXFGfGmAuNMdOMMT8aY+YaYx4yxrT0XVeSGGP+Vf0zbLjvWuLOGLO5Meau6lV/\nq4wxbxlj2tT3+bFtBowxf8MNOhwLtMLNSAi0iEKeKwX+i1sISlZtZ9xGWqcBfwb6A2cCV/gsKm6M\nMd2Aa4FBwJ7AW8AkY8zGXguLtw7AKNzMqoOBJsBkY8y6XqtKiOpm83TcZ01WwxizIfAi8AturZ5d\ngPOB7+r9GnEcQFi9h8GnwKXW2nF+q0keY8zhwDXA34D3gT2stW/7rSo5jDEDgDOttTv4riUujDGv\nAK9aa8+p/rcBPgdGWmtLvRaXENWN0zdAR2tthe964swYsz4wHTgLuBR4w1p7nt+q4ssYMwxob63d\nP+xrxPXMQBtgc4DqTY6+NMY8YYzZ1XNdsWeMaQ7cApwILPJcTlJtiNtTQ4DqSyZtgWeW32fdXxFP\nA+191ZVAG+LO1OmztWY3AI9aa5/1XUhCFAKvG2Puq74kVWmMOTXIC8S1GdgOd+p2EDAEOAJ3umNK\n9ekQWbXbgTHW2jd8F5JExpgdgH7ATb5riZGNgUbA3Fr3z8VtPiZrUH0mZQRQYa1933c9cWaM6Q7s\nAVzou5YE2Q53FuUDoDNwIzDSGHNSfV8g0mbAGHNV9WCQVd2WVg+wWV7XUGttefUvtl64rvrYKGuO\ng/rmZoz5J7A+ULL8qR7L9irAZ23F52wBTATutdb+20/lklJjcGNSuvsuJM6MMVvimqYeWqAukLWA\n6dbaS621b1lrbwVuxY1/qpdcbmFcl2twf7muzmyqLxEAv+3HaK1dbIyZDRTkqLY4q09unwAH4k7b\n/uL+EPnN68aYe6y1vXJUXxzV97MGuJG4wLO4v9zOyGVhCTQfWAo0r3V/c+Dr6MtJFmPMaKAL0MFa\n+5XvemKuLbAJUGl+/yHWCOhojOkH/MHGcaCbf1+xwu/LajOAY+r7ApE2A9W7KK1xJyVjzHTcqMid\ngJeq72uC243psxyWGEsBcjsbuHiFuzYHJgHHAdNyU1081Tcz+O2MwLPAa0DvXNaVRNbaJdX/TXYC\nHoHfTnt3Akb6rC3uqhuBI4H9rbVzfNeTAE/jZo+taBzuF9swNQKr9CLu9+WKdiLA78uozwzUi7X2\nJ2PMTcBlxpj/4t5QMe4ywf1ei4sxa+1/V/y3MWYh7lLBbGvtl36qirfqMwJTcGdWioFNl/9BYq2t\nfY08nw0HxlU3BdNwUzCb4n5QSx2MMWOA44EMsLB6cC/AD9ZabdFeB2vtQtwMqN9U/xz71lpb+y9f\n+d11wIvGmAuB+3DTWU/FTZmul1g2A9UGAEtwaw2sC7wKHKRdDwNTJ716h+AG32yHmyoHroGyuNOT\nAlhr76ueGjcEd3ngTeBQa+08v5XF2pm4z9GUWvf3wv1ck/rRz7A1sNa+bow5GhiGm4r5CXCOtXZC\nfV8jlusMiIiISHTiOrVQREREIqJmQEREJM+pGRAREclzagZERETynJoBERGRPKdmQEREJM+pGRAR\nEclzagZERETynJoBERGRPKdmQEREJM+pGRAREclz/w/osRhIM3hMDgAAAABJRU5ErkJggg==\n", "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "x = np.arange(-5, 5, 0.01)\n", "plt.plot(x, x**2, 'b')\n", "plt.grid(1)\n", "plt.scatter(1.5, 1.5**2)" ] }, { "cell_type": "code", "execution_count": 251, "metadata": { "collapsed": false }, "outputs": [ { "data": { "text/plain": [ "(2.9999999999996696, 3.0)" ] }, "execution_count": 251, "metadata": {}, "output_type": "execute_result" } ], "source": [ "def f(x):\n", " return x**2\n", "epsilon = 1e-4\n", "x = 1.5\n", "\n", "numericGradient = (f(x+epsilon) - f(x-epsilon))/(2 * epsilon)\n", "# verification:\n", "numericGradient, 2*x" ] }, { "cell_type": "code", "execution_count": 253, "metadata": { "collapsed": false }, "outputs": [ { "data": { "text/plain": [ "array([-0.10832181, 0.29157752, -2.32875267, -0.79858892, 0.77523413,\n", " 0.01438056, 0.45891691, -1.01902928, 0.55968482])" ] }, "execution_count": 253, "metadata": {}, "output_type": "execute_result" } ], "source": [ "NN.getParams()" ] }, { "cell_type": "code", "execution_count": 260, "metadata": { "collapsed": false }, "outputs": [ { "data": { "text/plain": [ "array([-0.0269274 , -0.00625294, 0.0535152 , -0.01712459, -0.00425763,\n", " 0.03704035, 0.0800578 , 0.12394559, 0.0490642 ])" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/plain": [ "array([ 0.02535875, 0.00589676, -0.05122405, 0.01697504, 0.00424728,\n", " -0.03729985, -0.07870872, -0.1205269 , -0.04933063])" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/plain": [ "73.212293905780498" ] }, "execution_count": 260, "metadata": {}, "output_type": "execute_result" } ], "source": [ "def computeNumericalGradients(N,X,y):\n", " paramsInitial = N.getParams()\n", " numgrad = np.zeros(paramsInitial.shape)\n", " perturb = np.zeros(paramsInitial.shape)\n", " e = 1e-4\n", " for p in range(len(paramsInitial)):\n", " # set perturbation vector\n", " perturb[p] = e\n", " N.setParams(paramsInitial + perturb)\n", " loss1 = N.costFunction(X,y)\n", " N.setParams(paramsInitial - perturb)\n", " loss2 = N.costFunction(X,y)\n", " \n", " #compute numerical gradient\n", " numgrad[p] = (loss2 - loss1) / (2*e)\n", " \n", " #reset\n", " perturb[p] = 0\n", " #reset\n", " N.setParams(paramsInitial)\n", " return numgrad\n", "\n", "numgrad = computeNumericalGradients(NN,X,y)\n", "grad = NN.computeGradients(X,y)\n", "display(numgrad, grad) # similar \n", "\n", "np.linalg.norm(grad - numgrad) / np.linalg.norm(grad+numgrad) # not sure about this" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# 6 Training\n", "BFGS Quasi-Newton optimization of SGD - leverages Hessian" ] }, { "cell_type": "code", "execution_count": 266, "metadata": { "collapsed": true }, "outputs": [], "source": [ "from scipy import optimize\n", "\n", "class Trainer(object):\n", " def __init__(self,N):\n", " self.N = N\n", " \n", " def costFuncWrapper(self, params, X, y):\n", " self.N.setParams(params)\n", " cost = self.N.costFunction(X,y)\n", " grad = self.N.computeGradients(X,y)\n", " return cost, grad\n", " \n", " def callbackF(self, params):\n", " self.N.setParams(params)\n", " self.J.append(self.N.costFunction(self.X, self.y))\n", " \n", " def train(self, X, y):\n", " self.X = X\n", " self.y = y\n", " self.J = [] # costs\n", " params0 = self.N.getParams()\n", " options = {'maxiter': 200, 'disp': True}\n", " # set jacobian\n", " _res = optimize.minimize( \\\n", " self.costFuncWrapper, params0, jac=True, method='BFGS', args=(X,y), options=options, callback=self.callbackF)\n", " self.N.setParams(_res.x)\n", " self.optimizationResults = _res" ] }, { "cell_type": "code", "execution_count": 280, "metadata": { "collapsed": false }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Warning: Desired error not necessarily achieved due to precision loss.\n", " Current function value: 0.040577\n", " Iterations: 3\n", " Function evaluations: 96\n", " Gradient evaluations: 84\n" ] }, { "data": { "text/plain": [ "" ] }, "execution_count": 280, "metadata": {}, "output_type": "execute_result" }, { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAAikAAAF5CAYAAABa9bp0AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAAPYQAAD2EBqD+naQAAIABJREFUeJzt3XmYFNX1//H3YRHBBRfUcWERN9QYldHEcY8LBtAWV0SN\nCC5RwQUNaBIjxHyNAeMKuCQSccsgJoq4BaLxp0FFzYyaTTRREFeEuKAOuMD5/XF7ZKZnhmZmqruq\naz6v5+knTPWt6lOeNHOoqnuuuTsiIiIiSdMu7gBEREREGqMiRURERBJJRYqIiIgkkooUERERSSQV\nKSIiIpJIKlJEREQkkVSkiIiISCKpSBEREZFEUpEiIiIiiaQiRURERBIpMUWKmY0ws/lmtszM5prZ\nnqsZW2Zmd5vZq2a2wsyuaWTMTmb2h+wxV5rZeYU9AxEREYlSIooUMxsMXA2MBXYHXgZmmVm3Jnbp\nBHwA/AJ4qYkxXYDXgYuB9yINWERERArOkrDAoJnNBZ5z9/OzPxvwFnCDu0/Is+8TwIvufuFqxswH\nrnX3GyIMW0RERAoo9ispZtYRKAcer93moXJ6DKiIKy4RERGJV+xFCtANaA8sytm+CCgrZiBm1sXM\n+ppZl2J+roiISKkrxO/QDlEdKCV2A54Gqs3ss5z3/gTMKn5IIiIiiXMY8P2cbesCfYF9gGei+JAk\nFClLgBXAZjnbNwPeL3IsvbL/27eR9/YHflm8UEREREpSL9JSpLj7V2ZWBRwMzIRvHpw9GCj2g64L\nAO666y523HHHIn+0FMKoUaO49tpr4w5DIqJ8povymS6vvPIKJ598MmR/l0Yh9iIl6xpgarZYeR4Y\nRZhCPBXAzK4EtnD3obU7mNmugBEuL22S/flLd38l+35HYKfsmLWALbNjPnP315uIYznAjjvuSN++\njV1MkVLTtWtX5TJFlM90UT5Ta3lUB0pEkeLu07M9US4n3OZ5CTjM3Rdnh5QB3XN2exGonT/dFzgR\neBPond22Rc6YH2VfTwIHFeA0JIHef7/YdwylkJTPdFE+JZ9EFCkA7n4jcGMT7w1rZNtqZya5+5sk\nY/aSxOidd96JOwSJkPKZLsqn5KNf4pJq5eXlcYcgEVI+00X5lHxUpEiqDRkyJO4QJELKZ7oon5KP\nihRJNf0lmC7KZ7oon5KPihQRERFJJBUpkmrDhjV45lpKmPKZLsqn5KMipREJWBhaItKvX7+4Q5AI\nKZ/ponxKPipSGnHzzXFHIFHRPe90UT7TRfmUfFSkNOLWW8NLRERE4qMipRHHHgtnnQV/+lPckYiI\niLRdKlIaMXo09O8Pxx0HL74YdzTSGnPmzIk7BImQ8pkuyqfkoyKlER06wLRp0KcPDBgAb74Zd0TS\nUhMmTIg7BImQ8pkuyqfkoyKlCeusAw89BGuvHQqVjz+OOyJpiWnTpsUdgkRI+UwX5VPyUZGyGptt\nBo8+Cu+9B0cdBV98EXdE0lxdunSJOwSJkPKZLsqn5KMiJY8+feCBB+DZZ2H4cPVQERERKRYVKWtg\nv/3gjjvg97+Hn/407mhERETaBhUpa+j44+Gqq+DKK+GWW+KORtbU6NGj4w5BIqR8povyKfl0iDuA\nUnLRRWGmzznnwFZbwcCBcUck+fTo0SPuECRCyme6KJ+Sj7kesviGmfUFqqqqqujbt2+jY1asgGOO\ngT//GZ58EvbYo7gxioiIJFF1dTXl5eUA5e5eHcUxdbunmdq3D8+mfOtbcPjhsGBB3BGJiIikk4qU\nFujSBR58MPRS6d8fPvww7ohERETSR0VKC226aeihsnixeqgk2bx58+IOQSKkfKaL8in5qEhphe23\nh5kz4bnnYOhQWLky7ogk15gxY+IOQSKkfKaL8in5qEhppb33hrvvhunT4cc/jjsayTVp0qS4Q5AI\nKZ/ponxKPipSInDMMXDNNTBhAkyeHHc0UpemOKaL8pkuyqfkoz4pEbnggjDT57zzoHt3yGTijkhE\nRKS06UpKhK6+GgYNghNOgBdeiDsaERGR0qYiJULt28Ndd8Guu4YeKm+8EXdEMn78+LhDkAgpn+mi\nfEo+KlIi1rlzmPGz/vqhh8r//hd3RG1bTU1N3CFIhJTPdFE+JR+1xa9jTdrir6n//hcqKmCHHUIL\n/c6do4lRREQkidQWv4Rsu23oSltdDaecoh4qIiIizaUipYD22ius8/PHP4JWJBcREWkeFSkFNmgQ\nXH996KNyww1xR9P2LFmyJO4QJELKZ7oon5KPipQiOPdcuOii0Etlxoy4o2lbhg8fHncIEiHlM12U\nT8knMUWKmY0ws/lmtszM5prZnqsZW2Zmd5vZq2a2wsyuaWLccWb2SvaYL5tZ/8KdwepNmBA60w4Z\nAnPnxhVF2zNu3Li4Q5AIKZ/ponxKPokoUsxsMHA1MBbYHXgZmGVm3ZrYpRPwAfAL4KUmjrk38Hvg\nt8BuwAPADDPbKdro10y7dnDnnVBeDkccEWb/SOG1dpaWJIvymS7Kp+STiCIFGAXc4u53uPs84Cyg\nBmj0WqC7v+nuo9z9LmBpE8c8D3jU3a9x91fd/TKgGhhZgPjXyNprwwMPwEYbhR4qixfHFYmIiEjy\nxV6kmFlHoBx4vHabh+YtjwEVrTh0RfYYdc1q5TFbbeON4dFHYenSsL7PsmVxRiMiIpJcsRcpQDeg\nPbAoZ/sioKwVxy0rwDEj0bs3PPQQvPwynHQSrFgRd0TpNWXKlLhDkAgpn+mifEo+SShS2qQ994R7\n7gm3fy66KO5o0qu6OpKmh5IQyme6KJ+STxKKlCXACmCznO2bAe+34rjvt/SYAwYMIJPJ1HtVVFQw\nI2f+8OzZs8lkMg32HzFiRIN/IVRXV5PJZOr1BTjiCDjssLFcf/14rrtu1diFCxeSyWSYN29evWNM\nnDiR0Tld4WpqashkMsyZM6fe9srKSoYNG9YgtsGDB0d+HgBjx45tsFhYEs5j8uTJqTgPSEc+Wnse\nkydPTsV5QDry0drzmDx5cirOo1ZbOo/KyspvfjeWlZWRyWQYNWpUg31aKxFr95jZXOA5dz8/+7MB\nC4Eb3P2qPPs+Abzo7hfmbJ8GdHb3I+tsexp42d3PaeJYka3d0xwXXwxXXQX33humKYuIiJSaQqzd\n0yGKg0TgGmCqmVUBzxNm+3QBpgKY2ZXAFu4+tHYHM9sVMGBdYJPsz1+6+yvZIdcD/8/MLgQeBoYQ\nHtA9oyhn1AxXXgkLF4bnUzbfHPbeO+6IRERE4peIIsXdp2d7olxOuCXzEnCYu9dO0i0Duufs9iJQ\nexmoL3Ai8CbQO3vMZ83sROCK7Os/wJHu/u9CnktLtGsHU6fCu++GGT/PPAPbbx93VCIiIvFKwjMp\nALj7je7ey907u3uFu/+tznvD3P2gnPHt3L19zqt3zpg/unuf7DG/7e6zinU+zdWpE9x/P2yySeih\n8sEHcUeUDo3d85XSpXymi/Ip+SSmSJHQ5O3RR+Hzz8NDtZ9/HndEpW/kyNh690kBKJ/ponxKPipS\nEqZXL3j4YfjXv+DEE9VDpbX69esXdwgSIeUzXZRPyUdFSgKVl8P06aHh2/nnQwImYImIiBSdipSE\nGjAAbroJJk+Gq6+OOxoREZHiU5GSYGeeCT/+MYweHa6sSPPlNlCS0qZ8povyKfmoSEm4K64Iz6b8\n4Afw17/GHU3pqaysjDsEiZDymS7Kp+STiI6zSRFXx9l8vvgCvv/9sCDhM89Anz5xRyQiIlJfITrO\n6kpKCajtobL55qGHyvutWdFIRESkRKhIKREbbBB6qHzxBRx+uHqoiIhI+qlIKSE9eoQeKq++Ciec\nAF9/HXdEIiIihaMipcTsvntYLfnRR+Hcc9VDJZ/GlhuX0qV8povyKfmoSClB3/8+3HIL3HwzTJgQ\ndzTJpo6W6aJ8povyKfkkYhVkab7TToM334RLLoHu3cM0ZWloyJAhcYcgEVI+00X5lHxUpJSwn/88\nFCrDhsGWW8IBB8QdkYiISHR0u6eEmcFvfwv77QeDBsG//x13RCIiItFRkVLi1loL/vhH2Gqr0EPl\nvffijihZ5syZE3cIEiHlM12UT8lHRUoKdO0KjzwCK1bAwIHw2WdxR5QcE/Rkcaoon+mifEo+KlJS\nonv30EPlv/+F449XD5Va06ZNizsEiZDymS7Kp+SjIiVFdt013Pr585/h7LPVQwWgS5cucYcgEVI+\n00X5lHxUpKTMoYeGh2lvvRV++cu4oxEREWk5TUFOoVNPDVOTL70UevaEk0+OOyIREZHm05WUlLrs\nstA/Zfhw+Mtf4o4mPqNHj447BImQ8pkuyqfkoyIlpcxC6/wDD4SjjoJ//jPuiOLRo0ePuEOQCCmf\n6aJ8Sj7merryG2bWF6iqqqqib9++cYcTiaVLYf/94X//g7lzQ2daERGRqFVXV1NeXg5Q7u7VURxT\nV1JSbv31w9RkCD1Uli6NNx4REZE1pSKlDdhyy9Dsbf58OO44+OqruCMSERHJT0VKG7HLLnD//fDE\nE/DDH7adHirz5s2LOwSJkPKZLsqn5KMipQ056CCYMgVuuw1+8Yu4oymOMWPGxB2CREj5TBflU/JR\nn5Q25gc/gIULQw+VHj1CT5U0mzRpUtwhSISUz3RRPiUfFSlt0E9+AgsWwBlnhOdVDj007ogKR1Mc\n00X5TBflU/LR7Z42yAxuvBEOOQSOOQb+/ve4IxIREWlIRUob1bEjTJ8O224LAwbA22/HHZGIiEh9\nKlLasPXWCz1U2rcPhconn8QdUfTGjx8fdwgSIeUzXZRPyUdFShu3+ebw6KPhYdpjjoEvv4w7omjV\n1NTEHYJESPlMF+VT8klMkWJmI8xsvpktM7O5ZrZnnvEHmlmVmS03s9fMbGjO+x3M7DIz+2/2mC+a\n2WGFPYvStNNOMGMG/PWv4WHaNPVQ+fnPfx53CBIh5TNdlE/JJxFFipkNBq4GxgK7Ay8Ds8ysWxPj\newEPAY8DuwLXA7eaWd15KlcAZwAjgB2BW4D7zWzXwpxFaTvwwNA/5Y47YOzYuKMRERFJSJECjAJu\ncfc73H0ecBZQAwxvYvzZwBvuPsbdX3X3ycAfssepdTJwhbvPcvcF7n4z8AhwUeFOo7SdeCJceWVo\n9DZlStzRiIhIWxd7kWJmHYFywlURADwszfwYUNHEbntl369rVs74TsAXOWOWAfu2Jt60u/ji0Db/\nhz+EWbPijqb1lixZEncIEiHlM12UT8kn9iIF6Aa0BxblbF8ElDWxT1kT49c3s07Zn2cBF5rZthYc\nChwNbB5N2OlkBpMmwfe/D8ceCy++GHdErTN8eFMX46QUKZ/ponxKPkkoUgrlfOA/wDzCFZUbgN8B\nK/PtOGDAADKZTL1XRUUFM2bMqDdu9uzZZDKZBvuPGDGCKTn3S6qrq8lkMg3+5TB27NgG0/AWLlxI\nJpNpsPjWxIkTGT16dL1tNTU1ZDIZ5syZU297ZWUlw4YNaxDb4MGD855Hhw4wbRp07jyC731vCgsX\nluZ5AIwbN67k81FL5xHymYbzgHTko7XnMW7cuFScR622dB6VlZXf/G4sKysjk8kwatSoBvu0lnnM\nUzmyt3tqgGPcfWad7VOBru5+VCP7PAlUufuFdbadClzr7hvmjF0L2Njd3zOzXwED3X2XJmLpC1RV\nVVXRt2/f1p9ciXv/faiogHXWgTlzYIMN4o5IRESSqrq6mvLycoByd6+O4pixX0lx96+AKuDg2m1m\nZtmfn2lit2frjs/ql92ee/wvswVKR+AYYEbuGGlcWVnoofLuu3DUUfBF7hM+IiIiBRR7kZJ1DXCG\nmZ1iZn2Am4EuwFQAM7vSzG6vM/5moLeZjTezHczsHODY7HHI7vMdMzvKzLY2s/2ARwEDrirOKaVD\nnz7wwAPwzDNw2mnp6qEiIiLJlogixd2nAz8CLgdeBL4NHObui7NDyoDudcYvAAYChwAvEaYen+bu\ndWf8rA38H/Av4I/AW8C+7r60oCeTQvvtF/qn3H03XHpp3NE0T+69XSltyme6KJ+STyKKFAB3v9Hd\ne7l7Z3evcPe/1XlvmLsflDP+KXcvz47fzt3vbOT9nd29i7tvmj3G+8U6n7QZPBgmTIBf/hJ+85u4\no1lz1dWR3BaVhFA+00X5lHxif3A2SfTg7Oq5w7nnws03w8yZYVFCERERSOmDs1I6zOD662HgQDj+\neKiqijsiERFJMxUp0izt20NlJey8cyhWFiyIOyIREUkrFSnSbF26wIMPhv4p/fvDRx/FHZGIiKSR\nihRpkU03DT1UPvgABg1Kbg+Vxjo6SulSPtNF+ZR8VKRIi22/fXiA9rnn4NRTYWXeBQeKb+TIkXGH\nIBFSPtNF+ZR8VKRIq+yzD9x1F9xzD/zkJ3FH01C/fv3iDkEipHymi/Ip+ahIkVY79li4+moYPx5u\nuinuaEREJC06xB2ApMMFF4SZPiNHwlZbwRFHxB2RiIiUOl1JkUiYwTXXwJFHwgknwAsvxB1RkLs8\nupQ25TNdlE/JR0WKRKZ9+7C+z7e/DYcfDm+8EXdEUFlZGXcIEiHlM12UT8lHbfHrUFv8aCxeDHvv\nHYqWp5+GjTeOOyIRESk0tcWXkrDJJqGHyv/+F27/LF8ed0QiIlKKVKRIQWy7behKW1UFp5ySzB4q\nIiKSbCpSpGD22gt+/3v4wx9gzJi4oxERkVKjIkUK6qij4LrrQh+ViROL//nDhg0r/odKwSif6aJ8\nSj7qkyIFd9558OabcP750L17WOunWNTRMl2Uz3RRPiUfze6pQ7N7CmflSjj+eHj4YXjiiXArSERE\n0kOze6RktWsHd94JffuGbrSvvx53RCIiknQqUqRoOncOqyZvuCH07w9LlsQdkYiIJJmKFCmqjTcO\nPVQ+/hgyGVi2rLCfN2fOnMJ+gBSV8pkuyqfkoyJFim6bbeChh+Cll+Dkk2HFisJ91oQJEwp3cCk6\n5TNdlE/JR0WKxOI734Fp02DGDPjRjwr3OdOmTSvcwaXolM90UT4lHxUpEptMBm64IfRRue66wnxG\nly5dCnNgiYXymS7Kp+SjPikSqxEjQg+VCy8MPVSOOSbuiEREJClUpEjsfvUrWLgwPJ+y+eZhBWUR\nERHd7pHYtWsHU6fCnnuGW0CvvRbdsUePHh3dwSR2yme6KJ+Sj4oUSYS11w4P0W6ySeih8sEH0Ry3\nR48e0RxIEkH5TBflU/JRW/w61BY/fvPnQ0UF9OwZ2ufruToRkdKgtviSeltvHXqo/POfcOKJhe2h\nIiIiyaYiRRJnjz1g+nR48EG44ALQxT4RkbZJRYok0sCBcOONMGkSXHNNy48zb9686IKS2Cmf6aJ8\nSj4qUiSxfvhDuOSS0JH23ntbdowxY8ZEG5TESvlMF+VT8klMkWJmI8xsvpktM7O5ZrZnnvEHmlmV\nmS03s9fMbGgjYy4ws3lmVmNmC83sGjPrVLizkKhdcUV4NuUHP4CWrEU2adKk6IOS2Cif6aJ8Sj6J\nKFLMbDBwNTAW2B14GZhlZt2aGN8LeAh4HNgVuB641cwOrTPmRODK7DH7AMOB44ErCnUeEr127eB3\nvwszfjIZePXV5u2vKY7ponymi/Ip+SSiSAFGAbe4+x3uPg84C6ghFBaNORt4w93HuPur7j4Z+EP2\nOLUqgDnufo+7L3T3x4BpwHcKdxpSCJ06wX33hW60/fvDokVxRyQiIsUQe5FiZh2BcsJVEQA8NG95\njFBoNGav7Pt1zcoZ/wxQXnvbyMx6AwOAh6OJXIppww3hkUdg2TI4/HD4/PO4IxIRkUKLvUgBugHt\ngdx/Hy8CyprYp6yJ8evXPnPi7pWEWz1zzOxL4D/AE+4+PqrApbh69gyFyiuvwAknwNdf599n/Hil\nO02Uz3RRPiWfJBQpBWFmBwI/Idw62h04GjjczC7Nt++AAQPIZDL1XhUVFcyYMaPeuNmzZ5PJZBrs\nP2LECKZMmVJvW3V1NZlMhiVLltTbPnbs2AZf1IULF5LJZBpMz5s4cWKDtS5qamrIZDLMyXmqtLKy\nkmHDhjWIbfDgwSV9HrvvHmb6PPzwYI44Yka9HiqNnUdNTU0iz6NWqeej2OdRU1OTivOAdOSjtedR\nU1OTivOo1ZbOo7Ky8pvfjWVlZWQyGUaNGtVgn9aKvS1+9nZPDXCMu8+ss30q0NXdj2pknyeBKne/\nsM62U4Fr3X3D7M9PAXPdfUydMScRnn1Zt4lY1Ba/RNx6K5xxBowfD5rFKCISv1S2xXf3r4Aq4ODa\nbWZm2Z+faWK3Z+uOz+qX3V6rC5B7Q2BlneNLCTv9dLj0Urj4YqisjDsaEREphA5xB5B1DTDVzKqA\n5wmzdLoAUwHM7EpgC3ev7YVyMzDCzMYDvyMULMcSHoyt9SAwysxeBp4DtgMuB2Z63JePJBKXXw5v\nvgmnngpbbAEHHBB3RCIiEqVEFCnuPj3bE+VyYDPgJeAwd1+cHVIGdK8zfoGZDQSuBc4D3gZOy04z\nrvULwpWTXwBbAouBmUDeZ1KkNJiF2z7vvAODBsHTT8NOO9Ufs2TJErp1a7TdjpQg5TNdlE/JJ/bb\nPbXc/UZ37+Xund29wt3/Vue9Ye5+UM74p9y9PDt+O3e/M+f9le7+C3ff3t3XyR77PHdfWqxzksJb\na63QQ2WrrWDAAHjvvfrvDx/eVKsdKUXKZ7oon5JPYooUkZbq2jVMTf7qq9BD5bPPVr03bty42OKS\n6Cmf6aJ8Sj4qUiQVuncPhcp//gPHH7+qh4pmaaWL8pkuyqfkoyJFUmPXXeEPf4A//xnOOQf0eLSI\nSGlrUZFiZpeZWZdGtnc2s8taH5ZIy/TrB7/5Dfz2t3DllXFHIyIirdHSKyljgcYaonXJvicSm2HD\nYOxY+OlP4fTTp+TfQUpGbidOKW3Kp+TT0iLFgMYupu8KfNjycESiMXZs6J9y223V/OUvcUcjUamu\njqSJpSSE8in5NKtPipl9RChOHHjNzOoWKu0JV1duji48kZYxC7d93nlnMkcfDXPmwLe+FXdU0lqT\nJ0+OOwSJkPIp+TS3mdsFhKsovyPc1vmkzntfAgvc/dnGdhQpto4dw4O0++0XeqjMnRs604qISGlo\nVpHi7rcDmNl84Gl3z10bRyRR1l8/TE3ea69QqPz1r7DeenFHJSIia6Klz6R8CuxY+4OZHWlmM8zs\nl2a2VjShiURjyy1DoTJ/Phx3XGj6JiIiydfSIuUWYHsAM+sN3APUAMcBE6IJTaT1MpkMALvsEtrn\nP/44nHWWeqiUqtp8Sjoon5JPS4uU7QmLAEIoTJ509xOBU4FjIohLJBIjR4785s8HHwxTpsDvfgf/\n938xBiUtVjefUvqUT8mnpasgG6sKnEOAh7J/fgvQkpaSGP369av38ymnwMKF8LOfQY8eMHRoTIFJ\ni+TmU0qb8in5tLRI+RtwqZk9BhwAnJ3dvjWwKIrARArlpz+FN9+E008Pz6scckjcEYmISGNaervn\nAqAvMAm4wt3/m91+LPBMFIGJFIoZ3HhjKE6OPhr+/ve4IxIRkca0qEhx97+7+y7u3tXdf17nrdGA\nLqBLYsyYMaPR7R07wvTpsM02YWry228XOTBpkabyKaVJ+ZR8WrUKspmVm9nJ2Vdfd1/u7prgKYlR\nWVnZ5HvrrQcPPwzt24dC5ZNPmhwqCbG6fErpUT4lH/MWzMU0s00J044PAD7Obt4AeAI4wd0XRxZh\nEZlZX6CqqqqKvn37xh2OFMm//gX77AN77hmKlrXU6UdEpNmqq6spLy8HKHf3SBZmaumVlImEdXp2\ndveN3H0j4FvA+sANUQQmUiw77wz33w9PPglnnqkeKiIiSdHSIuX7wDnu/krtBnf/NzAC6B9FYCLF\n9L3vwdSpcPvtMG5c3NGIiAi0fApyO6CxZ0++opXPuYjE5cQTw9Tkn/wEevaE4cPjjkhEpG1raUHx\nF+B6M/tmTVkz2xK4Fng8isBEojBs2LBmjb/kknDL58wzYdasAgUlLdbcfEqyKZ+ST0uLlJGE508W\nmNnrZvY6MD+77dyoghNpreZ2tDSDyZPhsMPg2GPhpZfy7yPFow6l6aJ8Sj4tmt0DYGZGaInfJ7vp\nFXd/LKrA4qDZPVLrs8/gwAPh3Xdh7tzQQl9ERJoW++weMzvIzP5tZut78Gd3n+juE4EXzOxfZnZY\nFIGJxGnddeGhh8J05AED4OOP8+8jIiLRau7tnguA37r70tw33P0T4BZ0u0dSoqwMHn00XE05+mj4\n8su4IxIRaVuaW6TsCvxpNe/PBr7d8nBEojVnzpxW7b/jjjBjBjz9NJx2mnqoxK21+ZRkUT4ln+YW\nKZvR+NTjWl8Dm7Q8HJFoTZgwodXH2H//0D/lrrvgZz+LIChpsSjyKcmhfEo+ze2T8g6hs+x/m3j/\n28B7rYpIJELTpk2L5DgnnAALF8LFF4eHaM88M5LDSjNFlU9JBuVT8mlukfII8Asz+5O7L6/7hpl1\nBn4OPBRVcCKt1aVLl8iONXp0aPZ2zjmw1VbhgVoprijzKfFTPiWf5hYp/wccDbxmZpOAV7Pb+xBa\n4rcHroguPJHkMIPrr4e33oLjj4enngLNVBcRKZxmPZPi7ouAvYF/AlcC92dfv8xu2zc7RiSVOnSA\nykrYaScYOBAWLIg7IhGR9Gp2x1l3f9PdBwDdgO8CewHd3H2Au8+POkCR1hg9enTkx1xnHXjwQejc\nOdzy+eijyD9CmlCIfEp8lE/Jp8WLAbr7R+7+grs/7+6t/mvazEaY2XwzW2Zmc81szzzjDzSzKjNb\nbmavmdnQnPefMLOVjbwebG2sUjp6FKhV7GabhR4qixbBUUfBF18U5GMkR6HyKfFQPiWfRKxYbGaD\ngauBscDuwMvALDPr1sT4XoQHdB8n9G65HrjVzA6tM+wooKzO61vACmB6QU5CEunccwvXW3CHHeCB\nB0Lb/FNPhZUrC/ZRklXIfErxKZ+STyKKFGAUcIu73+Hu84CzgBpgeBPjzwbecPcx7v6qu08G/pA9\nDgDu/rG7f1D7AvoBn2fHiURi333hzjvhnnvgJz+JOxoRkXRp7uyeyJlZR6Cc8PAtAO7uZvYYUNHE\nbnsBuYtvm/MgAAAgAElEQVQZzgKuXc1HDQcq3X1ZK8IVaeC448KMn4sugp494eyz445IRCQdknAl\npRth6nLurKBFhNs0jSlrYvz6ZtYpd7CZfQfYGbi1daFKqZk3b15RPmfUKDj3XBg5MixMKIVRrHxK\ncSifkk8SipRiOA34h7tXxR2IFNeYMWOK8jlmcO21kMnA4MHwwgtF+dg2p1j5lOJQPiWfJBQpSwgP\ntG6Ws30z4P0m9nm/ifFL3b3ePAsz6wIMphlXUQYMGEAmk6n3qqioYMaMGfXGzZ49m0wm02D/ESNG\nMGXKlHrbqquryWQyLFmypN72sWPHMn78+HrbFi5cSCaTafCvjIkTJzaYsldTU0Mmk2mwUFdlZSXD\nhg1rENvgwYPb1HlMmjSpaOfRvj389rc1dOqUoV+/OcyvMyFf+YjmPCZNmpSK84B05KO15zFp0qRU\nnEettnQelZWV3/xuLCsrI5PJMGrUqAb7tJZ5ApZ1NbO5wHPufn72ZwMWAje4+1WNjP8V0N/dd62z\n7ffABtkeLnXHngrcCGyZb6q0mfUFqqqqquirVqLSQosXQ0VFaPz29NOw8cZxRyQiUnjV1dWUl5cD\nlLt7dRTHTMKVFIBrgDPM7BQz6wPcDHQBpgKY2ZVmdnud8TcDvc1svJntYGbnAMdmj5PrNGBGFL1c\nRNbEJpuEHipLlsCgQbB8ef59RESkoUQUKe4+HfgRcDnwImE15cPcfXF2SBnQvc74BcBA4BDgJcLU\n49Pcvd6MHzPbntDGXw/MSlFtt13oSvu3v8HQoeqhIiLSEokoUgDc/UZ37+Xund29wt3/Vue9Ye5+\nUM74p9y9PDt+O3e/s5Fjvubu7d39L8U4B0me3Pu1xVRRAXffDffeCxdfHFsYqRJnPiV6yqfkk5gi\nRaQQampqYv38o48Os35+/WvIPiMorRB3PiVayqfkk4gHZ5NCD85KoVx4IVx/Pdx3Hxx5ZNzRiIhE\nL80Pzoqk2q9/HRYiHDIEnnsu7mhEREqDihSRImjXLqzxs/vucMQR8PrrcUckIpJ8KlIk1XKbH8Wp\nc+ewavIGG0D//mGKsjRPkvIprad8Sj4qUiTVhg9vaiHteHTrFnqofPxxaKG/TMtdNkvS8imto3xK\nPipSJNXGjRsXdwgNbLNN6KHy0ktw8smwYkXcEZWOJOZTWk75lHxUpEiqJXWW1ne/C5WVMGMG5Cyn\nIauR1HxKyyifko+KFJGYHHkk3HBD6KNy/fVxRyMikjwd4g5ApC0bMQIWLIBRo6B799D8TUREAl1J\nkVTLXfI8icaPh+OOg5NOgmeeiTuaZCuFfMqaUz4lHxUpkmrV1ZE0PSyodu3g9tthjz3CjJ///Cfu\niJKrFPIpa075lHzUFr8OtcWXOH34Iey9N3z9dbiisummcUckIrLm1BZfJMU22ij0UPn003BFRWuv\niUhbpyJFJEG23hoefhj+8Y/wjIp6qIhIW6YiRSRh9tgD7rkHZs4Ms350R1ZE2ioVKZJqmUwm7hBa\n5PDDYfJkmDgx9FGRoFTzKY1TPiUf9UmRVBs5cmTcIbTYWWfBm2/CRReFHirHHRd3RPEr5XxKQ8qn\n5KMiRVKtX79+cYfQKldcEQqVH/wANt8c9t037ojiVer5lPqUT8lHt3tEEqxdO7jtNthrr9BG/9VX\n445IRKR4VKSIJFynTnD//bDZZtC/PyxaFHdEIiLFoSJFUm3GjBlxhxCJDTcMPVSWLYMjjoDPP487\nonikJZ8SKJ+Sj4oUSbXKysq4Q4hMz56hh8q//w1DhoTOtG1NmvIpyqfkpyJFUu2ee+6JO4RI9e0L\n994LjzwC553X9nqopC2fbZ3yKfmoSBEpMf37w003hddVV8UdjYhI4WgKskgJOuOMMDX54ouhRw84\n4YS4IxIRiZ6KFJES9YtfhEJl6NDQQ+WAA+KOSEQkWrrdI6k2bNiwuEMoGDOYMgX22QcGDYJXXok7\nosJLcz7bIuVT8lGRIqmW9o6Wa60F990HW24ZnlV5//24IyqstOezrVE+JR8VKZJqQ4YMiTuEgttg\ng9BD5auvYOBA+OyzuCMqnLaQz7ZE+ZR8VKSIpED37qGHymuvweDBbbOHioikj4oUkZTYbTf4wx9g\n1iwYMaLt9VARkfRRkSKpNmfOnLhDKKrDDoPf/Ca8fvWruKOJXlvLZ9opn5JPYooUMxthZvPNbJmZ\nzTWzPfOMP9DMqsxsuZm9ZmZDGxnT1cwmm9m72XHzzOz7hTsLSZoJEybEHULRDR8Ol10GP/kJ3H13\n3NFEqy3mM82UT8knEX1SzGwwcDVwJvA8MAqYZWbbu/uSRsb3Ah4CbgROBA4BbjWzd939z9kxHYHH\ngPeBo4F3gZ7Ax4U+H0mOadOmxR1CLMaNCz1Uhg2DLbaA730v7oii0VbzmVbKp+STiCKFUJTc4u53\nAJjZWcBAYDjQWKl9NvCGu4/J/vyqme2bPc6fs9tOAzYA9nL3FdltCwsUvyRUly5d4g4hFmbhls87\n78BRR8HTT8POO8cdVeu11XymlfIp+cR+uyd7xaMceLx2m7s74SpIRRO77ZV9v65ZOeOPAJ4FbjSz\n983sH2b2YzOL/ZxFimGtteCPfwyrJ/fvD+++G3dEIiLNk4Rf2N2A9sCinO2LgLIm9ilrYvz6ZtYp\n+3Nv4DjCOfYHLgcuAn4aQcwiJWH99cPU5JUrQw+VTz+NOyIRkTWXhCKlUNoRCpcz3f1Fd78XuAI4\nK96wpJhGjx4ddwix22oreOQReOMNOO640PStVCmf6aJ8Sj5JKFKWACuAzXK2b0Z46LUx7zcxfqm7\nf5H9+T3gteyto1qvAGVmttpncQYMGEAmk6n3qqioYMaMGfXGzZ49m0wm02D/ESNGMGXKlHrbqqur\nyWQyLFlS/zngsWPHMn78+HrbFi5cSCaTYd68efW2T5w4scGXuqamhkwm02AqX2VlZaPrYgwePLhN\nnUePHj1ScR7Qunx8+9tw000LmT07w5Ah8+r1UCml8+jRo0cq8gHp+P9Va8+jR48eqTiPWm3pPCor\nK7/53VhWVkYmk2HUqFEN9mkt8wR0fDKzucBz7n5+9mcjPOR6g7tf1cj4XwH93X3XOtt+D2zg7gOy\nP18BDHH33nXGnA+MdvetmoijL1BVVVVF3759oztBkYS4/XY49dSwgvKll8YdjYikSXV1NeXl5QDl\n7l4dxTGTcCUF4BrgDDM7xcz6ADcDXYCpAGZ2pZndXmf8zUBvMxtvZjuY2TnAsdnj1LoJ2MjMbjCz\n7cxsIPBjYFIRzkckkYYOhcsvh5/9DO64I+5oRERWLxFTkN19upl1IzzcuhnwEnCYuy/ODikDutcZ\nvyBbdFwLnAe8DZzm7o/VGfO2mR2WHfMy8E72z+oeJG3apZeGHiqnnRZWTz744LgjEhFpXCKKFAB3\nv5HQnK2x9xrcHHP3pwhTl1d3zOeAvSMJUErSvHnz6NOnT9xhJIoZ3HQTvP02HH00zJkDu+wSd1Rr\nRvlMF+VT8knK7R6RghgzZkz+QW1Qx45w773Qu3foofL223FHtGaUz3RRPiUfFSmSapMm6RGkpqy3\nXuih0q5d6KGydGncEeWnfKaL8in5qEiRVKud4iiN22ILePTR8IzKMcckv4eK8pkuyqfkoyJFpI3b\neWe4/3548kk480xIQFcCERFARYqIEFZJvu02mDoVfv7zuKMREQlUpEiq5XZjlKaddBJccUUoUm67\nLe5oGqd8povyKfkkZgqySCHU1NTEHUJJ+fGPw/MpZ54Zeqj06xd3RPUpn+mifEo+iWiLnxRqiy8C\nX38NRx4JTz0Ff/0r7LZb3BGJSClIc1t8EUmIDh3gnntg++3D1OS33oo7IhFpq1SkiEgD664beqh0\n7BiavX38cdwRiUhbpCJFUi13aXNZc2VloYfKO++EHipffhl3RMpn2iifko+KFEm14cOHxx1CSdtx\nR3jggbC+z+mnx99DRflMF+VT8lGRIqk2bty4uEMoefvvD7ffDnfeCZddFm8syme6KJ+Sj6YgS6pp\nllY0TjgBFi6Eiy+GHj3gjDPiiUP5TBflU/JRkSIia2T0aFiwAM4+G7baKjxQKyJSSLrdIyJrxAxu\nuCEUJ8cdB9WRdEEQEWmaihRJtSlTpsQdQqp06ADTpoUHagcODN1pi0n5TBflU/JRkSKpVq1/7kdu\nnXXgoYegc+dwVeWjj4r32cpnuiifko/a4tehtvgia+7VV2HvvWGXXWDWLOjUKe6IRCROaosvIomx\nww6hh8rcuTBsGKxcGXdEIpI2KlJEpMX23Tf0T6mshJ/+NO5oRCRtVKSISKscdxz8+tfwq1/BzTfH\nHY2IpImKFEm1TCYTdwhtwoUXwsiRMGJEeKi2UJTPdFE+JR8VKZJqI0eOjDuENsEMrrsOjjgCBg+G\nv/2tMJ+jfKaL8in5qEiRVOvXr1/cIbQZ7dvD738fZvsMHAjz50f/Gcpnuiifko+KFBGJTJcu8OCD\nsN56oYfKhx/GHZGIlDIVKSISqU02gUcegSVL4MgjYfnyuCMSkVKlIkVSbcaMGXGH0CZtvz3MnBme\nTRk6NLoeKspnuiifko+KFEm1ysrKuENos/beG+66C+69Fy65JJpjKp/ponxKPipSJNXuueeeuENo\n0445Bq65Bq66CiZPbv3xlM90UT4lnw5xByAi6XbBBWG15PPOg622Cs+piIisCV1JEZGC+/WvYdAg\nGDIEnn8+7mhEpFSoSBGRgmvfPjyfsttucPjh8PrrcUckIqVARYqk2rBhw+IOQbI6dw4zfrp2DT1U\nlixp/jGUz3RRPiWfxBQpZjbCzOab2TIzm2tme+YZf6CZVZnZcjN7zcyG5rw/1MxWmtmK7P+uNLOa\nwp6FJI06WiZLt27w6KPw0Ufh2ZRly5q3v/KZLsqn5JOIIsXMBgNXA2OB3YGXgVlm1q2J8b2Ah4DH\ngV2B64FbzezQnKGfAGV1Xj0LEL4k2JAhQ+IOQXJsu21YhPDFF+EHP2heDxXlM12UT8knEUUKMAq4\nxd3vcPd5wFlADTC8ifFnA2+4+xh3f9XdJwN/yB6nLnf3xe7+Qfa1uGBnICJr7LvfDev83Hcf/OhH\ncUcjIkkVe5FiZh2BcsJVESBUFsBjQEUTu+2Vfb+uWY2MX9fMFpjZQjObYWY7RRS2iLTSoEFwww1w\n7bVw/fVxRyMiSRR7kQJ0A9oDi3K2LyLcomlMWRPj1zezTtmfXyVcickAJxHO9Rkz2yKKoKU0zJkz\nJ+4QZDVGjoSLLoJRo+D++/OPVz7TRfmUfJJQpBSEu89197vc/e/u/lfgaGAx8MN8+w4YMIBMJlPv\nVVFR0WCdidmzZ5PJZBrsP2LECKZMmVJvW3V1NZlMhiU5UxrGjh3L+PHj621buHAhmUyGefPm1ds+\nceJERo8eXW9bTU0NmUymwZe9srKy0SfnBw8e3KbOY8KECak4D0hHPho7jwkT4OCDqzn22AyPPrr6\n85gwYUJizwPSkY9inseECRNScR612tJ5VFZWfvO7saysjEwmw6hRuU9ctJ6FOyvxyd7uqQGOcfeZ\ndbZPBbq6+1GN7PMkUOXuF9bZdipwrbtvuJrPmg585e4nNfF+X6CqqqqKvn37tvCMJElqamro0qVL\n3GFIHsuXwyGHwLx58OyzsN12jY9TPtNF+UyX6upqysvLAcrdvTqKY8Z+JcXdvwKqgINrt5mZZX9+\npondnq07PqtfdnujzKwdsAvwXmvildKivwBLw9prwwMPhCnK/fvD4iYecVc+00X5lHxiL1KyrgHO\nMLNTzKwPcDPQBZgKYGZXmtntdcbfDPQ2s/FmtoOZnQMcmz0O2X1+ZmaHmtnWZrY7cDfQA7i1OKck\nIs2x8cbwyCPw6adwxBFQo65GIm1eIooUd58O/Ai4HHgR+DZwWJ0pw2VA9zrjFwADgUOAlwhTj09z\n97ozfjYEfgP8G3gYWBeoyE5xFpEE6t079FD5xz/gpJNgxYq4IxKROCWiSAFw9xvdvZe7d3b3Cnf/\nW533hrn7QTnjn3L38uz47dz9zpz3L3T3rbPvb+HuR7j734t1PpIMuQ+KSfLtuSdMmxZa6I8aBXUf\nm1M+00X5lHwSU6SIFEKPHj3iDkFa4IgjYNIkmDgRrrtu1XblM12UT8kn9tk9SaLZPSLJcsklYYry\n9Olw7LFxRyMiq1OI2T0dojiIiEgh/PKX8OabcPLJsPnmsM8+cUckIsWk2z0ikljt2sHUqWGtn0wG\nnnmm+Ssni0jp0pUUSbV58+bRp0+fuMOQVujUCWbMCFdR9tlnHtCHLbYIM4G22Sa86v65Wzcwiztq\nWRP6fko+KlIk1caMGcPMmTPzD5RE23BDeOEF6NdvDGecMZPXX4c33ggdah9+GOp2C19vvVVFS24h\n06MHdOwY33lIffp+Sj4qUiTVJk2aFHcIEpF11oHKykk0NiFk6dJQtNQWL6+/Hl733Reeaantt9K+\nPfTs2XQRs/76xT2ntk7fT8lHRYqkmqY4pktT+Vx/fdhtt/DK9dVXsHBh/eLljTdg7ly4+2747LNV\nY7t1a7x42Wab8OBuOz3FFyl9PyUfFSkikmodO64qOA49tP577uFWUd3ipfbPTz4J7767auzaa8PW\nWzf+HEyvXuF9EYmWihQRabPMYJNNwmuvvRq+X1MDCxY0LGL+9CeYPx++/HLVcbbcsumrMBttpId5\nRVpCRYqk2vjx47n44ovjDkMiUux8dukCO+0UXrlWroR33ml4Beaf/wwrOn/44aqxXbs2PRtpq62g\nQxv9m1jfT8mnjX41pK2o0VK6qZKkfLZrB927h9eBBzZ8/+OPGz4H8/rrYZbSW2+FIgdCgdKrV+NX\nYXr3hnXXLeZZFVeS8inJpLb4dagtvogUw5dfhllHuVdhav9c93f3Zps1fhWmd28oK9NtJEkOtcUX\nEUmBtdaC7bYLr1zusGhR41OqH3sM3n9/1dguXVZdccktYnr1Cp8jUspUpIiIJIhZuEJSVgZ7793w\n/c8/D4VL7hWYhx4KD/l+9VUY165deN6lsedgevcODfJEkk5FiqTakiVL6NatW9xhSESUz9DUbpdd\nwivXihXw9tsNbx9VV8O998Inn6wau+GGTc9G2nLL0Piu0JRPyUdFiqTa8OHD1XY7RZTP1avtqNuz\nJxx0UMP3P/yw8edgnn02FDe1jyiutdaqh3lzC5mttw63maKgfEo+KlIk1caNGxd3CBIh5bN1Ntoo\nvPbcs+F7y5evepi3biHzl7/ArbeG92ttvnnTU6o32WTNH+ZVPiUfFSmSapqllS7KZ+GsvTbssEN4\n5XKH995reAXmtdfg0Udh8eJVY9ddt+m1kXr2rL/Ao/Ip+ahIERGR1TKDLbYIr333bfj+p582Phtp\nxoxwdebrr8O49u3DStRNXYXRAo+SS0WKiIi0ynrrwa67hleur79ufIHH55+HyspQ4NTaeOOmZyNt\nsYUWeGyLVKRIqk2ZMoXTTjst7jAkIspn6enQYVUvl0MOqf/erbdOYdCg0xp9mPepp8KyA7VqF3hs\n7CrM1ltrgce0UpEiqVZdXa1faimifKbLiy9Wc/rpp9GtG3z3uw3fX7as8QUeZ88OCzx+8cWqsatb\n4HHjjdWZt1SpLX4daosvIlIaVq6Ed99temmB//1v1dj112/6OZju3dvuAo9RU1t8ERERVnXU3Wor\nOOCAhu9/8knjxcv06eEZmboLPPbs2fRVmDQv8FgKVKSIiEjqdO0Ku+8eXrm++irMOsotYp55Bu68\nMyw9UGvTTZu+CqMFHgtPRYqIiLQpHTvCttuGVy730Pcl9wpMbWO7995bNbZz54Y9Yeou8NipU9FO\nKbVUpEiqZTIZtd1OEeUzXZKYT7Nw9WTTTaGiouH7NTWNL/D4yCPhYd7aBR7NVr/A40YbFfe8SpWK\nFEm1kSNHxh2CREj5TJdSzGeXLvCtb4VXrhUrwrTp3KswL70E990HH320auwGGzT9HMxWWxVngcdS\noNk9dWh2j4iIFMpHHzU9G+mtt1Yt8Nix4+oXeFxnnVhPo0ma3SMiIlKiNtwQ9tgjvHJ98cWqBR7r\nFjFPPgm/+13oGVOrrKzpqzCbbpquh3lVpIiIiMSsUyfYfvvwyuUO77/f8ArMf/8Ls2bBBx+sGrvO\nOqtf4HGttYp3TlFQkSKpNmPGDAYNGhR3GBIR5TNdlM81Ywabbx5e++zT8P3PPmt8gceZM0PH3toF\nHtu1W/0Cj127FvW01khinkkxsxHAj4Ay4GXgXHd/YTXjDwSuBnYGFgJXuPvtTYw9Afg9MMPdj17N\nMfVMSspUVFTw7LPPxh2GRET5TBfls/C+/jo879JYEfP667B06aqxG23U9GykLbfMv8Bjap9JMbPB\nhILjTOB5YBQwy8y2d/cljYzvBTwE3AicCBwC3Gpm77r7nxsZexXwVOHOQJJqk002iTsEiZDymS7K\nZ+F16BAett16azj44PrvucOHHzb+MO+cOfD226vGduq0+gUeO3cuUPyFOWyzjQJucfc7AMzsLGAg\nMByY0Mj4s4E33H1M9udXzWzf7HG+KVLMrB1wF3AZsD+QwItZIiIixWcWFl/ceGP4zncavr98eeML\nPD72GPzmN/UXeNxiC9hss+hjjL1IMbOOQDnwy9pt7u5m9hjQSCsdAPYCHsvZNgu4NmfbWGCRu99m\nZvtHFLKIiEjqrb029OkTXrlWrgzdd+sWLy80+YBGy8VepADdgPbAopzti4AdmtinrInx65tZJ3f/\nIntlZRiwa5TBioiItHXt2oXnVLbcEvbPXgKorg6zjaKUhCIlcma2LnAHcIa7f5RvfB1rA7zyyisF\niUuK7/nnn6e6OpLntyQBlM90UT7Tpc7vzrWjOmYSipQlwAog927WZsD7TezzfhPjl2avovQBegIP\nmn3T1qYdgJl9Cezg7vMbOW4vgJNPPrm55yAJln3aXFJC+UwX5TOVegHPRHGg2IsUd//KzKqAg4GZ\nANnC4mDghiZ2exbon7OtX3Y7wDxgl5z3rwDWBc4D3mriuLOAk4AFwPI1PgkRERFZm1CgRHbTJxF9\nUszseGAqcBarpiAfC/Rx98VmdiWwhbsPzY7vBfyDMAX5d4SC5jpggLvnPlBb+xm3AV1X1ydFRERE\nkiP2KykA7j7dzLoBlxNu27wEHObui7NDyoDudcYvMLOBhNk85wFvA6c1VaCIiIhI6UnElRQRERGR\nXHma3IqIiIjEQ0WKiIiIJFKbKlLMbISZzTezZWY218z2zDP+QDOrMrPlZvaamQ0tVqyyZpqTUzM7\nwMxW5rxWmNmmxYxZGjKz/cxsppm9k81LZg320fczwZqbU30/k8vMfmxmz5vZUjNbZGb3m9n2a7Bf\nq7+jbaZIqbOI4Vhgd8JKy7OyD+w2Nr4XYRHDxwlda68nLGJ4aDHilfyam9MsB7YjPIxdBmzu7h8U\nOlbJax3CA/PnEHK0Wvp+loRm5TRL389k2g+YCHyXsKBvR2C2mTW5rGBU39E28+Csmc0FnnP387M/\nG6Ffyg3u3mARQzMbD/R392/X2VZJmMY8oEhhy2q0IKcHAH8BNnT3pbnvSzKY2UpgkLvPXM0YfT9L\nyBrmVN/PEpH9h+AHwP7uPqeJMZF8R9vElZQ6ixg+XrvNQ3XWkkUMmxovRdTCnAIY8JKZvWtms81s\n78JGKgWi72c66ftZGjYgXPX6cDVjIvmOtokihdUvYljWxD6rXcQw2vCkBVqS0/eAHwLHAEcTrrr8\nPzPbrVBBSsHo+5k++n6WgOwV6+uAOe7+79UMjeQ7mohmbiLF4O6vAa/V2TTXzLYhdDjWQ5ciMdL3\ns2TcCOwE7FOMD2srV1IiX8Qw2vCkBVqS08Y8D2wbVVBSNPp+tg36fiaImU0CBgAHuvt7eYZH8h1t\nE0WKu38F1C5iCNRbxLCplRqfrTs+q+4ihhKjFua0MbsRLjNLadH3s23Q9zMhsgXKkcD33H3hGuwS\nyXe0Ld3uuQaYml1xuXYRwy6EhQ3JXcQQuBkYkX1CuXYRw2MJVaQkQ7NyambnA/OBfxFW6zwD+B6g\naasxM7N1CP9ituym3ma2K/Chu7+l72fpaW5O9f1MLjO7ERgCZIDPzaz2Cskn7r48O+aXwJZRf0fb\nTJGiRQzTp7k5BdYi9FXZAqgB/g4c7O5PFS9qacIewBOEGQNOyBPA7cBw9P0sRc3KKfp+JtlZhBz+\nv5ztw4A7sn/enAJ8R9tMnxQREREpLW3imRQREREpPSpSREREJJFUpIiIiEgiqUgRERGRRFKRIiIi\nIomkIkVEREQSSUWKiIiIJJKKFBEREUkkFSkiUtLMbL6ZnRd3HCISPRUpIrLGzOw2M7sv++cnzOya\nIn72UDP7qJG39gB+U6w4RKR42szaPSKSTGbWMbuqdd6hhPVD6nH3/0UflYgkga6kiEizmdltwAHA\n+Wa20sxWmFmP7HvfMrNHzOxTM3vfzO4ws43r7PuEmU00s2vNbDHwp+z2UWb2dzP7zMwWmtlkM+uS\nfe8AwkqqXet83mXZ9+rd7jGz7mb2QPbzPzGze8xs0zrvjzWzF83s5Oy+H5tZZXbV3toxx2ZjqTGz\nJWY228w6F/Q/qog0oCJFRFriPOBZ4LeEFag3B94ys67A40AV0Bc4DNgUmJ6z/ynAF8DehBVWAVYA\n5wI7Zd//HjAh+94zwAXA0jqf9+vcoMzMgJnABsB+wCFAb2BaztBtgCMJy8YPJBRcl2SPUQb8HrgV\n6JN97z7ClRwRKSLd7hGRZnP3T83sS6DG3RfXbjezkUC1u/+szrbTgYVmtq27/ze7+T/ufknOMW+o\n8+NCM/sZcBMw0t2/MrNPwrBVn9eIQ4CdgV7u/m72808B/mVm5e5eVRsWMNTda7Jj7gQOBn5GKIDa\nA/e7+1vZ8f9a0/82IhIdXUkRkSjtChyUvdXyqZl9CrxCeJZkmzrjqnJ3NLNDzOwxM3vbzJYCdwIb\nm2Ad0pYAAAHzSURBVNnazfj8PsBbtQUKgLu/AnwM7Fhn3ILaAiXrPcIVH4CXCVeD/mlm083sdDPb\noBkxiEhEVKSISJTWJdxu+TahYKl9bQc8VWfc53V3MrOewIPAS8DRhFtFI7Jvr1WAOHMf1HWyfx+6\n+0p37wd8n3AF5VxgXjZGESkiFSki0lJfEm6L1FVNuN3ypru/kfNatppjlQPm7j9y9+ezt4W2XIPP\ny/UK0N3MvtnXzHYiPKPSrFs27v6su/8c2J1Q1BzVnP1FpPVUpIhISy0AvmtmPevM3pkMbARMM7M9\nzKy3mR1mZr/LPtTalP8CHc3sPDPb2sx+APywkc9b18wOMrONG5tt4+6PAf8E7jaz3c3sO8DtwBPu\n/uKanJSZfcfMfmxm5WbWHTgG6Ab8e032F5HoqEgRkZb6NWFGzr+BD8ysh7u/B+xD+LtlFvB34Brg\nI3ev7XHSWK+TvwMXAmOAfwBDyM62qTPmWeBm4B7gA2B0E8fLAB8BTwKzCQXQCc04r6XA/sDDwKvA\n5cCF7j67GccQkQjYqr83RERERJJDV1JEREQkkVSkiIiISCKpSBEREZFEUpEiIiIiiaQiRURERBJJ\nRYqIiIgkkooUERERSSQVKSIiIpJIKlJEREQkkVSkiIiISCKpSBEREZFEUpEiIiIiifT/AXc2+c9J\ntpfVAAAAAElFTkSuQmCC\n", "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "NN = Neural_Network()\n", "T = Trainer(NN)\n", "T.train(X,y)\n", "plt.plot(T.J)\n", "plt.grid(1)\n", "plt.ylabel('Cost')\n", "plt.xlabel('Iterations')\n", "#hmmmm ...." ] }, { "cell_type": "code", "execution_count": 281, "metadata": { "collapsed": false }, "outputs": [ { "data": { "text/plain": [ "(array([[-0.02306668, -0.05445037, -0.00996909],\n", " [-0.01709984, -0.02921199, -0.00573783]]), array([[-0.00477078],\n", " [-0.02088484],\n", " [-0.00580673]]))" ] }, "execution_count": 281, "metadata": {}, "output_type": "execute_result" } ], "source": [ "NN.costFunctionPrime(X,y)" ] }, { "cell_type": "code", "execution_count": 282, "metadata": { "collapsed": false }, "outputs": [ { "data": { "text/plain": [ "(array([[ 0.96101082],\n", " [ 0.96878685],\n", " [ 0.8296233 ]]), array([[ 0.75],\n", " [ 0.82],\n", " [ 0.95]]))" ] }, "execution_count": 282, "metadata": {}, "output_type": "execute_result" } ], "source": [ "NN.forward(X), y " ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# 7 Overfitting" ] }, { "cell_type": "code", "execution_count": 284, "metadata": { "collapsed": true }, "outputs": [], "source": [ "# fit the signal, not the noise\n", "# TODO: regularize the cost function" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": true }, "outputs": [], "source": [] } ], "metadata": { "anaconda-cloud": {}, "kernelspec": { "display_name": "Python [conda env:gcloud]", "language": "python", "name": "conda-env-gcloud-py" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.5.2" } }, "nbformat": 4, "nbformat_minor": 1 }