{ "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "

Week 7: Learning (Part II, Intro to Neural Nets)

\n", "\n", "

CSCI-UA 9473 - Introduction to Machine Learning

\n", "\n", "

Partial Solutions

" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Part 1. A simple linearly separable dataset (gradient) " ] }, { "cell_type": "code", "execution_count": 4, "metadata": {}, "outputs": [], "source": [ "# a simple neural network which takes as input a 1D latent variable\n", "\n", "import numpy as np\n", "import copy\n", "import matplotlib.pyplot as plt\n", "\n", "\n", "input_dim = 2\n", "output_dim = 1\n", "# number of neurons per layer\n", "network_size = [1]\n", "total_size = copy.deepcopy(network_size)\n", "total_size.append(output_dim)\n", "num_layers = len(network_size)\n" ] }, { "cell_type": "code", "execution_count": 8, "metadata": {}, "outputs": [], "source": [ "# defining the activation function\n", "def activation1(x):\n", " \n", " sigma = 1/(1+np.exp(-x))\n", " \n", " derivative = sigma*(1-sigma)\n", " \n", " return sigma, derivative" ] }, { "cell_type": "code", "execution_count": 13, "metadata": {}, "outputs": [], "source": [ "# forward and backward propagation\n", "\n", "x_in = np.random.normal(0,1,(input_dim,1))\n", "\n", "def SGD_neuralNet(x_in, target, weights, biases): \n", " \n", " # forward propagation\n", " current_input = x_in\n", "\n", " # adding the bias term\n", "\n", " preactivation = []\n", " postactivation = []\n", " \n", " for l in np.arange(len(weights)):\n", "\n", " output_s = np.shape(weights[l])[1]\n", " \n", " tmp = np.matmul(weights[l],current_input).reshape(-1,1) \\\n", " + biases[l].reshape(-1,1)\n", " \n", " tmp2 = activation1(tmp)[0]\n", " preactivation.append(tmp)\n", " postactivation.append(tmp2)\n", " current_output = tmp2\n", " current_input = current_output\n", " \n", " \n", " ### backpropagation \n", "\n", " loss = -target*np.log(current_output) -(1-target)*np.log(1-current_output)\n", " delta_out = current_output - target\n", " current_delta = delta_out\n", "\n", " weight_backp = weights[::-1]\n", " preactivation_backp = preactivation[:-1][::-1]\n", " postactivation_backp = postactivation[:-1][::-1]\n", "\n", " grad = []\n", " grad_biases = []\n", " \n", " grad.append(np.squeeze(delta_out)*np.squeeze(postactivation_backp[0]))\n", " grad_biases.append(delta_out)\n", " \n", " \n", " postactivation_backp.append(x_in)\n", "\n", " for l in np.arange(len(weights)-1):\n", "\n", " tmp = np.matmul(weight_backp[l].T, current_delta)\n", " \n", " sigmaPrime = np.squeeze(activation1(preactivation_backp[l])[1])\n", " \n", " current_delta = np.multiply(tmp.reshape(-1,1),sigmaPrime.reshape(-1,1))\n", " \n", " tmp1 = postactivation_backp[l+1].reshape(-1,1).T\n", " tmp2 = np.matmul(np.squeeze(current_delta).reshape(-1,1), tmp1) \n", " grad.append(np.squeeze(tmp2))\n", " \n", " grad_biases.append(np.squeeze(current_delta))\n", " \n", " grad = grad[::-1]\n", " grad_biases = grad_biases[::-1]\n", " \n", " return loss, current_output, grad, grad_biases" ] }, { "cell_type": "code", "execution_count": 15, "metadata": {}, "outputs": [ { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAD4CAYAAADiry33AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAABFqklEQVR4nO3dd3gc1dXA4d+Zma2SbLl3Y4rpzfQaAgFCC4SEGgKhBIfe+QgQakILCaFDSCAEQg819I4pphiwAWMMxjbY2NiWi2xLW2fO98fKQtKupJWsrvPy+EE7c2fmrMvZ2Tv3niuqijHGmJ7L6ewAjDHGtC9L9MYY08NZojfGmB7OEr0xxvRwluiNMaaH8zo7gEIGDBygo9cY3dlhmG4g6we4uJ0dhjFFCYKA2VPnQIHBjuLAmhuv0epzT5nycYWqDiq0r0sm+tFrjObVia92dhimG1iyrJq+Xt/ODsOYoqgqx25yOollybx9a2w6kmuevbjV5x40oOybxvZZ140xxnQQEeH4q46sl3kVRVzh+Ct+3W7X7ZJ39MYY01Pt8LOtKSsv4Z4rHqbi28WM3mgkR19yOGtu3H7d1ZbojTGmg22y84Zc+/ylHXY967oxxpgertlELyJREXlfRKaIyFQRuaxAGxGRG0Vkhoh8IiJb1Nm3l4hMr9n3+7Z+A8YY0xtk0hkWzV1MMpFq8bHFdN2kgN1UdaWIhIC3ROQ5VX23Tpu9gbE1v7YFbgO2FREXuAXYA5gLfCAiT6nq5y2O1BhjeqknbnuOR/7yFH7GR0T48WE78Nsrfo3rFTe0uNk7es1ZWfMyVPOr4SjQA4B7atq+C5SLyDBgG2CGqs5U1TTwYE1bY4wxRZjw6EQeuOpR/JQPAaivvHL/BP59+UNFn6OoPnoRcUVkMrAQeElV32vQZAQwp87ruTXbGtte6BrjRWSSiEyqqKgoMnxjjOnZ7r/mUSSon6oddXn53glkM9mizlFUoldVX1U3B0YC24jIxg2aSKHDmthe6Bp3qOpWqrrVwIEDiwnLGGN6vBWLqwpuD/yAZHVx/fUtGnWjqsuA14G9GuyaC4yq83okMK+J7cYYY4oweoPhBbfHSiOU9IkXdY5iRt0MEpHymp9jwO7AFw2aPQUcVTP6ZjugUlXnAx8AY0VkTREJA4fVtDXGGFOE4644EvEErdsZ4sAxVxyBSKFOk3zF3NEPA14TkU/IJe6XVPVpETlBRE6oafMsMBOYAfwDOAlAVbPAKcALwDTgYVWdWlRkxhSpMlvZ2SEY027W2WwMVz59Ietvvw7RPhFGbDiM8+89nR8duF3R55CuuGbsuC3HqRU1M8Vasqy69mcrcGZ6q0EDyj5U1a0K7bOZsabb618ep395rq/S7u6NyWeJ3vQYq5K9MaY+S/TGGNPDWaI3xpgezhK9Mcb0cJbojTGmh7NEb4wxPZwlemOM6eFsKUFjjOmm0qk0D//1SV6+940m21miN8aYburyw/7Clx98jUPTC5BY140xxnRDsz77lhkfzmo2yYMlemOM6ZZmT52DHwRFtbVEb4wx3dDg0QNxnLYrU2yMMaaL2XC7dRkwYgABzd/VW6I3xphuSES46ukL2WjH9ZrN5Dbqxhhj2tHi+Uu57Zx/MfXN6YgrbLPPOI6/8khK+q5+tdW+A/pwyUPnkkqmGDnigUbb2R296XGsJr3pKhLVSc7Z4xI+feNzNFCCTMA7T37A+fv9ibZc9CkSjTS53xK96VHqLkBiCd90tneemMTKZVVInVTr4LDgm4VMfafh0tvtp5jFwUeJyGsiMk1EporI6QXanCsik2t+fSYivoj0r9k3W0Q+rdk3qT3ehDF11V1xypjONGPK17gFxrkHgTL3y/kdFkcxd/RZ4GxV3QDYDjhZRDas20BVr1XVzVV1c+B84A1VXVKnya41+wuuZ2iMMT3RWhuPwcfP2+44woh1hnZYHM0melWdr6of1fy8ApgGjGjikMOBxp8KGGNML7HTL7alpCxebwhkgM/AkQPYaMf1OyyOFvXRi8gYYBzwXiP748BewKN1Nivwooh8KCLjmzj3eBGZJCKTKioqWhKWMcZ0SbGSKH9+8RLW33YsCOAIW+89jqufvQjH6bhHpEUPrxSRUnIJ/AxVXd5Is58BbzfottlRVeeJyGDgJRH5QlUnNDxQVe8A7gAYt+W4tnscbYwxnWjwqIFc/uh5taNsRIqbzdqWivpIEZEQuSR/n6o+1kTTw2jQbaOq82r+vxB4HNimdaEaY0z3JSKdkuShuFE3AtwJTFPV65po1xfYBXiyzrYSESlb9TOwJ/DZ6gZtjDGmeMV03ewIHAl8KiKTa7ZdAIwGUNXba7YdCLyoqlV1jh0CPF7zKeYB96vq820QtzHGmCI1m+hV9S1yjxGaa3c3cHeDbTOBzVoZm2knqmkgQCTa2aEYYzqA1brpRdRfQvXCi4jIpyCQzK5JyZDLEW+Nzg7NGNOOrARCL6EakPj+WEI6BcdRHFGi7tekFv4WDaqaP4ExptuyO/reIvM+QXYxXuyHXjjHEfx0NZp4Hin5ZScGZ0zvtWTBMh6/6Rk+fv1ThowexCFn/Zz1tlq7Ta9hib63yH6H4wQ0/BIXizpkUzMJl3ROWMb0ZhXzlnDWrheRrEri4LJo9mKmvnUNJ91wLD86cLs2u4513fQWobFQoLhSdULxYpt0fDzGGB645lGSVanaBb4FgQD+cd69+Nn8GjmtZYm+t/A2Rd21SKV+mHScySqu1x+J7taJgRnTe01+fSpOgTScSaZZMKftSsFYou8lRCA+9HackoNIZWKks2GC0B5Eh92DSLizw2s3VpPedGVl/Usb2SOUlbddf6ol+l5EnCiR/mcQG/Ei0eGvEBt0CeKUd3ZY7abuIiTGdEWHnvVz1Km/uHeAzwbbj6WsX2MfAi1nid70aLbilOnKtttvSw48bT/ElVzCd2Dslmtz7j9ObtPr2Kgb0+OtSvZLllV3ciTG1CciHHbOz9n/dz9l7lfz6T+kLwNHDGjROSY8OpH7r360yTaW6I0xppPFy2Ksu8VaLT7uqb+/wH1XPIIETXfOWNeNMcZ0Q9lMloevfbLZJA+W6I0xpltatmg52XS2qLaW6I0xphsq61+CFLkcoSV6Y4zphiLRCD85YicCaX4GrSV6Y4zppo65/HD2OW4PnJA9jDXGmB7JdV2OvvQw7v3qlibbWaI3xphuzvXyCxbWZYneGGN6uGYTvYiMEpHXRGSaiEwVkdMLtPmxiFSKyOSaXxfX2beXiEwXkRki8vu2fgPGGGOaVszM2Cxwtqp+JCJlwIci8pKqft6g3Zuqul/dDSLiArcAewBzgQ9E5KkCxxpjjGknzd7Rq+p8Vf2o5ucVwDRgRJHn3waYoaozVTUNPAgc0NpgjTGmp6hekeD72QvJpDPtfq0W1boRkTHAOOC9Aru3F5EpwDzgHFWdSu4DYU6dNnOBbRs593hgPMDI0SNbEpYxxnQb6XSGW864k/ee/hhQvLDHry74Bfscu3u7XbPoh7EiUgo8Cpyhqssb7P4IWENVNwNuAp5YdViBU2mBbajqHaq6lapuNXDgwGLDMsaYbuW2s//FxKc+gEAhgGwyyz2XPsT7z33UbtcsKtGLSIhckr9PVR9ruF9Vl6vqypqfnwVCIjKQ3B38qDpNR5K74zdtQDUgSLzIynknsnLeKWjqDbTgx6hZxWrSm86UqE4y8akPa9eIrRUI913TdKnh1dFs142ICHAnME1Vr2ukzVBggaqqiGxD7gNkMbAMGCsiawLfAYcBv2qj2Hs1VUgsPAdNfUA8mtuWWDAFie5GbPBlnRtcF9W/PM6SZdVUZivp6/Xt7HBML1S1rJrG7sYqF7TfTUgxffQ7AkcCn4rI5JptFwCjAVT1duAg4EQRyQIJ4DBVVSArIqcALwAucFdN371ZXdnJkJ5ELPrDpmgU0pnX0OxRiLd2p4XWlVmyN52p3+C+hKIhMtX1H8AGBKyzZfv9m2020avqWxTua6/b5mbg5kb2PQs826roTKOC5LuEwwH5fzQBpD8AS/SNWpXsjelorudy9GWH8Y/z7oEg9283IMALeRx98aHtdl1bYaqbEqecVFqJRuon+mwWItKnk6IyxjTnJ4fvTP8h/fjPVQ+zZP4yxm6xFkdddCgjxw5rt2taou+mJLoHnncbUH8F+VAoBJEfd0pMxpjijNttY8bttnGHXc8SfTcl7gC8/teQWXwhmUwaBDwvSmTQXxEn3tnhGdNjzJk+j2fufJGFcyrYfp+t+dHB2xGJRjo7rBaxRN+NOZHtCA97nnDmMxAHvI3JVZ0wxrSFiU9P4oaT7yDwAxwcPntzGo/f8gx/feVyYiXR5k/QRVj1ym5OJISExyGhzdDU26yceyCZ73dh5dxfEiTf7OzwjOm2MukMt5xxF/jg1KRKB5eKuYt57s6XOzm6lrFE30MEiVdIV/yBeLgC1wmIhxeSXnwhQfKNzg7NmG7pm8/nkkmn87Y7uLz+37frbZs3awHXn/x3Ttr+/7jm2Bv5ZtrcjgqzKNZ100MkFv+NWLj+RIxwSElU/I2Skbt0UlTGdF+x0ijSyMjyeNkPz8FmTf2WC/e7Ej+TRXComLOEj1+Zyh/uP5ONd1y/o8Jtkt3RdxEaVJKp/Ccr5/6G6oUXoNnpxR+rEA0XnlUXCS0mqH6alXOPZeV34wmSr6BWJ8GYZg1feyj9h/cnaDCyTZ2AA0/et/b1Hefdg5/xkdruHQd85Zaz7uzQeJtiib4LUH8pyfmHkV1+N/HwTMLBBNILxhMkXinqeBFIpUsKn1sdEhVXEw9/RTw0jeSiS0kuuqQtwzemRxIRLnnoHPoO7IM6Ab744MBeR+/GNnuPq203+7O5Be/8l86rJN0BJYiL0a27bjRIEFQ/TWLFW4Qiown3OQTxRjV/YBeTrrwLhxWEaiY/OY7gOAHppVcRif64qJE04fITSS79C3VHfWV9EAmIRX/4SxiNQDrzOpr92sokGNOMIWsM4u8f/YUv3p/B8orlrLv1OvQfUl6vTbQkRKIylXes4zp4zazl2lG67R29BstJzD+M5JIbiLsfQfJx0gt/TZB6t7NDa7FM1QRCofw7gsBPgz+nwBH5vNIDCJefSSoTz1U/VUEEHCe/m0Y1gPTHqx23Mb2B4zhsuN26bLffVnlJHmD/E/ciEL/etgCfHx+2A47TNVJs14iiFTKVd+Po4tq71ZAnhLyAVMUl3a8PupGSBZ4nIGVFn8Yr+wXR4S/ghw4kkwlwHQp+pfR9wLGCXsa0hQNO2ptdD98ZHEGdABzYeu9xHHP54Z0dWq1u23WTXvkK8UiBJ+KayN0Fe6M7PqhWig88huSiS+p1u2SySpaxhN0BLTqXCPjVLxINN16HLhSKQHjn1oZrjKnDcRxO/PPR/PqCg/h+5kIGjR5I+cCuVW+q2yZ6iANL8ra6joB0rxIAEtkNt3Q62eSDpNMBnif4Mob40L+06XUUJZ0pJTrkb4jTfWb1GdMdlJWXUrZFaWeHUVC3TfTxAb8mufjqBg8flbSOIex2/aUI1V8C/tfgDEW8UUT6nYgGRxDKfgnOQMQb0+pzeyX7kKp6mEidu/ogUNLBCGLDHyS3loxZtdqU1aU3PV23TfQS2xeJTiabeZF0xs899HAGU9LGd8FtTVVJLb4KJ/MCqXRAyBN8GUt86I2I0wfCW632NUJ9jyebeJdEcg6RsJJMgRcqITb4OkvyNfqX57712SIkpjfovoleIDboQtT/LaHsF+AMqCnq1dmRNS2oeoyg+jnCUfBiAEo6M53qRZdQMqRtPqTEiREbdh9k3ofMdErKh0JkF0S6V8W9jmCLkJjeoNsm+lXEHQLukM4Oo2iJpffWrvG6Sjgk+MEHaJBAnFirz61BJdnl95Na8QZ4AykZcAxSctRqRmyM6e6KWRx8FHAPMJTcKhd3qOoNDdocAZxX83IlcKKqTqnZNxtYAfhAVlVXv2+iG3OdwnePqgqaBFqX6DVYRnL+r0CX14xG+o7UwjPx+p6KV3Zw6wM2xrSZdCrN+89NpmLOYsZsMppNf7RBh4y1L+aOPgucraofiUgZ8KGIvKSqn9dpMwvYRVWXisjewB3AtnX276qqFW0Xdveloa3x/Tdw3QZLAPp9CDn9Wn3e7PL7QZfXewAbDivZlbegpT9DxEbZGNOZFnyziN/vfTlVK6pBBXFg2NpDuerpP7R7bftmP0pUdb6qflTz8wpgGjCiQZt3VHVpzct3gZFtHWjttbJfklp8NdULL0RTE3KzPLuR2IBT8TVGKp2b1JX1lUzWITro0tV6vpBa8Ua9JL9KOh1AdmbrT2yMaRPXjr+ZquUJHHVxcJDAYd5X83nwz4+3+7Vb9J1BRMYA44D3mmh2HPBcndcKvCgiH4rI+BZHWEd2xUOkFhyPk36aKBOoXnAhiQWntnmy1+xsNPMpGiTb9LwA4g4lMuRhvNIjqE6vje/9lPDgf+NEVrNHq5EhpaEQIOWrd25jzGqpXl7N3GnzaxcwWcXB5Y1H3mn36xf9MFZESoFHgTNUdXkjbXYll+h3qrN5R1WdJyKDgZdE5AtVnVDg2PHAeICRo/O/EGhQSbDyVsKhAGqm9ceikEh+Auk3IbL6NdfVX0T196fiMo9sFsJhB6f0NLyyX6z2uesStx+h8hMJlZ/YZucsGXgMyYVnEalTkz43u3YtIt7wNruOMaaNdUDJlqLu6EUkRC7J36eqjzXSZlPgn8ABqrp41XZVnVfz/4XA48A2hY5X1TtUdStV3WrgwAJ3p5mPSKf9vM2xKCSWPVvM22iSKlR/fwohZy7hkBKPKZ7rk11+PZr5ZLXP394kvBWhvieT9V2qE0I6I2RZm/iQ6zs7NGN6vXifOMPHDkEb1LYP8Nn5l9u3+/WbTfSSm2FzJzBNVa9rpM1o4DHgSFX9ss72kpoHuIhICbAn8FmrIpUoFCjQFQSK47ZBXQn/axydj9fgIannBVQvvmf1z98BvLJDCQ15npLhtxIZ8iAlw/+NuK1/wGuMaV5VZTWTX/uMrz6a2WRBxXP+eQrRkii++KgqgRMwZMxgDj+vbXsMCimm62ZH4EjgUxGZXLPtAmA0gKreDlwMDABurZl5uWoY5RDg8ZptHnC/qj7fqkhDW+GFIkD9us9Z3yEy4MBWnbKeoBLfz/9DckTQ9KLVP38HEScKzsadHYYxvcITtz7HQ9c8ga9ZHBzKBpRx+ePnMWzMD3N7/KzPK/e/yXP/epm+Q/uwzWZbkE6l+WLSDCoXLefa42/mmEt/xej1R+SdPwgCKiuWEyuLEY21fsKjdMWSvuO2HKevTnw1b7tmppJedDqZTApQwmEXKRlPqM8Rq31NDarJLNgbz63/1SqZUsLlv8UrO3q1r2G6piXLqq0EgmmxqROn88dD/wLBD70ASkD50HJu/+BaRARV5bJDruXzd6fjaG4RkoAAqfkvd4zihlyufu6iesn+3Wcm8fdz/01iZQoQttlvHCdfdwyRaOGEP2hA2YeNzVPqVjNjJbQR4WHPEU5/AFoNoS3brGtCnDhO6e9IVd5e+0AzmQoQdyBuiU04MsbU9+Ttz6FB/Q5lwaFyYSUzP/2GtTcdw/QPZvDFe1/VJnkgb+SNIPiZLNedcBvVKxKkqlKstdkYpk38kh+69JWJT31AqirJ+f8+o8WxdqtEDyASgsgO7XJur+xXuJF1qVp8D2QXEy3fHbfkEMQpvB6rMab3qly0vODCPgEB1ZW5GfDT3v0KDbRgu7oEh/kzFtS2+/zt6TXbfzjOxWXKa5+zrGJ5i+vdd7tE394kvBWlw3p1lQZjTBF2+cX23P3JgzjUXxfWdTzWGbcWAH0H9SFAKWbl2LpJvbEPBkVZumBZixN9t11KsKtSVTQzGU29gfpLmz/AGNMt7XbEjxg8ehABuWHfSm4ZwSMvPphYaa6kwXb7bUk4Em72XEpxz0odcRg6ZnCLY7U7+jak2bkkF5xA4FeimntYrNHDifQ7obNDM82wmvSmpaKxCH955TJef+htJjw2kX5D+nLACXuz7pZr17aJl8U447bfcfOp/yRVnUI1/25d0WY6dmraqRKJhltVF8cS/WpQBdJvUrXkEdAMIZlNyF2BExJyj2gCUlX3oyWbI+HtOjla05hVNeltxSnTUtFYhL2O3o29jt4tb18QBNxy5l28/fj7BIGPiFMzEidAajpTFCUU9Rg8eiBzv5yX1w1Ul4iQqkqTTqWL+pZQl3XdrIZkxeUkFl5A3P2YuPcZnrsCx6n/2RwJK1WL/9NJEZpi9S+P1646tSrhG7M6Xn3gLd587F1yPTouorkkH45FcpnXgWFjh3DVMxdx+eO/Z+OdN8htl6a6cpRUIt3iWOyOvpU0+zWSfplInSGtjT5Z91d0TFBmtdmKU6atPHHrszjacCilQyaV4foJf6RP/zLifeK1+y5+4Jzc8MpEirN3u4SqpdV5S39GyyKU9m35KMBee0evCkH1c6ycczBVc/cisfAS1F9Q/AnSk6CIqpnJpBIr33c1IjXGdEfJqlSj+wKlXpJfJV4Wo9/gci68/0zEkdo7e0VB4MzbT2zVus+9NtGnl95IcvGVxCPfEwtX4fqvkFrwa9Rf3PzBAFJGNr/GGooSBLk/nERSwRuJU7J/G0ZujOkOdvjZ1gTk3wzGyqIMXWNQk8euvekYbnn/GnY8cGv6j+jLuD024c8vXsxmu2zUqlh6ZdeNBsuR1KPUnUnsuUI6nSCz/D7C/U5r/iSRXQiFriW3ANcPfN8lCO1CNlNJfNDuSPSniLTswYkxpvs7+Kz9mfi/SVRWVOLgEhDgui6n3/K7opYPHDCsH6fdVH8Jj4rvFvPZ218QK42x+W4bNVoOoaFemejxZ5FOB3gNlmcNh4Xqle82mehVFTLvkl7xOm5sFzKJN8lkMoASCoUIDbgCJ9LysqOq5NaMlUirvpoZY9rf5Nen8vB1T7B04TK22n1zfnn6fpQPKjxKq6xfKTe+dSWvPfQWH7z0MSPWGsY+x+/B8DWHFGzfnP9c+V+evv1F/MBHBMKRCBc/fA7rbrFWs8d2q6JmbUWz80gvPIyQV/+9B4GSdnYiPvjqwsdpQGLBaWh6CrEoZDIK4uKV/Sq3QlRo01yJhhbKrnySTOWthLwEWT+MW3o0XtmvV2tpQdN6VuTMFPL0HS/xnz89XFvELMAnXhbn+gl/ajTZt5Wp70znj4fVL6AGEC4JcddnN+CFvCaLmvXKPnrxhpOVDWrXbV3F911i/Y9p9DhNvVab5AFCISHkBfgr7wdvw1Yl+aD6RTLL/kokVI0jSthLkan8O/7KB1p8LmNM+0gmUtx/1aP1Eq2DS/WKah6/efUXPmrOs/96qeDYj2QixbR3v8zf0UCvTPQA8SHXEXhb4/sO6YxDKhMnNOBPiLdeo8dUL/1fbZKvK50OIPNxq+KoXnxLveX/AKIRyC6/qyNWGDPGFGHu9Hn4fjZvu4PLBy82/W+/4rvF3Hbu3Zy8w3lcceTfmDF5Vouvn6xOFR6+rUo6lR9XQ72zjx4Qp4SSoX9Dg5V4WgXO4Gb7xh03SqCKU6hdKx+4RkLLCm73vCSQAVr+LcEY03KqytR3pvPlpK8pH9yX7fbbknhZ7kFe+aA+tbNZGxowvPFS6fNnL+Dc3S8jnUzj4FDx7WI+nTCNs/9xElvvuXnRse12yI/4dMK0vHH5rhtiw+3Wbfb4XpvoVxGnFCgtqm2s/CCSC9/JuwP3QlEIbd6q66eyQ4iH5+dtT2dKiVmSN6ZDZNIZLj34WmZ8PAuCXKnhuy66n8sfP4+1NlmDgSMGMGaTUcycMrt+mQJHOeTMnzd63n9dfH9tkofchCl8uPWsu7jr0xsK3lyurKziwT8/zttPvU8oHGKf3+7BPsftxvrbjmX6+18hgUNAgOM4HH/Nr2sLqDWl1yd69ReRWf4Q6epPCMc3ItTnMMQt/FRcwlvhxA8hm3yEdCYLCKFQmMigvyHSut/K+MDTyS79PV6dvzvptBDpd7o9jDWmg7xw92t89eHXtUncxSWbzHLVUTdwx0d/RUS48L4zufLX1zPrk29zq0K5HkdefBCb7LRBo+f94r0ZeQuNACSWJ1m2qJJ+g8vrbU8lU5z9k0tY+v0yHBwSJLn/ikf47K3PufThc/n41c+Y+L/3Ke1fxh5H/IgR6wwr6v316kSv2dmkFx6LBmniYSFd/TnpxBOEB/8D8dYpeEyk/ymofxCh9CRwSiG8PSKtW8tRNUty6T9wfIWaRckDBSe2A27JXq1+X8aYlnnu7lcKFhRbsXgl82ctYPhaQykrL+Wqp/9AxbwlLF+8ghFjhzY7jj3eN0a6KlNgjxAri+VtfeeJD1i6YGm9WBxcPnvzC+ZOn8eWu2/Klrtv2uL31+zDWBEZJSKvicg0EZkqIqcXaCMicqOIzBCRT0Rkizr79hKR6TX7ft/iCNtR1cIrcd004XAuyYZDgutmqFrwxyaPE3coEtsPify41UkeQFf+i7Azo/b6AI4A6Ymo330WJDemNxk4vD9rbbJGUZOVDj7zAFTqD5cJ8Nnqp5sWXOz74zc+q7fs4Cp+4PP1lNmtjrmYUTdZ4GxV3QDYDjhZRDZs0GZvYGzNr/HAbQAi4gK31OzfEDi8wLGdJupOz3uw6ogQC82k7vwC1QxB8mUSiy4ju/wO1P9+ta+t2XkE1fcWfLCbTvutHsVjjGm5vY/+Se0CInWVDShlWCsnOAHsdthO7P3b3RFXUCe3MMnGO23AyTccl9c2sTJJSVkMv0AcjiMMHNG/1XE023WjqvOB+TU/rxCRacAI4PM6zQ4A7tFcdnxXRMpFZBgwBpihqjMBROTBmrZ1j+00Wd8j7OWX/PR9F2fVCu2aJDH/WDQ7h1gU0ssVf+V9hAZcixPZptXXzqz4L44GFOqIdxwBp2VLhRljWu+nR+/KO09/wNeTZ0EgBAREohHOv+f01ZqpLiIcfclhHHzG/sz7+nsGDO9H/6E/jNLxfZ/ZU+fw8n0TeP3BtwnUz+vTDwgoH9iXjXZcv9VxtKiPXkTGAOOA9xrsGgHMqfN6bs22Qtu3beTc48l9G2Dk6JEtCavVJPozklX/JRr54Q8ylVIk9tPa/OuvfAT8b4lFa7p3wjULilT8gejw5xFp3VSEdOob4k7+XyBF8bwIhGzd2s5kK071LqFwiCuevICp73zBl5Nm5g2vbKkZk2fxyN+eZP6shYz78SYccPLejG1QqmDKG1P56/hbSVena1eeqrsgiSDgwJiNRnHaDcfz+kPvsHJZFRvtuB5rbzqmRfEUnehFpBR4FDhDVZc33F3gEG1ie/5G1TuAOyBXAqHYuFZHuN/JJNLfkMl+SDqthENCENqU+ICzatsklz1NPJL/NoIgCf7X4I1t1bVjpduTWPJe7QfIDwSv/59aPYrHrD5bcap3EhE23nEDNt6x8VE0xXj3mUlcf+IdaJBbSeq5mS/z2kNvcd2rlzNwxAAAFs9fytW/uRHN5tJkfhbIJfnrJ/yJZQsr+b+9/kgmnc59FDjClntuxtl3nFRUcTQocmas5Ob2Pwrcp6qPFWgyFxhV5/VIYF4T27sEkRDxoX8jPPhBSoddTXjIfZQMvaX+A9ZGHraGQ4qmPkC10BP15jnxfXHcfqTrlGFIpZUUu7SqKJppW3VXnDKmWL7vc9vZd0NA7d25g0uyKsV/rvpvbbtXH3wTP9v8jNZAA6468gaCtI+Lm+vWCYRJL0xm4v8+LDquYkbdCHAnME1Vr2uk2VPAUTWjb7YDKmv69j8AxorImpKr1XtYTdsuRbzhSGRHxBuVty8+4AiSDdYPUBTXgeqKW0nM/w2qyRZdT7NzIT2RyKBLkPgvSKT6UJ0aTLj8LGKDmh7xY4zpuiq+W0KqOv+5n4PDlNen1r5e9N2SJteHBQjHQixftIJkIj+/iDo8e9eLRcdVTP/AjsCRwKciMrlm2wXAaABVvR14FtgHmAFUA8fU7MuKyCnAC4AL3KWqU+lGJLonEplIJvsKnqc1/Wi5L1rxmJDNfou/4h94fU6td5z6iyH7BbiDwF0XEVD1SSz8Pa7/HplMgOs4BO5I4iPuRxzrHjCmuyvp2/i3wNI63xC33G1TXn/orbySBlDTP+8Iv/vzbwoO1lgl8Ivv4S5m1M1bFO5rr9tGgZMb2fcsuQ+CbklEiA2+lCC5M+nFlxJuUJXA84Qg8TBpLSHc91hUIbX4WpzM06TSAZ4rBDKc+NDb8KuehPREQhEh5AmgZDLfUr3wEkqGXt8Zb88Y04ZK+5aw4Y7r8tmbX9QbPaNOwMF1SiVstedmjFp/BHOmza29s1eUSDzMOluuyaFnH8h6W61NNpMlHI2Qqa7fRRxIwN5H/6TouHpt9cqWEm8MqoU/7xwBqv6NZuejyecJEk/huQElMYiElZDMpXrBOaSWP1JvhA/kSh1H+LjF3T/GmK7pnL+fxNgt1gQnl+BxhZ/97qfs/IsfBhy6nstVT1/IURcfypB1BjFiw2GcfMNx3P3FTVz8wDmst9XaAHghj/+782TEc/DxURR1lE122oAdDyx+eHevXHikNVQhMe/nREIVBcuFJpMBsYFnUbXkv8TDc/P2Z30hCMKEQ/kLBvuB4A1+GrGx812OLUJiiqGqzPniO3w/YPQGI3BdlwXfLmLp98sYtd6IJrt0irGsYjnvPPk+K5ZWsclOG7DBtmPzxvc3tfBIlxzDp/4iqubuByjh0n3x+hyLOM1XaGtPIhAbcgOZRcfguum8ZB8oKAK6ouDxvg8S3oys/x6eW//YVHYQoQZJXjULmkGc1o3jNcZ0jOkffs01R99IdWV1zRaHcMxj7JZr8ZuLDl3tJA9QPrAP+xy3e6uP75J39JttEtNXn8t9dUmlFHVHExv2n1ZPTmqMapPPOgoKsrPxFx2F6zZYhlBBVRBRIL9mfSoTJzLkXlILjyLIriAWdUinFXE8woNuQUIb1cSUJLnoGrzgVQQlle1PbPDFOOEtMB3P7uhNY5KJFDee+g8+en4KhUbDKwGO53HlMxew5kaj2z2ebreUYN0cGYkImp0D6Xfb7PxBehIr5x5MsGhnkt/tQWb5vyn2A8/xxiClp5P1HaoTSnVCCcg9AXcdxZFcvRytmRcWBLmfQl41ie9/R6TfuUT6nUS1vyVu6eGEhzxUm+QBqheci6RfwnUCHEeJhReTqTgTzbZ8VRpjTPv56/hbmfT8ZKgzEq8uwSHI+vz9/+7J25dKpvjs7Wl8+eHXBEGBNQLbWJfsumkoGlU0/TkS2WG1z6WZz8lUnE08nPvNDYeSJJb9kyC7jEj/vMKcBXllv0TjuxFKv4P6C0gu/VfeQ9bAB5U4otU41HT9hCtIL7mc8MBrKR12RN55syv+Q9SZjITrn0vEJ7nkbmKDL2vdmzbGtKlliyr59I1pBWvN1yUIs6d8y5IFy+g/pByAt598n1vO/BfZbAYB4n3iXPzQOe16198l7+gbSiRAvKFtcq7qir/hefWrw8WiIKnHyCy7lZXf/Y7UkuvQ7A8TeDVYhma/QoNE7TZx+yGxfRFvDQot7uq6gkMCt8GciHAooHrR9XntNf0R/oo7Co5j9VzBT80o+H40O4/EostZOefnrJx3EpqZ0vibN8a0iSXfL6v91t4cRfnv9f8D4Luv53PTaf8kSPs4gYMEDollSS75xTVk0rkhlDMmz+KWs+7iz7+9mfef+wjfz69m2VJd/o4+UM0t1RfZbbXPpapEnOkFv2a5bkBm5f3Ew0ImMZV08ilCA68nsfR+QsF7ZDJKKCQQO4JQ3/G13UuqGSIFnhNnMpqX5FcJOflljqsq/kncK/wXJ5NRQvFx+e8n+x3phUfiSppIRAi0gtTCUwn1+wNufM/GfyOMMatl2FpDcBwHDZpP9oLw8WufAPDSvRMI/CDvm0AykWTKG58z98vvePCaJwiCXJsPX5jMBtuty0UPno3bWEIpQpe8ow80TNYXsr6Qyo4gMvgfiNMGdUcyHzXRF6+1C4CEQkLI88lUnIuTfgfPDYhFFc8NyK64F02+UHtU9eJ7Cn9weEI2KPw5mgkK1Lf289eNzUWliHiEy4/M25dYchOOpGtH8TgihENKeulfUG3/fj9jeqtYSZT9T/wpKvXzSWN3+QOH52rJL124tGB3jwaw6NsKHrzmSQj4YY1ZdZj23pd89MqnqxVvl0z0bngtQoOfJDT4SUpGPIR4a7bNif15ZDKFE2ChZO15SSIN+t6jEaiuuKP2ddhtvEabW/KbvDo56YwQH5T/LMCLb0smm/+XRNXBG3xH4XVs0x/juoUeAiUhsBWq2sqqKpbG1PXzk/fOGwkoSH6yd5RDz/45ADvsuzWB5Ocg13ERAV/zC51J4DDhsYmrFWuXTPRQ0wfu9mu+YUuE1iO36FV9QQuHmLrOD2Pl0355wTZZP4xbdhTh8tNIZeIokEgPIDzgCpzIdnntI+XHoUTJ1kn2qbRA/AQcb92C1wgofG3XEZCyot+PadyqCpaV2UpL+KaeL97/moACiXnVTaOjuBGXY/90RG3p461+ujlrbjKaQH7od1dH+ekxuzJg5ICCw72VgNI+q9ej0eX76NuSeOui3iYkU1NqR8lksopIlHQ2WW/kTDqtuJ6L6zRY7zFQAneT2tfx/ieRWvInIuEfEnQyBaE+R+I4Dk7ZwXhlB6MKJU2N2ZcwgbMennyCAr4fItL/JJz4QY0eEh84ntSiS4nUqaScSika2YFQW3R1GeCHZL9kWXUzLU1vEo6FCvYEKMomu2zAURcdytC1BhOuUyDL9Vz+9MT5vPnoe7x0/+vESqP87Pi92HzXjcikM4TDEbLJ+h8e4jjsedSuqxVrl72jby+xIdcT6nMUyXQpqUwMDe+FN/ghiOxENitUJXLdK76zHm7Z+aQzTu0df9ZX/CBEfOBptedz4nsS6ns66UwU3xcy2RBe2W/w+hxd77pNTcxSher543H9KTiSqyDnOGkyy25Dg2WNHudEf4LX5ziyvksiKWR9hyC8HbFBl6zG75Axphjrb7MOsZL8meviwM9P2pvR64+ol+RXCYVD7Hb4Tlz1vz9w8QPnMG63jRERwpEwFz14Nl7MI3B8fPHBEY68+GDW3Hj1hl52yZmxnVXrRrPzwJ8F7vDa5wKa+ZSqir9Ddi5uZHOi/ccj3vD8Y1Vz5Q+kpGD3UJPXzUyhev6pxKL1/yySyYBwvxPwyvIfxNa/dhL8uSAD2r67y9SyWbKmoa8/mc1lB19LMpFEFVxx2f+kPfnV7w8inUoze+pc4qVRRowdVvTas+l0hs/enEaqOs3GO61PWb/Soo7rdrVuOot4w6FBEpfQJpQOuzmvbZB4lcTiG4iElpLK9CHa7xTckr1ad+Hsd6j6NPyCFY06JBPT8ZrpbheJgrdO665tjGm1tTcdw52fXs8nE6ZRVVnNxjutT/8h5Ux4dCJ/P/ee3KQoFeLlJVx43xlFrfUaDofY4iebtmmcluhbIUi8RnrxpcRq+uVj4UpSS68khI9Xsm/LTxhap2aMbP07+uqEEhuw+eoHbIxpU4mqJBMefZevPprBWhuvwY8P2ZF4zQPTWVO/5dYz/1VvOcGqJVVcuM+VnPvvU9iyQRL/+pPZzP50DoNG9WfjnTYoeh3YluhRiV7VB60CKSv6a1JrJBbfUJvkV4mEleTSm1uV6MVbl8Bdn2Tq89oHwllfcUOlOPF92iRmY0zbWDx/KefsfgkrK6twcXnjvxN58NonuOaFixk2ZghP//2F2glPq6zKR3897lb+9cUNRKIR0ukMlx9yLTM+moUfBDiO0G9wOVc+84facgltpUc8jFVV0stuJz1/d7IL9iM1b0+yK59ot+tFQksKbw+vLLo4WkPxoTfhlR5CKhMjkw2RdXcmOuTetpkoZoxpM7f/391UV1bj1qwM5dYs/n3Taf8EYOF3ixutgZP1M0x9ezoAj/3tf3w56WsIBBcXCRyWfL+M6064tc1j7hF39JnKO8iuuI9ozTBD102SWnYdjtcHJ7r6pRMaSmX6EQvnJ/tUupR4K79JiEQI9zsN+p3WfGNjTKf57M3ptV0yqzg4zPz4G3zfZ6efbcOX783Ia7PKqnvBF+59PW+BcAeHGR/OJrEySay07dbgaPaOXkTuEpGFIvJZI/vPFZHJNb8+ExFfRPrX7JstIp/W7JvUZlHXoRqg1Q/VJvlVImGluiL/IWpbiPU/NTeZqY5UGsLlJ7bL9YwxXYdTYDY6ACKICLscuiNlA8sKlkPwvBAb7bgeAH6m8WJlbVHIrK5ium7uBhodTqKq16rq5qq6OXA+8Iaq1r3d3bVmf8FhP6tNEzhO/uw0gJCztF0u6cR3J9z/DyTSfVAgmS4lVP5/eKX7t8v1jDFdx44/35qA+ok4wGfcHhvjOA7RWIQb37ySUesPz63xihLgI65w5m2/IxrL3ZVuu88WBNSfkKkog9YYQGnfkjaNudmuG1WdICJjijzf4cADqxVRS0kJWT+G6+TPWswEo4gUOKQtuPE9KYnviSrE2/C5rypo6nWqK+4CXU6o5EeE+x5j4+ON6SKOuexXfD15FnO/+j730NURBo8axEl/Paa2Tbwsxl9evowZk2fx8aufEi+Lsf3+W9d7yHrkRYcw+bWpLFtciasuPj7hSISzbm/7noGiJkzVJPqnVXXjJtrEgbnAOqvu6EVkFrCU3LjBv6vqHU0cPx4YDzBy9MgtP/nqk6LfRLbqGTJLr6lXhiCTdQgPuhEJbVb0ebqC9LK/5xYgqfmESmcUpYzosAcRxybrdCabMGVWUVWmT/qaOV/MY/jaQ9hw+3VbNdIvmUjxzhMf8NnEaYxefxS7HbYjffq3rk5VUxOm2jLRHwr8WlV/VmfbcFWdJyKDgZeAU1V1QnPXa83M2CD5JtUVN+E5FWR1NCWDz0JCbTvpoL1psILMgv3w3Ppf51IpxevzG0J9j++kyAxYojfFqVy8nFmfzqH/kL6M3mBkh123o2bGHkaDbhtVnVfz/4Ui8jiwDdBsom8NJ7ozpSN3bo9TN0qz30D2K3CHgrdxixcaz5P9inQ6wGtQPiMSEapWvGmJ3pguTFW5+9IHeeFfrxOQxcFh6NpDuPSR/6N8YJ8Wny8IAj6fOJ0F3yxmjY1Gss5mY1odW5skehHpC+wC/LrOthLAUdUVNT/vCVzeFtfrbKpZEgvPw81+QCYb5FaacYcTH3o74pS3/sTOADxPaDhDNggUN5xfX8cY03W8/cT7PH/XK4g6tcMm5331PVf/5gaufuYiIPdh8PUn35CqTjF2izUJR8IFz7WsYjkX7PtHFs9fmvv37zisvfkYLnnk3EaPaUqziV5EHgB+DAwUkbnAJUCoJujba5odCLyoqlV1Dh0CPF7Tb+UB96vq8y2OsAvyV/wH0u8RikhueUGUTHYu1QsvpmToja0+r3hr4MsaZLOzahJ+TtZ3iQ38TRtEblZXZbbSum9MQf+98SlE88fXz/50DksXLmPl0mouPeRaqpauRFFcL8RJ1x3NTj/fNu9c1590OxXf5VajcgEC+OqjmTx07RMc+YdDWhxbMaNuDi+izd3khmHW3TYT6LJPQlUDNPUa1UseR8Ql3v8gCO9cVPdLavnDxBqsPBXyBCeYggZJxGn9RIf40Juo+v4c8L8kmwXHCRPudx7ibdDqc5q20b88zpJl1ZbsTUFVyxKN7FFWLqniogOvJrEiidT8F6R9bj79TsZsNJqRY4fVtk4lU3zx7oy82bUOLq/cN6FVib5HlEBoKVVILPw/EgsvJe5NIeZ+RGLhBSQriutZciTT2JmB9GrFJk45pcP/SWjwY8SG/ZvI8Odx4nus1jlN27EVp0xjttlrXN74eoBwLMyi7xZTXVWdt1BJ4Ac8f/cr9bdltWHvba1stnVrQffKRE92CqTfJ1bnxjsaAUm/jGa/bv740I71lvxbJZUdjDgtf+hSiLgDEW9NRHpElYoepX95vDbhG7PKwWcfQEl5aW2yDwjAhVOuP47q5Qm0QI52cFjyff2JnbHSKEPXGZw3szYgYMs9WjeSsFcm+iA5kXC4wO+6BpD+oNnjowNOxdcyEsncH0QqrWSyHiVDesSzZmNMK5QP7MNNb1/JIeccwOhNR7D9/ltx9XMXsc3eW7DBduviOgXWqxafHfbbJm/7WbeeQCgaqvOh4VPWv5RjLmu2J72gXnm7KE456bQSadDPnvUhIs3fkYs7gOiw/+JX/Y/qlZOIlK2DV/pLxB3UXiEbY7qB0r4lHHTG/hx0Rv1yKAOG9WOv437Cc3e+jAS5vBPgM2q9EWy335Z55xm9wUhuff8aXn3gLWZ/MYeNtlmPnX65La7nkkykassoFKt3Jvro7rjebdCgzkQoFILILsWdwynBjW4Ay18hveJJUtWfEu+zPamqL/HCA/FKD0S8UbXtNaiE7NfgDEG8EW35dowxXVAQBLz0nzd48tbnSCXT7LDf1px20/G8dN/rVK9M8JNDf8Suh+2IFyqchvv0L+PnJ+8N5CZhXXvczXz+9pegMGydIZx52wmMXr+4XNJr14wNUu+QWfwHMpncg1UvFCEy6C9IaPOij09XnE84lPuwWNWfJgiZrAIuoX6XINHdSC+9AUk9RjodEAoJWVmP+JDrAYXUm0ASQtsWXIt2FdUsmnye6iWPAQ7x/ocg0T3adYEV0zSbKWuact2Jt/Pu0x/gaK7LJiCg78A+3Pj2lcRKih+ZFwQBp+zweyrmLqkdiaMooWiIW9+/prZkgq0ZW4AT2YHwsBcIZ6YALoQ2adGDz8Siq4nV6eev+zQ95AkQkFn6R0J9V+BX/ZdohJoZr4qmp5FceCquziSdzuKI4nkeGjuMSL/8gkaqkFhwGpr+hHhUaq7/RyT6OvHBV7byd8AY016+n72Q95/5qDbJQ+7B67KKSt545B32Orr4dTKmvfsVi+ctqTfcUhBSyRSvPvg2Pz+p+bWqe+XD2FVEQkh4KyQ8rkVJXjVDJNR8CeRMJkty6Z15tfLDYSHsfIXn+sRjQjTq4HkBQdUDaKZA2f/MJMh8Siz6w4dJLApO5i00+1XRcRtjOsbMT77B1/yhli4uk16e0qJzfT97EUGQ3/Pi4jL782+KOkevTvStF8L3Q822chxBJNnI3vw/uJAXkKp8Mm+7n/yASKRAF5sGkP6o2TiMMR2r/9BynALdqgEBI9ceVuCIxq2x0UjcAguG+/hsuM16RZ3DEn0riIDEDyaZbPz5RqCKOGUQ2o6sn9+u4KMRAQ3yJ1w5Tj9SqfwDsj6wOrV1jDHtYr2t16H/sH5ogwEfruey93E/adG51tlsDGttNqbeZKyAgLJ+pez8y+2KOocl+lYK9f0dTskvyPoOqbQQBEIQQHVCqU5A1o8THXIj0QGnEGicVDqXqINAyWQFNH9MbTotRMv3y9susZ/iufnfIEKhMIR/1PZvzphepqqymlvP+RdHrXcyR613Mnf8/h6qVzRW0qB5IsIVT13IGpuMymVZB+L9Y1x43xkMGV1/GHY6lWbezO+bvN4lj5zDvuP3JFIaxot6bLvvlvzl5cuKfqjba0fdtBUNkhAsAXdg7v+ZySB9ILxNbb+/+kvILL+f1Mp3cSOjifU/Bj/1JUHln1H1cT3IZBwI/4jooD8VrLcTpCaRXnw+fjaFAOLGiQ6+Dglt2KHv1/zARt30DH7W59Sdzq83qiXAZ+haQ7j+9T/hFOg2WUVV+WTC57xw72sEfsDuv9qFLXfftN5ouGWLKkkl0gweNbDedlXl0Rv/x2PXP0vg+yAOOx24NSdce3SjQy6bYqNu2pE4UXBqhkW6Q8HNfwIubn/C/U4h3O+U2m2eNxaNbI6feAGCKqL9fwTeZo0WVXMiWxEZ9jxkvwBxwF3fhlYa0wY+fGkKFd8tri0tDLkCYgtmL+STNz5n810bXW+J28/7N68/+HbtJKiPX/mEbfbekrNuP6H232f5oMI3A2888g6P/PUpJHCAXBXcCf+dSDgWZvxVR7XZ+8u9H9NpxBuBV3YsXt9TkVDjSb62vbhIaCPE28CSvDFtZNZnc2qSbX0awOzP5zZ63Jzp83jjwXdqkzyABA4fPPcRMz6e1ex1H/jzY3nXdXB57YF3yGayLXgHzbNEb4zp1YaNGUzgFKh9JcqQ0QMbPW7y61MJgvwhlBrAhy83v+b1yqXVBbdrEJCsTjV7fEtYojfG9Grb7LcFJSWxXLXJGgEBJX1L2fKnjS+pESuJFKwmrASU9G2+uukaG43Mq1AJEOsTpaRP21ZHtURvzGqwmvTdXzQW4ernL2bMpj+MkFl73BiufeFiwuHG58tsu98WeF7+ftf12PGArZu97vFXHonjufWHYDow/uqj2rxr1kbdGLMalizLff220Tc9Q6I6iYgUXR1y8utTufbYm8lkc/NfPDfEaTf/lu32LTj4Jc+30+ZyzxWPMPOT2QwePZAjLziEjXYobhJUQ02Numk20YvIXcB+wEJVzXv8LCI/Bp4EVj19eExVL6/ZtxdwA+AC/1TVq4sJ2BK96U4s2fduqWSKqW9/SRAEbLTjesTirV9KdHWs7vDKu4GbgXuaaPOmqtab6SMiLnALsAcwF/hARJ5S1c+LitqYbmLVWrKmd4pEI2zxk006O4wmNdtHr6oTgCWtOPc2wAxVnamqaeBB4IBWnKfH0KAazXyCZhsfsmWMMW2trSZMbS8iU4B5wDmqOhUYAcyp02YusG1jJxCR8cB4gJGjR7ZRWF1HZvm9aNU/SacDPE/wGUN86A2I269dr6uaRpOvkFrxHqHoaNyS/RG38SFjxpiepy1G3XwErKGqmwE3AU/UbC/02LjRBwKqeoeqbqWqWw0c2LMSkaYnkl3+Dzw3IB6DcEjxZCbVC85q3+sGK0nMO4zqhVcQ4RUyy+8iveCgwqWQjTE91monelVdrqora35+FgiJyEByd/Cj6jQdSe6Ov9epqvgXkXD9z7iQJ3h8jfrft9t1M5V3IbqQeCz3mRsJCyHPJ7HggsLVM40xrbZo7mLee+ZDZkyeRVcbzbjaXTciMhRYoKoqItuQ+/BYDCwDxorImsB3wGHAr4o5Z5Cdj2bnIl4P6cLxlxT8nc5mIazLgaHtctnMyheJRfK/WLnOMgi+z9XmMcasliAIuPHUf/Du/z7EVx9XHAatMZA/Pn4+5QP7dHZ4QBF39CLyADARWE9E5orIcSJygoicUNPkIOCzmj76G4HDNCcLnAK8AEwDHq7pu2+eVpJeeASaaX4acXcQLt2FdCb/E97zXHDHtNt1lcKTPXJzMVq2irwxprAX//0a7zz1PgTgqguBsGDWIq497ubODq1Ws3f0qnp4M/tvJjf8stC+Z4FnWxqUkFttqWrBZZSOfLSlh3c5ob5Hkqx+llR6BZGwEKiSzTp4fc9AJNxu142WH0py6c1E6yxB6PtKOliTcDs/BDamt3jytufqrQ0LufVhv/54NiuWraSsvLSTIqsbTxcW9RahQfcfnyxOOdFhD+KV/Zrq1ChSujWRwTfjlbbvaFO39GCIbEc6IySSNQuiBP0oGfrndr2uMb1Jsjp/VbhVUk3s60hduh69IiDNr83aHYjTl1DfEwj1PaH5xm11TXGJD/kLmp0JmWngDobQloh06c93Y7qVrfcax+sPvl27aMkqJf3iDBjWNb45d9l/8am0knF2RnpIou9M4q2FxPZFwltbkjemjf3qvF9Q2q8Ev2ZN14AA8YQzbvldl1k3okve0atC4G5EfOAFnR2KMcY0qXxQX25++ypevv9NPnxlMqPWHcG+x+/BsDFDOju0Wl2yeuXmW2ykr737VmeHYUzRbP1Y09maKmrWJb/Ht+dIFGOM6W26ZKI3xpjuLrEySVVl1xg12CX76I3pjlatNmVdOD1fKpmiYu4S+g0pJ14Wq7dv8fyl/OX4W5j1ybcADFlrEOf8/WRGrTe8M0IFumgfvS08YrorW4SkZ1NVHrr2CZ689QU08AGHnQ/alhP+/Btcz8XP+vxuq3NYXrGidrilooRjYW6fdG1Ra8m2Vrfrozemu+pfnvuHbGvJ9kwv3fsGj9/0DJoNIBAIlDcefpt7Ln8IyC0tuHzJ8npj6gUhmUjy5mPvdlbYluiNaWurkr3peR6+7klE66dNB5eX7p2A7/ss+rYCDfKPc3GZ+1XnFe+1RG+MMUWqrkwU3B5kA9LJDGM2HoXj5KfVQHzW22pse4fXKEv0xhhTpFEbFH6gWjogTjQeYb2t12GNjUYS1MyShdxM2f5D+7Htflt0VJh5LNEbY0yRjr/qKMQTlFz/jKLgwO/+fDQigojwx8d/z/4n7kWsb5RISYhdD9+Ja1+6lHC488q52KgbY9qBzZTtub6ZNpd7r3iEmZ/OZtiaQ/j1+Qezwbad1y2zSlOjbmwcvTHGtMAaG4zkD/85s7PDaBHrujHGmB7OEr0xxvRwluiNMaaHK2Zx8LtEZKGIfNbI/iNE5JOaX++IyGZ19s0WkU9FZLKITGrLwI0xxhSnmDv6u4G9mtg/C9hFVTcF/gjc0WD/rqq6eWNPg40xxrSvZkfdqOoEERnTxP536rx8FxjZBnEZY4xpI23dR38c8Fyd1wq8KCIfisj4pg4UkfEiMklEJlVUVLRxWMYY03u12Th6EdmVXKLfqc7mHVV1nogMBl4SkS9UdUKh41X1Dmq6fcZtOa7rzeIyxphuqk3u6EVkU+CfwAGqunjVdlWdV/P/hcDjwDZtcT1jjDHFW+1ELyKjgceAI1X1yzrbS0SkbNXPwJ5AwZE7xvREldlKq0tvuoRmu25E5AHgx8BAEZkLXAKEAFT1duBiYABwq4gAZGtG2AwBHq/Z5gH3q+rz7fAejOlyVtWkX7KsmspspdW9MZ2qmFE3hzez/7fAbwtsnwlsln+EMb1H//J47fKCxnQWmxlrjDE9nCV6Y4zp4SzRG2NMD2eJ3hhjejhL9MYY08NZojfGmB6uVywlqJmpJJbeR5BZRKzvHjjx/REn2tlhGWNMh+jxiT678nGylX8j7AY4ISGxZBpS+RCxYf9BnFhnh2eMMe2uR3fdaJAkWHED4ZDiOAJALCoQLMCveryTozPGmI7RoxM9/hek037e5mhESFZaNQZjTO8gql2vIrCILAK+6ew42tlAoDcX3rf3b+/f3n/bWkNVBxXa0SUTfW8gIpN68/KK9v7t/dv777j337O7bowxxliiN8aYns4Sfee5o7MD6GT2/ns3e/8dyProjTGmh7M7emOM6eEs0RtjTA9nib6TiMi1IvKFiHwiIo+LSHlnx9SRRORgEZkqIoGI9JphdiKyl4hMF5EZIvL7zo6no4nIXSKyUEQ+6+xYOoOIjBKR10RkWs3f/9M74rqW6DvPS8DGqrop8CVwfifH09E+A34BTOjsQDqKiLjALcDewIbA4SKyYedG1eHuBvbq7CA6URY4W1U3ALYDTu6IvwOW6DuJqr6oqtmal+8CIzszno6mqtNUdXpnx9HBtgFmqOpMVU0DDwIHdHJMHUpVJwBLOjuOzqKq81X1o5qfVwDTgBHtfV1L9F3DscBznR2EaXcjgDl1Xs+lA/6Rm65JRMYA44D32vtaPb5McWcSkZeBoQV2XaiqT9a0uZDc17n7OjK2jlDM++9lpMA2G9/cC4lIKfAocIaqLm/v61mib0equntT+0XkN8B+wE+0B05oaO7990JzgVF1Xo8E5nVSLKaTiEiIXJK/T1Uf64hrWtdNJxGRvYDzgP1Vtbqz4zEd4gNgrIisKSJh4DDgqU6OyXQgERHgTmCaql7XUde1RN95bgbKgJdEZLKI3N7ZAXUkETlQROYC2wPPiMgLnR1Te6t5+H4K8AK5h3APq+rUzo2qY4nIA8BEYD0RmSsix3V2TB1sR+BIYLeaf/eTRWSf9r6olUAwxpgezu7ojTGmh7NEb4wxPZwlemOM6eEs0RtjTA9nid4YY3o4S/TGGNPDWaI3xpge7v8BgHpWyK6AzK8AAAAASUVORK5CYII=\n", "text/plain": [ "
" ] }, "metadata": { "needs_background": "light" }, "output_type": "display_data" } ], "source": [ "from sklearn.datasets import make_classification\n", "X1, Y1 = make_classification(n_features=2, n_redundant=0, n_informative=2,\n", " n_clusters_per_class=1, class_sep=2, random_state=1)\n", "plt.scatter(X1[:, 0], X1[:, 1], marker='o', c=Y1,\n", " s=25, edgecolor='k')\n", "\n", "data = X1\n", "targets = Y1\n", "\n", "# applying the gradient step \n", "x_in = data[0,:]\n", "target = targets[0]\n", "weights = []\n", "biases = []\n", "\n", "current_is = input_dim\n", "current_os = network_size[0]\n", "weights.append(np.random.normal(0,1,(current_os, current_is)))\n", "biases.append(np.random.normal(0,1,(current_os,)))\n", "\n", "# random initialization of weights\n", "for l in np.arange(1,len(total_size)):\n", " \n", " current_is = current_os\n", " current_os = total_size[l]\n", " weights.append(np.random.normal(0,1,(current_os, current_is)))\n", " biases.append(np.random.normal(0,1,(current_os, 1)))\n", "\n", "# learning rate\n", "# learning rate\n", "eta = .01\n", "\n", "num_epochs = 2000\n", "total_loss = np.zeros(num_epochs)\n", "\n", "for e in range(num_epochs):\n", " \n", " # random swapping \n", " indices_epoch = np.arange(np.shape(data)[0])\n", " np.random.shuffle(indices_epoch)\n", " data_epoch = data[indices_epoch,:]\n", " target_epoch = targets[indices_epoch]\n", " \n", " # SGD\n", " \n", " grad_weights_tmp = []\n", " grad_biases_tmp = []\n", " \n", " for l in np.arange(len(weights)):\n", " grad_weights_tmp.append(np.zeros(np.shape(weights[l])))\n", " grad_biases_tmp.append(np.zeros(np.shape(biases[l])))\n", " \n", " for i in np.arange(len(target_epoch)):\n", " \n", " loss, f, g, b = SGD_neuralNet(data_epoch[i,:], target_epoch[i], weights, biases)\n", " \n", " total_loss[e] += loss \n", " #one gradient step \n", " \n", " \n", " for l in np.arange(len(weights)):\n", " \n", " grad_weights_tmp[l] = np.squeeze(grad_weights_tmp[l])+ np.squeeze(g[l])\n", " grad_biases_tmp[l] = np.squeeze(grad_biases_tmp[l]) + np.squeeze(b[l])\n", " \n", " \n", " for l in np.arange(len(weights)):\n", " \n", " weights[l] = weights[l] - (eta/len(target_epoch))*grad_weights_tmp[l] \n", " biases[l] = biases[l].reshape(-1,1) - (eta/len(target_epoch))*grad_biases_tmp[l].reshape(-1,1)\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", "x1min = np.min(data[:,0])\n", "x1max = np.max(data[:,0])\n", "x2min = np.min(data[:,1]) \n", "x2max = np.max(data[:,1])\n", "\n", "xx1 = np.linspace(x1min, x1max, 100)\n", "xx2 = np.linspace(x2min, x2max, 100)\n", "xx1,xx2 = np.meshgrid(xx1, xx2)\n", "\n", "data_grid = np.vstack((xx1.flatten(), xx2.flatten())).T\n", "\n", "prediction = np.zeros((np.shape(data_grid)[0],1))\n", "\n", "for sample in np.arange(np.shape(data_grid)[0]):\n", " \n", " prediction[sample,:] = SGD_neuralNet(data_grid[sample,:], 0, weights, biases)[0]\n", " \n", "\n", "plt.scatter(data[:,0], data[:,1], c = targets)\n", "plt.contourf(xx1,xx2, np.reshape(prediction>0.5, np.shape(xx1)), levels=1, alpha = .1)\n", "plt.show()\n" ] }, { "cell_type": "code", "execution_count": 16, "metadata": {}, "outputs": [ { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYcAAAD4CAYAAAAHHSreAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAAkNElEQVR4nO3deXxU9b3/8dcnO4EkbCFkZUtYAsgqsoiIoCIuUBUVtbVK3er6s7XVa+/tvbf3Xqtttfpz4SeKWhdQVERsRdyRTWSXRSBAIAv7voVs398fGXtjEiADyZxJ5v18PPJg5syZOe+cGeads5tzDhERkcrCvA4gIiLBR+UgIiLVqBxERKQalYOIiFSjchARkWoivA5QV1q3bu3at2/vdQwRkQZlyZIlu51ziVWHN5pyaN++PYsXL/Y6hohIg2JmW2oartVKIiJSjcpBRESqUTmIiEg1KgcREalG5SAiItWoHEREpBqVg4iIVBPy5fDZ2h28Oj/X6xgiIkEl5Mvh7yu38ehHa9l16LjXUUREgkbIl8PdF2RSXFrOpK83eR1FRCRohHw5dExsxhW9UnhtwRZ2H9bSg4gIqBwAuPuCLIpKy7T0ICLio3IAMts04/KzKpYe9h4p9jqOiIjnVA4+947I5FhJGS9q6UFEROXwg8w2cVzaM5lX5+eyT0sPIhLiVA6V3Dsii6MlZbw0d7PXUUREPKVyqKRzUhyjeyTzyvxc9h/V0oOIhC6VQxX3jMjk8PFSJmvpQURCmMqhiq5t4xnVvS0vz8vlwNESr+OIiHhC5VCDe0dkceh4KZPnaelBREKTyqEG2SnxXJSdxOR5mzlYpKUHEQk9KocTuHdEFoeKSnllXq7XUUREAk7lcAI9UhMY2S2Jl+Zq6UFEQo/K4STuH5nFgWMlvPS1tj2ISGhROZxEj9QELunRlpfmbtZR0yISUlQOp/DAhZ05UlzKxK82eh1FRCRgVA6nkJUUx9jeqby6IJedB4u8jiMiEhAqh1q4f2QWpWWOZ77I8TqKiEhAqBxqoV2rpozrn86URVvJ33fU6zgiIvVO5VBL947IxMx4+rMNXkcREal3QVkOZjbWzCaZ2Qwzu8jrPADJCU248Zx2vLu0gE27DnsdR0SkXgWsHMxsspntNLNVVYaPMrN1ZpZjZg8BOOfed87dCvwcuDZQGU/ll8M7ER0RxpOfaulBRBq3QC45vAKMqjzAzMKBZ4FLgGxgvJllVxrld77Hg0LrZtHcPKQ9M1cUsnbbQa/jiIjUm4CVg3NuDrC3yuABQI5zbpNzrhiYCoyxCo8BHznnlp7oNc3sNjNbbGaLd+3aVX/hK7ltaCfiYyJ4fNb3AZmeiIgXvN7mkArkVbqf7xt2DzASuNrM7jjRk51zLzjn+jvn+icmJtZvUp+E2EjuGp7JF+t2MT9nd0CmKSISaF6Xg9UwzDnnnnbO9XPO3eGcmxjwVKdw0+D2pDZvwqMffU95ufM6johInfO6HPKB9Er304BCj7LUWkxkOL+6qDPfFRxg5sqgjysi4jevy+FbIMvMOphZFHAd8IHHmWplbO9UspPj+dPH6zheWuZ1HBGROhXIXVmnAAuALmaWb2YTnHOlwN3Ax8Ba4G3n3OpAZToTYWHGv4zuRv6+Y7y2YIvXcURE6lREoCbknBt/guH/AP4RqBx16dys1pzXOZH/+3kO4/qlkxAb6XUkEZE64fVqpQbvoVFdOVhUwnNf6qR8ItJ4qBzOUHZKPFf2SePlebls3aOT8olI46ByqAO/vrgz4WHGf/9jjddRRETqhMqhDiQnNOGu4Z34ePUO5unAOBFpBFQOdeQXQzuS3rIJ/zFzNaVl5V7HERE5IyqHOhITGc4jo7NZv+Mwry/Urq0i0rCpHOrQxd2TGJLZiic+Wc/eI8VexxEROW0NvhzM7HIze+HAgQNeR8HM+LfLunOkuIy/zF7ndRwRkdPW4MvBOTfTOXdbQkKC11EA6NI2jhvPyWDKoq2sLvS+sERETkeDL4dg9MCFXWgRG8Uj01dRprO2ikgDpHKoBwmxkTxyaTeW5+3nzUVbvY4jIuI3lUM9+UmfVAZ3asXjs75n56Eir+OIiPhF5VBPzIw/jO3B8ZJy/uvDtV7HERHxi8qhHnVKbMad53figxWFzFkfmGtci4jUBZVDPbvz/E50aN2Uf52xiqISXRRIRBoGlUM9i4kM57/H9mDLnqM8+cl6r+OIiNSKyiEABme2ZvyADCZ9vYmlW/d5HUdE5JRUDgHyL6O70jY+hgenrdDqJREJeiqHAImLieSPV53Fxl1HePJTrV4SkeCmcgig8zonMn5AOpPmbGKZVi+JSBBTOQTYv4zuRtv4GH6t1UsiEsRUDgEWFxPJY1dXrF76n3/o4DgRCU4qBw8MzUpkwrkd+NuCLXy2dofXcUREqmnw5RBM13Pwx29GdaFbcjwPvrNS514SkaDT4Msh2K7nUFvREeE8fV1vjhwv5Vdvr6Bcp/YWkSDS4MuhIctKiuN3l2Xz9YbdTJ632es4IiL/pHLw2I3nZDCyWxKPzfpeu7eKSNBQOXjMzPjzuLNIio/hl28sZc/h415HEhFROQSD5rFRPH9DP/YcKeb+t5br0qIi4jmVQ5DomZbAf17Rna837OYpnV5DRDymcggi156dztX90nj68xwd/yAinlI5BBEz4w9jepCdHM99U5ezfschryOJSIhSOQSZJlHhvHhTf2Iiw5nw6rfsPVLsdSQRCUEqhyCU0rwJk37Wjx0Hj3PH60soLi33OpKIhBiVQ5Dqk9GCP119Fos27+XfZqzCOe3BJCKBE+F1ADmxMb1T2bDjMM98kUN6y1juGp7pdSQRCREqhyD3wIWdyd93lD99vI7EZtFcc3a615FEJASoHIJcWJjx+NW92HOkmIenf0erZlGM6JbkdSwRaeS0zaEBiIoI4/kb+5GdHM9dby5lyRadg0lE6pfKoYFoFh3ByzefTdv4GG555VvWFB70OpKINGIqhwakdbNoXptwDk2jwrnxpW9Yt10HyYlI/Wjw5dBQrwR3utJbxvLmrQOJCDNueHEhOTsPex1JRBqhBl8ODfVKcGeifeumvHnrQMC4ftJCNu8+4nUkEWlkGnw5hKrMNs1489ZzKC13XPfCAjboPEwiUodUDg1Y56Q4ptw6kHIH176wkFUFobFqTUTqn8qhgevSNo5ptw+iSWQ4419YyLe5e72OJCKNgMqhEWjfuinT7hhEYlw0P33pG75av8vrSCLSwKkcGomU5k146/ZBdGjdjAmvfMu0xXleRxKRBkzl0IgkxkXz1u0DGdixFQ++s5InPlmvs7mKyGlROTQy8TGRvHzz2RWXG/1sA7+atkLXgxARv+nEe41QZHgYf7r6LDJaxvLEJ+vZfqCIZ6/vS4umUV5HE5EGQksOjZSZce+ILP4yrheLc/dxxbNzWbtN52MSkdpROTRyV/VL463bB1JcWs6Vz83n7yu3eR1JRBoAlUMI6JPRgpl3n0u35DjuenMpj8/6nrJybagWkRNTOYSINvExTLltIOMHpPPclxv52eRv2HmoyOtYIhKkVA4hJDoinEevPIvHrurJki37GP3UXObl7PY6logEIZVDCLr27Axm3HUuzWMjufGlb3hi9jqtZhKRH1E5hKgubeP44O4hXNU3jac/z+H6SQsp2H/M61giEiSCuhzMrKOZvWRm73idpTGKjYrgz+N68ZdxvVhVcIBRT87hvaX5OqpaRGpXDmbW3MzeMbPvzWytmQ06nYmZ2WQz22lmq2p4bJSZrTOzHDN7CMA5t8k5N+F0piW1d1W/ND667zy6tI3jgbdX8Ms3lrL3SLHXsUTEQ7VdcngKmOWc6wr0AtZWftDM2phZXJVhmTW8zivAqKoDzSwceBa4BMgGxptZdi2zSR3IaBXLW7cP4rejuvLp2h1c9OQcPv9+h9exRMQjpywHM4sHzgNeAnDOFTvn9lcZbRgww8xifM+5FXi66ms55+YANV1wYACQ41tSKAamAmNq8wuE2jWk61N4mHHn+Z2Ycde5tG4WxS2vLOZXb69g/1EtRYiEmtosOXQEdgEvm9kyM3vRzJpWHsE5Nw2YBUw1sxuAW4Br/MiRClQ+x3Q+kGpmrcxsItDHzB6u6YmheA3p+padEs+Mu4dw1/BOzFhewMgnvmLmikJtixAJIbUphwigL/C8c64PcAR4qOpIzrnHgSLgeeAK59xhP3JYDcOcc26Pc+4O51wn59yjfryenKHoiHAevLgrH9x9LinNm3DPlGXc+rfFbDugPZpEQkFtyiEfyHfOfeO7/w4VZfEjZjYU6AFMB37vZ458IL3S/TSg0M/XkHqQnRLPe3cO5neXdmNuzm4ufGIOry3couMiRBq5U5aDc247kGdmXXyDRgBrKo9jZn2ASVRsJ7gZaGlm/+VHjm+BLDPrYGZRwHXAB348X+pRRHgYvxjakdn3D6N3enP+9f1VXPncPFbk7fc6mojUk9rurXQP8IaZrQR6A/9T5fFYYJxzbqNzrhy4CdhS9UXMbAqwAOhiZvlmNgHAOVcK3A18TMWeUG8751afxu8j9SijVSyvTRjAU9f1ZtuBIsY+N4+H3/uOfdrtVaTRscaykbF///5u8eLFXscIGYeKSnjq0w28PD+XuJgIfjuqK9f2TycsrKbNRyISrMxsiXOuf9XhQX2EtASvuJhIfndZNv+4dyidk+J4+L3v+Mlz81iuVU0ijYLKQc5Il7ZxvHXbQP56bW8KDxQx9tl53D91GYU6T5NIg6ZykDNmZoztk8rnvxrGL8/vxD9WbeeCv3zJE7PXceR4qdfxROQ0qBykzsTFRPKbUV357IFhjOyWxNOf5zD8z1/y9uI8yrXrq0iDonKQOpfeMpZnru/Lu3cOJqV5E37zzkouf2YuCzbu8TqaiNSSykHqTb92LXjvzsE8dV1v9h0pZvykhdz88iLWbjvodTQROQWVg9SrsDBjTO9UPv/1+fxmVBcWb9nH6Ke/5oG3lpO396jX8UTkBHScgwTU/qPFPP/lRl6enwsObhiYwd3DM2nVLNrraCIh6UTHOagcxBPbDhzjr59sYNqSPGKjIrjtvI5MOLcDTaMjvI4mElJUDhKUcnYe4vFZ65i9Zgetm0Vz74hMrjs7g6gIrfEUCQSVgwS1JVv28dis71m0eS+pzZtw74hMruybRmS4SkKkPun0GRLU+rVrwVu3DeTVWwbQulkUv333O0Y+8RXvLc3X6cFFPKBykKBhZgzrnMj7dw3hxZ/1p2lUBA+8vYKLnqy4Ep0OpBMJHJWDBB0zY2R2Eh/ecy7P39CX8DDjninLGP3018xatV2XKxUJAJWDBK2wMOOSnsl8dN95PHVdb4pLy7nj9SVc/sxcPv9+h0pCpB5pg7Q0GKVl5by/vJCnPltP3t5j9E5vzn0jszi/cyJmuo6EyOnQ3krSaJSUlfPOknye+TyHgv3H6JWWwL0jsrigaxuVhIifGm05mNnlwOWZmZm3btiwwes4EkDFpeVMX5bPM1/kkLf3GN1T4rl3RBYXZSepJERqqdGWww+05BC6SsrKeX9ZAc9+kUPunqN0S47n3gsyubh7W122VOQUVA7S6JWWlTNzZSH/9/McNu06QuekZtxzQRajeyYTrpIQqZHKQUJGWbnjQ19J5Ow8TKfEptxzQRaXnZVMhI64FvkRlYOEnPJyx0ertvP0ZxtYt+MQHVo35e7hmYzpnaKSEPFROUjIKi93zF6zg6c/28CabQdJb9mEO4Z14up+aURHhHsdT8RTKgcJec45Pl27k2e+yGFF3n6S4qO5dWhHxg/I0KnCJWSpHER8nHPM37iHZ7/IYf7GPbSIjeTmIR24aVB7EmIjvY4nElAqB5EaLN26j+e+yOHTtTtpGhXOjYPaMeHcDrSJi/E6mkhAqBxETmLttoM8/+VGPlxZSER4GNf2T+f2YR1JaxHrdTSReqVyEKmFzbuP8P++2si7S/NxDsb0TuXO8zuR2aaZ19FE6oXKQcQP2w4cY9Kczby5aAvHS8sZ1b0tdw3PpEdqgtfRROqUykHkNOw5fJyX5+Xy6oJcDhWVMqxzIncNz2RAh5ZeRxOpEyoHkTNwsKiE1xdu4aWvN7PnSDH92rXgjmGdGNG1jc7fJA2aykGkDhwrLuPtxXlM+noT+fuOkdWmGbed15ExvVOJitBR19LwqBxE6lBpWTl//24bE7/axNptB2kbH8OEczsw/pwMmumAOmlAVA4i9cA5x5wNu5n45UYWbNpDfEwEPx3Ujp8P7kBiXLTX8UROSeUgUs9W5O1n4lcbmbV6O5HhYVzdL43bhnakfeumXkcTOSGVg0iAbNp1mElfb+bdJfmUlpdzSY9k7hjWiZ5p2g1Wgo/KQSTAdh4q4uV5uby+cAuHikoZ3KkVdwzrxNCs1rqMqQQNlYOIRw4VlTBl0VZemruZHQePk50cz+3DOnJpT118SLynchDx2PHSMmYsK2TinI1s2nWE9JZN+MW5HRnXP43YKO3hJN5QOYgEifJyx6drdzDxq40s3bqfhCaR3Dgwg5sGtadNvM4GK4GlchAJQku27GXSnM18vGY7kWFhjOmdwi+GdqRL2zivo0mIOFE5aFlWxEP92rWk309bkrv7CJPnbWba4nymLclnWOdEbh3akSGZrbTxWjyhJQeRILLvSDFvfLOFV+ZvYffh43RLjufWoR247KwUnZ5D6kWDXK1kZh2BR4AE59zVJxtX5SCNyQ8bryd9vYkNOw+TFB/NzUM6MH5ABglNdClTqTsnKoda/yliZuFmtszMPjyDEJPNbKeZrarhsVFmts7McszsIQDn3Cbn3ITTnZ5IQxUdEc41Z6cz+/+cxys3n01mm2b88aPvGfzoZ/zHzNXk7T3qdURp5PxZTr0PWFvTA2bWxsziqgzLrGHUV4BRNTw/HHgWuATIBsabWbYf2UQaJTPj/C5teOMXA/n7vedyUfe2vLZgC8P+9AV3vbmU5Xn7vY4ojVStysHM0oBLgRdPMMowYIaZxfjGvxV4uupIzrk5wN4anj8AyPEtKRQDU4Extcx2uZm9cODAgdqMLtJgdU9J4Mlre/P1b4dz63kdmbN+F2Ofncc1Excwe/V2ysqDdxWxNDy1XXL4K/AboLymB51z04BZwFQzuwG4BbjGjxypQF6l+/lAqpm1MrOJQB8ze/gE057pnLstIUHnrZHQkJzQhIcv6caCh0fwr5dlU7D/GLe9toQL/vIlL8/bzOHjpV5HlEbglOVgZpcBO51zS042nnPucaAIeB64wjl32I8cNe2r55xze5xzdzjnOjnnHvXj9UQavWbREUw4twNfPXg+z1zfh9bNovmPmWsY9D+f8YcP12i7hJyR2hznMAS4wsxGAzFAvJm97py7sfJIZjYU6AFMB34P3O1HjnwgvdL9NKDQj+eLhKyI8DAuOyuFy85KYXnefl6et5lX5+fy8rzNXJidxC1DOjCgQ0sdLyF+8WtXVjM7H/i1c+6yKsP7AFOo2C6xGXgd2OSc+10Nr9Ee+NA516PSsAhgPTACKAC+Ba53zq2ubTbtyiryv7YfKOK1hbm88c1W9h8toXtKPDcP6cDlvZKJjgj3Op4EkTPelfUUYoFxzrmNzrly4CZgSw0hpgALgC5mlm9mEwCcc6VULGl8TMUeUW/7Uwwi8mNtE2J48OKuLHhoBI9e2ZPi0nJ+PW0FQ/74BX/9dD27Dh33OqIEuaA+CM4fWnIQOTHnHHNzdjN57ma+WLeLqPCK8zjdPKQD2SnxXscTD+ncSiIhzMwYmpXI0KxENu46zCvzcnlnScV5nAZ2bMktQzowolsS4WHaLiEVtOQgEqIOHC1h6rdbeXV+LoUHishoGcvPB7dnXP804mJ0io5Q0SDPreQPlYPI6SktK+fj1TuYPG8zS7bso2lUOFf2TeOmwe3IbKNThzd2KgcROaUVeft5dUEuH67YRnFZOUMyW3HToPZa5dSIqRxEpNb2HD7O1G/zeH3hFrYdKCK1eRN+Oqgd1/ZPp0XTKK/jSR1SOYiI30rLyvlkzQ5eXZDLwk17iY6o2MvppsHt6Z6iU9Y0BioHETkj328/yN8WbGH60gKOlZTRv10LbhrcnlE92hIZrgsRNVQqBxGpEweOljBtSR6vLdzClj1HaRMXzQ3ntGP8Oem0iYvxOp74SeUgInWqvNzx1fpdvDI/l6/W7yIy3BjdM5mfDWpP34zmOpdTA6GD4ESkToWFGcO7tmF41zZs3n2Evy3I5Z3F+cxYXkjP1AR+OrAdl/dKoUmUzuXUEGnJQUTqzJHjpby3rIDXFuSyfsdh4mMiuKpfGjec047MNs28jic10GolEQkY5xyLNu/l9W+2MmvVNkrKHIM6tuLGge24qHuSNmAHEZWDiHhi16HjvL04jze/2UrB/mMkxkVz3dnpjB+QQUrzJl7HC3kqBxHxVFm546v1O3l94Va+WLcTAy7omsSNAzM4LyuRMB2B7QltkBYRT4WHGRd0TeKCrknk7T3KlEVbeevbPD5du4N2rWK5fkAG4/qn01JHYAcFLTmIiGeOl5Yxa9V23li4lUW5e4mKCOPSnsncODCDvhkttDtsAGi1kogEtXXbD/HGN1t4b2kBh4+X0rVtHDcObMeY3ik6hXg9UjmISINw5HgpM5YX8vrCLazZdpDYqHAuPyuF8edk0CstQUsTdUzlICINinOO5Xn7mboojw9WFHKspIyubeO4/pwMxvROJaGJlibqgspBRBqsQ0UlfLCikCmLtrKq4CAxkWFc2jOF689J17aJM6RyEJFG4bv8A0z5diszlhVwpLiMrDbNGD8ggyv7ptI8Vns6+UvlICKNypHjpcxcUciUb/NYkbefqIgwRvdoy/gBGQzo0FJLE7WkchCRRmt14QGmLsrj/WUFHDpeSsfEpow/O4Or+qXpuIlTUDmISKN3tLiUv6/cxpRFW1m6dT9R4WFc1D2J6wdkMLBjKx2FXQOVg4iElHXbDzFl0VbeW5rPwaJS0ls2YVy/dK7ql0aqzun0TyoHEQlJRSUVR2G/vTiP+Rv3YAZDsxK5pn8aF2YnER0R2tebUDmISMjL23uUaUvyeWdxHoUHimgeG8nY3qlc0z+d7JR4r+N5QuUgIuJTVu6Yl7ObtxfnMXv1DorLyumRGs81/dMZ0yuVhNjQOcCuQZaDmXUEHgESnHNXn2xclYOInI79R4t5f1kBby3OZ+22g0RFhDGqe1uu6Z/O4E6NfyP2aZeDmcUAc4BoKk7x/Y5z7venGWIycBmw0znXo8pjo4CngHDgRefcHys99o7KQUTq26qCA0xbnMf7yws5cKyE1OZNuLpfGlf3SyO9ZazX8erFmZSDAU2dc4fNLBKYC9znnFtYaZw2wDHn3KFKwzKdczlVXus84DDwt8rlYGbhwHrgQiAf+BYY75xb43tc5SAiAVNUUsbsNTuYtjiPuTm7cQ6GZLbimv7pXNy9LTGRjWcj9mlf7MdVtMdh391I30/VRhkG3Glmo51zRWZ2K/ATYHSV15pjZu1rmMwAIMc5t8kXdiowBlhzqnxmdjlweWZm5qlGFRGplZjIcK7olcIVvVLI33eUd5cUMG1JHvdNXU5cdASXnpXMVf3S6N+u8Z7XqVZX+TazcDNbDuwEPnHOfVP5cefcNGAWMNXMbgBuAa7xI0cqkFfpfj6QamatzGwi0MfMHq7pic65mc652xISEvyYnIhI7aS1iOW+kVnMeXA4b/7iHC7snsQHKwoZN3EB5//5S576dAN5e496HbPO1eoyoc65MqC3mTUHpptZD+fcqirjPO77i/95oJNz7nANL3UiNVWvc87tAe7w43VEROpFWJgxOLM1gzNb84cxpXy0ajvvLsnnyU/X8+Sn6zmnQ0uu6pfG6J7JNItu+Fdg9us3cM7tN7MvgVHAj8rBzIYCPYDpwO+Bu/146XwgvdL9NKDQn2wiIoHSNDrinxuq8/cdZfrSAt5bVsBv3lnJ72esZlSPtlzZN5XBnVoT3kD3dqrNBulEoMRXDE2A2cBjzrkPK43TB5gCXApsBl4HNjnnflfD67UHPqyyQTqCig3SI4ACKjZIX++cW13bX0QbpEXES845lm7dz7tL85m5opBDRaUkJ8Qwtk8qV/VNI7NNM68j1uhM9lY6C3iVil1Mw4C3nXP/WWWcIcBB59x3vvuRwM+dc5OqjDcFOB9oDewAfu+ce8n32Gjgr77pTHbO/bc/v6DKQUSCRVFJGZ+u3cG7S/L5av0uyh30Sm/O1X1TubxXSlBdd6JBHgTnD5WDiASjnYeKmLGskHeX5vP99kNEhYcxolsbruybxrDOiURF1Gq/oHqjchAR8ZBzjtWFB3l3aT4fLC9kz5FimsdGcmnPZH7SJ5V+Hu0Wq3IQEQkSJWXlzN2wm+nLCpi9ZjtFJeWktWjC2N6pjO2TQmabuIBlUTmIiAShw8dLmb16O9OXFTAvZzflDnqkxjO2dypX9EqhTXxMvU5f5SAiEuR2Hipi5optvL+sgO8KDhBmMCSzNWN7p3Jxj7b1cvyEykFEpAHJ2XmYGcsLeH95AXl7jxETGcaF2W35SZ8UhmYlEhleNxuyVQ4iIg1QxfET+5i+rIC/r9zGvqMltGwaxWVnJTO2Typ90puf0YZslYOISANXXFrOnPW7mL68gE/X7OB4aTntWsXyzPi+9Ew7vfPLnfZZWUVEJDhERYQxMjuJkdlJHCoqYdaq7cxcuY2MerjWhMpBRKQBiouJZFz/dMb1Tz/1yKfB20PzREQkKKkcRESkGpWDiIhUo3IQEZFqVA4iIlKNykFERKpROYiISDUqBxERqabRnD7DzHYBW07z6a2B3XUYp64ol3+Uyz/K5b9gzXYmudo55xKrDmw05XAmzGxxTecW8Zpy+Ue5/KNc/gvWbPWRS6uVRESkGpWDiIhUo3Ko8ILXAU5AufyjXP5RLv8Fa7Y6z6VtDiIiUo2WHEREpBqVg4iIVBPy5WBmo8xsnZnlmNlDAZxuupl9YWZrzWy1md3nG/7vZlZgZst9P6MrPedhX851ZnZxPWbLNbPvfNNf7BvW0sw+MbMNvn9beJCrS6X5stzMDprZ/V7MMzObbGY7zWxVpWF+zyMz6+eb1zlm9rSdycWAT5zrT2b2vZmtNLPpZtbcN7y9mR2rNN8mBjiX3+9bgHK9VSlTrpkt9w0P5Pw60fdD4D5jzrmQ/QHCgY1ARyAKWAFkB2jayUBf3+04YD2QDfw78Osaxs/25YsGOvhyh9dTtlygdZVhjwMP+W4/BDwW6Fw1vHfbgXZezDPgPKAvsOpM5hGwCBgEGPARcEk95LoIiPDdfqxSrvaVx6vyOoHI5ff7FohcVR7/C/BvHsyvE30/BOwzFupLDgOAHOfcJudcMTAVGBOICTvntjnnlvpuHwLWAqknecoYYKpz7rhzbjOQQ0X+QBkDvOq7/Sow1uNcI4CNzrmTHRVfb9mcc3OAvTVMr9bzyMySgXjn3AJX8b/4b5WeU2e5nHOznXOlvrsLgbSTvUagcp2Ep/PrB76/sK8BppzsNeop14m+HwL2GQv1ckgF8irdz+fkX9D1wszaA32Ab3yD7vatAphcabExkFkdMNvMlpjZbb5hSc65bVDxwQXaeJCrsuv48X9ar+cZ+D+PUn23A5UP4BYq/nr8QQczW2ZmX5nZUN+wQOby530L9PwaCuxwzm2oNCzg86vK90PAPmOhXg41rXsL6L69ZtYMeBe43zl3EHge6AT0BrZRsVgLgc06xDnXF7gEuMvMzjvJuAGfh2YWBVwBTPMNCoZ5djInyhHQfGb2CFAKvOEbtA3IcM71AR4A3jSz+ADm8vd9C/T7OZ4f/wES8PlVw/fDCUc9QYbTzhbq5ZAPpFe6nwYUBmriZhZJxRv/hnPuPQDn3A7nXJlzrhyYxP+uBglYVudcoe/fncB0X4YdvkXUHxajdwY6VyWXAEudczt8OT2fZz7+zqN8fryKp97ymdlNwGXADb7VC/hWQezx3V5CxXrqzoHKdRrvWyDnVwRwJfBWpbwBnV81fT8QwM9YqJfDt0CWmXXw/TV6HfBBICbsW5/5ErDWOfdEpeHJlUb7CfDDXhQfANeZWbSZdQCyqNjQVNe5mppZ3A+3qdiYuco3/Zt8o90EzAhkrip+9Bed1/OsEr/mkW+1wCEzG+j7PPys0nPqjJmNAn4LXOGcO1ppeKKZhftud/Tl2hTAXH69b4HK5TMS+N45989VMoGcXyf6fiCQn7Ez2aLeGH6A0VTsCbAReCSA0z2XisW7lcBy389o4DXgO9/wD4DkSs95xJdzHWe4N8RJcnWkYq+HFcDqH+YJ0Ar4DNjg+7dlIHNVmlYssAdIqDQs4POMinLaBpRQ8dfZhNOZR0B/Kr4UNwLP4DtrQR3nyqFiffQPn7OJvnGv8r3HK4ClwOUBzuX3+xaIXL7hrwB3VBk3kPPrRN8PAfuM6fQZIiJSTaivVhIRkRqoHEREpBqVg4iIVKNyEBGRalQOIiJSjcpBRESqUTmIiEg1/x+0lhhYr6XEFgAAAABJRU5ErkJggg==\n", "text/plain": [ "
" ] }, "metadata": { "needs_background": "light" }, "output_type": "display_data" } ], "source": [ "import matplotlib.pyplot as plt\n", "\n", "plt.semilogy(total_loss)\n", "plt.show()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Part 2. The XOR Gate (gradient)" ] }, { "cell_type": "code", "execution_count": 17, "metadata": {}, "outputs": [ { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXAAAAD4CAYAAAD1jb0+AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAA2RklEQVR4nO19f2xlV53f55BkYqawMxlPgj3JhnEpyryIFoKtitahTWEheNYs00WDFpoqFKKo1ewqIy0socmAWhI2LeoqWzVtFQ2jBFKxJPsjBCsWpKFTmllgNQ5st/TOClVOogWbZDxJFmTMkJ3TP+679n3P9757fny/53zPe/cjWc++fvf8/J7v+Z7vr6O01mjRokWLFunhVbEb0KJFixYt3NAy8BYtWrRIFC0Db9GiRYtE0TLwFi1atEgULQNv0aJFi0RxccjK9u7erffv2xeyyhYtWJBl9f/rdMK1wxkbG9jAGC6M7YzdEnF4+WVgZQUoO+gpBUxOArt2xSkry5bOaq0v738elIHv37cPp7/4xZBVtmjBgvl5YGV1+/PJCWBhAIkvLgL33QesrgITE8CRI8DcnFmdPu/2vz87nuHvHu7gulumzQtgape0uufnexkukP994QJgy76oypqZUc9WPW9VKP1YXMxHfWYm/1xcjN2i0YXguThyBBgb6302NpY/r8PiInDX3Tnj18g/77rbrFs+7/a/fw0ynF0DTpzwH9LFReDuu3MGCuSfd1u0q65Mk2nnqLsox+Z5qLKq0DLwMrgoooU9hM/F3Bxw5x25xK2Qf955x2Dp7777gI2N3mcbG/nzJvi8W37/AHLdzxl08K3z08bvc7WrHzbTTl13gYkJu+ehyqpCy8DL4KKIFvZIYC7m5oCFBeD06fyz6ejuI435SnK7VzMcQIYz6OAMOnga01bvc7WrHzbTziXd1p2urr/e/kDoclKzQcvAy+A+7whWCbDBtc/ccxEBPtKYlySXZdi1C5uMu2Dexu9ztasCNtPOJd3OzQF33LFVzsRETroLC/YHwqqy7mg4qdmgZeBlcJ53JKoEuDcUnz5znz0jwEcac3o3y7D8eIYMHVz3wQ7+71ivwZJCEqSWMG2m3bZuG3LvP1099ZT7gdD2pGYDGQycg5G4lMl53pGmEgixofj0mfvsGQEuenPfdzemOljvTOO6W6ZZJEFqCdNm2m3q9iV3qQdCFTIb4cy11+ptboTFyJYX+tiYHxX4lMnlEzUzU/+/06fty/Nt5/x8/bl0YcG+PVXw7XNM/zQDCG8ekOXS93rH31UwJLhcA33InXq52PZxZkYtaa23LaigfuCVGCSluc6aT5lzczyrcGLCTpE3CP0bVCFOAOZtDyFS+PaZay4IULjlFVNQuPUBQpqcZVheBjBV/W/Jmw/HtPuS+5Ej1TKhy4GQYvkWiK9C4WAkEs87lCoBCnVMCB3zEKpBCkjTiG0iyzaZd6E+6YeLOiF1+7svuVOqiihpJz4D52AkEg1glBRAsUGFYK7cJviIcJ2CEIwwQwfnDt5UqzqxZSAS7e+2cA28Ks8VQGOMpJQv4zNwDkYiVfKjMkdTbFChmKtPnwWLfS5T4BtNSQVbBuIrMVJOo2tZtuTOuWlRypfxGTgHI6EsUyITodqgOP2bfCFc7HOZAna1S6H3boAtA/GRGCmn0bcsG3LnnCtK+TK+ERPgsVpQlElpbaBEUbdUKxQFOIzbhHCZAjbTTDc14iC9dxm2BjkfWzTlNIYkCW4z2qWXbvVl1y7gYx9z64MMBi4VkpmIYA8NEkg0RPfBdgomJqozGHqZZrrMO0MHmIKRy6Dt5uPjgZFSYqgyKJ3GyqjycP75z93Li69CkYwEmMjQQqIh2hNUR+eyVu+2o8DiM3kCcht/bxt1go9GMqXEUGVwmdGoVTPDx8ApddZDyESSgVRDtAd8IjEL9BtCz67lf586xdXqHDYMv7wEf/Yz4OK+c77rNIYkCZdNy4T1UMuEw6VCodZZU3rvl9tIpbsOFY0RI+pjSPX8vpqvsgRXpIbd2ACOPTqNhVsIGuiJ/iX48ss5A9+1K//d9xKL+fk8L0mICzFs5sqU9VCrZuKH0lOCIzycmuFSpQ3gSEEQs54WRpiZyS9kAPLsggA2swu6ZGSgBuUSTGm5mPbbtR11ofQyVChUag8OnTWlqx2lAixUKKDYkMPRRCGp9aeGlaLVo1yCKS0X035Te03HZ+CUjqLSddYpmuQlG3Il+ugz48gR4NIdvc8kmQYol2BKy8Wm35QyYXwGTrk1Sjd8pWiSl7opCg/04cLc/gzvfS+wdzz/m8q4RgXKJZjSchnUb87xj8/AKbdG6bk3KKk71GYldVMcNdVOKUnVgUMdfPZr08YeIS77XKiQ9UFIabnU9RvglTPiGzFD5KWmAJUxc5i9UEJ6q1DnV3cARXeNyigF69jm9nZZXpLs1ikulzJcx7+/nceOVRsx4zNwSdRShxTaGBuhxyjyxt+fDxzIu2vj121cRpYBnQ6yzC5YB3Db52yH1oUxSs5HTgnb8a9bRhsbly9r/cLf7v9+fBUKt9qDQgE1asd1F4Qeo8iqHYru2pTRFcKt4aL7tdFquuYWHxXzhe3419EEsO/Kqu/HZ+AAX1Y8KkqR7IlhCy6LSugximzvoOiuURmlDIMuV6O57HM2TMdlIxslech2/Ovp55IdVU9lMHAuUFGKVE8MW6SS5NgUEdPhUnR3YBmlG+VNMgzWwWWfs2E6LhvZoHd85IsQ3ja2ddiOfz39/OJ81dPhZuBUUqFUTwxbpJLkOAFQdLeqjDfvyPDJQ71XovleSmy7z9kwHZeNrO5/u3a5yxch1DKuddiMfx1dAT/6YdX3h5uBU0mF0t0TTcGp5hiWMTIERWKq/jKuH899vPfMbpe6TSQ/SgnUlOm4bGT1TMpdvghxa1AI1U+9O+LZc1Xfj++FwonWe6QXKbhsjop7QhWyrNJV0ISMY5I6lRfKsWP132/yDPXxKjUdO5s6qMm4LhfKcGUj7MeQZrRzBkd2RUpIvQEpMkzuFYl594hLhsWqd4pl2g+TA3OIW4NM66Ag4/4NANi7p+p7w61CAaIauowQMs5ZuppjlNwT+jHgPksTzdcwOEr52BV83jUdO9M6KNQ5/bp24OrXV313uCVw6YghcUq+im0YuJAtDK5EM5H8uK4ACwmfA7PPu6ZjZ1qHLxlXbQCAqhS25UvgUjPOSQgQ4h6b0GM/LO6aA9BzHdqNGU6d2gqRr/M2MZH8bCVQqcvK58Ds+q7N2JnU4UvGNvKKbAYuNWRLQoAQ99jEGPshd0Wsug7tdx7oNF6HZqL5stGOSV1WsUCtWfQlYxt5RbYXilSvCap2+ZTDPTaxxn6IvVDm53tvpT+ADGfQwd5x4LNf8/P1tm0H9dQO8bQ5wWc8qrxigOkLWi9d1P9d2TpwqTpRygAhV68Q7rGJNfaSdfSeKIauuMuywNfXpvHZCO0wfd6E1nloO3zIuErXvrr63LNV3zVWoSilLlJKfVcptdD9e49S6gml1A+6n5e5NXcApOpEJQQIcY+NS/mhFatSFbk1mJjYYt5n0Nm8Fi00OVOTTkjnIdspT4xENtGva68L5LHRgd8G9IgOtwN4Umv9RgBPdv+mBbVOlGo2KdsVwvLiAherWEjFaoKK3OI6tPJ9lk1TxsGAqEkn1GHNdsoTJBFrGDFwpdRVAH4VwPHS4/cBeLD7+4MADpG2DKC1LlDOpgR/au422JYf2oc7QZ/xuTngox8xvw6NiwFRk06og7LtlMckkVCSv5ERUyn1hwB+F8BrAXxMaz2vlHpJa7279J0Xtdbb1ChKqVsB3AoAV09MTD8by/go1SA6LAh9Q46AG3mskfUmqWpCKiQbKozfdspdSITqliXq8agLpW+UwJVS8wCe11ovuVSstb5faz2jtZ65/DJ6NbkxpBpEORFSARjaXiHVPlIFx9SwsUnWlHxcJHoX0rSdctvnVCeekJK/iQplFsCvKaWeAfAHAN6hlHoIwI+VUpMA0P18nr55hEhpwVMgtAIwtA93Kj7jmXtqWBcGRLVf25KPjSnHtOz+/lx/vd2U25IIFeMNufE2MnCt9Se11ldprfcD+A0A39Ba3wTgMQA3d792M4Cv0DePECkseMoVGFoBGNouIMEOMQgEFzLYkCz1fs1JPiZlV/VnYSFfFqZTbksiVIw3pKzo4wd+D4CHlVIfBfAcgMNeLeGOBJCemZDamTbG+Tu0D3dkn/FakrXUddfBhmSpsxFyko9J2XX9eeopO/2/DYlQ5ZMJmfTTioFrrU8CONn9fQ3AO0laESoSQHKQCPUKHIbsRoJRhMUXU7aymv8NAHP7/Zl3AVOSpWa4nORjUnYM+YOK8YaUFWXkQknQJWwbfNUf1BSbgsooYUgjWepjOyf5mJQdw2RFqZULlcVaBgOPbW73BYUCkppipeuIE0cVaR5Aht2r9Xm9y6B2EKJmuJzkY1J2LPlD+vUB/ZCRzCoVh9c6ULS/vf4tKVQlpgKAs+OdxsRUXFM9bAmlXPszTOOw1ZcZaH1a9f9fhgQeK5kxVTkUJ4hWYk4KVSS7vKODtx9t1ntzqV9Skx6b4NIfKm8capdMl7L6+1IFGdkIbbT+VAZPSsMplcVHspG1RQ/6SXbvOPBPDgPXRTA4ttgChS8AJWvwKav6Zp5eyJDAAfPtlkp8oRSDQirspKdXk94+QszNAQufy3D6ixmO3gbMzpq9N2oxZbbwISGKzZGSNfiUZdJmOQzcFFTiC6UYFEr9IT29GkX7bFZv7M2i6+9tG6wzCg5CVGoDWxKi2BwpWYNPWSZtTo+BU4kvHF4f3ApIab5r/eC4jrtu9cbezLqXEbuEyKdg7vDZG32mxpeEKDZHStbgU1ZVX/qRHgOnEl9CiUGUUiKn8pSinb7ts1m9EjazTsf5VckGR9+9kUNtYEpCFJujKWswWTI+bKa/L1WQYcS0AVWYU4hwKeoIU67wOKp2+rbPZvUKsARmWfN3UoSvIdBXbeBL4r6+ACaswXTJ+LKZoi8zM0uV2WDTk8ABOvGFWwyilhK5Tg1U7eS6jrvqeSxLYClJFQCScPlBiOEx67s3UqsNYtgHmliDzZLZfj0a3aE8TQaeCqilRC7lKVU7fdtns3pjrHSP1LAuoPRptinHd280mZq6DSUF+wDgvmSoTTfpqVBSAofKg8NXnLKdPu2zOW9Gyi5pm6TKJyqQKr+ZbTm+SZ2apqZJ/eBL4iEiMV2XDHXOOjkMfJjiXwuEzCvpA0nttFm9wgOffE0LsTxmKfbGQVPjwsRM2UOoxKauS4b6UC5DhRLbJYwLPudBH0da2/dSObfGQpZtqk9s4GtaiOkxy2kesmViNuzBdcxtl43rkqE23bTJrAYh1qnANdtRmxCLHh6XM/jeu0w1ndLIwna523zf9SLjUOPjWpfzpcZBIMAlbBtingpcxQgJvtHDgpLU7Xo5g6+0RXUwknbAsrU/27AHlzEPuWyo50KGDlzi7THU1gYbuG5oEjfCFFFm3AfdPU0oTAtUan5J5gJbHbsNe3AZ89DLhnIuZEjgLi5h3HkwYjJDV9FNcpak2HlLTJFlQMcst0lTl6RJvj6gnj4bHbsNe3AZc8nLpgkyJHDbLTmEqTnmqcBVdJPkTVJGKNeAgLCJxEu0i5uIPX227MF2zKUuGxPIMGLaIoTRM7blZ5iuI+GaL+q+dmPjTaIsQ9rdY08pZV99+8I1FrHHuAl1RkwZErgtQqg3IgWK9NTvUpdEkY9jvqjFwpLeG2gOkQ+lYYst/RZ12jyvg29fOMdC4rIxgQwduC1CKa2onWFT0QObwrQ/HPNF6TrgECIfigQlOBZR9dW3LxLGggKUbCBNBu6bByMGIx22YCWb/nDkLSEWgW1dBV27ZEt6EhyLqKbPty8SxsIX1GwgTQbuG+EYg5EOi/hQwDYdG7U7RmTXAZcuuZAeVTd9ZBaq6fPtS4gp55btqNlAmkZMH8SK+vQNy7MFt1UmdH/6QWFktjBaUsCF9Ci6GdseT9UO7n6EGCfXZSM7EjMkYp3DQkqMIU4ZsZ1nfcXC0n2WIVLDAm6kRyH9Ukh9FJKpb1+4/epDHJKpl02aXig+8PXvdpVsQzqbhogileA86+o6UATrgF/qLsOV9Hw9JHxlFkrvD9++cHqLhJDtqJfN6EngPhYZH8k2ZFheKDdLjv4Mm6dOCbFum/GV+obNfFOHEIfK4cyFEhI+/t2+km0oZ9NQUaTU/Qnh9FzovSPcZxkrtMBX6gutdYwVVBPqUEm5bEaPgQPuI5iKH5MPJcYMSeNW/VgG63AgRsCI78YRMqsE9R5uQ84u4xQ7glMOA489EiaQmDWxCq4rNnbYH+cG6ZkatgrSSHZQe3w2jlTNNy7kbDNOsZcLIEUHnkqQSyglJpXJ3zaKNLayk1kJSc28JZEsZ3tSNd9wk3OI5bIzW8LObKn2/zIYeGzGYYoQlByTM8RWEdlukBENntJIlrs91Fkl6kC5h3OTM3f5O7MljC1n6KDeYCNDhRKbcdiAW4kZ8yIJThWRib7BRvVjen7t03tTQRrJSmtPGTaqJkp1DbfGk7L8Kil7bDnD1BSATj3typDAXbbdYXU3i7kSuVRENqcKU1HPROR0SFJlithxTKb1xjbP2B4oKQ+53BpPivILFUkhaZd/pg52BjJvwEACV0qNAfgmgEu73/9DrfWnlVJ7AHwZwH4AzwD4gNb6RfOml2C77UqwHnCBYlt3ta5x+blxnCoGbXRlqXuKx9sklGHPdColxFVVwWXqqQ653G6btuUPlLIPup0QG3OhKKUUgL+ltf6pUuoSAE8BuA3ArwM4p7W+Ryl1O4DLtNafGFTWwFwoNkxnmG+xl54wwgUceVPqaGB8HMu33UtqsKwDtxeK7VRK84oB4qfMkYJCyp6aqvhng5QNAGpmpjIXilUyK6XUTuQM/F8B+AKAG7TWK0qpSQAntdbXDHqfLJkVFVVQUjwl4/Rpl8TNjaNNVeO9Ywdw7Biy/XNRfLypIXEqbTEMfbCFqz57EOoYuJEOXCl1kVLqewCeB/CE1vo7AF6ntV4BgO7nFTXv3qqUOq2UOv3Ci24alm2gUPhRe3tQugH4mPwlWrM4lJFVytKPfCS+yEkI6qmMYTbimHqp5i9ffbYLjLxQtNZ/A+AtSqndAP5EKfUm0wq01vcDuB/IJXCXRm4DhcKPWi8rhXFKDDbiUkaWlaURQ+S5QDmVscxG1FMv1fzVoyJx1Ge7wMqNUGv9klLqJID3APixUmqypEJ5nqOBlaCgCmqGK4VxhrZmmap7ON0vGaIsuRDLpS6mdyrl1MfsR4GCWffDR0XiChMvlMsB/KLLvF8N4FcA/DsAjwG4GcA93c+vcDZ0G3ypgprhUjNOaZ4kdW2MLQ5lWX4hA5O3CSVsh4tyKqUcEH0Rsx+FbttXn00JEwl8EsCDSqmLkOvMH9ZaLyilvgXgYaXURwE8B+AwYzvpQc1wKVebL2MMlTGJWhzyMN5KZ95AXJe6WGnwqd4vt5dC7mpqT5Uh8syjGU6eBE6tdbyXN5V81cjAtdb/G8B1Fc/XALzTrVoB4JBUqVabhHOiCSjFIZdN6/hx4JFHgLXLgYk3yPCbG4DYMVo+CSp95AnKgxqF3NXUniqXv1OngK9+Ffjz87nUvbIK3OXQh8XF/L2ibtdyCozenZgpIBXnWUofMduyjh/H8vH/jo1XchlkHTu7bpt3imXisV3qXCU/33ZT99tXgi23563YkrT3jgP33lutIpmfz5ltPyYt++BaTp0boYxcKC16IcUg2gRKNZSNeJplWP7St7HxysU54y4g8ZRSQuxoSdcDou/Jgfrk4XvQXV3dYtwHSomi1Bry660rvEio+kA9FjJyobToRay7t2xBmbjC0rd/4+WNXuZdILBVzsYn2We4uH2fB5XvG3Zh8z5XPwsf7Z3ZEt49voQDyDaZ9xl0cAYdvDRR76tNlWuGOmdNy8AlImQCZl9Q5Rm13bTG91Y/D3hKcYkFcxku7gzDTeX7yhOm73P0c2e2hD2PP9QTVPOpwxku3bHFuE36QyVTUctmcnTgEhM5pIJhGbumfpSTVJ06hfUHvlyRuiCcDjyUTpu7HpPyQ3ihuPRz0GUHQL3Ln0t/qJaZSzkkuVB8UcvAQydgGhaGB8hMXsWBqmCdyPMYytbMXY8Um7lNO+qSQ506BTz8CLC2BoyPAx84DMzeEt9f2xeyjZgh3eYkBJ9QIhWXQ1cMSg0byt+9BqFszSldTFCG7f5q2o4e5l2SrBcXgbseKC2HNeD0A8CdVw7HUqiCDB14SAdZaXdh+WJYQuz6kWWsFzJQIJStOYWLCfrhos+uasc/2LGE/zz7EPY8vvXTQbVaZNiWtglkSOAh3eaGjeGl4nLoAOkh8qGyFki7mMAErlGnxbv7Vpfwtl0ZPvRBYHYWRmHrw7a0TSCDgbs6yLroQEPF4vogVMYjqgsoOCw7hw4Bs/KTU4XS4nDXQ11+EzOtMz6+fz/w/s+55RuZmKgOkhkCWaYWMhi4iwjgqssOEYvrg1AZjyj6QDUOVeWcOAGgAwhn4C2qUScnvXt8CTsz1N9OU8AhWdSRI71h6oDM8AlKyPBCcYGPbxVlLK5t3THLpq6Hqq195SxjPzYwhvWJN5iVM0xeRYmjkKxPncr34J+f3/rfpTuAj783wz89BLZMfqmSQlO7ZXuhuMBH4UURi+tad8yyqethiC/OcABAN7eJSTmLi8Ddd5Wk95X8byDqyg3FSKQwrP5Uq53ZPCx9m0tfhT6bsg/F0i7KPHYs/5TMyH0SXIVl4BsbuXcBxe4b03jHWXdKvmlUbR0fx/Laa3Opuxweb1KOQDfKUJ6qIT1iv3t8CY88Apxdy5M+HS6YcRebKpGS6mO2A8ze0twHyux8HGVyB/34kHBQN8INjGF5GTT3XsXMF8JZd0q+ab5lZBmWH8+QHf4UNnb8Ui/zNi1HoOtBKHe2UPV89/gSTh3PsHctzx+ydy3Dn57IcO5U6c7HgnFbCmccfaAss9gMVlYBja3NYJA7pO07PiQcVAK/MLYzD8gAAQMPefNMyLpT8k0jKGNjqoP1g9PAlW90K2diIlebVD2PhJS0YCah6H/xCHD+FWzmDQEAnAd+91FgoUHCbgLHWFGW6SId277j4z2Trg4ciBuJx1l3Sr5pVG11LefIkV4dOBDd9SAVLVhdOHoPpvIbaKpcHSg2JA7XP8oyXTYD23d8vGeiRGKSqVFapItulKU35ubyBFYTkwBU/hn5UgfpWrAirWqPr/WAH+oUqBR9CFWmS99t35mbA+68I7/UQSH/vNMwlVFwCbwIzMgAdAomLuBy0BaB0J1z0lvkI+dE6YckLVjZO6SMqSlUXlxQBU7/ao6xoizTpe8u77iScFA/8GuvndFf/OJWWrG6pDQtEsYg83tVRkHfMlvUgnJ9jfIUDOx7zT9d3hkEEelk+xk4sN1/tGXkCaMute2HPwzMzroz7yodt+C7L2OgyhjZrilmuKRydkz/XMfAo2cjXH/meax//NPY+NTdWP7N/5DfNE6BmHdQjVK5ZdSY35e/9G1k6ODcwZvsVSajmGLOAmV9dvnWmQ4yTB20d+sbKfiuCRfaJKbnuF4oJelqHTuBl9eRnfhTjL3wGkwderM78XFHOHCVn1q5/agws2c4ALy84a7rFujnHQsDpWxDfTYbuHQsnOX6rokQLioNiCuB9+1G69iJ9fMXY+Pkt7c8VVy8VbilNq7yDcp1EhoM2+stpJfM7MvYjwwH8jmdeINlQdVlGj13RYgTikc9tVK2q4qEsr9cl3ZyXgZKsYZDuKg0ICgDz7I+WqnZddbXfoaNqZxE/+RR4LYbMzs6K5W7iBsxj8cwgz/D/Or9NOuyot2LuBHzq/f7rYeG3dklKsykXK+yyzh0CMsX/x0cx7/Av8R/wbX4PuaxgMXrP2NRSB9C+OQVJ8HVFQB6K58Kh9rNoJ7yDepGLn9O7SBkjBEFGmdQSMIutElMz8FVKD0nlQFRdOudaSwuAo9/FZg6n+EaZMAqcNfdOcEOPOV0IxwWcSPuwp3YwKsBACvY551noVx+gW31rDrmXmiIzHDOmWAQ8eGdUiTLsLxvFk/+4+vxW0/++uZYrGISdy9MAm92HPMQPnmh8qk01LPNoF8GpYqEur9cai5O9RlFtJULbRLTcxQVyuYm2rAb3Xcf8K3z0ziDzmYY7/6NrHkD7pZ7H45sMpJtdfugr91k9TSMhzM9G+z6zmUX+UzQwcZUB8e+/8/ox3xuLk8re/p0/umakahOZRAh9n1nrlzKf1b/X4+KpFLSZmqH0fMmcKm5UoggcqFNCnruIpoRc3UVjbtRQU9PY8sAdgAZdq9m3UigGsIu3j9WPdHe67Kv3asgqqdhPJxDhA12feuyawJyRNocm1LOhsqn0q1nJ9Yxhg1M4Zn8+fg4gMzpEgP3dnhKn2VQXJISslwgbi4lQgT1A1dqRgO5H7hJzv+6+wLePb6E/3pbTQ6H0gKYn69mSpMGddsgVD39aTKBnJ5Nw269yu4zJm/eFI/eOytD3Udhhfn5GgY9mTcqkK/5zuP/ETjxeYyd/+st5m3gA0wOR1/kxjJT8kJJDKIudDDdROs24LcfncbG/u05DceWM0xhK994qCuWQtUTJQni/gzLj28x603UXDbMKTQ5o+lYwDCwlSHs+/Zg6iP/EHjkEWANbvVQMDSu+HUOxiosTYI0BJfAJyZOW9GKDb1uy67W6YzczShk8MhZIm4smiRwSiwuYue9n8XY2l9havwn+c0Ht3jmXC2V7S05i5ucRBF4HMWG0nOgzbHiiVLOEqBa0k4KjCqSnuCaU6d4VSS++ikO1ckownUcPZi+CAZ+5ZUz+sKF00E2rYG5jlumvoWKQCnSTIEDEFSIIa6s0uXv6NH8Ash+UBkAZrat3y2cNhCMYhoohknydxlHz81ThA58ZQUo9gvO+/uAXGpc70xjoy/8eFNPDow2I+9TkfSgRr9dBdd1GfI+x81CCRh2gao7ICuZN0DnguPrPRLLRSjkZIfYKFzGkSnWIKgfeL+wT+GT3RQRXDDy4ufcwZuQoVN7qQR1hHGI6GzrOrsqksJ3u3+MbJi3a0BfKjmqdmZL2PP4Q/Uh7GUhgDvs39d3OVRagn6EmmzO0PsyXMaRafOMfqWaT/tdN/aCQS2XpXHkKsxHTgAr5/NF6XObNcdt26Z17t/IsAsAVoFHPgP80g97bxCvU5HYCi8+QoVIf/ESnNIcc7vg+HqPxHIRCjXZoSJqXcaR6Z69Rh24UuqXAXwBwASACwDu11r/vlJqD4AvA9gP4BkAH9Bavzi4rC0/8AI+6jcKlV75aHz0KLB3bYuhF9GfLv7coXzDy7jtxgxn1/ounwWwdxy4997e71Yxb1sVnY9KVqS/eBdV3kzGkK7rjdG+UJPtayOwge04RtSBvwLgt7XWTyulXgtgSSn1BIAPA3hSa32PUup2ALcD+MTARqheNYrv5k+xsZcZ2dfXAGAab8USDiDDga50rlax3em8AbtXkUvBfXApyxQF8y5HrgIA1oD1Bh7kIrz4CBVS/MXJL0KQ7rcco32hJjvUbdKA/TgyBXE0MnCt9QqAle7vP1FKZQCuBPA+ADd0v/YggJNoYOCTk8CFC3Ttp56vorx+Brh33J7nvjCeM9R+uJRlirPjwNNr2/XXJuPhshn6rMvYkcwDE0f1M2/pUnUVqNvsU16oyZYiFdSBYfO00oErpfYDuA7AdwC8rsvcobVeUUpdUfPOrQBuBYCJiatJT0zU81Uur2DiY2PAHUebJdh+vP1ozYnJoSybOr/pOB4um6HvuowlrPaoSJok7eDuMgSgbjNFeSEmO7ZUEAHGfuBKqdcA+J8A7tZa/7FS6iWt9e7S/1/UWl82qAwOP3BJggZFWb71U7n1AcMX4+FkmPTR3/pMps+71DrnkAaLFE87AeAVyKOUugTAAoCvaa1/r/vsLwHc0JW+JwGc1FpfM6icV71qRmu9ZUwwZRCjMqexmajpOKc4H86GSVfDmM9k+hICtTEvlHEw9gIQDOdLjZVSCsDnAWQF8+7iMQA3d3+/GcBXmspy8QMP5dopAbF9o03SFKcwH0Y32pjC1XfaZzJ9CYHa3zuU/3joBRA6UIOhPpNAnlkA/xzAO5RS3+v+HARwD4B3KaV+AOBd3b+t0eQxEpuphYR032hA9nyw3NDuGjzjM5m+hEB9DV2Ia+2AsAsgtCTCVJ+JF8pTAFTNv9/pVTuaN3GOOZWqAgjpBeUKn/ngHHcrw6QNXA1jPpPpSwjUxrxQxsGQCyBU0A9zfUEjMV38wKnnlNupwIdJSfeCAtzng2vcjQ2Tvm5wto30mUwKQqD2+gjhRRJyAYQ+7jLVFzQXyuTk1kKfmDCzTVCf3jhVAL6npLm5fExsx6iuLRzqPdf54Bj3nnskB6lIYijufSaTkhBiw4YQQ/Y7dF4YpvqSyAdOefTmNKhLCQ/nTlfsMh+U427tDihlYkYNkr1KQrdtGNLJuoLy9MapZpNihHRRt9moOFzmg2rce3TdBw313FImJmW47Nqh9cw27Qwd9BMrlH7YwKlmk2KEDJWu2GZN+467l5FSysSkClcDRuiN07adoUOBGeoLqgOXAE41m41+2FdHPej9EOmKbdXKFOPu5McNhHODG1a4GjBC65lj+LjGSPpfwshJ4EDzRuiqczc9Jfl6ZDS9HyJdsYvEHi1RX+o5MmL7vbpK0qHdqqRL/AwYijsxqfOXcNs2fG1qJu9zpysOmXo5+iXVPglmfAlTgiEwVj6YkO0UXp+IS41dc6EMAjV9h5gTX+bHxTxt1lqIcXJKPkUNH5ceCsKU4EEjYRMxQeh2+uTJsdzUnHOhUILjTkxqtVeIU5ivapBLtWiSC6UAt1rZ2MebG64ERkWYEjxoUvFLD91Ol4VIHJMQ3YjpS4fU9B3C7uLL/CTY5LjWSmXyKU40GaFcCYyKMEMbAutgs7vHRMh2uixEYokzOgP3pUNq+g7BHH2ZXyoCkS2CS90m0pArgVERJhVBRvaWGEq4LERiiVN8LpQmUBu6Qzks+HpkRPPo6MLVAN+k/guq6zZxpXElMCrCpCBIAd4SzojtgdME24VIHJPQeqG0cIKLba3JxrQzW0IHWTgGbmqEiumFQgEJhlAXpGI8tYFjn0R4objmQhklhFrzvvW4GOAH8ZFvfC6Cq2CqjM0WIX0+KTGs80PohTKSgTxSEeqkS1GPy0mw6vtvxRIOrGboALR5vE2QQv5eCsRKJeArJUjwwOEAof4zuhHTBNz2F+ryXcsLFQlMUY+Lba2OX+wdh1uIvC+G1RrcD5fJosj14OsuR+2hMISGXPEM3JYObOeIOlW0T3mhBA6Kelx4XxUfuXQH8IHD5vWSIxX3OB/YThbFooglJdSBok8CNwDxOnAbNZiLfYBazeZTXiiVX0zVYv+p+jOHlnDLbEDDpQt8VAG+aoQYhlAKAqHSu1P137dPkQ2qIiIxXWAjLbps+tRSr095oQJ0bOuhFDz6Bd7ZWfeygsBHcvOV+mLcJFTUY/O8ClTqD6oTkm+fhN7mLZ6B29CByxxRq9l8ygulkrWpJxYPEQOfheu76GMxDYpFISFcuAzfPlFLekRSkXgGbkMHLnNETWe+5YVSyZrWI1TwcIPLovFZuL6LPpYXBsWikGYg9u0TpaRHKBUFZ+C2a8iGDlzmiJrOpNGtLzh5SJFtMAhcF43PwpWatawJVETMIY24Sq6+faKU9AilouCRmOfOnWa1A0gJfhsWcBk8g+f4du2Ij/HK1/A1jJGIPog9HlTMxcHAKyIS85JLZvQrr2xvYOqBVcMM6jUTLce3j1fEqHmhSMWwRGY69ENEJOYrr1Q/Tz2wapjBkdxrk2eHdB30iUb0iZxLPWuZJAxLZCZhBHBQHfjFNdtF6NTGgyDQVx+Ae7so+mOqypQ6dgDkeUWkBgmTKyU3ui8IDWVBJfArrgDOnZObeoIjFwnHtYg2qVtDZREVn7E0VJ5gqfBVA0mY3GHKXUN0sgoeifmhD50Wu4aoVWyxr0UMqTI0rWtntpRrTjLh0ZfDBF9ClKR7HlGbgAgdOCBbpUetYjO5L4CzXSFVhiZ1bboNZhl9A7gwDAzDlxAl6Z4lM5AIEB/IExLUKjYquo99q5cJmuoqX5cWJfOgC6gCLnz0xxS6Z19CTFX3HFpvH8FO0DLwEqjtXLGvRQxptxtUV3CfbypQBFzEzKVSwJcQUzQAh84BESnnxNAxcJ9NkDqKkoruXdvlkkWUa+ySY94AzREqZi6VAhT5HVILLw6dAyJSzomhupGHwlhep2JzUYVSOj64qv5M3+Mcu2RBcZNNzFwqBSgI0WVyY9oPQuvtI9kJxDNwGxqgMhpWtcGVuaXC1LjGLmlQuK35bAKUV6GFJsTYroehr5GLdG2daBWKrVqJaxMcqox8NQgtQEiIC2kEherAR32Rou65QOxFE3rsIs1VowSulDoBYB7A81rrN3Wf7QHwZQD7ATwD4ANa6xepG2crFXJtgpK8qLjAMXblvCeY2nq+uAjcVRLOVlbzvwGB0j5FKDzgpkpIOfgo9qIJPXaR5qoxkEcp9Y8A/BTAF0oM/N8DOKe1vkcpdTuAy7TWn2iqzPZKNdv8Q1zJyiTFMXCBI2lVnefJ/HzOtPsxOUTjOfIYhUUTEM5XqmmtvwngXN/j9wF4sPv7gwAO+TawCrbeT1zG8pRPsqbgGLs6z5PYwlmLABiFRSMArkbM12mtVwBAa72ilLqi7otKqVsB3AoAu3Zdvbkxm5wwXGxIHLYaqSdZaiN/KDvXxES1BC49LqSFBbgWTUjPlgSicI1yoSil9gNYKKlQXtJa7y79/0Wt9WVN5bzqVTNa6y3dh8kRPYExjILYue0HoSlwp18HDuRtv1NA21sIRkii567LkrF5XehQwcD/EsANXel7EsBJrfU1zeXMaKBXed2qxNwgUcXYw7iBgYE77cbcwhoSs7O5wGFzoE5m9RiAmwHc0/38imM5rd7TEZL0yC637KTiH99CEKRlZ3MFYdBFoxFTKfUlAN8CcI1S6q+UUh9FzrjfpZT6AYB3df92Qki9J4XvsRT/Ze78Qqb9LCepSjJcPgVIIbrYkJSdzQeEm4OJF8oHtdaTWutLtNZXaa0/r7Ve01q/U2v9xu5nv5dKJZTq/TukUZoi10ykfDWVqDLyA8D11/uXbdrP7x5fwn/6zQzHPgXMf7yDxWcSYt6pMEVqokul31WQkp3NF4SbQ9BIzMlJvnw4TXRJERhGHVzmmzxqfn7784UF/zVp0s/FReDECeCll4EMnc1gnCT4gS9TDMkEKYlOkgTigpBJtTjrItwcgt/IYxPIYwoTm4DPpeQFKMooQGHk5rKzmPRzfh7Yt7qEA8hwBluSdxLBOD4DZzpxVFZaSqKTaPkeBrjMNZEXiuhcKKYwEVIoTi2UajEKwWqQKs1HQDS5nOEdqw/hALbfrJOEUdpHB2l6PKGSdCmJjtMwx3Uqka7ycZ1r05vCGzAUDNyELilOLZRqMYq1NGgN+/AOk8sZ9o4DZ9Dpkb6b2iQGPkzRZOIo1R6URMdlmONSzaSg8omctGsoGLgJXVKotCjVYhRrqc6Q2Q9beqrr5/v35xcST00B7znaSTdS2ocpmkwcpaRLSXRchjkuJsZRLrVEH9mfV3w+cBOYhtxT+B5T+S9TpJquilamvANgWz+z3v/3151MMI5P400mjjq1IxXRcU0aFxOjLpcjR3mkPOAFhoKBp8hMqNrcv7YH2akokGXIlSZZhrm5jugxHgifK46AwRNnujvHCEfliKDiYmLU5XLcWkIhiXkgGQbeROspRvZxtJmTntY70wByJj62nGGquGHeAUmH0jdNnAmTj31jDSW4iI66XI6TQmTpMQk3QsmJmyQiBHN0CZ8vt2/kk1kNm0ufDdFxfbcJCY+5VzIrKrgycI5x52RySUuXlmjKPFiF9kIH0AcVpEJwMaWxhCVB6mRWQZGCLSNE2ZKwxTOm8e5x4PBh4J2GjFxSIq5ooNLvpkZwMW/PTtFY1oCgboQvv+zmwUPtvsrpuhn7LtcQ6HfP/fraNH7rgWk8+aMOlpeRK8mz7UE+BULmJLJCyKARKpc+boIbMrc7pwAayjEgHs+gDHxlxc0nn9p9lZOGYtNnCNTxjGOPTmNjqoMMJUZeAZG3bYUOGqHy7+aOrqQeE7G7dw0ox4BhPIMy8H51u6mgQJ1XhpOGUqNPFwziGeudaax3ckZeJ43PzeUGy8kJQCH/jG7AjHF0ogin5iQ4jjHh2L05T06UY8AwntEjMU0FBaLUAQB4JUCfsmOkfXCp04RnFEy8ThqnnE8SpHp04iRmLrc7SmmM++REOQYM4xndiBlDMuW0ZbiWHcMW5VqnqXtu4TcOAMuF3zgg89KHkBF1lF4jnMTMNSaUARDcRlHKMWAYz6BuhC6XGo8KYrio+mZVteUZLi6HwWDjYubDgFNyZUuhrZTumFWgHAOPskS4EU5OAhcuyPTgie1KG+ME71OnixBVSOQipXFTSdb3qBTTjc4WKbjdcZ+cKMeAYTyTCOThRihBY9AmkZoE7gvR0vgg+A4at8Q4akjhlECApC90sDW02X4/hANCk60lhmtdTHe+spGzyW9cFHyPSqPgphQS1EZRCgT0RhDPwG2NzC5G6RDqi6ZNIgYdVtU5P5+3yYf2TOm3cDls8htngesi82XAIp3gE4ckl6bA8QTiVSi2J1aXE24IVQLXyZlSd09xGnUtwyc5ljV8Oko1SLH1yhLaMIxgYibJqlBspWMXaTqEUMRxcqbe7ClUSaZl9AvAf/RMRQAQF3w6SnFUii0xpnBVWaoI7I0gPheKLeNzYZSma9JHtcWxSVDr7iloz6SMQfyjJwDocSbduG9HKRkwhb5UqtGHQg8s/VLjfgS2cYjPhWLL+FwZZdOa9BVaOHTc1Js9Be2ZlNHEP0zC8b0gxZBIIQlLNPpQSfgpnhQC2zjE50KxZXxcxkAKoYX65EzNhyhoz6QMU/7RFI7vDCmGxJA6qzK4NzAqCd+3nBjSe2BvhOih9BxBIxxXlUlMlUF94xRFnIFJGTaxF7Xh+GXYGjylBKiE0ln1g/seR6rF4lNOzDzpAe93jM7AU3F/DZkqwxQcfIiC9prKcOEfm/dx9j0fkxjVaQoKonIpg3sDo1osPuWkFPHqgaAMXKleNUpK7q+RL5+uRcDNngyu/KMsjZeRwfKSZSm32FAQlWsZnIRDtVh8ypF4ZK6Dh0tnmwvFEFJO3cMCKv7hlPFQinQWSmcVGlRt8inH9xQQyk/eU5gQEchDHYxCPe6plMlRbkrxHsbBQFRRVSkNzqghdrCWKQwDf0RkI6wC5WmW42ScSpkc5UrRNJjCWBqn0NGmNjijBh/pPeQJzVPVEz0SU/iNRcmUyVEuZ7wHp4fXemca5w7eVO9+SOFGOAq3V6cOV7/dkPpzT5fO6Ayccqw4xj2VMjnK5WpnqPiM2mAgCl/dlIxkLewQMtDLU5iIzsApx4pj3FMpk6NcrnaGFF5rpXHfqCop0Zwt6BEy0MtTmAhqxFRKvQDg2d6ne/cAV78eUKXNRF8AnnsWOHvOrgbKssjL3AvgLF87Ocrlaud0V1n9AoDL+/63tORebjjsBfZcDbxelYQgDVx4Dnj2LOAxNmU6aYFI47EX2LMPuPISYMcvgPM/An7oOa++eL3Wun+xhGXgowyl1OkqK/Ioox2T7WjHpBfteAxGdBVKixYtWrRwQ8vAW7Ro0SJRtAw8HO6P3QCBaMdkO9ox6UU7HgPQ6sBbtGjRIlG0EniLFi1aJIqWgbdo0aJFomgZOAOUUieUUs8rpf5P6dkepdQTSqkfdD8vi9nGkFBK/bJS6n8opTKl1PeVUrd1n4/ymIwppf5MKfXn3TH5N93nIzsmAKCUukgp9V2l1EL375Eejya0DJwHDwB4T9+z2wE8qbV+I4Anu3+PCl4B8Nta6w6AtwE4opS6FqM9Jj8H8A6t9ZsBvAXAe5RSb8NojwkA3IbeeztGfTwGomXgDNBafxPbo7beB+DB7u8PAjgUsk0xobVe0Vo/3f39J8gX6JUY7THRWuufdv+8pPujMcJjopS6CsCvAjheejyy42GCloGHw+u01itAztAAXBG5PVGglNoP4DoA38GIj0lXXfA9AM8DeEJrPepjci+A3wFwofRslMejES0DbxEMSqnXAPgjAEe11n8duz2xobX+G631WwBcBeDvK6XeFLlJ0aCUmgfwvNY6iVw4UtAy8HD4sVJqEgC6n89Hbk9QKKUuQc68/5vW+o+7j0d6TAporV8CcBK53WRUx2QWwK8ppZ4B8AcA3qGUegijOx5GaBl4ODwG4Obu7zcD+ErEtgSFUkoB+DyATGv9e6V/jfKYXK6U2t39/dUAfgXAGYzomGitP6m1vkprvR/AbwD4htb6JozoeJiijcRkgFLqSwBuQJ4K88cAPg3gUQAPA7gawHMADmutY6anDAal1PUA/heAv8CWfvNfI9eDj+qY/D3kRrmLkAtSD2ut/61SahwjOiYFlFI3APiY1nq+HY/BaBl4ixYtWiSKVoXSokWLFomiZeAtWrRokShaBt6iRYsWiaJl4C1atGiRKFoG3qJFixaJomXgLVq0aJEoWgbeokWLFoni/wPNqDV5qNnymgAAAABJRU5ErkJggg==\n", "text/plain": [ "
" ] }, "metadata": { "needs_background": "light" }, "output_type": "display_data" } ], "source": [ "#\n", "\n", "from scipy.io import loadmat\n", "data1 = loadmat('neural_net_class1.mat')['neural_net_class1']\n", "data2 = loadmat('neural_net_class2.mat')['neural_net_class2']\n", "\n", "targets1 = np.ones((np.shape(data1)[0],1))\n", "targets0 = np.zeros((np.shape(data2)[0],1))\n", "\n", "targets = np.vstack((targets1, targets0))\n", "\n", "data = np.vstack((data1, data2))\n", "\n", "# applying the SGD step \n", "\n", "input_dim = 2\n", "output_dim = 1\n", "# number of neurons per layer\n", "network_size = [20,20]\n", "total_size = copy.deepcopy(network_size)\n", "total_size.append(output_dim)\n", "num_layers = len(network_size)\n", "\n", "weights = []\n", "biases = []\n", "\n", "\n", "\n", "current_is = input_dim\n", "current_os = network_size[0]\n", "weights.append(np.random.normal(0,1,(current_os, current_is)))\n", "biases.append(np.random.normal(0,1,(current_os,)))\n", "\n", "# random initialization of weights\n", "for l in np.arange(1,len(total_size)):\n", " \n", " current_is = current_os\n", " current_os = total_size[l]\n", " weights.append(np.random.normal(0,1,(current_os, current_is)))\n", " biases.append(np.random.normal(0,1,(current_os, 1)))\n", "\n", "# learning rate\n", "eta = .01\n", "\n", "num_epochs = 40000\n", "total_loss = np.zeros(num_epochs)\n", "\n", "for e in range(num_epochs):\n", " \n", " # random swapping \n", " indices_epoch = np.arange(np.shape(data)[0])\n", " np.random.shuffle(indices_epoch)\n", " data_epoch = data[indices_epoch,:]\n", " target_epoch = targets[indices_epoch]\n", " \n", " # SGD\n", " \n", " grad_weights_tmp = []\n", " grad_biases_tmp = []\n", " \n", " for l in np.arange(len(weights)):\n", " grad_weights_tmp.append(np.zeros(np.shape(weights[l])))\n", " grad_biases_tmp.append(np.zeros(np.shape(biases[l])))\n", " \n", " for i in np.arange(len(target_epoch)):\n", " \n", " loss, f, g, b = SGD_neuralNet(data_epoch[i,:], target_epoch[i], weights, biases)\n", " \n", " total_loss[e] += loss \n", " #one gradient step \n", " \n", " \n", " for l in np.arange(len(weights)):\n", " \n", " grad_weights_tmp[l] = np.squeeze(grad_weights_tmp[l])+ np.squeeze(g[l])\n", " grad_biases_tmp[l] = np.squeeze(grad_biases_tmp[l]) + np.squeeze(b[l])\n", " \n", " \n", " for l in np.arange(len(weights)):\n", " \n", " weights[l] = weights[l] - (eta/len(target_epoch))*grad_weights_tmp[l] \n", " biases[l] = biases[l].reshape(-1,1) - (eta/len(target_epoch))*grad_biases_tmp[l].reshape(-1,1)\n", "\n", "\n", "x1min = np.min(data[:,0])\n", "x1max = np.max(data[:,0])\n", "x2min = np.min(data[:,1]) \n", "x2max = np.max(data[:,1])\n", "\n", "from matplotlib.colors import ListedColormap\n", "cm_bright = ListedColormap(['#0000FF', '#FF0000'])\n", "\n", "xx1 = np.linspace(x1min, x1max, 100)\n", "xx2 = np.linspace(x2min, x2max, 100)\n", "xx1,xx2 = np.meshgrid(xx1, xx2)\n", "\n", "data_grid = np.vstack((xx1.flatten(), xx2.flatten())).T\n", "\n", "prediction = np.zeros((np.shape(data_grid)[0],1))\n", "\n", "for sample in np.arange(np.shape(data_grid)[0]):\n", " \n", " prediction[sample,:] = SGD_neuralNet(data_grid[sample,:], 0, weights, biases)[0]\n", " \n", " \n", "plt.scatter(data1[:,0], data1[:,1], c='r')\n", "plt.scatter(data2[:,0], data2[:,1], c='b')\n", "plt.contourf(xx1,xx2, np.reshape(prediction>0.5, np.shape(xx1)), levels = 2,alpha=0.2, cmap=cm_bright)\n", "plt.show()" ] }, { "cell_type": "code", "execution_count": 18, "metadata": {}, "outputs": [ { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD4CAYAAAAXUaZHAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAAaDklEQVR4nO3de3Bc533e8e9vFzfiSoBYXAhQJEDwKpCUSJpmpIphdLEpWbRimZNKTSaxxy2ndZ2008m09ngmTabTGdeTdBo5cRQ5VjJOHSmOZMViIouRKFGqHEUSryIgECR4EQmAuPGCG4Xb7ts/dgFigSVFXM+exfOZ2dnF2d2zP74DPHz3Pe95jznnEBGR1BLwugAREZl9CncRkRSkcBcRSUEKdxGRFKRwFxFJQWleFwBQXFzsVqxY4XUZIiK+cvjw4S7nXCjRc0kR7itWrODQoUNelyEi4itm9vHNntOwjIhIClK4i4ikIIW7iEgKUriLiKQghbuISApSuIuIpCCFu4hICvJ1uB9oaOf7B5u8LkNEJOn4OtzfOtXJD94+63UZIiJJx9fhHjAjomuNiIhM4utwN4OIriQlIjKJr8M9YIayXURkMp+Hu3ruIiKJ+DzcTeEuIpKAr8PdzIhEvK5CRCT5+DrcgwENy4iIJOLrcNewjIhIYr4Od9M8dxGRhHwd7gGL3jv13kVE4vg83KPprt67iEi8WQ93M1tnZk+b2Qtm9h9me//jjfbcNe4uIhLvtsLdzJ41sw4zq5uwfZeZNZpZk5l9E8A51+Cc+/fArwFbZ7/kuM8HIKyuu4hInNvtuf8VsGv8BjMLAn8KPAysB540s/Wx574IvAMcmLVKEwjGuu7quIuIxLutcHfOvQ1cmbB5G9DknDvrnBsCngcei73+ZefcPcCv32yfZrbXzA6Z2aHOzs7pFa9hGRGRhNJm8N4K4OK4n5uBz5rZTuBxIBN45WZvds49AzwDsHXr1mml840Dqgp3EZHxZhLulmCbc84dBA7OYL+3X4Bmy4iIJDST2TLNwLJxP1cCrTMrZ2o0z11EJLGZhPsHwCozqzKzDOAJ4OXZKev2BDRbRkQkodudCvkc8C6wxsyazexrzrkR4BvAfqAB+Ilzrn7uSp3sxgHV+fxUEZHkd1tj7s65J2+y/RVucdB0rgXGpkIq3UVExvN0+QEz221mz3R3d0/r/Vp+QEQkMU/D3Tm3zzm3t6CgYFrv1zx3EZHEfL1wmGmeu4hIQr4O99FhGWW7iEg8n4d79F5TIUVE4vk83DUsIyKSiL/DPaDZMiIiifg73LX8gIhIQprnLiKSgjTPXUQkBfl6WEbz3EVEEvN1uI8Ny0Q8LkREJMn4OtyDserVcxcRiefrcNewjIhIYr4Od82WERFJzOfhHr3XPHcRkXg+D3f13EVEEvF1uJvmuYuIJJQaZ6iq6y4iEsfXZ6gGtXCYiEhCvh6W0fIDIiKJ+TrcNc9dRCQxX4e7LrMnIpKYz8M9eq+eu4hIPJ+Huw6oiogk4utwN10gW0QkIV+H++hUSC0/ICISz9fhrmEZEZHEfH6GavReB1RFROL5+gxVzXMXEUksJYZllO0iIvF8Hu7Re/XcRUTi+Tzco+muqZAiIvH8He4BDcuIiCTi73DXsIyISEI+D3fNcxcRScTX4a7L7ImIJObrcL8xFVLhLiIyXkqEu4ZlRETi+Tzco/eaCikiEs/f4R7Q8gMiIon4fOEwzXMXEUnE1wuHaZ67iEhi/h6W0QFVEZGEfB3umucuIpKYr8N9rOeurruISBxfh3tQwzIiIgn5OtxHh2XCGpYREYnj83A3AqblB0REJvJ1uEN03F0HVEVE4qVEuIcjXlchIpJc/B/uAQ3LiIhM5P9wN9PCYSIiE/g+3INmmgopIjKB78PdTGeoiohM5PtwDwQ0W0ZEZCJfL/kLo8MyCncRkfF8veQvRE9k0lRIEZF4vh+WCWoqpIjIJL4Pd52hKiIyWUqEu4ZlRETi+T/cNSwjIjKJ/8PdTEv+iohMkBLhrjNURUTipUC46wxVEZGJUiDcTddQFRGZwPfhHtTyAyIik/g+3HWGqojIZL4Pd11DVURkMt+Hu4ZlREQm8324mxlhZbuISBzfh3tQwzIiIpP4Pty1cJiIyGQpEe66QLaISDz/h3sALT8gIjKB/8NdZ6iKiEzi/2uoaiqkiMgkKXENVXXcRUTipcCwjFaFFBGZyPfhHtRUSBGRSXwf7lo4TERkMt+He1DXUBURmcT34a4zVEVEJkuJcNcZqiIi8fwf7gFDHXcRkXj+D3eDsNJdRCSO78NdUyFFRCbzfbibGRFNhRQRieP7cA8G0AFVEZEJfB/umWlBBkfCXpchIpJUUiDcAwyOaFxGRGQ834d7VnqQgeGwzlIVERknBcI9QMTBcFjhLiIyyvfhnpkWBNC4u4jIOL4P96z06D9hYFjj7iIio3wf7qM994Fh9dxFREb5PtyzM6Ph3j804nElIiLJw/fhXpKXBUBHz6DHlYiIJA/fh3tZfjTc27oHPK5ERCR5+D/cC7LICAZo6uzzuhQRkaTh+3DPSAuwsbKA985e9roUEZGk4ftwB3hofSnHm7tpuNTjdSkiIkkhJcL9X39mGbmZafzh/kavSxERSQopEe6LszP47ftrOHCyg9c+ave6HBERz3ka7ma228ye6e7unvG+vnpvFevL8/lvL35IR49mzojIwuZpuDvn9jnn9hYUFMx4XxlpAZ568i6uD43wb390iJ6B4VmoUETEn1JiWGZUTUke3//1zTRc6uHXnn6Xc139XpckIuKJlAp3gPvXlvLsVz5DW88Aj/zx/+P7B5u0YqSILDgpF+4A960K8crv3MeO1cV899VGdnz3TX7w9lkN1YjIgmHJcAWjrVu3ukOHDs3Jvn/R1MX3Dzbxi6bLZKYF+NydZTx+dwX31CwZW1FSRMSPzOywc25roufS5ruY+XZvTTH31hRT19LNTw5d5OXjrew73kpuZhq/vDrEg+tL2Lm6hMKcDK9LFRGZNSnfc59oaCTCO02dvPZRBwca2unoHcQM1pTmsXVFIZsqF7O2LJ+aklwWZahnLyLJ61Y99wUX7uNFIo4TLd0cbOzk0MdXOHrhGn2D0XXhzWB5UTZryvJYXXrjVlWcQ0ZaSh6qEBGfWdDDMrcSCBibli1m07LFAIQjjo8v93OqvZfGtj4a23s42dbL6w0dhCPR/wTTAsaK4hzWlOaxqjQ3Fvq5LF+SQ3pQoS8iyWFBh/tEwYBRHcqlOpTLrtob2wdHwpztjIZ+9NZHXWs3r9RdYvSLT3rQWBmKhv0vrVzCzjUhygsWefMPEZEFT+F+GzLTgqwrz2ddeX7c9k+Gwpzp7Iv29Nt7Od3ex3vnLvPy8VYAtlUV8fWdK9m5psSLskVkAVO4z8CijCC1FQXUVtxYPsE5x6n2Pl5vaOdv3rvAV/7yAx7fXMH/eKyWnEw1t4jMD6XNLDMz1pTlsaYsj707qvnegdP8yZtNHL94jf/5pQ1sW1FEIGBelykiKW5Bz5aZL//c1MXvPH+Mrr5BinMz+dydpey6s4zt1Us080ZEpk1TIZNA3+AIb5zsYH99GwdPdtA/FCY/K40H15Xy+doydqwKaV69iEyJwj3JDAyH+UVTF6/WtfFaQzvXrg+TlR5g5+oSdtWW8StrSyhYlO51mSKS5DTPPclkpQd5YF0pD6wrZSQc4f1zV3i1vo399W28Wt9GetC4Z2Uxn7+zjIfWlxLKy/S6ZBHxGfXck0gk4jjWfC0a8nVtfHz5OmbwmeVFPLyhjC9sLKckL8vrMkUkSWhYxoecczS29/JqXTToT7b1EjDYXr2EL25ayq7aMhZna7EzkYVM4Z4CTrf3su94Ky8fb+X85eukB40dq0Ls3rSUh9aXag69yAKkcE8hzjnqWnrY92F06eJL3QNkpQd4YG0puzct5VfWhrROvcgCoXBPUZGI4/CFq7x8rJVXTlzicv8QBYvS2b2pnMc3V3L3ssWY6YQpkVSlcF8ARsIR3mnq4qWjLeyvb2NgOEJ1cQ6Pb67gV++uoLIw2+sSRWSWKdwXmN6BYX5+oo0XjzTz3rkrAGyvLuLLmyt5eEM5uRqfF0kJCvcF7OKV67x0tIWfHmnm/OXrZKUH2HVnGY9vruTemmKCWudGxLcU7oJzjiMXrvHTI83sO95Kz8AIZflZfGlzBV/eXElNSa7XJYrIFCncJc7AcJgDDR28eKSZt051Eo447lq2mD1bKtm9cSkF2Vr6QMQPFO5yUx29A/zsaCsvHG6msb2XjLQAD60vZc/mSu5bVUyaLh0okrQU7vKpnHPUt/bwwuFmfnashavXhwnlZfKlu6PDNmvK8rwuUUQmULjLlAyNRHizsYMXDjfz5skORiKODRUF7NlSyRc3LaUwR8seiCQDhbtM2+W+QX52rJUXjzRT39pDetB4YG0pX95Syc41IdI1bCPiGYW7zIqGSz28eLiZvz/WQlffEEtyMnjsrgr2bKlk/dL8T9+BiMwqhbvMquFwhLdPdfLC4WYONHQwFI6wrjyfPVsqeeyupRTnav15kfmgcJc5c7V/iH0ftvLi4WaON3eTFjB2rilhz5YK7l9bqmvEiswhhbvMi9PtvbxwpJmXjrTQ0TvI4ux0Ht1Yzhc2LGVbVZHOhhWZZQp3mVeji5i9cLiZ1xvaGRiOUJybycO1ZTyyoVxBLzJLFO7imetDI7x5spN/PNHKGyc7FPQis0jhLknh+tAIb5zs4JUTl+KC/qH1pTy0voR7VhaTla4LjYjcLoW7JJ3+wRHebIwG/VuNnfQPhVmUHmTH6mIeXFfKA+tKKdLJUiK3dKtw18Le4omczDQe3biURzcuZXAkzLtnLvN6Qzuvf9TB/vp2AgZblhfGgr6ElaFcXVVKZArUc5ekMnqN2Nca2nnto3YaLvUAsLQgix2rQ+xYHeLelcVauVIEDcuIjzVfvc7bp7p4+1Qnv2jqondwhIDBXcsWj4X9psrFOigrC5LCXVLCSDjCsYvXePtUJ2+d7uLD5ms4B3lZaXy2qojt1UvYXr2EdeX5CntZEBTukpKu9g/xTlMX/3zmMu+dvczZrn5AYS8Lx7weUDWzXwW+AJQAf+qc+6fZ/gwRgMKcDHZvWsruTUsBaOse4L1zl/mXs5f5l7NXeL2hA4iG/eY7CtmyvJDNdxSyaVkBeVkas5fUdls9dzN7FngU6HDO1Y7bvgv4YyAI/IVz7jvjnisE/tA597VP27967jIXboT9FY5euEpjey/OgRmsKc1jcyzstywvZMWSbM3GEd+Z8bCMme0A+oAfjYa7mQWBU8BDQDPwAfCkc+6j2PN/BPzYOXfk0/avcJf50DMwzPGL1zj88VWOXLjG0QtX6R0YAaAwO50NlYvZWFFAbUUBGyoLWFqQpcCXpDbjYRnn3NtmtmLC5m1Ak3PubOxDngceM7MG4DvAz28V7Ga2F9gLcMcdd9xOGSIzkp+Vzn2rQty3KgRAJOJo6uzjyMdXOXLhKidaevizt84QjkQ7PEU5GWyoKGCDAl98aCZj7hXAxXE/NwOfBX4beBAoMLMa59zTid7snHsGeAaiPfcZ1CEyLYGAsbo0j9WleTyxLdrBGBgO03Cph7qWbk60dPNhczfvNHXFBf7asjzWluWztjyPtWXR92vZBEk2Mwn3RN0X55x7CnhqBvsV8UxWepC77yjk7jsKx7aNBv6Jlm7qW3o42d7Lc+9f4JPhMAABgxXFOawry2dtWR5ryvJYV55PxeJFBDRLRzwyk3BvBpaN+7kSaJ1ZOSLJJ1HghyOOC1eu09jWQ8OlXk629VDX2s0/nrg09prsjCArQ7msKsllZUkuNbHb8qJs0nTtWZljMwn3D4BVZlYFtABPAP9mVqoSSXLBgFFVnENVcQ67asvHtvcPjnCqvZeGS72c7uilqaOPd89e5qdHW8Zekx6MvremJJea0I3gXxnK1fCOzJrbCnczew7YCRSbWTPw351zPzSzbwD7iU6FfNY5Vz9nlYr4QE5m2qRePkDf4AhnOvo43dFHU+zWcKmXV+vaiA3nYwaVhYuoKs6lOvYfx4riHKqLc1i6eJFOxJIp0RmqIh4aHAlzvuv6WC//bGc/57qit77BkbHXZQQDLF+SHf22EIoG/ool0ceh3EzN4FmgknbJXzPbDeyuqanxsgwRz2SmBVkTOwg7nnOOzr5Bzndd51xXH2e7+jkXC/6DjZ0MhSNjr83NTBsbIqoqzqE6dKPXn68zcRcs9dxFfCYccbRe+yQW+H2c6+qPPu7qp+XaJ4z/ky7OzaC6OJeVJTlUF+dSHcqhOpTLssJFOqibApK25y4iUxcMGMuKsllWlM0vrw7FPTcwHObiletjYX+2MzrUs7++nSv9N05LSQ8ay5dEh3eqQ9HQXxnKZWUoh8XZugJWKlC4i6SQrPQgq0rzWFWaN+m5q/1DnO3q40xnP2c7Y8Hf1c+bjR0Mh29094tyMmKhH53Rs6o0jzWleZTr7FxfUbiLLBCFORlsySliy/KiuO0j4QgXr34y1ssf/Q/gjZOd/ORQ89jr8jLTWFWaO3ZW7+rSPFaX5eqAbpLSmLuI3NS160Ocau+jsb2X0+29NLb1cqq9l6vXh8deszg7ndWx3v3qceFfqAuczzmNuYvItCzOzmBbVRHbqm709p1zdPUNcbo9GvSN7X2cbu/l74+1jK2yCRDKy2RtWR53Li3gzqX51FYUsLwoW0syzBOFu4hMiZkRyssklJfJPTXFY9udc7T3DMb18hvaevjhO2fHxvRzM9NYVx4f+DUluaRr5s6sU7iLyKwwM8oKsigryIqbxTM0EuF0Ry/1LT3Ut3ZT19rDTw5d5PpQdOG1jLTAWA+/tiKf2qUFrCmbvNLmwHCYgeGwZvPcJk/H3MedxPTvTp8+7VkdIjK/whHHua5+6lu7qW+NLrFc39pD9yfRsfxgwFhVkkttRQG1sR7+nqffven+NlYWsKu2jJ2rS6gpySUjbWF8E9AFskUk6TnnaL76SbR33xJdZbOupZuuviGvS0uotiKfu5cVsrEyejGX6lAOmWnzu/Cbwl1EfMk5R0fvIHUt0cBv6xngufcveF3WrHp+73a2Vy+Z1nsV7iKS8sIRx0tHW/jdvzvudSlTEsrL5INvPzit92oqpIikvGDA2LOlkj1bKhM+75zjrVOdfOUvP5jnym6tf9zqn7NJ4S4iC4KZsXNNCee/84WEzzvn+HldG1//8ZF5rWt01tBsU7iLiBAN/0c2lN80/MMRx4/ePc8f7PtoniubHoW7iMhtCAaMr95bxVfvrUr4/HA4wvcOnOapN5rmubLEdEBVRGQeDIcjfO+NJp46EH9Oz/Hf+xwF2dO7qErSzpbRSUwiItN3q3D39DQu59w+59zegoICL8sQEUk5C+McXRGRBUbhLiKSghTuIiIpSOEuIpKCFO4iIilI4S4ikoIU7iIiKSgpzlA1s07g42m+vRjomsVyZovqmhrVNTWqa2qStS6YWW3LnXOhRE8kRbjPhJkdutkZWl5SXVOjuqZGdU1NstYFc1ebhmVERFKQwl1EJAWlQrg/43UBN6G6pkZ1TY3qmppkrQvmqDbfj7mLiMhkqdBzFxGRCRTuIiIpyNfhbma7zKzRzJrM7Jvz8HnnzeyEmR0zs0OxbUVm9pqZnY7dF457/bditTWa2efHbd8S20+TmT1lZjbFOp41sw4zqxu3bdbqMLNMM/vb2Pb3zGzFDOr6fTNribXZMTN7xIO6lpnZm2bWYGb1ZvafkqHNblGXp21mZllm9r6ZHY/V9QdJ0l43q8vz37HYe4NmdtTM/iEZ2gvnnC9vQBA4A1QDGcBxYP0cf+Z5oHjCtu8C34w9/ibwv2KP18dqygSqYrUGY8+9D/wSYMDPgYenWMcOYDNQNxd1AF8Hno49fgL42xnU9fvA7yZ47XzWVQ5sjj3OA07FPt/TNrtFXZ62WWwfubHH6cB7wPYkaK+b1eX571js9f8F+BvgH5Lhb9KzcJ7pLdYA+8f9/C3gW3P8meeZHO6NQHnscTnQmKgeYH+s5nLg5LjtTwJ/Po1aVhAforNWx+hrYo/TiJ49Z9Os62Z/ePNa14TP/hnwULK0WYK6kqbNgGzgCPDZZGqvCXV53l5AJXAAuJ8b4e5pe/l5WKYCuDju5+bYtrnkgH8ys8Nmtje2rdQ5dwkgdl/yKfVVxB5P3D5Ts1nH2HuccyNAN7BkBrV9w8w+tOiwzehXU0/qin2dvZtory9p2mxCXeBxm8WGGI4BHcBrzrmkaK+b1AXe/479H+C/ApFx2zxtLz+He6Jx6rme13mvc24z8DDwH81sxy1ee7P65rvu6dQxmzX+GbASuAu4BPyRV3WZWS7wIvCfnXM9t3rpfNaWoC7P28w5F3bO3UW0R7rNzGpv9U/wuC5P28vMHgU6nHOHP63++azLz+HeDCwb93Ml0DqXH+ica43ddwAvAduAdjMrB4jdd3xKfc2xx7Nd92zWMfYeM0sDCoAr0ynKOdce+4OMAD8g2mbzXpeZpRMN0B87534a2+x5myWqK1naLFbLNeAgsIskaK9EdSVBe90LfNHMzgPPA/eb2f/F4/byc7h/AKwysyozyyB6kOHlufowM8sxs7zRx8DngLrYZ/5W7GW/RXTclNj2J2JHuauAVcD7sa9nvWa2PXYk/DfHvWcmZrOO8fvaA7zhYoN9UzX6yx3zJaJtNq91xfbzQ6DBOfe/xz3laZvdrC6v28zMQma2OPZ4EfAgcBLv2ythXV63l3PuW865SufcCqI59IZz7je8bq8pHYhKthvwCNEZBmeAb8/xZ1UTPcJ9HKgf/Tyi414HgNOx+6Jx7/l2rLZGxs2IAbYS/QU8A/wJUz/w9hzRr5/DRP9H/9ps1gFkAX8HNBE9el89g7r+GjgBfBj7BS33oK5/RfQr7IfAsdjtEa/b7BZ1edpmwEbgaOzz64Dfm+3f9Vmuy/PfsXH73cmNA6qetpeWHxARSUF+HpYREZGbULiLiKQghbuISApSuIuIpCCFu4hIClK4i4ikIIW7iEgK+v9mhJ1Woq6e/QAAAABJRU5ErkJggg==\n", "text/plain": [ "
" ] }, "metadata": { "needs_background": "light" }, "output_type": "display_data" } ], "source": [ "plt.semilogy(total_loss)\n", "plt.show()" ] } ], "metadata": { "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.8.5" } }, "nbformat": 4, "nbformat_minor": 4 }