{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Neural Networks from scratch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: matplotlib in /usr/local/lib/python3.6/dist-packages (3.1.3)\n",
      "Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.6/dist-packages (from matplotlib) (1.1.0)\n",
      "Requirement already satisfied: numpy>=1.11 in /usr/local/lib/python3.6/dist-packages (from matplotlib) (1.18.1)\n",
      "Requirement already satisfied: python-dateutil>=2.1 in /usr/local/lib/python3.6/dist-packages (from matplotlib) (2.8.1)\n",
      "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.6/dist-packages (from matplotlib) (0.10.0)\n",
      "Requirement already satisfied: pyparsing!=2.0.4,!=2.1.2,!=2.1.6,>=2.0.1 in /usr/local/lib/python3.6/dist-packages (from matplotlib) (2.4.6)\n",
      "Requirement already satisfied: setuptools in /usr/local/lib/python3.6/dist-packages (from kiwisolver>=1.0.1->matplotlib) (45.2.0)\n",
      "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.6/dist-packages (from python-dateutil>=2.1->matplotlib) (1.14.0)\n"
     ]
    }
   ],
   "source": [
    "!pip3 install matplotlib"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from math import e\n",
    "from typing import List, Callable\n",
    "from random import random\n",
    "from matplotlib import pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Function to calculate the dot product\n",
    "# See: https://en.wikipedia.org/wiki/Dot_product\n",
    "def dot(a: List[float], b: List[float]) -> float:\n",
    "    return sum(a_i * b_i for a_i, b_i in zip(a, b))\n",
    "\n",
    "assert dot([1, 2, 3, 4], [5, 6, 7, 8]) == 70"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# The Step activation function\n",
    "# See: https://en.wikipedia.org/wiki/Step_function\n",
    "def step(x: float) -> int:\n",
    "    return 0.0 if x < 0.0 else 1.0\n",
    "\n",
    "assert step(-0.1) == 0.0\n",
    "assert step(0.0) == 1.0\n",
    "assert step(0.1) == 1.0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAASrElEQVR4nO3dfYxl9V3H8fcHttRoH1B3WxGWLtWt6WrU4gSbtGqTYgWirM+BxPjUSEzEaOpDaDDYYGJSGzVpROs2NtVGi9THja5BrfUhRipDSykPYrdYZRHLWpvWpAri/frHPQM/DjM7d2bv3Du/8f1KJnvvOWfnfnPu4cNvv+ec30lVIUnq3znLLkCSNB8GuiTtEQa6JO0RBrok7REGuiTtEfuW9cH79++vQ4cOLevjJalLd911179X1YH11i0t0A8dOsTq6uqyPl6SupTknzdaZ8tFkvYIA12S9ggDXZL2CANdkvYIA12S9ohNAz3JO5I8luTeDdYnyVuTnExyT5JL51+mJGkzs4zQ3wlccYb1VwKHh5/rgF85+7IkSVu16XXoVfXXSQ6dYZOjwG/UdB7eO5Kcn+SCqnp0TjVKC3Xqk5/hPauncGpp7ZTXvvzFfMXB8+f+e+dxY9GFwMPN+1PDsmcFepLrmI7iufjii+fw0dL83bZ6ire+9yMky65Ee9WLXvBZuzbQZ1ZVx4BjACsrKw5/tCs9+b8T9p0TTv7sVcsuRdqSeVzl8ghwsHl/0bBM6tKk4ByH5+rQPAL9OPDdw9UurwQ+Zf9cPasq2y3q0qYtlyTvBl4D7E9yCvhp4DkAVfU24ARwFXAS+AzwfTtVrLQIkypH6OrSLFe5XLvJ+gJ+aG4VSUs2bbksuwpp67xTVBpxhK5eGejSSBX20NUlA10amVRxjj0XdchAl0ZsuahXBro04klR9cpAl0am16Gb6OqPgS6NTCaO0NUnA10asYeuXhno0ohzuahXBro04lwu6pWBLo3YclGvDHRpxMsW1SsDXRpxhK5eGejSiHO5qFcGujTiCF29MtClEQNdvTLQpZGJLRd1ykCXRsoRujploEsjk4Jz/C9DHfKwlUbsoatXBro0Mu2hG+jqj4EujUx76MuuQto6A10aKWdbVKcMdGlk4ghdnTLQpZGJj6BTpwx0acTZFtUrA10a8cYi9cpAl0Z8BJ16ZaBLIxMfQadOGejSiCN09cpAl0a8sUi9minQk1yR5MEkJ5PcsM76i5O8L8kHk9yT5Kr5lyothnO5qFebBnqSc4FbgCuBI8C1SY6MNvsp4LaqegVwDfDL8y5UWpTJxLlc1KdZRuiXASer6qGqegK4FTg62qaAFwyvXwj86/xKlBbLO0XVq1kC/ULg4eb9qWFZ603AdyU5BZwAfni9X5TkuiSrSVZPnz69jXKlnedcLurVvE6KXgu8s6ouAq4C3pXkWb+7qo5V1UpVrRw4cGBOHy3N16TKB1yoS7Mcto8AB5v3Fw3LWq8HbgOoqr8DPgvYP48CpUVzLhf1apZAvxM4nOSSJOcxPel5fLTNvwCvBUjycqaBbk9FXbLlol5tGuhV9SRwPXA78ADTq1nuS3JzkquHzX4M+IEkHwLeDXxvVdVOFS3tJE+Kqlf7Ztmoqk4wPdnZLrupeX0/8Kr5liYth3eKqlee+pFGnMtFvTLQpRF76OqVgS6N2ENXrwx0acS5XNQrA10amZRzuahPBro04vS56pWBLo1MR+jLrkLaOgNdGrGHrl4Z6NLIZGKgq08GujRStlzUKQNdGrHlol4Z6NLIdC6XZVchbZ2BLo04QlevDHRppLyxSJ0y0KUR53JRrwx0acSWi3ploEsjnhRVrwx0qbH25ER76OqRgS41JsOTcG25qEcGutSYDCN0Wy7qkYEuNZ4KdBNdHTLQpcaQ587loi4Z6FLj6ZaLia7+GOhS4+mTosutQ9oOA11qOEJXzwx0qVGT6Z9eh64eGehSw8sW1TMDXWrYclHPDHSp4UlR9cxAlxrO5aKezRToSa5I8mCSk0lu2GCb70xyf5L7kvzWfMuUFsO5XNSzfZttkORc4Bbg64FTwJ1JjlfV/c02h4E3Aq+qqk8medFOFSztJE+KqmezjNAvA05W1UNV9QRwK3B0tM0PALdU1ScBquqx+ZYpLcYwQHeEri7NEugXAg83708Ny1ovA16W5G+T3JHkivV+UZLrkqwmWT19+vT2KpZ20GSy1kNfciHSNszrpOg+4DDwGuBa4O1Jzh9vVFXHqmqlqlYOHDgwp4+W5qfsoatjswT6I8DB5v1Fw7LWKeB4Vf1PVf0T8I9MA17qytPT5y65EGkbZjls7wQOJ7kkyXnANcDx0TZ/wHR0TpL9TFswD82xTmkhvLFIPds00KvqSeB64HbgAeC2qrovyc1Jrh42ux34RJL7gfcBP1FVn9ipoqWdMnlqPnQDXf3Z9LJFgKo6AZwYLbupeV3AG4YfqVvlZYvqmJ1CqeGNReqZgS41vLFIPTPQpcbEuVzUMQNdangdunpmoEsNWy7qmYEuNTwpqp4Z6FLj6R76kguRtsFAlxrlnaLqmIEuNWy5qGcGutRYmz7Xk6LqkYEuNZzLRT0z0KWGc7moZwa61Hiqh26iq0MGutTwxiL1zECXGs7lop4Z6FLDuVzUMwNdathyUc8MdKnhjUXqmYEuNdZG6FKPDHSp4Vwu6pmBLjWevg59uXVI2+FhKzUmjtDVMQNdajx9UnS5dUjbYaBLjfLGInXMQJcatlzUMwNdakwm0z9tuahHBrrUcISunhnoUqOeesDFcuuQtsNAlxqO0NUzA11qOJeLemagSw1nW1TPZgr0JFckeTDJySQ3nGG7b0tSSVbmV6K0OF6Hrp5tGuhJzgVuAa4EjgDXJjmyznbPB34EeP+8i5QWxTtF1bNZRuiXASer6qGqegK4FTi6znY/A7wZ+O851ictlCdF1bNZAv1C4OHm/alh2VOSXAocrKo/PtMvSnJdktUkq6dPn95ysdJO86SoenbWJ0WTnAP8AvBjm21bVceqaqWqVg4cOHC2Hy3N3VM9dC8XUIdmOWwfAQ427y8alq15PvBlwF8m+RjwSuC4J0bVI1su6tksgX4ncDjJJUnOA64Bjq+trKpPVdX+qjpUVYeAO4Crq2p1RyqWdpAnRdWzTQO9qp4ErgduBx4Abquq+5LcnOTqnS5QWiRH6OrZvlk2qqoTwInRsps22PY1Z1+WtBzO5aKeeepHakwmjtDVLwNdanjZonpmoEsN53JRzwx0qeFcLuqZgS41Ckfn6peBLjUmVfbP1S0DXWpMyhOi6peBLjUmVV6Drm4Z6FKjHKGrYwa61JhMypOi6paBLjXsoatnBrrUsIeunhnoUqOqOMeeizploEsNWy7qmYEuNaY3Fi27Cml7DHSpMSnncVG/DHSpUY7Q1TEDXWo4l4t6ZqBLDU+KqmcGutTwOnT1zECXGs7lop4Z6FLDyxbVMwNdathDV88MdKlhD109M9ClRnnZojpmoEuNycSWi/ploEsNWy7qmYEuNTwpqp4Z6FJjOh/6squQtsdDV2pMqgiO0NWnmQI9yRVJHkxyMskN66x/Q5L7k9yT5L1JXjL/UqWdN225LLsKaXs2DfQk5wK3AFcCR4BrkxwZbfZBYKWqvhz4HeDn5l2otAjTk6Imuvo0ywj9MuBkVT1UVU8AtwJH2w2q6n1V9Znh7R3ARfMtU1qMcoSujs0S6BcCDzfvTw3LNvJ64E/WW5HkuiSrSVZPnz49e5XSgjgfuno215OiSb4LWAHest76qjpWVStVtXLgwIF5frQ0Fwa6erZvhm0eAQ427y8alj1DksuBG4Gvq6rH51OetFjTZ4ouuwppe2YZod8JHE5ySZLzgGuA4+0GSV4B/CpwdVU9Nv8ypcVwLhf1bNNAr6ongeuB24EHgNuq6r4kNye5etjsLcDzgPckuTvJ8Q1+nbSrTQpvLFK3Zmm5UFUngBOjZTc1ry+fc13SUthDV88ci0iNaQ/dQFefDHSpUT6CTh0z0KWGLRf1zECXGtMHXCy7Cml7DHSp4Vwu6pmBLjWcy0U9M9Clhj109cxAlxoGunpmoEuNci4XdcxAlxqO0NUzA11q+Ag69cxAlxqO0NUzA11qlHO5qGMGutSYOJeLOmagSw1bLuqZgS41fMCFeuahKzXKuVzUMQNdajiXi3pmoEsNe+jqmYEuNaY3Fhno6pOBLjWm86Evuwppewx0qVGO0NUxA11qeGORemagSw1PiqpnBrrUmDiXizpmoEuNsuWijhnoUsPLFtUzA11qeFJUPTPQpUFVOR+6umagS4Oq6Z+2XNQrA10aTIZEt+WiXs0U6EmuSPJgkpNJblhn/XOT/Paw/v1JDs27UGmnTdZG6Ca6OrVpoCc5F7gFuBI4Alyb5Mhos9cDn6yqLwZ+EXjzvAuVdtraCN2Oi3q1b4ZtLgNOVtVDAEluBY4C9zfbHAXeNLz+HeCXkqRqrSs5P7fd+TBv/5uH5v1rpacDHRNdfZol0C8EHm7enwK+eqNtqurJJJ8CPh/493ajJNcB1wFcfPHF2yr4/M9+Dodf/Lxt/V1pM0e+8IVc/vIXLbsMaVtmCfS5qapjwDGAlZWVbY3eX/elX8DrvvQL5lqXJO0Fs5wUfQQ42Ly/aFi27jZJ9gEvBD4xjwIlSbOZJdDvBA4nuSTJecA1wPHRNseB7xlefzvwFzvRP5ckbWzTlsvQE78euB04F3hHVd2X5GZgtaqOA78GvCvJSeA/mIa+JGmBZuqhV9UJ4MRo2U3N6/8GvmO+pUmStsI7RSVpjzDQJWmPMNAlaY8w0CVpj8iyri5Mchr4523+9f2M7kLdRXZrbda1Nda1dbu1tr1W10uq6sB6K5YW6GcjyWpVrSy7jvXs1tqsa2usa+t2a23/n+qy5SJJe4SBLkl7RK+BfmzZBZzBbq3NurbGurZut9b2/6auLnvokqRn63WELkkaMdAlaY/Y9YGe5DuS3JdkkmRltO6Nw4OpH0zyDc3yMz7Uegdq/O0kdw8/H0ty97D8UJL/ata9badrGdX1piSPNJ9/VbNu3X23wNrekuQfktyT5PeTnD8sX+o+G2pY6PFzhjoOJnlfkvuH/wZ+ZFi+4fe6wNo+luTDw+evDss+L8mfJfnI8OfnLrimL2n2yd1JPp3kR5e1v5K8I8ljSe5tlq27jzL11uGYuyfJpdv60Kra1T/Ay4EvAf4SWGmWHwE+BDwXuAT4KNPpfc8dXr8UOG/Y5sgC6/154Kbh9SHg3iXuuzcBP77O8nX33YJrex2wb3j9ZuDNu2SfLfX4GdVyAXDp8Pr5wD8O39263+uCa/sYsH+07OeAG4bXN6x9p0v8Hv8NeMmy9hfwtcCl7fG80T4CrgL+BAjwSuD92/nMXT9Cr6oHqurBdVYdBW6tqser6p+Ak0wfaP3UQ62r6glg7aHWOy5JgO8E3r2IzzsLG+27hamqP62qJ4e3dzB9EtZusLTjZ6yqHq2qDwyv/xN4gOnze3ero8CvD69/HfjmJdbyWuCjVbXdu9HPWlX9NdPnQ7Q22kdHgd+oqTuA85NcsNXP3PWBfgbrPbz6wjMsX4SvAT5eVR9pll2S5INJ/irJ1yyojtb1wz/h3tH8E3iZ+2g93890dLJmmftst+0bYNqKAl4BvH9YtN73ukgF/GmSuzJ9+DvAi6vq0eH1vwEvXkJda67hmQOrZe+vNRvto7kcd7si0JP8eZJ71/lZyshoPTPWeC3PPIgeBS6uqlcAbwB+K8kLFljXrwBfBHzlUMvPz/Ozz7K2tW1uBJ4EfnNYtOP7rDdJngf8LvCjVfVplvy9Dl5dVZcCVwI/lORr25U17SMs5ZroTB+VeTXwnmHRbthfz7IT+2imJxbttKq6fBt/7UwPr97sodZbtlmNmT4c+1uBr2r+zuPA48Pru5J8FHgZsHq29cxaV1Pf24E/Gt7O8uDvszbDPvte4BuB1w4H90L22SYWsm9mleQ5TMP8N6vq9wCq6uPN+vZ7XZiqemT487Ekv8+0VfXxJBdU1aNDu+CxRdc1uBL4wNp+2g37q7HRPprLcbcrRujbdBy4Jslzk1wCHAb+ntkear0TLgf+oapOrS1IciDJucPrlw41PrSAWtY+v+3BfQuwdrZ9o323MEmuAH4SuLqqPtMsX+o+Y3nHz7MM52R+DXigqn6hWb7R97qouj4nyfPXXjM9wX0vz3xY/PcAf7jIuhrP+JfysvfXyEb76Djw3cPVLq8EPtW0Zma36DO/2zhT/C1M+0mPAx8Hbm/W3cj0ioQHgSub5VcxvSLgo8CNC6rzncAPjpZ9G3AfcDfwAeCbFrzv3gV8GLhnOGAu2GzfLbC2k0x7hncPP2/bDftsWcfPBnW8muk/ye9p9tNVZ/peF1TXS5le/fOh4bu6cVj++cB7gY8Afw583hL22ecAnwBe2Cxbyv5i+j+VR4H/GTLs9RvtI6ZXt9wyHHMfprmibys/3vovSXtEzy0XSVLDQJekPcJAl6Q9wkCXpD3CQJekPcJAl6Q9wkCXpD3i/wAmSRRoAvz5hwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot([x for x in range(-100, 100)], [step(x) for x in range(-100, 100)]);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# The `Neuron` class represents a single Neuron / Perceptron\n",
    "class Neuron:\n",
    "    def __init__(self, activation: Callable) -> None:\n",
    "        self._weights: List[float] = [random() for i in range(2)]\n",
    "        self._bias: float = random()\n",
    "        self._activation: Callable = activation\n",
    "    \n",
    "    def forward(self, x: List[float]) -> float:\n",
    "        return self._activation(dot(x, self._weights) + self._bias)\n",
    "\n",
    "    @property\n",
    "    def weights(self) -> List[float]:\n",
    "        return self._weights\n",
    "\n",
    "    @property\n",
    "    def bias(self) -> float:\n",
    "        return self._bias\n",
    "\n",
    "    @weights.setter\n",
    "    def weights(self, weights: List[float]) -> None:\n",
    "        self._weights: List[float] = weights\n",
    "    \n",
    "    @bias.setter\n",
    "    def bias(self, bias: float) -> None:\n",
    "        self._bias: float = bias"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Using a Perceptron to \"learn\" AND\n",
    "# See: https://en.wikipedia.org/wiki/Logical_conjunction\n",
    "x_1: List[float] = [0, 0]\n",
    "x_2: List[float] = [0, 1]\n",
    "x_3: List[float] = [1, 0]\n",
    "x_4: List[float] = [1, 1]\n",
    "\n",
    "y_1: float = 0.0\n",
    "y_2: float = 0.0\n",
    "y_3: float = 0.0\n",
    "y_4: float = 1.0\n",
    "\n",
    "# Creating a single `Neuron` (with a `step` activation function)\n",
    "# and setting the weights and bias\n",
    "perceptron_and: Neuron = Neuron(step)\n",
    "perceptron_and.weights = [1.5, 1.5]\n",
    "perceptron_and.bias = -2\n",
    "\n",
    "assert perceptron_and.forward(x_1) == y_1\n",
    "assert perceptron_and.forward(x_2) == y_2\n",
    "assert perceptron_and.forward(x_3) == y_3\n",
    "assert perceptron_and.forward(x_4) == y_4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# The Sigmoid activation function\n",
    "# See: https://en.wikipedia.org/wiki/Sigmoid_function\n",
    "def sigmoid(x: float) -> float:\n",
    "    return 1 / (1 + (e ** -x))\n",
    "\n",
    "assert sigmoid(0) == 0.5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAfSUlEQVR4nO3deXRc9X338fdXo82SvEtekC3LDl4wZrERBpKmAYzBOMQO0KaQJg0JDSdtSNLSpCUnLc1J2vOEJE2bNCQp2SB5EihtAlaIwUACD2lSwI6RvGEZ2XjRYlsS3iRZy2i+zx8zNoOQrLE9M3dm9HmdM2fu8puZ77kz89HVb+69P3N3REQk++UFXYCIiCSHAl1EJEco0EVEcoQCXUQkRyjQRURyRH5QL1xeXu7V1dVBvbyISFb6/e9/3+7uFUOtCyzQq6ur2bBhQ1AvLyKSlcxsz3Dr1OUiIpIjFOgiIjlCgS4ikiMU6CIiOUKBLiKSI0YMdDP7gZkdNLMtw6w3M/uGmTWa2SYzW5L8MkVEZCSJ7KE/AKw4xfrrgbmx2x3At8++LBEROV0jHofu7s+bWfUpmqwGfuTR6/C+YGYTzGy6u7cmqUYREQDcnZ7+CH3hCOFIhHDE6R+IEB5wwpEI/QNOeMDpj8SWDUToj8TuB5yBSLRdxJ1IBCLuuIPjRDw6H/Ho6/ig+Temo+3jrzx+oj2AQ9z0m5efmFl23lQumjkh6dsnGScWVQL74uabYsveEuhmdgfRvXiqqqqS8NIikk2OdPez71A3R4/309kbprM3TFdvmGOx+67eAY71RKfj18dPR7J8CAczmDKuOGMDPWHufj9wP0BNTU2Wvy0iMpi7097Zx56OLnZ3dLM3dr+no4s9r3dzuLt/2MeG8oyyonzKivIpLQpRVpTP2OJ8po8vji2LrispClEYyqMglEd+yCjIi97nh/IoyIvexy8vCBn5J9rE7kNm5JlhFg3YvNh8noHFlsfP58XmzcCI3gMn59+YPrHc4qaj8+mQjEBvBmbGzc+ILRORHBSJOK1He9jTHg3p3R1d7O3oPhnc3X0DJ9vmGVROHEP15FJuuHA6syaVMnNSCRNKCk6Gd1lx9L4oPy9twZerkhHotcCdZvYwcBlwRP3nIrklPBDhtzs7eHRjE09tO/Cm0C4M5TFjUjS0L58ziVmTSphVXkr15FIqJ4yhMF9HR6fLiIFuZg8BVwLlZtYE/CNQAODu3wHWAiuBRqAb+HCqihWR9HF3trUe5dGNzaypb6HtWC/jivNZffE5LKocT/XkUmZNLmH6+DGE8rRnnQkSOcrl1hHWO/DxpFUkIoFqPXKcNXUtPLqxmYYDxygIGVfNn8JNSyq5asEUivJDQZcowwjs8rkikjk6e8M8uWU/j77cxO92duAOS6om8MX3LuKGC6YzsbQw6BIlAQp0kVEqPBDhfxrbefTlZtZt3U9Pf4SqSSV88uq53Li4kury0qBLlNOkQBcZZba1HOVnG5tYU9dCe2cv48cUcPOSGdy0pJIlVRN1pEkWU6CLjBLuzr//upGvPb2DgpBx9YIp3Lh4BlctqFC/eI5QoIuMAuGBCP+wZgsPvbSPmxZXcs97FjKhRP3iuUaBLpLjuvvC3PnTl/n19oN8/Kq38elr56tbJUcp0EVyWHtnL7c/sJ7NzUf4p/cu4gOXzwq6JEkhBbpIjtrd3sWHfvgSB4728B8frGH5wqlBlyQppkAXyUEv7z3E7Q9uAOCnH72cJVUTA65I0kGBLpJjntl2gDsf2siUscU8+JGlzNbx5KOGAl0kh/zkxT38w2NbWFQ5nu9/6FIqxhYFXZKkkQJdJAe4O//y1A6++WwjV82v4JvvX0Jpkb7eo43ecZEs1z8Q4e6fbeZnG5v4k5qZ/PONi8gP6ZK1o5ECXSSLdfaG+Yv/+3t+82o7f3XNXD61bK6OMR/FFOgiWerg0R4+/MB6tu8/xpdvvpD3XTpz5AdJTlOgi2ShxoOdfOgHL3Gou4/vfaiGq+ZPCbokyQAKdJEss2H36/z5jzaQn2c8fMflXDgj+aPHS3ZSoItkkVdaj/Kn33uRcyaM4cEPL6VqcknQJUkGUaCLZJH/88R2igtC/NfHrqC8TMeYy5vp2CaRLPHbxnae39HGnVedqzCXISnQRbJAJOJ86YntVE4Ywwev0BUTZWgKdJEs8PjmVjY3H+Gu5fMoLtDoQjI0BbpIhusLR/jqugYWTBvLexdXBl2OZDAFukiG++mLe9j7ejd/d/0CQnk6C1SGp0AXyWDHevr5xq8buWLOZK6cVxF0OZLhFOgiGey7z+/i9a4+7r5+ga7RIiNSoItkqIPHevjub17j3RdO56KZOhtURqZAF8lQX3/mVfoHInzm2vlBlyJZQoEukoF2tXXy8Pp9vP+yKqo1hJwkSIEukoG+sq6B4vw8PnH13KBLkSyiQBfJMBv3HuKJLfv56B/O0ZigcloSCnQzW2FmDWbWaGZ3D7G+ysyeNbOXzWyTma1Mfqkiuc89eop/eVkhf/7OOUGXI1lmxEA3sxBwH3A9sBC41cwWDmr298Aj7r4YuAX4VrILFRkNnm04yEuvvc6nls2lTIM8y2lKZA99KdDo7rvcvQ94GFg9qI0D42LT44GW5JUoMjoMRJx7n2igenIJtyytCrocyUKJBHolsC9uvim2LN7ngQ+YWROwFvjEUE9kZneY2QYz29DW1nYG5Yrkrp9vbKLhwDE+c90CCkL6eUtOX7I+NbcCD7j7DGAl8GMze8tzu/v97l7j7jUVFTqNWeSEnv4Bvvb0Di6aOYGVF0wLuhzJUokEejMQP5z4jNiyeLcDjwC4+/8CxUB5MgoUGQ0e/N1uWo/0cPcKneIvZy6RQF8PzDWz2WZWSPRHz9pBbfYCywDM7Dyiga4+FZEEHO7u475nG7lqfgVXvG1y0OVIFhsx0N09DNwJrANeIXo0y1Yz+4KZrYo1+xvgo2ZWDzwE3ObunqqiRXLJt5/bybHeMH+7YkHQpUiWS+i4KHdfS/THzvhl98RNbwPekdzSRHJf8+Hj/PB3u7lp8QzOmz5u5AeInIJ+ShcJ0L8+vQOAu66dF3AlkgsU6CIB2b7/KD/b2MRtb6+mcsKYoMuRHKBAFwnIvU9sZ2xRPn955duCLkVyhAJdJAD/u7ODZxva+MurzmVCSWHQ5UiOUKCLpJm786UntzN9fDG3vb066HIkhyjQRdLsiS37qd93mL9ePo/iglDQ5UgOUaCLpFH/QISvrGtg3tQybl4yI+hyJMco0EXS6KmtB3itvYtPXzufUJ5O8ZfkUqCLpNGaumamjC1i2XlTgy5FcpACXSRNjhzv57mGNm648BztnUtKKNBF0mTdlv30DURYffE5QZciOUqBLpIma+qbqZ5cwoUzxgddiuQoBbpIGhw82sPvdnaw6uJKXe9cUkaBLpIGj29qxR1WXaTuFkkdBbpIGqypb+H8c8Zx7pSyoEuRHKZAF0mx3e1d1O87rB9DJeUU6CIpVlvfghm8R90tkmIKdJEUcnfW1DWztHoS08frmueSWgp0kRTa1nqUnW1drFJ3i6SBAl0khWrrWsjPM1Yumh50KTIKKNBFUiQScWrrW3jXvAomlmoQC0k9BbpIiqzf/TqtR3rU3SJpo0AXSZHa+hbGFIRYvlBXVpT0UKCLpEBfOMIvN7eyfOFUSgrzgy5HRgkFukgK/E9jG4e7+3UykaSVAl0kBdbUtTChpIB3zq0IuhQZRRToIknW3Rfm6W0HWHnBdArz9RWT9NGnTSTJnnnlIN19A7qyoqSdAl0kyWrrmpk2rpil1ZOCLkVGGQW6SBId6urjuYY2Vl18DnkaN1TSLKFAN7MVZtZgZo1mdvcwbd5nZtvMbKuZ/TS5ZYpkhye27CcccXW3SCBGPEDWzELAfcByoAlYb2a17r4trs1c4LPAO9z9kJlNSVXBIpmstr6ZORWlnH/OuKBLkVEokT30pUCju+9y9z7gYWD1oDYfBe5z90MA7n4wuWWKZL7WI8d58bXXWX2Rxg2VYCQS6JXAvrj5ptiyePOAeWb2WzN7wcxWDPVEZnaHmW0wsw1tbW1nVrFIhnq8PjZuqE4mkoAk60fRfGAucCVwK/BdM5swuJG73+/uNe5eU1GhEy4kt6ypb+aiGeOZXV4adCkySiUS6M3AzLj5GbFl8ZqAWnfvd/fXgB1EA15kVNjZ1smW5qOsunjwP68i6ZNIoK8H5prZbDMrBG4Bage1eYzo3jlmVk60C2ZXEusUyWi1ddFxQ2+4UANZSHBGDHR3DwN3AuuAV4BH3H2rmX3BzFbFmq0DOsxsG/As8Bl370hV0SKZxD06kMUVcyYzdVxx0OXIKJbQdT3dfS2wdtCye+KmHbgrdhMZVTY3H+G19i4+9q45QZcio5zOFBU5S2vqWigM5bHifHW3SLAU6CJnYSDiPL6phXfNr2B8SUHQ5cgop0AXOQsvvtbBgaO9GshCMoICXeQs1Na1UFoYYtkCjRsqwVOgi5yh3vAAaze3ct350xhTGAq6HBEFusiZen5HO0d7wjrVXzKGAl3kDK2pa2ZSaSHvOLc86FJEAAW6yBnp7A3zzCsHePcF0ykI6WskmUGfRJEz8PS2/fT0R3R0i2QUBbrIGaita6FywhiWVE0MuhSRkxToIqepo7OX519t5z0XadxQySwKdJHTtHbLfgYiru4WyTgKdJHTVFvXzLypZSyYNjboUkTeRIEuchqaDx9n/e5DrL5Y44ZK5lGgi5yGX9S3APCeC9XdIplHgS5yGtbUtbC4agJVk0uCLkXkLRToIgnaceAYr7QeZfVF2juXzKRAF0lQbV0LeQbvVneLZCgFukgCTowb+o5zy6kYWxR0OSJDUqCLJKBu32H2vt7NKnW3SAZToIskYE1dC4X5eVy3aFrQpYgMS4EuMoLwQITHN7WybMEUxhVr3FDJXAp0kRG8sOt12js1bqhkPgW6yAjW1DUztiifK+dPCboUkVNSoIucQk//AE9u2c91i6ZRXKBxQyWzKdBFTuG5hoMc6w2ru0WyggJd5BTW1LVQXlbEFXMmB12KyIgU6CLDONbTz6+2H+SGC6eTr3FDJQvoUyoyjHVbD9AXjrBK3S2SJRToIsNYU9fMzEljWDxzQtCliCREgS4yhLZjvfy2sZ3VF2kgC8keCQW6ma0wswYzazSzu0/R7mYzczOrSV6JIum3dnMrEUdHt0hWGTHQzSwE3AdcDywEbjWzhUO0Gwt8Cngx2UWKpNuaumYWTBvL3KkaN1SyRyJ76EuBRnff5e59wMPA6iHafRG4F+hJYn0iabe3o5uNew+z+uLKoEsROS2JBHolsC9uvim27CQzWwLMdPdfnuqJzOwOM9tgZhva2tpOu1iRdPjFpti4oRdND7gSkdNz1j+Kmlke8DXgb0Zq6+73u3uNu9dUVFSc7UuLpMSaumYurZ7IjIkaN1SySyKB3gzMjJufEVt2wlhgEfCcme0GLgdq9cOoZKPt+4+y40CnBrKQrJRIoK8H5prZbDMrBG4Bak+sdPcj7l7u7tXuXg28AKxy9w0pqVgkhdbUtRDKM1ZeoO4WyT4jBrq7h4E7gXXAK8Aj7r7VzL5gZqtSXaBIukQiTm1dC++cW87kMo0bKtknP5FG7r4WWDto2T3DtL3y7MsSSb+New/RfPg4n75uXtCliJwRnSkqElNb30JxQR7LF2rcUMlOCnQRoH8gwi83tbLsvKmUFSX0j6tIxlGgiwC/bWyno6uP1Tq6RbKYAl0EqK1rYVxxPu+ar/MjJHsp0GXUO943wLqt+1l5wXSK8jVuqGQvBbqMer/efpCuvgGdTCRZT4Euo96aumamjC3iMo0bKllOgS6j2pHufp5raOM9F51DKE8DWUh2U6DLqPbk1lb6BiIayEJyggJdRrXa+haqJ5dwQeX4oEsROWsKdBm1Dh7t4Xc7O1h1scYNldygQJdR6xebWnFHR7dIzlCgy6hVW9fMospxnDulLOhSRJJCgS6j0mvtXdQ3HWH1RRo3VHKHAl1GpV/Ut2AGN2jcUMkhCnQZddydx+qaWVo9ienjxwRdjkjSKNBl1NnacpRdbV2svljdLZJbFOgy6tTWt1AQMq5fpIEsJLco0GVUiUScX9S38IdzK5hYWhh0OSJJpUCXUeV/GttpPdLDKp3qLzlIgS6jhrvz1acaOGd8Mdedr+4WyT0KdBk1frm5lU1NR7jr2vkUF2ggC8k9CnQZFfoHInxlXQMLpo3lxsU6ukVykwJdRoWHXtrLno5u/m7FAl33XHKWAl1yXmdvmG/86lUumz2JKzUItOQwBbrkvO8+v4v2zj4+u/I8XSZXcpoCXXJa27FevvubXay8YBoXz5wQdDkiKaVAl5z2jV+9Sl84wmeuWxB0KSIpp0CXnPVaexcPvbSXW5dWMbu8NOhyRFJOgS4566vrGijMz+OTy+YGXYpIWiQU6Ga2wswazKzRzO4eYv1dZrbNzDaZ2a/MbFbySxVJXN2+w/xycysffeccKsYWBV2OSFqMGOhmFgLuA64HFgK3mtnCQc1eBmrc/ULgv4EvJ7tQkUS5O1964hXKywr56B/OCbockbRJZA99KdDo7rvcvQ94GFgd38Ddn3X37tjsC8CM5JYpkrjndrTxwq7X+eSyuZQV5QddjkjaJBLolcC+uPmm2LLh3A48MdQKM7vDzDaY2Ya2trbEqxRJ0EDEufeJ7cyaXMItl1YFXY5IWiX1R1Ez+wBQA3xlqPXufr+717h7TUWFztiT5Hv05Wa27z/GZ66bT2G+fvOX0SWR/0ebgZlx8zNiy97EzK4BPge8y917k1OeSOJ6+gf42lMNXDhjPCsXafBnGX0S2YVZD8w1s9lmVgjcAtTGNzCzxcB/AKvc/WDyyxQZ2Y/+dzctR3q4+/oF5OkCXDIKjRjo7h4G7gTWAa8Aj7j7VjP7gpmtijX7ClAG/JeZ1ZlZ7TBPJ5ISR7r7ue/ZnbxrXgVvf1t50OWIBCKhQwDcfS2wdtCye+Kmr0lyXSKn5Vv/r5GjPf383Qqd4i+jl341kqzXcvg4P/ztbm68uJKF54wLuhyRwCjQJev969M7wOGua+cFXYpIoBToktUa9h/jZxub+LMrZjFjYknQ5YgESoEuWe3LT26ntCifj191btCliAROgS5Z68VdHfxq+0H+4sq3MbG0MOhyRAKnQJes5O586cntTBtXzEfeMTvockQyggJdstK6rft5ee9h/nr5XIoLQkGXI5IRFOiSdfoHInz5yQbmTinj5iW6sKfICQp0yTqPbNjHrvYu/nbFAvJD+giLnKBvg2SVrt4w//bMq1xaPZFrzpsSdDkiGUWBLlnj9a4+Pvj9F2nv7OXu68/DTBfgEomn4VwkK+zt6Oa2H75E0+HjfOv9S7hk1sSgSxLJOAp0yXibm47w4Qdeon/A+emfX0ZN9aSgSxLJSAp0yWjPNhzk4z/ZyMSSQh6+YynnTikLuiSRjKVAl4z1yPp9fPbRzSyYNpYf3nYpU8YVB12SSEZToEvGcXe+/qtX+bdnXuWdc8v59gcuoaxIH1WRkehbIhklPBDh7x/bwsPr93Hzkhl86eYLKNCx5iIJUaBLxujqDXPnTzfybEMbn7j6XO5aPk+HJoqcBgW6ZIS2Y73c/uB6tjQf4Z9vXMSfXjYr6JJEso4CXQK3q62T2364noPHerj/gzVcs3Bq0CWJZCUFugRq495D3P7AesyMh++4gotnTgi6JJGspUCXwDy97QCfeGgjU8cV8+CHl1JdXhp0SSJZTYEugfjxC3v4xzVbuKByPN+/7VLKy4qCLkkk6ynQJa3CAxG+9vQOvvXcTpYtmMK/v38xJYX6GIokg75JknLuztaWozz6cjO19S20Hevl1qVVfHH1+bqeuUgSKdAlZVqPHGdNXQuPbmym4cAxCkLGVfOn8EeXzGD5wqk6xlwkyRToklSdvWGe3LKfR19u4nc7O3CHJVUT+OJ7F3HDBdOZWFoYdIkiOUuBLmctPBDhN43tPPZyM+u27qenP0LVpBI+efVcblxcqaNXRNJEgS5n5ES/+M83RvvF2zt7GT+mgJuXzOCmJZUsqZqoLhWRNFOgS0IGIk7L4ePs6eimvukwj73czKsHOykIGVcvmMKNi2dw1YIKivJDQZcqMmop0OWkvnCEpkPd7OnoZndHF3s6utkTu993qJv+AT/Z9pJZE/mn9y7ihgunM6FE/eIimSChQDezFcDXgRDwPXf/0qD1RcCPgEuADuBP3H13ckuVsxGJOF19Ybp6Bzh8vO9NYX0iwFsOHyfyRmZTWhhi1uRS5k8by7XnT6N6cglVk0t4W0UZUzXYhEjGGTHQzSwE3AcsB5qA9WZW6+7b4prdDhxy93PN7BbgXuBPUlFwtnN3Ig79AxHCESc8EKF/wBmI+FuWhSOx+9jy/oEI4YHofWdvmM7eMF29YTp7B+js7aerdyC6vCdMV1/4jeneMF19A0PWM7GkgKrJpVwyayI3LZnBrEklVJeXUDWplPKyQvWDi2SRRPbQlwKN7r4LwMweBlYD8YG+Gvh8bPq/gW+ambm7k2SPrN/H/b/ZdXJ+8EsM+YL+1tkTj4tOn1jub0yfvB+6XcSj69whEgvpSGzeB81H3HHemE+2wlAeZcX5lBaFKC3MZ2xxPpNKC5k5qYSxRfmUFuVTFruVFuUzfkwBMyeNYdakUsaXFCS/IBEJRCKBXgnsi5tvAi4bro27h83sCDAZaI9vZGZ3AHcAVFVVnVHBE0sLmT917JsX2ilnT7z2W9qcWGRx6y3uCQzD7I3ni05H5/LyouvyDPLMyDM7uT7P4tdHHxFtE32dPDPyQ0ZByMjPyyM/7v7EsoI3LcsjP8/ID0WXF4TyToZzaVFIP0SKCJDmH0Xd/X7gfoCampoz2lddvnAqy3W9bBGRt0jkQhrNwMy4+RmxZUO2MbN8YDzRH0dFRCRNEgn09cBcM5ttZoXALUDtoDa1wIdi038E/DoV/eciIjK8EbtcYn3idwLriB62+AN332pmXwA2uHst8H3gx2bWCLxONPRFRCSNEupDd/e1wNpBy+6Jm+4B/ji5pYmIyOnQxahFRHKEAl1EJEco0EVEcoQCXUQkR1hQRxeaWRuw5wwfXs6gs1AzjOo7O6rv7GV6jarvzM1y94qhVgQW6GfDzDa4e03QdQxH9Z0d1Xf2Mr1G1Zca6nIREckRCnQRkRyRrYF+f9AFjED1nR3Vd/YyvUbVlwJZ2YcuIiJvla176CIiMogCXUQkR2RsoJvZH5vZVjOLmFnNoHWfNbNGM2sws+uGefxsM3sx1u4/Y5f+TVWt/2lmdbHbbjOrG6bdbjPbHGu3IVX1DPG6nzez5rgaVw7TbkVsmzaa2d1prO8rZrbdzDaZ2aNmNmGYdmndfiNtDzMrir33jbHPWnWqa4p77Zlm9qyZbYt9Tz41RJsrzexI3Pt+z1DPlcIaT/l+WdQ3Yttvk5ktSWNt8+O2S52ZHTWzvxrUJtDtd0ai42Jm3g04D5gPPAfUxC1fCNQDRcBsYCcQGuLxjwC3xKa/A/xFmur+F+CeYdbtBsoD2JafBz49QptQbFvOAQpj23hhmuq7FsiPTd8L3Bv09ktkewB/CXwnNn0L8J9pfE+nA0ti02OBHUPUdyXweLo/b4m+X8BK4AmiozxeDrwYUJ0hYD/RE3YyZvudyS1j99Dd/RV3bxhi1WrgYXfvdffXgEaiA1mfZNEBQq8mOmA1wIPAe1NZb9zrvg94KNWvlQInBwN39z7gxGDgKefuT7l7ODb7AtFRsYKWyPZYTfSzBdHP2jIbPHhtirh7q7tvjE0fA14hOrZvNlkN/MijXgAmmNn0AOpYBux09zM9cz1jZGygn8JQg1YP/iBPBg7HhcRQbVLhncABd391mPUOPGVmv48NmJ1Od8b+rf2BmU0cYn0i2zUdPkJ0r20o6dx+iWyPNw2ODpwYHD2tYl09i4EXh1h9hZnVm9kTZnZ+Wgsb+f3KlM/cLQy/Exbk9jttaR0kejAzewaYNsSqz7n7mnTXcyoJ1norp947/wN3bzazKcDTZrbd3Z9PdX3At4EvEv2CfZFot9BHkvG6iUpk+5nZ54Aw8JNhniZl2y9bmVkZ8DPgr9z96KDVG4l2I3TGfjd5DJibxvIy/v2K/ba2CvjsEKuD3n6nLdBAd/drzuBhiQxa3UH037f82J7TUG1Oy0i1WnRw7JuAS07xHM2x+4Nm9ijRf+uT8gFPdFua2XeBx4dYlch2PWMJbL/bgBuAZR7rwBziOVK2/YZwOoOjN1kAg6ObWQHRMP+Ju/988Pr4gHf3tWb2LTMrd/e0XHQqgfcrpZ+5BF0PbHT3A4NXBL39zkQ2drnUArfEjjCYTfQv5kvxDWKB8CzRAashOoB1qvf4rwG2u3vTUCvNrNTMxp6YJvpD4JYU13TiteP7JW8c5nUTGQw8VfWtAP4WWOXu3cO0Sff2y+jB0WN99d8HXnH3rw3TZtqJPn0zW0r0+56WPzgJvl+1wJ/Fjna5HDji7q3pqC/OsP9VB7n9zljQv8oOdyMaPE1AL3AAWBe37nNEj0BoAK6PW74WOCc2PYdo0DcC/wUUpbjeB4CPDVp2DrA2rp762G0r0a6GdG3LHwObgU1Ev0TTB9cXm19J9GiJnWmur5FoX2pd7PadwfUFsf2G2h7AF4j+4QEojn22GmOftTlp3GZ/QLQLbVPcdlsJfOzE5xC4M7at6on+2Pz2NNY35Ps1qD4D7ott383EHc2WphpLiQb0+LhlGbH9zvSmU/9FRHJENna5iIjIEBToIiI5QoEuIpIjFOgiIjlCgS4ikiMU6CIiOUKBLiKSI/4/sutj6ozHdTYAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot([x for x in range(-10, 10)], [sigmoid(x) for x in range(-10, 10)]);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# The derivative of the Sigmoid activation function\n",
    "def d_sigmoid(x: float) -> float:\n",
    "    return sigmoid(x) * (1 - sigmoid(x))\n",
    "\n",
    "assert d_sigmoid(0) == 0.25"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAD4CAYAAADiry33AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO3deXhcd33v8fd3RhrJ2iVLtrV6ibd4kS1ZmBAgBLLgxCZOIQaH9rnQ0ifQNlAebmnD5T7QJ/ReoKG9YUkL6YXblhZCHAgYx4mTkIQAaYK1WPJuy5t2W7JWS5ZGmvndP+bIGU9G0kiamTPL9/U8E8+cc2bmmzNHnznzO+d3fmKMQSmlVOJy2F2AUkqpyNKgV0qpBKdBr5RSCU6DXimlEpwGvVJKJbgUuwsIVFhYaJYtW2Z3GUopFVfq6up6jDFFwebFXNAvW7aM2tpau8tQSqm4IiIXppqnTTdKKZXgNOiVUirBadArpVSC06BXSqkEp0GvlFIJLqSgF5FtInJSRJpF5KEg8z8nIsdEpElEfiUiS/3meUTkkHXbG87ilVJKzWzG0ytFxAk8BtwBtAEHRWSvMeaY32INQI0xZkRE/gz4e+Aj1ryrxpjNYa5bKaVUiELZo98KNBtjzhpj3MATwE7/BYwxLxtjRqyHrwNl4S1Tqdj38olLnL44ZHcZSr1FKEFfCrT6PW6zpk3lE8Czfo/TRaRWRF4XkXuDPUFEHrCWqe3u7g6hJKViS/+Im0/+sI7/8fRhu0tR6i3CejBWRP4IqAEe8Zu81BhTA3wUeFREbgh8njHmcWNMjTGmpqgoaA9epWLaLw514PZ4OXi+j3M9w3aXo9R1Qgn6dqDc73GZNe06InI78EXgHmPM2OR0Y0y79e9Z4BWgah71KhWT9tS1snRhBg6Bp+paZ36CUlEUStAfBFaJyHIRcQG7gevOnhGRKuB7+EL+kt/0fBFJs+4XAu8E/A/iKhX3jncOcqR9kD++eRm3rC7ip3XteLw6RKeKHTMGvTFmAngQOAAcB540xhwVkYdF5B5rsUeALGBPwGmUNwK1ItIIvAx8LeBsHaXi3p7aNlxOBzs3l7JrSzldg6P8trnH7rKUuiakq1caY/YD+wOmfcnv/u1TPO81YON8ClQqlrknvPz8UDu3r1tEfqaL29ctIi8jlT21rbxntR5vUrFBe8YqNQ8vnbhI77CbXVt8h7HSUpzs3FTC88cu0j/itrk6pXw06JWahz21bSzKTuPdqwqvTdtVU457wsvexg4bK1PqTRr0Ss3RpaFRXjnVzQery0hxvvmntL4kh7VLstlT22ZjdUq9SYNeqTl6ut53ds2umus7gosIH64p53D7ACe6Bm2qTqk3adArNQfGGPbUtbFlaT43FGW9Zf69VaWkOkX36lVM0KBXag4OtfbTfOkKu7YEv6xTQaaL29Yu5ucN7Yx7vFGuTqnradArNQd76tpIT3WwvbJ4ymV21ZRxedjNSycuTbmMUtGgQa/ULF11e/jloQ7u3lBMdnrqlMu9Z3URRdlp2nyjbKdBr9QsHTjaxdDYBPfVTH817hSngw9WlfLyyUt0D41Nu6xSkaRBr9Qs7alrpSx/ATctXzjjsrtqyvB4DT9veMt1AJWKGg16pWahrW+E185c5r4tZTgcMuPyKxdls7k8jydrWzFGL3Sm7KFBr9Qs/LTOt2d+3xRn2wTz4ZpyTl+6QmPbQKTKUmpaGvRKhcjrNTxV38rNNyykLD8j5Oft2FRMeqqDPbV6nXplDw16pUL0xrleWnuvXruAWahy0lPZtn4Jexs7GB33RKg6paamQa9UiPbUtZKdlsL71y+Z9XN31ZQzNDrBgaNdEahMqelp0CsVgqHRcZ493MWOTSUscDln/fx3rFhIad4CnqrTc+pV9GnQKxWCZ5o6uTruecsFzELlcAgf2lLGb5t7aO+/GubqlJqeBr1SIdhT18YNRZlUlefN+TV2bSnDGPiZ7tWrKNOgV2oGZ7qvUHehj1015YjMfO78VMoLMrhpRQFP1bfpOfUqqjTolZrBU3VtOB3CB6tK5/1aH64p58LlEX5/rjcMlSkVGg16pabh8Rp+Vt/GrauLWJSTPu/Xu2tDMVlpKezR5hsVRRr0Sk3j1dPdXBwcm/NB2EALXE52VBbzTFMnV8YmwvKaSs1Eg16paTxV20ZBpov3rV0cttfcVVPG1XEP+5s6w/aaSk1Hg16pKfQNu3nh2EV2bi7BlRK+P5XqinxWFGWyp04viaCiQ4NeqSn84lA7bo931pc8mImIcN+WMg6e7+Ncz3BYX1upYDTolZrCnro21pfksK4kJ+yv/aHqMhwCT+levYoCDXqlgjjWMcjRjsEpB/+er8U56dyyuoif1rXj8eo59SqyNOiVCmJPXSsup4Odm+d/7vxUPlxTTtfgKL9t7onYeygFGvRKvYV7wssvDnVwx7rF5Ge6IvY+t924iLyMVL1OvYo4DXqlArx04iK9w+4ZB/+er7QUJ/duLuX5YxfpH3FH9L1UctOgVyrAnto2Fuekccuqooi/131bynBPeNnb2BHx91LJK6SgF5FtInJSRJpF5KEg8z8nIsdEpElEfiUiS/3mfUxETlu3j4WzeKXC7dLgKC+fvMQHq8twhjD493xtKM3lxuIc9tTqJRFU5MwY9CLiBB4D7gLWAfeLyLqAxRqAGmNMJfAU8PfWcwuALwNvB7YCXxaR/PCVr1R4/ayhHa8hYmfbBLNrSxmH2wc40TUYtfdUySWUPfqtQLMx5qwxxg08Aez0X8AY87IxZsR6+Dow+VfyfuAFY0yvMaYPeAHYFp7SlQovYwx7alvZsjSfFUVZUXvfe6tKSXWK7tWriAkl6EsB/9MC2qxpU/kE8OxsnisiD4hIrYjUdnd3h1CSUuHX0NrPme7hqO7NAxRkurht7WJ+3tDOuMcb1fdWySGsB2NF5I+AGuCR2TzPGPO4MabGGFNTVBT5A2BKBbOnto30VAfbK4uj/t67asq4POzmpROXov7eKvGFEvTtgP/FPsqsadcRkduBLwL3GGPGZvNcpezm9RqePdLJtvVLyE5Pjfr7v2d1EYVZLp7RK1qqCAgl6A8Cq0RkuYi4gN3AXv8FRKQK+B6+kPffJTkA3Cki+dZB2DutaUrFlLM9w/SPjHPzykJb3j/F6eDtyxdS39Jny/urxDZj0BtjJoAH8QX0ceBJY8xREXlYRO6xFnsEyAL2iMghEdlrPbcX+Aq+L4uDwMPWNKViymTAVlfMffDv+aqqyKOt7yqXhkZtq0ElppRQFjLG7Af2B0z7kt/926d57g+AH8y1QKWioaGlj5z0FFYURu9sm0BVFb4zj+sv9LNtwxLb6lCJR3vGKgU0tPSzuSIfRxQ6SU1lfUkOqU6hoVWbb1R4adCrpDc0Os7Ji0O2NtsApKc6WV+SS8OFflvrUIlHg14lvaa2AYzxDfFnt+qKfJra+/V8ehVWGvQq6dVf8DWVbCq3d48efAdkR8e9nOgcsrsUlUA06FXSq2/pY9WiLHIXRP/8+UDVS60DsnqapQojDXqV1IwxNLT2U2Vz+/ykktx0FmWn0aBBr8JIg14ltXNWR6lYaJ8HEBGqK/Kpb9EDsip8NOhVUpsM1Mkmk1hQvTSPlt4Req6MzbywUiHQoFdJraGlj+y0FFZG8bLEM5nsONWge/UqTDToVVKrb+lnc0WerR2lAm0szSXFIXpAVoWNBr1KWsNjE5zsGqQqBk6r9Jee6mRdSY4ekFVho0GvklZjWz9eA1Ux1D4/qboin8bWASa045QKAw16lbQm28BjbY8efB2nro57ONGlHafU/GnQq6TV0NLHiqJM8jJcdpfyFpOneza06gFZNX8a9CopGWOob+mPmfPnA5XlL6AwK42GC9pOr+ZPg14lpZbeEXqH3THTIzaQiFBVkad79CosNOhVUnpzRKnY3KMHX23neobpHXbbXYqKcxr0KinVX+gn0+Vk9eJsu0uZ0uT18fU0SzVfGvQqKTW09rGpPA9nDHWUCrSxLBenQ7SHrJo3DXqVdEbcExzvHIrpZhuADFcKNxZnaw9ZNW8a9CrpHG4bwOM1MXsg1l9VeT6Nrf14vMbuUlQc06BXSWfyipVVMb5HD74rWQ67PZy6qB2n1Nxp0KukU9/Sx/LCTAoyY6+jVKDJ5iVtvlHzoUGvkooxhoaW/pi87EEwFQUZFGS69ICsmhcNepVU2vqu0nNlLCYvZBaMb8SpPN2jV/OiQa+SymRgxssePfiOJZztHqZ/RDtOqbnRoFdJpaGlnwyXk7VLYrejVKDJs4P0cghqrjToVVKpb+mjsiyXFGf8bPqbyvJwCHqBMzVn8bO1KzVPo+MejnUMxsVplf4y01JYsyRH9+jVnGnQq6RxuH2ACa+J+R6xwVRX5HGopR+vdpxScxBS0IvINhE5KSLNIvJQkPm3iEi9iEyIyH0B8zwicsi67Q1X4UrN1uTFweKhR2ygqop8hsYmaO6+YncpKg6lzLSAiDiBx4A7gDbgoIjsNcYc81usBfg48FdBXuKqMWZzGGpVal7qL/RTUZBBYVaa3aXM2uSVLOsv9MX0FTdVbAplj34r0GyMOWuMcQNPADv9FzDGnDfGNAE6krGKSb4RpfquBWa8WV6YSV5Gqp5Pr+YklKAvBVr9HrdZ00KVLiK1IvK6iNwbbAERecBapra7u3sWL61UaDoGRrk0NBZ3B2IniQhV5XnaQ1bNSTQOxi41xtQAHwUeFZEbAhcwxjxujKkxxtQUFRVFoSSVbOovxP6IUjOprsjn9KUrDFwdt7sUFWdCCfp2oNzvcZk1LSTGmHbr37PAK0DVLOpTKiwaWvpJT3Wwtjh+27cnf4006mmWapZCCfqDwCoRWS4iLmA3ENLZMyKSLyJp1v1C4J3AsemfpVT41bf0UVmaR2ocdZQKtKk8FxG9kqWavRm3emPMBPAgcAA4DjxpjDkqIg+LyD0AIvI2EWkDdgHfE5Gj1tNvBGpFpBF4GfhawNk6SkXc6LiHox0DVC2NzwOxk7LTU1mzOPva9fSVCtWMp1cCGGP2A/sDpn3J7/5BfE06gc97Ddg4zxqVmpejHYOMewxV5fHbPj+pqiKPZ5o68XoNjhge71bFlvj9HatUiCY7SsXrqZX+qiryGRyd4GyPdpxSodOgVwmvoaWf0rwFLMpJt7uUebvWcUqbb9QsaNCrhFff0kd1nAw0MpMVhVnkpKdc+5WiVCg06FVC6xy4SufAaEI02wA4HEJVRT71F3SPXoVOg14ltMmepPHaIzaYqoo8Tl0aYmhUO06p0GjQq4RWf6EPV4qDdcU5dpcSNtUV+RgDja0Ddpei4oQGvUpo9S19bCzNxZWSOJv6pvLJA7LaTq9Ckzhbv1IBxiY8HOkYTJj2+Um5C1JZtShLD8iqkGnQq4R1rGMQ94Q3ri9kNpXqinwaWvsxRkecUjPToFcJKxEPxE6qqsijf2Sccz3Ddpei4oAGvUpY9S19lOSmsyQ3/jtKBZrsF6Adp1QoNOhVwmpo6U/IvXmAlUVZZKel6AFZFRINepWQLg6O0t5/NS4HAg+FwyFsrtARp1RoNOhVQrp2IbMEufRBMFUV+ZzsGuTK2ITdpagYp0GvElJDSz8up4P1JYnTUSpQVUUeXgNNbbpXr6anQa8SUn1LH+tLc0hLcdpdSsRUWR2ntPlGzUSDXiUc94SXpraBhBhoZDp5GS5WFGVeG/hcqalo0KuEc6JrkLEJL9VxPnRgKLTjlAqFBr1KOJN7uInYIzZQdUU+vcNuLlwesbsUFcM06FXCaWjtZ3FOGsUJ2FEq0OTpow2t2nyjpqZBrxJOfUsf1RX5iCT+4NmrF2eT6XLqQCRqWhr0KqF0D43R2pu4HaUCOR3CpvI87SGrpqVBrxLKtY5SSdA+P6m6Ip8TXUOMuLXjlApOg14llPqWflKdwobSXLtLiZrqpXl4vIamNh1xSgWnQa8SSkNLH+uKc0hPTdyOUoE2W/0FtOOUmooGvUoYEx6ro1QSNdsAFGS6WF6Yqe30akoa9CphnOga4uq4J2kOxPqrKs+joaVPO06poDToVcJIxgOxk6qW5tNzxU1b31W7S1ExSINeJYz6ln6KstMoy19gdylRNzkAujbfqGA06FXCqLvQR1V5XlJ0lAq0ZnE2GS4ndXqBMxWEBr1KCCe7hmjpHeFdqwrtLsUWKU4H71ixkBePXcTr1XZ6db2Qgl5EtonISRFpFpGHgsy/RUTqRWRCRO4LmPcxETlt3T4WrsKV8vdMUwcOgbs2FNtdim22VxbTMTBKQ6ueZqmuN2PQi4gTeAy4C1gH3C8i6wIWawE+Dvwo4LkFwJeBtwNbgS+LSPIdKVMRZYxhX1MnN61YSFF2mt3l2OaOdYtxpTjY19RhdykqxoSyR78VaDbGnDXGuIEngJ3+CxhjzhtjmgBvwHPfD7xgjOk1xvQBLwDbwlC3Utcc6xzkbM8wOypL7C7FVtnpqdy6uoj9hzu1+UZdJ5SgLwVa/R63WdNCEdJzReQBEakVkdru7u4QX1opn2eaOnE6hG0blthdiu22VxZzcXCMWj0oq/zExMFYY8zjxpgaY0xNUVGR3eWoODLZbHPzDQspyHTZXY7tbr9xMemp2nyjrhdK0LcD5X6Py6xpoZjPc5Wa0eH2AVp6R/hAkjfbTMpMS+F9axex/3AXHm2+UZZQgv4gsEpElouIC9gN7A3x9Q8Ad4pIvnUQ9k5rmlJh8UxTJykO4c71i+0uJWZs31hCz5Ux3jh32e5SVIyYMeiNMRPAg/gC+jjwpDHmqIg8LCL3AIjI20SkDdgFfE9EjlrP7QW+gu/L4iDwsDVNqXmbbLZ596pC8jK02WbS+9YuIsPlZF9Tp92lqBiREspCxpj9wP6AaV/yu38QX7NMsOf+APjBPGpUKqiG1n7a+6/yuTtW211KTFngcnLbjYt57kgXD9+znhRnTByKUzbSLUDFrWeaOnE5HdyhzTZvsaOymN5hN/91VptvlAa9ilNer+GZpk5uWV1ETnqq3eXEnPesLiIrLYV9jdp8ozToVZyqa+mja3CUD2xK3kseTCc91ckd6xbz3NEu3BOB/RhVstGgV3FpX2MHaSkObrtRm22msqOymIGr4/yuucfuUpTNNOhV3PF4DfuPdPHeNYvISgvpfIKk9K5VhWSnp+jZN0qDXsWf35/rpXtojB3abDOttBQn71+/hOePdTE24bG7HGUjDXoVd/Y1dbAg1cn71i6yu5SYt6OymKHRCV49pc03yUyDXsWVCY+X54508b4bF5Hh0mabmbxzZSF5Gak8o9e+SWoa9CquvH62l8vDbj5Qqc02oUh1Oti2fgkvHLvI6Lg23yQrDXoVV/Y1dZDpcnLrGm22CdWOyhKG3R5eOXnJ7lKUTTToVdwY93h57mgXt69bTHqq0+5y4sZNKwpYmOnSs2+SmAa9ihu/a+6hf2Q86UeSmq0Up4NtG5bwq+OXGHFP2F2OsoEGvYob+5o6yU5P4ZbVhXaXEnd2VJZwddzDSye0+SYZadCruOCe8HLgaBd3rltCWoo228zW1uUFFGWn8Yw23yQlDXoVF35zupuh0Ql26Nk2c+J0CHdvWMJLJy5xZUybb5KNBr2KC/uaOsldkMo7V2qzzVzt2FTC2ISXXx2/aHcpKso06FXMGx338MKxi2xbvwRXim6yc7WlIp8lOen8Ui9dnHT0r0bFvF+f6ubK2ATbtdlmXhwO4e6Nxbx6qpvB0XG7y1FRpEGvYt6+pk4KMl3cfMNCu0uJezs2FeP2eHnhqDbfJBMNehXTrro9/Or4RbZtWKJjn4ZBVXkepXkL2KfXvkkq+pejYtrLJy8x4vawY6M224SDiLC9spjfnO5hYESbb5KFBr2KafuaOijMSuPtK7TZJlx2VBYz4TUcONpldykqSjToVcwaHpvgpROXuHvjEpwOsbuchLGxNJeKggx+qc03SUODXsWsX524xOi4l+3abBNWk803r525TO+w2+5yVBRo0KuYta+xg8U5abxtWYHdpSScHZXFeLyG545o800y0KBXMWlodJxXTnVz98ZiHNpsE3brinNYUZipZ98kCQ16FZNePH4R94RXL0kcISLCjspiXj97me6hMbvLURGmQa9i0r7GTkpy06kqz7O7lIS1vbIEr4HnjuglERKdBr2KOQMj47x6upvtldpsE0lrlmSzalEWv9RLFyc8DXoVcw4c62LcY7TZJgp2VJZw8HwvFwdH7S5FRVBIQS8i20TkpIg0i8hDQeanichPrPlviMgya/oyEbkqIoes23fDW75KRM80dVJesIDKsly7S0l42yuLMQb2H9a9+kQ2Y9CLiBN4DLgLWAfcLyLrAhb7BNBnjFkJ/B/g637zzhhjNlu3T4WpbpWg+obd/K65h+0bSxDRZptIW7koi7VLsnXg8AQXyh79VqDZGHPWGOMGngB2BiyzE/g36/5TwG2if6VqDp472sWE1+hIUlH0gU0l1F3oo6P/qt2lqAgJJehLgVa/x23WtKDLGGMmgAFg8uIky0WkQUR+LSLvDvYGIvKAiNSKSG13d/es/gdUYnmmqZNlCzNYX5JjdylJY7LnsTbfJK5IH4ztBCqMMVXA54Afichb/oKNMY8bY2qMMTVFRUURLknFqp4rY7x2pocdldpsE03LCjPZUJqjZ98ksFCCvh0o93tcZk0LuoyIpAC5wGVjzJgx5jKAMaYOOAOsnm/RKjE9e6QLr/ENjqGia0dlCY2t/bT2jthdioqAUIL+ILBKRJaLiAvYDewNWGYv8DHr/n3AS8YYIyJF1sFcRGQFsAo4G57SVaLZ19jBykVZrFmcbXcpSWey+UavaJmYZgx6q839QeAAcBx40hhzVEQeFpF7rMW+DywUkWZ8TTSTp2DeAjSJyCF8B2k/ZYzpDff/hIp/+5o6eONcLx+qLtNmGxuUF2Rw04oCvvvKGdr6dK8+0Ygxxu4arlNTU2Nqa2vtLkNFUcvlEbZ/6zesWpzFTz75DlJ1yEBbXLg8zPZv/ZbV+jnEJRGpM8bUBJunn6SylXvCy6efaEAEvrm7SsPFRksXZvLVD26kvqWfR188ZXc5Koz0r0rZ6hvPn6SxtZ+vf6iS8oIMu8tJeh/YVMLut5XzT6+c4bene+wuR4WJBr2yzcsnL/H4q2f5o5squEtHkYoZX/7AelYWZfHZnxzSSxgnCA16ZYuLg6P89ycbWbskm/+5PfCKGspOC1xOvvPRaoZGx/nck4fwemPrOJ6aPQ16FXUer+GzTxziqtvDdz5aTXqq0+6SVIA1S7L58gfW85vTPXzvVT0jOt5p0Kuoe+zlZv7r7GUe3rmelYuy7C5HTeH+reVsryzmG8+fpO5Cn93lqHnQoFdR9ftzvTz64inu3VzCfVvK7C5HTUNE+OoHN1KSl85nftzAwMi43SWpOdKgV1HTN+zmL59ooKIgg7/7g43aMSoO5KSn8u37q7k4OMrf/LSJWOt3o0KjQa+iwhjD559q5PIVN9/5aDVZaSl2l6RCtLk8j7/etobnjnbxH2+02F2OmgMNehUV/+9353nx+CW+cPdaNpTqyFHx5k/ftYJb1xTxlX3HONYxaHc5apY06FXEHW4b4KvPHuf2Gxfx8ZuX2V2OmgOHQ/jGrk3kLUjlwR/XM+KesLskNQsa9CqiroxN8Okf11OYlcYj923Sdvk4VpiVxqMf2cy5nmG+9IujdpejZkGDXkWMMYYvPn2Ylt4Rvrm7ivxMl90lqXm6eWUhn37vSp6qa+Pphja7y1Eh0qBXEbOnro1fHOrgs7evZuvyArvLUWHymdtWsXVZAf/z6SOc6xm2uxwVAg16FRHNl4b48i+O8o4VC/mL9660uxwVRilOB4/u3kxqioMHf1TP2ITH7pLUDDToVdiNjnt48EcNZLicPLp7M06HtssnmpK8BTxy3yaOdgzy1f0n7C5HzUCDXoXd3z1zjBNdQ3zjw5tYnJNudzkqQu5Yt5g/fucy/vW187xw7KLd5ahpaNCrsHr2cCf/8XoLD9yygveuWWR3OSrCHrprLRtKc/j8U4109F+1uxw1BQ16FTatvSP89U+b2FSex1/ducbuclQUpKU4+fb91YxPePnLJxqY8HjtLkkFoUGvwuJYxyCf/GEdGPj27ipcKbppJYvlhZn8rz/YyMHzffz1T5voHXbbXZIKoBccUfPS1jfCPz5/iqcPtZOTnsqjuzdTsVCHBEw291aVcuriEN/99RleOHqRT916A3/yzuUscOlYA7FAYu1qdDU1Naa2ttbuMtQMBkbGeeyVZv71tfMA/PE7l/Hn71lJbkaqvYUpW52+OMTXnzvJi8cvsjgnjc/dsZoPVZeRooO+R5yI1BljaoLO06BXszE67uHfXjvPYy83MzQ2wYeqy/jcHaspyVtgd2kqhhw838v/3n+chpZ+Vi3K4m+2reW2GxfpJTAiSINezZvHa/h5Qzv/8PxJOgZGuXVNEX+zbS03FufYXZqKUcYYnjvSxd8fOMm5nmG2LivgC3evpaoi3+7SEpIGvZozYwy/PtXN1549wYmuISrLcnnorrXcfEOh3aWpODHu8fLEwVa++eJpeq6McdeGJXz+/WtYUaTDSIaTBr2ak8nLC7925jIVBRl8/v1r2L6xGIf2dFVzMDw2wb/85iyPv3oW94SX+7dW8JnbVlGUnWZ3aQlBg17NSmvvCI8cOMnexg4KMl18+n0r+cO3L9VTJlVYdA+N8a1fneZHv28hPcXBA7fcwJ++ezmZOurYvGjQq5D0Drv59kun+Y/XL+B0CH/6rhV88j0ryE7XM2lU+J3tvsIjB07y7JEuCrPS+Oztq/jI28pJ1TN05kSDXr3FuMfLya4hmtoGaGrrp7FtgFMXhzDG8OGacj57+2qW5Op1alTk1V3o42vPHufg+T4yXU7Wl+ayqSyXyrI8NpXlUV6wQM/WCYEGfZLzeg1ne67Q2PpmqB/rHMQ94euunpeRysbSXDaV5bFzcwmrFmfbXLFKNsYYXjnVzSsnLk27fVaW5bKpPE8vlheEBn0SMcbQ1neVxrZ+mtoGaGzt50j7AMNu3zXDM1xONugek4pxU/3i9Hh9eYCXs9cAAAjuSURBVLU4J42NpXm+7bg8j8rS3KQfwWzeQS8i24BvAk7g/xpjvhYwPw34d2ALcBn4iDHmvDXvC8AnAA/wGWPMgeneS4P+rYwxXB330DcyTt+wm95hN30jbt99a1rfiJvLV9yc6Bqkb2QcAJfTwY0lOX6hnsuKoiy9PryKS6PjHo52DNI0uRPT1s/Z7jdHuKooyGBFUSb5GS7yM1wUZKaSl+GiINP3OD8zlYIMF3kZroQ8sWC6oJ/xMLeIOIHHgDuANuCgiOw1xhzzW+wTQJ8xZqWI7Aa+DnxERNYBu4H1QAnwooisNsbE9ZA0xhi8xteJyGt8N4/X4PWC2+P13SZ8t7EJj999321yvv+8yfkDV8d9IT7ipnd4nP4RX7CPTQS/KqAI5C6Y3IBTuXPdEirLfT9zVy/OTsgNWiWn9FQnW5bms2Xpmx2uBkfHOdI+cG3Pv63vKs2XrtA/Ms6VsYkpXysrLeVa8OdPfhFkuMhKTyEtxYHL6cCV4vDdn7w5HaSlOq+bFzg/xenA6RCcIohw7b7dpySHcj7TVqDZGHMWQESeAHYC/kG/E/hb6/5TwHfE1xawE3jCGDMGnBORZuv1/is85b+pf8TNh/75NQCM9R+DL5R9/4LB+P61fsQEm+c1BIS3FerGd99jDJFq7XJYoZ1vbXyleQvYUJLj2yPJdJGfkWrtqbiu7ankLkjVPXSVtHLSU7n5hsKgHfjGJjz0j4xbO01u+obH/X4Ju+kfGbemuznTfYW+4em/HObLYQW/Q3w3331w+H0ZVJbm8v2Pvy3s7x1K0JcCrX6P24C3T7WMMWZCRAaAhdb01wOeWxr4BiLyAPAAQEVFRai1X8fpENYuyQEr88T3uta/1z9GQBC/6X6PRXA6sL6RfR+G02F9O8vkfbHu+z4kh7z5Qbmccu0bPi3FGXTPIG1y3rW9hDf3BpRS4ZGW4mRxjnNWB26NMYx7DG6Pl7Fxj9+v78B/rV/jHi9j42/+Sh/3eK2dRGuH0do59Br87lvTvQE7lQbK8iNzzaiY6KFgjHkceBx8bfRzeY3s9FQe+8PqsNallEouIoIrxbezlpVAHbhC2YVsB8r9HpdZ04IuIyIpQC6+g7KhPFcppVQEhRL0B4FVIrJcRFz4Dq7uDVhmL/Ax6/59wEvGdzrPXmC3iKSJyHJgFfD78JSulFIqFDP+NrHa3B8EDuA7vfIHxpijIvIwUGuM2Qt8H/ihdbC1F9+XAdZyT+I7cDsB/EW8n3GjlFLxRjtMKaVUApjuPHo9zUMppRKcBr1SSiU4DXqllEpwGvRKKZXgYu5grIh0Axfm8RKFQE+YyokErW9+tL750frmJ5brW2qMKQo2I+aCfr5EpHaqI8+xQOubH61vfrS++Yn1+qaiTTdKKZXgNOiVUirBJWLQP253ATPQ+uZH65sfrW9+Yr2+oBKujV4ppdT1EnGPXimllB8NeqWUSnBxGfQisktEjoqIV0RqAuZ9QUSaReSkiLx/iucvF5E3rOV+Yl1+OVK1/kREDlm38yJyaIrlzovIYWu5qF3VTUT+VkTa/Wq8e4rltlnrtFlEHopifY+IyAkRaRKRp0Ukb4rlorr+Zlof1qW5f2LNf0NElkW6Jr/3LheRl0XkmPV38pdBlrlVRAb8PvcvRas+6/2n/bzE51vW+msSkaiNKiQia/zWyyERGRSRzwYsY+v6mzVjTNzdgBuBNcArQI3f9HVAI5AGLAfOAM4gz38S2G3d/y7wZ1Gq+x+AL00x7zxQaMO6/Fvgr2ZYxmmtyxWAy1rH66JU351AinX/68DX7V5/oawP4M+B71r3dwM/ieJnWgxUW/ezgVNB6rsV2Bft7S3Uzwu4G3gW32ifNwFv2FSnE+jC1xkpZtbfbG9xuUdvjDlujDkZZNa1wciNMeeAycHIr7EGLX8fvkHMAf4NuDeS9fq974eBH0f6vSLg2gDxxhg3MDlAfMQZY543xkyO2Pw6vlHK7BbK+tiJb9sC37Z2m7UNRJwxptMYU2/dHwKOE2Ss5hi3E/h34/M6kCcixTbUcRtwxhgzn976tovLoJ9GsIHMAzfwhUC/X3gEHbA8At4NXDTGnJ5ivgGeF5E6a7D0aHrQ+nn8AxHJDzI/lPUaDX+Cby8vmGiuv1DWx7VlrG1tAN+2F1VWk1EV8EaQ2e8QkUYReVZE1ke1sJk/r1jZ5nYz9c6ZnetvVmJ29FsReRFYEmTWF40xv4h2PdMJsdb7mX5v/l3GmHYRWQS8ICInjDGvRro+4J+Br+D7w/sKvualPwnH+4YqlPUnIl/EN0rZf07xMhFbf/FKRLKAnwKfNcYMBsyux9ccccU6LvNzfEN9RkvMf17Wsbt7gC8EmW33+puVmA16Y8ztc3haKIORX8b3MzDF2tOa94DlM9UqvgHTPwhsmeY12q1/L4nI0/iaB8Ky4Ye6LkXkX4B9QWZFdJD3ENbfx4EdwG3GaiAN8hoRW39BhLI+Jpdpsz7/XHzbXlSISCq+kP9PY8zPAuf7B78xZr+I/JOIFBpjonLBrhA+r4hucyG6C6g3xlwMnGH3+putRGu6mXEwcisoXsY3iDn4BjWP9C+E24ETxpi2YDNFJFNEsifv4zsAeSTCNU2+t3+75x9M8b6hDBAfqfq2AX8N3GOMGZlimWivv1DWx1582xb4trWXpvqSCjfrWMD3gePGmH+cYpklk8cMRGQrviyIyhdRiJ/XXuC/WWff3AQMGGM6o1Gfnyl/hdu5/ubE7qPBc7nhC6Q2YAy4CBzwm/dFfGdEnATu8pu+Hyix7q/A9wXQDOwB0iJc778CnwqYVgLs96un0bodxddkEa11+UPgMNCE74+rOLA+6/Hd+M7eOBPl+prxtdUesm7fDazPjvUXbH0AD+P7QgJIt7atZmtbWxHFdfYufE1xTX7r7W7gU5PbIfCgta4a8R3kvjmK9QX9vALqE+Axa/0exu/suijVmIkvuHP9psXE+pvLTS+BoJRSCS7Rmm6UUkoF0KBXSqkEp0GvlFIJToNeKaUSnAa9UkolOA16pZRKcBr0SimV4P4/WbjdKqi+Hz0AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot([x for x in range(-10, 10)], [d_sigmoid(x) for x in range(-10, 10)]);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Sum of squared errors function\n",
    "# See: https://en.wikipedia.org/wiki/Residual_sum_of_squares\n",
    "def sum_squared_error(ys: List[float], ys_pred: List[float]) -> float:\n",
    "    return sum((y - y_pred) ** 2 for y, y_pred in zip(ys, ys_pred))\n",
    "\n",
    "assert sum_squared_error([0, 1], [0, 1]) == 0\n",
    "assert sum_squared_error([0, 0], [1, 1]) == 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Derivative of the squared error function\n",
    "# NOTE: The squared error function is expressed as `(y - y_pred) ** 2`\n",
    "def d_squared_error(y: float, y_pred: float) -> float:\n",
    "    return -2 * (y - y_pred)\n",
    "\n",
    "assert d_squared_error(1, 4) == 6\n",
    "assert d_squared_error(1, 1) == 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--- Before training ---\n",
      "Prediction for [0, 0] (should be 0): 0.59\n",
      "Prediction for [0, 1] (should be 1): 0.67\n",
      "Prediction for [1, 0] (should be 1): 0.76\n",
      "Prediction for [1, 1] (should be 1): 0.81\n",
      "\n",
      "Training Perceptron for 5000 epochs with Learning Rate 0.1\n",
      "\n",
      "--- After training ---\n",
      "Prediction for [0, 0] (should be 0): 0.05\n",
      "Prediction for [0, 1] (should be 1): 0.97\n",
      "Prediction for [1, 0] (should be 1): 0.97\n",
      "Prediction for [1, 1] (should be 1): 1.00\n"
     ]
    }
   ],
   "source": [
    "# Using a Perceptron to \"learn\" OR\n",
    "# See: https://en.wikipedia.org/wiki/Logical_disjunction\n",
    "xs: List[List[float]] = [[0, 0], [0, 1], [1, 0], [1, 1]]\n",
    "ys: List[float] = [0, 1, 1, 1]\n",
    "\n",
    "# Creating a single `Neuron` (with a `sigmoid` activation function)\n",
    "perceptron_or: Neuron = Neuron(sigmoid)\n",
    "\n",
    "# Print out what the randomized weights and bias initializations cause the Perceptron to predict\n",
    "print('--- Before training ---')\n",
    "print('Prediction for [0, 0] (should be 0): {num:.{digits}f}'.format(num=perceptron_or.forward([0, 0]), digits=2))\n",
    "print('Prediction for [0, 1] (should be 1): {num:.{digits}f}'.format(num=perceptron_or.forward([0, 1]), digits=2))\n",
    "print('Prediction for [1, 0] (should be 1): {num:.{digits}f}'.format(num=perceptron_or.forward([1, 0]), digits=2))\n",
    "print('Prediction for [1, 1] (should be 1): {num:.{digits}f}'.format(num=perceptron_or.forward([1, 1]), digits=2))\n",
    "\n",
    "# Train the Perceptron\n",
    "epochs: int = 5000\n",
    "learning_rate: int = 0.1\n",
    "\n",
    "print(f'\\nTraining Perceptron for {epochs} epochs with Learning Rate {learning_rate}\\n')\n",
    "    \n",
    "for epoch in range(epochs):\n",
    "    for x, y in zip(xs, ys):\n",
    "        y_pred: float = perceptron_or.forward(x)\n",
    "\n",
    "        d_error_d_y_pred: float = d_squared_error(y, y_pred)\n",
    "\n",
    "        w1: float = perceptron_or.weights[0]\n",
    "        w2: float = perceptron_or.weights[1]\n",
    "        b: float = perceptron_or.bias\n",
    "\n",
    "        sum_p: float = dot([x[0], x[1]], [w1, w2]) + b\n",
    "        d_y_pred_d_w1: float = x[0] * d_sigmoid(sum_p)\n",
    "        d_y_pred_d_w2: float = x[1] * d_sigmoid(sum_p)\n",
    "        d_y_pred_d_b: float = d_sigmoid(sum_p)\n",
    "\n",
    "        perceptron_or.weights[0] += -learning_rate * d_error_d_y_pred * d_y_pred_d_w1\n",
    "        perceptron_or.weights[1] += -learning_rate * d_error_d_y_pred * d_y_pred_d_w2\n",
    "        perceptron_or.bias += -learning_rate * d_error_d_y_pred * d_y_pred_d_b\n",
    "\n",
    "# Print out what the Perceptron predicts after training (updating the weights and biases)\n",
    "print('--- After training ---')\n",
    "print('Prediction for [0, 0] (should be 0): {num:.{digits}f}'.format(num=perceptron_or.forward([0, 0]), digits=2))\n",
    "print('Prediction for [0, 1] (should be 1): {num:.{digits}f}'.format(num=perceptron_or.forward([0, 1]), digits=2))\n",
    "print('Prediction for [1, 0] (should be 1): {num:.{digits}f}'.format(num=perceptron_or.forward([1, 0]), digits=2))\n",
    "print('Prediction for [1, 1] (should be 1): {num:.{digits}f}'.format(num=perceptron_or.forward([1, 1]), digits=2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Our 3 Layer (input, hidden, output) Neural Network\n",
    "class NeuralNetwork:\n",
    "    def __init__(self, activation: Callable):\n",
    "        self._activation: Callable = activation\n",
    "        # `Neuron`s of the hidden layer\n",
    "        self._n1: Neuron = Neuron(self._activation)\n",
    "        self._n2: Neuron = Neuron(self._activation)\n",
    "        # `Neuron` of the output layer\n",
    "        self._n3: Neuron = Neuron(self._activation)\n",
    "        # The most recent x values we've seen\n",
    "        self._x1: float = 0\n",
    "        self._x2: float = 0\n",
    "        # The results from the most recent forward-pass\n",
    "        self._res_n1: float = 0\n",
    "        self._res_n2: float = 0\n",
    "        self._res_n3: float = 0\n",
    "\n",
    "    # Forward-pass the data through the network\n",
    "    def forward(self, x: List[float]) -> float:\n",
    "        # Store a copy of the x values\n",
    "        self._x1: float = x[0]\n",
    "        self._x2: float = x[1]\n",
    "        # Construct the data flow by combining the NN layers\n",
    "        res_n1: float = self._n1.forward(x)\n",
    "        res_n2: float = self._n2.forward(x)\n",
    "        res_n3: float = self._n3.forward([res_n1, res_n2])\n",
    "        # Store the most recent result of every single `Neuron`\n",
    "        self._res_n1: float = res_n1\n",
    "        self._res_n2: float = res_n2\n",
    "        self._res_n3: float = res_n3\n",
    "        # Return the overall result (the prediction)\n",
    "        return res_n3\n",
    "\n",
    "    # Back-propagating weight and bias updates across the network\n",
    "    def backward(self, y: float, lr: float) -> None:\n",
    "        # The most recent x values we've used\n",
    "        x1: float = self._x1\n",
    "        x2: float = self._x2\n",
    "        \n",
    "        # The individual `Neuron` results\n",
    "        n1: float = self._res_n1\n",
    "        n2: float = self._res_n2\n",
    "        n3: float = self._res_n3\n",
    "        y_pred: float = n3\n",
    "        \n",
    "        # The individual `Neuron`s weights and biases\n",
    "        # `Neuron` 1\n",
    "        b1: float = self._n1.bias\n",
    "        w1: float = self._n1.weights[0]\n",
    "        w2: float = self._n1.weights[1]\n",
    "        # `Neuron` 2\n",
    "        b2: float = self._n2.bias\n",
    "        w3: float = self._n2.weights[0]\n",
    "        w4: float = self._n2.weights[1]\n",
    "        # `Neuron` 3 (output)\n",
    "        b3: float = self._n3.bias\n",
    "        w5: float = self._n3.weights[0]\n",
    "        w6: float = self._n3.weights[1]\n",
    "\n",
    "        # The partial derivative for the error function is used in every computation\n",
    "        d_error_d_y_pred: float = d_squared_error(y, y_pred)\n",
    "        \n",
    "        # Calculate the partial derivatives for the individual weights and biases\n",
    "        # `Neuron` 1\n",
    "        sum_n1: float = dot([x1, x2], [w1, w2]) + b1\n",
    "        d_n1_d_w1: float = x1 * d_sigmoid(sum_n1)\n",
    "        d_n1_d_w2: float = x2 * d_sigmoid(sum_n1)\n",
    "        d_n1_d_b1: float = d_sigmoid(sum_n1)\n",
    "        # `Neuron` 2\n",
    "        sum_n2: float = dot([x1, x2], [w3, w4]) + b2\n",
    "        d_n2_d_w3: float = x1 * d_sigmoid(sum_n2)\n",
    "        d_n2_d_w4: float = x2 * d_sigmoid(sum_n2)\n",
    "        d_n2_d_b2: float = d_sigmoid(sum_n2)\n",
    "        # `Neuron` 3 (output)\n",
    "        sum_n3: float = dot([n1, n2], [w5, w6]) + b3\n",
    "        d_y_pred_d_n1: float = w5 * d_sigmoid(sum_n3)\n",
    "        d_y_pred_d_n2: float = w6 * d_sigmoid(sum_n3)\n",
    "        d_y_pred_d_w5: float = n1 * d_sigmoid(sum_n3)\n",
    "        d_y_pred_d_w6: float = n2 * d_sigmoid(sum_n3)\n",
    "        d_y_pred_d_b3: float = d_sigmoid(sum_n3)\n",
    "\n",
    "        # Update the weights and biases\n",
    "        # `Neuron` 1\n",
    "        self._n1.weights[0] += -lr * d_error_d_y_pred * d_y_pred_d_n1 * d_n1_d_w1\n",
    "        self._n1.weights[1] += -lr * d_error_d_y_pred * d_y_pred_d_n1 * d_n1_d_w2\n",
    "        self._n1.bias += -lr * d_error_d_y_pred * d_y_pred_d_n1 * d_n1_d_b1\n",
    "        # `Neuron` 2\n",
    "        self._n2.weights[0] += -lr * d_error_d_y_pred * d_y_pred_d_n2 * d_n2_d_w3\n",
    "        self._n2.weights[1] += -lr * d_error_d_y_pred * d_y_pred_d_n2 * d_n2_d_w4\n",
    "        self._n2.bias += -lr * d_error_d_y_pred * d_y_pred_d_n2 * d_n2_d_b2\n",
    "        # `Neuron` 3 (output)\n",
    "        self._n3.weights[0] += -lr * d_error_d_y_pred * d_y_pred_d_w5\n",
    "        self._n3.weights[1] += -lr * d_error_d_y_pred * d_y_pred_d_w6\n",
    "        self._n3.bias += -lr * d_error_d_y_pred * d_y_pred_d_b3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create a new Neural Network with a `sigmoid` activation function\n",
    "nn: NeuralNetwork = NeuralNetwork(sigmoid)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--- Before training ---\n",
      "Prediction for [0, 0] (should be 0): 0.79\n",
      "Prediction for [0, 1] (should be 1): 0.83\n",
      "Prediction for [1, 0] (should be 1): 0.83\n",
      "Prediction for [1, 1] (should be 0): 0.85\n"
     ]
    }
   ],
   "source": [
    "# Print out what the randomized weights and bias initializations cause the NN to predict\n",
    "print('--- Before training ---')\n",
    "print('Prediction for [0, 0] (should be 0): {num:.{digits}f}'.format(num=nn.forward([0, 0]), digits=2))\n",
    "print('Prediction for [0, 1] (should be 1): {num:.{digits}f}'.format(num=nn.forward([0, 1]), digits=2))\n",
    "print('Prediction for [1, 0] (should be 1): {num:.{digits}f}'.format(num=nn.forward([1, 0]), digits=2))\n",
    "print('Prediction for [1, 1] (should be 0): {num:.{digits}f}'.format(num=nn.forward([1, 1]), digits=2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 0 --> loss: 1.4036893181184023\n",
      "Epoch 5000 --> loss: 0.014574431740830018\n",
      "Epoch 10000 --> loss: 0.004078498610061902\n",
      "Epoch 15000 --> loss: 0.0023138620933954145\n",
      "Epoch 20000 --> loss: 0.0016032268987062385\n",
      "Epoch 25000 --> loss: 0.001222352177475733\n"
     ]
    }
   ],
   "source": [
    "# Train the Neural Network to learn the XOR function\n",
    "# See: https://en.wikipedia.org/wiki/Exclusive_or\n",
    "\n",
    "# XOR function data\n",
    "xs: List[List[float]] = [[0, 0], [0, 1], [1, 0], [1, 1]]\n",
    "ys: List[float] = [0, 1, 1, 0]\n",
    "\n",
    "epochs: int = 30000\n",
    "learning_rate: float = 0.1\n",
    "\n",
    "# This list is used to record the loss at every epoch to plot it later on\n",
    "losses: List[float] = []\n",
    "\n",
    "# This list is used to record our current y predictions\n",
    "ys_pred: List[float] = []\n",
    "\n",
    "for epoch in range(epochs):\n",
    "    # Show the NN the whole data set once\n",
    "    for x, y in zip(xs, ys):\n",
    "        # Let it make a prediction\n",
    "        y_pred: float = nn.forward(x)\n",
    "        # Record the prediction\n",
    "        ys_pred.append(y_pred)\n",
    "        # Do a backward-pass to update the weights and biases\n",
    "        nn.backward(y, lr=learning_rate)\n",
    "\n",
    "    # Calculate and record the loss for every epoch\n",
    "    # so that we can plot it later on\n",
    "    loss: float = sum_squared_error(ys, ys_pred)\n",
    "    losses.append(loss)\n",
    "\n",
    "    # Print the loss every once in a while\n",
    "    if epoch % 5000 == 0:\n",
    "        print(f'Epoch {epoch} --> loss: {loss}')\n",
    "\n",
    "    # Clear the list of predictions after the NN has seen the whole data set\n",
    "    ys_pred.clear()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEGCAYAAABo25JHAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAd60lEQVR4nO3deXgc9Z3n8fdHp+VLtmXZOD6wzZHEBBJAIeRYQm4gWUgWdoBnspsQMs5Fkp3M5AnzZB8my+w+T45nswkzTBgnSwjMBELOx5N1TsLADFcsEnBsM4AwdpAxWDbGt2VL+u4fXbJbcuuw6eruUn1ez6Onq35VXfUttaSPqn7Vv1ZEYGZm+VVX7QLMzKy6HARmZjnnIDAzyzkHgZlZzjkIzMxyrqHaBRyr2bNnx+LFi6tdhplZpjz88MPbIqK91LLMBcHixYvp7OysdhlmZpkiadNIy3xpyMws5xwEZmY55yAwM8s5B4GZWc45CMzMcs5BYGaWc6kFgaSbJW2VtHaM9V4rqU/SZWnVYmZmI0vzjOAW4ILRVpBUD3wJ+GWKdQDw+HO7+eovH2fbnt60d2VmlimpBUFE3Au8MMZqnwR+CGxNq45BT27dzQ2/6eKFvQfT3pWZWaZUrY9A0nzgfcA3xrHuckmdkjp7enqOb38IAH8Oj5nZUNXsLP4a8LmIGBhrxYhYEREdEdHR3l5yqIwxScm2cBKYmRWr5lhDHcAdKvyFng1cJKkvIn6Sxs7qkiAYGDN2zMzypWpBEBFLBqcl3QL8NK0QSPZS2K/PCMzMhkgtCCTdDpwPzJbUDfw10AgQETeltd+R6yk8uo/AzGyo1IIgIq48hnU/mFYdg5T2DszMMio37yxO+iJ8RmBmNkx+giB5dB+BmdlQ+QkC9xGYmZWUmyCoS5JgwElgZjZEboKAw28oMzOzYrkJgsN9BE4CM7Mh8hMEOtJdbGZmR+QnCJJHnxGYmQ2VnyBwH4GZWUn5CQIPQ21mVlJugqDu8PsInARmZsVyEwSDnQQDzgEzsyFyEwTyMNRmZiXlJwh896iZWUn5CYLk0TlgZjZUfoLAw1CbmZWUoyAoPLqPwMxsqNwEQZ2HoTYzKyk3QTDYS+BhqM3MhkotCCTdLGmrpLUjLP9TSWsk/UHS/ZJenVYthf0VHh0DZmZDpXlGcAtwwSjLnwbeHBGnA38DrEixliMfXu8kMDMboiGtDUfEvZIWj7L8/qLZB4EFadUCRXcNOQnMzIaolT6Cq4GfjbRQ0nJJnZI6e3p6jmsHHobazKy0qgeBpLdQCILPjbRORKyIiI6I6Ghvbz/O/Qxu67iebmY2YaV2aWg8JJ0BfAu4MCK2p7mvusOXhszMrFjVzggkLQJ+BPyXiHiiUvv17aNmZkOldkYg6XbgfGC2pG7gr4FGgIi4CbgOaAP+PunI7YuIjvTqKTw6B8zMhkrzrqErx1j+YeDDae1/OHnYOTOzkqreWVwpPiMwMystf0FQ3TLMzGpOfoLAH15vZlZSboKgzsNQm5mVlJsgkD+83syspNwEAYcvDTkJzMyK5SYIDn94vZmZDZGfIEgefUJgZjZUfoLAw1CbmZWUnyBIHn1GYGY2VG6C4PDoow4CM7MhchMER24fdRKYmRXLTRAMcgyYmQ2VmyCQBx81MyspR0Hgu4bMzErJTxAkj+4iMDMbKj9B4GGozcxKyk0QDN4+6ruGzMyGyk0QtLY0AnDr/Zu4+9+3cuBQf5UrMjOrDakFgaSbJW2VtHaE5ZJ0g6QuSWsknZVWLQCTGuu5vGMhG7fv5apbVvOGL/6Gr//6SQeCmeVemmcEtwAXjLL8QuCU5Gs58I0UawHgS5edwaN//U5uueq1nLVoJv/n10/w7hv+lT9u35f2rs3MalZqQRAR9wIvjLLKJcCtUfAgMEPSvLTqGTSpsZ7zXz6Hb32gg1s/dA7b9x7kspvuZ8vO/Wnv2sysJlWzj2A+8EzRfHfSdhRJyyV1Surs6ekpWwHnndrO95a/nn0H+/nobQ/T1z9Qtm2bmWVFJjqLI2JFRHREREd7e3tZt/3yE6bxxUtP59HunXzngU1l3baZWRZUMwg2AwuL5hckbRX37tPncd6p7Xz910+wp7evGiWYmVVNNYNgJfBfk7uHzgV2RsSWahQiic+841R2Hejjuw/5rMDM8iXN20dvBx4AXi6pW9LVkj4q6aPJKquADUAX8E3g42nVMh6vWTiDc5fO4tYHNjEw4DedmVl+NKS14Yi4cozlAXwirf0fj8tfu5A//96jrN74Aq9b2lbtcszMKiITncWV8q7TTmBKUz0/+l1VuirMzKrCQVBkclMDb33lXO769+d9ecjMcsNBMMzbXjGHbXsOsmbzzmqXYmZWEQ6CYd58ajt1gt889ny1SzEzqwgHwTAzpzRx5qKZ3PNE+d7BbGZWyxwEJbzhpDbWPruL3QcOVbsUM7PUOQhKeN2SNvoHgs5NO6pdiplZ6hwEJZx14gwa68WDG7ZXuxQzs9Q5CEqY3NTAqxfM4MENo42ibWY2MTgIRnDu0jbWbt7pfgIzm/AcBCM4d6n7CcwsHxwEI3A/gZnlhYNgBO4nMLO8cBCM4vUnuZ/AzCY+B8Eo3E9gZnngIBjFWYtmFvoJnnI/gZlNXA6CUbQ01fOahTPcYWxmE5qDYAyvX9rGH9xPYGYTmINgDOcubWMgYPVG3z1kZhNTqkEg6QJJj0vqknRtieWLJN0t6feS1ki6KM16jsdZJ85kUmMd9zzuYanNbGJKLQgk1QM3AhcCy4ArJS0bttp/B+6MiDOBK4C/T6ue4zWpsZ43ndzOr9Y/T4Q/vtLMJp40zwjOAboiYkNEHATuAC4Ztk4A05PpVuDZFOs5bu88bS7P7jzAumd3VbsUM7OySzMI5gPPFM13J23FvgC8X1I3sAr4ZKkNSVouqVNSZ09P5S/RvO0Vc6gT/HLdcxXft5lZ2qrdWXwlcEtELAAuAm6TdFRNEbEiIjoioqO9vb3iRbZNbea1i2fxz2u2+PKQmU04aQbBZmBh0fyCpK3Y1cCdABHxADAJmJ1iTcftsrMX8PS2vTzsdxmb2QSTZhCsBk6RtERSE4XO4JXD1vkj8DYASa+kEAQ1eXvORafPY3JTPXd2PjP2ymZmGZJaEEREH3AN8AvgMQp3B62TdL2ki5PV/gL4M0mPArcDH4wavfYypbmB95wxj5+u2cLOfX5zmZlNHA1pbjwiVlHoBC5uu65oej3wxjRrKKer3riEOzu7+ceHNvGJt5xc7XLMzMpiXGcEkk6S1JxMny/pU5JmpFta7XnlvOmcd2o7375vIwcO9Ve7HDOzshjvpaEfAv2STgZWUOgE/m5qVdWwj5y3lG17evnR74b3e5uZZdN4g2Agueb/PuBvI+KzwLz0yqpdbzipjdPnt7Li3qfoH6jJ7gwzs2My3iA4JOlK4APAT5O2xnRKqm2S+Pj5J7Fx+z5+vtZvMDOz7BtvEFwFvB74XxHxtKQlwG3plVXb3nnaCSydPYVv3NPlN5iZWeaNKwgiYn1EfCoibpc0E5gWEV9KubaaVV8nPvLmpazdvIt/69pW7XLMzF6S8d419C+SpkuaBfwO+Kakr6ZbWm1775nzmTu9mX+4Z0O1SzEze0nGe2moNSJ2Af8JuDUiXge8Pb2yal9zQz1XnrOI+57axvO7DlS7HDOz4zbeIGiQNA/4E450Fufee86YRwSs+sOWapdiZnbcxhsE11MYKuKpiFgtaSnwZHplZcPJc6Zxypyp3PXY1mqXYmZ23MbbWfz9iDgjIj6WzG+IiEvTLS0b3njybB7etIODfQPVLsXM7LiMt7N4gaQfS9qafP1Q0oK0i8uCc5fOYv+hftZ0v1jtUszMjst4Lw19m8IQ0i9Lvv45acu9s0+cBcCj3TurXImZ2fEZbxC0R8S3I6Iv+boFqPxHhdWg9mnNtE9rZr0/z9jMMmq8QbBd0vsl1Sdf7we2p1lYliybN53HtjgIzCybxhsEH6Jw6+hzwBbgMuCDKdWUOa+cN50nt+7mUL87jM0se8Z719CmiLg4ItojYk5EvBfwXUOJk+dM5VB/0L1jf7VLMTM7Zi/loyo/U7YqMm5x22QANm3fW+VKzMyO3UsJApWtioxbdDgI9lW5EjOzY/dSgmDM8ZclXSDpcUldkq4dYZ0/kbRe0jpJmfzUs/apzUxuqmejzwjMLING/fB6Sbsp/QdfQMsYz60HbgTeAXQDqyWtTD6wfnCdU4C/At4YETskzTnG+muCJE5sm+IzAjPLpFGDICKmvYRtnwN0RcQGAEl3AJcA64vW+TPgxojYkewvs4P2zJ/RQvcOB4GZZc9LuTQ0lvnAM0Xz3UlbsVOBUyXdJ+lBSRekWE+qTmht5jkPR21mGTTqGUGF9n8KcD6wALhX0ukRMWTgHknLgeUAixYtqnSN43LC9Em8uO8QBw71M6mxvtrlmJmNW5pnBJuBhUXzC5K2Yt3Ayog4FBFPA09QCIYhImJFRHREREd7e22ObDF3+iQAf0iNmWVOmkGwGjhF0hJJTcAVFAauK/YTCmcDSJpN4VJRJj/78YTWQhA8t9NBYGbZkloQREQfcA2FD7R5DLgzItZJul7Sxclqv6AwjtF64G7gsxGRyTGMTkjOCNxPYGZZk2ofQUSsAlYNa7uuaDoovEM58+9SntvqS0Nmlk1pXhrKlWnNDTQ11LF9z8Fql2JmdkwcBGUiibYpTbyw10FgZtniICijmZMdBGaWPQ6CMmqb2sR2B4GZZYyDoIxm+dKQmWWQg6CMZk1pYoeDwMwyxkFQRrMmN7G7t4/evv5ql2JmNm4OgjKaNbUJgB17D1W5EjOz8XMQlFHblEIQuJ/AzLLEQVBGMyc7CMwsexwEZdQ6uRGAnft9acjMssNBUEatLQ4CM8seB0EZOQjMLIscBGXU0lhPU32dg8DMMsVBUEaSmN7S6CAws0xxEJRZa0sDuxwEZpYhDoIya21p5MX9vn3UzLLDQVBmrb40ZGYZ4yAoMweBmWWNg6DMWlsa2bnPQWBm2ZFqEEi6QNLjkrokXTvKepdKCkkdadZTCa0tjezu7WNgIKpdipnZuKQWBJLqgRuBC4FlwJWSlpVYbxrwaeChtGqppOktjUTA7gN91S7FzGxc0jwjOAfoiogNEXEQuAO4pMR6fwN8CTiQYi0V43cXm1nWpBkE84Fniua7k7bDJJ0FLIyI/zfahiQtl9QpqbOnp6f8lZbRjGQEUgeBmWVF1TqLJdUBXwX+Yqx1I2JFRHREREd7e3v6xb0EPiMws6xJMwg2AwuL5hckbYOmAa8C/kXSRuBcYGXWO4wHg8BvKjOzrEgzCFYDp0haIqkJuAJYObgwInZGxOyIWBwRi4EHgYsjojPFmlLnMwIzy5rUgiAi+oBrgF8AjwF3RsQ6SddLujit/Vabg8DMsqYhzY1HxCpg1bC260ZY9/w0a6mUSY11HorazDLF7ywus8GhqD0CqZllhYMgBTMmN/Kih5kws4xwEKRgpoPAzDLEQZCCGZOb2LHPt4+aWTY4CFIwy0FgZhniIEjBjCmN7Nh7iAiPQGpmtc9BkIKZk5s42D/AvoP91S7FzGxMDoIUzEoGnvPlITPLAgdBCmZMLry7eMde3zlkZrXPQZCCWVN8RmBm2eEgSMEMXxoyswxxEKTg8BnBXgeBmdU+B0EKWlsakWCH311sZhngIEhBfZ1obWn0pSEzywQHQUpmTm7yGYGZZYKDICVtU5rYtru32mWYmY3JQZCSudMnsXX3gWqXYWY2JgdBSuZMb+b5XT4jMLPa5yBIydzpk9jT28ee3r5ql2JmNqpUg0DSBZIel9Ql6doSyz8jab2kNZLuknRimvVU0tzpzQBs3eXLQ2ZW21ILAkn1wI3AhcAy4EpJy4at9nugIyLOAH4AfDmteipt7vRJADznIDCzGpfmGcE5QFdEbIiIg8AdwCXFK0TE3RGxL5l9EFiQYj0VNRgEW91PYGY1Ls0gmA88UzTfnbSN5GrgZ6UWSFouqVNSZ09PTxlLTM9gEDzvMwIzq3E10Vks6f1AB/CVUssjYkVEdERER3t7e2WLO05TmxuYNqmBzS/ur3YpZmajakhx25uBhUXzC5K2ISS9Hfg88OaImFDXURa3TWHT9n1jr2hmVkVpnhGsBk6RtERSE3AFsLJ4BUlnAv8AXBwRW1OspSpObJvMxu17q12GmdmoUguCiOgDrgF+ATwG3BkR6yRdL+niZLWvAFOB70t6RNLKETaXSYvbptC9Yz+H+geqXYqZ2YjSvDRERKwCVg1ru65o+u1p7r/aTmybTP9AsHnHfhbPnlLtcszMSqqJzuKJavCP/9PbfHnIzGqXgyBFp86dBsBjz+2qciVmZiNzEKSotaWRhbNaWLfZQWBmtctBkLJXvayVtc/urHYZZmYjchCk7FXzW9m0fR87/WllZlajHAQp6zhxJgAPPr29ypWYmZXmIEjZmYtmMrmpnvu6tlW7FDOzkhwEKWtqqON1S2Zx7xM9RES1yzEzO4qDoALesewENm7fx1rfPWRmNchBUAHvPn0eTfV1/Oj33dUuxczsKA6CCmid3Mg7T5vLDzq72bnfdw+ZWW1xEFTIx84/id29fdz8b09XuxQzsyEcBBVy2stauej0E7jpnqc89pCZ1RQHQQV94T+eRlNDHdd893fs7e2rdjlmZoCDoKLmTJ/EDVecyWNbdvGR2x5m9wH3F5hZ9TkIKuwtr5jDVy57NQ9s2M6l37iftZs9DpGZVZeDoAouPXsB37nqHF7cd4hLbryPv/z+o2zo2VPtsswsp5S1d7t2dHREZ2dntcsoi537DvG1u57guw/9kd6+Ac4+cSbvOWMebzp5NifPmYqkapdoZhOEpIcjoqPkMgdB9W3dfYAfPryZH/++myeeL5wZtE9r5vT5rbxy3jReccJ0FrdNYcHMFmZMbnRAmNkxq1oQSLoA+DpQD3wrIr44bHkzcCtwNrAduDwiNo62zYkYBMWeeWEf93Vt46GnX2D9s7vo6tlD/8CR12hKUz3zZ7Ywe2ozs6Y0HX6cNaWJ1pZGpjY3MKW5gSnN9YenpzY30NxQ5wAxy7GqBIGkeuAJ4B1AN7AauDIi1het83HgjIj4qKQrgPdFxOWjbXeiB8FwvX39PLV1L8/s2Ef3jv1079jH5h372b73INv39LJ970F2Hxj7VtT6OjGpoY6mhjqaG+qTxzqaG+toqh/a1tRQaKuvEw31KjzWJfN1KnqsK1qePNbXHZ6ul6irgzoJSdSpMF0nkvkjbTq8rHg51NUVHqH4+UXr1x29TQESiMJ6g5Sso+L5ZJ3Dq5Vo07BtHl4vWWcwYEfdb1Gbhm2Tom2YpWW0IGhIcb/nAF0RsSEp4g7gEmB90TqXAF9Ipn8A/J0kRdauV6WouaGeZS+bzrKXTR9xnYN9A+zYd5Bd+w+xp7ePvb397Ok9xJ7efvb29iVtffT2DXCwb4Devv7kcWDI44v7DtKbzPcNDNDfH/QNBP0DxY8D9A8Eh/r9EqVhXAEEDM+NI/F2ZDtDlxcv04jLSjUMX178/NH2M559HZ1/x7Lt4ctH/h6MVddRVQzf9pBtjf97XWpfGnFm7Ode8dqFfPg/LD2q3pcqzSCYDzxTNN8NvG6kdSKiT9JOoA0YMni/pOXAcoBFixalVW9mNTXUMXf6JOZOn1TR/Q4kAdE3MFAIiv6jA2MgYCCCCIg4Mj/YNhDF6yTTyfNKrR8EAwNHnjfSNgvrFkREYTp5fqGtsDxKtDG4/mBbMj/470kk22RIWwzZZnEbw9Yf934ZfhzJc4Zl8PBIHv5/VPHs0esO39bIzx3uqP28xG3HKMuGb/2o546yr9H2M546GfX7d6zfg/E/9+gygtlTm4evVRZpBkHZRMQKYAUULg1VuRxL1NWJpjrR5LuQzTItzd/gzcDCovkFSVvJdSQ1AK0UOo3NzKxC0gyC1cApkpZIagKuAFYOW2cl8IFk+jLgN+4fMDOrrNQuDSXX/K8BfkHh9tGbI2KdpOuBzohYCfxf4DZJXcALFMLCzMwqKNU+gohYBawa1nZd0fQB4D+nWYOZmY3OvXxmZjnnIDAzyzkHgZlZzjkIzMxyLnOjj0rqATYd59NnM+xdyxnmY6lNE+VYJspxgI9l0IkR0V5qQeaC4KWQ1DnSoEtZ42OpTRPlWCbKcYCPZTx8acjMLOccBGZmOZe3IFhR7QLKyMdSmybKsUyU4wAfy5hy1UdgZmZHy9sZgZmZDeMgMDPLudwEgaQLJD0uqUvStdWupxRJGyX9QdIjkjqTtlmSfiXpyeRxZtIuSTckx7NG0llF2/lAsv6Tkj4w0v7KXPvNkrZKWlvUVrbaJZ2dfG+6kuem9iG/IxzLFyRtTl6bRyRdVLTsr5K6Hpf0rqL2kj9zydDsDyXt30uGaU/jOBZKulvSeknrJH06ac/c6zLKsWTxdZkk6beSHk2O5X+Mtn9Jzcl8V7J88fEe44gi+YjAifxFYRjsp4ClQBPwKLCs2nWVqHMjMHtY25eBa5Ppa4EvJdMXAT+j8DGn5wIPJe2zgA3J48xkemYFaj8POAtYm0btwG+TdZU898IKH8sXgL8sse6y5OepGViS/JzVj/YzB9wJXJFM3wR8LKXjmAeclUxPA55I6s3c6zLKsWTxdREwNZluBB5Kvocl9w98HLgpmb4C+N7xHuNIX3k5IzgH6IqIDRFxELgDuKTKNY3XJcB3kunvAO8tar81Ch4EZkiaB7wL+FVEvBARO4BfARekXWRE3EvhMyXKXnuybHpEPBiF34Bbi7ZVqWMZySXAHRHRGxFPA10Uft5K/swl/zG/FfhB8vzi70tZRcSWiPhdMr0beIzC54Rn7nUZ5VhGUsuvS0TEnmS2MfmKUfZf/Hr9AHhbUu8xHeNoNeUlCOYDzxTNdzP6D1G1BPBLSQ9LWp60zY2ILcn0c8DcZHqkY6qlYy1X7fOT6eHtlXZNcsnk5sHLKRz7sbQBL0ZE37D2VCWXE86k8N9npl+XYccCGXxdJNVLegTYSiFYnxpl/4drTpbvTOot29+AvARBVrwpIs4CLgQ+Iem84oXJf12ZvN83y7UnvgGcBLwG2AL87+qWM36SpgI/BP5bROwqXpa116XEsWTydYmI/oh4DYXPcj8HeEU168lLEGwGFhbNL0jaakpEbE4etwI/pvAD8nxyCk7yuDVZfaRjqqVjLVftm5Pp4e0VExHPJ7+8A8A3Kbw2cOzHsp3CJZeGYe2pkNRI4Q/nP0XEj5LmTL4upY4lq6/LoIh4EbgbeP0o+z9cc7K8Nam3fH8D0ugMqbUvCh/JuYFCh8pg58lp1a5rWI1TgGlF0/dTuLb/FYZ27H05mX43Qzv2fpu0zwKeptCpNzOZnlWhY1jM0A7WstXO0Z2SF1X4WOYVTf85hWuzAKcxtMNuA4XOuhF/5oDvM7RT8OMpHYMoXLf/2rD2zL0uoxxLFl+XdmBGMt0C/CvwnpH2D3yCoZ3Fdx7vMY5YU5q/TLX0ReGOiCcoXIv7fLXrKVHf0uQFexRYN1gjhWuBdwFPAr8u+gUUcGNyPH8AOoq29SEKHUddwFUVqv92Cqfmhyhck7y6nLUDHcDa5Dl/R/Ku+Aoey21JrWuAlcP+AH0+qetxiu6aGelnLnmtf5sc4/eB5pSO400ULvusAR5Jvi7K4usyyrFk8XU5A/h9UvNa4LrR9g9MSua7kuVLj/cYR/ryEBNmZjmXlz4CMzMbgYPAzCznHARmZjnnIDAzyzkHgZlZzjkIzBKS+otGsXxkXKM2jn/bi1U0mqlZLWkYexWz3Ngfhbf9m+WKzwjMxqDC50R8ORl3/7eSTk7aF0v6TTLg2V2SFiXtcyX9OBlv/lFJb0g2VS/pm8kY9L+U1JKs/6lknP01ku6o0mFajjkIzI5oGXZp6PKiZTsj4nQK7579WtL2t8B3IuIM4J+AG5L2G4B7IuLVFD7XYF3SfgpwY0ScBrwIXJq0XwucmWzno2kdnNlI/M5is4SkPRExtUT7RuCtEbEhGfjsuYhok7SNwpAGh5L2LRExW1IPsCAieou2sZjCmP6nJPOfAxoj4n9K+jmwB/gJ8JM4Mla9WUX4jMBsfGKE6WPRWzTdz5E+undTGOPnLGB10QiUZhXhIDAbn8uLHh9Ipu+nMBokwJ9SGEUSCgO6fQwOfwBJ60gblVQHLIyIu4HPURhi+KizErM0+T8PsyNakk+NGvTziBi8hXSmpDUU/qu/Mmn7JPBtSZ8FeoCrkvZPAyskXU3hP/+PURjNtJR64B+TsBBwQxTGqDerGPcRmI0h6SPoiIht1a7FLA2+NGRmlnM+IzAzyzmfEZiZ5ZyDwMws5xwEZmY55yAwM8s5B4GZWc79f1IRuS3sir74AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot([i for i in range(epochs)], losses)\n",
    "plt.xlabel('Epochs')\n",
    "plt.ylabel('Loss');"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--- After training ---\n",
      "Prediction for [0, 0] (should be 0): 0.02\n",
      "Prediction for [0, 1] (should be 1): 0.99\n",
      "Prediction for [1, 0] (should be 1): 0.99\n",
      "Prediction for [1, 1] (should be 0): 0.02\n"
     ]
    }
   ],
   "source": [
    "# Print out what the NN predicts after training (updating the weights and biases)\n",
    "print('--- After training ---')\n",
    "print('Prediction for [0, 0] (should be 0): {num:.{digits}f}'.format(num=nn.forward([0, 0]), digits=2))\n",
    "print('Prediction for [0, 1] (should be 1): {num:.{digits}f}'.format(num=nn.forward([0, 1]), digits=2))\n",
    "print('Prediction for [1, 0] (should be 1): {num:.{digits}f}'.format(num=nn.forward([1, 0]), digits=2))\n",
    "print('Prediction for [1, 1] (should be 0): {num:.{digits}f}'.format(num=nn.forward([1, 1]), digits=2))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}