{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "NeuralNetworks.ipynb",
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "OduD_pshW6Kb"
      },
      "source": [
        "from keras.datasets import mnist\n",
        "import numpy as np\n",
        "import pandas as pd\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uM3w-SXXrn4u"
      },
      "source": [
        "# Base class\n",
        "class Layer:\n",
        "    def __init__(self):\n",
        "        self.input = None\n",
        "        self.output = None\n",
        "\n",
        "    # computes the output Y of a layer for a given input X\n",
        "    def forward_propagation(self, input):\n",
        "        raise NotImplementedError\n",
        "\n",
        "    # computes dE/dX for a given dE/dY (and update parameters if any)\n",
        "    def backward_propagation(self, output_error, learning_rate):\n",
        "        raise NotImplementedError"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Lbh6q6qvrt2q"
      },
      "source": [
        "class FCLayer(Layer):\n",
        "    def __init__(self, input_size, output_size):\n",
        "      self.weights = np.random.rand(input_size, output_size) - 0.5\n",
        "      self.bias = np.random.rand(1, output_size) - 0.5\n",
        "    \n",
        "    def forward_propagation(self,input_data):\n",
        "      self.input = input_data\n",
        "      self.output = np.dot(self.input,self.weights)+self.bias\n",
        "      return(self.output)\n",
        "\n",
        "    ## compute dE/dw, de/dB for a given outputt erroor. Returns input error to be given t the next layer\n",
        "    def backward_propagation(self,output_error, learning_rate):\n",
        "      input_error = np.dot(output_error, self.weights.T)\n",
        "      weights_error = np.dot(self.input.T, output_error)\n",
        "\n",
        "      self.weights -= learning_rate * weights_error\n",
        "      self.bias -= learning_rate * output_error\n",
        "      return(input_error)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KatIfnmxtxmW"
      },
      "source": [
        "\n",
        "class ActivationLayer(Layer):\n",
        "    def __init__(self):\n",
        "      self.activation = lambda x: np.tanh(x)\n",
        "      self.activation_prime = lambda x: 1-np.tanh(x)**2;\n",
        "    \n",
        "    def forward_propagation(self, input_data):\n",
        "      self.input = input_data\n",
        "      self.output = self.activation(self.input)\n",
        "      return(self.output)\n",
        "    \n",
        "    def backward_propagation(self, output_error, learning_rate):\n",
        "      return(self.activation_prime(self.input) * output_error)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LplgFhjivh8S"
      },
      "source": [
        "# loss function and its derivative\n",
        "def mse(y_true, y_pred):\n",
        "    return np.mean(np.power(y_true - y_pred, 2));\n",
        "\n",
        "def mse_prime(y_true, y_pred):\n",
        "    return 2*(y_pred-y_true)/y_true.size;"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jMuCe0CgwieD"
      },
      "source": [
        "class Network:\n",
        "  def __init__(self):\n",
        "    self.layers=[]\n",
        "    self.loss=None\n",
        "    self.loss_prime=None\n",
        "\n",
        "  def add(self,layer):\n",
        "    self.layers.append(layer)\n",
        "\n",
        "  def use(self,loss,loss_prime):\n",
        "    self.loss = loss\n",
        "    self.loss_prime = loss_prime\n",
        "\n",
        "  def predict(self,input_data):\n",
        "    result=[]\n",
        "    for i in range(len(input_data)):\n",
        "      output=input_data[i]\n",
        "      for layer in self.layers:\n",
        "        output= layer.forward_propagation(output)\n",
        "      result.append(output)\n",
        "    return(result)\n",
        "\n",
        "  def fit(self, x_train, y_train, epochs, learning_rate):\n",
        "    samples = len(x_train)\n",
        "    errors=[]\n",
        "\n",
        "    for i in range(epochs):\n",
        "      err = 0\n",
        "      for j in range(samples):\n",
        "        output = x_train[j]\n",
        "        for layer in self.layers:\n",
        "          output = layer.forward_propagation(output)\n",
        "        \n",
        "        err += self.loss(y_train[j], output)\n",
        "\n",
        "        error = self.loss_prime(y_train[j], output)\n",
        "        for layer in reversed(self.layers):\n",
        "          error = layer.backward_propagation(error, learning_rate)\n",
        "      errors.append(err/samples)\n",
        "\n",
        "      print('epoch %d/%d   error=%f' % (i+1, epochs, err))\n",
        "    return(errors)\n",
        "\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TK_LwCo3xITw"
      },
      "source": [
        "x_train = np.array([[[0,0]], [[0,1]], [[1,0]], [[1,1]]])\n",
        "y_train = np.array([[[0]], [[1]], [[1]], [[0]]])\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tFvsRjeuzLIr"
      },
      "source": [
        "net = Network()\n",
        "net.add(FCLayer(2,5))\n",
        "net.add(ActivationLayer())\n",
        "net.add(FCLayer(5,1))\n",
        "net.add(ActivationLayer())\n",
        "net.use(mse, mse_prime)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "l2tIbwjFz5yO",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "10cb3f69-d635-4e2b-9c72-214b74e1ca0c"
      },
      "source": [
        "err = net.fit(x_train, y_train, epochs=1000, learning_rate=0.1)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "epoch 1/1000   error=3.439531\n",
            "epoch 2/1000   error=1.469290\n",
            "epoch 3/1000   error=1.249098\n",
            "epoch 4/1000   error=1.221891\n",
            "epoch 5/1000   error=1.213275\n",
            "epoch 6/1000   error=1.208269\n",
            "epoch 7/1000   error=1.204283\n",
            "epoch 8/1000   error=1.200643\n",
            "epoch 9/1000   error=1.197125\n",
            "epoch 10/1000   error=1.193641\n",
            "epoch 11/1000   error=1.190152\n",
            "epoch 12/1000   error=1.186639\n",
            "epoch 13/1000   error=1.183090\n",
            "epoch 14/1000   error=1.179496\n",
            "epoch 15/1000   error=1.175854\n",
            "epoch 16/1000   error=1.172157\n",
            "epoch 17/1000   error=1.168403\n",
            "epoch 18/1000   error=1.164588\n",
            "epoch 19/1000   error=1.160709\n",
            "epoch 20/1000   error=1.156765\n",
            "epoch 21/1000   error=1.152753\n",
            "epoch 22/1000   error=1.148673\n",
            "epoch 23/1000   error=1.144523\n",
            "epoch 24/1000   error=1.140304\n",
            "epoch 25/1000   error=1.136016\n",
            "epoch 26/1000   error=1.131659\n",
            "epoch 27/1000   error=1.127236\n",
            "epoch 28/1000   error=1.122749\n",
            "epoch 29/1000   error=1.118200\n",
            "epoch 30/1000   error=1.113594\n",
            "epoch 31/1000   error=1.108935\n",
            "epoch 32/1000   error=1.104229\n",
            "epoch 33/1000   error=1.099480\n",
            "epoch 34/1000   error=1.094695\n",
            "epoch 35/1000   error=1.089882\n",
            "epoch 36/1000   error=1.085048\n",
            "epoch 37/1000   error=1.080200\n",
            "epoch 38/1000   error=1.075348\n",
            "epoch 39/1000   error=1.070499\n",
            "epoch 40/1000   error=1.065664\n",
            "epoch 41/1000   error=1.060849\n",
            "epoch 42/1000   error=1.056065\n",
            "epoch 43/1000   error=1.051319\n",
            "epoch 44/1000   error=1.046620\n",
            "epoch 45/1000   error=1.041977\n",
            "epoch 46/1000   error=1.037396\n",
            "epoch 47/1000   error=1.032884\n",
            "epoch 48/1000   error=1.028448\n",
            "epoch 49/1000   error=1.024094\n",
            "epoch 50/1000   error=1.019825\n",
            "epoch 51/1000   error=1.015647\n",
            "epoch 52/1000   error=1.011563\n",
            "epoch 53/1000   error=1.007575\n",
            "epoch 54/1000   error=1.003685\n",
            "epoch 55/1000   error=0.999895\n",
            "epoch 56/1000   error=0.996204\n",
            "epoch 57/1000   error=0.992613\n",
            "epoch 58/1000   error=0.989120\n",
            "epoch 59/1000   error=0.985724\n",
            "epoch 60/1000   error=0.982424\n",
            "epoch 61/1000   error=0.979217\n",
            "epoch 62/1000   error=0.976100\n",
            "epoch 63/1000   error=0.973071\n",
            "epoch 64/1000   error=0.970126\n",
            "epoch 65/1000   error=0.967262\n",
            "epoch 66/1000   error=0.964476\n",
            "epoch 67/1000   error=0.961764\n",
            "epoch 68/1000   error=0.959122\n",
            "epoch 69/1000   error=0.956546\n",
            "epoch 70/1000   error=0.954034\n",
            "epoch 71/1000   error=0.951581\n",
            "epoch 72/1000   error=0.949184\n",
            "epoch 73/1000   error=0.946839\n",
            "epoch 74/1000   error=0.944543\n",
            "epoch 75/1000   error=0.942292\n",
            "epoch 76/1000   error=0.940085\n",
            "epoch 77/1000   error=0.937916\n",
            "epoch 78/1000   error=0.935783\n",
            "epoch 79/1000   error=0.933684\n",
            "epoch 80/1000   error=0.931615\n",
            "epoch 81/1000   error=0.929575\n",
            "epoch 82/1000   error=0.927559\n",
            "epoch 83/1000   error=0.925565\n",
            "epoch 84/1000   error=0.923592\n",
            "epoch 85/1000   error=0.921636\n",
            "epoch 86/1000   error=0.919696\n",
            "epoch 87/1000   error=0.917768\n",
            "epoch 88/1000   error=0.915851\n",
            "epoch 89/1000   error=0.913942\n",
            "epoch 90/1000   error=0.912039\n",
            "epoch 91/1000   error=0.910140\n",
            "epoch 92/1000   error=0.908242\n",
            "epoch 93/1000   error=0.906344\n",
            "epoch 94/1000   error=0.904442\n",
            "epoch 95/1000   error=0.902535\n",
            "epoch 96/1000   error=0.900620\n",
            "epoch 97/1000   error=0.898696\n",
            "epoch 98/1000   error=0.896758\n",
            "epoch 99/1000   error=0.894806\n",
            "epoch 100/1000   error=0.892836\n",
            "epoch 101/1000   error=0.890845\n",
            "epoch 102/1000   error=0.888832\n",
            "epoch 103/1000   error=0.886793\n",
            "epoch 104/1000   error=0.884725\n",
            "epoch 105/1000   error=0.882626\n",
            "epoch 106/1000   error=0.880491\n",
            "epoch 107/1000   error=0.878319\n",
            "epoch 108/1000   error=0.876105\n",
            "epoch 109/1000   error=0.873845\n",
            "epoch 110/1000   error=0.871537\n",
            "epoch 111/1000   error=0.869176\n",
            "epoch 112/1000   error=0.866757\n",
            "epoch 113/1000   error=0.864278\n",
            "epoch 114/1000   error=0.861732\n",
            "epoch 115/1000   error=0.859115\n",
            "epoch 116/1000   error=0.856422\n",
            "epoch 117/1000   error=0.853648\n",
            "epoch 118/1000   error=0.850786\n",
            "epoch 119/1000   error=0.847830\n",
            "epoch 120/1000   error=0.844773\n",
            "epoch 121/1000   error=0.841609\n",
            "epoch 122/1000   error=0.838330\n",
            "epoch 123/1000   error=0.834928\n",
            "epoch 124/1000   error=0.831394\n",
            "epoch 125/1000   error=0.827718\n",
            "epoch 126/1000   error=0.823891\n",
            "epoch 127/1000   error=0.819901\n",
            "epoch 128/1000   error=0.815738\n",
            "epoch 129/1000   error=0.811388\n",
            "epoch 130/1000   error=0.806838\n",
            "epoch 131/1000   error=0.802073\n",
            "epoch 132/1000   error=0.797079\n",
            "epoch 133/1000   error=0.791837\n",
            "epoch 134/1000   error=0.786330\n",
            "epoch 135/1000   error=0.780538\n",
            "epoch 136/1000   error=0.774440\n",
            "epoch 137/1000   error=0.768014\n",
            "epoch 138/1000   error=0.761234\n",
            "epoch 139/1000   error=0.754075\n",
            "epoch 140/1000   error=0.746509\n",
            "epoch 141/1000   error=0.738505\n",
            "epoch 142/1000   error=0.730032\n",
            "epoch 143/1000   error=0.721055\n",
            "epoch 144/1000   error=0.711538\n",
            "epoch 145/1000   error=0.701443\n",
            "epoch 146/1000   error=0.690730\n",
            "epoch 147/1000   error=0.679356\n",
            "epoch 148/1000   error=0.667280\n",
            "epoch 149/1000   error=0.654457\n",
            "epoch 150/1000   error=0.640844\n",
            "epoch 151/1000   error=0.626401\n",
            "epoch 152/1000   error=0.611087\n",
            "epoch 153/1000   error=0.594869\n",
            "epoch 154/1000   error=0.577720\n",
            "epoch 155/1000   error=0.559625\n",
            "epoch 156/1000   error=0.540582\n",
            "epoch 157/1000   error=0.520606\n",
            "epoch 158/1000   error=0.499736\n",
            "epoch 159/1000   error=0.478038\n",
            "epoch 160/1000   error=0.455608\n",
            "epoch 161/1000   error=0.432576\n",
            "epoch 162/1000   error=0.409107\n",
            "epoch 163/1000   error=0.385399\n",
            "epoch 164/1000   error=0.361679\n",
            "epoch 165/1000   error=0.338191\n",
            "epoch 166/1000   error=0.315186\n",
            "epoch 167/1000   error=0.292907\n",
            "epoch 168/1000   error=0.271572\n",
            "epoch 169/1000   error=0.251362\n",
            "epoch 170/1000   error=0.232410\n",
            "epoch 171/1000   error=0.214797\n",
            "epoch 172/1000   error=0.198556\n",
            "epoch 173/1000   error=0.183675\n",
            "epoch 174/1000   error=0.170107\n",
            "epoch 175/1000   error=0.157781\n",
            "epoch 176/1000   error=0.146609\n",
            "epoch 177/1000   error=0.136496\n",
            "epoch 178/1000   error=0.127345\n",
            "epoch 179/1000   error=0.119061\n",
            "epoch 180/1000   error=0.111556\n",
            "epoch 181/1000   error=0.104747\n",
            "epoch 182/1000   error=0.098560\n",
            "epoch 183/1000   error=0.092927\n",
            "epoch 184/1000   error=0.087789\n",
            "epoch 185/1000   error=0.083091\n",
            "epoch 186/1000   error=0.078788\n",
            "epoch 187/1000   error=0.074837\n",
            "epoch 188/1000   error=0.071202\n",
            "epoch 189/1000   error=0.067850\n",
            "epoch 190/1000   error=0.064754\n",
            "epoch 191/1000   error=0.061887\n",
            "epoch 192/1000   error=0.059227\n",
            "epoch 193/1000   error=0.056755\n",
            "epoch 194/1000   error=0.054454\n",
            "epoch 195/1000   error=0.052307\n",
            "epoch 196/1000   error=0.050301\n",
            "epoch 197/1000   error=0.048423\n",
            "epoch 198/1000   error=0.046663\n",
            "epoch 199/1000   error=0.045010\n",
            "epoch 200/1000   error=0.043456\n",
            "epoch 201/1000   error=0.041993\n",
            "epoch 202/1000   error=0.040613\n",
            "epoch 203/1000   error=0.039310\n",
            "epoch 204/1000   error=0.038079\n",
            "epoch 205/1000   error=0.036913\n",
            "epoch 206/1000   error=0.035809\n",
            "epoch 207/1000   error=0.034761\n",
            "epoch 208/1000   error=0.033766\n",
            "epoch 209/1000   error=0.032820\n",
            "epoch 210/1000   error=0.031920\n",
            "epoch 211/1000   error=0.031062\n",
            "epoch 212/1000   error=0.030244\n",
            "epoch 213/1000   error=0.029464\n",
            "epoch 214/1000   error=0.028718\n",
            "epoch 215/1000   error=0.028005\n",
            "epoch 216/1000   error=0.027324\n",
            "epoch 217/1000   error=0.026671\n",
            "epoch 218/1000   error=0.026045\n",
            "epoch 219/1000   error=0.025445\n",
            "epoch 220/1000   error=0.024869\n",
            "epoch 221/1000   error=0.024316\n",
            "epoch 222/1000   error=0.023784\n",
            "epoch 223/1000   error=0.023273\n",
            "epoch 224/1000   error=0.022782\n",
            "epoch 225/1000   error=0.022308\n",
            "epoch 226/1000   error=0.021852\n",
            "epoch 227/1000   error=0.021412\n",
            "epoch 228/1000   error=0.020988\n",
            "epoch 229/1000   error=0.020579\n",
            "epoch 230/1000   error=0.020184\n",
            "epoch 231/1000   error=0.019802\n",
            "epoch 232/1000   error=0.019433\n",
            "epoch 233/1000   error=0.019076\n",
            "epoch 234/1000   error=0.018731\n",
            "epoch 235/1000   error=0.018397\n",
            "epoch 236/1000   error=0.018073\n",
            "epoch 237/1000   error=0.017760\n",
            "epoch 238/1000   error=0.017456\n",
            "epoch 239/1000   error=0.017162\n",
            "epoch 240/1000   error=0.016876\n",
            "epoch 241/1000   error=0.016599\n",
            "epoch 242/1000   error=0.016330\n",
            "epoch 243/1000   error=0.016069\n",
            "epoch 244/1000   error=0.015815\n",
            "epoch 245/1000   error=0.015568\n",
            "epoch 246/1000   error=0.015329\n",
            "epoch 247/1000   error=0.015095\n",
            "epoch 248/1000   error=0.014869\n",
            "epoch 249/1000   error=0.014648\n",
            "epoch 250/1000   error=0.014433\n",
            "epoch 251/1000   error=0.014224\n",
            "epoch 252/1000   error=0.014020\n",
            "epoch 253/1000   error=0.013822\n",
            "epoch 254/1000   error=0.013628\n",
            "epoch 255/1000   error=0.013440\n",
            "epoch 256/1000   error=0.013256\n",
            "epoch 257/1000   error=0.013076\n",
            "epoch 258/1000   error=0.012901\n",
            "epoch 259/1000   error=0.012730\n",
            "epoch 260/1000   error=0.012564\n",
            "epoch 261/1000   error=0.012401\n",
            "epoch 262/1000   error=0.012242\n",
            "epoch 263/1000   error=0.012086\n",
            "epoch 264/1000   error=0.011935\n",
            "epoch 265/1000   error=0.011786\n",
            "epoch 266/1000   error=0.011641\n",
            "epoch 267/1000   error=0.011500\n",
            "epoch 268/1000   error=0.011361\n",
            "epoch 269/1000   error=0.011225\n",
            "epoch 270/1000   error=0.011092\n",
            "epoch 271/1000   error=0.010963\n",
            "epoch 272/1000   error=0.010835\n",
            "epoch 273/1000   error=0.010711\n",
            "epoch 274/1000   error=0.010589\n",
            "epoch 275/1000   error=0.010470\n",
            "epoch 276/1000   error=0.010353\n",
            "epoch 277/1000   error=0.010238\n",
            "epoch 278/1000   error=0.010126\n",
            "epoch 279/1000   error=0.010016\n",
            "epoch 280/1000   error=0.009908\n",
            "epoch 281/1000   error=0.009802\n",
            "epoch 282/1000   error=0.009699\n",
            "epoch 283/1000   error=0.009597\n",
            "epoch 284/1000   error=0.009497\n",
            "epoch 285/1000   error=0.009399\n",
            "epoch 286/1000   error=0.009303\n",
            "epoch 287/1000   error=0.009209\n",
            "epoch 288/1000   error=0.009116\n",
            "epoch 289/1000   error=0.009025\n",
            "epoch 290/1000   error=0.008936\n",
            "epoch 291/1000   error=0.008849\n",
            "epoch 292/1000   error=0.008763\n",
            "epoch 293/1000   error=0.008678\n",
            "epoch 294/1000   error=0.008595\n",
            "epoch 295/1000   error=0.008513\n",
            "epoch 296/1000   error=0.008433\n",
            "epoch 297/1000   error=0.008354\n",
            "epoch 298/1000   error=0.008277\n",
            "epoch 299/1000   error=0.008201\n",
            "epoch 300/1000   error=0.008126\n",
            "epoch 301/1000   error=0.008052\n",
            "epoch 302/1000   error=0.007980\n",
            "epoch 303/1000   error=0.007909\n",
            "epoch 304/1000   error=0.007839\n",
            "epoch 305/1000   error=0.007770\n",
            "epoch 306/1000   error=0.007702\n",
            "epoch 307/1000   error=0.007635\n",
            "epoch 308/1000   error=0.007569\n",
            "epoch 309/1000   error=0.007505\n",
            "epoch 310/1000   error=0.007441\n",
            "epoch 311/1000   error=0.007378\n",
            "epoch 312/1000   error=0.007317\n",
            "epoch 313/1000   error=0.007256\n",
            "epoch 314/1000   error=0.007196\n",
            "epoch 315/1000   error=0.007137\n",
            "epoch 316/1000   error=0.007079\n",
            "epoch 317/1000   error=0.007022\n",
            "epoch 318/1000   error=0.006966\n",
            "epoch 319/1000   error=0.006910\n",
            "epoch 320/1000   error=0.006855\n",
            "epoch 321/1000   error=0.006802\n",
            "epoch 322/1000   error=0.006748\n",
            "epoch 323/1000   error=0.006696\n",
            "epoch 324/1000   error=0.006644\n",
            "epoch 325/1000   error=0.006594\n",
            "epoch 326/1000   error=0.006543\n",
            "epoch 327/1000   error=0.006494\n",
            "epoch 328/1000   error=0.006445\n",
            "epoch 329/1000   error=0.006397\n",
            "epoch 330/1000   error=0.006349\n",
            "epoch 331/1000   error=0.006303\n",
            "epoch 332/1000   error=0.006256\n",
            "epoch 333/1000   error=0.006211\n",
            "epoch 334/1000   error=0.006166\n",
            "epoch 335/1000   error=0.006122\n",
            "epoch 336/1000   error=0.006078\n",
            "epoch 337/1000   error=0.006035\n",
            "epoch 338/1000   error=0.005992\n",
            "epoch 339/1000   error=0.005950\n",
            "epoch 340/1000   error=0.005908\n",
            "epoch 341/1000   error=0.005867\n",
            "epoch 342/1000   error=0.005827\n",
            "epoch 343/1000   error=0.005787\n",
            "epoch 344/1000   error=0.005747\n",
            "epoch 345/1000   error=0.005709\n",
            "epoch 346/1000   error=0.005670\n",
            "epoch 347/1000   error=0.005632\n",
            "epoch 348/1000   error=0.005594\n",
            "epoch 349/1000   error=0.005557\n",
            "epoch 350/1000   error=0.005521\n",
            "epoch 351/1000   error=0.005485\n",
            "epoch 352/1000   error=0.005449\n",
            "epoch 353/1000   error=0.005414\n",
            "epoch 354/1000   error=0.005379\n",
            "epoch 355/1000   error=0.005344\n",
            "epoch 356/1000   error=0.005310\n",
            "epoch 357/1000   error=0.005276\n",
            "epoch 358/1000   error=0.005243\n",
            "epoch 359/1000   error=0.005210\n",
            "epoch 360/1000   error=0.005178\n",
            "epoch 361/1000   error=0.005146\n",
            "epoch 362/1000   error=0.005114\n",
            "epoch 363/1000   error=0.005082\n",
            "epoch 364/1000   error=0.005051\n",
            "epoch 365/1000   error=0.005021\n",
            "epoch 366/1000   error=0.004990\n",
            "epoch 367/1000   error=0.004960\n",
            "epoch 368/1000   error=0.004931\n",
            "epoch 369/1000   error=0.004901\n",
            "epoch 370/1000   error=0.004872\n",
            "epoch 371/1000   error=0.004843\n",
            "epoch 372/1000   error=0.004815\n",
            "epoch 373/1000   error=0.004787\n",
            "epoch 374/1000   error=0.004759\n",
            "epoch 375/1000   error=0.004732\n",
            "epoch 376/1000   error=0.004704\n",
            "epoch 377/1000   error=0.004678\n",
            "epoch 378/1000   error=0.004651\n",
            "epoch 379/1000   error=0.004625\n",
            "epoch 380/1000   error=0.004598\n",
            "epoch 381/1000   error=0.004573\n",
            "epoch 382/1000   error=0.004547\n",
            "epoch 383/1000   error=0.004522\n",
            "epoch 384/1000   error=0.004497\n",
            "epoch 385/1000   error=0.004472\n",
            "epoch 386/1000   error=0.004448\n",
            "epoch 387/1000   error=0.004423\n",
            "epoch 388/1000   error=0.004399\n",
            "epoch 389/1000   error=0.004376\n",
            "epoch 390/1000   error=0.004352\n",
            "epoch 391/1000   error=0.004329\n",
            "epoch 392/1000   error=0.004306\n",
            "epoch 393/1000   error=0.004283\n",
            "epoch 394/1000   error=0.004260\n",
            "epoch 395/1000   error=0.004238\n",
            "epoch 396/1000   error=0.004216\n",
            "epoch 397/1000   error=0.004194\n",
            "epoch 398/1000   error=0.004172\n",
            "epoch 399/1000   error=0.004150\n",
            "epoch 400/1000   error=0.004129\n",
            "epoch 401/1000   error=0.004108\n",
            "epoch 402/1000   error=0.004087\n",
            "epoch 403/1000   error=0.004066\n",
            "epoch 404/1000   error=0.004046\n",
            "epoch 405/1000   error=0.004025\n",
            "epoch 406/1000   error=0.004005\n",
            "epoch 407/1000   error=0.003985\n",
            "epoch 408/1000   error=0.003965\n",
            "epoch 409/1000   error=0.003946\n",
            "epoch 410/1000   error=0.003926\n",
            "epoch 411/1000   error=0.003907\n",
            "epoch 412/1000   error=0.003888\n",
            "epoch 413/1000   error=0.003869\n",
            "epoch 414/1000   error=0.003850\n",
            "epoch 415/1000   error=0.003832\n",
            "epoch 416/1000   error=0.003813\n",
            "epoch 417/1000   error=0.003795\n",
            "epoch 418/1000   error=0.003777\n",
            "epoch 419/1000   error=0.003759\n",
            "epoch 420/1000   error=0.003741\n",
            "epoch 421/1000   error=0.003724\n",
            "epoch 422/1000   error=0.003706\n",
            "epoch 423/1000   error=0.003689\n",
            "epoch 424/1000   error=0.003672\n",
            "epoch 425/1000   error=0.003655\n",
            "epoch 426/1000   error=0.003638\n",
            "epoch 427/1000   error=0.003621\n",
            "epoch 428/1000   error=0.003605\n",
            "epoch 429/1000   error=0.003588\n",
            "epoch 430/1000   error=0.003572\n",
            "epoch 431/1000   error=0.003556\n",
            "epoch 432/1000   error=0.003540\n",
            "epoch 433/1000   error=0.003524\n",
            "epoch 434/1000   error=0.003508\n",
            "epoch 435/1000   error=0.003493\n",
            "epoch 436/1000   error=0.003477\n",
            "epoch 437/1000   error=0.003462\n",
            "epoch 438/1000   error=0.003447\n",
            "epoch 439/1000   error=0.003431\n",
            "epoch 440/1000   error=0.003416\n",
            "epoch 441/1000   error=0.003402\n",
            "epoch 442/1000   error=0.003387\n",
            "epoch 443/1000   error=0.003372\n",
            "epoch 444/1000   error=0.003358\n",
            "epoch 445/1000   error=0.003343\n",
            "epoch 446/1000   error=0.003329\n",
            "epoch 447/1000   error=0.003315\n",
            "epoch 448/1000   error=0.003301\n",
            "epoch 449/1000   error=0.003287\n",
            "epoch 450/1000   error=0.003273\n",
            "epoch 451/1000   error=0.003259\n",
            "epoch 452/1000   error=0.003246\n",
            "epoch 453/1000   error=0.003232\n",
            "epoch 454/1000   error=0.003219\n",
            "epoch 455/1000   error=0.003205\n",
            "epoch 456/1000   error=0.003192\n",
            "epoch 457/1000   error=0.003179\n",
            "epoch 458/1000   error=0.003166\n",
            "epoch 459/1000   error=0.003153\n",
            "epoch 460/1000   error=0.003140\n",
            "epoch 461/1000   error=0.003128\n",
            "epoch 462/1000   error=0.003115\n",
            "epoch 463/1000   error=0.003103\n",
            "epoch 464/1000   error=0.003090\n",
            "epoch 465/1000   error=0.003078\n",
            "epoch 466/1000   error=0.003066\n",
            "epoch 467/1000   error=0.003053\n",
            "epoch 468/1000   error=0.003041\n",
            "epoch 469/1000   error=0.003029\n",
            "epoch 470/1000   error=0.003018\n",
            "epoch 471/1000   error=0.003006\n",
            "epoch 472/1000   error=0.002994\n",
            "epoch 473/1000   error=0.002982\n",
            "epoch 474/1000   error=0.002971\n",
            "epoch 475/1000   error=0.002959\n",
            "epoch 476/1000   error=0.002948\n",
            "epoch 477/1000   error=0.002937\n",
            "epoch 478/1000   error=0.002926\n",
            "epoch 479/1000   error=0.002914\n",
            "epoch 480/1000   error=0.002903\n",
            "epoch 481/1000   error=0.002892\n",
            "epoch 482/1000   error=0.002882\n",
            "epoch 483/1000   error=0.002871\n",
            "epoch 484/1000   error=0.002860\n",
            "epoch 485/1000   error=0.002849\n",
            "epoch 486/1000   error=0.002839\n",
            "epoch 487/1000   error=0.002828\n",
            "epoch 488/1000   error=0.002818\n",
            "epoch 489/1000   error=0.002807\n",
            "epoch 490/1000   error=0.002797\n",
            "epoch 491/1000   error=0.002787\n",
            "epoch 492/1000   error=0.002777\n",
            "epoch 493/1000   error=0.002766\n",
            "epoch 494/1000   error=0.002756\n",
            "epoch 495/1000   error=0.002746\n",
            "epoch 496/1000   error=0.002737\n",
            "epoch 497/1000   error=0.002727\n",
            "epoch 498/1000   error=0.002717\n",
            "epoch 499/1000   error=0.002707\n",
            "epoch 500/1000   error=0.002698\n",
            "epoch 501/1000   error=0.002688\n",
            "epoch 502/1000   error=0.002679\n",
            "epoch 503/1000   error=0.002669\n",
            "epoch 504/1000   error=0.002660\n",
            "epoch 505/1000   error=0.002650\n",
            "epoch 506/1000   error=0.002641\n",
            "epoch 507/1000   error=0.002632\n",
            "epoch 508/1000   error=0.002623\n",
            "epoch 509/1000   error=0.002614\n",
            "epoch 510/1000   error=0.002605\n",
            "epoch 511/1000   error=0.002596\n",
            "epoch 512/1000   error=0.002587\n",
            "epoch 513/1000   error=0.002578\n",
            "epoch 514/1000   error=0.002569\n",
            "epoch 515/1000   error=0.002560\n",
            "epoch 516/1000   error=0.002552\n",
            "epoch 517/1000   error=0.002543\n",
            "epoch 518/1000   error=0.002535\n",
            "epoch 519/1000   error=0.002526\n",
            "epoch 520/1000   error=0.002518\n",
            "epoch 521/1000   error=0.002509\n",
            "epoch 522/1000   error=0.002501\n",
            "epoch 523/1000   error=0.002493\n",
            "epoch 524/1000   error=0.002484\n",
            "epoch 525/1000   error=0.002476\n",
            "epoch 526/1000   error=0.002468\n",
            "epoch 527/1000   error=0.002460\n",
            "epoch 528/1000   error=0.002452\n",
            "epoch 529/1000   error=0.002444\n",
            "epoch 530/1000   error=0.002436\n",
            "epoch 531/1000   error=0.002428\n",
            "epoch 532/1000   error=0.002420\n",
            "epoch 533/1000   error=0.002412\n",
            "epoch 534/1000   error=0.002404\n",
            "epoch 535/1000   error=0.002397\n",
            "epoch 536/1000   error=0.002389\n",
            "epoch 537/1000   error=0.002381\n",
            "epoch 538/1000   error=0.002374\n",
            "epoch 539/1000   error=0.002366\n",
            "epoch 540/1000   error=0.002359\n",
            "epoch 541/1000   error=0.002351\n",
            "epoch 542/1000   error=0.002344\n",
            "epoch 543/1000   error=0.002337\n",
            "epoch 544/1000   error=0.002329\n",
            "epoch 545/1000   error=0.002322\n",
            "epoch 546/1000   error=0.002315\n",
            "epoch 547/1000   error=0.002308\n",
            "epoch 548/1000   error=0.002300\n",
            "epoch 549/1000   error=0.002293\n",
            "epoch 550/1000   error=0.002286\n",
            "epoch 551/1000   error=0.002279\n",
            "epoch 552/1000   error=0.002272\n",
            "epoch 553/1000   error=0.002265\n",
            "epoch 554/1000   error=0.002258\n",
            "epoch 555/1000   error=0.002252\n",
            "epoch 556/1000   error=0.002245\n",
            "epoch 557/1000   error=0.002238\n",
            "epoch 558/1000   error=0.002231\n",
            "epoch 559/1000   error=0.002225\n",
            "epoch 560/1000   error=0.002218\n",
            "epoch 561/1000   error=0.002211\n",
            "epoch 562/1000   error=0.002205\n",
            "epoch 563/1000   error=0.002198\n",
            "epoch 564/1000   error=0.002192\n",
            "epoch 565/1000   error=0.002185\n",
            "epoch 566/1000   error=0.002179\n",
            "epoch 567/1000   error=0.002172\n",
            "epoch 568/1000   error=0.002166\n",
            "epoch 569/1000   error=0.002159\n",
            "epoch 570/1000   error=0.002153\n",
            "epoch 571/1000   error=0.002147\n",
            "epoch 572/1000   error=0.002141\n",
            "epoch 573/1000   error=0.002134\n",
            "epoch 574/1000   error=0.002128\n",
            "epoch 575/1000   error=0.002122\n",
            "epoch 576/1000   error=0.002116\n",
            "epoch 577/1000   error=0.002110\n",
            "epoch 578/1000   error=0.002104\n",
            "epoch 579/1000   error=0.002098\n",
            "epoch 580/1000   error=0.002092\n",
            "epoch 581/1000   error=0.002086\n",
            "epoch 582/1000   error=0.002080\n",
            "epoch 583/1000   error=0.002074\n",
            "epoch 584/1000   error=0.002068\n",
            "epoch 585/1000   error=0.002062\n",
            "epoch 586/1000   error=0.002057\n",
            "epoch 587/1000   error=0.002051\n",
            "epoch 588/1000   error=0.002045\n",
            "epoch 589/1000   error=0.002039\n",
            "epoch 590/1000   error=0.002034\n",
            "epoch 591/1000   error=0.002028\n",
            "epoch 592/1000   error=0.002023\n",
            "epoch 593/1000   error=0.002017\n",
            "epoch 594/1000   error=0.002011\n",
            "epoch 595/1000   error=0.002006\n",
            "epoch 596/1000   error=0.002000\n",
            "epoch 597/1000   error=0.001995\n",
            "epoch 598/1000   error=0.001989\n",
            "epoch 599/1000   error=0.001984\n",
            "epoch 600/1000   error=0.001979\n",
            "epoch 601/1000   error=0.001973\n",
            "epoch 602/1000   error=0.001968\n",
            "epoch 603/1000   error=0.001963\n",
            "epoch 604/1000   error=0.001957\n",
            "epoch 605/1000   error=0.001952\n",
            "epoch 606/1000   error=0.001947\n",
            "epoch 607/1000   error=0.001942\n",
            "epoch 608/1000   error=0.001937\n",
            "epoch 609/1000   error=0.001931\n",
            "epoch 610/1000   error=0.001926\n",
            "epoch 611/1000   error=0.001921\n",
            "epoch 612/1000   error=0.001916\n",
            "epoch 613/1000   error=0.001911\n",
            "epoch 614/1000   error=0.001906\n",
            "epoch 615/1000   error=0.001901\n",
            "epoch 616/1000   error=0.001896\n",
            "epoch 617/1000   error=0.001891\n",
            "epoch 618/1000   error=0.001886\n",
            "epoch 619/1000   error=0.001881\n",
            "epoch 620/1000   error=0.001876\n",
            "epoch 621/1000   error=0.001872\n",
            "epoch 622/1000   error=0.001867\n",
            "epoch 623/1000   error=0.001862\n",
            "epoch 624/1000   error=0.001857\n",
            "epoch 625/1000   error=0.001852\n",
            "epoch 626/1000   error=0.001848\n",
            "epoch 627/1000   error=0.001843\n",
            "epoch 628/1000   error=0.001838\n",
            "epoch 629/1000   error=0.001834\n",
            "epoch 630/1000   error=0.001829\n",
            "epoch 631/1000   error=0.001824\n",
            "epoch 632/1000   error=0.001820\n",
            "epoch 633/1000   error=0.001815\n",
            "epoch 634/1000   error=0.001811\n",
            "epoch 635/1000   error=0.001806\n",
            "epoch 636/1000   error=0.001802\n",
            "epoch 637/1000   error=0.001797\n",
            "epoch 638/1000   error=0.001793\n",
            "epoch 639/1000   error=0.001788\n",
            "epoch 640/1000   error=0.001784\n",
            "epoch 641/1000   error=0.001779\n",
            "epoch 642/1000   error=0.001775\n",
            "epoch 643/1000   error=0.001771\n",
            "epoch 644/1000   error=0.001766\n",
            "epoch 645/1000   error=0.001762\n",
            "epoch 646/1000   error=0.001758\n",
            "epoch 647/1000   error=0.001753\n",
            "epoch 648/1000   error=0.001749\n",
            "epoch 649/1000   error=0.001745\n",
            "epoch 650/1000   error=0.001741\n",
            "epoch 651/1000   error=0.001736\n",
            "epoch 652/1000   error=0.001732\n",
            "epoch 653/1000   error=0.001728\n",
            "epoch 654/1000   error=0.001724\n",
            "epoch 655/1000   error=0.001720\n",
            "epoch 656/1000   error=0.001716\n",
            "epoch 657/1000   error=0.001711\n",
            "epoch 658/1000   error=0.001707\n",
            "epoch 659/1000   error=0.001703\n",
            "epoch 660/1000   error=0.001699\n",
            "epoch 661/1000   error=0.001695\n",
            "epoch 662/1000   error=0.001691\n",
            "epoch 663/1000   error=0.001687\n",
            "epoch 664/1000   error=0.001683\n",
            "epoch 665/1000   error=0.001679\n",
            "epoch 666/1000   error=0.001675\n",
            "epoch 667/1000   error=0.001672\n",
            "epoch 668/1000   error=0.001668\n",
            "epoch 669/1000   error=0.001664\n",
            "epoch 670/1000   error=0.001660\n",
            "epoch 671/1000   error=0.001656\n",
            "epoch 672/1000   error=0.001652\n",
            "epoch 673/1000   error=0.001648\n",
            "epoch 674/1000   error=0.001645\n",
            "epoch 675/1000   error=0.001641\n",
            "epoch 676/1000   error=0.001637\n",
            "epoch 677/1000   error=0.001633\n",
            "epoch 678/1000   error=0.001630\n",
            "epoch 679/1000   error=0.001626\n",
            "epoch 680/1000   error=0.001622\n",
            "epoch 681/1000   error=0.001618\n",
            "epoch 682/1000   error=0.001615\n",
            "epoch 683/1000   error=0.001611\n",
            "epoch 684/1000   error=0.001607\n",
            "epoch 685/1000   error=0.001604\n",
            "epoch 686/1000   error=0.001600\n",
            "epoch 687/1000   error=0.001597\n",
            "epoch 688/1000   error=0.001593\n",
            "epoch 689/1000   error=0.001589\n",
            "epoch 690/1000   error=0.001586\n",
            "epoch 691/1000   error=0.001582\n",
            "epoch 692/1000   error=0.001579\n",
            "epoch 693/1000   error=0.001575\n",
            "epoch 694/1000   error=0.001572\n",
            "epoch 695/1000   error=0.001568\n",
            "epoch 696/1000   error=0.001565\n",
            "epoch 697/1000   error=0.001561\n",
            "epoch 698/1000   error=0.001558\n",
            "epoch 699/1000   error=0.001555\n",
            "epoch 700/1000   error=0.001551\n",
            "epoch 701/1000   error=0.001548\n",
            "epoch 702/1000   error=0.001545\n",
            "epoch 703/1000   error=0.001541\n",
            "epoch 704/1000   error=0.001538\n",
            "epoch 705/1000   error=0.001535\n",
            "epoch 706/1000   error=0.001532\n",
            "epoch 707/1000   error=0.001528\n",
            "epoch 708/1000   error=0.001526\n",
            "epoch 709/1000   error=0.001522\n",
            "epoch 710/1000   error=0.001520\n",
            "epoch 711/1000   error=0.001516\n",
            "epoch 712/1000   error=0.001515\n",
            "epoch 713/1000   error=0.001512\n",
            "epoch 714/1000   error=0.001511\n",
            "epoch 715/1000   error=0.001508\n",
            "epoch 716/1000   error=0.001509\n",
            "epoch 717/1000   error=0.001508\n",
            "epoch 718/1000   error=0.001511\n",
            "epoch 719/1000   error=0.001512\n",
            "epoch 720/1000   error=0.001519\n",
            "epoch 721/1000   error=0.001525\n",
            "epoch 722/1000   error=0.001540\n",
            "epoch 723/1000   error=0.001554\n",
            "epoch 724/1000   error=0.001583\n",
            "epoch 725/1000   error=0.001613\n",
            "epoch 726/1000   error=0.001667\n",
            "epoch 727/1000   error=0.001727\n",
            "epoch 728/1000   error=0.001824\n",
            "epoch 729/1000   error=0.001943\n",
            "epoch 730/1000   error=0.002115\n",
            "epoch 731/1000   error=0.002341\n",
            "epoch 732/1000   error=0.002645\n",
            "epoch 733/1000   error=0.003069\n",
            "epoch 734/1000   error=0.003592\n",
            "epoch 735/1000   error=0.004367\n",
            "epoch 736/1000   error=0.005231\n",
            "epoch 737/1000   error=0.006599\n",
            "epoch 738/1000   error=0.007921\n",
            "epoch 739/1000   error=0.010194\n",
            "epoch 740/1000   error=0.011947\n",
            "epoch 741/1000   error=0.015368\n",
            "epoch 742/1000   error=0.017131\n",
            "epoch 743/1000   error=0.021579\n",
            "epoch 744/1000   error=0.022405\n",
            "epoch 745/1000   error=0.027193\n",
            "epoch 746/1000   error=0.026114\n",
            "epoch 747/1000   error=0.030312\n",
            "epoch 748/1000   error=0.027216\n",
            "epoch 749/1000   error=0.030249\n",
            "epoch 750/1000   error=0.025948\n",
            "epoch 751/1000   error=0.027774\n",
            "epoch 752/1000   error=0.023250\n",
            "epoch 753/1000   error=0.024124\n",
            "epoch 754/1000   error=0.020011\n",
            "epoch 755/1000   error=0.020246\n",
            "epoch 756/1000   error=0.016800\n",
            "epoch 757/1000   error=0.016656\n",
            "epoch 758/1000   error=0.013900\n",
            "epoch 759/1000   error=0.013560\n",
            "epoch 760/1000   error=0.011418\n",
            "epoch 761/1000   error=0.011000\n",
            "epoch 762/1000   error=0.009362\n",
            "epoch 763/1000   error=0.008933\n",
            "epoch 764/1000   error=0.007694\n",
            "epoch 765/1000   error=0.007292\n",
            "epoch 766/1000   error=0.006359\n",
            "epoch 767/1000   error=0.006001\n",
            "epoch 768/1000   error=0.005301\n",
            "epoch 769/1000   error=0.004992\n",
            "epoch 770/1000   error=0.004466\n",
            "epoch 771/1000   error=0.004205\n",
            "epoch 772/1000   error=0.003810\n",
            "epoch 773/1000   error=0.003592\n",
            "epoch 774/1000   error=0.003296\n",
            "epoch 775/1000   error=0.003116\n",
            "epoch 776/1000   error=0.002893\n",
            "epoch 777/1000   error=0.002744\n",
            "epoch 778/1000   error=0.002576\n",
            "epoch 779/1000   error=0.002454\n",
            "epoch 780/1000   error=0.002327\n",
            "epoch 781/1000   error=0.002227\n",
            "epoch 782/1000   error=0.002130\n",
            "epoch 783/1000   error=0.002048\n",
            "epoch 784/1000   error=0.001974\n",
            "epoch 785/1000   error=0.001907\n",
            "epoch 786/1000   error=0.001851\n",
            "epoch 787/1000   error=0.001795\n",
            "epoch 788/1000   error=0.001752\n",
            "epoch 789/1000   error=0.001706\n",
            "epoch 790/1000   error=0.001672\n",
            "epoch 791/1000   error=0.001634\n",
            "epoch 792/1000   error=0.001608\n",
            "epoch 793/1000   error=0.001576\n",
            "epoch 794/1000   error=0.001556\n",
            "epoch 795/1000   error=0.001529\n",
            "epoch 796/1000   error=0.001513\n",
            "epoch 797/1000   error=0.001490\n",
            "epoch 798/1000   error=0.001478\n",
            "epoch 799/1000   error=0.001458\n",
            "epoch 800/1000   error=0.001449\n",
            "epoch 801/1000   error=0.001432\n",
            "epoch 802/1000   error=0.001424\n",
            "epoch 803/1000   error=0.001409\n",
            "epoch 804/1000   error=0.001403\n",
            "epoch 805/1000   error=0.001390\n",
            "epoch 806/1000   error=0.001385\n",
            "epoch 807/1000   error=0.001373\n",
            "epoch 808/1000   error=0.001369\n",
            "epoch 809/1000   error=0.001359\n",
            "epoch 810/1000   error=0.001355\n",
            "epoch 811/1000   error=0.001346\n",
            "epoch 812/1000   error=0.001343\n",
            "epoch 813/1000   error=0.001335\n",
            "epoch 814/1000   error=0.001332\n",
            "epoch 815/1000   error=0.001325\n",
            "epoch 816/1000   error=0.001323\n",
            "epoch 817/1000   error=0.001316\n",
            "epoch 818/1000   error=0.001314\n",
            "epoch 819/1000   error=0.001307\n",
            "epoch 820/1000   error=0.001305\n",
            "epoch 821/1000   error=0.001299\n",
            "epoch 822/1000   error=0.001298\n",
            "epoch 823/1000   error=0.001292\n",
            "epoch 824/1000   error=0.001290\n",
            "epoch 825/1000   error=0.001285\n",
            "epoch 826/1000   error=0.001284\n",
            "epoch 827/1000   error=0.001279\n",
            "epoch 828/1000   error=0.001277\n",
            "epoch 829/1000   error=0.001273\n",
            "epoch 830/1000   error=0.001271\n",
            "epoch 831/1000   error=0.001267\n",
            "epoch 832/1000   error=0.001265\n",
            "epoch 833/1000   error=0.001261\n",
            "epoch 834/1000   error=0.001259\n",
            "epoch 835/1000   error=0.001255\n",
            "epoch 836/1000   error=0.001254\n",
            "epoch 837/1000   error=0.001250\n",
            "epoch 838/1000   error=0.001248\n",
            "epoch 839/1000   error=0.001245\n",
            "epoch 840/1000   error=0.001243\n",
            "epoch 841/1000   error=0.001239\n",
            "epoch 842/1000   error=0.001238\n",
            "epoch 843/1000   error=0.001234\n",
            "epoch 844/1000   error=0.001233\n",
            "epoch 845/1000   error=0.001229\n",
            "epoch 846/1000   error=0.001228\n",
            "epoch 847/1000   error=0.001225\n",
            "epoch 848/1000   error=0.001223\n",
            "epoch 849/1000   error=0.001220\n",
            "epoch 850/1000   error=0.001218\n",
            "epoch 851/1000   error=0.001215\n",
            "epoch 852/1000   error=0.001213\n",
            "epoch 853/1000   error=0.001210\n",
            "epoch 854/1000   error=0.001209\n",
            "epoch 855/1000   error=0.001206\n",
            "epoch 856/1000   error=0.001204\n",
            "epoch 857/1000   error=0.001201\n",
            "epoch 858/1000   error=0.001200\n",
            "epoch 859/1000   error=0.001197\n",
            "epoch 860/1000   error=0.001195\n",
            "epoch 861/1000   error=0.001192\n",
            "epoch 862/1000   error=0.001191\n",
            "epoch 863/1000   error=0.001188\n",
            "epoch 864/1000   error=0.001186\n",
            "epoch 865/1000   error=0.001184\n",
            "epoch 866/1000   error=0.001182\n",
            "epoch 867/1000   error=0.001179\n",
            "epoch 868/1000   error=0.001178\n",
            "epoch 869/1000   error=0.001175\n",
            "epoch 870/1000   error=0.001174\n",
            "epoch 871/1000   error=0.001171\n",
            "epoch 872/1000   error=0.001169\n",
            "epoch 873/1000   error=0.001167\n",
            "epoch 874/1000   error=0.001165\n",
            "epoch 875/1000   error=0.001163\n",
            "epoch 876/1000   error=0.001161\n",
            "epoch 877/1000   error=0.001159\n",
            "epoch 878/1000   error=0.001157\n",
            "epoch 879/1000   error=0.001155\n",
            "epoch 880/1000   error=0.001153\n",
            "epoch 881/1000   error=0.001150\n",
            "epoch 882/1000   error=0.001149\n",
            "epoch 883/1000   error=0.001146\n",
            "epoch 884/1000   error=0.001145\n",
            "epoch 885/1000   error=0.001142\n",
            "epoch 886/1000   error=0.001141\n",
            "epoch 887/1000   error=0.001139\n",
            "epoch 888/1000   error=0.001137\n",
            "epoch 889/1000   error=0.001135\n",
            "epoch 890/1000   error=0.001133\n",
            "epoch 891/1000   error=0.001131\n",
            "epoch 892/1000   error=0.001129\n",
            "epoch 893/1000   error=0.001127\n",
            "epoch 894/1000   error=0.001125\n",
            "epoch 895/1000   error=0.001123\n",
            "epoch 896/1000   error=0.001121\n",
            "epoch 897/1000   error=0.001119\n",
            "epoch 898/1000   error=0.001118\n",
            "epoch 899/1000   error=0.001115\n",
            "epoch 900/1000   error=0.001114\n",
            "epoch 901/1000   error=0.001112\n",
            "epoch 902/1000   error=0.001110\n",
            "epoch 903/1000   error=0.001108\n",
            "epoch 904/1000   error=0.001106\n",
            "epoch 905/1000   error=0.001104\n",
            "epoch 906/1000   error=0.001103\n",
            "epoch 907/1000   error=0.001101\n",
            "epoch 908/1000   error=0.001099\n",
            "epoch 909/1000   error=0.001097\n",
            "epoch 910/1000   error=0.001095\n",
            "epoch 911/1000   error=0.001093\n",
            "epoch 912/1000   error=0.001092\n",
            "epoch 913/1000   error=0.001090\n",
            "epoch 914/1000   error=0.001088\n",
            "epoch 915/1000   error=0.001086\n",
            "epoch 916/1000   error=0.001085\n",
            "epoch 917/1000   error=0.001082\n",
            "epoch 918/1000   error=0.001081\n",
            "epoch 919/1000   error=0.001079\n",
            "epoch 920/1000   error=0.001077\n",
            "epoch 921/1000   error=0.001075\n",
            "epoch 922/1000   error=0.001074\n",
            "epoch 923/1000   error=0.001072\n",
            "epoch 924/1000   error=0.001070\n",
            "epoch 925/1000   error=0.001068\n",
            "epoch 926/1000   error=0.001067\n",
            "epoch 927/1000   error=0.001065\n",
            "epoch 928/1000   error=0.001064\n",
            "epoch 929/1000   error=0.001062\n",
            "epoch 930/1000   error=0.001060\n",
            "epoch 931/1000   error=0.001058\n",
            "epoch 932/1000   error=0.001057\n",
            "epoch 933/1000   error=0.001055\n",
            "epoch 934/1000   error=0.001053\n",
            "epoch 935/1000   error=0.001052\n",
            "epoch 936/1000   error=0.001050\n",
            "epoch 937/1000   error=0.001048\n",
            "epoch 938/1000   error=0.001047\n",
            "epoch 939/1000   error=0.001045\n",
            "epoch 940/1000   error=0.001044\n",
            "epoch 941/1000   error=0.001042\n",
            "epoch 942/1000   error=0.001040\n",
            "epoch 943/1000   error=0.001038\n",
            "epoch 944/1000   error=0.001037\n",
            "epoch 945/1000   error=0.001035\n",
            "epoch 946/1000   error=0.001034\n",
            "epoch 947/1000   error=0.001032\n",
            "epoch 948/1000   error=0.001031\n",
            "epoch 949/1000   error=0.001029\n",
            "epoch 950/1000   error=0.001028\n",
            "epoch 951/1000   error=0.001026\n",
            "epoch 952/1000   error=0.001024\n",
            "epoch 953/1000   error=0.001022\n",
            "epoch 954/1000   error=0.001021\n",
            "epoch 955/1000   error=0.001019\n",
            "epoch 956/1000   error=0.001018\n",
            "epoch 957/1000   error=0.001016\n",
            "epoch 958/1000   error=0.001015\n",
            "epoch 959/1000   error=0.001013\n",
            "epoch 960/1000   error=0.001012\n",
            "epoch 961/1000   error=0.001010\n",
            "epoch 962/1000   error=0.001009\n",
            "epoch 963/1000   error=0.001007\n",
            "epoch 964/1000   error=0.001006\n",
            "epoch 965/1000   error=0.001004\n",
            "epoch 966/1000   error=0.001003\n",
            "epoch 967/1000   error=0.001001\n",
            "epoch 968/1000   error=0.001000\n",
            "epoch 969/1000   error=0.000998\n",
            "epoch 970/1000   error=0.000997\n",
            "epoch 971/1000   error=0.000995\n",
            "epoch 972/1000   error=0.000995\n",
            "epoch 973/1000   error=0.000993\n",
            "epoch 974/1000   error=0.000992\n",
            "epoch 975/1000   error=0.000990\n",
            "epoch 976/1000   error=0.000989\n",
            "epoch 977/1000   error=0.000987\n",
            "epoch 978/1000   error=0.000986\n",
            "epoch 979/1000   error=0.000984\n",
            "epoch 980/1000   error=0.000984\n",
            "epoch 981/1000   error=0.000982\n",
            "epoch 982/1000   error=0.000981\n",
            "epoch 983/1000   error=0.000979\n",
            "epoch 984/1000   error=0.000978\n",
            "epoch 985/1000   error=0.000977\n",
            "epoch 986/1000   error=0.000976\n",
            "epoch 987/1000   error=0.000974\n",
            "epoch 988/1000   error=0.000974\n",
            "epoch 989/1000   error=0.000972\n",
            "epoch 990/1000   error=0.000971\n",
            "epoch 991/1000   error=0.000969\n",
            "epoch 992/1000   error=0.000969\n",
            "epoch 993/1000   error=0.000967\n",
            "epoch 994/1000   error=0.000967\n",
            "epoch 995/1000   error=0.000965\n",
            "epoch 996/1000   error=0.000965\n",
            "epoch 997/1000   error=0.000963\n",
            "epoch 998/1000   error=0.000963\n",
            "epoch 999/1000   error=0.000961\n",
            "epoch 1000/1000   error=0.000961\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 281
        },
        "id": "7YBSf_OY0PLi",
        "outputId": "074e9e90-6185-47a7-bd4f-6e8a5c0e10b5"
      },
      "source": [
        "pd.DataFrame(err).plot()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<matplotlib.axes._subplots.AxesSubplot at 0x7f8f771ea6d0>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 30
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAZxUlEQVR4nO3de3xU9Z3/8ddnZnJBbhISEQiYWFIqWLuUgEi1q1stlLZof62/hd1tbfVXHj5WW39uf799aLfbbu3utnt59LIr25a11m236621LbVYdn9WV6sWCVpv4WIQkCBoQOSihFzm8/tjzoSZyQQmySQnZ/J+Ph55ZM453znzOZzwnu9851zM3RERkeiLhV2AiIgUhwJdRKREKNBFREqEAl1EpEQo0EVESkQirBeurq72urq6sF5eRCSSNm3atN/da/ItCy3Q6+rqaGpqCuvlRUQiycx29bVMQy4iIiVCgS4iUiIU6CIiJSK0MXQRkbB0dnbS2tpKe3t72KX0qbKyktraWsrKygp+jgJdREad1tZWxo8fT11dHWYWdjm9uDsHDhygtbWV+vr6gp+nIRcRGXXa29uZPHnyiAxzADNj8uTJ/f4EoUAXkVFppIZ52kDqi1ygb9z5Ol//z610dCXDLkVEZESJXKA/tesg//TrFjq7FegiEl2/+tWvmD17NrNmzeJrX/taUdYZuUAf4Z+SREROqbu7m+uuu44HHniA5uZm7rzzTpqbmwe93sgFeprusyQiUfXkk08ya9Yszj77bMrLy1mxYgU///nPB73eyB22aKS66Lp1nogUw5d/8QLNrxwu6jrnTJvAlz48t8/le/bsYcaMGT3TtbW1bNiwYdCvG7keuoZcRETyi1wPPU39cxEphpP1pIfK9OnT2b17d890a2sr06dPH/R6I9dDT9OIi4hE1YIFC3jxxRfZsWMHHR0d3HXXXSxfvnzQ641cD32knwwgInIqiUSCW2+9lSVLltDd3c3VV1/N3LmD/6QQuUDvoR66iETYsmXLWLZsWVHXGbkhl3T/3JXoIiJZohfoGnEREcmroEA3s6VmttXMWszspjzLZ5rZQ2b2tJk9a2bF/RyRh74UFZHBGOnnsgykvlMGupnFgdXAB4A5wEozm5PT7AvAPe4+D1gB/Eu/KynQiSEXEZGBqays5MCBAyM21NPXQ6+srOzX8wr5UnQh0OLuLwGY2V3A5UDmhQccmBA8ngi80q8q+kFHuYjIYNXW1tLa2kpbW1vYpfQpfcei/igk0KcDuzOmW4Hzc9r8FfCfZvYZYCxwab4VmdkqYBXAzJkz+1VorpH6zioiI19ZWVm/7gQUFcX6UnQlcIe71wLLgB+aWa91u/sad29098aampoBvVC6g644FxHJVkig7wFmZEzXBvMyXQPcA+DuTwCVQHUxCsylARcRkfwKCfSNQIOZ1ZtZOakvPdfmtHkZeB+AmZ1DKtCHdHBKIy4iItlOGeju3gVcD6wHNpM6muUFM7vFzNIXH/gc8Gkzewa4E/ikD9UgdzDmohOLRESyFXTqv7uvA9blzPtixuNm4D3FLS2/niEX5bmISBadKSoiUiIiF+hp6qCLiGSLXKCfuAVdyIWIiIww0Qt0DbmIiOQVuUBP01EuIiLZIhfoPRfnUp6LiGSJXqBryEVEJK/IBXqaOugiItkiF+gnjnJRpIuIZIpcoOvqXCIi+UUv0APqoIuIZItcoKuDLiKSX/QCXYe5iIjkFblAT9OQi4hItsgFes+JRTpwUUQkS/QCXSMuIiJ5RS7Q0zTkIiKSLXKBnu6hK89FRLJFL9B1pqiISF7RC3SNoYuI5BW5QE9T/1xEJFt0A12JLiKSJXKBrjNFRUTyi1ygn6AuuohIpsgFum5BJyKSX/QCXSMuIiJ5RS7Q09RBFxHJFrlAP3FiUciFiIiMMNELdA25iIjkFblAT9Plc0VEskUu0HWUi4hIftELdA25iIjkFblAT1MPXUQkWwQDPTjKRWPoIiJZIhfoGnIREckvcoGepiEXEZFskQt0ddBFRPIrKNDNbKmZbTWzFjO7qY82/9PMms3sBTP7j+KWmfU6gHroIiK5EqdqYGZxYDVwGdAKbDSzte7enNGmAbgZeI+7HzSzM4aqYPXQRUTyK6SHvhBocfeX3L0DuAu4PKfNp4HV7n4QwN1fK26ZvekoFxGRbIUE+nRgd8Z0azAv09uBt5vZY2b2WzNbmm9FZrbKzJrMrKmtrW1ABaePctGQi4hItmJ9KZoAGoCLgZXAv5rZ6bmN3H2Nuze6e2NNTc2AXkiHLYqI5FdIoO8BZmRM1wbzMrUCa9290913ANtIBfyQUQddRCRbIYG+EWgws3ozKwdWAGtz2vyMVO8cM6smNQTzUhHr7HHieuiKdBGRTKcMdHfvAq4H1gObgXvc/QUzu8XMlgfN1gMHzKwZeAj4v+5+YEgq1pCLiEhepzxsEcDd1wHrcuZ9MeOxA38W/AwL9c9FRLJF9kxRjbiIiGSLXqDrMBcRkbwiF+gnqIsuIpIpcoGuIRcRkfyiF+gacRERyStygZ6mDrqISLbIBfqJE4tCLkREZISJXqBryEVEJK/IBXqaTv0XEckWuUDvOcol1CpEREaeyAU6uh66iEhekQt009W5RETyilygp+kWdCIi2SIX6KZBdBGRvKIX6GEXICIyQkUu0NPUQRcRyRa5QE9fPldHuYiIZItgoIddgYjIyBS5QE/TUS4iItkiF+i6HrqISH7RC3QNuYiI5BW5QE9TB11EJFsEAz19lIsiXUQkU+QCXUMuIiL5RS7Q09Q/FxHJFrlA7+mgK9FFRLJEL9A15iIiklfkAj1NJxaJiGSLXKDrxCIRkfyiF+i6BZ2ISF7RC3RdEV1EJK/IBXqaOugiItkiF+gnhlwU6SIimSIX6CIikl9kA139cxGRbJELdB3lIiKSX/QCXUe5iIjkVVCgm9lSM9tqZi1mdtNJ2n3UzNzMGotXYl/URRcRyXTKQDezOLAa+AAwB1hpZnPytBsP3ABsKHaR2a+T+q0hFxGRbIX00BcCLe7+krt3AHcBl+dp9xXg74D2ItbXi67NJSKSXyGBPh3YnTHdGszrYWbvBma4+y9PtiIzW2VmTWbW1NbW1u9iM6mDLiKSbdBfippZDPg68LlTtXX3Ne7e6O6NNTU1A3u9nlvQDejpIiIlq5BA3wPMyJiuDealjQfOBR42s53AImDtUH0xqiEXEZH8Cgn0jUCDmdWbWTmwAlibXujuh9y92t3r3L0O+C2w3N2bhqTi9Otq0EVEJMspA93du4DrgfXAZuAed3/BzG4xs+VDXWAuXQ9dRCS/RCGN3H0dsC5n3hf7aHvx4Mvqm4ZcRETyi9yZomnqoIuIZItcoKdvEq3L54qIZItcoJfFUiV3divQRUQyRS7QyxPpQE+GXImIyMgSuUAvi6eGXBToIiLZohfoQQ+9o0uBLiKSKXKBXh4PAl09dBGRLJEL9LIg0Du79KWoiEimyAV6PGbETGPoIiK5IhfokDrSRYEuIpItkoFeFo9pDF1EJEckA708HtNRLiIiOSIZ6GVxDbmIiOSKZqAnTKf+i4jkiGSgTzqtnNeODOm9qEVEIieSgT532gSe3X2ItiPHwy5FRGTEiGSgX7W4js5kkitWP8bjLfvDLkdEZESIZKC/48wJ3PnpRVQkYvzRbRu45RfNtHd2h12WiEioIhnoAPNmTuKXn72IT1xwFrc/toMrVj/G1n1Hwi5LRCQ0kQ10gDHlcW65/Fy+/8kF7D96nA/f+ht++nRr2GWJiIQi0oGedsk7zuCBG97LvBmnc+Pdz/DVBzbTndRhjSIyupREoAPUjK/g3//X+fzx+TP57n+/xKd/0MSbx7vCLktEZNiUTKBD6gzSv/nIO/nK5XP5721tXPmdJ9h3SMeri8joUFKBnvbxC+q47apGdh14kytWP0bzK4fDLklEZMiVZKADXDL7DO69djEAV37ncR7e+lrIFYmIDK2SDXSAOdMm8NPrFjNz8liu+bcm/mPDy2GXJCIyZEo60AGmThzDvddewIWzqvn8T5/jqw9sJqkjYESkBJV8oAOMq0jwvasae46A+cydT3OkvTPsskREimpUBDpAIh7jr684l88vewfrnt/L0m8+yiPb2sIuS0SkaEZNoAOYGave+zZ+fO1iKstifOL2J7n2h5vY3nY07NJERAZtVAV62vyzUteBufHSt/Poi228/xuPcOPdv+P5PYfCLk1EZMDMPZwvCBsbG72pqSmU187UduQ43354O3dvfJk3O7o5v76KlQtnsmTumYwpj4ddnohIFjPb5O6NeZeN9kBPO3Ssk7s3vswPnthF68FjjKtI8KHzpvKx+bXMP2sSZhZ2iSIiCvT+SCadDTte5ydPtbLuub281dFNffVYPja/lo/Mm86008eEXaKIjGIK9AF683gXDzy/j3ubdrNhx+uYwYWzqvnY/FqWzD2TyjINyYjI8FKgF8GuA2/yk02t/OSpPex54xjV48r51Hvq+ZNFZzFxTFnY5YnIKKFAL6Jk0nls+37+9dEdPLKtjfEVCT7zvll8cnE95YlRedCQiAyjkwV6QQlkZkvNbKuZtZjZTXmW/5mZNZvZs2b2oJmdNdiiR6pYzLiooYYfXL2QX372QhbUV/G367aw5JuP8NTLB8MuT0RGsVMGupnFgdXAB4A5wEozm5PT7Gmg0d3PA34M/H2xCx2J5k6byO2fXMD3P7WAzu4kV37nCdY8sp2wPvWIyOhWSA99IdDi7i+5ewdwF3B5ZgN3f8jd3womfwvUFrfMke2S2Wew7oaLWDJ3Cn+7bgtf/kWzLgAmIsOukECfDuzOmG4N5vXlGuCBfAvMbJWZNZlZU1tbaV1HZUJlGbeufDfXXFjPHY/v5G/WbQ67JBEZZRLFXJmZ/QnQCPx+vuXuvgZYA6kvRYv52iNBLGZ84YPnkHTne7/ZQV31WD6+qGS/ThCREaaQHvoeYEbGdG0wL4uZXQr8BbDc3Y8Xp7zoMTP+8oNzuGR2DV+5v5mt+46EXZKIjBKFBPpGoMHM6s2sHFgBrM1sYGbzgO+SCvNRf6+3WMz4hyvfxYTKMv7Pvc9oPF1EhsUpA93du4DrgfXAZuAed3/BzG4xs+VBs38AxgH3mtnvzGxtH6sbNarHVfCFD57Dc3sOcd/TvT7QiIgUnU4sGkLJpPM/vv04ew8d49E//wOdeCQigzboE4tkYGIx48bL3s6rh4+z9plXwi5HREqcAn2IvbehmtlTxnPboy/phCMRGVIK9CFmZlxzUT1b9h2haZcuDSAiQ0eBPgyWvXMqlWUxfqYvR0VkCCnQh8G4igSXzTmTXz63l46uZNjliEiJUqAPkyt+bxpvvNXJb1pK65IHIjJyKNCHyUUNNZxWHufBzaP+vCsRGSIK9GFSnoix+G3VPLy1TUe7iMiQUKAPo4tn17DnjWNsbzsadikiUoIU6MPo4tk1ADy8VePoIlJ8CvRhVDvpNM6uHssT2w+EXYqIlCAF+jBbUFdF066DugKjiBSdAn2YLaiv4tCxTra9puuki0hxKdCH2cK6KgA27ng95EpEpNQo0IfZjKoxTJlQwZM7dV0XESkuBfowMzMW1FWxccfrOh5dRIpKgR6CBXVV7DvcTuvBY2GXIiIlRIEeggXBOHrTLo2ji0jxKNBDMPvM8YyvSLBR4+giUkQK9BDEY8b8ukk60kVEikqBHpIFdVW8+NpRDr7ZEXYpIlIiFOghaTxrEgCbdFs6ESkSBXpI3jXjdMrjMTbqi1ERKRIFekgqy+K8s3YiTfpiVESKRIEeosa6STzb+gbtnd1hlyIiJUCBHqKFdVV0djtPaRxdRIpAgR6iRWdPpjwR48Etus+oiAyeAj1EYysSLH7bZB7c/Kqu6yIig6ZAD9ml50xh54G32LJP10cXkcFRoIds2TunUh6PcU/T7rBLEZGIU6CHrGpsOZfNncJPn97DsQ4d7SIiA6dAHwE+tbiON97q5N+e2Bl2KSISYQr0EaCxropLZtfwLw+1sO9Qe9jliEhEKdBHiL/80Bw6u50b7/4dHV3JsMsRkQhSoI8QZ9eM46+vOJcnXjrAn/5oE0ePd4VdkohEjAJ9BPno/Fq+cvlcHtraxof/+Tf8eouOTxeRwiXCLkCyffyCOhqmjOfm+57j6juaaDhjHB+dX8vFs2uYPWU8ZhZ2iSIyQlkhPUAzWwp8C4gDt7n713KWVwA/AOYDB4A/dPedJ1tnY2OjNzU1DbDs0tfRleRnT+/hRxt28UzrIQAmnVbGOVMncM7UCdRVj2XaxErOnFjJ1IljmDimjHhMYS/DZ3vbUe7ZuJuxFQk+eN5U3lYzLuySRgUz2+TujXmXnSrQzSwObAMuA1qBjcBKd2/OaPOnwHnufq2ZrQA+4u5/eLL1KtALt/fQMR7dtp9Nuw6yZd9htr56hPbO7C9OzWBcRYLTTytj4pjUz5iyBJVlMSrL4owpi/c8riyLU5GIUZ6IEY8ZZbHU70TcSASPy+IW/A6WxYxEPEbcDDOImRGLgWHEDMxSv2MZy3vaBY8zp2OWeq7F6JmOBZ8+LL0seH+ynm20jMcn5snw6exOsnnvYe54fCf3PbUna9ncaRO4bM4UzqudyNSJYxhXkWBCZRnjKhPqbBTRYAP9AuCv3H1JMH0zgLt/NaPN+qDNE2aWAPYBNX6SlSvQB6476ew/epxX3jjG3kPt7DvUzhvHOjl8rJNDGT/HOrpp7wx+upI9j5MlOiyf7w0gPX1iWXajfMtSbyi915F+0GtZ1rqy21vGE3Pfe3r/7/A+l2VOZv63yl1F5vNy//t5nxOFr/94V5LupFMWNz6+qI4r5k3jaHsXD29r45FtbWx99Uie7YKx5XHGlCeCN/8Tb9jpfzPLM/9UCn0zL6hVge83xarrhvc18OF3TSvsRXuvv89AL2QMfTqQeV56K3B+X23cvcvMDgGTgf05hawCVgHMnDmzoOKlt3jMmDKhkikTKpnXz+e6Ox3dSdo7k3R1p/5zdiad7m6nMxlMB/O7kk5Xt9OVTNLV7T3z3J2kc+I3mdNOMpkKgqSfaJt6TNZ0+jmeMZ0M0sB7fgd1Zz3uvSw9kc6SzLZZ7XLX02vZydv3CtqM1z1ZfSee52TGQu7//czJ3svyPy83Pk4WKNnPs5Msyz+/IhGnYco4LmqooWpsec/8xbOq+fyyczh6vIvNew+z/8hxjrR3cbi9k6PHuzjS3sVbHd1A+u8j+HuAnr+L1GPv9SaST6HHChS2rsJWVlCrAuuaOKassIb9NKxfirr7GmANpHrow/nakmJmVCTiVCTiYZciJWhcRYIFdVVhlzFqFXLY4h5gRsZ0bTAvb5tgyGUiqS9HRURkmBQS6BuBBjOrN7NyYAWwNqfNWuCq4PHHgF+fbPxcRESK75RDLsGY+PXAelKHLd7u7i+Y2S1Ak7uvBb4H/NDMWoDXSYW+iIgMo4LG0N19HbAuZ94XMx63A1cWtzQREekPnfovIlIiFOgiIiVCgS4iUiIU6CIiJaKgi3MNyQubtQG7Bvj0anLOQh0FtM2jg7Z5dBjMNp/l7jX5FoQW6INhZk19XcugVGmbRwdt8+gwVNusIRcRkRKhQBcRKRFRDfQ1YRcQAm3z6KBtHh2GZJsjOYYuIiK9RbWHLiIiORToIiIlInKBbmZLzWyrmbWY2U1h11MsZjbDzB4ys2Yze8HMbgjmV5nZf5nZi8HvScF8M7N/Cv4dnjWzd4e7BQNjZnEze9rM7g+m681sQ7BddweXbMbMKoLplmB5XZh1D5SZnW5mPzazLWa22cwuGAX7+Mbgb/p5M7vTzCpLcT+b2e1m9pqZPZ8xr9/71syuCtq/aGZX5XutvkQq0IMbVq8GPgDMAVaa2ZxwqyqaLuBz7j4HWARcF2zbTcCD7t4APBhMQ+rfoCH4WQV8e/hLLoobgM0Z038HfMPdZwEHgWuC+dcAB4P53wjaRdG3gF+5+zuAd5Ha9pLdx2Y2Hfgs0Oju55K6BPcKSnM/3wEszZnXr31rZlXAl0jd5nMh8KX0m0BBPLjvYxR+gAuA9RnTNwM3h13XEG3rz4HLgK3A1GDeVGBr8Pi7wMqM9j3tovJD6u5XDwJ/ANxP6laW+4FE7v4mdT3+C4LHiaCdhb0N/dzeicCO3LpLfB+n7zdcFey3+4ElpbqfgTrg+YHuW2Al8N2M+VntTvUTqR46+W9YPT2kWoZM8DFzHrABmOLue4NF+4ApweNS+Lf4JvDnQDKYngy84e5dwXTmNmXdiBxI34g8SuqBNuD7wTDTbWY2lhLex+6+B/hH4GVgL6n9tonS3s+Z+rtvB7XPoxboJc/MxgE/Af63ux/OXOapt+ySOM7UzD4EvObum8KuZRglgHcD33b3ecCbnPgIDpTWPgYIhgsuJ/VmNg0YS+9hiVFhOPZt1AK9kBtWR5aZlZEK8x+5+33B7FfNbGqwfCrwWjA/6v8W7wGWm9lO4C5Swy7fAk4PbjQO2dtUCjcibwVa3X1DMP1jUgFfqvsY4FJgh7u3uXsncB+pfV/K+zlTf/ftoPZ51AK9kBtWR5KZGal7s252969nLMq8AfdVpMbW0/M/EXxbvgg4lPHRbsRz95vdvdbd60jtx1+7+x8DD5G60Tj03t5I34jc3fcBu81sdjDrfUAzJbqPAy8Di8zstOBvPL3NJbufc/R3364H3m9mk4JPN+8P5hUm7C8RBvClwzJgG7Ad+Iuw6ynidl1I6uPYs8Dvgp9lpMYPHwReBP4fUBW0N1JH/GwHniN1FEHo2zHAbb8YuD94fDbwJNAC3AtUBPMrg+mWYPnZYdc9wG39PaAp2M8/AyaV+j4GvgxsAZ4HfghUlOJ+Bu4k9T1BJ6lPY9cMZN8CVwfb3wJ8qj816NR/EZESEbUhFxER6YMCXUSkRCjQRURKhAJdRKREKNBFREqEAl1EpEQo0EVESsT/B772XdPezGMKAAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Se9nejmt1p3L",
        "outputId": "5bb6b356-2d7e-43a9-8726-9782374160f8"
      },
      "source": [
        "out = net.predict(x_train)\n",
        "print(x_train, out)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[[[0 0]]\n",
            "\n",
            " [[0 1]]\n",
            "\n",
            " [[1 0]]\n",
            "\n",
            " [[1 1]]] [array([[0.00043794]]), array([[0.97852756]]), array([[0.97771572]]), array([[-0.00452103]])]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Fe0xmRlK2cZf",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "ed7fa44f-f37e-43b6-f180-4dbb26cc4187"
      },
      "source": [
        "###### Digits recognitin\n",
        "from keras.datasets import mnist\n",
        "from keras.utils import np_utils\n",
        "\n",
        "(x_train, y_train), (x_test, y_test) = mnist.load_data()\n",
        "x_train = x_train.reshape(x_train.shape[0], 1, 28*28).astype('float32')/255\n",
        "y_train = np_utils.to_categorical(y_train)\n",
        "\n",
        "x_test = x_test.reshape(x_test.shape[0], 1, 28*28).astype('float32')/255\n",
        "y_test = np_utils.to_categorical(y_test)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Downloading data from https://storage.googleapis.com/tensorflow/tf-keras-datasets/mnist.npz\n",
            "11493376/11490434 [==============================] - 0s 0us/step\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "JGhSWfaU5o7G",
        "outputId": "5855c39d-a041-46dc-db63-4aca90f3040c"
      },
      "source": [
        "y_train[0]"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([0., 0., 0., 0., 0., 1., 0., 0., 0., 0.], dtype=float32)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 33
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "PfILnMdC3uqL",
        "outputId": "37d66d00-1a71-4910-e0f2-8828923d3811"
      },
      "source": [
        "net = Network()\n",
        "net.add(FCLayer(28*28, 100))\n",
        "net.add(ActivationLayer())\n",
        "net.add(FCLayer(100, 50)) \n",
        "net.add(ActivationLayer())\n",
        "net.add(FCLayer(50, 10)) \n",
        "net.add(ActivationLayer())\n",
        "net.use(mse, mse_prime)\n",
        "\n",
        "errors = net.fit(x_train[0:5000], y_train[0:5000], epochs=35, learning_rate=0.1)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "epoch 1/35   error=597.229832\n",
            "epoch 2/35   error=294.311087\n",
            "epoch 3/35   error=222.290965\n",
            "epoch 4/35   error=182.827325\n",
            "epoch 5/35   error=156.561311\n",
            "epoch 6/35   error=137.362874\n",
            "epoch 7/35   error=122.995554\n",
            "epoch 8/35   error=111.681362\n",
            "epoch 9/35   error=102.327054\n",
            "epoch 10/35   error=94.795020\n",
            "epoch 11/35   error=88.419284\n",
            "epoch 12/35   error=82.838767\n",
            "epoch 13/35   error=78.053786\n",
            "epoch 14/35   error=73.588500\n",
            "epoch 15/35   error=69.571904\n",
            "epoch 16/35   error=66.085256\n",
            "epoch 17/35   error=62.907259\n",
            "epoch 18/35   error=59.947271\n",
            "epoch 19/35   error=57.193714\n",
            "epoch 20/35   error=54.729624\n",
            "epoch 21/35   error=52.550492\n",
            "epoch 22/35   error=50.488494\n",
            "epoch 23/35   error=48.678247\n",
            "epoch 24/35   error=47.048339\n",
            "epoch 25/35   error=45.462000\n",
            "epoch 26/35   error=43.884629\n",
            "epoch 27/35   error=42.421490\n",
            "epoch 28/35   error=41.161154\n",
            "epoch 29/35   error=39.919483\n",
            "epoch 30/35   error=38.800290\n",
            "epoch 31/35   error=37.824970\n",
            "epoch 32/35   error=36.865852\n",
            "epoch 33/35   error=35.948138\n",
            "epoch 34/35   error=35.107472\n",
            "epoch 35/35   error=34.259474\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QElfsG8W6TEk"
      },
      "source": [
        "errors=[]\n",
        "for i in range(1000):\n",
        "  out=sum((net.predict(x_test[i]) - y_test[i])[0][0])\n",
        "  errors.append(0 if out<0.5 else 1)\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "04Fj4DyV8OIt",
        "outputId": "30906536-20a2-4843-e5e3-00c4ffdd3477"
      },
      "source": [
        "np.mean(errors)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.028"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 36
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vjTT4Da59tWz"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "nHLxeJrU9zGz"
      },
      "source": [
        "Extend Digits network to hhandwritten characters \n",
        "\n",
        "Use NIST dataset: EMNIST Letters: 145,600 characters. 26 balanced classes.\n",
        "https://www.nist.gov/itl/products-and-services/emnist-dataset\n",
        "\n",
        "* Play with NN configurations -- number of layers, ssize of layers\n",
        "* Please try to break it\n",
        "* Throw in some random noise \n",
        "* Replace tangent with another function (logit/ sigmioid???)\n"
      ]
    }
  ]
}