{
"nbformat": 4,
"nbformat_minor": 0,
"metadata": {
"colab": {
"name": "NeuralNetworks.ipynb",
"provenance": []
},
"kernelspec": {
"name": "python3",
"display_name": "Python 3"
}
},
"cells": [
{
"cell_type": "code",
"metadata": {
"id": "OduD_pshW6Kb"
},
"source": [
"from keras.datasets import mnist\n",
"import numpy as np\n",
"import pandas as pd\n"
],
"execution_count": null,
"outputs": []
},
{
"cell_type": "code",
"metadata": {
"id": "uM3w-SXXrn4u"
},
"source": [
"# Base class\n",
"class Layer:\n",
" def __init__(self):\n",
" self.input = None\n",
" self.output = None\n",
"\n",
" # computes the output Y of a layer for a given input X\n",
" def forward_propagation(self, input):\n",
" raise NotImplementedError\n",
"\n",
" # computes dE/dX for a given dE/dY (and update parameters if any)\n",
" def backward_propagation(self, output_error, learning_rate):\n",
" raise NotImplementedError"
],
"execution_count": null,
"outputs": []
},
{
"cell_type": "code",
"metadata": {
"id": "Lbh6q6qvrt2q"
},
"source": [
"class FCLayer(Layer):\n",
" def __init__(self, input_size, output_size):\n",
" self.weights = np.random.rand(input_size, output_size) - 0.5\n",
" self.bias = np.random.rand(1, output_size) - 0.5\n",
" \n",
" def forward_propagation(self,input_data):\n",
" self.input = input_data\n",
" self.output = np.dot(self.input,self.weights)+self.bias\n",
" return(self.output)\n",
"\n",
" ## compute dE/dw, de/dB for a given outputt erroor. Returns input error to be given t the next layer\n",
" def backward_propagation(self,output_error, learning_rate):\n",
" input_error = np.dot(output_error, self.weights.T)\n",
" weights_error = np.dot(self.input.T, output_error)\n",
"\n",
" self.weights -= learning_rate * weights_error\n",
" self.bias -= learning_rate * output_error\n",
" return(input_error)"
],
"execution_count": null,
"outputs": []
},
{
"cell_type": "code",
"metadata": {
"id": "KatIfnmxtxmW"
},
"source": [
"\n",
"class ActivationLayer(Layer):\n",
" def __init__(self):\n",
" self.activation = lambda x: np.tanh(x)\n",
" self.activation_prime = lambda x: 1-np.tanh(x)**2;\n",
" \n",
" def forward_propagation(self, input_data):\n",
" self.input = input_data\n",
" self.output = self.activation(self.input)\n",
" return(self.output)\n",
" \n",
" def backward_propagation(self, output_error, learning_rate):\n",
" return(self.activation_prime(self.input) * output_error)"
],
"execution_count": null,
"outputs": []
},
{
"cell_type": "code",
"metadata": {
"id": "LplgFhjivh8S"
},
"source": [
"# loss function and its derivative\n",
"def mse(y_true, y_pred):\n",
" return np.mean(np.power(y_true - y_pred, 2));\n",
"\n",
"def mse_prime(y_true, y_pred):\n",
" return 2*(y_pred-y_true)/y_true.size;"
],
"execution_count": null,
"outputs": []
},
{
"cell_type": "code",
"metadata": {
"id": "jMuCe0CgwieD"
},
"source": [
"class Network:\n",
" def __init__(self):\n",
" self.layers=[]\n",
" self.loss=None\n",
" self.loss_prime=None\n",
"\n",
" def add(self,layer):\n",
" self.layers.append(layer)\n",
"\n",
" def use(self,loss,loss_prime):\n",
" self.loss = loss\n",
" self.loss_prime = loss_prime\n",
"\n",
" def predict(self,input_data):\n",
" result=[]\n",
" for i in range(len(input_data)):\n",
" output=input_data[i]\n",
" for layer in self.layers:\n",
" output= layer.forward_propagation(output)\n",
" result.append(output)\n",
" return(result)\n",
"\n",
" def fit(self, x_train, y_train, epochs, learning_rate):\n",
" samples = len(x_train)\n",
" errors=[]\n",
"\n",
" for i in range(epochs):\n",
" err = 0\n",
" for j in range(samples):\n",
" output = x_train[j]\n",
" for layer in self.layers:\n",
" output = layer.forward_propagation(output)\n",
" \n",
" err += self.loss(y_train[j], output)\n",
"\n",
" error = self.loss_prime(y_train[j], output)\n",
" for layer in reversed(self.layers):\n",
" error = layer.backward_propagation(error, learning_rate)\n",
" errors.append(err/samples)\n",
"\n",
" print('epoch %d/%d error=%f' % (i+1, epochs, err))\n",
" return(errors)\n",
"\n"
],
"execution_count": null,
"outputs": []
},
{
"cell_type": "code",
"metadata": {
"id": "TK_LwCo3xITw"
},
"source": [
"x_train = np.array([[[0,0]], [[0,1]], [[1,0]], [[1,1]]])\n",
"y_train = np.array([[[0]], [[1]], [[1]], [[0]]])\n"
],
"execution_count": null,
"outputs": []
},
{
"cell_type": "code",
"metadata": {
"id": "tFvsRjeuzLIr"
},
"source": [
"net = Network()\n",
"net.add(FCLayer(2,5))\n",
"net.add(ActivationLayer())\n",
"net.add(FCLayer(5,1))\n",
"net.add(ActivationLayer())\n",
"net.use(mse, mse_prime)"
],
"execution_count": null,
"outputs": []
},
{
"cell_type": "code",
"metadata": {
"id": "l2tIbwjFz5yO",
"colab": {
"base_uri": "https://localhost:8080/"
},
"outputId": "10cb3f69-d635-4e2b-9c72-214b74e1ca0c"
},
"source": [
"err = net.fit(x_train, y_train, epochs=1000, learning_rate=0.1)"
],
"execution_count": null,
"outputs": [
{
"output_type": "stream",
"text": [
"epoch 1/1000 error=3.439531\n",
"epoch 2/1000 error=1.469290\n",
"epoch 3/1000 error=1.249098\n",
"epoch 4/1000 error=1.221891\n",
"epoch 5/1000 error=1.213275\n",
"epoch 6/1000 error=1.208269\n",
"epoch 7/1000 error=1.204283\n",
"epoch 8/1000 error=1.200643\n",
"epoch 9/1000 error=1.197125\n",
"epoch 10/1000 error=1.193641\n",
"epoch 11/1000 error=1.190152\n",
"epoch 12/1000 error=1.186639\n",
"epoch 13/1000 error=1.183090\n",
"epoch 14/1000 error=1.179496\n",
"epoch 15/1000 error=1.175854\n",
"epoch 16/1000 error=1.172157\n",
"epoch 17/1000 error=1.168403\n",
"epoch 18/1000 error=1.164588\n",
"epoch 19/1000 error=1.160709\n",
"epoch 20/1000 error=1.156765\n",
"epoch 21/1000 error=1.152753\n",
"epoch 22/1000 error=1.148673\n",
"epoch 23/1000 error=1.144523\n",
"epoch 24/1000 error=1.140304\n",
"epoch 25/1000 error=1.136016\n",
"epoch 26/1000 error=1.131659\n",
"epoch 27/1000 error=1.127236\n",
"epoch 28/1000 error=1.122749\n",
"epoch 29/1000 error=1.118200\n",
"epoch 30/1000 error=1.113594\n",
"epoch 31/1000 error=1.108935\n",
"epoch 32/1000 error=1.104229\n",
"epoch 33/1000 error=1.099480\n",
"epoch 34/1000 error=1.094695\n",
"epoch 35/1000 error=1.089882\n",
"epoch 36/1000 error=1.085048\n",
"epoch 37/1000 error=1.080200\n",
"epoch 38/1000 error=1.075348\n",
"epoch 39/1000 error=1.070499\n",
"epoch 40/1000 error=1.065664\n",
"epoch 41/1000 error=1.060849\n",
"epoch 42/1000 error=1.056065\n",
"epoch 43/1000 error=1.051319\n",
"epoch 44/1000 error=1.046620\n",
"epoch 45/1000 error=1.041977\n",
"epoch 46/1000 error=1.037396\n",
"epoch 47/1000 error=1.032884\n",
"epoch 48/1000 error=1.028448\n",
"epoch 49/1000 error=1.024094\n",
"epoch 50/1000 error=1.019825\n",
"epoch 51/1000 error=1.015647\n",
"epoch 52/1000 error=1.011563\n",
"epoch 53/1000 error=1.007575\n",
"epoch 54/1000 error=1.003685\n",
"epoch 55/1000 error=0.999895\n",
"epoch 56/1000 error=0.996204\n",
"epoch 57/1000 error=0.992613\n",
"epoch 58/1000 error=0.989120\n",
"epoch 59/1000 error=0.985724\n",
"epoch 60/1000 error=0.982424\n",
"epoch 61/1000 error=0.979217\n",
"epoch 62/1000 error=0.976100\n",
"epoch 63/1000 error=0.973071\n",
"epoch 64/1000 error=0.970126\n",
"epoch 65/1000 error=0.967262\n",
"epoch 66/1000 error=0.964476\n",
"epoch 67/1000 error=0.961764\n",
"epoch 68/1000 error=0.959122\n",
"epoch 69/1000 error=0.956546\n",
"epoch 70/1000 error=0.954034\n",
"epoch 71/1000 error=0.951581\n",
"epoch 72/1000 error=0.949184\n",
"epoch 73/1000 error=0.946839\n",
"epoch 74/1000 error=0.944543\n",
"epoch 75/1000 error=0.942292\n",
"epoch 76/1000 error=0.940085\n",
"epoch 77/1000 error=0.937916\n",
"epoch 78/1000 error=0.935783\n",
"epoch 79/1000 error=0.933684\n",
"epoch 80/1000 error=0.931615\n",
"epoch 81/1000 error=0.929575\n",
"epoch 82/1000 error=0.927559\n",
"epoch 83/1000 error=0.925565\n",
"epoch 84/1000 error=0.923592\n",
"epoch 85/1000 error=0.921636\n",
"epoch 86/1000 error=0.919696\n",
"epoch 87/1000 error=0.917768\n",
"epoch 88/1000 error=0.915851\n",
"epoch 89/1000 error=0.913942\n",
"epoch 90/1000 error=0.912039\n",
"epoch 91/1000 error=0.910140\n",
"epoch 92/1000 error=0.908242\n",
"epoch 93/1000 error=0.906344\n",
"epoch 94/1000 error=0.904442\n",
"epoch 95/1000 error=0.902535\n",
"epoch 96/1000 error=0.900620\n",
"epoch 97/1000 error=0.898696\n",
"epoch 98/1000 error=0.896758\n",
"epoch 99/1000 error=0.894806\n",
"epoch 100/1000 error=0.892836\n",
"epoch 101/1000 error=0.890845\n",
"epoch 102/1000 error=0.888832\n",
"epoch 103/1000 error=0.886793\n",
"epoch 104/1000 error=0.884725\n",
"epoch 105/1000 error=0.882626\n",
"epoch 106/1000 error=0.880491\n",
"epoch 107/1000 error=0.878319\n",
"epoch 108/1000 error=0.876105\n",
"epoch 109/1000 error=0.873845\n",
"epoch 110/1000 error=0.871537\n",
"epoch 111/1000 error=0.869176\n",
"epoch 112/1000 error=0.866757\n",
"epoch 113/1000 error=0.864278\n",
"epoch 114/1000 error=0.861732\n",
"epoch 115/1000 error=0.859115\n",
"epoch 116/1000 error=0.856422\n",
"epoch 117/1000 error=0.853648\n",
"epoch 118/1000 error=0.850786\n",
"epoch 119/1000 error=0.847830\n",
"epoch 120/1000 error=0.844773\n",
"epoch 121/1000 error=0.841609\n",
"epoch 122/1000 error=0.838330\n",
"epoch 123/1000 error=0.834928\n",
"epoch 124/1000 error=0.831394\n",
"epoch 125/1000 error=0.827718\n",
"epoch 126/1000 error=0.823891\n",
"epoch 127/1000 error=0.819901\n",
"epoch 128/1000 error=0.815738\n",
"epoch 129/1000 error=0.811388\n",
"epoch 130/1000 error=0.806838\n",
"epoch 131/1000 error=0.802073\n",
"epoch 132/1000 error=0.797079\n",
"epoch 133/1000 error=0.791837\n",
"epoch 134/1000 error=0.786330\n",
"epoch 135/1000 error=0.780538\n",
"epoch 136/1000 error=0.774440\n",
"epoch 137/1000 error=0.768014\n",
"epoch 138/1000 error=0.761234\n",
"epoch 139/1000 error=0.754075\n",
"epoch 140/1000 error=0.746509\n",
"epoch 141/1000 error=0.738505\n",
"epoch 142/1000 error=0.730032\n",
"epoch 143/1000 error=0.721055\n",
"epoch 144/1000 error=0.711538\n",
"epoch 145/1000 error=0.701443\n",
"epoch 146/1000 error=0.690730\n",
"epoch 147/1000 error=0.679356\n",
"epoch 148/1000 error=0.667280\n",
"epoch 149/1000 error=0.654457\n",
"epoch 150/1000 error=0.640844\n",
"epoch 151/1000 error=0.626401\n",
"epoch 152/1000 error=0.611087\n",
"epoch 153/1000 error=0.594869\n",
"epoch 154/1000 error=0.577720\n",
"epoch 155/1000 error=0.559625\n",
"epoch 156/1000 error=0.540582\n",
"epoch 157/1000 error=0.520606\n",
"epoch 158/1000 error=0.499736\n",
"epoch 159/1000 error=0.478038\n",
"epoch 160/1000 error=0.455608\n",
"epoch 161/1000 error=0.432576\n",
"epoch 162/1000 error=0.409107\n",
"epoch 163/1000 error=0.385399\n",
"epoch 164/1000 error=0.361679\n",
"epoch 165/1000 error=0.338191\n",
"epoch 166/1000 error=0.315186\n",
"epoch 167/1000 error=0.292907\n",
"epoch 168/1000 error=0.271572\n",
"epoch 169/1000 error=0.251362\n",
"epoch 170/1000 error=0.232410\n",
"epoch 171/1000 error=0.214797\n",
"epoch 172/1000 error=0.198556\n",
"epoch 173/1000 error=0.183675\n",
"epoch 174/1000 error=0.170107\n",
"epoch 175/1000 error=0.157781\n",
"epoch 176/1000 error=0.146609\n",
"epoch 177/1000 error=0.136496\n",
"epoch 178/1000 error=0.127345\n",
"epoch 179/1000 error=0.119061\n",
"epoch 180/1000 error=0.111556\n",
"epoch 181/1000 error=0.104747\n",
"epoch 182/1000 error=0.098560\n",
"epoch 183/1000 error=0.092927\n",
"epoch 184/1000 error=0.087789\n",
"epoch 185/1000 error=0.083091\n",
"epoch 186/1000 error=0.078788\n",
"epoch 187/1000 error=0.074837\n",
"epoch 188/1000 error=0.071202\n",
"epoch 189/1000 error=0.067850\n",
"epoch 190/1000 error=0.064754\n",
"epoch 191/1000 error=0.061887\n",
"epoch 192/1000 error=0.059227\n",
"epoch 193/1000 error=0.056755\n",
"epoch 194/1000 error=0.054454\n",
"epoch 195/1000 error=0.052307\n",
"epoch 196/1000 error=0.050301\n",
"epoch 197/1000 error=0.048423\n",
"epoch 198/1000 error=0.046663\n",
"epoch 199/1000 error=0.045010\n",
"epoch 200/1000 error=0.043456\n",
"epoch 201/1000 error=0.041993\n",
"epoch 202/1000 error=0.040613\n",
"epoch 203/1000 error=0.039310\n",
"epoch 204/1000 error=0.038079\n",
"epoch 205/1000 error=0.036913\n",
"epoch 206/1000 error=0.035809\n",
"epoch 207/1000 error=0.034761\n",
"epoch 208/1000 error=0.033766\n",
"epoch 209/1000 error=0.032820\n",
"epoch 210/1000 error=0.031920\n",
"epoch 211/1000 error=0.031062\n",
"epoch 212/1000 error=0.030244\n",
"epoch 213/1000 error=0.029464\n",
"epoch 214/1000 error=0.028718\n",
"epoch 215/1000 error=0.028005\n",
"epoch 216/1000 error=0.027324\n",
"epoch 217/1000 error=0.026671\n",
"epoch 218/1000 error=0.026045\n",
"epoch 219/1000 error=0.025445\n",
"epoch 220/1000 error=0.024869\n",
"epoch 221/1000 error=0.024316\n",
"epoch 222/1000 error=0.023784\n",
"epoch 223/1000 error=0.023273\n",
"epoch 224/1000 error=0.022782\n",
"epoch 225/1000 error=0.022308\n",
"epoch 226/1000 error=0.021852\n",
"epoch 227/1000 error=0.021412\n",
"epoch 228/1000 error=0.020988\n",
"epoch 229/1000 error=0.020579\n",
"epoch 230/1000 error=0.020184\n",
"epoch 231/1000 error=0.019802\n",
"epoch 232/1000 error=0.019433\n",
"epoch 233/1000 error=0.019076\n",
"epoch 234/1000 error=0.018731\n",
"epoch 235/1000 error=0.018397\n",
"epoch 236/1000 error=0.018073\n",
"epoch 237/1000 error=0.017760\n",
"epoch 238/1000 error=0.017456\n",
"epoch 239/1000 error=0.017162\n",
"epoch 240/1000 error=0.016876\n",
"epoch 241/1000 error=0.016599\n",
"epoch 242/1000 error=0.016330\n",
"epoch 243/1000 error=0.016069\n",
"epoch 244/1000 error=0.015815\n",
"epoch 245/1000 error=0.015568\n",
"epoch 246/1000 error=0.015329\n",
"epoch 247/1000 error=0.015095\n",
"epoch 248/1000 error=0.014869\n",
"epoch 249/1000 error=0.014648\n",
"epoch 250/1000 error=0.014433\n",
"epoch 251/1000 error=0.014224\n",
"epoch 252/1000 error=0.014020\n",
"epoch 253/1000 error=0.013822\n",
"epoch 254/1000 error=0.013628\n",
"epoch 255/1000 error=0.013440\n",
"epoch 256/1000 error=0.013256\n",
"epoch 257/1000 error=0.013076\n",
"epoch 258/1000 error=0.012901\n",
"epoch 259/1000 error=0.012730\n",
"epoch 260/1000 error=0.012564\n",
"epoch 261/1000 error=0.012401\n",
"epoch 262/1000 error=0.012242\n",
"epoch 263/1000 error=0.012086\n",
"epoch 264/1000 error=0.011935\n",
"epoch 265/1000 error=0.011786\n",
"epoch 266/1000 error=0.011641\n",
"epoch 267/1000 error=0.011500\n",
"epoch 268/1000 error=0.011361\n",
"epoch 269/1000 error=0.011225\n",
"epoch 270/1000 error=0.011092\n",
"epoch 271/1000 error=0.010963\n",
"epoch 272/1000 error=0.010835\n",
"epoch 273/1000 error=0.010711\n",
"epoch 274/1000 error=0.010589\n",
"epoch 275/1000 error=0.010470\n",
"epoch 276/1000 error=0.010353\n",
"epoch 277/1000 error=0.010238\n",
"epoch 278/1000 error=0.010126\n",
"epoch 279/1000 error=0.010016\n",
"epoch 280/1000 error=0.009908\n",
"epoch 281/1000 error=0.009802\n",
"epoch 282/1000 error=0.009699\n",
"epoch 283/1000 error=0.009597\n",
"epoch 284/1000 error=0.009497\n",
"epoch 285/1000 error=0.009399\n",
"epoch 286/1000 error=0.009303\n",
"epoch 287/1000 error=0.009209\n",
"epoch 288/1000 error=0.009116\n",
"epoch 289/1000 error=0.009025\n",
"epoch 290/1000 error=0.008936\n",
"epoch 291/1000 error=0.008849\n",
"epoch 292/1000 error=0.008763\n",
"epoch 293/1000 error=0.008678\n",
"epoch 294/1000 error=0.008595\n",
"epoch 295/1000 error=0.008513\n",
"epoch 296/1000 error=0.008433\n",
"epoch 297/1000 error=0.008354\n",
"epoch 298/1000 error=0.008277\n",
"epoch 299/1000 error=0.008201\n",
"epoch 300/1000 error=0.008126\n",
"epoch 301/1000 error=0.008052\n",
"epoch 302/1000 error=0.007980\n",
"epoch 303/1000 error=0.007909\n",
"epoch 304/1000 error=0.007839\n",
"epoch 305/1000 error=0.007770\n",
"epoch 306/1000 error=0.007702\n",
"epoch 307/1000 error=0.007635\n",
"epoch 308/1000 error=0.007569\n",
"epoch 309/1000 error=0.007505\n",
"epoch 310/1000 error=0.007441\n",
"epoch 311/1000 error=0.007378\n",
"epoch 312/1000 error=0.007317\n",
"epoch 313/1000 error=0.007256\n",
"epoch 314/1000 error=0.007196\n",
"epoch 315/1000 error=0.007137\n",
"epoch 316/1000 error=0.007079\n",
"epoch 317/1000 error=0.007022\n",
"epoch 318/1000 error=0.006966\n",
"epoch 319/1000 error=0.006910\n",
"epoch 320/1000 error=0.006855\n",
"epoch 321/1000 error=0.006802\n",
"epoch 322/1000 error=0.006748\n",
"epoch 323/1000 error=0.006696\n",
"epoch 324/1000 error=0.006644\n",
"epoch 325/1000 error=0.006594\n",
"epoch 326/1000 error=0.006543\n",
"epoch 327/1000 error=0.006494\n",
"epoch 328/1000 error=0.006445\n",
"epoch 329/1000 error=0.006397\n",
"epoch 330/1000 error=0.006349\n",
"epoch 331/1000 error=0.006303\n",
"epoch 332/1000 error=0.006256\n",
"epoch 333/1000 error=0.006211\n",
"epoch 334/1000 error=0.006166\n",
"epoch 335/1000 error=0.006122\n",
"epoch 336/1000 error=0.006078\n",
"epoch 337/1000 error=0.006035\n",
"epoch 338/1000 error=0.005992\n",
"epoch 339/1000 error=0.005950\n",
"epoch 340/1000 error=0.005908\n",
"epoch 341/1000 error=0.005867\n",
"epoch 342/1000 error=0.005827\n",
"epoch 343/1000 error=0.005787\n",
"epoch 344/1000 error=0.005747\n",
"epoch 345/1000 error=0.005709\n",
"epoch 346/1000 error=0.005670\n",
"epoch 347/1000 error=0.005632\n",
"epoch 348/1000 error=0.005594\n",
"epoch 349/1000 error=0.005557\n",
"epoch 350/1000 error=0.005521\n",
"epoch 351/1000 error=0.005485\n",
"epoch 352/1000 error=0.005449\n",
"epoch 353/1000 error=0.005414\n",
"epoch 354/1000 error=0.005379\n",
"epoch 355/1000 error=0.005344\n",
"epoch 356/1000 error=0.005310\n",
"epoch 357/1000 error=0.005276\n",
"epoch 358/1000 error=0.005243\n",
"epoch 359/1000 error=0.005210\n",
"epoch 360/1000 error=0.005178\n",
"epoch 361/1000 error=0.005146\n",
"epoch 362/1000 error=0.005114\n",
"epoch 363/1000 error=0.005082\n",
"epoch 364/1000 error=0.005051\n",
"epoch 365/1000 error=0.005021\n",
"epoch 366/1000 error=0.004990\n",
"epoch 367/1000 error=0.004960\n",
"epoch 368/1000 error=0.004931\n",
"epoch 369/1000 error=0.004901\n",
"epoch 370/1000 error=0.004872\n",
"epoch 371/1000 error=0.004843\n",
"epoch 372/1000 error=0.004815\n",
"epoch 373/1000 error=0.004787\n",
"epoch 374/1000 error=0.004759\n",
"epoch 375/1000 error=0.004732\n",
"epoch 376/1000 error=0.004704\n",
"epoch 377/1000 error=0.004678\n",
"epoch 378/1000 error=0.004651\n",
"epoch 379/1000 error=0.004625\n",
"epoch 380/1000 error=0.004598\n",
"epoch 381/1000 error=0.004573\n",
"epoch 382/1000 error=0.004547\n",
"epoch 383/1000 error=0.004522\n",
"epoch 384/1000 error=0.004497\n",
"epoch 385/1000 error=0.004472\n",
"epoch 386/1000 error=0.004448\n",
"epoch 387/1000 error=0.004423\n",
"epoch 388/1000 error=0.004399\n",
"epoch 389/1000 error=0.004376\n",
"epoch 390/1000 error=0.004352\n",
"epoch 391/1000 error=0.004329\n",
"epoch 392/1000 error=0.004306\n",
"epoch 393/1000 error=0.004283\n",
"epoch 394/1000 error=0.004260\n",
"epoch 395/1000 error=0.004238\n",
"epoch 396/1000 error=0.004216\n",
"epoch 397/1000 error=0.004194\n",
"epoch 398/1000 error=0.004172\n",
"epoch 399/1000 error=0.004150\n",
"epoch 400/1000 error=0.004129\n",
"epoch 401/1000 error=0.004108\n",
"epoch 402/1000 error=0.004087\n",
"epoch 403/1000 error=0.004066\n",
"epoch 404/1000 error=0.004046\n",
"epoch 405/1000 error=0.004025\n",
"epoch 406/1000 error=0.004005\n",
"epoch 407/1000 error=0.003985\n",
"epoch 408/1000 error=0.003965\n",
"epoch 409/1000 error=0.003946\n",
"epoch 410/1000 error=0.003926\n",
"epoch 411/1000 error=0.003907\n",
"epoch 412/1000 error=0.003888\n",
"epoch 413/1000 error=0.003869\n",
"epoch 414/1000 error=0.003850\n",
"epoch 415/1000 error=0.003832\n",
"epoch 416/1000 error=0.003813\n",
"epoch 417/1000 error=0.003795\n",
"epoch 418/1000 error=0.003777\n",
"epoch 419/1000 error=0.003759\n",
"epoch 420/1000 error=0.003741\n",
"epoch 421/1000 error=0.003724\n",
"epoch 422/1000 error=0.003706\n",
"epoch 423/1000 error=0.003689\n",
"epoch 424/1000 error=0.003672\n",
"epoch 425/1000 error=0.003655\n",
"epoch 426/1000 error=0.003638\n",
"epoch 427/1000 error=0.003621\n",
"epoch 428/1000 error=0.003605\n",
"epoch 429/1000 error=0.003588\n",
"epoch 430/1000 error=0.003572\n",
"epoch 431/1000 error=0.003556\n",
"epoch 432/1000 error=0.003540\n",
"epoch 433/1000 error=0.003524\n",
"epoch 434/1000 error=0.003508\n",
"epoch 435/1000 error=0.003493\n",
"epoch 436/1000 error=0.003477\n",
"epoch 437/1000 error=0.003462\n",
"epoch 438/1000 error=0.003447\n",
"epoch 439/1000 error=0.003431\n",
"epoch 440/1000 error=0.003416\n",
"epoch 441/1000 error=0.003402\n",
"epoch 442/1000 error=0.003387\n",
"epoch 443/1000 error=0.003372\n",
"epoch 444/1000 error=0.003358\n",
"epoch 445/1000 error=0.003343\n",
"epoch 446/1000 error=0.003329\n",
"epoch 447/1000 error=0.003315\n",
"epoch 448/1000 error=0.003301\n",
"epoch 449/1000 error=0.003287\n",
"epoch 450/1000 error=0.003273\n",
"epoch 451/1000 error=0.003259\n",
"epoch 452/1000 error=0.003246\n",
"epoch 453/1000 error=0.003232\n",
"epoch 454/1000 error=0.003219\n",
"epoch 455/1000 error=0.003205\n",
"epoch 456/1000 error=0.003192\n",
"epoch 457/1000 error=0.003179\n",
"epoch 458/1000 error=0.003166\n",
"epoch 459/1000 error=0.003153\n",
"epoch 460/1000 error=0.003140\n",
"epoch 461/1000 error=0.003128\n",
"epoch 462/1000 error=0.003115\n",
"epoch 463/1000 error=0.003103\n",
"epoch 464/1000 error=0.003090\n",
"epoch 465/1000 error=0.003078\n",
"epoch 466/1000 error=0.003066\n",
"epoch 467/1000 error=0.003053\n",
"epoch 468/1000 error=0.003041\n",
"epoch 469/1000 error=0.003029\n",
"epoch 470/1000 error=0.003018\n",
"epoch 471/1000 error=0.003006\n",
"epoch 472/1000 error=0.002994\n",
"epoch 473/1000 error=0.002982\n",
"epoch 474/1000 error=0.002971\n",
"epoch 475/1000 error=0.002959\n",
"epoch 476/1000 error=0.002948\n",
"epoch 477/1000 error=0.002937\n",
"epoch 478/1000 error=0.002926\n",
"epoch 479/1000 error=0.002914\n",
"epoch 480/1000 error=0.002903\n",
"epoch 481/1000 error=0.002892\n",
"epoch 482/1000 error=0.002882\n",
"epoch 483/1000 error=0.002871\n",
"epoch 484/1000 error=0.002860\n",
"epoch 485/1000 error=0.002849\n",
"epoch 486/1000 error=0.002839\n",
"epoch 487/1000 error=0.002828\n",
"epoch 488/1000 error=0.002818\n",
"epoch 489/1000 error=0.002807\n",
"epoch 490/1000 error=0.002797\n",
"epoch 491/1000 error=0.002787\n",
"epoch 492/1000 error=0.002777\n",
"epoch 493/1000 error=0.002766\n",
"epoch 494/1000 error=0.002756\n",
"epoch 495/1000 error=0.002746\n",
"epoch 496/1000 error=0.002737\n",
"epoch 497/1000 error=0.002727\n",
"epoch 498/1000 error=0.002717\n",
"epoch 499/1000 error=0.002707\n",
"epoch 500/1000 error=0.002698\n",
"epoch 501/1000 error=0.002688\n",
"epoch 502/1000 error=0.002679\n",
"epoch 503/1000 error=0.002669\n",
"epoch 504/1000 error=0.002660\n",
"epoch 505/1000 error=0.002650\n",
"epoch 506/1000 error=0.002641\n",
"epoch 507/1000 error=0.002632\n",
"epoch 508/1000 error=0.002623\n",
"epoch 509/1000 error=0.002614\n",
"epoch 510/1000 error=0.002605\n",
"epoch 511/1000 error=0.002596\n",
"epoch 512/1000 error=0.002587\n",
"epoch 513/1000 error=0.002578\n",
"epoch 514/1000 error=0.002569\n",
"epoch 515/1000 error=0.002560\n",
"epoch 516/1000 error=0.002552\n",
"epoch 517/1000 error=0.002543\n",
"epoch 518/1000 error=0.002535\n",
"epoch 519/1000 error=0.002526\n",
"epoch 520/1000 error=0.002518\n",
"epoch 521/1000 error=0.002509\n",
"epoch 522/1000 error=0.002501\n",
"epoch 523/1000 error=0.002493\n",
"epoch 524/1000 error=0.002484\n",
"epoch 525/1000 error=0.002476\n",
"epoch 526/1000 error=0.002468\n",
"epoch 527/1000 error=0.002460\n",
"epoch 528/1000 error=0.002452\n",
"epoch 529/1000 error=0.002444\n",
"epoch 530/1000 error=0.002436\n",
"epoch 531/1000 error=0.002428\n",
"epoch 532/1000 error=0.002420\n",
"epoch 533/1000 error=0.002412\n",
"epoch 534/1000 error=0.002404\n",
"epoch 535/1000 error=0.002397\n",
"epoch 536/1000 error=0.002389\n",
"epoch 537/1000 error=0.002381\n",
"epoch 538/1000 error=0.002374\n",
"epoch 539/1000 error=0.002366\n",
"epoch 540/1000 error=0.002359\n",
"epoch 541/1000 error=0.002351\n",
"epoch 542/1000 error=0.002344\n",
"epoch 543/1000 error=0.002337\n",
"epoch 544/1000 error=0.002329\n",
"epoch 545/1000 error=0.002322\n",
"epoch 546/1000 error=0.002315\n",
"epoch 547/1000 error=0.002308\n",
"epoch 548/1000 error=0.002300\n",
"epoch 549/1000 error=0.002293\n",
"epoch 550/1000 error=0.002286\n",
"epoch 551/1000 error=0.002279\n",
"epoch 552/1000 error=0.002272\n",
"epoch 553/1000 error=0.002265\n",
"epoch 554/1000 error=0.002258\n",
"epoch 555/1000 error=0.002252\n",
"epoch 556/1000 error=0.002245\n",
"epoch 557/1000 error=0.002238\n",
"epoch 558/1000 error=0.002231\n",
"epoch 559/1000 error=0.002225\n",
"epoch 560/1000 error=0.002218\n",
"epoch 561/1000 error=0.002211\n",
"epoch 562/1000 error=0.002205\n",
"epoch 563/1000 error=0.002198\n",
"epoch 564/1000 error=0.002192\n",
"epoch 565/1000 error=0.002185\n",
"epoch 566/1000 error=0.002179\n",
"epoch 567/1000 error=0.002172\n",
"epoch 568/1000 error=0.002166\n",
"epoch 569/1000 error=0.002159\n",
"epoch 570/1000 error=0.002153\n",
"epoch 571/1000 error=0.002147\n",
"epoch 572/1000 error=0.002141\n",
"epoch 573/1000 error=0.002134\n",
"epoch 574/1000 error=0.002128\n",
"epoch 575/1000 error=0.002122\n",
"epoch 576/1000 error=0.002116\n",
"epoch 577/1000 error=0.002110\n",
"epoch 578/1000 error=0.002104\n",
"epoch 579/1000 error=0.002098\n",
"epoch 580/1000 error=0.002092\n",
"epoch 581/1000 error=0.002086\n",
"epoch 582/1000 error=0.002080\n",
"epoch 583/1000 error=0.002074\n",
"epoch 584/1000 error=0.002068\n",
"epoch 585/1000 error=0.002062\n",
"epoch 586/1000 error=0.002057\n",
"epoch 587/1000 error=0.002051\n",
"epoch 588/1000 error=0.002045\n",
"epoch 589/1000 error=0.002039\n",
"epoch 590/1000 error=0.002034\n",
"epoch 591/1000 error=0.002028\n",
"epoch 592/1000 error=0.002023\n",
"epoch 593/1000 error=0.002017\n",
"epoch 594/1000 error=0.002011\n",
"epoch 595/1000 error=0.002006\n",
"epoch 596/1000 error=0.002000\n",
"epoch 597/1000 error=0.001995\n",
"epoch 598/1000 error=0.001989\n",
"epoch 599/1000 error=0.001984\n",
"epoch 600/1000 error=0.001979\n",
"epoch 601/1000 error=0.001973\n",
"epoch 602/1000 error=0.001968\n",
"epoch 603/1000 error=0.001963\n",
"epoch 604/1000 error=0.001957\n",
"epoch 605/1000 error=0.001952\n",
"epoch 606/1000 error=0.001947\n",
"epoch 607/1000 error=0.001942\n",
"epoch 608/1000 error=0.001937\n",
"epoch 609/1000 error=0.001931\n",
"epoch 610/1000 error=0.001926\n",
"epoch 611/1000 error=0.001921\n",
"epoch 612/1000 error=0.001916\n",
"epoch 613/1000 error=0.001911\n",
"epoch 614/1000 error=0.001906\n",
"epoch 615/1000 error=0.001901\n",
"epoch 616/1000 error=0.001896\n",
"epoch 617/1000 error=0.001891\n",
"epoch 618/1000 error=0.001886\n",
"epoch 619/1000 error=0.001881\n",
"epoch 620/1000 error=0.001876\n",
"epoch 621/1000 error=0.001872\n",
"epoch 622/1000 error=0.001867\n",
"epoch 623/1000 error=0.001862\n",
"epoch 624/1000 error=0.001857\n",
"epoch 625/1000 error=0.001852\n",
"epoch 626/1000 error=0.001848\n",
"epoch 627/1000 error=0.001843\n",
"epoch 628/1000 error=0.001838\n",
"epoch 629/1000 error=0.001834\n",
"epoch 630/1000 error=0.001829\n",
"epoch 631/1000 error=0.001824\n",
"epoch 632/1000 error=0.001820\n",
"epoch 633/1000 error=0.001815\n",
"epoch 634/1000 error=0.001811\n",
"epoch 635/1000 error=0.001806\n",
"epoch 636/1000 error=0.001802\n",
"epoch 637/1000 error=0.001797\n",
"epoch 638/1000 error=0.001793\n",
"epoch 639/1000 error=0.001788\n",
"epoch 640/1000 error=0.001784\n",
"epoch 641/1000 error=0.001779\n",
"epoch 642/1000 error=0.001775\n",
"epoch 643/1000 error=0.001771\n",
"epoch 644/1000 error=0.001766\n",
"epoch 645/1000 error=0.001762\n",
"epoch 646/1000 error=0.001758\n",
"epoch 647/1000 error=0.001753\n",
"epoch 648/1000 error=0.001749\n",
"epoch 649/1000 error=0.001745\n",
"epoch 650/1000 error=0.001741\n",
"epoch 651/1000 error=0.001736\n",
"epoch 652/1000 error=0.001732\n",
"epoch 653/1000 error=0.001728\n",
"epoch 654/1000 error=0.001724\n",
"epoch 655/1000 error=0.001720\n",
"epoch 656/1000 error=0.001716\n",
"epoch 657/1000 error=0.001711\n",
"epoch 658/1000 error=0.001707\n",
"epoch 659/1000 error=0.001703\n",
"epoch 660/1000 error=0.001699\n",
"epoch 661/1000 error=0.001695\n",
"epoch 662/1000 error=0.001691\n",
"epoch 663/1000 error=0.001687\n",
"epoch 664/1000 error=0.001683\n",
"epoch 665/1000 error=0.001679\n",
"epoch 666/1000 error=0.001675\n",
"epoch 667/1000 error=0.001672\n",
"epoch 668/1000 error=0.001668\n",
"epoch 669/1000 error=0.001664\n",
"epoch 670/1000 error=0.001660\n",
"epoch 671/1000 error=0.001656\n",
"epoch 672/1000 error=0.001652\n",
"epoch 673/1000 error=0.001648\n",
"epoch 674/1000 error=0.001645\n",
"epoch 675/1000 error=0.001641\n",
"epoch 676/1000 error=0.001637\n",
"epoch 677/1000 error=0.001633\n",
"epoch 678/1000 error=0.001630\n",
"epoch 679/1000 error=0.001626\n",
"epoch 680/1000 error=0.001622\n",
"epoch 681/1000 error=0.001618\n",
"epoch 682/1000 error=0.001615\n",
"epoch 683/1000 error=0.001611\n",
"epoch 684/1000 error=0.001607\n",
"epoch 685/1000 error=0.001604\n",
"epoch 686/1000 error=0.001600\n",
"epoch 687/1000 error=0.001597\n",
"epoch 688/1000 error=0.001593\n",
"epoch 689/1000 error=0.001589\n",
"epoch 690/1000 error=0.001586\n",
"epoch 691/1000 error=0.001582\n",
"epoch 692/1000 error=0.001579\n",
"epoch 693/1000 error=0.001575\n",
"epoch 694/1000 error=0.001572\n",
"epoch 695/1000 error=0.001568\n",
"epoch 696/1000 error=0.001565\n",
"epoch 697/1000 error=0.001561\n",
"epoch 698/1000 error=0.001558\n",
"epoch 699/1000 error=0.001555\n",
"epoch 700/1000 error=0.001551\n",
"epoch 701/1000 error=0.001548\n",
"epoch 702/1000 error=0.001545\n",
"epoch 703/1000 error=0.001541\n",
"epoch 704/1000 error=0.001538\n",
"epoch 705/1000 error=0.001535\n",
"epoch 706/1000 error=0.001532\n",
"epoch 707/1000 error=0.001528\n",
"epoch 708/1000 error=0.001526\n",
"epoch 709/1000 error=0.001522\n",
"epoch 710/1000 error=0.001520\n",
"epoch 711/1000 error=0.001516\n",
"epoch 712/1000 error=0.001515\n",
"epoch 713/1000 error=0.001512\n",
"epoch 714/1000 error=0.001511\n",
"epoch 715/1000 error=0.001508\n",
"epoch 716/1000 error=0.001509\n",
"epoch 717/1000 error=0.001508\n",
"epoch 718/1000 error=0.001511\n",
"epoch 719/1000 error=0.001512\n",
"epoch 720/1000 error=0.001519\n",
"epoch 721/1000 error=0.001525\n",
"epoch 722/1000 error=0.001540\n",
"epoch 723/1000 error=0.001554\n",
"epoch 724/1000 error=0.001583\n",
"epoch 725/1000 error=0.001613\n",
"epoch 726/1000 error=0.001667\n",
"epoch 727/1000 error=0.001727\n",
"epoch 728/1000 error=0.001824\n",
"epoch 729/1000 error=0.001943\n",
"epoch 730/1000 error=0.002115\n",
"epoch 731/1000 error=0.002341\n",
"epoch 732/1000 error=0.002645\n",
"epoch 733/1000 error=0.003069\n",
"epoch 734/1000 error=0.003592\n",
"epoch 735/1000 error=0.004367\n",
"epoch 736/1000 error=0.005231\n",
"epoch 737/1000 error=0.006599\n",
"epoch 738/1000 error=0.007921\n",
"epoch 739/1000 error=0.010194\n",
"epoch 740/1000 error=0.011947\n",
"epoch 741/1000 error=0.015368\n",
"epoch 742/1000 error=0.017131\n",
"epoch 743/1000 error=0.021579\n",
"epoch 744/1000 error=0.022405\n",
"epoch 745/1000 error=0.027193\n",
"epoch 746/1000 error=0.026114\n",
"epoch 747/1000 error=0.030312\n",
"epoch 748/1000 error=0.027216\n",
"epoch 749/1000 error=0.030249\n",
"epoch 750/1000 error=0.025948\n",
"epoch 751/1000 error=0.027774\n",
"epoch 752/1000 error=0.023250\n",
"epoch 753/1000 error=0.024124\n",
"epoch 754/1000 error=0.020011\n",
"epoch 755/1000 error=0.020246\n",
"epoch 756/1000 error=0.016800\n",
"epoch 757/1000 error=0.016656\n",
"epoch 758/1000 error=0.013900\n",
"epoch 759/1000 error=0.013560\n",
"epoch 760/1000 error=0.011418\n",
"epoch 761/1000 error=0.011000\n",
"epoch 762/1000 error=0.009362\n",
"epoch 763/1000 error=0.008933\n",
"epoch 764/1000 error=0.007694\n",
"epoch 765/1000 error=0.007292\n",
"epoch 766/1000 error=0.006359\n",
"epoch 767/1000 error=0.006001\n",
"epoch 768/1000 error=0.005301\n",
"epoch 769/1000 error=0.004992\n",
"epoch 770/1000 error=0.004466\n",
"epoch 771/1000 error=0.004205\n",
"epoch 772/1000 error=0.003810\n",
"epoch 773/1000 error=0.003592\n",
"epoch 774/1000 error=0.003296\n",
"epoch 775/1000 error=0.003116\n",
"epoch 776/1000 error=0.002893\n",
"epoch 777/1000 error=0.002744\n",
"epoch 778/1000 error=0.002576\n",
"epoch 779/1000 error=0.002454\n",
"epoch 780/1000 error=0.002327\n",
"epoch 781/1000 error=0.002227\n",
"epoch 782/1000 error=0.002130\n",
"epoch 783/1000 error=0.002048\n",
"epoch 784/1000 error=0.001974\n",
"epoch 785/1000 error=0.001907\n",
"epoch 786/1000 error=0.001851\n",
"epoch 787/1000 error=0.001795\n",
"epoch 788/1000 error=0.001752\n",
"epoch 789/1000 error=0.001706\n",
"epoch 790/1000 error=0.001672\n",
"epoch 791/1000 error=0.001634\n",
"epoch 792/1000 error=0.001608\n",
"epoch 793/1000 error=0.001576\n",
"epoch 794/1000 error=0.001556\n",
"epoch 795/1000 error=0.001529\n",
"epoch 796/1000 error=0.001513\n",
"epoch 797/1000 error=0.001490\n",
"epoch 798/1000 error=0.001478\n",
"epoch 799/1000 error=0.001458\n",
"epoch 800/1000 error=0.001449\n",
"epoch 801/1000 error=0.001432\n",
"epoch 802/1000 error=0.001424\n",
"epoch 803/1000 error=0.001409\n",
"epoch 804/1000 error=0.001403\n",
"epoch 805/1000 error=0.001390\n",
"epoch 806/1000 error=0.001385\n",
"epoch 807/1000 error=0.001373\n",
"epoch 808/1000 error=0.001369\n",
"epoch 809/1000 error=0.001359\n",
"epoch 810/1000 error=0.001355\n",
"epoch 811/1000 error=0.001346\n",
"epoch 812/1000 error=0.001343\n",
"epoch 813/1000 error=0.001335\n",
"epoch 814/1000 error=0.001332\n",
"epoch 815/1000 error=0.001325\n",
"epoch 816/1000 error=0.001323\n",
"epoch 817/1000 error=0.001316\n",
"epoch 818/1000 error=0.001314\n",
"epoch 819/1000 error=0.001307\n",
"epoch 820/1000 error=0.001305\n",
"epoch 821/1000 error=0.001299\n",
"epoch 822/1000 error=0.001298\n",
"epoch 823/1000 error=0.001292\n",
"epoch 824/1000 error=0.001290\n",
"epoch 825/1000 error=0.001285\n",
"epoch 826/1000 error=0.001284\n",
"epoch 827/1000 error=0.001279\n",
"epoch 828/1000 error=0.001277\n",
"epoch 829/1000 error=0.001273\n",
"epoch 830/1000 error=0.001271\n",
"epoch 831/1000 error=0.001267\n",
"epoch 832/1000 error=0.001265\n",
"epoch 833/1000 error=0.001261\n",
"epoch 834/1000 error=0.001259\n",
"epoch 835/1000 error=0.001255\n",
"epoch 836/1000 error=0.001254\n",
"epoch 837/1000 error=0.001250\n",
"epoch 838/1000 error=0.001248\n",
"epoch 839/1000 error=0.001245\n",
"epoch 840/1000 error=0.001243\n",
"epoch 841/1000 error=0.001239\n",
"epoch 842/1000 error=0.001238\n",
"epoch 843/1000 error=0.001234\n",
"epoch 844/1000 error=0.001233\n",
"epoch 845/1000 error=0.001229\n",
"epoch 846/1000 error=0.001228\n",
"epoch 847/1000 error=0.001225\n",
"epoch 848/1000 error=0.001223\n",
"epoch 849/1000 error=0.001220\n",
"epoch 850/1000 error=0.001218\n",
"epoch 851/1000 error=0.001215\n",
"epoch 852/1000 error=0.001213\n",
"epoch 853/1000 error=0.001210\n",
"epoch 854/1000 error=0.001209\n",
"epoch 855/1000 error=0.001206\n",
"epoch 856/1000 error=0.001204\n",
"epoch 857/1000 error=0.001201\n",
"epoch 858/1000 error=0.001200\n",
"epoch 859/1000 error=0.001197\n",
"epoch 860/1000 error=0.001195\n",
"epoch 861/1000 error=0.001192\n",
"epoch 862/1000 error=0.001191\n",
"epoch 863/1000 error=0.001188\n",
"epoch 864/1000 error=0.001186\n",
"epoch 865/1000 error=0.001184\n",
"epoch 866/1000 error=0.001182\n",
"epoch 867/1000 error=0.001179\n",
"epoch 868/1000 error=0.001178\n",
"epoch 869/1000 error=0.001175\n",
"epoch 870/1000 error=0.001174\n",
"epoch 871/1000 error=0.001171\n",
"epoch 872/1000 error=0.001169\n",
"epoch 873/1000 error=0.001167\n",
"epoch 874/1000 error=0.001165\n",
"epoch 875/1000 error=0.001163\n",
"epoch 876/1000 error=0.001161\n",
"epoch 877/1000 error=0.001159\n",
"epoch 878/1000 error=0.001157\n",
"epoch 879/1000 error=0.001155\n",
"epoch 880/1000 error=0.001153\n",
"epoch 881/1000 error=0.001150\n",
"epoch 882/1000 error=0.001149\n",
"epoch 883/1000 error=0.001146\n",
"epoch 884/1000 error=0.001145\n",
"epoch 885/1000 error=0.001142\n",
"epoch 886/1000 error=0.001141\n",
"epoch 887/1000 error=0.001139\n",
"epoch 888/1000 error=0.001137\n",
"epoch 889/1000 error=0.001135\n",
"epoch 890/1000 error=0.001133\n",
"epoch 891/1000 error=0.001131\n",
"epoch 892/1000 error=0.001129\n",
"epoch 893/1000 error=0.001127\n",
"epoch 894/1000 error=0.001125\n",
"epoch 895/1000 error=0.001123\n",
"epoch 896/1000 error=0.001121\n",
"epoch 897/1000 error=0.001119\n",
"epoch 898/1000 error=0.001118\n",
"epoch 899/1000 error=0.001115\n",
"epoch 900/1000 error=0.001114\n",
"epoch 901/1000 error=0.001112\n",
"epoch 902/1000 error=0.001110\n",
"epoch 903/1000 error=0.001108\n",
"epoch 904/1000 error=0.001106\n",
"epoch 905/1000 error=0.001104\n",
"epoch 906/1000 error=0.001103\n",
"epoch 907/1000 error=0.001101\n",
"epoch 908/1000 error=0.001099\n",
"epoch 909/1000 error=0.001097\n",
"epoch 910/1000 error=0.001095\n",
"epoch 911/1000 error=0.001093\n",
"epoch 912/1000 error=0.001092\n",
"epoch 913/1000 error=0.001090\n",
"epoch 914/1000 error=0.001088\n",
"epoch 915/1000 error=0.001086\n",
"epoch 916/1000 error=0.001085\n",
"epoch 917/1000 error=0.001082\n",
"epoch 918/1000 error=0.001081\n",
"epoch 919/1000 error=0.001079\n",
"epoch 920/1000 error=0.001077\n",
"epoch 921/1000 error=0.001075\n",
"epoch 922/1000 error=0.001074\n",
"epoch 923/1000 error=0.001072\n",
"epoch 924/1000 error=0.001070\n",
"epoch 925/1000 error=0.001068\n",
"epoch 926/1000 error=0.001067\n",
"epoch 927/1000 error=0.001065\n",
"epoch 928/1000 error=0.001064\n",
"epoch 929/1000 error=0.001062\n",
"epoch 930/1000 error=0.001060\n",
"epoch 931/1000 error=0.001058\n",
"epoch 932/1000 error=0.001057\n",
"epoch 933/1000 error=0.001055\n",
"epoch 934/1000 error=0.001053\n",
"epoch 935/1000 error=0.001052\n",
"epoch 936/1000 error=0.001050\n",
"epoch 937/1000 error=0.001048\n",
"epoch 938/1000 error=0.001047\n",
"epoch 939/1000 error=0.001045\n",
"epoch 940/1000 error=0.001044\n",
"epoch 941/1000 error=0.001042\n",
"epoch 942/1000 error=0.001040\n",
"epoch 943/1000 error=0.001038\n",
"epoch 944/1000 error=0.001037\n",
"epoch 945/1000 error=0.001035\n",
"epoch 946/1000 error=0.001034\n",
"epoch 947/1000 error=0.001032\n",
"epoch 948/1000 error=0.001031\n",
"epoch 949/1000 error=0.001029\n",
"epoch 950/1000 error=0.001028\n",
"epoch 951/1000 error=0.001026\n",
"epoch 952/1000 error=0.001024\n",
"epoch 953/1000 error=0.001022\n",
"epoch 954/1000 error=0.001021\n",
"epoch 955/1000 error=0.001019\n",
"epoch 956/1000 error=0.001018\n",
"epoch 957/1000 error=0.001016\n",
"epoch 958/1000 error=0.001015\n",
"epoch 959/1000 error=0.001013\n",
"epoch 960/1000 error=0.001012\n",
"epoch 961/1000 error=0.001010\n",
"epoch 962/1000 error=0.001009\n",
"epoch 963/1000 error=0.001007\n",
"epoch 964/1000 error=0.001006\n",
"epoch 965/1000 error=0.001004\n",
"epoch 966/1000 error=0.001003\n",
"epoch 967/1000 error=0.001001\n",
"epoch 968/1000 error=0.001000\n",
"epoch 969/1000 error=0.000998\n",
"epoch 970/1000 error=0.000997\n",
"epoch 971/1000 error=0.000995\n",
"epoch 972/1000 error=0.000995\n",
"epoch 973/1000 error=0.000993\n",
"epoch 974/1000 error=0.000992\n",
"epoch 975/1000 error=0.000990\n",
"epoch 976/1000 error=0.000989\n",
"epoch 977/1000 error=0.000987\n",
"epoch 978/1000 error=0.000986\n",
"epoch 979/1000 error=0.000984\n",
"epoch 980/1000 error=0.000984\n",
"epoch 981/1000 error=0.000982\n",
"epoch 982/1000 error=0.000981\n",
"epoch 983/1000 error=0.000979\n",
"epoch 984/1000 error=0.000978\n",
"epoch 985/1000 error=0.000977\n",
"epoch 986/1000 error=0.000976\n",
"epoch 987/1000 error=0.000974\n",
"epoch 988/1000 error=0.000974\n",
"epoch 989/1000 error=0.000972\n",
"epoch 990/1000 error=0.000971\n",
"epoch 991/1000 error=0.000969\n",
"epoch 992/1000 error=0.000969\n",
"epoch 993/1000 error=0.000967\n",
"epoch 994/1000 error=0.000967\n",
"epoch 995/1000 error=0.000965\n",
"epoch 996/1000 error=0.000965\n",
"epoch 997/1000 error=0.000963\n",
"epoch 998/1000 error=0.000963\n",
"epoch 999/1000 error=0.000961\n",
"epoch 1000/1000 error=0.000961\n"
],
"name": "stdout"
}
]
},
{
"cell_type": "code",
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/",
"height": 281
},
"id": "7YBSf_OY0PLi",
"outputId": "074e9e90-6185-47a7-bd4f-6e8a5c0e10b5"
},
"source": [
"pd.DataFrame(err).plot()"
],
"execution_count": null,
"outputs": [
{
"output_type": "execute_result",
"data": {
"text/plain": [
"<matplotlib.axes._subplots.AxesSubplot at 0x7f8f771ea6d0>"
]
},
"metadata": {
"tags": []
},
"execution_count": 30
},
{
"output_type": "display_data",
"data": {
"image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAZxUlEQVR4nO3de3xU9Z3/8ddnZnJBbhISEQiYWFIqWLuUgEi1q1stlLZof62/hd1tbfVXHj5WW39uf799aLfbbu3utnt59LIr25a11m236621LbVYdn9WV6sWCVpv4WIQkCBoQOSihFzm8/tjzoSZyQQmySQnZ/J+Ph55ZM453znzOZzwnu9851zM3RERkeiLhV2AiIgUhwJdRKREKNBFREqEAl1EpEQo0EVESkQirBeurq72urq6sF5eRCSSNm3atN/da/ItCy3Q6+rqaGpqCuvlRUQiycx29bVMQy4iIiVCgS4iUiIU6CIiJSK0MXQRkbB0dnbS2tpKe3t72KX0qbKyktraWsrKygp+jgJdREad1tZWxo8fT11dHWYWdjm9uDsHDhygtbWV+vr6gp+nIRcRGXXa29uZPHnyiAxzADNj8uTJ/f4EoUAXkVFppIZ52kDqi1ygb9z5Ol//z610dCXDLkVEZESJXKA/tesg//TrFjq7FegiEl2/+tWvmD17NrNmzeJrX/taUdYZuUAf4Z+SREROqbu7m+uuu44HHniA5uZm7rzzTpqbmwe93sgFeprusyQiUfXkk08ya9Yszj77bMrLy1mxYgU///nPB73eyB22aKS66Lp1nogUw5d/8QLNrxwu6jrnTJvAlz48t8/le/bsYcaMGT3TtbW1bNiwYdCvG7keuoZcRETyi1wPPU39cxEphpP1pIfK9OnT2b17d890a2sr06dPH/R6I9dDT9OIi4hE1YIFC3jxxRfZsWMHHR0d3HXXXSxfvnzQ641cD32knwwgInIqiUSCW2+9lSVLltDd3c3VV1/N3LmD/6QQuUDvoR66iETYsmXLWLZsWVHXGbkhl3T/3JXoIiJZohfoGnEREcmroEA3s6VmttXMWszspjzLZ5rZQ2b2tJk9a2bF/RyRh74UFZHBGOnnsgykvlMGupnFgdXAB4A5wEozm5PT7AvAPe4+D1gB/Eu/KynQiSEXEZGBqays5MCBAyM21NPXQ6+srOzX8wr5UnQh0OLuLwGY2V3A5UDmhQccmBA8ngi80q8q+kFHuYjIYNXW1tLa2kpbW1vYpfQpfcei/igk0KcDuzOmW4Hzc9r8FfCfZvYZYCxwab4VmdkqYBXAzJkz+1VorpH6zioiI19ZWVm/7gQUFcX6UnQlcIe71wLLgB+aWa91u/sad29098aampoBvVC6g644FxHJVkig7wFmZEzXBvMyXQPcA+DuTwCVQHUxCsylARcRkfwKCfSNQIOZ1ZtZOakvPdfmtHkZeB+AmZ1DKtCHdHBKIy4iItlOGeju3gVcD6wHNpM6muUFM7vFzNIXH/gc8Gkzewa4E/ikD9UgdzDmohOLRESyFXTqv7uvA9blzPtixuNm4D3FLS2/niEX5bmISBadKSoiUiIiF+hp6qCLiGSLXKCfuAVdyIWIiIww0Qt0DbmIiOQVuUBP01EuIiLZIhfoPRfnUp6LiGSJXqBryEVEJK/IBXqaOugiItkiF+gnjnJRpIuIZIpcoOvqXCIi+UUv0APqoIuIZItcoKuDLiKSX/QCXYe5iIjkFblAT9OQi4hItsgFes+JRTpwUUQkS/QCXSMuIiJ5RS7Q0zTkIiKSLXKBnu6hK89FRLJFL9B1pqiISF7RC3SNoYuI5BW5QE9T/1xEJFt0A12JLiKSJXKBrjNFRUTyi1ygn6AuuohIpsgFum5BJyKSX/QCXSMuIiJ5RS7Q09RBFxHJFrlAP3FiUciFiIiMMNELdA25iIjkFblAT9Plc0VEskUu0HWUi4hIftELdA25iIjkFblAT1MPXUQkWwQDPTjKRWPoIiJZIhfoGnIREckvcoGepiEXEZFskQt0ddBFRPIrKNDNbKmZbTWzFjO7qY82/9PMms3sBTP7j+KWmfU6gHroIiK5EqdqYGZxYDVwGdAKbDSzte7enNGmAbgZeI+7HzSzM4aqYPXQRUTyK6SHvhBocfeX3L0DuAu4PKfNp4HV7n4QwN1fK26ZvekoFxGRbIUE+nRgd8Z0azAv09uBt5vZY2b2WzNbmm9FZrbKzJrMrKmtrW1ABaePctGQi4hItmJ9KZoAGoCLgZXAv5rZ6bmN3H2Nuze6e2NNTc2AXkiHLYqI5FdIoO8BZmRM1wbzMrUCa9290913ANtIBfyQUQddRCRbIYG+EWgws3ozKwdWAGtz2vyMVO8cM6smNQTzUhHr7HHieuiKdBGRTKcMdHfvAq4H1gObgXvc/QUzu8XMlgfN1gMHzKwZeAj4v+5+YEgq1pCLiEhepzxsEcDd1wHrcuZ9MeOxA38W/AwL9c9FRLJF9kxRjbiIiGSLXqDrMBcRkbwiF+gnqIsuIpIpcoGuIRcRkfyiF+gacRERyStygZ6mDrqISLbIBfqJE4tCLkREZISJXqBryEVEJK/IBXqaTv0XEckWuUDvOcol1CpEREaeyAU6uh66iEhekQt009W5RETyilygp+kWdCIi2SIX6KZBdBGRvKIX6GEXICIyQkUu0NPUQRcRyRa5QE9fPldHuYiIZItgoIddgYjIyBS5QE/TUS4iItkiF+i6HrqISH7RC3QNuYiI5BW5QE9TB11EJFsEAz19lIsiXUQkU+QCXUMuIiL5RS7Q09Q/FxHJFrlA7+mgK9FFRLJEL9A15iIiklfkAj1NJxaJiGSLXKDrxCIRkfyiF+i6BZ2ISF7RC3RdEV1EJK/IBXqaOugiItkiF+gnhlwU6SIimSIX6CIikl9kA139cxGRbJELdB3lIiKSX/QCXUe5iIjkVVCgm9lSM9tqZi1mdtNJ2n3UzNzMGotXYl/URRcRyXTKQDezOLAa+AAwB1hpZnPytBsP3ABsKHaR2a+T+q0hFxGRbIX00BcCLe7+krt3AHcBl+dp9xXg74D2ItbXi67NJSKSXyGBPh3YnTHdGszrYWbvBma4+y9PtiIzW2VmTWbW1NbW1u9iM6mDLiKSbdBfippZDPg68LlTtXX3Ne7e6O6NNTU1A3u9nlvQDejpIiIlq5BA3wPMyJiuDealjQfOBR42s53AImDtUH0xqiEXEZH8Cgn0jUCDmdWbWTmwAlibXujuh9y92t3r3L0O+C2w3N2bhqTi9Otq0EVEJMspA93du4DrgfXAZuAed3/BzG4xs+VDXWAuXQ9dRCS/RCGN3H0dsC5n3hf7aHvx4Mvqm4ZcRETyi9yZomnqoIuIZItcoKdvEq3L54qIZItcoJfFUiV3divQRUQyRS7QyxPpQE+GXImIyMgSuUAvi6eGXBToIiLZohfoQQ+9o0uBLiKSKXKBXh4PAl09dBGRLJEL9LIg0Du79KWoiEimyAV6PGbETGPoIiK5IhfokDrSRYEuIpItkoFeFo9pDF1EJEckA708HtNRLiIiOSIZ6GVxDbmIiOSKZqAnTKf+i4jkiGSgTzqtnNeODOm9qEVEIieSgT532gSe3X2ItiPHwy5FRGTEiGSgX7W4js5kkitWP8bjLfvDLkdEZESIZKC/48wJ3PnpRVQkYvzRbRu45RfNtHd2h12WiEioIhnoAPNmTuKXn72IT1xwFrc/toMrVj/G1n1Hwi5LRCQ0kQ10gDHlcW65/Fy+/8kF7D96nA/f+ht++nRr2GWJiIQi0oGedsk7zuCBG97LvBmnc+Pdz/DVBzbTndRhjSIyupREoAPUjK/g3//X+fzx+TP57n+/xKd/0MSbx7vCLktEZNiUTKBD6gzSv/nIO/nK5XP5721tXPmdJ9h3SMeri8joUFKBnvbxC+q47apGdh14kytWP0bzK4fDLklEZMiVZKADXDL7DO69djEAV37ncR7e+lrIFYmIDK2SDXSAOdMm8NPrFjNz8liu+bcm/mPDy2GXJCIyZEo60AGmThzDvddewIWzqvn8T5/jqw9sJqkjYESkBJV8oAOMq0jwvasae46A+cydT3OkvTPsskREimpUBDpAIh7jr684l88vewfrnt/L0m8+yiPb2sIuS0SkaEZNoAOYGave+zZ+fO1iKstifOL2J7n2h5vY3nY07NJERAZtVAV62vyzUteBufHSt/Poi228/xuPcOPdv+P5PYfCLk1EZMDMPZwvCBsbG72pqSmU187UduQ43354O3dvfJk3O7o5v76KlQtnsmTumYwpj4ddnohIFjPb5O6NeZeN9kBPO3Ssk7s3vswPnthF68FjjKtI8KHzpvKx+bXMP2sSZhZ2iSIiCvT+SCadDTte5ydPtbLuub281dFNffVYPja/lo/Mm86008eEXaKIjGIK9AF683gXDzy/j3ubdrNhx+uYwYWzqvnY/FqWzD2TyjINyYjI8FKgF8GuA2/yk02t/OSpPex54xjV48r51Hvq+ZNFZzFxTFnY5YnIKKFAL6Jk0nls+37+9dEdPLKtjfEVCT7zvll8cnE95YlRedCQiAyjkwV6QQlkZkvNbKuZtZjZTXmW/5mZNZvZs2b2oJmdNdiiR6pYzLiooYYfXL2QX372QhbUV/G367aw5JuP8NTLB8MuT0RGsVMGupnFgdXAB4A5wEozm5PT7Gmg0d3PA34M/H2xCx2J5k6byO2fXMD3P7WAzu4kV37nCdY8sp2wPvWIyOhWSA99IdDi7i+5ewdwF3B5ZgN3f8jd3womfwvUFrfMke2S2Wew7oaLWDJ3Cn+7bgtf/kWzLgAmIsOukECfDuzOmG4N5vXlGuCBfAvMbJWZNZlZU1tbaV1HZUJlGbeufDfXXFjPHY/v5G/WbQ67JBEZZRLFXJmZ/QnQCPx+vuXuvgZYA6kvRYv52iNBLGZ84YPnkHTne7/ZQV31WD6+qGS/ThCREaaQHvoeYEbGdG0wL4uZXQr8BbDc3Y8Xp7zoMTP+8oNzuGR2DV+5v5mt+46EXZKIjBKFBPpGoMHM6s2sHFgBrM1sYGbzgO+SCvNRf6+3WMz4hyvfxYTKMv7Pvc9oPF1EhsUpA93du4DrgfXAZuAed3/BzG4xs+VBs38AxgH3mtnvzGxtH6sbNarHVfCFD57Dc3sOcd/TvT7QiIgUnU4sGkLJpPM/vv04ew8d49E//wOdeCQigzboE4tkYGIx48bL3s6rh4+z9plXwi5HREqcAn2IvbehmtlTxnPboy/phCMRGVIK9CFmZlxzUT1b9h2haZcuDSAiQ0eBPgyWvXMqlWUxfqYvR0VkCCnQh8G4igSXzTmTXz63l46uZNjliEiJUqAPkyt+bxpvvNXJb1pK65IHIjJyKNCHyUUNNZxWHufBzaP+vCsRGSIK9GFSnoix+G3VPLy1TUe7iMiQUKAPo4tn17DnjWNsbzsadikiUoIU6MPo4tk1ADy8VePoIlJ8CvRhVDvpNM6uHssT2w+EXYqIlCAF+jBbUFdF066DugKjiBSdAn2YLaiv4tCxTra9puuki0hxKdCH2cK6KgA27ng95EpEpNQo0IfZjKoxTJlQwZM7dV0XESkuBfowMzMW1FWxccfrOh5dRIpKgR6CBXVV7DvcTuvBY2GXIiIlRIEeggXBOHrTLo2ji0jxKNBDMPvM8YyvSLBR4+giUkQK9BDEY8b8ukk60kVEikqBHpIFdVW8+NpRDr7ZEXYpIlIiFOghaTxrEgCbdFs6ESkSBXpI3jXjdMrjMTbqi1ERKRIFekgqy+K8s3YiTfpiVESKRIEeosa6STzb+gbtnd1hlyIiJUCBHqKFdVV0djtPaRxdRIpAgR6iRWdPpjwR48Etus+oiAyeAj1EYysSLH7bZB7c/Kqu6yIig6ZAD9ml50xh54G32LJP10cXkcFRoIds2TunUh6PcU/T7rBLEZGIU6CHrGpsOZfNncJPn97DsQ4d7SIiA6dAHwE+tbiON97q5N+e2Bl2KSISYQr0EaCxropLZtfwLw+1sO9Qe9jliEhEKdBHiL/80Bw6u50b7/4dHV3JsMsRkQhSoI8QZ9eM46+vOJcnXjrAn/5oE0ePd4VdkohEjAJ9BPno/Fq+cvlcHtraxof/+Tf8eouOTxeRwiXCLkCyffyCOhqmjOfm+57j6juaaDhjHB+dX8vFs2uYPWU8ZhZ2iSIyQlkhPUAzWwp8C4gDt7n713KWVwA/AOYDB4A/dPedJ1tnY2OjNzU1DbDs0tfRleRnT+/hRxt28UzrIQAmnVbGOVMncM7UCdRVj2XaxErOnFjJ1IljmDimjHhMYS/DZ3vbUe7ZuJuxFQk+eN5U3lYzLuySRgUz2+TujXmXnSrQzSwObAMuA1qBjcBKd2/OaPOnwHnufq2ZrQA+4u5/eLL1KtALt/fQMR7dtp9Nuw6yZd9htr56hPbO7C9OzWBcRYLTTytj4pjUz5iyBJVlMSrL4owpi/c8riyLU5GIUZ6IEY8ZZbHU70TcSASPy+IW/A6WxYxEPEbcDDOImRGLgWHEDMxSv2MZy3vaBY8zp2OWeq7F6JmOBZ8+LL0seH+ynm20jMcn5snw6exOsnnvYe54fCf3PbUna9ncaRO4bM4UzqudyNSJYxhXkWBCZRnjKhPqbBTRYAP9AuCv3H1JMH0zgLt/NaPN+qDNE2aWAPYBNX6SlSvQB6476ew/epxX3jjG3kPt7DvUzhvHOjl8rJNDGT/HOrpp7wx+upI9j5MlOiyf7w0gPX1iWXajfMtSbyi915F+0GtZ1rqy21vGE3Pfe3r/7/A+l2VOZv63yl1F5vNy//t5nxOFr/94V5LupFMWNz6+qI4r5k3jaHsXD29r45FtbWx99Uie7YKx5XHGlCeCN/8Tb9jpfzPLM/9UCn0zL6hVge83xarrhvc18OF3TSvsRXuvv89AL2QMfTqQeV56K3B+X23cvcvMDgGTgf05hawCVgHMnDmzoOKlt3jMmDKhkikTKpnXz+e6Ox3dSdo7k3R1p/5zdiad7m6nMxlMB/O7kk5Xt9OVTNLV7T3z3J2kc+I3mdNOMpkKgqSfaJt6TNZ0+jmeMZ0M0sB7fgd1Zz3uvSw9kc6SzLZZ7XLX02vZydv3CtqM1z1ZfSee52TGQu7//czJ3svyPy83Pk4WKNnPs5Msyz+/IhGnYco4LmqooWpsec/8xbOq+fyyczh6vIvNew+z/8hxjrR3cbi9k6PHuzjS3sVbHd1A+u8j+HuAnr+L1GPv9SaST6HHChS2rsJWVlCrAuuaOKassIb9NKxfirr7GmANpHrow/nakmJmVCTiVCTiYZciJWhcRYIFdVVhlzFqFXLY4h5gRsZ0bTAvb5tgyGUiqS9HRURkmBQS6BuBBjOrN7NyYAWwNqfNWuCq4PHHgF+fbPxcRESK75RDLsGY+PXAelKHLd7u7i+Y2S1Ak7uvBb4H/NDMWoDXSYW+iIgMo4LG0N19HbAuZ94XMx63A1cWtzQREekPnfovIlIiFOgiIiVCgS4iUiIU6CIiJaKgi3MNyQubtQG7Bvj0anLOQh0FtM2jg7Z5dBjMNp/l7jX5FoQW6INhZk19XcugVGmbRwdt8+gwVNusIRcRkRKhQBcRKRFRDfQ1YRcQAm3z6KBtHh2GZJsjOYYuIiK9RbWHLiIiORToIiIlInKBbmZLzWyrmbWY2U1h11MsZjbDzB4ys2Yze8HMbgjmV5nZf5nZi8HvScF8M7N/Cv4dnjWzd4e7BQNjZnEze9rM7g+m681sQ7BddweXbMbMKoLplmB5XZh1D5SZnW5mPzazLWa22cwuGAX7+Mbgb/p5M7vTzCpLcT+b2e1m9pqZPZ8xr9/71syuCtq/aGZX5XutvkQq0IMbVq8GPgDMAVaa2ZxwqyqaLuBz7j4HWARcF2zbTcCD7t4APBhMQ+rfoCH4WQV8e/hLLoobgM0Z038HfMPdZwEHgWuC+dcAB4P53wjaRdG3gF+5+zuAd5Ha9pLdx2Y2Hfgs0Oju55K6BPcKSnM/3wEszZnXr31rZlXAl0jd5nMh8KX0m0BBPLjvYxR+gAuA9RnTNwM3h13XEG3rz4HLgK3A1GDeVGBr8Pi7wMqM9j3tovJD6u5XDwJ/ANxP6laW+4FE7v4mdT3+C4LHiaCdhb0N/dzeicCO3LpLfB+n7zdcFey3+4ElpbqfgTrg+YHuW2Al8N2M+VntTvUTqR46+W9YPT2kWoZM8DFzHrABmOLue4NF+4ApweNS+Lf4JvDnQDKYngy84e5dwXTmNmXdiBxI34g8SuqBNuD7wTDTbWY2lhLex+6+B/hH4GVgL6n9tonS3s+Z+rtvB7XPoxboJc/MxgE/Af63ux/OXOapt+ySOM7UzD4EvObum8KuZRglgHcD33b3ecCbnPgIDpTWPgYIhgsuJ/VmNg0YS+9hiVFhOPZt1AK9kBtWR5aZlZEK8x+5+33B7FfNbGqwfCrwWjA/6v8W7wGWm9lO4C5Swy7fAk4PbjQO2dtUCjcibwVa3X1DMP1jUgFfqvsY4FJgh7u3uXsncB+pfV/K+zlTf/ftoPZ51AK9kBtWR5KZGal7s252969nLMq8AfdVpMbW0/M/EXxbvgg4lPHRbsRz95vdvdbd60jtx1+7+x8DD5G60Tj03t5I34jc3fcBu81sdjDrfUAzJbqPAy8Di8zstOBvPL3NJbufc/R3364H3m9mk4JPN+8P5hUm7C8RBvClwzJgG7Ad+Iuw6ynidl1I6uPYs8Dvgp9lpMYPHwReBP4fUBW0N1JH/GwHniN1FEHo2zHAbb8YuD94fDbwJNAC3AtUBPMrg+mWYPnZYdc9wG39PaAp2M8/AyaV+j4GvgxsAZ4HfghUlOJ+Bu4k9T1BJ6lPY9cMZN8CVwfb3wJ8qj816NR/EZESEbUhFxER6YMCXUSkRCjQRURKhAJdRKREKNBFREqEAl1EpEQo0EVESsT/B772XdPezGMKAAAAAElFTkSuQmCC\n",
"text/plain": [
"<Figure size 432x288 with 1 Axes>"
]
},
"metadata": {
"tags": [],
"needs_background": "light"
}
}
]
},
{
"cell_type": "code",
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "Se9nejmt1p3L",
"outputId": "5bb6b356-2d7e-43a9-8726-9782374160f8"
},
"source": [
"out = net.predict(x_train)\n",
"print(x_train, out)"
],
"execution_count": null,
"outputs": [
{
"output_type": "stream",
"text": [
"[[[0 0]]\n",
"\n",
" [[0 1]]\n",
"\n",
" [[1 0]]\n",
"\n",
" [[1 1]]] [array([[0.00043794]]), array([[0.97852756]]), array([[0.97771572]]), array([[-0.00452103]])]\n"
],
"name": "stdout"
}
]
},
{
"cell_type": "code",
"metadata": {
"id": "Fe0xmRlK2cZf",
"colab": {
"base_uri": "https://localhost:8080/"
},
"outputId": "ed7fa44f-f37e-43b6-f180-4dbb26cc4187"
},
"source": [
"###### Digits recognitin\n",
"from keras.datasets import mnist\n",
"from keras.utils import np_utils\n",
"\n",
"(x_train, y_train), (x_test, y_test) = mnist.load_data()\n",
"x_train = x_train.reshape(x_train.shape[0], 1, 28*28).astype('float32')/255\n",
"y_train = np_utils.to_categorical(y_train)\n",
"\n",
"x_test = x_test.reshape(x_test.shape[0], 1, 28*28).astype('float32')/255\n",
"y_test = np_utils.to_categorical(y_test)"
],
"execution_count": null,
"outputs": [
{
"output_type": "stream",
"text": [
"Downloading data from https://storage.googleapis.com/tensorflow/tf-keras-datasets/mnist.npz\n",
"11493376/11490434 [==============================] - 0s 0us/step\n"
],
"name": "stdout"
}
]
},
{
"cell_type": "code",
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "JGhSWfaU5o7G",
"outputId": "5855c39d-a041-46dc-db63-4aca90f3040c"
},
"source": [
"y_train[0]"
],
"execution_count": null,
"outputs": [
{
"output_type": "execute_result",
"data": {
"text/plain": [
"array([0., 0., 0., 0., 0., 1., 0., 0., 0., 0.], dtype=float32)"
]
},
"metadata": {
"tags": []
},
"execution_count": 33
}
]
},
{
"cell_type": "code",
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "PfILnMdC3uqL",
"outputId": "37d66d00-1a71-4910-e0f2-8828923d3811"
},
"source": [
"net = Network()\n",
"net.add(FCLayer(28*28, 100))\n",
"net.add(ActivationLayer())\n",
"net.add(FCLayer(100, 50)) \n",
"net.add(ActivationLayer())\n",
"net.add(FCLayer(50, 10)) \n",
"net.add(ActivationLayer())\n",
"net.use(mse, mse_prime)\n",
"\n",
"errors = net.fit(x_train[0:5000], y_train[0:5000], epochs=35, learning_rate=0.1)"
],
"execution_count": null,
"outputs": [
{
"output_type": "stream",
"text": [
"epoch 1/35 error=597.229832\n",
"epoch 2/35 error=294.311087\n",
"epoch 3/35 error=222.290965\n",
"epoch 4/35 error=182.827325\n",
"epoch 5/35 error=156.561311\n",
"epoch 6/35 error=137.362874\n",
"epoch 7/35 error=122.995554\n",
"epoch 8/35 error=111.681362\n",
"epoch 9/35 error=102.327054\n",
"epoch 10/35 error=94.795020\n",
"epoch 11/35 error=88.419284\n",
"epoch 12/35 error=82.838767\n",
"epoch 13/35 error=78.053786\n",
"epoch 14/35 error=73.588500\n",
"epoch 15/35 error=69.571904\n",
"epoch 16/35 error=66.085256\n",
"epoch 17/35 error=62.907259\n",
"epoch 18/35 error=59.947271\n",
"epoch 19/35 error=57.193714\n",
"epoch 20/35 error=54.729624\n",
"epoch 21/35 error=52.550492\n",
"epoch 22/35 error=50.488494\n",
"epoch 23/35 error=48.678247\n",
"epoch 24/35 error=47.048339\n",
"epoch 25/35 error=45.462000\n",
"epoch 26/35 error=43.884629\n",
"epoch 27/35 error=42.421490\n",
"epoch 28/35 error=41.161154\n",
"epoch 29/35 error=39.919483\n",
"epoch 30/35 error=38.800290\n",
"epoch 31/35 error=37.824970\n",
"epoch 32/35 error=36.865852\n",
"epoch 33/35 error=35.948138\n",
"epoch 34/35 error=35.107472\n",
"epoch 35/35 error=34.259474\n"
],
"name": "stdout"
}
]
},
{
"cell_type": "code",
"metadata": {
"id": "QElfsG8W6TEk"
},
"source": [
"errors=[]\n",
"for i in range(1000):\n",
" out=sum((net.predict(x_test[i]) - y_test[i])[0][0])\n",
" errors.append(0 if out<0.5 else 1)\n"
],
"execution_count": null,
"outputs": []
},
{
"cell_type": "code",
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "04Fj4DyV8OIt",
"outputId": "30906536-20a2-4843-e5e3-00c4ffdd3477"
},
"source": [
"np.mean(errors)"
],
"execution_count": null,
"outputs": [
{
"output_type": "execute_result",
"data": {
"text/plain": [
"0.028"
]
},
"metadata": {
"tags": []
},
"execution_count": 36
}
]
},
{
"cell_type": "code",
"metadata": {
"id": "vjTT4Da59tWz"
},
"source": [
""
],
"execution_count": null,
"outputs": []
},
{
"cell_type": "markdown",
"metadata": {
"id": "nHLxeJrU9zGz"
},
"source": [
"Extend Digits network to hhandwritten characters \n",
"\n",
"Use NIST dataset: EMNIST Letters: 145,600 characters. 26 balanced classes.\n",
"https://www.nist.gov/itl/products-and-services/emnist-dataset\n",
"\n",
"* Play with NN configurations -- number of layers, ssize of layers\n",
"* Please try to break it\n",
"* Throw in some random noise \n",
"* Replace tangent with another function (logit/ sigmioid???)\n"
]
}
]
}