{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Neural Network\n",
    "\n",
    "Nbviewer [https://nbviewer.jupyter.org/github/shaundsouza/ai-ecosystems-enabling/blob/master/nnet.ipynb](https://nbviewer.jupyter.org/github/shaundsouza/ai-ecosystems-enabling/blob/master/nnet.ipynb)\n",
	"\n",
    "Execute on Binder [https://mybinder.org/v2/gh/shaundsouza/ai-ecosystems-enabling/master?filepath=nnet.ipynb](https://mybinder.org/v2/gh/shaundsouza/ai-ecosystems-enabling/master?filepath=nnet.ipynb)\n",
    "\n",
    "Code [https://nbviewer.jupyter.org/format/script/github/shaundsouza/ai-ecosystems-enabling/blob/master/nnet.ipynb](https://nbviewer.jupyter.org/format/script/github/shaundsouza/ai-ecosystems-enabling/blob/master/nnet.ipynb)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD8CAYAAACMwORRAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAFzZJREFUeJzt3X2QXfVdx/H3Z5/zuAGy1JiEJrXpQ+y0QlcK1hGq1AZUohU1mWofpGZ8wKc6KkwdWtE/LD7VjmlpBhGtLUgfpmaYdFJt0aoVyjJtKYGmbIHCCjZLSkJJSDab/frHORtubu7dezZ7s2fP735eM3fOOb/zu/d+z57ks7/93YejiMDMzNLSVXYBZmbWfg53M7MEOdzNzBLkcDczS5DD3cwsQQ53M7MEOdzNzBLkcDczS5DD3cwsQT1lPfHKlStj3bp1ZT29mVkl3XfffU9HxFCrfqWF+7p16xgZGSnr6c3MKknSt4r087SMmVmCHO5mZglyuJuZJcjhbmaWIIe7mVmCHO5mZglyuJuZJah64f7tPfC5G+Dwd8quxMxswapeuH/nUfjPv4QDj5ddiZnZglW9cF/6omz53L5y6zAzW8AqGO7nZsvnvl1uHWZmC1gFw3165O5wNzNrpnrh3jsAA4MOdzOzGVQv3CEbvTvczcyaqma4LzoLnj9QdhVmZgtWNcN9YAUccbibmTXTMtwl3SJpn6QHWvT7QUnHJV3VvvKaGBiEIwfP+NOYmVVVkZH7rcCmmTpI6gbeB+xuQ02tOdzNzGbUMtwj4gtAq8/6/ybwSWB+Plk0He4R8/J0ZmZVM+c5d0mrgZ8Bbpp7OQUNDEJMwcRz8/aUZmZV0o4XVN8P/GFEHG/VUdI2SSOSRsbHx0//GRetyJaemjEza6gd4T4M3C7pMeAq4IOSfrpRx4jYERHDETE8NDR0+s84MJgt/XZIM7OGeub6ABGxfnpd0q3AnRHx6bk+7oymw90jdzOzhlqGu6TbgEuBlZLGgPcAvQARMX/z7LUc7mZmM2oZ7hGxteiDRcTb51RNUQ53M7MZVfcTquBwNzNroprh3r88W/orCMzMGqpmuHf3QN8yj9zNzJqoZrgD9C/1h5jMzJqobrj3LYGjDnczs0YqHO5LYeJQ2VWYmS1IDnczswRVONyXeM7dzKwJh7uZWYIqHu6eljEza6S64d6/zOFuZtZEdcN9elrGV2MyMztFtcM9puDY82VXYma24FQ43JdmS0/NmJmdosLhviRb+h0zZmanqHC4e+RuZtZMhcPdI3czs2YqHO7TI3eHu5lZvZbhLukWSfskPdBk/1sk3Z/fvijpNe0vs4ETI3dPy5iZ1Ssycr8V2DTD/keBSyLi1cCfADvaUFdr0+Hur/01MztFkQtkf0HSuhn2f7Fm825gzdzLKqB/WbY8dnhens7MrEraPed+NfCZZjslbZM0ImlkfHx8bs/kF1TNzJpqW7hLegNZuP9hsz4RsSMihiNieGhoaG5P2DMA6vKcu5lZAy2nZYqQ9GrgZuDyiNjfjscs8KS+YIeZWRNzHrlLOg/4FPBLEfGNuZc0C/5OdzOzhlqO3CXdBlwKrJQ0BrwH6AWIiJuA64FzgA9KApiMiOEzVfBJ/J3uZmYNFXm3zNYW+98JvLNtFc2Gw93MrKHqfkIVPOduZtZExcPdc+5mZo0kEO4euZuZ1XO4m5klqOLhvtTTMmZmDVQ83PORuy+SbWZ2kuqH+9QkHJ8ouxIzswWl4uHuS+2ZmTVS8XD3N0OamTWSSLh75G5mVqvi4e5pGTOzRioe7p6WMTNrJJFw98jdzKxWxcPd0zJmZo1UPNw9LWNm1kgi4e6Ru5lZrWqHe+/ibOlwNzM7Sctwl3SLpH2SHmiyX5I+IGlU0v2SLmh/mU10dWcB72kZM7OTFBm53wpsmmH/5cCG/LYN+NDcy5oFf+2vmdkpWoZ7RHwB+M4MXTYD/xiZu4EVkla1q8CWHO5mZqdox5z7auCJmu2xvG1++DqqZmanaEe4q0Fbwy9Yl7RN0oikkfHx8TY8Nb6OqplZA+0I9zFgbc32GuDJRh0jYkdEDEfE8NDQUBueGk/LmJk10I5w3wm8NX/XzEXAwYh4qg2PW4zD3czsFD2tOki6DbgUWClpDHgP0AsQETcBu4ArgFHgMPCOM1VsQ76OqpnZKVqGe0RsbbE/gN9oW0Wz5ZG7mdkpqv0JVXC4m5k1kEC4L4XJI3B8suxKzMwWjOqHe/+ybDnx3XLrMDNbQKof7gOD2fLIwXLrMDNbQBzuZmYJcribmSXI4W5mliCHu5lZghzuZmYJqn649y0D5HA3M6tR/XDv6oL+5Q53M7Ma1Q93yKZmHO5mZic43M3MEuRwNzNLkMPdzCxBCYX7s2VXYWa2YCQU7h65m5lNSyfcjz4LU1NlV2JmtiAUCndJmyTtlTQq6doG+8+TdJekL0u6X9IV7S91BgODQGQBb2ZmrcNdUjewHbgc2AhslbSxrtsfAXdExPnAFuCD7S50Rv4KAjOzkxQZuV8IjEbEIxExAdwObK7rE8DyfH0QeLJ9JRZwItwPzOvTmpktVD0F+qwGnqjZHgNeV9fnvcBnJf0msAS4rC3VFTWQ/1456kvtmZlBsZG7GrRF3fZW4NaIWANcAXxE0imPLWmbpBFJI+Pj47Ovtpnp66g63M3MgGLhPgasrdlew6nTLlcDdwBExP8AA8DK+geKiB0RMRwRw0NDQ6dXcSP9HrmbmdUqEu73AhskrZfUR/aC6c66Po8DPwYg6ZVk4d7GoXkLJ0bufreMmRkUCPeImASuAXYDD5G9K2aPpBskXZl3+z3gVyR9FbgNeHtE1E/dnDmeljEzO0mRF1SJiF3Arrq262vWHwRe397SZqFnALp6/BUEZma5ND6hKmWjd4/czcyAVMIdshdVHe5mZoDD3cwsSQmF+zK/W8bMLJdYuHvkbmYGDnczsyQ53M3MEpRYuHvO3cwMUgr3vqUweQSmjpddiZlZ6dIJ995F2fLY8+XWYWa2ADjczcwSlFC4L86Wxw6VW4eZ2QKQTrj3TYe7R+5mZumE+4mR++Fy6zAzWwASCnfPuZuZTUso3POR+4RH7mZmCYX79Mjd4W5mllC4+wVVM7NphcJd0iZJeyWNSrq2SZ+fl/SgpD2SPtbeMgvwC6pmZie0vIaqpG5gO/BGYAy4V9LO/Lqp0302ANcBr4+IZySde6YKbsrTMmZmJxQZuV8IjEbEIxExAdwObK7r8yvA9oh4BiAi9rW3zAL8bhkzsxOKhPtq4Ima7bG8rdbLgJdJ+m9Jd0va1OiBJG2TNCJpZHx8/PQqbqa7F7p6PXI3M6NYuKtBW9Rt9wAbgEuBrcDNklaccqeIHRExHBHDQ0NDs621tb7FHrmbmVEs3MeAtTXba4AnG/T5l4g4FhGPAnvJwn5+9S6GCX+3jJlZkXC/F9ggab2kPmALsLOuz6eBNwBIWkk2TfNIOwstpHeRp2XMzCgQ7hExCVwD7AYeAu6IiD2SbpB0Zd5tN7Bf0oPAXcDvR8T+M1V0U72L4diReX9aM7OFpuVbIQEiYhewq67t+pr1AN6V38rjkbuZGZDSJ1QhH7n7BVUzswTD3SN3M7PEwt3TMmZmkFy4e1rGzAySC3eP3M3MILVw9ydUzcyA1MK9dzFMHoGpqbIrMTMrVWLh7q/9NTOD5MLdV2MyM4Nkw90jdzPrbImFuy/YYWYGyYW7R+5mZpBcuHvkbmYGyYW7R+5mZpBauPc53M3MILVw97SMmRmQXLh75G5mBgXDXdImSXsljUq6doZ+V0kKScPtK3EWPHI3MwMKhLukbmA7cDmwEdgqaWODfsuA3wLuaXeRhU2P3Cc8cjezzlZk5H4hMBoRj0TEBHA7sLlBvz8BbgTKu0J1dy909Xpaxsw6XpFwXw08UbM9lredIOl8YG1E3NnG2k6PL9hhZlYo3NWgLU7slLqAvwZ+r+UDSdskjUgaGR8fL17lbPiCHWZmhcJ9DFhbs70GeLJmexnwKuDfJT0GXATsbPSiakTsiIjhiBgeGho6/apn0rvII3cz63hFwv1eYIOk9ZL6gC3AzumdEXEwIlZGxLqIWAfcDVwZESNnpOJWehd75G5mHa9luEfEJHANsBt4CLgjIvZIukHSlWe6wFnrWwITh8quwsysVD1FOkXELmBXXdv1TfpeOvey5mBgOTz/TKklmJmVLa1PqAL0L4cjB8uuwsysVOmF+8AgHHm27CrMzEqVYLh75G5mlmC4D8Lxo3CsvA/KmpmVLb1w71+eLY96asbMOld64T4wmC09725mHSzhcPe8u5l1rvTC/cS0jMPdzDpXeuHukbuZWcLh/vyBcuswMytReuG+ZGW2PPx0uXWYmZUovXDv6Yf+QTjkcDezzpVeuAMsOQcOnaGLgZiZVUCi4T7kcDezjpZwuHtaxsw6V6LhvtIjdzPraImG+xAc3g9Tx8uuxMysFImG+7kQU1nAm5l1oELhLmmTpL2SRiVd22D/uyQ9KOl+SZ+T9OL2lzoLg6uz5cGxUsswMytLy3CX1A1sBy4HNgJbJW2s6/ZlYDgiXg18Arix3YXOyuDabHng8VLLMDMrS5GR+4XAaEQ8EhETwO3A5toOEXFXRBzON+8G1rS3zFlakYf7wSdKLcPMrCxFwn01UJuSY3lbM1cDn2m0Q9I2SSOSRsbHz+C7WQZWQN8yOOBwN7POVCTc1aAtGnaUfhEYBv680f6I2BERwxExPDQ0VLzK2ZKy0btH7mbWoXoK9BkD1tZsrwGerO8k6TLg3cAlEXG0PeXNwYrz4JlvlV2FmVkpiozc7wU2SFovqQ/YAuys7SDpfODDwJURsa/9ZZ6GlS+D/Q/D8cmyKzEzm3ctwz0iJoFrgN3AQ8AdEbFH0g2Srsy7/TmwFPi4pK9I2tnk4ebPuRvh+AQ882jZlZiZzbsi0zJExC5gV13b9TXrl7W5rrk79xXZct+DsHJDubWYmc2zND+hCrDy5YBg30NlV2JmNu/SDfe+xdmI/ckvl12Jmdm8SzfcAda+Dh6/G6amyq7EzGxepR3u510MRw7A098ouxIzs3mVeLhflC0f/UK5dZiZzbO0w/2c74NzNsDX7yy7EjOzeZV2uAO88ifhsf+Cw98puxIzs3mTfrh//89AHIevfbzsSszM5k364b7qNbDmB+FLO/yuGTPrGOmHO8DrfhX2j8KeT5VdiZnZvOiMcP/+N8P3vBr+7b1w9LmyqzEzO+M6I9y7uuDyG7Nrqu76/bKrMTM74zoj3AFefDFc8gfw1Y/BfzS8loiZWTIKfStkMi65NruAx11/Cs8/A2+8Abo760dgZp2hs5Ktqws2b4eBQbh7Ozz+RbjiL2HNa8uuzMysrTpnWmZadw9ccSNc9ffw7FNw84/CP/0sPPxvvmqTmSWjs0butV71ZnjpZXDPh7P3wH/0Z2HR2fCKn4D1l2Rz9INryq7SzOy0KCJad5I2AX8DdAM3R8Sf1e3vB/4ReC2wH/iFiHhspsccHh6OkZGR0yy7zSaPwsOfhT2fzpZHn83al62Cc1+ZXbJv6OXZRbcH18Ly1dA7UG7NZtaRJN0XEcOt+rUcuUvqBrYDbwTGgHsl7YyIB2u6XQ08ExEvlbQFeB/wC6dXegl6+uGVP5Xdpo7Dtx+Ax/4bnvpqdpm+e2+GySMn32fJECx9ESw6Cxafk9/Ozpb9y6FvCfQthf6l+foS6FuWLXsXgVTOsZpZRygyLXMhMBoRjwBIuh3YDNSG+2bgvfn6J4C/laQo8mfBQtPVnX1lwarXvNA2dRwOPJ69T/7E7Qk49DQc3p/9Aji8P3sHThT8ioPuPugZyJf9Lyx7+qG7/+S27j7o6slv3fmtp66twba6G/TpBnUBypbqyn7RnFhv1tagD63upwb9VbPMndSmNrXV76PJ88+yzb+UrSKKhPtq4Ima7THgdc36RMSkpIPAOcDT7SiydF3dcPb67DaTqans4iBHvwsTh2Diufx2KPtk7PT6scPZVNDxiXx5NFvWtk0ehWMHYHIi2z91HKYma5aT2Rei1W5PTRb/5WJt0OIXw0ld638pzLR/LvdttH8u953D/lIfmxb75/N8NOhzwVvhh65p0K99ioR7o0rrR+RF+iBpG7AN4Lzzzivw1BXT1ZVPzZxdXg1TU3noT9b8MpjePpaF/4lb5Lepxjeipl+TPlGgz0mPE/n29D+PdrTxwr7a9Tm1McfHq1W3PdP+Vn/stvWxZ3HfWe9v92O3q65Z7p/rYzfsAyw999S2NisS7mPA2prtNcCTTfqMSeoBBoFTvkA9InYAOyB7QfV0CrYWurqALujuLbsSMytRkfe53wtskLReUh+wBdhZ12cn8LZ8/Srg85WcbzczS0TLkXs+h34NsJvsrZC3RMQeSTcAIxGxE/g74COSRslG7FvOZNFmZjazQh9iiohdwK66tutr1o8AP9fe0szM7HR13tcPmJl1AIe7mVmCHO5mZglyuJuZJcjhbmaWoELfCnlGnlgaB751mndfSSpfbVCcj7kz+Jg7w1yO+cURMdSqU2nhPheSRop85WVKfMydwcfcGebjmD0tY2aWIIe7mVmCqhruO8ouoAQ+5s7gY+4MZ/yYKznnbmZmM6vqyN3MzGZQuXCXtEnSXkmjkq4tu57TJWmtpLskPSRpj6TfztvPlvSvkh7Ol2fl7ZL0gfy475d0Qc1jvS3v/7CktzV7zoVCUrekL0u6M99eL+mevP5/zr9aGkn9+fZovn9dzWNcl7fvlfSmco6kGEkrJH1C0tfz831x6udZ0u/m/64fkHSbpIHUzrOkWyTtk/RATVvbzquk10r6Wn6fD0izvMZjRFTmRvaVw98EXgL0AV8FNpZd12keyyrggnx9GfANYCNwI3Bt3n4t8L58/QrgM2RXvboIuCdvPxt4JF+ela+fVfbxtTj2dwEfA+7Mt+8AtuTrNwG/lq//OnBTvr4F+Od8fWN+7vuB9fm/ie6yj2uG4/0H4J35eh+wIuXzTHbZzUeBRTXn9+2pnWfgR4ALgAdq2tp2XoEvARfn9/kMcPms6iv7BzTLH+bFwO6a7euA68quq03H9i/AG4G9wKq8bRWwN1//MLC1pv/efP9W4MM17Sf1W2g3sit5fQ74UeDO/B/u00BP/Tkmu4bAxfl6T95P9ee9tt9CuwHL86BTXXuy55kXrql8dn7e7gTelOJ5BtbVhXtbzmu+7+s17Sf1K3Kr2rRMo4t1ry6plrbJ/ww9H7gHeFFEPAWQL6cvttjs2Kv2M3k/8AfA9JW8zwEORMRkvl1b/0kXXgemL7xepWN+CTAO/H0+FXWzpCUkfJ4j4n+BvwAeB54iO2/3kfZ5ntau87o6X69vL6xq4V7oQtxVImkp8EngdyLi2Zm6NmiLGdoXHEk/CeyLiPtqmxt0jRb7KnPMZCPRC4APRcT5wCGyP9ebqfwx5/PMm8mmUr4XWAJc3qBrSue5ldke45yPvWrhXuRi3ZUhqZcs2D8aEZ/Km78taVW+fxWwL29vduxV+pm8HrhS0mPA7WRTM+8HVii7sDqcXP+JY9PJF16v0jGPAWMRcU++/QmysE/5PF8GPBoR4xFxDPgU8EOkfZ6nteu8juXr9e2FVS3ci1ysuxLyV77/DngoIv6qZlftxcbfRjYXP93+1vxV94uAg/mffbuBH5d0Vj5i+vG8bcGJiOsiYk1ErCM7d5+PiLcAd5FdWB1OPeZGF17fCWzJ32WxHthA9uLTghMR/wc8IenledOPAQ+S8Hkmm465SNLi/N/59DEne55rtOW85vu+K+mi/Gf41prHKqbsFyRO4wWMK8jeWfJN4N1l1zOH4/hhsj+z7ge+kt+uIJtr/BzwcL48O+8vYHt+3F8Dhmse65eB0fz2jrKPreDxX8oL75Z5Cdl/2lHg40B/3j6Qb4/m+19Sc/935z+LvczyXQQlHOsPACP5uf402bsikj7PwB8DXwceAD5C9o6XpM4zcBvZawrHyEbaV7fzvALD+c/vm8DfUveifKubP6FqZpagqk3LmJlZAQ53M7MEOdzNzBLkcDczS5DD3cwsQQ53M7MEOdzNzBLkcDczS9D/AxVgYO2rlMiUAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from matplotlib import animation, rc\n",
    "from IPython.display import HTML\n",
    "\n",
    "get_ipython().magic('matplotlib inline')\n",
    "\n",
    "def sigmoid(x):\n",
    "    return 1 / (1 + np.exp(-x))\n",
    "\n",
    "def diff_sigmoid(t):\n",
    "    return t * (1 - t)\n",
    "\n",
    "def od(t, y):\n",
    "    return diff_sigmoid(t) * (y - t)\n",
    "\n",
    "def nn_error(x, w1, w2, y):\n",
    "    h = sigmoid(np.matmul(x, w1))\n",
    "    t = sigmoid(np.matmul(h, w2))\n",
    "    return sum((y - t)**2)\n",
    "\n",
    "fig, ax = plt.subplots()\n",
    "\n",
    "x = np.array([1.5,2,1,3.5,3,4,3,1,2,4,3,2,1,1,1,1,1,1]).reshape(3,6).T\n",
    "size_x = x.shape\n",
    "\n",
    "y = np.array([0,0,0,1,1,1]).reshape(6,1)\n",
    "size_y = y.shape\n",
    "\n",
    "size_h = 2\n",
    "err = []\n",
    "\n",
    "line, = ax.plot([], [], lw=2)\n",
    "\n",
    "def init():\n",
    "    line.set_data([], [])\n",
    "    return (line,)\n",
    "\n",
    "def animate(i):\n",
    "    line.set_data(err[i][0], err[i][1])\n",
    "    return (line,)\n",
    "\n",
    "# w1 = np.array([.2,-.1,.4,.7,-1.2,1.2]).reshape(2,3).T\n",
    "# w2 = np.array([1.1,.1]).reshape(1,2).T\n",
    "# w2 = np.array([1.1,.1,3.1,1.17]).reshape(2,2).T\n",
    "w1 = np.random.randn(3, 2)\n",
    "w2 = np.random.randn(2, 1)\n",
    "\n",
    "err_prev = 100\n",
    "# print(x)\n",
    "# print(w1)\n",
    "# print(w2)\n",
    "\n",
    "for n in range(10000):\n",
    "    e = nn_error(x, w1, w2, y) \n",
    "    err.append(e)\n",
    "#     print(n, e)\n",
    "\n",
    "    if (e > err_prev):\n",
    "        break\n",
    "\n",
    "    err_prev = e\n",
    "\n",
    "    for i in range(size_x[0]):\n",
    "        h = sigmoid(np.dot(x[i,:], w1))\n",
    "        t = sigmoid(np.dot(h, w2))\n",
    "\n",
    "        d_o = od(t, y[i,:])\n",
    "        backp = d_o * w2\n",
    "        d_h = h * (1 - h) * backp.ravel()\n",
    "\n",
    "        a=0.3\n",
    "\n",
    "        for j in range(size_x[1]):\n",
    "            d_w1 = a * x[i, j] * d_h\n",
    "            w1[j,:] = w1[j,:] + d_w1\n",
    "\n",
    "        for j in range(size_h):\n",
    "            d_w2 = a * h[j] * d_o\n",
    "            w2[j,:] = w2[j,:] + d_w2\n",
    "\n",
    "h = sigmoid(np.matmul(x, w1))\n",
    "t = sigmoid(np.matmul(h, w2))\n",
    "# print(w1)\n",
    "# print(w2)\n",
    "# print(t)\n",
    "\n",
    "plt.plot(err)\n",
    "plt.show()\n",
    "\n",
    "# anim = animation.FuncAnimation(fig, animate, init_func=init, frames=len(err))\n",
    "\n",
    "# HTML(anim.to_html5_video())\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}