{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "toc": "true"
   },
   "source": [
    "# Table of Contents\n",
    " <p>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "import math,sys,os,numpy as np\n",
    "from numpy.random import random\n",
    "from matplotlib import pyplot as plt, rcParams, animation, rc\n",
    "from __future__ import print_function, division\n",
    "from ipywidgets import interact, interactive, fixed\n",
    "from ipywidgets.widgets import *\n",
    "rc('animation', html='html5')\n",
    "rcParams['figure.figsize'] = 3, 3\n",
    "%precision 4\n",
    "np.set_printoptions(precision=4, linewidth=100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def lin(a,b,x): return a*x+b"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "a=3.\n",
    "b=8."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "n=30\n",
    "x = random(n)\n",
    "y = lin(a,b,x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 0.2764,  0.3057,  0.4897,  0.787 ,  0.6627,  0.6034,  0.3722,  0.1886,  0.4145,  0.6801,\n",
       "        0.5178,  0.4675,  0.3708,  0.9557,  0.1604,  0.4065,  0.8752,  0.6775,  0.1818,  0.2346,\n",
       "        0.591 ,  0.9049,  0.9551,  0.3547,  0.7319,  0.5636,  0.7542,  0.7464,  0.2111,  0.1858])"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([  8.8293,   8.9171,   9.4692,  10.3611,   9.9881,   9.8102,   9.1167,   8.5659,   9.2436,\n",
       "        10.0403,   9.5534,   9.4025,   9.1125,  10.8671,   8.4813,   9.2196,  10.6257,  10.0326,\n",
       "         8.5453,   8.7038,   9.7731,  10.7148,  10.8654,   9.064 ,  10.1956,   9.6907,  10.2627,\n",
       "        10.2393,   8.6334,   8.5573])"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.collections.PathCollection at 0x7f57280e7dd0>"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAANgAAADICAYAAACK9i92AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAEoNJREFUeJzt3X2QVNWZx/HvIzLDZBVEnWBEGLK+xBl3LQcWnLLyAhFK\nY2VDEpMglvFtghgK1GzFMpukVt1k3ZiY7GoSCrRGNMk6TioxanzZqBF0LReHIIpKo7g6w4su6UTU\nUIsw4rN/3DvSND1D07dPv/4+VV3cvn27z2noh3vuOec+x9wdEQnjoHJXQKSWKcBEAlKAiQSkABMJ\nSAEmEpACTCSg/QaYmXWZ2VYzW5ux7wtm9ryZ7TazycO890wzW29mL5nZVcWqtEi1yOcMtgw4I2vf\nc8DngMeGepOZHQT8JH7vScBcMzuxwHqKVKX9Bpi7PwFsy9r3ortvAGyYt04DNrh7v7sPAHcCs5NU\nVqTahLwGGw9syni+Od4nUjcOLncFAMxM87Wkorn7cK21IYU8g20BJmY8Pybel5O7l+xx9dVXqzyV\nl/cjiXwDzBj6emuo/auA48ysxcwagHOAew+wfiJVLZ9u+juAJ4ETzGyjmV1kZp81s01AB3CfmT0Y\nH/shM7sPwN13AwuBh4AXgDvdPRXqi4hUov1eg7n7uUO8dHeOY18HPp3x/D+BjxRcu0CmT5+u8lRe\nSVjSNmZRKmHmlVAPqRypVIre3l6mTZtGa2trWetiZngFdnKIFGTRoitoa5vChRdeR1vbFBYturzc\nVSqYzmBSUVKpFG1tU4CVwMnAWqCDdetWl+1MpjOY1Ize3l5gAlFwEf95TLy/+ijApKJMmzaNaALQ\n4NzytcDmeH/1UYBJRWltbWXhwnlEI0AnAB0sXDiv7B0dhdI1mFSkWulFVICJ7Ic6OaQqpdNpVq1a\nRTqdLndVglGASVl0d/fQ0nIis2ZdSkvLiXR395S7SkGoiSgll06naWk5kR07ljM41tXUNIP+/vU0\nNzeXu3r7UBNRqkpfXx8NDZPIHOsaObKFvr6+8lUqEAWYlNykSZPYtauPzLGugYF+Jk2aVL5KBaIA\nk5IZ7NQA6OpaTFPTDEaPnkxT0wy6uhZXZPMwqYpIGSC1r7u7h87OBTQ0RGevrq7F9Pevp6+vj0mT\nJtVkcIE6OaQEqq1TI5s6OaSi1VOnRjYFmARXT50a2RRgElxzc3PddGpk2+81mJl1EeXZ2OruJ8f7\nxgI9QAvQB3zJ3d/K8d7dwLNEmaf63f2zQ5Sha7A6kE6nq7JTI+hkXzP7KLAd+FlGgF0P/Nndvx8v\n6jDW3b+R471vu/voPL6AAkwqVtBODs+Rm54ox/zt8fbtQM4zE8PnrhepeYVeg33Q3bcCuPv/Ah8c\n4rhGM+s1syfNTAs/SN0p1kDzUO27Fnd/3cw+DDxqZmvd/dVcB15zzTXvb0+fPr2qct9JbVmxYgUr\nVqwoymflNdBsZi3AbzOuwVLAdHffamZHAcvdfdjbTs1sWfwZd+V4TddgUrFKMdCcnZv+XuDCePsC\n4J4clToszkmPmR0JnAasK6SSItWqoNz0wPeAWWb2InB6/Bwzm2JmN8dvbQX+YGZrgN8D/+ru60N8\nCZFKpbmIIvuhuYhScvWQT6MYFGBywOoln0YxqIkoB6Tabz0phJqIUjL1fOtJIRRgckDq+daTQijA\n5IDU860nhdA1mOQl+1aTar31pBC6BpOgli69hQkTjuP00zvf7zVsbm5m6tSpNR9cSekMJsNauvQW\nLr30cqKlhDYBV9HUdH1N9xpmS3IGU9o2GVIqleKyy65k7+VcZzBixNH09fXVTYAloSai5NTd3UN7\newe7do0jeznXgYGN6jXMkwJM9pFOp+nsXMDOnfcAf2Lv5Vw3cOONN+jslSc1EWUfg4PJO3ZMBxYD\nM4AjaGj4IzfddCPz588rbwWriAJM9rH3YPIcYByNjbNZs2Zl2ZdzrTZqIso+9h1MPptly25WcBVA\n3fQypHoaTB6OFkEXCUgzOUQqlAKsjqVSKW6//XZSqVS5q1Kz8kl602VmW81sbca+sWb2kJm9aGa/\nM7MxQ7z3AjN7KT7u/GJWXJJZtOgK2tqmcOGF19HWNoVFiy4vd5VqUrDc9PECEX8AJhOlfFsNTB5i\nkQhdg5VQKpWirW0Ke0+B6mDdutXqKcyhUnPTnwE85O5vufubwEPAmYVUUorrkUceASaQPQWqt7e3\nfJWqUSFz048nmn49aEu8T8qou7uHK6/8NtE/TeYUqM1MmzatfBWrUaFz0+dNuenD2zPH8L+ALqCD\n6P+8LSxcOE/Nw1gxc9Pj7vt9EC20tzbjeQoYF28fBaRyvOccYEnG8yXAnCE+3yW83t5eHzNmsoPH\nj3U+atRE7+npKXfVKlr8+8wrVrIfwXLTA78jSq89Ju7wmBXvkzLZN2HNAGbbmTFjRhlrVduC5aZ3\n923Ad4h6Ep8CrvWos0PKRAlrSk9TpeqQ5hgeGM1FFAlIcxFFKpQCTCQgBVgN0tJClUMBVmO0tFBl\nUSdHDUmlUrS3d8TZoKZTD0sLlYI6OSTOY3gaO3ceBZwN9KClhcpPWaVqwJ45ho+RmYEXxmlpoTLT\nGawG5FoUD46gsXG2ZmqUmc5gVWxwRsYhhxySMccwOoM1NqaVx7ACKMCqVHd3D52dC2hoiCbwdnae\nR1fXDEaObGFgoJ+uriUKrgqgXsQqNNRC5KtXP8H27ds1x7DItHxRndmTO37vhci3b9/O1KlTy1o3\n2Zs6OaqQFiKvHgqwKqT7uqqHrsGqmO7rKg3dDyYSkKZK1TjNjq9eCrAKp9nx1S1RE9HMLge+Ej+9\nxd1vynr9E0QZp16Jd93l7t/N8TlqIuYw1HiXZseXVlnGwczsJKAT+DvgXeBBM7vP3V/JOvRxd/9M\noeXUs6HGu/r6+hRgVSJJE7EVeMrdd7r7buBx4PM5jiso8kXjXbUgSYA9D3wsXsroA8BZRCsKZOsw\nszVmdr+ZtSUor24MdmoAGu+qcgU3Ed19fbyM0cNEyxutAXZnHbYaaHH3/zOzTwF3Ayfk+jzlpo9k\nT+Lt6lpMf/96jXeVUDFz0xdtHMzM/gXY5O5LhjnmVWCKu7+RtV+dHKhTo1KVbRzMzJrjPycCnwPu\nyHp9XMb2NKKA3iu4ZI9cN07qlv/qlnQ2/a/N7HBgAFjg7m+b2Xyi1ShuBr5gZl+NX98BzElYXs1K\np9Ns27aNnTtfIfPGSXVqVDdNlaoAmdddO3b8D+67aWo6Pr5xcjFz5+r/pXLSXMQqNtR11913d9Pe\n3q5rrwqguYhVbKjrrrFjxyq4aoACrMw0mFzbFGBlppsna5uuwSqEbp6sXOrkEAlInRxVQDdN1icF\nWAnopsn6pSZiYOl0mokTT+Cdd34KzAJe1/zCKqMmYgVbuvQW3nlnF/BD4EQgpfmFdURnsIByzdKA\n6Ywa5Wzc+JLOYFVCqbMrVK5b/uFwvvWtixVcdUJNxIByzdJoatrG/PnzylgrKSUFWJFldsdrlobo\nGqyIurt7uPjiSxkxYhy7d2/l1luXMHfuHM3SqHKayVEB0uk048cfy8DAwcCHgVcZOXKALVteUVBV\nOXXTV4Dly5czMLALuIso188KBgbeY82aNWWumZSTAqwIurt7OP/8+cBE4Gygh6jH8ENlrZeUn5qI\nCeUe65oB/JqGhs+yefMGNRGrXDmzSl1uZs/Fj8uGOOYmM9tgZs+Y2SlJyqtEue5IhiNobJzNbbct\nVXDVuaC56eNko8e6+/FmdiqwBOhIWOeKsvdYV3QGa2xMs2bNSlpbW8tbOSm70LnpZwM/A3D3p4Ax\nmbkSa0Gusa5ly5YouARINlXqeeC7ZjYW2EmUm35V1jHjgU0Zz7fE+7YmKLfizJ07h5kzP6mxLtlH\n6Nz0eav23PTNzc0KrBpRNbnpzWwJsNzde+Ln64FPuPvWrPdWbS+i1L6KzU0P3AucHx/TAbyZHVzV\nRLf9y4FKOtD8azN7nmiZ2Pdz05vZJQDu/gDwqpm9DCwFFiQsr2x0278UQgPNedCyQvVNcxED07JC\nUigFWB6U3loKpQAbhtZKlqSUk2MIWitZikGdHDmoU0MyqZOjyNSpIcWiAMtBnRpSLAqwHJQNSopF\n12DDUDYoAWWVSkyBJMNRJ0cCmmMoIdX1GUzd8ZIPncEKkE6neeCBBzj44PGoO15CqcsAG2wWLlp0\nI3/5y8vA9+NX1B0vxVV3TcTceQw7OOSQY9m9+zW6uhYzd+6cktRFqoPWBzsAudbsOvTQj/DjH1/B\nWWedpWsvKaq6aiKm02m2bdu2zyyNd9/dqOCSIOrmDHbDDT/im9/8JxoaJvLuu7toaPg4o0b9NQMD\n/ZqlIcHUxTVYZ+d8br3158AEojSNn2PUqAe4554e2tvbFVwyrHJmlfqamT1vZmvN7D/MrCHr9QvM\n7I9m9nT8uDhJeYVIpVJxcK0EXoz//A0jRhzF2LFjFVwSVMEBZmZHA4uAye5+MlFz85wch97p7pPj\nx62Flleo3t5eojNX5uIMx7Br1yZ1x0twSTs5RgB/ZWYHAx8AXstxTEGn1mJIp9M0NTURNQv3dGrA\nZq677p919pLgCg4wd38N+CGwkSjn/Jvu/kiOQz8fL130SzM7ptDyDtTgYPIll1zPQQc50aIuJwAd\nXHzxeXz96/9QqqpIHUuyfNFhRKuntABvAb8ys3PdPTO7773AHe4+ECcjvR04PdfnFTM3fTqdprNz\nATt2LI/Hu9bS2PgxfvCDRcycOVMrn8iwKiI3vZl9ATjD3efFz78MnOruC4c4/iDgDXc/LMdrRe1F\nXLVqFbNmXcpbb61+f9/o0ZN55JGlTJ06tWjlSH0oVy/iRqDDzEaZmRGdmVJZFTsq4+lsYF2C8vKm\nW/6lUiS5BusFfkW0bNGz8e6bzexaM/t0/PyyuBt/DbAQuDBJZfOlW/6lUtT0QLPuVJZiUMoAkYB0\nw6VIhVKAiQSkABMJSAEmEpACTCQgBZhIQAowkYAUYCIBKcBEAlKAiQSkABMJSAEmEpACTCQgBZhI\nQAowkYAUYCIBKcBEAlKAiQQUOjd9g5ndaWYbzOy/zWxisuoWR7Fy3qm8+igvidC56TuJciEeD/w7\ne9ZqLata/0GovMoROjf9bKJsvhCleMuZ1VekVoXOTT+eaOUF3H038KaZHV5omSJVx90LegCHAb8H\nDic6k/0GODfrmOeAozOevwwcnuOzXA89KvlRaJwkWUJ2JvCKu78BYGZ3AacBmYs/bCZanOs1MxsB\njB48PlOhOedEKl3Q3PTAb4EL4u0vAo8mKE+k6iTK7GtmVxP1HA4ATwPzgG8Dq9z9PjNrBH4OtAN/\nBs5x976klRapFhWROlukVpV0JoeZnWlm683sJTO7KsfrRR2YzqO8r5nZC/EKnA+b2YSQ5WUcd7aZ\nvWdmk0OXZ2Zfir/jc2b2i5DlmdkEM3s0XvD+GTP7VIKyusxsq5mtHeaYm+LfyjNmdkqhZeVTnpmd\na2bPxo8nzOxv8/rgQntHCuh1PIioF7EFGAk8A5yYdcxXgcXx9hyiBdRDlvcJYFS8fWno8uLjDgEe\nA54kGqQP+f2OA1YTdS4BHBm4vKXA/Hi7FXg1QXkfBU4B1g7x+qeA++PtU4GVCX+f+yuvAxgTb5+Z\nb3mlPINNAza4e7+7DwB3Eg1EZyrmwPR+y3P3x9z9nfjpSqJxu2Dlxb4DfA/YmaCsfMubB/zU3d8G\ncPc/BS7vPWB0vH0Y0fhoQdz9CWDbMIfMBn4WH/sUMMbMxoUqz91Xuvtb8dO8fyulDLD3B51jm9m3\nksUcmM6nvEydwIMFlpVXeWbWDhzj7knKybs8olXfPxI3aZ40szMCl3ct8GUz2wTcRzSVLpTs+mzJ\nUZ9QvkKev5Uk42ClUJLxMTM7D5hC1GQMVYYBP2LPsAWE/34HEzUTPw5MBB43s78ZPKMFMBdY5u7/\nZmYdwC+AkwKVVRZmNgO4iKhJuV+lPINtIfpHHnQM+zYhBgemGW5guojlYWYzgX8E/j5u+hRqf+Ud\nSvRjW2FmrxK16e9J0NGR79/nve7+nkfDIy8BxwcsrxP4JURNKmCUmR1ZYHn51CezUyrnv28xmdnJ\nwM3AZ9x9uObrHkkuDA/wInIEey6SG4gukluzjlnAnk6Oc0jW6ZBPee3xMceW4vtlHb8caA/8/c4A\nbou3jwT6gbEBy7sfuCDebgU2J/w7nQQ8N8RrZ7Gnk6ODhJ0ceZQ3EdgAdBzQZyat1AF+gTOBF+OK\nfiPedy3w6Xi7keh/wA1EF5KTApf3MPA60SD5GuDukOVlHfsoCXoR8y2PaEL2C0QL1X8x8N9nK/BE\nHHxPA6cnKOsOorszdhLNGroImA9cknHMT+Kgf7YIf5fDlgfcQjRZYvC30pvP52qgWSQgpQwQCUgB\nJhKQAkwkIAWYSEAKMJGAFGAiASnARAL6fwYejWFDRfGKAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f56ecc14b50>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.scatter(x,y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def sse(y,y_pred): return ((y-y_pred)**2).sum()\n",
    "def loss(y,a,b,x): return sse(y, lin(a,b,x))\n",
    "def avg_loss(y,a,b,x): return np.sqrt(loss(y,a,b,x)/n)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "9.1365"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a_guess=-1.\n",
    "b_guess=1.\n",
    "avg_loss(y, a_guess, b_guess, x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "lr=0.01\n",
    "# d[(y-(a*x+b))**2,b] = 2 (b + a x - y)      = 2 (y_pred - y)\n",
    "# d[(y-(a*x+b))**2,a] = 2 x (b + a x - y)    = x * dy/db"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def upd():\n",
    "    global a_guess, b_guess\n",
    "    y_pred = lin(a_guess, b_guess, x)\n",
    "    dydb = 2 * (y_pred - y)\n",
    "    dyda = x*dydb\n",
    "    a_guess -= lr*dyda.mean()\n",
    "    b_guess -= lr*dydb.mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "fig = plt.figure(dpi=100, figsize=(5, 4))\n",
    "plt.scatter(x,y)\n",
    "line, = plt.plot(x,lin(a_guess,b_guess,x))\n",
    "plt.close()\n",
    "\n",
    "def animate(i):\n",
    "    line.set_ydata(lin(a_guess,b_guess,x))\n",
    "    for i in range(10): upd()\n",
    "    return line,\n",
    "\n",
    "ani = animation.FuncAnimation(fig, animate, np.arange(0, 40), interval=100)\n",
    "ani"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  },
  "nav_menu": {},
  "toc": {
   "navigate_menu": true,
   "number_sections": true,
   "sideBar": true,
   "threshold": 6,
   "toc_cell": true,
   "toc_section_display": "block",
   "toc_window_display": false
  },
  "widgets": {
   "state": {},
   "version": "1.1.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}