{
 "metadata": {
  "name": "COURSE_Andrew_Ng_1"
 },
 "nbformat": 3,
 "nbformat_minor": 0,
 "worksheets": [
  {
   "cells": [
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "## Linear Regression With Theano\n",
      "import theano\n",
      "from theano import tensor as T\n",
      "import numpy as np\n",
      "import matplotlib.pylab as plt\n",
      "from scipy.optimize import fmin_cg\n",
      "\n",
      "## data\n",
      "n_samples, n_feats = 50, 3\n",
      "X = np.random.uniform(low = 0.0, high = 10.0, size = (n_samples, n_feats))\n",
      "y = X[:, 0]* 2 + X[:, 1]*(-.15) + 11. + np.random.random(size = n_samples)\n",
      "\n",
      "## equations\n",
      "var_X = theano.shared(value = X, name = 'X', borrow = True)\n",
      "var_y = theano.shared(value = y, name = 'y', borrow = True)\n",
      "var_theta = theano.shared(value = np.zeros(shape = (n_feats+1, ), dtype = theano.config.floatX), \n",
      "            name = 'theta', borrow = True)\n",
      "var_W = var_theta[:n_feats].reshape((n_feats, ))\n",
      "var_b = var_theta[n_feats:].reshape((1, 1))\n",
      "var_pred = T.dot(var_X, var_W) + var_b\n",
      "var_negative_ll = T.mean((var_pred - var_y) ** 2) \n",
      "var_grad_negative_ll = T.grad(var_negative_ll, var_theta)\n",
      "\n",
      "## optimization with scipy optimization\n",
      "def f(theta_value):\n",
      "    var_theta.set_value(theta_value, borrow = True)\n",
      "    return var_negative_ll.eval()\n",
      "def fprime(theta_value):\n",
      "    var_theta.set_value(theta_value, borrow = True)\n",
      "    return var_grad_negative_ll.eval()\n",
      "def callback(theta_value):\n",
      "    var_theta.set_value(theta_value, borrow = True)\n",
      "    print 'current error', var_negative_ll.eval()\n",
      "\n",
      "theta0 = np.random.random(size = ((n_feats+1, )))\n",
      "fmin_cg(f = f, x0 = theta0, fprime = fprime, callback=callback)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "current error 40.2442576854\n",
        "current error 5.92511275102\n",
        "current error 5.76317650001\n",
        "current error 0.97021338516\n",
        "current error 0.94558943818\n",
        "current error 0.270532848531\n",
        "current error 0.185224566318\n",
        "current error 0.165724807222\n",
        "current error 0.159587591232\n",
        "current error 0.155863769982\n",
        "current error 0.146783303033\n",
        "current error 0.0873606708328\n",
        "current error 0.0866631777669\n",
        "current error 0.0864241409458\n",
        "current error 0.0861089020591\n",
        "current error 0.0858309320522\n",
        "current error 0.0856858008653\n",
        "current error 0.0856623119626\n",
        "current error 0.0855841821725\n",
        "current error 0.0820605716395\n",
        "current error 0.0816609707845\n",
        "Optimization terminated successfully.\n",
        "         Current function value: 0.081661\n",
        "         Iterations: 21\n",
        "         Function evaluations: 45\n",
        "         Gradient evaluations: 45\n"
       ]
      },
      {
       "output_type": "pyout",
       "prompt_number": 28,
       "text": [
        "array([  2.01504435,  -0.1444453 ,   0.01872027,  11.27019332])"
       ]
      }
     ],
     "prompt_number": 28
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [],
     "language": "python",
     "metadata": {},
     "outputs": []
    }
   ],
   "metadata": {}
  }
 ]
}