{
 "metadata": {
  "name": ""
 },
 "nbformat": 3,
 "nbformat_minor": 0,
 "worksheets": [
  {
   "cells": [
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# load the ex3data1\n",
      "# it's in mat format in the course, but can be convert to txt in octave: \n",
      "# save('ex3data1.txt', 'X', '-ascii')\n",
      "# save('ex3data1.y.txt', 'y', '-ascii')\n",
      "\n",
      "import numpy as np\n",
      "\n",
      "def load_dataset():\n",
      "    data = []\n",
      "    y = []    \n",
      "    for line in open('data/ex3data1.txt'):\n",
      "        data.append(map(double, line.split()))\n",
      "    for line in open('data/ex3data1.y.txt'):\n",
      "        y.append(int(float(line.strip())))\n",
      "        \n",
      "    return np.array(data, np.double), np.array(y, np.int)\n",
      "    \n",
      "X, y = load_dataset()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 115
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "%%time\n",
      "\n",
      "from sklearn.base import BaseEstimator\n",
      "from lbfgs import LBFGS\n",
      "import lbfgs\n",
      "\n",
      "def sigmoid(X, theta):\n",
      "    return 1 / (1 + np.exp(-np.dot(X, theta)))\n",
      "\n",
      "def f(theta, g, lr, X, y):\n",
      "    lr.theta[:] = theta\n",
      "    m = X.shape[0]\n",
      "    predicted = sigmoid(X, theta)\n",
      "    g[:] = gradient(theta, X, y)\n",
      "    return -np.sum(1.0* y * log(predicted) + (1 - y) * log(1 - predicted)) / m\n",
      "\n",
      "def gradient(theta, X, y):\n",
      "    m = X.shape[0]\n",
      "    predicted = sigmoid(X, theta)\n",
      "    error = predicted - y\n",
      "    return np.dot(X.T, error) / m                \n",
      "\n",
      "class LogisticRegression(BaseEstimator):\n",
      "    \n",
      "    def fit(self, X, y):\n",
      "        n = X.shape[1]\n",
      "        x0 = np.zeros(n)\n",
      "        self.theta = np.zeros(n)\n",
      "        opt = LBFGS()\n",
      "        try:\n",
      "            opt.minimize(f, x0, args=[self, X, y])\n",
      "        except Exception, e:\n",
      "            print repr(e)\n",
      "        return self\n",
      "        \n",
      "    def predict_proba(self, X):\n",
      "        positive = sigmoid(X, self.theta).reshape(X.shape[0], 1)\n",
      "        return np.hstack((1-positive, positive))\n",
      "    \n",
      "lr = LogisticRegression()\n",
      "lr.fit(X, y == 10)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "LBFGSError('The line-search routine reaches the maximum number of evaluations.',)\n",
        "CPU times: user 1.19 s, sys: 27.6 ms, total: 1.22 s\n",
        "Wall time: 438 ms\n"
       ]
      }
     ],
     "prompt_number": 116
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "%%time\n",
      "from sklearn.cross_validation import train_test_split\n",
      "from sklearn.multiclass import OneVsRestClassifier\n",
      "ovr = OneVsRestClassifier(LogisticRegression())\n",
      "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.4)\n",
      "ovr.fit(X_train, y_train)\n",
      "print ovr.estimators_"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "LBFGSError('The line-search routine reaches the maximum number of evaluations.',)\n",
        "LBFGSError('The line-search routine reaches the maximum number of evaluations.',)"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "LBFGSError('The line-search routine reaches the maximum number of evaluations.',)"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "LBFGSError('The line-search routine reaches the maximum number of evaluations.',)"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "LBFGSError('A rounding error occurred; alternatively, no line-search step satisfies the sufficient decrease and curvature conditions.',)"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "LBFGSError('The line-search routine reaches the maximum number of evaluations.',)"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "LBFGSError('The line-search routine reaches the maximum number of evaluations.',)"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "LBFGSError('The line-search routine reaches the maximum number of evaluations.',)"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "[LogisticRegression(), LogisticRegression(), LogisticRegression(), LogisticRegression(), LogisticRegression(), LogisticRegression(), LogisticRegression(), LogisticRegression(), LogisticRegression(), LogisticRegression()]\n",
        "CPU times: user 12 s, sys: 400 ms, total: 12.4 s\n",
        "Wall time: 4.14 s\n"
       ]
      }
     ],
     "prompt_number": 117
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "from sklearn.metrics import confusion_matrix, accuracy_score, recall_score, precision_score\n",
      "predicted = ovr.predict(X_test)\n",
      "print 'accuracy', accuracy_score(predicted, y_test)\n",
      "print confusion_matrix(predicted, y_test)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "accuracy 0.8645\n",
        "[[186   3   1   1   0   1   1   2   0   0]\n",
        " [  2 164  13   4   3   3   4   5   1   0]\n",
        " [  0   2 167   1  10   0   8   3   2   4]\n",
        " [  0   1   2 162   0   1   1   4  12   0]\n",
        " [  0   1  14   0 165   2   0   6   2   1]\n",
        " [  0   2   0   3   2 195   1   2   0   1]\n",
        " [  1   3   4   2   0   0 163   1  15   0]\n",
        " [  6  11  10   8  24   3   3 157   2   1]\n",
        " [  2   4   3  10   6   0   6   4 174   2]\n",
        " [  0   3   0   2   0   0   1   1   1 196]]\n"
       ]
      }
     ],
     "prompt_number": 118
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [],
     "language": "python",
     "metadata": {},
     "outputs": []
    }
   ],
   "metadata": {}
  }
 ]
}