{
 "metadata": {
  "name": "",
  "signature": "sha256:4a195ae44b4a73879282646f8181c8e44c202ec3d98599c5dbd34984a45547a7"
 },
 "nbformat": 3,
 "nbformat_minor": 0,
 "worksheets": [
  {
   "cells": [
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "# Education Data Mining Testing System\n",
      "\n",
      "We need an in-house testing system to validate our machine learning algorithm. We need this in order to iterate towards better solutions. I am basing this in-house testing system on the Yu et al. JMLR Workshop and Conference Proceedings paper that the winning team submitted. The [leaderboard](https://pslcdatashop.web.cmu.edu/KDDCup/results_full.jsp) contains the full list of submissions and links to papers.\n",
      "\n",
      "In the Yu et al. paper, the main reason why they built their own testing system instead of just submitting to their answers and having the KDD Cup server score it was to avoid overfitting the solution."
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "%matplotlib inline\n",
      "import pandas as pd\n",
      "import numpy as np\n",
      "import matplotlib.pyplot as plt\n",
      "import sklearn\n",
      "from sklearn.cross_validation import cross_val_score"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 137
    },
    {
     "cell_type": "heading",
     "level": 1,
     "metadata": {},
     "source": [
      "Feature engineering"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# Get the data: Algebra 2005-2006 (A56) and/or Algebra 2008-2009 (A89)\n",
      "a56_train_filepath = 'data/algebra0506/algebra_2005_2006_train.txt'\n",
      "#a89_train_filepath = 'data/algebra0809/algebra_2008_2009_train.txt'\n",
      "\n",
      "a56data = pd.read_table(a56_train_filepath)\n",
      "#a89data = pd.read_table(a89_train_filepath)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 104
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "hierarchy = a56data['Problem Hierarchy']"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 105
    },
    {
     "cell_type": "heading",
     "level": 2,
     "metadata": {},
     "source": [
      "Split the problem hierarchy into 'Units' and 'Sections'"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "units, sections = [], []\n",
      "for i in range(len(hierarchy)):\n",
      "    units.append(hierarchy[i].split(',')[0].strip())\n",
      "    sections.append(hierarchy[i].split(',')[1].strip())"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 106
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# Now add 'Units' and 'Sections' as columns within the dataframe\n",
      "a56data['Problem Unit'] = pd.Series(units, index=a56data.index)\n",
      "a56data['Problem Section'] = pd.Series(sections, index=a56data.index)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 107
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# Rearrange order of columns\n",
      "cols = a56data.columns.tolist()\n",
      "cols = cols[0:3]+cols[-2::]+cols[3:-2]\n",
      "a56data = a56data[cols]"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 108
    },
    {
     "cell_type": "heading",
     "level": 2,
     "metadata": {},
     "source": [
      "Create a temporary dataframe for the addition of new binary features"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "df = a56data"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 109
    },
    {
     "cell_type": "heading",
     "level": 2,
     "metadata": {},
     "source": [
      "Map string category values to integers"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "cats = ['Anon Student Id', 'Problem Hierarchy', 'Problem Unit', 'Problem Section', 'Problem Name']\n"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 110
    },
    {
     "cell_type": "heading",
     "level": 3,
     "metadata": {},
     "source": [
      "Student IDs"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "sids = list(set(df['Anon Student Id']))\n",
      "sid_dict = {}\n",
      "for idx,sid in enumerate(sids):\n",
      "    sid_dict[idx] = sid\n",
      "    df.loc[df['Anon Student Id'] == sid,'Anon Student Id'] = idx"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 111
    },
    {
     "cell_type": "heading",
     "level": 3,
     "metadata": {},
     "source": [
      "Problem Hierarchy"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "cat = 'Problem Hierarchy'\n",
      "prhs = list(set(df[cat]))\n",
      "prh_dict = {}\n",
      "for idx,prh in enumerate(prhs):\n",
      "    prh_dict[idx] = prh\n",
      "    df.loc[df[cat] == prh,cat] = idx"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 112
    },
    {
     "cell_type": "heading",
     "level": 3,
     "metadata": {},
     "source": [
      "Problem Unit"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "cat = 'Problem Unit'\n",
      "prus = list(set(df[cat]))\n",
      "pru_dict = {}\n",
      "for idx,pru in enumerate(prus):\n",
      "    pru_dict[idx] = pru\n",
      "    df.loc[df[cat] == pru,cat] = idx"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 113
    },
    {
     "cell_type": "heading",
     "level": 3,
     "metadata": {},
     "source": [
      "Problem Section"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "cat = 'Problem Section'\n",
      "prss = list(set(df[cat]))\n",
      "prs_dict = {}\n",
      "for idx,prs in enumerate(prss):\n",
      "    prs_dict[idx] = prs\n",
      "    df.loc[df[cat] == prs,cat] = idx"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 114
    },
    {
     "cell_type": "heading",
     "level": 3,
     "metadata": {},
     "source": [
      "Problem Name"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "cat = 'Problem Name'\n",
      "prns = list(set(df[cat]))\n",
      "prn_dict = {}\n",
      "for idx,prn in enumerate(prns):\n",
      "    prn_dict[idx] = prn\n",
      "    df.loc[df[cat] == prn,cat] = idx"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 115
    },
    {
     "cell_type": "heading",
     "level": 3,
     "metadata": {},
     "source": [
      "Step Name"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "cat = 'Step Name'\n",
      "stns = list(set(df[cat]))\n",
      "stn_dict = {}\n",
      "for idx,stn in enumerate(stns):\n",
      "    stn_dict[idx] = stn\n",
      "    df.loc[df[cat] == stn,cat] = idx"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "ename": "KeyboardInterrupt",
       "evalue": "",
       "output_type": "pyerr",
       "traceback": [
        "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m\n\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
        "\u001b[0;32m<ipython-input-160-531a7478134b>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0midx\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mstn\u001b[0m \u001b[0;32min\u001b[0m \u001b[0menumerate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstns\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m     \u001b[0mstn_dict\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0midx\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mstn\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 6\u001b[0;31m     \u001b[0mdf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mloc\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mdf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mcat\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0mstn\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mcat\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0midx\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
        "\u001b[0;32m/Users/mikhail/anaconda/lib/python2.7/site-packages/pandas/core/indexing.pyc\u001b[0m in \u001b[0;36m__setitem__\u001b[0;34m(self, key, value)\u001b[0m\n\u001b[1;32m     96\u001b[0m             \u001b[0mindexer\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_convert_to_indexer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mis_setter\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     97\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 98\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_setitem_with_indexer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mindexer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     99\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    100\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_has_valid_type\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mk\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0maxis\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
        "\u001b[0;32m/Users/mikhail/anaconda/lib/python2.7/site-packages/pandas/core/indexing.pyc\u001b[0m in \u001b[0;36m_setitem_with_indexer\u001b[0;34m(self, indexer, value)\u001b[0m\n\u001b[1;32m    404\u001b[0m                 \u001b[0;31m# scalar\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    405\u001b[0m                 \u001b[0;32mfor\u001b[0m \u001b[0mitem\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mlabels\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 406\u001b[0;31m                     \u001b[0msetter\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mitem\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    407\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    408\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
        "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
       ]
      }
     ],
     "prompt_number": 160
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "df"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "html": [
        "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
        "<table border=\"1\" class=\"dataframe\">\n",
        "  <thead>\n",
        "    <tr style=\"text-align: right;\">\n",
        "      <th></th>\n",
        "      <th>Row</th>\n",
        "      <th>Anon Student Id</th>\n",
        "      <th>Problem Hierarchy</th>\n",
        "      <th>Problem Unit</th>\n",
        "      <th>Problem Section</th>\n",
        "      <th>Problem Name</th>\n",
        "      <th>Problem View</th>\n",
        "      <th>Step Name</th>\n",
        "      <th>Step Start Time</th>\n",
        "      <th>First Transaction Time</th>\n",
        "      <th>Correct Transaction Time</th>\n",
        "      <th>Step End Time</th>\n",
        "      <th>Step Duration (sec)</th>\n",
        "      <th>Correct Step Duration (sec)</th>\n",
        "      <th>Error Step Duration (sec)</th>\n",
        "      <th>Correct First Attempt</th>\n",
        "      <th>Incorrects</th>\n",
        "      <th>Hints</th>\n",
        "      <th>Corrects</th>\n",
        "      <th>KC(Default)</th>\n",
        "      <th></th>\n",
        "    </tr>\n",
        "  </thead>\n",
        "  <tbody>\n",
        "    <tr>\n",
        "      <th>0 </th>\n",
        "      <td>  1</td>\n",
        "      <td> 104</td>\n",
        "      <td> 87</td>\n",
        "      <td> 3</td>\n",
        "      <td> 17</td>\n",
        "      <td> 1046</td>\n",
        "      <td> 1</td>\n",
        "      <td>          3(x+2) = 15</td>\n",
        "      <td> 2005-09-09 12:24:35.0</td>\n",
        "      <td> 2005-09-09 12:24:49.0</td>\n",
        "      <td> 2005-09-09 12:25:15.0</td>\n",
        "      <td> 2005-09-09 12:25:15.0</td>\n",
        "      <td>  40</td>\n",
        "      <td> NaN</td>\n",
        "      <td>  40</td>\n",
        "      <td> 0</td>\n",
        "      <td> 2</td>\n",
        "      <td> 3</td>\n",
        "      <td> 1</td>\n",
        "      <td> [SkillRule: Eliminate Parens; {CLT nested; CLT...</td>\n",
        "      <td>...</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>1 </th>\n",
        "      <td>  2</td>\n",
        "      <td> 104</td>\n",
        "      <td> 87</td>\n",
        "      <td> 3</td>\n",
        "      <td> 17</td>\n",
        "      <td> 1046</td>\n",
        "      <td> 1</td>\n",
        "      <td>              x+2 = 5</td>\n",
        "      <td> 2005-09-09 12:25:15.0</td>\n",
        "      <td> 2005-09-09 12:25:31.0</td>\n",
        "      <td> 2005-09-09 12:25:31.0</td>\n",
        "      <td> 2005-09-09 12:25:31.0</td>\n",
        "      <td>  16</td>\n",
        "      <td>  16</td>\n",
        "      <td> NaN</td>\n",
        "      <td> 1</td>\n",
        "      <td> 0</td>\n",
        "      <td> 0</td>\n",
        "      <td> 1</td>\n",
        "      <td> [SkillRule: Remove constant; {ax+b=c, positive...</td>\n",
        "      <td>...</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>2 </th>\n",
        "      <td>  3</td>\n",
        "      <td> 104</td>\n",
        "      <td> 87</td>\n",
        "      <td> 3</td>\n",
        "      <td> 17</td>\n",
        "      <td>  346</td>\n",
        "      <td> 1</td>\n",
        "      <td>            2-8y = -4</td>\n",
        "      <td> 2005-09-09 12:25:36.0</td>\n",
        "      <td> 2005-09-09 12:25:43.0</td>\n",
        "      <td> 2005-09-09 12:26:12.0</td>\n",
        "      <td> 2005-09-09 12:26:12.0</td>\n",
        "      <td>  36</td>\n",
        "      <td> NaN</td>\n",
        "      <td>  36</td>\n",
        "      <td> 0</td>\n",
        "      <td> 2</td>\n",
        "      <td> 3</td>\n",
        "      <td> 1</td>\n",
        "      <td> [SkillRule: Remove constant; {ax+b=c, positive...</td>\n",
        "      <td>...</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>3 </th>\n",
        "      <td>  4</td>\n",
        "      <td> 104</td>\n",
        "      <td> 87</td>\n",
        "      <td> 3</td>\n",
        "      <td> 17</td>\n",
        "      <td>  346</td>\n",
        "      <td> 1</td>\n",
        "      <td>             -8y = -6</td>\n",
        "      <td> 2005-09-09 12:26:12.0</td>\n",
        "      <td> 2005-09-09 12:26:34.0</td>\n",
        "      <td> 2005-09-09 12:26:34.0</td>\n",
        "      <td> 2005-09-09 12:26:34.0</td>\n",
        "      <td>  22</td>\n",
        "      <td>  22</td>\n",
        "      <td> NaN</td>\n",
        "      <td> 1</td>\n",
        "      <td> 0</td>\n",
        "      <td> 0</td>\n",
        "      <td> 1</td>\n",
        "      <td> [SkillRule: Remove coefficient; {ax+b=c, divid...</td>\n",
        "      <td>...</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>4 </th>\n",
        "      <td>  5</td>\n",
        "      <td> 104</td>\n",
        "      <td> 87</td>\n",
        "      <td> 3</td>\n",
        "      <td> 17</td>\n",
        "      <td>  346</td>\n",
        "      <td> 2</td>\n",
        "      <td>           -7y-5 = -4</td>\n",
        "      <td> 2005-09-09 12:26:38.0</td>\n",
        "      <td> 2005-09-09 12:28:36.0</td>\n",
        "      <td> 2005-09-09 12:28:36.0</td>\n",
        "      <td> 2005-09-09 12:28:36.0</td>\n",
        "      <td> 118</td>\n",
        "      <td> 118</td>\n",
        "      <td> NaN</td>\n",
        "      <td> 1</td>\n",
        "      <td> 0</td>\n",
        "      <td> 0</td>\n",
        "      <td> 1</td>\n",
        "      <td> [SkillRule: Remove constant; {ax+b=c, positive...</td>\n",
        "      <td>...</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>5 </th>\n",
        "      <td>  6</td>\n",
        "      <td> 104</td>\n",
        "      <td> 87</td>\n",
        "      <td> 3</td>\n",
        "      <td> 17</td>\n",
        "      <td>  346</td>\n",
        "      <td> 2</td>\n",
        "      <td>              -7y = 1</td>\n",
        "      <td> 2005-09-09 12:28:36.0</td>\n",
        "      <td> 2005-09-09 12:28:43.0</td>\n",
        "      <td> 2005-09-09 12:28:51.0</td>\n",
        "      <td> 2005-09-09 12:28:51.0</td>\n",
        "      <td>  15</td>\n",
        "      <td> NaN</td>\n",
        "      <td>  15</td>\n",
        "      <td> 0</td>\n",
        "      <td> 1</td>\n",
        "      <td> 0</td>\n",
        "      <td> 1</td>\n",
        "      <td> [SkillRule: Remove coefficient; {ax+b=c, divid...</td>\n",
        "      <td>...</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>6 </th>\n",
        "      <td>  7</td>\n",
        "      <td> 104</td>\n",
        "      <td> 87</td>\n",
        "      <td> 3</td>\n",
        "      <td> 17</td>\n",
        "      <td>  346</td>\n",
        "      <td> 3</td>\n",
        "      <td>             7y+4 = 7</td>\n",
        "      <td> 2005-09-09 12:28:57.0</td>\n",
        "      <td> 2005-09-09 12:29:09.0</td>\n",
        "      <td> 2005-09-09 12:29:09.0</td>\n",
        "      <td> 2005-09-09 12:29:09.0</td>\n",
        "      <td>  12</td>\n",
        "      <td>  12</td>\n",
        "      <td> NaN</td>\n",
        "      <td> 1</td>\n",
        "      <td> 0</td>\n",
        "      <td> 0</td>\n",
        "      <td> 1</td>\n",
        "      <td> [SkillRule: Remove constant; {ax+b=c, positive...</td>\n",
        "      <td>...</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>7 </th>\n",
        "      <td>  8</td>\n",
        "      <td> 104</td>\n",
        "      <td> 87</td>\n",
        "      <td> 3</td>\n",
        "      <td> 17</td>\n",
        "      <td>  346</td>\n",
        "      <td> 3</td>\n",
        "      <td>               7y = 3</td>\n",
        "      <td> 2005-09-09 12:29:09.0</td>\n",
        "      <td> 2005-09-09 12:29:14.0</td>\n",
        "      <td> 2005-09-09 12:29:14.0</td>\n",
        "      <td> 2005-09-09 12:29:14.0</td>\n",
        "      <td>   5</td>\n",
        "      <td>   5</td>\n",
        "      <td> NaN</td>\n",
        "      <td> 1</td>\n",
        "      <td> 0</td>\n",
        "      <td> 0</td>\n",
        "      <td> 1</td>\n",
        "      <td> [SkillRule: Remove positive coefficient; {ax/b...</td>\n",
        "      <td>...</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>8 </th>\n",
        "      <td>  9</td>\n",
        "      <td> 104</td>\n",
        "      <td> 87</td>\n",
        "      <td> 3</td>\n",
        "      <td> 17</td>\n",
        "      <td>  346</td>\n",
        "      <td> 4</td>\n",
        "      <td>           -5+9y = -6</td>\n",
        "      <td> 2005-09-09 12:29:19.0</td>\n",
        "      <td> 2005-09-09 12:29:31.0</td>\n",
        "      <td> 2005-09-09 12:29:31.0</td>\n",
        "      <td> 2005-09-09 12:29:31.0</td>\n",
        "      <td>  12</td>\n",
        "      <td>  12</td>\n",
        "      <td> NaN</td>\n",
        "      <td> 1</td>\n",
        "      <td> 0</td>\n",
        "      <td> 0</td>\n",
        "      <td> 1</td>\n",
        "      <td> [SkillRule: Remove constant; {ax+b=c, positive...</td>\n",
        "      <td>...</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>9 </th>\n",
        "      <td> 10</td>\n",
        "      <td> 104</td>\n",
        "      <td> 87</td>\n",
        "      <td> 3</td>\n",
        "      <td> 17</td>\n",
        "      <td>  346</td>\n",
        "      <td> 4</td>\n",
        "      <td>              9y = -1</td>\n",
        "      <td> 2005-09-09 12:29:31.0</td>\n",
        "      <td> 2005-09-09 12:29:36.0</td>\n",
        "      <td> 2005-09-09 12:29:36.0</td>\n",
        "      <td> 2005-09-09 12:29:36.0</td>\n",
        "      <td>   5</td>\n",
        "      <td>   5</td>\n",
        "      <td> NaN</td>\n",
        "      <td> 1</td>\n",
        "      <td> 0</td>\n",
        "      <td> 0</td>\n",
        "      <td> 1</td>\n",
        "      <td> [SkillRule: Remove positive coefficient; {ax/b...</td>\n",
        "      <td>...</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>10</th>\n",
        "      <td> 11</td>\n",
        "      <td> 104</td>\n",
        "      <td> 90</td>\n",
        "      <td> 3</td>\n",
        "      <td> 15</td>\n",
        "      <td>  346</td>\n",
        "      <td> 1</td>\n",
        "      <td>           -7-3x = -2</td>\n",
        "      <td> 2005-09-09 12:29:41.0</td>\n",
        "      <td> 2005-09-09 12:30:27.0</td>\n",
        "      <td> 2005-09-09 12:30:27.0</td>\n",
        "      <td> 2005-09-09 12:30:27.0</td>\n",
        "      <td>  46</td>\n",
        "      <td>  46</td>\n",
        "      <td> NaN</td>\n",
        "      <td> 1</td>\n",
        "      <td> 0</td>\n",
        "      <td> 0</td>\n",
        "      <td> 1</td>\n",
        "      <td> [SkillRule: Remove constant; {ax+b=c, positive...</td>\n",
        "      <td>...</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>11</th>\n",
        "      <td> 12</td>\n",
        "      <td> 104</td>\n",
        "      <td> 90</td>\n",
        "      <td> 3</td>\n",
        "      <td> 15</td>\n",
        "      <td>  346</td>\n",
        "      <td> 1</td>\n",
        "      <td>       -7-3x+7 = -2+7</td>\n",
        "      <td> 2005-09-09 12:30:27.0</td>\n",
        "      <td> 2005-09-09 12:30:34.0</td>\n",
        "      <td> 2005-09-09 12:30:45.0</td>\n",
        "      <td> 2005-09-09 12:30:49.0</td>\n",
        "      <td>  22</td>\n",
        "      <td> NaN</td>\n",
        "      <td>  22</td>\n",
        "      <td> 0</td>\n",
        "      <td> 1</td>\n",
        "      <td> 0</td>\n",
        "      <td> 2</td>\n",
        "      <td> [SkillRule: Add/Subtract; [Typein Skill: {Isol...</td>\n",
        "      <td>...</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>12</th>\n",
        "      <td> 13</td>\n",
        "      <td> 104</td>\n",
        "      <td> 90</td>\n",
        "      <td> 3</td>\n",
        "      <td> 15</td>\n",
        "      <td>  346</td>\n",
        "      <td> 1</td>\n",
        "      <td>              -3x = 5</td>\n",
        "      <td> 2005-09-09 12:30:49.0</td>\n",
        "      <td> 2005-09-09 12:31:04.0</td>\n",
        "      <td> 2005-09-09 12:31:04.0</td>\n",
        "      <td> 2005-09-09 12:31:04.0</td>\n",
        "      <td>  15</td>\n",
        "      <td>  15</td>\n",
        "      <td> NaN</td>\n",
        "      <td> 1</td>\n",
        "      <td> 0</td>\n",
        "      <td> 0</td>\n",
        "      <td> 1</td>\n",
        "      <td> [SkillRule: Remove coefficient; {ax+b=c, divid...</td>\n",
        "      <td>...</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>13</th>\n",
        "      <td> 14</td>\n",
        "      <td> 104</td>\n",
        "      <td> 90</td>\n",
        "      <td> 3</td>\n",
        "      <td> 15</td>\n",
        "      <td>  346</td>\n",
        "      <td> 1</td>\n",
        "      <td>        -3x/-3 = 5/-3</td>\n",
        "      <td> 2005-09-09 12:31:04.0</td>\n",
        "      <td> 2005-09-09 12:31:07.0</td>\n",
        "      <td> 2005-09-09 12:31:07.0</td>\n",
        "      <td> 2005-09-09 12:31:12.0</td>\n",
        "      <td>   8</td>\n",
        "      <td>   8</td>\n",
        "      <td> NaN</td>\n",
        "      <td> 1</td>\n",
        "      <td> 0</td>\n",
        "      <td> 0</td>\n",
        "      <td> 2</td>\n",
        "      <td> [SkillRule: Multiply/Divide; [Typein Skill: {R...</td>\n",
        "      <td>...</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>14</th>\n",
        "      <td> 15</td>\n",
        "      <td> 104</td>\n",
        "      <td> 90</td>\n",
        "      <td> 3</td>\n",
        "      <td> 15</td>\n",
        "      <td>  346</td>\n",
        "      <td> 2</td>\n",
        "      <td>            -9 = 8y+9</td>\n",
        "      <td> 2005-09-09 12:31:16.0</td>\n",
        "      <td> 2005-09-09 12:31:29.0</td>\n",
        "      <td> 2005-09-09 12:31:29.0</td>\n",
        "      <td> 2005-09-09 12:31:29.0</td>\n",
        "      <td>  13</td>\n",
        "      <td>  13</td>\n",
        "      <td> NaN</td>\n",
        "      <td> 1</td>\n",
        "      <td> 0</td>\n",
        "      <td> 0</td>\n",
        "      <td> 1</td>\n",
        "      <td> [SkillRule: Remove constant; {ax+b=c, positive...</td>\n",
        "      <td>...</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>15</th>\n",
        "      <td> 16</td>\n",
        "      <td> 104</td>\n",
        "      <td> 90</td>\n",
        "      <td> 3</td>\n",
        "      <td> 15</td>\n",
        "      <td>  346</td>\n",
        "      <td> 2</td>\n",
        "      <td>        -9-9 = 8y+9-9</td>\n",
        "      <td> 2005-09-09 12:31:29.0</td>\n",
        "      <td> 2005-09-09 12:31:32.0</td>\n",
        "      <td> 2005-09-09 12:31:32.0</td>\n",
        "      <td> 2005-09-09 12:31:39.0</td>\n",
        "      <td>  10</td>\n",
        "      <td>  10</td>\n",
        "      <td> NaN</td>\n",
        "      <td> 1</td>\n",
        "      <td> 0</td>\n",
        "      <td> 0</td>\n",
        "      <td> 2</td>\n",
        "      <td> [SkillRule: Add/Subtract; [Typein Skill: {Isol...</td>\n",
        "      <td>...</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>16</th>\n",
        "      <td> 17</td>\n",
        "      <td> 104</td>\n",
        "      <td> 90</td>\n",
        "      <td> 3</td>\n",
        "      <td> 15</td>\n",
        "      <td>  346</td>\n",
        "      <td> 2</td>\n",
        "      <td>             -18 = 8y</td>\n",
        "      <td> 2005-09-09 12:31:39.0</td>\n",
        "      <td> 2005-09-09 12:31:44.0</td>\n",
        "      <td> 2005-09-09 12:31:44.0</td>\n",
        "      <td> 2005-09-09 12:31:44.0</td>\n",
        "      <td>   5</td>\n",
        "      <td>   5</td>\n",
        "      <td> NaN</td>\n",
        "      <td> 1</td>\n",
        "      <td> 0</td>\n",
        "      <td> 0</td>\n",
        "      <td> 1</td>\n",
        "      <td> [SkillRule: Remove positive coefficient; {ax/b...</td>\n",
        "      <td>...</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>17</th>\n",
        "      <td> 18</td>\n",
        "      <td> 104</td>\n",
        "      <td> 90</td>\n",
        "      <td> 3</td>\n",
        "      <td> 15</td>\n",
        "      <td>  346</td>\n",
        "      <td> 2</td>\n",
        "      <td>         -18/8 = 8y/8</td>\n",
        "      <td> 2005-09-09 12:31:44.0</td>\n",
        "      <td> 2005-09-09 12:31:46.0</td>\n",
        "      <td> 2005-09-09 12:31:46.0</td>\n",
        "      <td> 2005-09-09 12:32:56.0</td>\n",
        "      <td>  72</td>\n",
        "      <td>  72</td>\n",
        "      <td> NaN</td>\n",
        "      <td> 1</td>\n",
        "      <td> 2</td>\n",
        "      <td> 0</td>\n",
        "      <td> 2</td>\n",
        "      <td> [SkillRule: Multiply/Divide; [Typein Skill: {R...</td>\n",
        "      <td>...</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>18</th>\n",
        "      <td> 19</td>\n",
        "      <td> 104</td>\n",
        "      <td> 90</td>\n",
        "      <td> 3</td>\n",
        "      <td> 15</td>\n",
        "      <td>  346</td>\n",
        "      <td> 3</td>\n",
        "      <td>            -2-2x = 9</td>\n",
        "      <td> 2005-09-09 12:33:01.0</td>\n",
        "      <td> 2005-09-09 12:33:22.0</td>\n",
        "      <td> 2005-09-09 12:33:32.0</td>\n",
        "      <td> 2005-09-09 12:33:32.0</td>\n",
        "      <td>  31</td>\n",
        "      <td> NaN</td>\n",
        "      <td>  31</td>\n",
        "      <td> 0</td>\n",
        "      <td> 1</td>\n",
        "      <td> 0</td>\n",
        "      <td> 1</td>\n",
        "      <td> [SkillRule: Remove constant; {ax+b=c, positive...</td>\n",
        "      <td>...</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>19</th>\n",
        "      <td> 20</td>\n",
        "      <td> 104</td>\n",
        "      <td> 90</td>\n",
        "      <td> 3</td>\n",
        "      <td> 15</td>\n",
        "      <td>  346</td>\n",
        "      <td> 3</td>\n",
        "      <td>        -2-2x+2 = 9+2</td>\n",
        "      <td> 2005-09-09 12:33:32.0</td>\n",
        "      <td> 2005-09-09 12:33:37.0</td>\n",
        "      <td> 2005-09-09 12:33:37.0</td>\n",
        "      <td> 2005-09-09 12:33:40.0</td>\n",
        "      <td>   8</td>\n",
        "      <td>   8</td>\n",
        "      <td> NaN</td>\n",
        "      <td> 1</td>\n",
        "      <td> 0</td>\n",
        "      <td> 0</td>\n",
        "      <td> 2</td>\n",
        "      <td> [SkillRule: Add/Subtract; [Typein Skill: {Isol...</td>\n",
        "      <td>...</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>20</th>\n",
        "      <td> 21</td>\n",
        "      <td> 104</td>\n",
        "      <td> 90</td>\n",
        "      <td> 3</td>\n",
        "      <td> 15</td>\n",
        "      <td>  346</td>\n",
        "      <td> 3</td>\n",
        "      <td>             -2x = 11</td>\n",
        "      <td> 2005-09-09 12:33:40.0</td>\n",
        "      <td> 2005-09-09 12:33:46.0</td>\n",
        "      <td> 2005-09-09 12:33:46.0</td>\n",
        "      <td> 2005-09-09 12:33:46.0</td>\n",
        "      <td>   6</td>\n",
        "      <td>   6</td>\n",
        "      <td> NaN</td>\n",
        "      <td> 1</td>\n",
        "      <td> 0</td>\n",
        "      <td> 0</td>\n",
        "      <td> 1</td>\n",
        "      <td> [SkillRule: Remove coefficient; {ax+b=c, divid...</td>\n",
        "      <td>...</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>21</th>\n",
        "      <td> 22</td>\n",
        "      <td> 104</td>\n",
        "      <td> 90</td>\n",
        "      <td> 3</td>\n",
        "      <td> 15</td>\n",
        "      <td>  346</td>\n",
        "      <td> 3</td>\n",
        "      <td>       -2x/-2 = 11/-2</td>\n",
        "      <td> 2005-09-09 12:33:46.0</td>\n",
        "      <td> 2005-09-09 12:33:51.0</td>\n",
        "      <td> 2005-09-09 12:33:51.0</td>\n",
        "      <td> 2005-09-09 12:33:55.0</td>\n",
        "      <td>   9</td>\n",
        "      <td>   9</td>\n",
        "      <td> NaN</td>\n",
        "      <td> 1</td>\n",
        "      <td> 0</td>\n",
        "      <td> 0</td>\n",
        "      <td> 2</td>\n",
        "      <td> [SkillRule: Multiply/Divide; [Typein Skill: {R...</td>\n",
        "      <td>...</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>22</th>\n",
        "      <td> 23</td>\n",
        "      <td> 104</td>\n",
        "      <td> 90</td>\n",
        "      <td> 3</td>\n",
        "      <td> 15</td>\n",
        "      <td>  346</td>\n",
        "      <td> 4</td>\n",
        "      <td>            4+4y = -6</td>\n",
        "      <td> 2005-09-09 12:33:59.0</td>\n",
        "      <td> 2005-09-09 12:34:06.0</td>\n",
        "      <td> 2005-09-09 12:34:06.0</td>\n",
        "      <td> 2005-09-09 12:34:06.0</td>\n",
        "      <td>   7</td>\n",
        "      <td>   7</td>\n",
        "      <td> NaN</td>\n",
        "      <td> 1</td>\n",
        "      <td> 0</td>\n",
        "      <td> 0</td>\n",
        "      <td> 1</td>\n",
        "      <td> [SkillRule: Remove constant; {ax+b=c, positive...</td>\n",
        "      <td>...</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>23</th>\n",
        "      <td> 24</td>\n",
        "      <td> 104</td>\n",
        "      <td> 90</td>\n",
        "      <td> 3</td>\n",
        "      <td> 15</td>\n",
        "      <td>  346</td>\n",
        "      <td> 4</td>\n",
        "      <td>        4+4y-4 = -6-4</td>\n",
        "      <td> 2005-09-09 12:34:06.0</td>\n",
        "      <td> 2005-09-09 12:34:09.0</td>\n",
        "      <td> 2005-09-09 12:34:09.0</td>\n",
        "      <td> 2005-09-09 12:34:17.0</td>\n",
        "      <td>  11</td>\n",
        "      <td>  11</td>\n",
        "      <td> NaN</td>\n",
        "      <td> 1</td>\n",
        "      <td> 0</td>\n",
        "      <td> 0</td>\n",
        "      <td> 2</td>\n",
        "      <td> [SkillRule: Add/Subtract; [Typein Skill: {Isol...</td>\n",
        "      <td>...</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>24</th>\n",
        "      <td> 25</td>\n",
        "      <td> 104</td>\n",
        "      <td> 90</td>\n",
        "      <td> 3</td>\n",
        "      <td> 15</td>\n",
        "      <td>  346</td>\n",
        "      <td> 4</td>\n",
        "      <td>            4y = -6-4</td>\n",
        "      <td> 2005-09-09 12:34:17.0</td>\n",
        "      <td> 2005-09-09 12:34:25.0</td>\n",
        "      <td> 2005-09-09 12:34:25.0</td>\n",
        "      <td> 2005-09-09 12:34:25.0</td>\n",
        "      <td>   8</td>\n",
        "      <td>   8</td>\n",
        "      <td> NaN</td>\n",
        "      <td> 1</td>\n",
        "      <td> 0</td>\n",
        "      <td> 0</td>\n",
        "      <td> 1</td>\n",
        "      <td>     [SkillRule: Consolidate vars with coeff; CLT]</td>\n",
        "      <td>...</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>25</th>\n",
        "      <td> 26</td>\n",
        "      <td> 104</td>\n",
        "      <td> 90</td>\n",
        "      <td> 3</td>\n",
        "      <td> 15</td>\n",
        "      <td>  346</td>\n",
        "      <td> 4</td>\n",
        "      <td>          FinalAnswer</td>\n",
        "      <td> 2005-09-09 12:34:25.0</td>\n",
        "      <td> 2005-09-09 12:34:29.0</td>\n",
        "      <td> 2005-09-09 12:34:33.0</td>\n",
        "      <td> 2005-09-09 12:34:33.0</td>\n",
        "      <td>   8</td>\n",
        "      <td> NaN</td>\n",
        "      <td>   8</td>\n",
        "      <td> 0</td>\n",
        "      <td> 1</td>\n",
        "      <td> 0</td>\n",
        "      <td> 1</td>\n",
        "      <td>                             combine-like-terms-sp</td>\n",
        "      <td>...</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>26</th>\n",
        "      <td> 27</td>\n",
        "      <td> 104</td>\n",
        "      <td> 90</td>\n",
        "      <td> 3</td>\n",
        "      <td> 15</td>\n",
        "      <td>  346</td>\n",
        "      <td> 4</td>\n",
        "      <td>             4y = -10</td>\n",
        "      <td> 2005-09-09 12:34:33.0</td>\n",
        "      <td> 2005-09-09 12:34:42.0</td>\n",
        "      <td> 2005-09-09 12:34:42.0</td>\n",
        "      <td> 2005-09-09 12:34:42.0</td>\n",
        "      <td>   9</td>\n",
        "      <td>   9</td>\n",
        "      <td> NaN</td>\n",
        "      <td> 1</td>\n",
        "      <td> 0</td>\n",
        "      <td> 0</td>\n",
        "      <td> 1</td>\n",
        "      <td> [SkillRule: Remove positive coefficient; {ax/b...</td>\n",
        "      <td>...</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>27</th>\n",
        "      <td> 28</td>\n",
        "      <td> 104</td>\n",
        "      <td> 90</td>\n",
        "      <td> 3</td>\n",
        "      <td> 15</td>\n",
        "      <td>  346</td>\n",
        "      <td> 4</td>\n",
        "      <td>         4y/4 = -10/4</td>\n",
        "      <td> 2005-09-09 12:34:42.0</td>\n",
        "      <td> 2005-09-09 12:34:46.0</td>\n",
        "      <td> 2005-09-09 12:34:46.0</td>\n",
        "      <td> 2005-09-09 12:35:02.0</td>\n",
        "      <td>  20</td>\n",
        "      <td>  20</td>\n",
        "      <td> NaN</td>\n",
        "      <td> 1</td>\n",
        "      <td> 0</td>\n",
        "      <td> 0</td>\n",
        "      <td> 2</td>\n",
        "      <td> [SkillRule: Multiply/Divide; [Typein Skill: {R...</td>\n",
        "      <td>...</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>28</th>\n",
        "      <td> 29</td>\n",
        "      <td> 104</td>\n",
        "      <td> 89</td>\n",
        "      <td> 3</td>\n",
        "      <td> 16</td>\n",
        "      <td>  668</td>\n",
        "      <td> 1</td>\n",
        "      <td>         -7 = -5(y+7)</td>\n",
        "      <td> 2005-09-09 12:35:08.0</td>\n",
        "      <td> 2005-09-09 12:36:27.0</td>\n",
        "      <td> 2005-09-09 12:37:01.0</td>\n",
        "      <td> 2005-09-09 12:37:01.0</td>\n",
        "      <td> 113</td>\n",
        "      <td> NaN</td>\n",
        "      <td> 113</td>\n",
        "      <td> 0</td>\n",
        "      <td> 2</td>\n",
        "      <td> 3</td>\n",
        "      <td> 1</td>\n",
        "      <td> [SkillRule: Eliminate Parens; {CLT nested; CLT...</td>\n",
        "      <td>...</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>29</th>\n",
        "      <td> 30</td>\n",
        "      <td> 104</td>\n",
        "      <td> 89</td>\n",
        "      <td> 3</td>\n",
        "      <td> 16</td>\n",
        "      <td>  668</td>\n",
        "      <td> 1</td>\n",
        "      <td>            7/5 = y+7</td>\n",
        "      <td> 2005-09-09 12:37:01.0</td>\n",
        "      <td> 2005-09-09 12:37:09.0</td>\n",
        "      <td> 2005-09-09 12:37:09.0</td>\n",
        "      <td> 2005-09-09 12:37:09.0</td>\n",
        "      <td>   8</td>\n",
        "      <td>   8</td>\n",
        "      <td> NaN</td>\n",
        "      <td> 1</td>\n",
        "      <td> 0</td>\n",
        "      <td> 0</td>\n",
        "      <td> 1</td>\n",
        "      <td> [SkillRule: Remove constant; {ax+b=c, positive...</td>\n",
        "      <td>...</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>30</th>\n",
        "      <td> 31</td>\n",
        "      <td> 104</td>\n",
        "      <td> 89</td>\n",
        "      <td> 3</td>\n",
        "      <td> 16</td>\n",
        "      <td>  668</td>\n",
        "      <td> 2</td>\n",
        "      <td>         -7(x+9) = -5</td>\n",
        "      <td> 2005-09-09 12:37:17.0</td>\n",
        "      <td> 2005-09-09 12:38:15.0</td>\n",
        "      <td> 2005-09-09 12:38:15.0</td>\n",
        "      <td> 2005-09-09 12:38:15.0</td>\n",
        "      <td>  58</td>\n",
        "      <td>  58</td>\n",
        "      <td> NaN</td>\n",
        "      <td> 1</td>\n",
        "      <td> 0</td>\n",
        "      <td> 0</td>\n",
        "      <td> 1</td>\n",
        "      <td> [SkillRule: Eliminate Parens; {CLT nested; CLT...</td>\n",
        "      <td>...</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>31</th>\n",
        "      <td> 32</td>\n",
        "      <td> 104</td>\n",
        "      <td> 89</td>\n",
        "      <td> 3</td>\n",
        "      <td> 16</td>\n",
        "      <td>  668</td>\n",
        "      <td> 2</td>\n",
        "      <td>            x+9 = 5/7</td>\n",
        "      <td> 2005-09-09 12:38:15.0</td>\n",
        "      <td> 2005-09-09 12:38:21.0</td>\n",
        "      <td> 2005-09-09 12:38:21.0</td>\n",
        "      <td> 2005-09-09 12:38:21.0</td>\n",
        "      <td>   6</td>\n",
        "      <td>   6</td>\n",
        "      <td> NaN</td>\n",
        "      <td> 1</td>\n",
        "      <td> 0</td>\n",
        "      <td> 0</td>\n",
        "      <td> 1</td>\n",
        "      <td> [SkillRule: Remove constant; {ax+b=c, positive...</td>\n",
        "      <td>...</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>32</th>\n",
        "      <td> 33</td>\n",
        "      <td> 104</td>\n",
        "      <td> 89</td>\n",
        "      <td> 3</td>\n",
        "      <td> 16</td>\n",
        "      <td>  668</td>\n",
        "      <td> 3</td>\n",
        "      <td>           5 = 8(y+1)</td>\n",
        "      <td> 2005-09-09 12:38:25.0</td>\n",
        "      <td> 2005-09-09 12:38:33.0</td>\n",
        "      <td> 2005-09-09 12:38:33.0</td>\n",
        "      <td> 2005-09-09 12:38:33.0</td>\n",
        "      <td>   8</td>\n",
        "      <td>   8</td>\n",
        "      <td> NaN</td>\n",
        "      <td> 1</td>\n",
        "      <td> 0</td>\n",
        "      <td> 0</td>\n",
        "      <td> 1</td>\n",
        "      <td> [SkillRule: Eliminate Parens; {CLT nested; CLT...</td>\n",
        "      <td>...</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>33</th>\n",
        "      <td> 34</td>\n",
        "      <td> 104</td>\n",
        "      <td> 89</td>\n",
        "      <td> 3</td>\n",
        "      <td> 16</td>\n",
        "      <td>  668</td>\n",
        "      <td> 3</td>\n",
        "      <td>            5/8 = y+1</td>\n",
        "      <td> 2005-09-09 12:38:33.0</td>\n",
        "      <td> 2005-09-09 12:38:40.0</td>\n",
        "      <td> 2005-09-09 12:38:40.0</td>\n",
        "      <td> 2005-09-09 12:38:40.0</td>\n",
        "      <td>   7</td>\n",
        "      <td>   7</td>\n",
        "      <td> NaN</td>\n",
        "      <td> 1</td>\n",
        "      <td> 0</td>\n",
        "      <td> 0</td>\n",
        "      <td> 1</td>\n",
        "      <td> [SkillRule: Remove constant; {ax+b=c, positive...</td>\n",
        "      <td>...</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>34</th>\n",
        "      <td> 35</td>\n",
        "      <td> 104</td>\n",
        "      <td> 89</td>\n",
        "      <td> 3</td>\n",
        "      <td> 16</td>\n",
        "      <td>  668</td>\n",
        "      <td> 4</td>\n",
        "      <td>          0 = -3(x-5)</td>\n",
        "      <td> 2005-09-09 12:38:44.0</td>\n",
        "      <td> 2005-09-09 12:38:49.0</td>\n",
        "      <td> 2005-09-09 12:38:49.0</td>\n",
        "      <td> 2005-09-09 12:38:49.0</td>\n",
        "      <td>   5</td>\n",
        "      <td>   5</td>\n",
        "      <td> NaN</td>\n",
        "      <td> 1</td>\n",
        "      <td> 0</td>\n",
        "      <td> 0</td>\n",
        "      <td> 1</td>\n",
        "      <td> [SkillRule: Eliminate Parens; {CLT nested; CLT...</td>\n",
        "      <td>...</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>35</th>\n",
        "      <td> 36</td>\n",
        "      <td> 104</td>\n",
        "      <td> 89</td>\n",
        "      <td> 3</td>\n",
        "      <td> 16</td>\n",
        "      <td>  668</td>\n",
        "      <td> 4</td>\n",
        "      <td>              0 = x-5</td>\n",
        "      <td> 2005-09-09 12:38:49.0</td>\n",
        "      <td> 2005-09-09 12:38:56.0</td>\n",
        "      <td> 2005-09-09 12:38:56.0</td>\n",
        "      <td> 2005-09-09 12:38:56.0</td>\n",
        "      <td>   7</td>\n",
        "      <td>   7</td>\n",
        "      <td> NaN</td>\n",
        "      <td> 1</td>\n",
        "      <td> 0</td>\n",
        "      <td> 0</td>\n",
        "      <td> 1</td>\n",
        "      <td> [SkillRule: Remove constant; {ax+b=c, positive...</td>\n",
        "      <td>...</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>36</th>\n",
        "      <td> 37</td>\n",
        "      <td> 104</td>\n",
        "      <td> 84</td>\n",
        "      <td> 3</td>\n",
        "      <td> 13</td>\n",
        "      <td>  668</td>\n",
        "      <td> 1</td>\n",
        "      <td>         -5(y-10) = 3</td>\n",
        "      <td> 2005-09-09 12:39:01.0</td>\n",
        "      <td> 2005-09-09 12:39:07.0</td>\n",
        "      <td> 2005-09-09 12:39:07.0</td>\n",
        "      <td> 2005-09-09 12:39:07.0</td>\n",
        "      <td>   6</td>\n",
        "      <td>   6</td>\n",
        "      <td> NaN</td>\n",
        "      <td> 1</td>\n",
        "      <td> 0</td>\n",
        "      <td> 0</td>\n",
        "      <td> 1</td>\n",
        "      <td> [SkillRule: Eliminate Parens; {CLT nested; CLT...</td>\n",
        "      <td>...</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>37</th>\n",
        "      <td> 38</td>\n",
        "      <td> 104</td>\n",
        "      <td> 84</td>\n",
        "      <td> 3</td>\n",
        "      <td> 13</td>\n",
        "      <td>  668</td>\n",
        "      <td> 1</td>\n",
        "      <td>   -5(y-10)/-5 = 3/-5</td>\n",
        "      <td> 2005-09-09 12:39:07.0</td>\n",
        "      <td> 2005-09-09 12:39:13.0</td>\n",
        "      <td> 2005-09-09 12:39:13.0</td>\n",
        "      <td> 2005-09-09 12:39:18.0</td>\n",
        "      <td>  11</td>\n",
        "      <td>  11</td>\n",
        "      <td> NaN</td>\n",
        "      <td> 1</td>\n",
        "      <td> 0</td>\n",
        "      <td> 0</td>\n",
        "      <td> 2</td>\n",
        "      <td> [SkillRule: Calculate Eliminate Parens; [Typei...</td>\n",
        "      <td>...</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>38</th>\n",
        "      <td> 39</td>\n",
        "      <td> 104</td>\n",
        "      <td> 84</td>\n",
        "      <td> 3</td>\n",
        "      <td> 13</td>\n",
        "      <td>  668</td>\n",
        "      <td> 1</td>\n",
        "      <td>          y-10 = 3/-5</td>\n",
        "      <td> 2005-09-09 12:39:18.0</td>\n",
        "      <td> 2005-09-09 12:39:26.0</td>\n",
        "      <td> 2005-09-09 12:39:26.0</td>\n",
        "      <td> 2005-09-09 12:39:26.0</td>\n",
        "      <td>   8</td>\n",
        "      <td>   8</td>\n",
        "      <td> NaN</td>\n",
        "      <td> 1</td>\n",
        "      <td> 0</td>\n",
        "      <td> 0</td>\n",
        "      <td> 1</td>\n",
        "      <td> [SkillRule: Remove constant; {ax+b=c, positive...</td>\n",
        "      <td>...</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>39</th>\n",
        "      <td> 40</td>\n",
        "      <td> 104</td>\n",
        "      <td> 84</td>\n",
        "      <td> 3</td>\n",
        "      <td> 13</td>\n",
        "      <td>  668</td>\n",
        "      <td> 1</td>\n",
        "      <td>    y-10+10 = 3/-5+10</td>\n",
        "      <td> 2005-09-09 12:39:26.0</td>\n",
        "      <td> 2005-09-09 12:39:29.0</td>\n",
        "      <td> 2005-09-09 12:39:29.0</td>\n",
        "      <td> 2005-09-09 12:39:36.0</td>\n",
        "      <td>  10</td>\n",
        "      <td>  10</td>\n",
        "      <td> NaN</td>\n",
        "      <td> 1</td>\n",
        "      <td> 0</td>\n",
        "      <td> 0</td>\n",
        "      <td> 2</td>\n",
        "      <td> [SkillRule: Add/Subtract; [Typein Skill: {Isol...</td>\n",
        "      <td>...</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>40</th>\n",
        "      <td> 41</td>\n",
        "      <td> 104</td>\n",
        "      <td> 84</td>\n",
        "      <td> 3</td>\n",
        "      <td> 13</td>\n",
        "      <td>  668</td>\n",
        "      <td> 1</td>\n",
        "      <td>          y = 3/-5+10</td>\n",
        "      <td> 2005-09-09 12:39:36.0</td>\n",
        "      <td> 2005-09-09 12:39:44.0</td>\n",
        "      <td> 2005-09-09 12:40:14.0</td>\n",
        "      <td> 2005-09-09 12:40:14.0</td>\n",
        "      <td>  38</td>\n",
        "      <td> NaN</td>\n",
        "      <td>  38</td>\n",
        "      <td> 0</td>\n",
        "      <td> 1</td>\n",
        "      <td> 3</td>\n",
        "      <td> 1</td>\n",
        "      <td>                                               NaN</td>\n",
        "      <td>...</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>41</th>\n",
        "      <td> 42</td>\n",
        "      <td> 104</td>\n",
        "      <td> 84</td>\n",
        "      <td> 3</td>\n",
        "      <td> 13</td>\n",
        "      <td>  668</td>\n",
        "      <td> 1</td>\n",
        "      <td>          FinalAnswer</td>\n",
        "      <td> 2005-09-09 12:40:14.0</td>\n",
        "      <td> 2005-09-09 12:40:36.0</td>\n",
        "      <td> 2005-09-09 12:40:36.0</td>\n",
        "      <td> 2005-09-09 12:41:14.0</td>\n",
        "      <td>  39</td>\n",
        "      <td>  39</td>\n",
        "      <td> NaN</td>\n",
        "      <td> 1</td>\n",
        "      <td> 0</td>\n",
        "      <td> 0</td>\n",
        "      <td> 2</td>\n",
        "      <td>      simplify-fractions-sp~~combine-like-terms-sp</td>\n",
        "      <td>...</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>42</th>\n",
        "      <td> 43</td>\n",
        "      <td> 104</td>\n",
        "      <td> 84</td>\n",
        "      <td> 3</td>\n",
        "      <td> 13</td>\n",
        "      <td>  668</td>\n",
        "      <td> 1</td>\n",
        "      <td>          y = -3/5+10</td>\n",
        "      <td> 2005-09-09 12:40:36.0</td>\n",
        "      <td> 2005-09-09 12:40:57.0</td>\n",
        "      <td> 2005-09-09 12:40:57.0</td>\n",
        "      <td> 2005-09-09 12:40:57.0</td>\n",
        "      <td>  21</td>\n",
        "      <td>  21</td>\n",
        "      <td> NaN</td>\n",
        "      <td> 1</td>\n",
        "      <td> 0</td>\n",
        "      <td> 0</td>\n",
        "      <td> 1</td>\n",
        "      <td>      [SkillRule: Consolidate vars, no coeff; CLT]</td>\n",
        "      <td>...</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>43</th>\n",
        "      <td> 44</td>\n",
        "      <td> 104</td>\n",
        "      <td> 84</td>\n",
        "      <td> 3</td>\n",
        "      <td> 13</td>\n",
        "      <td>  668</td>\n",
        "      <td> 2</td>\n",
        "      <td>          4(x-4) = -8</td>\n",
        "      <td> 2005-09-09 12:41:48.0</td>\n",
        "      <td> 2005-09-09 12:42:13.0</td>\n",
        "      <td> 2005-09-09 12:42:13.0</td>\n",
        "      <td> 2005-09-09 12:42:13.0</td>\n",
        "      <td>  25</td>\n",
        "      <td>  25</td>\n",
        "      <td> NaN</td>\n",
        "      <td> 1</td>\n",
        "      <td> 0</td>\n",
        "      <td> 0</td>\n",
        "      <td> 1</td>\n",
        "      <td> [SkillRule: Eliminate Parens; {CLT nested; CLT...</td>\n",
        "      <td>...</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>44</th>\n",
        "      <td> 45</td>\n",
        "      <td> 104</td>\n",
        "      <td> 84</td>\n",
        "      <td> 3</td>\n",
        "      <td> 13</td>\n",
        "      <td>  668</td>\n",
        "      <td> 2</td>\n",
        "      <td>      4(x-4)/4 = -8/4</td>\n",
        "      <td> 2005-09-09 12:42:13.0</td>\n",
        "      <td> 2005-09-09 12:42:18.0</td>\n",
        "      <td> 2005-09-09 12:42:18.0</td>\n",
        "      <td> 2005-09-09 12:42:23.0</td>\n",
        "      <td>  10</td>\n",
        "      <td>  10</td>\n",
        "      <td> NaN</td>\n",
        "      <td> 1</td>\n",
        "      <td> 0</td>\n",
        "      <td> 0</td>\n",
        "      <td> 2</td>\n",
        "      <td> [SkillRule: Calculate Eliminate Parens; [Typei...</td>\n",
        "      <td>...</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>45</th>\n",
        "      <td> 46</td>\n",
        "      <td> 104</td>\n",
        "      <td> 84</td>\n",
        "      <td> 3</td>\n",
        "      <td> 13</td>\n",
        "      <td>  668</td>\n",
        "      <td> 2</td>\n",
        "      <td>           x-4 = -8/4</td>\n",
        "      <td> 2005-09-09 12:42:23.0</td>\n",
        "      <td> 2005-09-09 12:42:32.0</td>\n",
        "      <td> 2005-09-09 12:42:32.0</td>\n",
        "      <td> 2005-09-09 12:42:32.0</td>\n",
        "      <td>   9</td>\n",
        "      <td>   9</td>\n",
        "      <td> NaN</td>\n",
        "      <td> 1</td>\n",
        "      <td> 0</td>\n",
        "      <td> 0</td>\n",
        "      <td> 1</td>\n",
        "      <td> [SkillRule: Remove constant; {ax+b=c, positive...</td>\n",
        "      <td>...</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>46</th>\n",
        "      <td> 47</td>\n",
        "      <td> 104</td>\n",
        "      <td> 84</td>\n",
        "      <td> 3</td>\n",
        "      <td> 13</td>\n",
        "      <td>  668</td>\n",
        "      <td> 2</td>\n",
        "      <td>       x-4+4 = -8/4+4</td>\n",
        "      <td> 2005-09-09 12:42:32.0</td>\n",
        "      <td> 2005-09-09 12:42:35.0</td>\n",
        "      <td> 2005-09-09 12:42:35.0</td>\n",
        "      <td> 2005-09-09 12:42:55.0</td>\n",
        "      <td>  23</td>\n",
        "      <td>  23</td>\n",
        "      <td> NaN</td>\n",
        "      <td> 1</td>\n",
        "      <td> 0</td>\n",
        "      <td> 0</td>\n",
        "      <td> 2</td>\n",
        "      <td> [SkillRule: Add/Subtract; [Typein Skill: {Isol...</td>\n",
        "      <td>...</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>47</th>\n",
        "      <td> 48</td>\n",
        "      <td> 104</td>\n",
        "      <td> 84</td>\n",
        "      <td> 3</td>\n",
        "      <td> 13</td>\n",
        "      <td>  668</td>\n",
        "      <td> 2</td>\n",
        "      <td>           x = -8/4+4</td>\n",
        "      <td> 2005-09-09 12:42:55.0</td>\n",
        "      <td> 2005-09-09 12:43:02.0</td>\n",
        "      <td> 2005-09-09 12:43:02.0</td>\n",
        "      <td> 2005-09-09 12:43:02.0</td>\n",
        "      <td>   7</td>\n",
        "      <td>   7</td>\n",
        "      <td> NaN</td>\n",
        "      <td> 1</td>\n",
        "      <td> 0</td>\n",
        "      <td> 0</td>\n",
        "      <td> 1</td>\n",
        "      <td>      [SkillRule: Consolidate vars, no coeff; CLT]</td>\n",
        "      <td>...</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>48</th>\n",
        "      <td> 49</td>\n",
        "      <td> 104</td>\n",
        "      <td> 84</td>\n",
        "      <td> 3</td>\n",
        "      <td> 13</td>\n",
        "      <td>  668</td>\n",
        "      <td> 2</td>\n",
        "      <td>          FinalAnswer</td>\n",
        "      <td> 2005-09-09 12:43:02.0</td>\n",
        "      <td> 2005-09-09 12:43:25.0</td>\n",
        "      <td> 2005-09-09 12:43:25.0</td>\n",
        "      <td> 2005-09-09 12:43:25.0</td>\n",
        "      <td>  23</td>\n",
        "      <td>  23</td>\n",
        "      <td> NaN</td>\n",
        "      <td> 1</td>\n",
        "      <td> 0</td>\n",
        "      <td> 0</td>\n",
        "      <td> 1</td>\n",
        "      <td>      simplify-fractions-sp~~combine-like-terms-sp</td>\n",
        "      <td>...</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>49</th>\n",
        "      <td> 50</td>\n",
        "      <td> 104</td>\n",
        "      <td> 84</td>\n",
        "      <td> 3</td>\n",
        "      <td> 13</td>\n",
        "      <td>  668</td>\n",
        "      <td> 3</td>\n",
        "      <td>           5 = 4(x-3)</td>\n",
        "      <td> 2005-09-09 12:43:34.0</td>\n",
        "      <td> 2005-09-09 12:43:41.0</td>\n",
        "      <td> 2005-09-09 12:43:41.0</td>\n",
        "      <td> 2005-09-09 12:43:41.0</td>\n",
        "      <td>   7</td>\n",
        "      <td>   7</td>\n",
        "      <td> NaN</td>\n",
        "      <td> 1</td>\n",
        "      <td> 0</td>\n",
        "      <td> 0</td>\n",
        "      <td> 1</td>\n",
        "      <td> [SkillRule: Eliminate Parens; {CLT nested; CLT...</td>\n",
        "      <td>...</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>50</th>\n",
        "      <td> 51</td>\n",
        "      <td> 104</td>\n",
        "      <td> 84</td>\n",
        "      <td> 3</td>\n",
        "      <td> 13</td>\n",
        "      <td>  668</td>\n",
        "      <td> 3</td>\n",
        "      <td>       5/4 = 4(x-3)/4</td>\n",
        "      <td> 2005-09-09 12:43:41.0</td>\n",
        "      <td> 2005-09-09 12:43:45.0</td>\n",
        "      <td> 2005-09-09 12:43:45.0</td>\n",
        "      <td> 2005-09-09 12:43:49.0</td>\n",
        "      <td>   8</td>\n",
        "      <td>   8</td>\n",
        "      <td> NaN</td>\n",
        "      <td> 1</td>\n",
        "      <td> 0</td>\n",
        "      <td> 0</td>\n",
        "      <td> 2</td>\n",
        "      <td> [SkillRule: Calculate Eliminate Parens; [Typei...</td>\n",
        "      <td>...</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>51</th>\n",
        "      <td> 52</td>\n",
        "      <td> 104</td>\n",
        "      <td> 84</td>\n",
        "      <td> 3</td>\n",
        "      <td> 13</td>\n",
        "      <td>  668</td>\n",
        "      <td> 3</td>\n",
        "      <td>            5/4 = x-3</td>\n",
        "      <td> 2005-09-09 12:43:49.0</td>\n",
        "      <td> 2005-09-09 12:44:09.0</td>\n",
        "      <td> 2005-09-09 12:44:09.0</td>\n",
        "      <td> 2005-09-09 12:44:09.0</td>\n",
        "      <td>  20</td>\n",
        "      <td>  20</td>\n",
        "      <td> NaN</td>\n",
        "      <td> 1</td>\n",
        "      <td> 0</td>\n",
        "      <td> 0</td>\n",
        "      <td> 1</td>\n",
        "      <td> [SkillRule: Remove constant; {ax+b=c, positive...</td>\n",
        "      <td>...</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>52</th>\n",
        "      <td> 53</td>\n",
        "      <td> 104</td>\n",
        "      <td> 84</td>\n",
        "      <td> 3</td>\n",
        "      <td> 13</td>\n",
        "      <td>  668</td>\n",
        "      <td> 3</td>\n",
        "      <td>        5/4+3 = x-3+3</td>\n",
        "      <td> 2005-09-09 12:44:09.0</td>\n",
        "      <td> 2005-09-09 12:44:12.0</td>\n",
        "      <td> 2005-09-09 12:44:12.0</td>\n",
        "      <td> 2005-09-09 12:44:19.0</td>\n",
        "      <td>  10</td>\n",
        "      <td>  10</td>\n",
        "      <td> NaN</td>\n",
        "      <td> 1</td>\n",
        "      <td> 0</td>\n",
        "      <td> 0</td>\n",
        "      <td> 2</td>\n",
        "      <td> [SkillRule: Add/Subtract; [Typein Skill: {Isol...</td>\n",
        "      <td>...</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>53</th>\n",
        "      <td> 54</td>\n",
        "      <td> 104</td>\n",
        "      <td> 84</td>\n",
        "      <td> 3</td>\n",
        "      <td> 13</td>\n",
        "      <td>  668</td>\n",
        "      <td> 3</td>\n",
        "      <td>            5/4+3 = x</td>\n",
        "      <td> 2005-09-09 12:44:19.0</td>\n",
        "      <td> 2005-09-09 12:44:35.0</td>\n",
        "      <td> 2005-09-09 12:44:35.0</td>\n",
        "      <td> 2005-09-09 12:44:35.0</td>\n",
        "      <td>  16</td>\n",
        "      <td>  16</td>\n",
        "      <td> NaN</td>\n",
        "      <td> 1</td>\n",
        "      <td> 0</td>\n",
        "      <td> 0</td>\n",
        "      <td> 1</td>\n",
        "      <td>      [SkillRule: Consolidate vars, no coeff; CLT]</td>\n",
        "      <td>...</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>54</th>\n",
        "      <td> 55</td>\n",
        "      <td> 104</td>\n",
        "      <td> 84</td>\n",
        "      <td> 3</td>\n",
        "      <td> 13</td>\n",
        "      <td>  668</td>\n",
        "      <td> 3</td>\n",
        "      <td>          FinalAnswer</td>\n",
        "      <td> 2005-09-09 12:44:35.0</td>\n",
        "      <td> 2005-09-09 12:44:48.0</td>\n",
        "      <td> 2005-09-09 12:44:48.0</td>\n",
        "      <td> 2005-09-09 12:44:48.0</td>\n",
        "      <td>  13</td>\n",
        "      <td>  13</td>\n",
        "      <td> NaN</td>\n",
        "      <td> 1</td>\n",
        "      <td> 0</td>\n",
        "      <td> 0</td>\n",
        "      <td> 1</td>\n",
        "      <td>      simplify-fractions-sp~~combine-like-terms-sp</td>\n",
        "      <td>...</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>55</th>\n",
        "      <td> 56</td>\n",
        "      <td> 104</td>\n",
        "      <td> 83</td>\n",
        "      <td> 3</td>\n",
        "      <td> 14</td>\n",
        "      <td>  936</td>\n",
        "      <td> 1</td>\n",
        "      <td>  0.1 = -42.3(y-83.7)</td>\n",
        "      <td> 2005-09-09 12:44:58.0</td>\n",
        "      <td> 2005-09-09 12:45:42.0</td>\n",
        "      <td> 2005-09-09 12:45:42.0</td>\n",
        "      <td> 2005-09-09 12:45:42.0</td>\n",
        "      <td>  44</td>\n",
        "      <td>  44</td>\n",
        "      <td> NaN</td>\n",
        "      <td> 1</td>\n",
        "      <td> 0</td>\n",
        "      <td> 0</td>\n",
        "      <td> 1</td>\n",
        "      <td> [SkillRule: Eliminate Parens; {CLT nested; CLT...</td>\n",
        "      <td>...</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>56</th>\n",
        "      <td> 57</td>\n",
        "      <td> 104</td>\n",
        "      <td> 83</td>\n",
        "      <td> 3</td>\n",
        "      <td> 14</td>\n",
        "      <td>  936</td>\n",
        "      <td> 1</td>\n",
        "      <td> -0.00236407 = y-83.7</td>\n",
        "      <td> 2005-09-09 12:45:42.0</td>\n",
        "      <td> 2005-09-09 12:45:56.0</td>\n",
        "      <td> 2005-09-09 12:45:56.0</td>\n",
        "      <td> 2005-09-09 12:45:56.0</td>\n",
        "      <td>  14</td>\n",
        "      <td>  14</td>\n",
        "      <td> NaN</td>\n",
        "      <td> 1</td>\n",
        "      <td> 0</td>\n",
        "      <td> 0</td>\n",
        "      <td> 1</td>\n",
        "      <td> [SkillRule: Remove constant; {ax+b=c, positive...</td>\n",
        "      <td>...</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>57</th>\n",
        "      <td> 58</td>\n",
        "      <td> 104</td>\n",
        "      <td> 83</td>\n",
        "      <td> 3</td>\n",
        "      <td> 14</td>\n",
        "      <td>  667</td>\n",
        "      <td> 1</td>\n",
        "      <td>    1.8(y-9.8) = -2.4</td>\n",
        "      <td> 2005-09-09 12:46:01.0</td>\n",
        "      <td> 2005-09-09 12:46:48.0</td>\n",
        "      <td> 2005-09-09 12:46:57.0</td>\n",
        "      <td> 2005-09-09 12:46:57.0</td>\n",
        "      <td>  56</td>\n",
        "      <td> NaN</td>\n",
        "      <td>  56</td>\n",
        "      <td> 0</td>\n",
        "      <td> 1</td>\n",
        "      <td> 0</td>\n",
        "      <td> 1</td>\n",
        "      <td> [SkillRule: Eliminate Parens; {CLT nested; CLT...</td>\n",
        "      <td>...</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>58</th>\n",
        "      <td> 59</td>\n",
        "      <td> 104</td>\n",
        "      <td> 83</td>\n",
        "      <td> 3</td>\n",
        "      <td> 14</td>\n",
        "      <td>  667</td>\n",
        "      <td> 1</td>\n",
        "      <td>  y-9.8 = -1.33333333</td>\n",
        "      <td> 2005-09-09 12:46:57.0</td>\n",
        "      <td> 2005-09-09 12:47:49.0</td>\n",
        "      <td> 2005-09-09 12:48:21.0</td>\n",
        "      <td> 2005-09-09 12:48:21.0</td>\n",
        "      <td>  84</td>\n",
        "      <td> NaN</td>\n",
        "      <td>  84</td>\n",
        "      <td> 0</td>\n",
        "      <td> 1</td>\n",
        "      <td> 0</td>\n",
        "      <td> 1</td>\n",
        "      <td> [SkillRule: Remove constant; {ax+b=c, positive...</td>\n",
        "      <td>...</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>59</th>\n",
        "      <td> 60</td>\n",
        "      <td> 104</td>\n",
        "      <td> 83</td>\n",
        "      <td> 3</td>\n",
        "      <td> 14</td>\n",
        "      <td>  667</td>\n",
        "      <td> 2</td>\n",
        "      <td>   -5.5(y-1.9) = -9.6</td>\n",
        "      <td> 2005-09-09 12:48:26.0</td>\n",
        "      <td> 2005-09-09 12:49:24.0</td>\n",
        "      <td> 2005-09-09 12:49:24.0</td>\n",
        "      <td> 2005-09-09 12:49:24.0</td>\n",
        "      <td>  58</td>\n",
        "      <td>  58</td>\n",
        "      <td> NaN</td>\n",
        "      <td> 1</td>\n",
        "      <td> 0</td>\n",
        "      <td> 0</td>\n",
        "      <td> 1</td>\n",
        "      <td> [SkillRule: Eliminate Parens; {CLT nested; CLT...</td>\n",
        "      <td>...</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th></th>\n",
        "      <td>...</td>\n",
        "      <td>...</td>\n",
        "      <td>...</td>\n",
        "      <td>...</td>\n",
        "      <td>...</td>\n",
        "      <td>...</td>\n",
        "      <td>...</td>\n",
        "      <td>...</td>\n",
        "      <td>...</td>\n",
        "      <td>...</td>\n",
        "      <td>...</td>\n",
        "      <td>...</td>\n",
        "      <td>...</td>\n",
        "      <td>...</td>\n",
        "      <td>...</td>\n",
        "      <td>...</td>\n",
        "      <td>...</td>\n",
        "      <td>...</td>\n",
        "      <td>...</td>\n",
        "      <td>...</td>\n",
        "    </tr>\n",
        "  </tbody>\n",
        "</table>\n",
        "<p>809694 rows \u00d7 21 columns</p>\n",
        "</div>"
       ],
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 116,
       "text": [
        "    Row Anon Student Id Problem Hierarchy Problem Unit Problem Section  \\\n",
        "0     1             104                87            3              17   \n",
        "1     2             104                87            3              17   \n",
        "2     3             104                87            3              17   \n",
        "3     4             104                87            3              17   \n",
        "4     5             104                87            3              17   \n",
        "5     6             104                87            3              17   \n",
        "6     7             104                87            3              17   \n",
        "7     8             104                87            3              17   \n",
        "8     9             104                87            3              17   \n",
        "9    10             104                87            3              17   \n",
        "10   11             104                90            3              15   \n",
        "11   12             104                90            3              15   \n",
        "12   13             104                90            3              15   \n",
        "13   14             104                90            3              15   \n",
        "14   15             104                90            3              15   \n",
        "15   16             104                90            3              15   \n",
        "16   17             104                90            3              15   \n",
        "17   18             104                90            3              15   \n",
        "18   19             104                90            3              15   \n",
        "19   20             104                90            3              15   \n",
        "20   21             104                90            3              15   \n",
        "21   22             104                90            3              15   \n",
        "22   23             104                90            3              15   \n",
        "23   24             104                90            3              15   \n",
        "24   25             104                90            3              15   \n",
        "25   26             104                90            3              15   \n",
        "26   27             104                90            3              15   \n",
        "27   28             104                90            3              15   \n",
        "28   29             104                89            3              16   \n",
        "29   30             104                89            3              16   \n",
        "30   31             104                89            3              16   \n",
        "31   32             104                89            3              16   \n",
        "32   33             104                89            3              16   \n",
        "33   34             104                89            3              16   \n",
        "34   35             104                89            3              16   \n",
        "35   36             104                89            3              16   \n",
        "36   37             104                84            3              13   \n",
        "37   38             104                84            3              13   \n",
        "38   39             104                84            3              13   \n",
        "39   40             104                84            3              13   \n",
        "40   41             104                84            3              13   \n",
        "41   42             104                84            3              13   \n",
        "42   43             104                84            3              13   \n",
        "43   44             104                84            3              13   \n",
        "44   45             104                84            3              13   \n",
        "45   46             104                84            3              13   \n",
        "46   47             104                84            3              13   \n",
        "47   48             104                84            3              13   \n",
        "48   49             104                84            3              13   \n",
        "49   50             104                84            3              13   \n",
        "50   51             104                84            3              13   \n",
        "51   52             104                84            3              13   \n",
        "52   53             104                84            3              13   \n",
        "53   54             104                84            3              13   \n",
        "54   55             104                84            3              13   \n",
        "55   56             104                83            3              14   \n",
        "56   57             104                83            3              14   \n",
        "57   58             104                83            3              14   \n",
        "58   59             104                83            3              14   \n",
        "59   60             104                83            3              14   \n",
        "    ...             ...               ...          ...             ...   \n",
        "\n",
        "   Problem Name  Problem View             Step Name        Step Start Time  \\\n",
        "0          1046             1           3(x+2) = 15  2005-09-09 12:24:35.0   \n",
        "1          1046             1               x+2 = 5  2005-09-09 12:25:15.0   \n",
        "2           346             1             2-8y = -4  2005-09-09 12:25:36.0   \n",
        "3           346             1              -8y = -6  2005-09-09 12:26:12.0   \n",
        "4           346             2            -7y-5 = -4  2005-09-09 12:26:38.0   \n",
        "5           346             2               -7y = 1  2005-09-09 12:28:36.0   \n",
        "6           346             3              7y+4 = 7  2005-09-09 12:28:57.0   \n",
        "7           346             3                7y = 3  2005-09-09 12:29:09.0   \n",
        "8           346             4            -5+9y = -6  2005-09-09 12:29:19.0   \n",
        "9           346             4               9y = -1  2005-09-09 12:29:31.0   \n",
        "10          346             1            -7-3x = -2  2005-09-09 12:29:41.0   \n",
        "11          346             1        -7-3x+7 = -2+7  2005-09-09 12:30:27.0   \n",
        "12          346             1               -3x = 5  2005-09-09 12:30:49.0   \n",
        "13          346             1         -3x/-3 = 5/-3  2005-09-09 12:31:04.0   \n",
        "14          346             2             -9 = 8y+9  2005-09-09 12:31:16.0   \n",
        "15          346             2         -9-9 = 8y+9-9  2005-09-09 12:31:29.0   \n",
        "16          346             2              -18 = 8y  2005-09-09 12:31:39.0   \n",
        "17          346             2          -18/8 = 8y/8  2005-09-09 12:31:44.0   \n",
        "18          346             3             -2-2x = 9  2005-09-09 12:33:01.0   \n",
        "19          346             3         -2-2x+2 = 9+2  2005-09-09 12:33:32.0   \n",
        "20          346             3              -2x = 11  2005-09-09 12:33:40.0   \n",
        "21          346             3        -2x/-2 = 11/-2  2005-09-09 12:33:46.0   \n",
        "22          346             4             4+4y = -6  2005-09-09 12:33:59.0   \n",
        "23          346             4         4+4y-4 = -6-4  2005-09-09 12:34:06.0   \n",
        "24          346             4             4y = -6-4  2005-09-09 12:34:17.0   \n",
        "25          346             4           FinalAnswer  2005-09-09 12:34:25.0   \n",
        "26          346             4              4y = -10  2005-09-09 12:34:33.0   \n",
        "27          346             4          4y/4 = -10/4  2005-09-09 12:34:42.0   \n",
        "28          668             1          -7 = -5(y+7)  2005-09-09 12:35:08.0   \n",
        "29          668             1             7/5 = y+7  2005-09-09 12:37:01.0   \n",
        "30          668             2          -7(x+9) = -5  2005-09-09 12:37:17.0   \n",
        "31          668             2             x+9 = 5/7  2005-09-09 12:38:15.0   \n",
        "32          668             3            5 = 8(y+1)  2005-09-09 12:38:25.0   \n",
        "33          668             3             5/8 = y+1  2005-09-09 12:38:33.0   \n",
        "34          668             4           0 = -3(x-5)  2005-09-09 12:38:44.0   \n",
        "35          668             4               0 = x-5  2005-09-09 12:38:49.0   \n",
        "36          668             1          -5(y-10) = 3  2005-09-09 12:39:01.0   \n",
        "37          668             1    -5(y-10)/-5 = 3/-5  2005-09-09 12:39:07.0   \n",
        "38          668             1           y-10 = 3/-5  2005-09-09 12:39:18.0   \n",
        "39          668             1     y-10+10 = 3/-5+10  2005-09-09 12:39:26.0   \n",
        "40          668             1           y = 3/-5+10  2005-09-09 12:39:36.0   \n",
        "41          668             1           FinalAnswer  2005-09-09 12:40:14.0   \n",
        "42          668             1           y = -3/5+10  2005-09-09 12:40:36.0   \n",
        "43          668             2           4(x-4) = -8  2005-09-09 12:41:48.0   \n",
        "44          668             2       4(x-4)/4 = -8/4  2005-09-09 12:42:13.0   \n",
        "45          668             2            x-4 = -8/4  2005-09-09 12:42:23.0   \n",
        "46          668             2        x-4+4 = -8/4+4  2005-09-09 12:42:32.0   \n",
        "47          668             2            x = -8/4+4  2005-09-09 12:42:55.0   \n",
        "48          668             2           FinalAnswer  2005-09-09 12:43:02.0   \n",
        "49          668             3            5 = 4(x-3)  2005-09-09 12:43:34.0   \n",
        "50          668             3        5/4 = 4(x-3)/4  2005-09-09 12:43:41.0   \n",
        "51          668             3             5/4 = x-3  2005-09-09 12:43:49.0   \n",
        "52          668             3         5/4+3 = x-3+3  2005-09-09 12:44:09.0   \n",
        "53          668             3             5/4+3 = x  2005-09-09 12:44:19.0   \n",
        "54          668             3           FinalAnswer  2005-09-09 12:44:35.0   \n",
        "55          936             1   0.1 = -42.3(y-83.7)  2005-09-09 12:44:58.0   \n",
        "56          936             1  -0.00236407 = y-83.7  2005-09-09 12:45:42.0   \n",
        "57          667             1     1.8(y-9.8) = -2.4  2005-09-09 12:46:01.0   \n",
        "58          667             1   y-9.8 = -1.33333333  2005-09-09 12:46:57.0   \n",
        "59          667             2    -5.5(y-1.9) = -9.6  2005-09-09 12:48:26.0   \n",
        "            ...           ...                   ...                    ...   \n",
        "\n",
        "   First Transaction Time Correct Transaction Time          Step End Time  \\\n",
        "0   2005-09-09 12:24:49.0    2005-09-09 12:25:15.0  2005-09-09 12:25:15.0   \n",
        "1   2005-09-09 12:25:31.0    2005-09-09 12:25:31.0  2005-09-09 12:25:31.0   \n",
        "2   2005-09-09 12:25:43.0    2005-09-09 12:26:12.0  2005-09-09 12:26:12.0   \n",
        "3   2005-09-09 12:26:34.0    2005-09-09 12:26:34.0  2005-09-09 12:26:34.0   \n",
        "4   2005-09-09 12:28:36.0    2005-09-09 12:28:36.0  2005-09-09 12:28:36.0   \n",
        "5   2005-09-09 12:28:43.0    2005-09-09 12:28:51.0  2005-09-09 12:28:51.0   \n",
        "6   2005-09-09 12:29:09.0    2005-09-09 12:29:09.0  2005-09-09 12:29:09.0   \n",
        "7   2005-09-09 12:29:14.0    2005-09-09 12:29:14.0  2005-09-09 12:29:14.0   \n",
        "8   2005-09-09 12:29:31.0    2005-09-09 12:29:31.0  2005-09-09 12:29:31.0   \n",
        "9   2005-09-09 12:29:36.0    2005-09-09 12:29:36.0  2005-09-09 12:29:36.0   \n",
        "10  2005-09-09 12:30:27.0    2005-09-09 12:30:27.0  2005-09-09 12:30:27.0   \n",
        "11  2005-09-09 12:30:34.0    2005-09-09 12:30:45.0  2005-09-09 12:30:49.0   \n",
        "12  2005-09-09 12:31:04.0    2005-09-09 12:31:04.0  2005-09-09 12:31:04.0   \n",
        "13  2005-09-09 12:31:07.0    2005-09-09 12:31:07.0  2005-09-09 12:31:12.0   \n",
        "14  2005-09-09 12:31:29.0    2005-09-09 12:31:29.0  2005-09-09 12:31:29.0   \n",
        "15  2005-09-09 12:31:32.0    2005-09-09 12:31:32.0  2005-09-09 12:31:39.0   \n",
        "16  2005-09-09 12:31:44.0    2005-09-09 12:31:44.0  2005-09-09 12:31:44.0   \n",
        "17  2005-09-09 12:31:46.0    2005-09-09 12:31:46.0  2005-09-09 12:32:56.0   \n",
        "18  2005-09-09 12:33:22.0    2005-09-09 12:33:32.0  2005-09-09 12:33:32.0   \n",
        "19  2005-09-09 12:33:37.0    2005-09-09 12:33:37.0  2005-09-09 12:33:40.0   \n",
        "20  2005-09-09 12:33:46.0    2005-09-09 12:33:46.0  2005-09-09 12:33:46.0   \n",
        "21  2005-09-09 12:33:51.0    2005-09-09 12:33:51.0  2005-09-09 12:33:55.0   \n",
        "22  2005-09-09 12:34:06.0    2005-09-09 12:34:06.0  2005-09-09 12:34:06.0   \n",
        "23  2005-09-09 12:34:09.0    2005-09-09 12:34:09.0  2005-09-09 12:34:17.0   \n",
        "24  2005-09-09 12:34:25.0    2005-09-09 12:34:25.0  2005-09-09 12:34:25.0   \n",
        "25  2005-09-09 12:34:29.0    2005-09-09 12:34:33.0  2005-09-09 12:34:33.0   \n",
        "26  2005-09-09 12:34:42.0    2005-09-09 12:34:42.0  2005-09-09 12:34:42.0   \n",
        "27  2005-09-09 12:34:46.0    2005-09-09 12:34:46.0  2005-09-09 12:35:02.0   \n",
        "28  2005-09-09 12:36:27.0    2005-09-09 12:37:01.0  2005-09-09 12:37:01.0   \n",
        "29  2005-09-09 12:37:09.0    2005-09-09 12:37:09.0  2005-09-09 12:37:09.0   \n",
        "30  2005-09-09 12:38:15.0    2005-09-09 12:38:15.0  2005-09-09 12:38:15.0   \n",
        "31  2005-09-09 12:38:21.0    2005-09-09 12:38:21.0  2005-09-09 12:38:21.0   \n",
        "32  2005-09-09 12:38:33.0    2005-09-09 12:38:33.0  2005-09-09 12:38:33.0   \n",
        "33  2005-09-09 12:38:40.0    2005-09-09 12:38:40.0  2005-09-09 12:38:40.0   \n",
        "34  2005-09-09 12:38:49.0    2005-09-09 12:38:49.0  2005-09-09 12:38:49.0   \n",
        "35  2005-09-09 12:38:56.0    2005-09-09 12:38:56.0  2005-09-09 12:38:56.0   \n",
        "36  2005-09-09 12:39:07.0    2005-09-09 12:39:07.0  2005-09-09 12:39:07.0   \n",
        "37  2005-09-09 12:39:13.0    2005-09-09 12:39:13.0  2005-09-09 12:39:18.0   \n",
        "38  2005-09-09 12:39:26.0    2005-09-09 12:39:26.0  2005-09-09 12:39:26.0   \n",
        "39  2005-09-09 12:39:29.0    2005-09-09 12:39:29.0  2005-09-09 12:39:36.0   \n",
        "40  2005-09-09 12:39:44.0    2005-09-09 12:40:14.0  2005-09-09 12:40:14.0   \n",
        "41  2005-09-09 12:40:36.0    2005-09-09 12:40:36.0  2005-09-09 12:41:14.0   \n",
        "42  2005-09-09 12:40:57.0    2005-09-09 12:40:57.0  2005-09-09 12:40:57.0   \n",
        "43  2005-09-09 12:42:13.0    2005-09-09 12:42:13.0  2005-09-09 12:42:13.0   \n",
        "44  2005-09-09 12:42:18.0    2005-09-09 12:42:18.0  2005-09-09 12:42:23.0   \n",
        "45  2005-09-09 12:42:32.0    2005-09-09 12:42:32.0  2005-09-09 12:42:32.0   \n",
        "46  2005-09-09 12:42:35.0    2005-09-09 12:42:35.0  2005-09-09 12:42:55.0   \n",
        "47  2005-09-09 12:43:02.0    2005-09-09 12:43:02.0  2005-09-09 12:43:02.0   \n",
        "48  2005-09-09 12:43:25.0    2005-09-09 12:43:25.0  2005-09-09 12:43:25.0   \n",
        "49  2005-09-09 12:43:41.0    2005-09-09 12:43:41.0  2005-09-09 12:43:41.0   \n",
        "50  2005-09-09 12:43:45.0    2005-09-09 12:43:45.0  2005-09-09 12:43:49.0   \n",
        "51  2005-09-09 12:44:09.0    2005-09-09 12:44:09.0  2005-09-09 12:44:09.0   \n",
        "52  2005-09-09 12:44:12.0    2005-09-09 12:44:12.0  2005-09-09 12:44:19.0   \n",
        "53  2005-09-09 12:44:35.0    2005-09-09 12:44:35.0  2005-09-09 12:44:35.0   \n",
        "54  2005-09-09 12:44:48.0    2005-09-09 12:44:48.0  2005-09-09 12:44:48.0   \n",
        "55  2005-09-09 12:45:42.0    2005-09-09 12:45:42.0  2005-09-09 12:45:42.0   \n",
        "56  2005-09-09 12:45:56.0    2005-09-09 12:45:56.0  2005-09-09 12:45:56.0   \n",
        "57  2005-09-09 12:46:48.0    2005-09-09 12:46:57.0  2005-09-09 12:46:57.0   \n",
        "58  2005-09-09 12:47:49.0    2005-09-09 12:48:21.0  2005-09-09 12:48:21.0   \n",
        "59  2005-09-09 12:49:24.0    2005-09-09 12:49:24.0  2005-09-09 12:49:24.0   \n",
        "                      ...                      ...                    ...   \n",
        "\n",
        "    Step Duration (sec)  Correct Step Duration (sec)  \\\n",
        "0                    40                          NaN   \n",
        "1                    16                           16   \n",
        "2                    36                          NaN   \n",
        "3                    22                           22   \n",
        "4                   118                          118   \n",
        "5                    15                          NaN   \n",
        "6                    12                           12   \n",
        "7                     5                            5   \n",
        "8                    12                           12   \n",
        "9                     5                            5   \n",
        "10                   46                           46   \n",
        "11                   22                          NaN   \n",
        "12                   15                           15   \n",
        "13                    8                            8   \n",
        "14                   13                           13   \n",
        "15                   10                           10   \n",
        "16                    5                            5   \n",
        "17                   72                           72   \n",
        "18                   31                          NaN   \n",
        "19                    8                            8   \n",
        "20                    6                            6   \n",
        "21                    9                            9   \n",
        "22                    7                            7   \n",
        "23                   11                           11   \n",
        "24                    8                            8   \n",
        "25                    8                          NaN   \n",
        "26                    9                            9   \n",
        "27                   20                           20   \n",
        "28                  113                          NaN   \n",
        "29                    8                            8   \n",
        "30                   58                           58   \n",
        "31                    6                            6   \n",
        "32                    8                            8   \n",
        "33                    7                            7   \n",
        "34                    5                            5   \n",
        "35                    7                            7   \n",
        "36                    6                            6   \n",
        "37                   11                           11   \n",
        "38                    8                            8   \n",
        "39                   10                           10   \n",
        "40                   38                          NaN   \n",
        "41                   39                           39   \n",
        "42                   21                           21   \n",
        "43                   25                           25   \n",
        "44                   10                           10   \n",
        "45                    9                            9   \n",
        "46                   23                           23   \n",
        "47                    7                            7   \n",
        "48                   23                           23   \n",
        "49                    7                            7   \n",
        "50                    8                            8   \n",
        "51                   20                           20   \n",
        "52                   10                           10   \n",
        "53                   16                           16   \n",
        "54                   13                           13   \n",
        "55                   44                           44   \n",
        "56                   14                           14   \n",
        "57                   56                          NaN   \n",
        "58                   84                          NaN   \n",
        "59                   58                           58   \n",
        "                    ...                          ...   \n",
        "\n",
        "    Error Step Duration (sec)  Correct First Attempt  Incorrects  Hints  \\\n",
        "0                          40                      0           2      3   \n",
        "1                         NaN                      1           0      0   \n",
        "2                          36                      0           2      3   \n",
        "3                         NaN                      1           0      0   \n",
        "4                         NaN                      1           0      0   \n",
        "5                          15                      0           1      0   \n",
        "6                         NaN                      1           0      0   \n",
        "7                         NaN                      1           0      0   \n",
        "8                         NaN                      1           0      0   \n",
        "9                         NaN                      1           0      0   \n",
        "10                        NaN                      1           0      0   \n",
        "11                         22                      0           1      0   \n",
        "12                        NaN                      1           0      0   \n",
        "13                        NaN                      1           0      0   \n",
        "14                        NaN                      1           0      0   \n",
        "15                        NaN                      1           0      0   \n",
        "16                        NaN                      1           0      0   \n",
        "17                        NaN                      1           2      0   \n",
        "18                         31                      0           1      0   \n",
        "19                        NaN                      1           0      0   \n",
        "20                        NaN                      1           0      0   \n",
        "21                        NaN                      1           0      0   \n",
        "22                        NaN                      1           0      0   \n",
        "23                        NaN                      1           0      0   \n",
        "24                        NaN                      1           0      0   \n",
        "25                          8                      0           1      0   \n",
        "26                        NaN                      1           0      0   \n",
        "27                        NaN                      1           0      0   \n",
        "28                        113                      0           2      3   \n",
        "29                        NaN                      1           0      0   \n",
        "30                        NaN                      1           0      0   \n",
        "31                        NaN                      1           0      0   \n",
        "32                        NaN                      1           0      0   \n",
        "33                        NaN                      1           0      0   \n",
        "34                        NaN                      1           0      0   \n",
        "35                        NaN                      1           0      0   \n",
        "36                        NaN                      1           0      0   \n",
        "37                        NaN                      1           0      0   \n",
        "38                        NaN                      1           0      0   \n",
        "39                        NaN                      1           0      0   \n",
        "40                         38                      0           1      3   \n",
        "41                        NaN                      1           0      0   \n",
        "42                        NaN                      1           0      0   \n",
        "43                        NaN                      1           0      0   \n",
        "44                        NaN                      1           0      0   \n",
        "45                        NaN                      1           0      0   \n",
        "46                        NaN                      1           0      0   \n",
        "47                        NaN                      1           0      0   \n",
        "48                        NaN                      1           0      0   \n",
        "49                        NaN                      1           0      0   \n",
        "50                        NaN                      1           0      0   \n",
        "51                        NaN                      1           0      0   \n",
        "52                        NaN                      1           0      0   \n",
        "53                        NaN                      1           0      0   \n",
        "54                        NaN                      1           0      0   \n",
        "55                        NaN                      1           0      0   \n",
        "56                        NaN                      1           0      0   \n",
        "57                         56                      0           1      0   \n",
        "58                         84                      0           1      0   \n",
        "59                        NaN                      1           0      0   \n",
        "                          ...                    ...         ...    ...   \n",
        "\n",
        "    Corrects                                        KC(Default)      \n",
        "0          1  [SkillRule: Eliminate Parens; {CLT nested; CLT... ...  \n",
        "1          1  [SkillRule: Remove constant; {ax+b=c, positive... ...  \n",
        "2          1  [SkillRule: Remove constant; {ax+b=c, positive... ...  \n",
        "3          1  [SkillRule: Remove coefficient; {ax+b=c, divid... ...  \n",
        "4          1  [SkillRule: Remove constant; {ax+b=c, positive... ...  \n",
        "5          1  [SkillRule: Remove coefficient; {ax+b=c, divid... ...  \n",
        "6          1  [SkillRule: Remove constant; {ax+b=c, positive... ...  \n",
        "7          1  [SkillRule: Remove positive coefficient; {ax/b... ...  \n",
        "8          1  [SkillRule: Remove constant; {ax+b=c, positive... ...  \n",
        "9          1  [SkillRule: Remove positive coefficient; {ax/b... ...  \n",
        "10         1  [SkillRule: Remove constant; {ax+b=c, positive... ...  \n",
        "11         2  [SkillRule: Add/Subtract; [Typein Skill: {Isol... ...  \n",
        "12         1  [SkillRule: Remove coefficient; {ax+b=c, divid... ...  \n",
        "13         2  [SkillRule: Multiply/Divide; [Typein Skill: {R... ...  \n",
        "14         1  [SkillRule: Remove constant; {ax+b=c, positive... ...  \n",
        "15         2  [SkillRule: Add/Subtract; [Typein Skill: {Isol... ...  \n",
        "16         1  [SkillRule: Remove positive coefficient; {ax/b... ...  \n",
        "17         2  [SkillRule: Multiply/Divide; [Typein Skill: {R... ...  \n",
        "18         1  [SkillRule: Remove constant; {ax+b=c, positive... ...  \n",
        "19         2  [SkillRule: Add/Subtract; [Typein Skill: {Isol... ...  \n",
        "20         1  [SkillRule: Remove coefficient; {ax+b=c, divid... ...  \n",
        "21         2  [SkillRule: Multiply/Divide; [Typein Skill: {R... ...  \n",
        "22         1  [SkillRule: Remove constant; {ax+b=c, positive... ...  \n",
        "23         2  [SkillRule: Add/Subtract; [Typein Skill: {Isol... ...  \n",
        "24         1      [SkillRule: Consolidate vars with coeff; CLT] ...  \n",
        "25         1                              combine-like-terms-sp ...  \n",
        "26         1  [SkillRule: Remove positive coefficient; {ax/b... ...  \n",
        "27         2  [SkillRule: Multiply/Divide; [Typein Skill: {R... ...  \n",
        "28         1  [SkillRule: Eliminate Parens; {CLT nested; CLT... ...  \n",
        "29         1  [SkillRule: Remove constant; {ax+b=c, positive... ...  \n",
        "30         1  [SkillRule: Eliminate Parens; {CLT nested; CLT... ...  \n",
        "31         1  [SkillRule: Remove constant; {ax+b=c, positive... ...  \n",
        "32         1  [SkillRule: Eliminate Parens; {CLT nested; CLT... ...  \n",
        "33         1  [SkillRule: Remove constant; {ax+b=c, positive... ...  \n",
        "34         1  [SkillRule: Eliminate Parens; {CLT nested; CLT... ...  \n",
        "35         1  [SkillRule: Remove constant; {ax+b=c, positive... ...  \n",
        "36         1  [SkillRule: Eliminate Parens; {CLT nested; CLT... ...  \n",
        "37         2  [SkillRule: Calculate Eliminate Parens; [Typei... ...  \n",
        "38         1  [SkillRule: Remove constant; {ax+b=c, positive... ...  \n",
        "39         2  [SkillRule: Add/Subtract; [Typein Skill: {Isol... ...  \n",
        "40         1                                                NaN ...  \n",
        "41         2       simplify-fractions-sp~~combine-like-terms-sp ...  \n",
        "42         1       [SkillRule: Consolidate vars, no coeff; CLT] ...  \n",
        "43         1  [SkillRule: Eliminate Parens; {CLT nested; CLT... ...  \n",
        "44         2  [SkillRule: Calculate Eliminate Parens; [Typei... ...  \n",
        "45         1  [SkillRule: Remove constant; {ax+b=c, positive... ...  \n",
        "46         2  [SkillRule: Add/Subtract; [Typein Skill: {Isol... ...  \n",
        "47         1       [SkillRule: Consolidate vars, no coeff; CLT] ...  \n",
        "48         1       simplify-fractions-sp~~combine-like-terms-sp ...  \n",
        "49         1  [SkillRule: Eliminate Parens; {CLT nested; CLT... ...  \n",
        "50         2  [SkillRule: Calculate Eliminate Parens; [Typei... ...  \n",
        "51         1  [SkillRule: Remove constant; {ax+b=c, positive... ...  \n",
        "52         2  [SkillRule: Add/Subtract; [Typein Skill: {Isol... ...  \n",
        "53         1       [SkillRule: Consolidate vars, no coeff; CLT] ...  \n",
        "54         1       simplify-fractions-sp~~combine-like-terms-sp ...  \n",
        "55         1  [SkillRule: Eliminate Parens; {CLT nested; CLT... ...  \n",
        "56         1  [SkillRule: Remove constant; {ax+b=c, positive... ...  \n",
        "57         1  [SkillRule: Eliminate Parens; {CLT nested; CLT... ...  \n",
        "58         1  [SkillRule: Remove constant; {ax+b=c, positive... ...  \n",
        "59         1  [SkillRule: Eliminate Parens; {CLT nested; CLT... ...  \n",
        "         ...                                                ...      \n",
        "\n",
        "[809694 rows x 21 columns]"
       ]
      }
     ],
     "prompt_number": 116
    },
    {
     "cell_type": "heading",
     "level": 1,
     "metadata": {},
     "source": [
      "Create the testing dataframe"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# Create an empty testing dataframe\n",
      "testdf = pd.DataFrame(columns=df.columns)\n",
      "\n",
      "# Create the testing set\n",
      "unique_units = list(set(df['Problem Unit']))\n",
      "for i in range(len(unique_units)):\n",
      "    # Get the last problem of the current problem unit\n",
      "    lastProb = list(df[df['Problem Unit'] == unique_units[i]]['Problem Name'])[-1]\n",
      "    \n",
      "    # Get all the rows corresponding to the last problem for the given problem unit\n",
      "    lastProbRows = a56data[(df['Problem Unit'] == unique_units[i]) & (df['Problem Name']==lastProb)]\n",
      "    \n",
      "    # Concatenate test dataframe with the rows just found\n",
      "    testdf = pd.concat([testdf,lastProbRows])"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 118
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# Create a training dataframe that is equal to original dataframe with all the test cases removed\n",
      "trainIndex = df.index - testdf.index\n",
      "traindf = df.loc[trainIndex]"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 119
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# Get the target feature within the test set: the Correct First Attmpt\n",
      "CFAs = np.array(testdf['Correct First Attempt'])"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 120
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "# Test functions"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# Define a helper function for calculating the root-mean-square error\n",
      "def RMSE(p,y):\n",
      "    ''' The Root-Mean-Square Error takes the predicted values p for the target\n",
      "    variable y and takes the square root of the mean of the square of their\n",
      "    differences. '''\n",
      "    return np.sqrt(np.sum(np.square(p-y))/len(y))"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 121
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# Test the RMSE for an array of all zeros\n",
      "p = np.zeros(len(CFAs))\n",
      "print 'An array of all zeros gives an RMSE of:',RMSE(p,CFAs)\n",
      "\n",
      "# Test the RMSE for an array of all ones\n",
      "p = np.ones(len(CFAs))\n",
      "print 'An array of all ones gives an RMSE of:',RMSE(p,CFAs)\n",
      "\n",
      "# Test the RMSE for an array of random 0s and 1s\n",
      "p = np.random.randint(0,2,len(CFAs)).astype(float)\n",
      "print 'An array of random ones and zeros gives an RMSE of:',RMSE(p,CFAs)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "An array of all zeros gives an RMSE of: 0.863841709437\n",
        "An array of all ones gives an RMSE of: 0.503763338322\n",
        "An array of random ones and zeros gives an RMSE of: 0.70685723912\n"
       ]
      }
     ],
     "prompt_number": 122
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "def error_metrics(p,yy):\n",
      "    '''Calculates the error metrics, i.e. the precision and recall.\n",
      "    Precision = True positives / Predicted positives\n",
      "    Recall    = True positives / Actual positives'''\n",
      "    predicted_positives = len(p[p==1])\n",
      "    actual_positives    = len(yy[yy==1])\n",
      "    # The predicted values for when actual values are 1\n",
      "    pp = p[yy==1]\n",
      "    # True positives are when these predicted values are also 1\n",
      "    true_positives      = len(pp[pp==1])\n",
      "    false_positives     = len(yy) - true_positives\n",
      "    \n",
      "    precision = float(true_positives) / float(predicted_positives)\n",
      "    recall    = float(true_positives) / float(actual_positives)\n",
      "    \n",
      "    F_1score  = 2.0 * precision * recall / (precision + recall)\n",
      "    \n",
      "    print 'Root-mean-square error: ', RMSE(p,yy)\n",
      "    \n",
      "    print '\\nPrecision: Of all predicted CFAs, what fraction actually succeeded?'\n",
      "    print precision\n",
      "    \n",
      "    print '\\nRecall: Of all actual CFAs, what fraction did we predict correctly?'\n",
      "    print recall\n",
      "    \n",
      "    print '\\nF_1 Score: ', F_1score"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 125
    },
    {
     "cell_type": "heading",
     "level": 1,
     "metadata": {},
     "source": [
      "Machine learning"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "traindf.columns"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 20,
       "text": [
        "Index([u'Row', u'Anon Student Id', u'Problem Hierarchy', u'Problem Unit', u'Problem Section', u'Problem Name', u'Problem View', u'Step Name', u'Step Start Time', u'First Transaction Time', u'Correct Transaction Time', u'Step End Time', u'Step Duration (sec)', u'Correct Step Duration (sec)', u'Error Step Duration (sec)', u'Correct First Attempt', u'Incorrects', u'Hints', u'Corrects', u'KC(Default)', u'Opportunity(Default)'], dtype='object')"
       ]
      }
     ],
     "prompt_number": 20
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# Define a helper function to normalize the feature matrix X\n",
      "import numba\n",
      "def autonorm(X):\n",
      "    ''' Calculates the mean and range of values of each column\n",
      "    in the matrix (features) subtracts the mean from each value\n",
      "    and divides by the range, thereby normalizing all values to\n",
      "    fall between -1 and 1.'''\n",
      "    x_means = np.mean(X,axis=0)\n",
      "    x_means = np.ones(np.shape(X))*x_means\n",
      "    x_maxs  = np.max(X,axis=0)\n",
      "    x_mins  = np.min(X,axis=0)\n",
      "    x_range = x_maxs - x_mins\n",
      "    X_normd = (X - x_means) / x_range\n",
      "    return X_normd\n",
      "\n",
      "autonorm_jit = numba.jit(autonorm)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 127
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "features_to_norm = ['Step Duration (sec)','Hints','Problem View']\n",
      "category_features = ['Anon Student Id', 'Problem Hierarchy', 'Problem Unit', 'Problem Section', 'Problem Name']\n",
      "target_feature = ['Correct First Attempt']\n",
      "features = features_to_norm + category_features\n",
      "all_features = features_to_norm + category_features + target_feature"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 150
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "X = traindf[all_features].dropna()\n",
      "y = np.array(X[target_feature]).astype(int).ravel()\n",
      "X_to_norm = np.array(X[features_to_norm])\n",
      "X_nonnorm = np.array(X[category_features])\n",
      "X_to_norm = autonorm(X_to_norm)\n",
      "X = np.concatenate((X_to_norm,X_nonnorm), axis=1)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 129
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "XX = testdf[all_features].dropna()\n",
      "yy = np.array(XX[target_feature]).astype(int).ravel()\n",
      "XX_to_norm = np.array(XX[features_to_norm])\n",
      "XX_nonnorm = np.array(XX[category_features])\n",
      "XX_to_norm = autonorm(XX_to_norm)\n",
      "XX = np.concatenate((XX_to_norm,XX_nonnorm), axis=1)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 130
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "from sklearn import tree\n",
      "model = tree.DecisionTreeClassifier()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 131
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "model = model.fit(X,y)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 133
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "p = model.predict(XX).astype(float)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 134
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "error_metrics(p,yy)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Root-mean-square error:  0.59582966275\n",
        "\n",
        "Precision: Of all predicted CFAs, what fraction actually succeeded?\n",
        "0.823697236354\n",
        "\n",
        "Recall: Of all actual CFAs, what fraction did we predict correctly?\n",
        "0.667139953785\n",
        "\n",
        "F_1 Score:  0.737198320284\n"
       ]
      }
     ],
     "prompt_number": 135
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "scores = cross_val_score(model, X, y)\n",
      "print 'Accuracy: {0:5.2f} (+/-{1:5.2f})'.format(scores.mean(), scores.std()*2)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Accuracy:  0.78 (+/- 0.01)\n"
       ]
      }
     ],
     "prompt_number": 139
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "from sklearn.ensemble import RandomForestClassifier\n",
      "\n",
      "n_ests = 70\n",
      "model = RandomForestClassifier(n_estimators=n_ests, criterion=\"entropy\", max_features=None)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 155
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "model = model.fit(X,y)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 156
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "p = model.predict(XX).astype(float)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 157
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "error_metrics(p,yy)\n",
      "importances = model.feature_importances_\n",
      "n_feats = len(features)\n",
      "feat_std = np.std([tree.feature_importances_ for tree in model.estimators_],axis=0)\n",
      "indices = np.argsort(importances)[::-1]\n",
      "\n",
      "# Print the feature ranking\n",
      "print(\"\\nFeature ranking:\")\n",
      "\n",
      "for f in range(n_feats):\n",
      "    print '{0:2} - {1:20}: {2:5.4f} (std: {3:5.4f})'.format(f+1,features[indices[f]],importances[indices[f]],feat_std[indices[f]])"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Root-mean-square error:  0.500098691559\n",
        "\n",
        "Precision: Of all predicted CFAs, what fraction actually succeeded?\n",
        "0.834598896018\n",
        "\n",
        "Recall: Of all actual CFAs, what fraction did we predict correctly?\n",
        "0.829254711991\n",
        "\n",
        "F_1 Score:  0.831918221428\n",
        "\n",
        "Feature ranking:"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        " 1 - Anon Student Id     : 0.2705 (std: 0.0026)\n",
        " 2 - Hints               : 0.2593 (std: 0.0011)\n",
        " 3 - Step Duration (sec) : 0.2261 (std: 0.0032)\n",
        " 4 - Problem Name        : 0.1125 (std: 0.0022)\n",
        " 5 - Problem View        : 0.0444 (std: 0.0010)\n",
        " 6 - Problem Section     : 0.0358 (std: 0.0014)\n",
        " 7 - Problem Hierarchy   : 0.0306 (std: 0.0011)\n",
        " 8 - Problem Unit        : 0.0208 (std: 0.0011)\n"
       ]
      }
     ],
     "prompt_number": 161
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "scores = cross_val_score(model, X, y)\n",
      "print 'Accuracy: {0:5.2f} (+/-{1:5.2f})'.format(scores.mean(), scores.std()*2)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "ename": "KeyboardInterrupt",
       "evalue": "",
       "output_type": "pyerr",
       "traceback": [
        "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m\n\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
        "\u001b[0;32m<ipython-input-162-8de7c3d2ecea>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mscores\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcross_val_score\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mX\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m \u001b[0;32mprint\u001b[0m \u001b[0;34m'Accuracy: {0:5.2f} (+/-{1:5.2f})'\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mscores\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmean\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mscores\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstd\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0;36m2\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
        "\u001b[0;32m/Users/mikhail/anaconda/lib/python2.7/site-packages/sklearn/cross_validation.pyc\u001b[0m in \u001b[0;36mcross_val_score\u001b[0;34m(estimator, X, y, scoring, cv, n_jobs, verbose, fit_params, score_func, pre_dispatch)\u001b[0m\n\u001b[1;32m   1149\u001b[0m                                               \u001b[0mtrain\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtest\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mverbose\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mNone\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1150\u001b[0m                                               fit_params)\n\u001b[0;32m-> 1151\u001b[0;31m                       for train, test in cv)\n\u001b[0m\u001b[1;32m   1152\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0marray\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mscores\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1153\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
        "\u001b[0;32m/Users/mikhail/anaconda/lib/python2.7/site-packages/sklearn/externals/joblib/parallel.pyc\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, iterable)\u001b[0m\n\u001b[1;32m    651\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_iterating\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mTrue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    652\u001b[0m             \u001b[0;32mfor\u001b[0m \u001b[0mfunction\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwargs\u001b[0m \u001b[0;32min\u001b[0m \u001b[0miterable\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 653\u001b[0;31m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdispatch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfunction\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    654\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    655\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mpre_dispatch\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m\"all\"\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mn_jobs\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
        "\u001b[0;32m/Users/mikhail/anaconda/lib/python2.7/site-packages/sklearn/externals/joblib/parallel.pyc\u001b[0m in \u001b[0;36mdispatch\u001b[0;34m(self, func, args, kwargs)\u001b[0m\n\u001b[1;32m    398\u001b[0m         \"\"\"\n\u001b[1;32m    399\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_pool\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 400\u001b[0;31m             \u001b[0mjob\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mImmediateApply\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfunc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    401\u001b[0m             \u001b[0mindex\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_jobs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    402\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0m_verbosity_filter\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mindex\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mverbose\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
        "\u001b[0;32m/Users/mikhail/anaconda/lib/python2.7/site-packages/sklearn/externals/joblib/parallel.pyc\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, func, args, kwargs)\u001b[0m\n\u001b[1;32m    136\u001b[0m         \u001b[0;31m# Don't delay the application, to avoid keeping the input\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    137\u001b[0m         \u001b[0;31m# arguments in memory\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 138\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mresults\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    139\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    140\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
        "\u001b[0;32m/Users/mikhail/anaconda/lib/python2.7/site-packages/sklearn/cross_validation.pyc\u001b[0m in \u001b[0;36m_fit_and_score\u001b[0;34m(estimator, X, y, scorer, train, test, verbose, parameters, fit_params, return_train_score, return_parameters)\u001b[0m\n\u001b[1;32m   1237\u001b[0m         \u001b[0mestimator\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX_train\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mfit_params\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1238\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1239\u001b[0;31m         \u001b[0mestimator\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX_train\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my_train\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mfit_params\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1240\u001b[0m     \u001b[0mtest_score\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_score\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mestimator\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mX_test\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my_test\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mscorer\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1241\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mreturn_train_score\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
        "\u001b[0;32m/Users/mikhail/anaconda/lib/python2.7/site-packages/sklearn/ensemble/forest.pyc\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, X, y, sample_weight)\u001b[0m\n\u001b[1;32m    277\u001b[0m                 \u001b[0msample_weight\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    278\u001b[0m                 verbose=self.verbose)\n\u001b[0;32m--> 279\u001b[0;31m             for i in range(n_jobs))\n\u001b[0m\u001b[1;32m    280\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    281\u001b[0m         \u001b[0;31m# Reduce\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
        "\u001b[0;32m/Users/mikhail/anaconda/lib/python2.7/site-packages/sklearn/externals/joblib/parallel.pyc\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, iterable)\u001b[0m\n\u001b[1;32m    651\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_iterating\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mTrue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    652\u001b[0m             \u001b[0;32mfor\u001b[0m \u001b[0mfunction\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwargs\u001b[0m \u001b[0;32min\u001b[0m \u001b[0miterable\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 653\u001b[0;31m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdispatch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfunction\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    654\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    655\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mpre_dispatch\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m\"all\"\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mn_jobs\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
        "\u001b[0;32m/Users/mikhail/anaconda/lib/python2.7/site-packages/sklearn/externals/joblib/parallel.pyc\u001b[0m in \u001b[0;36mdispatch\u001b[0;34m(self, func, args, kwargs)\u001b[0m\n\u001b[1;32m    398\u001b[0m         \"\"\"\n\u001b[1;32m    399\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_pool\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 400\u001b[0;31m             \u001b[0mjob\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mImmediateApply\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfunc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    401\u001b[0m             \u001b[0mindex\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_jobs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    402\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0m_verbosity_filter\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mindex\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mverbose\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
        "\u001b[0;32m/Users/mikhail/anaconda/lib/python2.7/site-packages/sklearn/externals/joblib/parallel.pyc\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, func, args, kwargs)\u001b[0m\n\u001b[1;32m    136\u001b[0m         \u001b[0;31m# Don't delay the application, to avoid keeping the input\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    137\u001b[0m         \u001b[0;31m# arguments in memory\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 138\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mresults\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    139\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    140\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
        "\u001b[0;32m/Users/mikhail/anaconda/lib/python2.7/site-packages/sklearn/ensemble/forest.pyc\u001b[0m in \u001b[0;36m_parallel_build_trees\u001b[0;34m(trees, forest, X, y, sample_weight, verbose)\u001b[0m\n\u001b[1;32m     87\u001b[0m             tree.fit(X, y,\n\u001b[1;32m     88\u001b[0m                      \u001b[0msample_weight\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcurr_sample_weight\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 89\u001b[0;31m                      check_input=False)\n\u001b[0m\u001b[1;32m     90\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     91\u001b[0m             \u001b[0mtree\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mindices_\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msample_counts\u001b[0m \u001b[0;34m>\u001b[0m \u001b[0;36m0.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
        "\u001b[0;32m/Users/mikhail/anaconda/lib/python2.7/site-packages/sklearn/tree/tree.pyc\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, X, y, sample_mask, X_argsorted, check_input, sample_weight)\u001b[0m\n\u001b[1;32m    265\u001b[0m                                            max_leaf_nodes)\n\u001b[1;32m    266\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 267\u001b[0;31m         \u001b[0mbuilder\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbuild\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtree_\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mX\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msample_weight\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    268\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    269\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mn_outputs_\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
        "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
       ]
      }
     ],
     "prompt_number": 162
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [],
     "language": "python",
     "metadata": {},
     "outputs": []
    }
   ],
   "metadata": {}
  }
 ]
}