{
 "metadata": {
  "name": "",
  "signature": "sha256:42df5f46b05d4fc6e4f3c223cf21fcdf01441c8c91dd3db2a4f1f22095e979b9"
 },
 "nbformat": 3,
 "nbformat_minor": 0,
 "worksheets": [
  {
   "cells": [
    {
     "cell_type": "heading",
     "level": 1,
     "metadata": {},
     "source": [
      "Otto Group Product Classification Challenge using nolearn/lasagne"
     ]
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "This short notebook is meant to help you getting started with nolearn and lasagne in order to train a neural net and make a submission to the Otto Group Product Classification Challenge.\n",
      "\n",
      "* [Otto Group Product Classification Challenge](https://www.kaggle.com/c/otto-group-product-classification-challenge)\n",
      "* [Get the notebook from the Otto Group repository](https://github.com/ottogroup)\n",
      "* [Nolearn repository](https://github.com/dnouri/nolearn)\n",
      "* [Lasagne repository](https://github.com/benanne/Lasagne)\n",
      "* [A nolearn/lasagne tutorial for convolutional nets](http://danielnouri.org/notes/2014/12/17/using-convolutional-neural-nets-to-detect-facial-keypoints-tutorial/)"
     ]
    },
    {
     "cell_type": "heading",
     "level": 2,
     "metadata": {},
     "source": [
      "Imports"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "import numpy as np\n",
      "import pandas as pd\n",
      "from sklearn.preprocessing import LabelEncoder\n",
      "from sklearn.preprocessing import StandardScaler"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 1
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "from lasagne.layers import DenseLayer\n",
      "from lasagne.layers import InputLayer\n",
      "from lasagne.layers import DropoutLayer\n",
      "from lasagne.nonlinearities import softmax\n",
      "from lasagne.updates import nesterov_momentum\n",
      "from nolearn.lasagne import NeuralNet"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 2
    },
    {
     "cell_type": "heading",
     "level": 2,
     "metadata": {},
     "source": [
      "Utility functions"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "def load_train_data(path):\n",
      "    df = pd.read_csv(path)\n",
      "    X = df.values.copy()\n",
      "    np.random.shuffle(X)\n",
      "    X, labels = X[:, 1:-1].astype(np.float32), X[:, -1]\n",
      "    encoder = LabelEncoder()\n",
      "    y = encoder.fit_transform(labels).astype(np.int32)\n",
      "    scaler = StandardScaler()\n",
      "    X = scaler.fit_transform(X)\n",
      "    return X, y, encoder, scaler"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 3
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "def load_test_data(path, scaler):\n",
      "    df = pd.read_csv(path)\n",
      "    X = df.values.copy()\n",
      "    X, ids = X[:, 1:].astype(np.float32), X[:, 0].astype(str)\n",
      "    X = scaler.transform(X)\n",
      "    return X, ids"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 4
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "def make_submission(clf, X_test, ids, encoder, name='my_neural_net_submission.csv'):\n",
      "    y_prob = clf.predict_proba(X_test)\n",
      "    with open(name, 'w') as f:\n",
      "        f.write('id,')\n",
      "        f.write(','.join(encoder.classes_))\n",
      "        f.write('\\n')\n",
      "        for id, probs in zip(ids, y_prob):\n",
      "            probas = ','.join([id] + map(str, probs.tolist()))\n",
      "            f.write(probas)\n",
      "            f.write('\\n')\n",
      "    print(\"Wrote submission to file {}.\".format(name))"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 5
    },
    {
     "cell_type": "heading",
     "level": 2,
     "metadata": {},
     "source": [
      "Load Data"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "X, y, encoder, scaler = load_train_data('data/train.csv')"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 6
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "X_test, ids = load_test_data('data/test.csv', scaler)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 7
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "num_classes = len(encoder.classes_)\n",
      "num_features = X.shape[1]"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 8
    },
    {
     "cell_type": "heading",
     "level": 2,
     "metadata": {},
     "source": [
      "Train Neural Net"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "layers0 = [('input', InputLayer),\n",
      "           ('dense0', DenseLayer),\n",
      "           ('dropout', DropoutLayer),\n",
      "           ('dense1', DenseLayer),\n",
      "           ('output', DenseLayer)]"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 9
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "net0 = NeuralNet(layers=layers0,\n",
      "                 \n",
      "                 input_shape=(None, num_features),\n",
      "                 dense0_num_units=200,\n",
      "                 dropout_p=0.5,\n",
      "                 dense1_num_units=200,\n",
      "                 output_num_units=num_classes,\n",
      "                 output_nonlinearity=softmax,\n",
      "                 \n",
      "                 update=nesterov_momentum,\n",
      "                 update_learning_rate=0.01,\n",
      "                 update_momentum=0.9,\n",
      "                 \n",
      "                 eval_size=0.2,\n",
      "                 verbose=1,\n",
      "                 max_epochs=20)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 10
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "net0.fit(X, y)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "  input             \t(None, 93L)         \tproduces      93 outputs\n",
        "  dense0            \t(None, 200)         \tproduces     200 outputs\n",
        "  dropout           \t(None, 200)         \tproduces     200 outputs\n",
        "  dense1            \t(None, 200)         \tproduces     200 outputs\n",
        "  output            \t(None, 9)           \tproduces       9 outputs\n",
        "\n",
        " Epoch  |  Train loss  |  Valid loss  |  Train / Val  |  Valid acc  |  Dur\n",
        "--------|--------------|--------------|---------------|-------------|-------"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "     1  |  \u001b[94m  0.929630\u001b[0m  |  \u001b[32m  0.674008\u001b[0m  |     1.379258  |     75.32%  |  1.6s"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "     2  |  \u001b[94m  0.706279\u001b[0m  |  \u001b[32m  0.627249\u001b[0m  |     1.125994  |     76.74%  |  1.2s"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "     3  |  \u001b[94m  0.665223\u001b[0m  |  \u001b[32m  0.602289\u001b[0m  |     1.104491  |     76.96%  |  1.2s"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "     4  |  \u001b[94m  0.640945\u001b[0m  |  \u001b[32m  0.587731\u001b[0m  |     1.090540  |     77.73%  |  1.2s"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "     5  |  \u001b[94m  0.623396\u001b[0m  |  \u001b[32m  0.577516\u001b[0m  |     1.079444  |     77.79%  |  1.5s"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "     6  |  \u001b[94m  0.612419\u001b[0m  |  \u001b[32m  0.568359\u001b[0m  |     1.077520  |     77.85%  |  1.3s"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "     7  |  \u001b[94m  0.599983\u001b[0m  |  \u001b[32m  0.566710\u001b[0m  |     1.058712  |     78.34%  |  1.3s"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "     8  |  \u001b[94m  0.592689\u001b[0m  |  \u001b[32m  0.559909\u001b[0m  |     1.058544  |     78.17%  |  1.2s"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "     9  |  \u001b[94m  0.581145\u001b[0m  |  \u001b[32m  0.557094\u001b[0m  |     1.043171  |     78.57%  |  1.6s"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "    10  |  \u001b[94m  0.578483\u001b[0m  |  \u001b[32m  0.553404\u001b[0m  |     1.045318  |     78.39%  |  1.8s"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "    11  |  \u001b[94m  0.568782\u001b[0m  |  \u001b[32m  0.550518\u001b[0m  |     1.033176  |     78.74%  |  1.8s"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "    12  |  \u001b[94m  0.563738\u001b[0m  |  \u001b[32m  0.546480\u001b[0m  |     1.031582  |     78.88%  |  1.8s"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "    13  |  \u001b[94m  0.559736\u001b[0m  |  \u001b[32m  0.542942\u001b[0m  |     1.030933  |     78.92%  |  1.9s"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "    14  |  \u001b[94m  0.554280\u001b[0m  |  \u001b[32m  0.540654\u001b[0m  |     1.025202  |     79.07%  |  1.8s"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "    15  |  \u001b[94m  0.553027\u001b[0m  |  \u001b[32m  0.536958\u001b[0m  |     1.029927  |     79.03%  |  1.8s"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "    16  |  \u001b[94m  0.548043\u001b[0m  |    0.537584  |     1.019456  |     79.13%  |  1.9s"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "    17  |  \u001b[94m  0.547400\u001b[0m  |  \u001b[32m  0.534357\u001b[0m  |     1.024408  |     79.39%  |  1.8s"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "    18  |  \u001b[94m  0.544515\u001b[0m  |  \u001b[32m  0.532123\u001b[0m  |     1.023289  |     79.27%  |  1.9s"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "    19  |  \u001b[94m  0.537075\u001b[0m  |  \u001b[32m  0.531905\u001b[0m  |     1.009719  |     79.42%  |  1.8s"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "    20  |  \u001b[94m  0.536718\u001b[0m  |  \u001b[32m  0.531486\u001b[0m  |     1.009845  |     79.16%  |  2.2s"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n"
       ]
      },
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 11,
       "text": [
        "NeuralNet(X_tensor_type=<function matrix at 0x0000000013F15518>,\n",
        "     batch_iterator_test=<nolearn.lasagne.BatchIterator object at 0x000000001492D470>,\n",
        "     batch_iterator_train=<nolearn.lasagne.BatchIterator object at 0x00000000147E3AC8>,\n",
        "     dense0_num_units=200, dense1_num_units=200, dropout_p=0.5,\n",
        "     eval_size=0.2, input_shape=(None, 93L),\n",
        "     layers=[('input', <class 'lasagne.layers.input.InputLayer'>), ('dense0', <class 'lasagne.layers.dense.DenseLayer'>), ('dropout', <class 'lasagne.layers.noise.DropoutLayer'>), ('dense1', <class 'lasagne.layers.dense.DenseLayer'>), ('output', <class 'lasagne.layers.dense.DenseLayer'>)],\n",
        "     loss=None, max_epochs=20, more_params={},\n",
        "     objective=<nolearn.lasagne.LossObjective object at 0x000000001492DDD8>,\n",
        "     on_epoch_finished=(), on_training_finished=(),\n",
        "     output_nonlinearity=<theano.tensor.nnet.nnet.Softmax object at 0x00000000141FB828>,\n",
        "     output_num_units=9, regression=False,\n",
        "     update=<function nesterov_momentum at 0x00000000147A8E48>,\n",
        "     update_learning_rate=0.01, update_momentum=0.9,\n",
        "     use_label_encoder=False, verbose=1,\n",
        "     y_tensor_type=TensorType(int32, vector))"
       ]
      }
     ],
     "prompt_number": 11
    },
    {
     "cell_type": "heading",
     "level": 2,
     "metadata": {},
     "source": [
      "Prepare Submission File"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "make_submission(net0, X_test, ids, encoder)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Wrote submission to file my_neural_net_submission.csv.\n"
       ]
      }
     ],
     "prompt_number": 12
    }
   ],
   "metadata": {}
  }
 ]
}