{
 "metadata": {
  "name": ""
 },
 "nbformat": 3,
 "nbformat_minor": 0,
 "worksheets": [
  {
   "cells": [
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "## Try different feature extraction strategy for binary classification problems\n",
      "- data from sklearn-london workshop hosted in [Kaggle](http://www.kaggle.com/c/data-science-london-scikit-learn)\n",
      "- Things to try:\n",
      "    - discretization of features - by decision tree and log odd coding\n",
      "    - sparse filtering & forests feature selection"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "import numpy as np\n",
      "import pandas as pd\n",
      "from sklearn.utils import shuffle\n",
      "from sklearn.base import BaseEstimator, TransformerMixin\n",
      "from sklearn.cross_validation import train_test_split\n",
      "from sklearn.preprocessing import StandardScaler, normalize\n",
      "from sklearn.linear_model import SGDClassifier\n",
      "from sklearn.svm import SVC\n",
      "from sklearn.grid_search import GridSearchCV\n",
      "from sklearn.ensemble import ExtraTreesClassifier\n",
      "from sklearn.ensemble import GradientBoostingClassifier\n",
      "from sklearn.decomposition import PCA\n",
      "\n",
      "\n",
      "from autodiff import optimize\n",
      "from scipy import stats\n",
      "import pylab as pl\n",
      "\n",
      "%pylab inline --no-import-all"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Populating the interactive namespace from numpy and matplotlib\n"
       ]
      }
     ],
     "prompt_number": 2
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "## load data\n",
      "train_data = pd.read_csv('../data/sklearn-london/train.csv', header=None)\n",
      "train_labels = pd.read_csv('../data/sklearn-london/trainLabels.csv', header=None)\n",
      "\n",
      "X = np.asarray(train_data)\n",
      "y = np.asarray(train_labels).ravel()\n",
      "\n",
      "print X.shape, y.shape\n",
      "print X.dtype, y.dtype, np.unique(y)\n",
      "\n",
      "## shuffle data\n",
      "X, y = shuffle(X, y)\n",
      "print X.shape, y.shape"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "(1000, 40) (1000,)\n",
        "float64 int64 [0 1]\n",
        "(1000, 40) (1000,)\n"
       ]
      }
     ],
     "prompt_number": 7
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "train_index, test_index = train_test_split(np.arange(X.shape[0]), test_size = 0.2)\n",
      "print train_index.shape, test_index.shape"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "(800,) (200,)\n"
       ]
      }
     ],
     "prompt_number": 8
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "## base case - on raw features"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "## analzye min/max, center, shape of different features\n",
      "def statistics(feat):\n",
      "    return (feat.min(), feat.max(), feat.std(), stats.skew(feat))\n",
      "mins, maxs, stds, skewness = zip(*[statistics(X[:, i]) for i in xrange(X.shape[1])])\n",
      "print np.std(mins), np.min(mins), np.max(mins)\n",
      "print np.std(maxs), np.min(maxs), np.max(maxs)\n",
      "print np.std(stds), np.min(stds), np.max(stds)\n",
      "print np.std(skewness), np.min(skewness), np.max(skewness)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "3.52909765249 -16.4219014729 -2.6956019378\n",
        "3.29453132536 2.54650651811 17.5653445056\n",
        "0.967315549334 0.966798559976 4.53656367456\n",
        "0.135242790356 -0.412842893417 0.333415441978\n"
       ]
      }
     ],
     "prompt_number": 9
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "ss = StandardScaler()\n",
      "train_X = ss.fit_transform(X[train_index, :])\n",
      "test_X = ss.transform(X[test_index, :])\n",
      "train_y = y[train_index]\n",
      "test_y = y[test_index]\n",
      "\n",
      "svc = SVC()\n",
      "gammas = [1e-4, 3e-4, 1e-3, 3e-3, 1e-2, 3e-2, 1e-1, 3e-1, 1., 3., 10.]\n",
      "gs = GridSearchCV(svc, {'gamma': gammas}, n_jobs=-1, cv = 5, scoring='accuracy')\n",
      "gs.fit(train_X, train_y)\n",
      "print gs.best_params_\n",
      "print gs.best_score_\n",
      "\n",
      "svc = SVC(**gs.best_params_)\n",
      "svc.fit(train_X, train_y)\n",
      "print svc.score(test_X, test_y)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "{'gamma': 0.03}\n",
        "0.86125\n",
        "0.9"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n"
       ]
      }
     ],
     "prompt_number": 10
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "## Base case - on PCA"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "pca = PCA(whiten=True)\n",
      "pca.fit(X)\n",
      "feature_importances = pd.DataFrame(pca.explained_variance_ratio_)\n",
      "feature_importances.plot(kind = 'bar')"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 11,
       "text": [
        "<matplotlib.axes.AxesSubplot at 0x9346c10>"
       ]
      },
      {
       "metadata": {},
       "output_type": "display_data",
       "png": "iVBORw0KGgoAAAANSUhEUgAAAXYAAAEACAYAAACnJV25AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3X9UVHX6B/A3Bq0VP0RLM4ZEBfktsCJY6TdOmqTHKLUt\nTC0NjCy33KOlttsqtSelss0fp5VtS09qZJmJ7eqsmdy0UsjUaMNMWwnEnykEYong8/0DmWUGuDPM\nDPi58H6dM6e58/nc+zzzmemZmWcuo4eICIiIqMPocqUTICIi92JhJyLqYFjYiYg6GBZ2IqIOhoWd\niKiDYWEnIupg7BZ2s9mMsLAwhISEICsrq8l4bm4uYmJiEBcXh0GDBmH79u0O70tERO7noXcee11d\nHUJDQ7Ft2zYEBARg8ODByMnJQXh4uGVOdXU1rrvuOgDAN998g7Fjx+Lw4cMO7UtERO6n+469oKAA\nwcHBCAoKgpeXF1JTU5Gbm2s1p6GoA8C5c+dw/fXXO7wvERG5n6feYFlZGQIDAy3bJpMJ+fn5TeZt\n3LgR8+bNw/Hjx7F169ZW7evh4eF08kREnVlLDRfdd+yOFt17770XBw4cwEcffYTJkye3GEwvucaX\n+fPnN7mttXNcHWcMtWKokANjMIZKOejRLewBAQEoLS21bJeWlsJkMrU4f9iwYaitrcXZs2dhMpla\ntW9jxcXFLs9xdZwx1IqhQg6MwRiq5mBLt7DHx8fj0KFDKC4uRk1NDdatW4eUlBSrOT/88IPl1WPv\n3r0AgB49eji0LxERtQGxY/PmzTJgwADp37+/vPjiiyIismLFClmxYoWIiGRlZUlkZKTExsbK0KFD\npaCgQHdfW82lkJeXZy8tu3NcHWcMtWKokANjMIZKOeiVb93THduDh4eH3X4RERFZ06udSv7lqaZp\nLs9xdZwx1IqhQg6MYbwYfn5+8PDwMPSle/fuDq1FY7qnOxIRGVllZaXhOwLOnBLOVgwRdVgdob60\ndB8M14ohIiLnKVnYjdK/Y4z2i6FCDoxhvBgdRWvvp5KFnYiInMceOxF1WB2hvrDHTkRkh69v9zY9\nPdHXt7tDeZw9exZjx46Ft7c3goKCkJOT47b7qGRhN0r/jjHaL4YKOTCG8WI0p6qqHIC02aX++PY9\n8cQT6Nq1K06dOoW1a9di+vTpKCoqcsv9VLKwExF1ZNXV1diwYQNeeOEFXHvttbjttttwzz33YPXq\n1W45PnvsRNRhNVdf6v/gpy1rjv2atm/fPgwdOhTV1dWW21599VVomoZNmzZZH409diIi9Z07dw6+\nvr5Wt/n4+KCqqsotx1eysBulf8cY7RdDhRwYw3gxVOXt7Y3Kykqr237++Wf4+Pg0O589diIixQ0Y\nMAC1tbU4fPiw5bavv/4aUVFRbjk+e+xE1GGp2mMHgAkTJsDDwwP/+Mc/sHfvXowZMwa7du1CeHi4\n9dHYYyci0ufj4w/Ao80u9ce37/XXX8cvv/yCnj17YtKkSVixYkWTou4sJQu7Ufp3jNF+MVTIgTGM\nF6M5lZVn7f7j0a5cKivPOpSHv78/PvzwQ5w7dw7FxcVITU112/1UsrATEZHz2GMnog6rI9QX9tiJ\niEjNwm6U/h1jtF8MFXJgDOPF8PX1veL/ZqmrF39/f/bYiYga5Obm6n7RmZeXZ/fLUHtz2nr87FnH\nvoxtjD12IiIDYo+diKgTUbKwG6V/xxjtF0OFHBiDMVTNwZaShZ2IiJzHHjsRkQGxx05E1InYLexm\nsxlhYWEICQlBVlZWk/G1a9ciJiYGAwcOxG233YbCwkLLWFBQEAYOHIi4uDgkJCQ4nJQKfS/GUCuG\nCjkwBmOomoMtT73Buro6zJgxA9u2bUNAQAAGDx6MlJQUq18g69evH3bs2AE/Pz+YzWY8+uij2L17\nN4D6jwqapqF7d8f+1W4iInKdbo99165dyMzMhNlsBgAsWrQIADB37txm55eXlyM6OhpHjx4FAPTt\n2xd79uxBjx49Wk6APXYiolbTq52679jLysoQGBho2TaZTMjPz29x/ptvvonRo0dbBR4xYgSuuuoq\nZGRkYNq0ac3uN2XKFOTkrENNza/Njl9zjTfOn6//twAbPpIkJSVxm9vc5nan2dY0DatWrQJQ3+bW\nJTrWr18v6enplu3Vq1fLjBkzmp27fft2CQ8Pl7Nnz1puO3bsmIiInDp1SmJiYmTHjh1N9mtIAYAA\ncvmS1+j6/+Y0lpeXp5e6y+OMoVYMFXJgDMZQKQe98q375WlAQABKS0st26WlpTCZTE3mFRYWYtq0\nadi0aRP8/f/3r4f07t0bAHDDDTdg7NixKCgo0H+VISIil+n22GtraxEaGopPPvkEN910ExISEpCT\nk2P15WlJSQnuuOMOrFmzBkOGDLHcfv78edTV1cHHxwfV1dUYOXIk5s+fj5EjR1oncLlPpP/vELIP\nT0TUmNM9dk9PTyxfvhzJycmoq6tDWloawsPDkZ2dDQDIyMjA888/j/LyckyfPh0A4OXlhYKCApw4\ncQLjxo0DUP8CMXHixCZFnYiI2oDd5k4bA3vsjGGQHBiDMVTKQa988y9PiYg6GGV+K4Y9diIix/G3\nYoiIOhFFC7tmf4YCv8/AGO0XQ4UcGIMxVM3BlqKFnYiInMUeOxGRAbHHTkTUiSha2DX7MxToazFG\n+8VQIQfGYAxVc7ClaGEnIiJnscdORGRA7LETEXUiihZ2zf4MBfpajNF+MVTIgTEYQ9UcbCla2ImI\nyFnssRMRGRB77EREnYiihV2zP0OBvhZjtF8MFXJgDMZQNQdbihZ2IiJyFnvsREQGxB47EVEnomhh\n1+zPUKCvxRjtF0OFHBiDMVTNwZaihZ2IiJzFHjsRkQGxx05E1IkoWtg1+zMU6GsxRvvFUCEHxmAM\nVXOwpWhhJyIiZ7HHTkRkQOyxExF1IooWds3+DAX6WozRfjFUyIExGEPVHGx5tmr2Febr2x1VVeVN\nbvfx8Udl5dkrkBERkXrs9tjNZjNmzpyJuro6pKenY86cOVbja9euxUsvvQQRgY+PD/72t79h4MCB\nDu0LtK7H3vIc9uCJqHPR67FDdNTW1kr//v3lyJEjUlNTIzExMVJUVGQ154svvpCKigoREdmyZYsk\nJiY6vO/lFxXLfwFp4WJvju7dICLqcPTqnm6PvaCgAMHBwQgKCoKXlxdSU1ORm5trNeeWW26Bn58f\nACAxMRFHjx51eN+WaS7PMUJvjTGMlQNjMIaqOdjS7bGXlZUhMDDQsm0ymZCfn9/i/DfffBOjR49u\n9b5Tpky5fG0BgG6Xrydd/q9mM1t/23YBGraTkpJaNd5e2/v372/TcU3TsH///jYdb+xKjfPxcny8\nMT4eV/7xcPTx0jQNq1atAgAEBQVBj26P/YMPPoDZbMYbb7wBAFizZg3y8/OxbNmyJnPz8vLwxBNP\n4PPPP4e/v7/D+7LHTkTUeno9dt137AEBASgtLbVsl5aWwmQyNZlXWFiIadOmwWw2w9/fv1X7EhGR\ne+n22OPj43Ho0CEUFxejpqYG69atQ0pKitWckpISjBs3DmvWrEFwcHCr9m2Z5vIc24+MrR13xzEY\no2PlwBiMoWoOtnTfsXt6emL58uVITk5GXV0d0tLSEB4ejuzsbABARkYGnn/+eZSXl2P69OkAAC8v\nLxQUFLS4LxERtS1D/VYMe+xERPX4WzFERJ2IooVdc3mOEXprjGGsHBiDMVTNwZaihZ2IiJzFHjsR\nkQGxx05E1IkoWtg1l+cYobfGGMbKgTEYQ9UcbCla2ImIyFnssRMRGRB77EREnYiihV1zeY4RemuM\nYawcGIMxVM3BlqKFnYiInMUeOxGRAbHHTkTUiSha2DWX5xiht8YYxsqBMRhD1RxsKVrYiYjIWeyx\nExEZEHvsRESdiKKFXXN5jhF6a4xhrBwYgzFUzcGWooWdiIicxR47EZEBscdORNSJKFrYNZfnGKG3\nxhjGyoExGEPVHGwpWtiJiMhZ7LETERkQe+xERJ2IooVdc3mOEXprjGGsHBiDMVTNwZaihZ2IiJzF\nHjsRkQG51GM3m80ICwtDSEgIsrKymox/9913uOWWW9C1a1csXrzYaiwoKAgDBw5EXFwcEhISnEyf\niIhaQ7ew19XVYcaMGTCbzSgqKkJOTg4OHDhgNadHjx5YtmwZZs+e3WR/Dw8PaJqGffv2oaCgoBVp\naS7PMUJvjTGMlQNjMIaqOdjSLewFBQUIDg5GUFAQvLy8kJqaitzcXKs5N9xwA+Lj4+Hl5dXsMdgi\nISJqX556g2VlZQgMDLRsm0wm5OfnO3xwDw8PjBgxAldddRUyMjIwbdq0ZudNmTLl8rUFALoBiG00\nqtnMbthOana84ZUtKSnJ5e2kpKQ2HW+cc1uN277St9W4Ctt8vNR6vPh4uPfx0jQNq1atAlDf5taj\n++XpBx98ALPZjDfeeAMAsGbNGuTn52PZsmVN5mZmZsLb2xuzZs2y3Hb8+HH07t0bp0+fxp133oll\ny5Zh2LBh1gnwy1MiolZz+svTgIAAlJaWWrZLS0thMpkcDty7d28A9e2asWPHtqLPrrk8x/aVsLXj\n7jgGY3SsHBiDMVTNwZZuYY+Pj8ehQ4dQXFyMmpoarFu3DikpKc3OtX3lOH/+PKqqqgAA1dXV2Lp1\nK6Kjo1uVHBERtZ7d89i3bNmCmTNnoq6uDmlpaZg3bx6ys7MBABkZGThx4gQGDx6MyspKdOnSBT4+\nPigqKsKpU6cwbtw4AEBtbS0mTpyIefPmNU2ArRgiolbTa8XwD5SIiAzIgD8Cprk8xwi9NcYwVg6M\nwRiq5mBL0cJORETOYiuGiMiADNiKISIiZyla2DWX5xiht8YYxsqBMRhD1RxsKVrYiYjIWeyxExEZ\nEHvsRESdiKKFXXN5jhF6a4xhrBwYgzFUzcGWooWdiIicxR47EZEBscdORNSJKFrYNZfnGKG3xhjG\nyoExGEPVHGwpWtiJiMhZ7LETERkQe+xERJ2IooVdc3mOEXprjGGsHBiDMVTNwZaihZ2IiJzFHjsR\nkQGxx05E1IkoWtg1l+cYobfGGMbKgTEYQ9UcbCla2ImIyFnssRMRGRB77EREnYiihV1zeY4RemuM\nYawcGIMxVM3BlqKFnYiInMUeOxGRAbHHTkTUidgt7GazGWFhYQgJCUFWVlaT8e+++w633HILunbt\nisWLF7dq35ZpLs8xQm+NMYyVA2Mwhqo52NIt7HV1dZgxYwbMZjOKioqQk5ODAwcOWM3p0aMHli1b\nhtmzZ7d6XyIicj/dHvuuXbuQmZkJs9kMAFi0aBEAYO7cuU3mZmZmwtvbG7NmzWrVvuyxExG1nl6P\n3VNvx7KyMgQGBlq2TSYT8vPzHQramn2nTJly+doCAN0AxAJIunybZjO7Ybv58YaPLElJSdzmNre5\n3WG2NU3DqlWrAABBQUHQJTrWr18v6enplu3Vq1fLjBkzmp27YMECeeWVV1q9b0MKAASQy5e8Rtcd\nmdP0buTl5endNbvj7jgGY3SsHBiDMVTKQa986/bYAwICUFpaatkuLS2FyWTSf6Vww75EROQ83R57\nbW0tQkND8cknn+Cmm25CQkICcnJyEB4e3mTuggUL4OPjY+mxO7ove+xERK3ndI/d09MTy5cvR3Jy\nMurq6pCWlobw8HBkZ2cDADIyMnDixAkMHjwYlZWV6NKlC5YsWYKioiJ4e3s3uy8REbUxu82dNgb2\n2BnDIDkwBmOolINe+eZfnhIRdTD8rRgiIgPib8UQEXUiihZ2zeU5DSf2OzvujmMwRsfKgTEYQ9Uc\nbCla2ImIyFnssRMRGRB77EREnYiihV1zeY4RemuMYawcGIMxVM3BlqKFnYiInNWheuy+vt1RVVXe\nZNTHxx+VlWfdmDUR0ZWl12PvUIWdX64SUWdhwC9PNTfM0R9XobfGGMbKgTEYQ9UcbCla2ImIyFls\nxRARGZABWzFEROQsRQu75oY5+uMq9NYYw1g5MAZjqJqDLUULOxEROYs9diIiA2KPnYioE1G0sGtu\nmKM/rkJvjTGMlQNjMIaqOdhStLATEZGz2GMnIjIg9tiJiDoRRQu75oY5+uMq9NYYw1g5MAZjqJqD\nLUULOxEROYs9diIiA2KPnYioE1G0sGtumKM/rkJvjTGMlQNjMIaqOdiyW9jNZjPCwsIQEhKCrKys\nZuc8+eSTCAkJQUxMDPbt22e5PSgoCAMHDkRcXBwSEhJalRgRETlHt8deV1eH0NBQbNu2DQEBARg8\neDBycnIQHh5umbN582YsX74cmzdvRn5+Pp566ins3r0bANC3b1989dVX6N69e8sJsMdORNRqTvfY\nCwoKEBwcjKCgIHh5eSE1NRW5ublWczZt2oSHH34YAJCYmIiKigqcPHnSMs6CSkTUvjz1BsvKyhAY\nGGjZNplMyM/PtzunrKwMvXr1goeHB0aMGIGrrroKGRkZmDZtWrNxpkyZcvnaAgDdLl+fefm/ms3s\nxttJrRr39e2OqqryJvGvucYb589X1e99uZeVlJRk1ddKSkpy+zgAvPbaa4iNjW2zcU3TsH//fsyc\nObPNxhvfxys1zsfL8fHG95GPx5V/PBx9vABg1apVAOrb3LpEx/r16yU9Pd2yvXr1apkxY4bVnDFj\nxshnn31m2R4+fLh89dVXIiJSVlYmIiKnTp2SmJgY2bFjR5MYDSkAEEAuX/IaXXdkjnvGRUR8fPwv\nz7O++Pj4N8k9Ly9Pb/nsjrvjGJ0lhgo5MAZjqJSDXvnW7bHv3r0bCxYsgNlsBgAsXLgQXbp0wZw5\ncyxzHnvsMSQlJSE1NRUAEBYWhk8//RS9evWyOlZmZia8vb0xa9Ysq9vbs8fuSA+efXoiMgKne+zx\n8fE4dOgQiouLUVNTg3Xr1iElJcVqTkpKCt5++20A9S8E3bp1Q69evXD+/HlUVdW3N6qrq7F161ZE\nR0e74/4QEZEO3cLu6emJ5cuXIzk5GREREXjggQcQHh6O7OxsZGdnAwBGjx6Nfv36ITg4GBkZGXj9\n9dcBACdOnMCwYcMQGxuLxMREjBkzBiNHjnQwLc0Nc1wdb36Or293eHh4NLn4+jY988e2P9ZsBAXO\ngTVCDBVyYAzGUDUHW7pfngLAqFGjMGrUKKvbMjIyrLaXL1/eZL9+/fph//79rUrGCOq/fG34+KOh\n/gtaoKrK48okRERko1P9Vow7euz2xls688bHxx+VlWebv3tERK2k12O3+46dWsf6HX3j2/mOnoja\nR6f9rRhVYqjQnzNCDBVyYAzGUDUHW4oWdiIichZ77I3GG/Jp6xhERK7i77ETEXUiihZ2zQ1zXB1v\nnxgq9OeMEEOFHBiDMVTNwZaihZ2IiJzFHnuj8YZ82joGz3UnIlfxPHbF8Fx3ImpLirZiNDfMcXVc\njRhG6BG2RwwVcmAMxlA1B1uKFnYiInIWe+yNxhvyudIxiIjs4XnsRESdiKKFXXPDHFfHr1wM/ua7\nmjkwBmOomoMtRQt75/a/s2YEQJ7lenOnSBIR2WKPvdF4Qz6qx+B58ETE89g7GJ4HT0R6FG3FaG6Y\n4+p4x4mhQg+wI+TAGIyhag62+I69A2qpVQOwXUPUGbDH3mi8IR+jx3BkLYnI2HgeOxFRJ6JoYdfc\nMMfV8Y4dw2jnyquQA2Mwhqo52FK0sFNb47nyRB0Xe+yNxhvyMXoMd6wlEamNPXZqNXutmpbGHZnj\n6nhLLSMiqqdoYdfcMMfV8c4dw16rxnrckTnuG288pz1ePIwco/ELoDu/V1GhN91RYrRFjx1yhTWk\nAEAAuXz5a6Prjsxxz3hHiWE93lFitPdzoqPEcPx55+Pjf3mO9cXHx9+hcXccw9nx9o7R2F//+tcm\nt7lzvKU5euXb7jt2s9mMsLAwhISEICsrq9k5Tz75JEJCQhATE4N9+/a1at/mVbhhjqvjjKFWDBVy\n6NgxrD8hzbdcb/4TVNNxdxyjdeNXLkbjTz9/+MMfdD/9VFToPx72xh2d05huYa+rq8OMGTNgNptR\nVFSEnJwcHDhwwGrO5s2bcfjwYRw6dAh///vfMX36dIf3JSIyIkde5K4k3cJeUFCA4OBgBAUFwcvL\nC6mpqcjNzbWas2nTJjz88MMAgMTERFRUVODEiRMO7duyYjfMcXWcMdSKoUIOjMEYrR8vLnZt3NE5\nVvT6Ou+//76kp6dbtlevXi0zZsywmjNmzBj5/PPPLdvDhw+XPXv2yPr16+3u29An4oUXXnjhpfWX\nluj+CFj9uc721ddn57iyLxERNaVb2AMCAlBaWmrZLi0thclk0p1z9OhRmEwmXLx40e6+RETkfro9\n9vj4eBw6dAjFxcWoqanBunXrkJKSYjUnJSUFb7/9NgBg9+7d6NatG3r16uXQvkRE5H6679g9PT2x\nfPlyJCcno66uDmlpaQgPD0d2djYAICMjA6NHj8bmzZsRHByM6667DitXrtTdl4iI2tYV/60YADhw\n4AByc3NRVlYGADCZTEhJSWnVC8GBAwdw7NgxJCYmwtvb23K72WzGXXfdhc8++wzdu3dHREQENE3D\nnj17EBcXh+HDhzuV886dO1FQUIDo6GiMHDkSu3fvRnh4OPz8/HD+/HksWrQIe/fuRWRkJJ599ln4\n+flh6dKlGDt2LAIDA5s95oULF/Duu+8iICAAI0aMwNq1a/HFF18gIiICjz76KLy8vPDDDz9gw4YN\nOHr0KLp06YLQ0FA8+OCD8PX1dep+EFHHc8V/UiArKwsTJkwAUH+6ZGJiIi5duoQJEyZg4cKFdvdf\nuXIlli5dinvvvRfLli1DZGQkNm7caBmfN28e5s2bh9mzZ+Phhx/GM888g7lz5+KXX35BZmYmXn75\n5RaP/dBDD1muJyQkWK6/8cYb+P3vf49z584hMzMTCxcuxCOPPILrrrsOAPDUU0+hsrISc+fOxTXX\nXIOpU6cCAJ577jkkJCRg6NCheP3113H69GmreFOnTsXmzZuxZMkSTJ48GevXr8eQIUNQUFCA9PR0\nLFmyBI899hguXLiAgoICXLhwASUlJUhMTEReXp4Dq33lnDp1yqX9z5w546ZM2seoUaPw888/Y+7c\nuZg0aRLeeecdq/HHH3/c6WO7upaAsdZz1KhRANAm69lh11LvdMf2EBwcLDU1NU1uv3DhgvTv39/u\n/iaTSSIjI6WqqkpERI4cOSKDBg2y/AlubGyshIeHy8WLF6W6ulq8vb2loqJCRETOnz8v0dHRIlJ/\n2ubdd98tY8aMsVyuvfZay+2xsbGWmIMGDZJTp06JiMi5c+ckMjJSwsLCLONxcXFWOQ4cONCSS11d\nnfz73/+WqVOnyvXXXy/JycmyatUqqayslKioKBERuXjxotxwww1y8eJFERG5dOmSREVFSWRkpNTW\n1oqISHV1tfzf//2fiIj8+OOPEhMTIyIi5eXlMmfOHAkNDZVu3bqJv7+/hIaGypw5c6S8vFx3Le+6\n6y6pqKiQOXPmyMSJE2Xt2rVW49OnTxcRkZKSEklLS7Mcc8qUKRIZGSmTJk2SkydPypkzZ6wuP/30\nk/Tp08eyvWXLFssxy8vL5ZFHHpGoqCiZMGGCnDhxQp555hnL+n755ZfSt29f6d+/vwQGBkpeXp5l\nLV944QU5fPhws/eloKBAkpKSZOLEiVJSUiIjRowQX19fiY+Pl71790plZaU899xzEhERIT4+PtKj\nRw9JSEiQlStXWuWmt5ZfffVVs5c9e/ZIr169ZOzYsTJnzhzZsGGDjBkzRsaNGye//PKLJf/2WEsR\nsbuerq6liNhdT1fXUkTsrqeraykiLj8322MtHXHFC3toaKgcOXKkye1HjhyRAQMGiIhIVFRUi5er\nr75aIiIirPatqqqSkSNHysyZMyUmJsZS9ETE6nrj7djYWHnwwQdl+/btomma5OXlyY033iiapomm\naRIdHW15MjQu8g3HGD9+vLz55psiIjJlyhQpKCgQEZGDBw9KfHy8JUZjFy5ckI0bN8oDDzwgPXr0\nkIiICPn111/l7Nmz4u3tLT/99JOI1L8ARURESFRUlOWJfObMGRk0aJDlWA1rcOedd8qiRYvk+PHj\ncunSJREROXbsmCxcuFDuvPNOtxSjO+64Q5YuXSovvviihIaGysKFC+XHH3+UpUuXyrhx48TDw0OC\ngoKsLp6enhIUFCR9+/a1WodHHnlE/vjHP8qRI0fk1VdflXvuuUciIyMt47fffrvVWv72t78VEZGg\noCCZNWuWBAYGSnx8vLz66qtSVlZm2S8+Pl42b94s77zzjgQEBMh7770nly5dkm3btsmQIUPk7rvv\nlrfeektKSkpk8eLFkpmZKQcPHpTJkyfLvHnzHFrLLl26SFJSUrOXrl27Wl7QG/zlL3+RW2+9VU6f\nPt1uaykidtfT1bUUEbvr6epaiojd9XR1LRs/x519brbHWjriihf2LVu2SP/+/SU5OVnS09MlPT1d\nkpOTpV+/frJ582YREenZs6fs3btXjhw50uTSu3dvSUpKkn379lkdt6amRiZPniweHh6SkJAg1dXV\nIiJSV1dnmVNeXm55d11bWyuLFy+W4cOHW145g4KCLHP79OljeTL07dtXjh07JiL1r64xMTFSXl4u\nDz30kPTt21cSEhIsT5hhw4bJ/v37RaRpYW/s3Llz8uKLL0rfvn1lwIABkp2dLeHh4ZKWliaRkZGS\nlZUlr732mkRFRUlaWpoMGDDA8kJy8uRJGTZsmIiIhISEtBgjJCTELcWo8YtjYGCg1fyYmBh55ZVX\nJDk5Wb7++mvL7Y3XsvE6DBw40PI/esN2WFiY5VNcYmKi1fEbPtU0HOPSpUvy6aefymOPPSa9evWS\npKQkyc7OtorRXI4Nn9QaNLxI1tXVWd5Q2FvLiIgIOXjwYLPjJpNJwsLCrJ5vIiIrV66UiIgIufnm\nmy256OXp6lqKiN31dHUtRcTuerq6lg33Q289XV1LEdefm+2xlo644oVdpL6ofvHFF/L+++/L+vXr\nZdeuXZY2hIjI1KlTZceOHc3um5qaKiUlJXL8+PEmY5cuXZKdO3da3nHaOn36tBQWFlrdVlpaKvfd\nd588/vhC3DxxAAAFYklEQVTjlieUnurqavnvf/9r2a6oqJB9+/bJl19+2SSn7777zu7xjhw5YvlY\nePjwYXn33XctLwwiIt988428//77cuDAgWb3HzFihGRlZVk+houIHD9+XBYtWiTDhw93SzFqXPyf\nffZZq7kNhbekpETuu+8+mTlzpvz8889W/wMFBATI4sWL5ZVXXpE+ffpY/c8THR0tS5culREjRsgn\nn3wi8+fPlyeffFI0TZM///nPMmnSJBFp/kXy4sWLsmXLFpkyZYoMHjxYzGazrFu3Tkwmk2zYsEFE\nRDRNk4SEBBkyZIjlObVx40YZOXKk5TgN//PYW8v33nuvxcfhww8/lNmzZ8vWrVubjG3ZskWCg4Pb\nZS1FxO56urqWImJ3PV1Zy4Z49tbT1bV0ZD1VWEtHKFHYVfTRRx85/LFHJWfOnJGnn37a0svs1q2b\nhIaGytNPPy1nzpxxSzH605/+JJWVlU3mfP/99zJ+/Hir2zZu3CgJCQnSs2dPy23z58+XBQsWWC4n\nT54UkfqP5pMnTxYRke3bt8vvfvc7iY2NlaioKLnrrrtkxYoVlndLDzzwgO465Ofny+233y6pqalS\nXFwsw4cPFx8fH4mLi5Mvv/xS9u/fL/Hx8eLn5ye33nqr5UX31KlTsmTJEofWUkSkqKhItm3bZvmO\np0HDp0174+2xlg3ref/99ze7nvfff79LaykidtfTkbXcuXOnfPvttyIikpeXJy+//LJs27bNKhe9\nOS2t5aFDh5pdy8TERKu11FvP48ePWz03XVnLpKQkp9fytdde0z1+Axb2TuStt97SHW9o7Tg73tKc\n6upqyycjV3Owt78jc1qTQ1FRkXz88cdNCsaWLVtkyZIlMmDAALnnnnvk5ptvlg8//NAyHhsba3e8\ncQxHXxwar2Xj8Y8//rjF/e3dD70cGo87G6Nhzs6dO+U///mPiNQXx8ZFee7cuZKYmCjx8fHy9NNP\nS2Jiojz//PMybNgweemllxyeY+/FofH4v/71L3nhhReaffFoyLOlY9gbt5eD7f4ff/xxq3Kwh4W9\nE7HXWnJ1vD1itGcO9gqzvbOx7I07EsMdLx4qxLBXlB05c83eHHsx3PHi0dbjjuTgCCX+QIncJzo6\nusWxgwcPIjQ01KXxmpoal2J8//33GDBggEv7X7hwQTcHV2M03M+oqCjs3r0b3t7eKC4uxvjx4zF5\n8mTMnDkTcXFxqKmpwbfffmvZ79y5cxg/fjwiIiKQl5eHixcv6o7v37+/SYz77rsPkyZNssS4ePGi\nS+P79u1TIsaFCxdQWFiImpoa9OrVC0ePHoWfnx9++eUXJCYmokuXLti/fz8AIDY21nK98Xbj25ub\nU1NToxujtrZWd7ywsBAREREuHcPVcUdyKCwsbPG520D3JwXIeE6dOgWz2Qx/f/8mY7feeqvL4+0R\nQ4UcgPpfHm34K+agoCB8+umnGD9+PH788UeICHr27GkpOADg7e2Nf/7zn0hLS0NhYSFuv/123fHm\nYmiaZhXD1XFVYlx99dXw9PSEp6cn+vfvDz8/PwDANddcgy5duuA3v/kNzp8/j2uvvRZ79+61PBYV\nFRXo0qX+7yjtzbEXw944AJeP0R45OMSh9/VkGPbOIHJ1vD1iqJCDiNg9jdbe2Vj2xh2J4eq4KjHs\nnXLsyJlr9ubYi+HIac+uHqM9cnAEWzFELSgtLYWXlxduvPFGq9tFBJ9//jmGDh3a5jH69Onj0vjQ\noUOViBEfH4+uXbs2uf8//fQTjh8/rttac9Svv/6qGyMkJMRuDq4ew9VxR3JwZK1Y2ImIOpgr/iNg\nRETkXizsREQdDAs7EVEHw8JORNTB/D8K6CUHgIQ5/QAAAABJRU5ErkJggg==\n",
       "text": [
        "<matplotlib.figure.Figure at 0x9346950>"
       ]
      }
     ],
     "prompt_number": 11
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "pca = PCA(n_components=15, whiten=True)\n",
      "pca_X = pca.fit_transform(X)\n",
      "train_X = pca_X[train_index, :]\n",
      "test_X = pca_X[test_index, :]\n",
      "train_y = y[train_index]\n",
      "test_y = y[test_index]\n",
      "print train_X.shape, test_X.shape\n",
      "print train_y.shape, test_y.shape"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "(800, 15) (200, 15)\n",
        "(800,) (200,)\n"
       ]
      }
     ],
     "prompt_number": 12
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "svc = SVC()\n",
      "gammas = [1e-4, 3e-4, 1e-3, 3e-3, 1e-2, 3e-2, 1e-1, 3e-1, 1., 3., 10.]\n",
      "gs = GridSearchCV(svc, {'gamma': gammas}, n_jobs=-1, cv = 5, scoring='accuracy')\n",
      "gs.fit(train_X, train_y)\n",
      "print gs.best_params_\n",
      "print gs.best_score_\n",
      "\n",
      "svc = SVC(**gs.best_params_)\n",
      "svc.fit(train_X, train_y)\n",
      "print svc.score(test_X, test_y)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "{'gamma': 0.1}\n",
        "0.91625\n",
        "0.93\n"
       ]
      }
     ],
     "prompt_number": 13
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "## benchmark 1 - sparse filtering (feature extraction) + decision forest (feature selection)"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "def soft_absolute(u):\n",
      "    epsilon = 1e-8\n",
      "    return np.sqrt(u*u + epsilon)\n",
      "\n",
      "def logistic(u):\n",
      "    return 1. / (1. + np.exp(-u))\n",
      "\n",
      "class SparseFilter(BaseEstimator, TransformerMixin):\n",
      "    def __init__(self, n_features = 200, n_iterations = 300, activate=soft_absolute):\n",
      "        self.epsilon = 1e-8\n",
      "        self.n_features = n_features\n",
      "        self.n_iterations = n_iterations\n",
      "        self.activate = activate\n",
      "    def fit(self, X, y = None):\n",
      "        n_samples, n_dim = X.shape\n",
      "        W = np.random.randn(n_dim, self.n_features)\n",
      "        b = np.random.randn(self.n_features)\n",
      "        obj_fn = self.get_objective_fn(X)\n",
      "        self.W_, self.b_ = optimize.fmin_l_bfgs_b(obj_fn, (W, b), \n",
      "                                iprint = 1, \n",
      "                                maxfun = self.n_iterations)\n",
      "        return self\n",
      "    def get_objective_fn(self, X):\n",
      "        def _objective_fn(W, b):\n",
      "            Y = self.activate(np.dot(X, W) + b)\n",
      "            Y = Y / np.sqrt(np.sum(Y*Y, axis = 0) + self.epsilon)\n",
      "            Y = Y / np.sqrt(np.sum(Y*Y, axis = 1)[:, np.newaxis] + self.epsilon)\n",
      "            return np.sum(Y)\n",
      "        return _objective_fn\n",
      "    def transform(self, X):\n",
      "        Y = self.activate(np.dot(X, self.W_) + self.b_)\n",
      "        Y = Y / np.sqrt(np.sum(Y*Y, axis=0) + self.epsilon)\n",
      "        Y = Y / np.sqrt(np.sum(Y*Y, axis=1)[:, np.newaxis] + self.epsilon)\n",
      "        return Y\n",
      "    "
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 10
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "sf = SparseFilter(n_features=50, n_iterations=5000)\n",
      "sf_X = sf.fit_transform(X)\n",
      "sf_X = ss.fit_transform(sf_X)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 11
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "train_X = sf_X[train_index, :]\n",
      "train_y = y[train_index, :]\n",
      "test_X = sf_X[test_index, :]\n",
      "test_y = y[test_index, :]\n",
      "print train_X.shape, test_X.shape\n",
      "print train_y.shape, test_y.shape"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "(800, 50) (200, 50)\n",
        "(800,) (200,)\n"
       ]
      }
     ],
     "prompt_number": 12
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "trees = ExtraTreesClassifier(n_estimators=100, max_features=50, n_jobs=-1, random_state=0)\n",
      "trees.fit(train_X, train_y)\n",
      "feature_importances = pd.DataFrame(trees.feature_importances_)\n",
      "feature_importances.plot(kind='bar')"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 13,
       "text": [
        "<matplotlib.axes.AxesSubplot at 0x7de04d0>"
       ]
      },
      {
       "metadata": {},
       "output_type": "display_data",
       "png": "iVBORw0KGgoAAAANSUhEUgAAAXYAAAEBCAYAAABseY4cAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJztnX14VGV6xu/ECWC+SCJfdiY6kYEwAYUsCdHd2h2F3SiL\n0UtwG+SbpLK4lM1WXbAfimtXsMh2A9jL6FYoqIhL3QRqkhZK4mIxRBbibg1gaAkk4RuJQbAEhqd/\nYIb5PDOZOUnek9y/6zoXnPO8732e951znjNzn5OZKBEREEII6TVE93QChBBC9IWFnRBCehks7IQQ\n0stgYSeEkF4GCzshhPQyWNgJIaSXEbSwV1ZWYtSoURgxYgRefvlln/jBgwdxzz33YMCAAVi1apVP\n3Ol0IjMzEw899JA+GRNCCNHEpBV0Op1YtGgRduzYAbPZjOzsbOTl5cFut7va3HLLLVizZg1KS0v9\nahQXFyMjIwMXLlzQN3NCCCF+0XzHXltbC5vNBqvVipiYGOTn56OsrMyjzeDBg5GVlYWYmBif/s3N\nzSgvL0dhYSH4d1CEENI9aBb2lpYWpKamutYtFgtaWlpCFv/pT3+KlStXIjqaVj4hhHQXmlZMVFRU\n2ML/9m//hiFDhiAzMxPV1dVdsg9CCOnLBHJCNN9Km81mNDU1udabmppgsVhC2uHu3buxdetWpKWl\nYfr06di5cydmz54dMLmO5fnnn/dYjzRGPepRj3q9UU8LzcKelZWFhoYGNDY2or29HZs3b0ZeXl7A\n4uzOSy+9hKamJhw5cgTvvvsu7r//fmzYsEEzGQBobGzUNUY96lGPer1dzxtNK8ZkMmHt2rXIzc2F\n0+lEQUEB7HY7SkpKAAALFizAyZMnkZ2djba2NkRHR6O4uBj19fWIj4/30KLlQggh3YT0MN4pVFVV\nBWwbTox61KMe9Xqjnlb5jvqmQY8RFRUV1C8ihBDiiVbtVO45RK0naMKJUY961Ou7eikpKYiKijL0\nkpKSEnS83mh67IQQYmTOnz9veEcgnPuTtGIIIb2W3lBfAo3BUFYMIYSQyFCusKvm0VGPetTrPXpG\npjNjUq6wE0IIiQx67ISQXou/+pKYmIILF8532T4TEpLR1vZF0HZffPEFCgoKsH37dgwaNAjLly/H\n9OnTfdqF47HzqRhCSJ/ielHvujeTFy6E9hTLj3/8YwwYMACnT5/G/v378YMf/ABjx45FRkZGxDko\nZ8Wo7tFRj3rUM66eKly8eBHvv/8+XnzxRcTGxuI73/kOHn74YWzcuDFgH3rshBCiMJ9//jlMJhNs\nNptr29ixY/HZZ5/pok+PnRDSa/FXX67/wU9X1pzgNW3Xrl344Q9/iBMnTri2vfHGG3jnnXdQVVXl\nqcbn2AkhRH3i4+PR1tbmse3LL79EQkKCLvrKFXbVPTrqUY96xtVThZEjR+Lq1as4fPiwa9unn36K\nMWPGBOxDj50QQhQmLi4Ojz76KJ577jlcunQJH330EbZt24ZZs2bpok+PnRDSa1HVYweuf0HZ/Pnz\nXc+xr1ixAvn5+b5qYXjsLOyEkF6Lyn+gFCq94uap6h4d9ahHPePqAUBb2xcBfzBaj0XPoh7qmLxR\nrrATQgiJDFoxhJBeS2+oL73CiiGEEBIZyhV21T066lGPesbRS05O7vHfLI10SU5ODjpeb5Qr7IQQ\nohdffOF5o7SqqirgTc9wYt2h98UXnb8ZG5LHXllZiaKiIjidThQWFmLJkiUe8YMHD2LevHnYv38/\nfvGLX+Cpp54CADQ1NWH27Nk4ffo0oqKi8MQTT2Dx4sWeCfQCD4wQQrqbiJ5jdzqdSE9Px44dO2A2\nm5GdnY1NmzbBbre72pw5cwZHjx5FaWkpkpOTXYX95MmTOHnyJMaNG4evvvoK48ePR2lpqUdfFnZC\nCOk8Ed08ra2thc1mg9VqRUxMDPLz81FWVubRZvDgwcjKykJMTIzH9mHDhmHcuHEArn/pjd1ux/Hj\nxzX3p5pHRz3qUS8yvcTEFB/fODExRZn8eoOeN0ELe0tLC1JTU13rFosFLS0tIe+gg8bGRuzfvx85\nOTmd7ksIMS43frGo6pt/pUv/8pOE8NN4179XITK++uorTJs2DcXFxYiPj/eJz507F1arFQCQlJQE\nAHA4HABuXKUcDgccDofHunfc33rHNupRj3o9p3cdBwD3dXXyM4IecL1WAnDVy0AE9dhramqwbNky\nVFZWAgCWL1+O6OhonxuoAPDCCy8gPj7e5bEDwJUrVzBlyhQ8+OCDKCoq8k2AHjshvRr/X7rF8z5S\nIvLYs7Ky0NDQgMbGRrS3t2Pz5s3Iy8vz29Z7JyKCgoICZGRk+C3q/vC8wkceox71qKeGnve79Uj1\nVB9vd+p5E9SKMZlMWLt2LXJzc+F0OlFQUAC73Y6SkhIAwIIFC3Dy5ElkZ2ejra0N0dHRKC4uRn19\nPerq6vDWW2/hrrvuQmZmJoDr7/gfeOCBkBMkhBDSOfhdMYSQLoVWTNfA74ohhJA+hHKFXRXPinrU\no56+evTYu07Pm6AeOyGEAP5/eUjvXwsi+kCPnRASEuF65fTYuwZ67IQQ0odQrrCr4llRj3rUCxwL\n5JfTY+8ZPW+UK+yEEEIigx47ISQk6LGrBT12QgjpQyhX2FXxrKhHPeoFjtFjV0vPG+UKOyGEkMig\nx04ICQl67GpBj50QQvoQyhV2VTwr6lGPeoFj9NjV0vNGucJOCCEkMuixE0JCgh67WtBjJ4SQPoRy\nhV0Vz4p61KNe4Bg9drX0vFGusBNCCIkMeuxu8IcECAkMPXa10KqdLOxu8AAkJDAs7GphqJun6nhW\n+uqp7tFRj3qdidFjV0vPG+UKOyGEkAiRIFRUVEh6errYbDZZsWKFT/zAgQNy9913S//+/eWVV17p\nVN9vbKBgKXQbAAQQr0Wd/AjpScI9P3hedQ1ac6jpsTudTqSnp2PHjh0wm83Izs7Gpk2bYLfbXW3O\nnDmDo0ePorS0FMnJyXjqqadC7gvQYyfEKNBjV4uwPfba2lrYbDZYrVbExMQgPz8fZWVlHm0GDx6M\nrKwsxMTEdLqvP9TxrPTVU92jox71OhOjx66Wnjeahb2lpQWpqamudYvFgpaWlpCEI+lLCCEkfExa\nwesfocKjM33nzp0Lq9UKAEhKSgIAOBwOADeuUg6HAw6Hw2PdO+5vvWNbqHo33lX4799ZPb3zox71\nelIv3PPjRh/3dfXHq5IecL1WAnDVy0Boeuw1NTVYtmwZKisrAQDLly9HdHQ0lixZ4tP2hRdeQHx8\nvMtjD7UvPXZCjAE9drUI22PPyspCQ0MDGhsb0d7ejs2bNyMvL89vW+8ddKavO55X+Mhj4eqF4yF2\nZ37Uo15P6oVzfgTqo9VPlfGqrueNphVjMpmwdu1a5Obmwul0oqCgAHa7HSUlJQCABQsW4OTJk8jO\nzkZbWxuio6NRXFyM+vp6xMfH++1LCCGka+FXCrjBj4yEBIZWjFoY6isFCCGERIZyhV0dz0pfPdU9\nOupRrzMxeuxq6XmjXGEnhBASGfTY3aAXSEhg6LGrBT12QgjpQyhX2NXxrPTVU92jox71OhOjx66W\nnjfKFXZCCCGRQY/dDXqBhASGHrta0GMnhJA+hHKFXR3PSl891T066lGvMzF67GrpeaNcYSeEEBIZ\n9NjdoBdISGDosasFPXZCCOlDKFfY1fGs9NVT3aOjHvU6E6PHrpaeN8oVdkIIIZFBj90NeoGEBIYe\nu1rQYyeEkD6EcoVdHc9KXz3VPTrqUa8zMXrsaul5o1xhJ4QQEhn02N2gF0hIYOixqwU9dkII6UMo\nV9jV8az01VPdo6Me9ToTo8eulp43yhV2QgghkRHUY6+srERRURGcTicKCwuxZMkSnzaLFy9GRUUF\nYmNjsX79emRmZgIAli9fjrfeegvR0dG48847sW7dOvTv398zAXrsupCYmIILF857bEtISEZb2xc9\nlBHpbdBjV4uwPXan04lFixahsrIS9fX12LRpEw4cOODRpry8HIcPH0ZDQwNef/11LFy4EADQ2NiI\nN954A/v27cMf//hHOJ1OvPvuuzoNiXhzvaiLx+Jd6AkhfQPNwl5bWwubzQar1YqYmBjk5+ejrKzM\no83WrVsxZ84cAEBOTg5aW1tx6tQpJCYmIiYmBpcuXcLVq1dx6dIlmM3moAmp41npq9ddHl04PqZW\nTPXxUq9n9Oixq6XnjWZhb2lpQWpqqmvdYrGgpaUlpDYpKSl46qmncNttt+FP/uRPkJSUhEmTJoWc\nGCGEkPAwaQWve2PB8efz/M///A9+9atfobGxEQMHDsRjjz2Gt99+GzNmzPBpO3fuXFitVgBAUlIS\nAMDhcAC4cZVyOBxwOBwe695xf+sd20LVu/Guwn//zurpnV+gdU+qXfmrkh/1eodeuOfHjT7u6+qP\nVyU94HqtBOCql4HQvHlaU1ODZcuWobKyEsD1m6HR0dEeN1B/9KMfweFwID8/HwAwatQofPjhh6iu\nrsb27dvx61//GgCwceNG1NTU4NVXX/VMgDdPdcHIuRNjwJunahH2zdOsrCw0NDSgsbER7e3t2Lx5\nM/Ly8jza5OXlYcOGDQCuXwiSkpIwdOhQpKeno6amBl9//TVEBDt27EBGRkbQZP2/Aw0/Fq5eOB5i\nd+YXOKavnurjpV7P6IVzfoRzbKoyXtX1vNG0YkwmE9auXYvc3Fw4nU4UFBTAbrejpKQEALBgwQJM\nnjwZ5eXlsNlsiIuLw7p16wAA48aNw+zZs5GVlYXo6Gh861vfwhNPPBFyYoQQQsKD3xXjhpE/Mho5\nd2IMaMWoBb8rhhBC+hDKFXZ1PCt99eixU6836dFjV0vPG+UKOyGEkMigx+6Gkb1AI+dOjAE9drWg\nx04IIX0I5Qq7Op6Vvnr02KnXm/Tosaul543mc+yEEP3hVyyTroYeuxtG9gKNnHtfw6ivFT12taDH\nTgghfQjlCrs6npW+evTYqecnqque6ueHkcerup43yhV2QgghkUGP3Q0je4FGzr2vYdTXih67WtBj\nJ4SQPoRyhV0dz0pfPXrs1PMT1VVP9fPDyONVXc8b5Qo7IYSQyKDH7oaRvUAj597XMOprRY9dLeix\nE0JIH0K5wq6OZ6WvHj126vmJ6qqn+vlh5PGqrueNcoWdEEJIZNBjd8PIXqCRc+9rGPW1oseuFvTY\nCSGkD6FcYVfHs9JXjx479fxEddVT/fww8nhV1/NGucJOCCEkMoJ67JWVlSgqKoLT6URhYSGWLFni\n02bx4sWoqKhAbGws1q9fj8zMTABAa2srCgsL8dlnnyEqKgpvvvkm7r77bs8E6LHrgpFz1xvVf8jC\nqK8VPXa10Kqdmr+g5HQ6sWjRIuzYsQNmsxnZ2dnIy8uD3W53tSkvL8fhw4fR0NCAPXv2YOHChaip\nqQEA/OQnP8HkyZOxZcsWXL16FRcvXtRxWIT453pRF69tUT2TDCE9gKYVU1tbC5vNBqvVipiYGOTn\n56OsrMyjzdatWzFnzhwAQE5ODlpbW3Hq1Cl8+eWX2LVrF+bPnw8AMJlMGDhwYNCE1PGs9NWjx979\neqp7uqrnR4/dOHreaL5jb2lpQWpqqmvdYrFgz549Qds0NzfjpptuwuDBgzFv3jx8+umnGD9+PIqL\nixEbG+uzn7lz58JqtQIAzp49CwBwOBwegwm23oF3vK6urlN6/g6+6urqsPX0zi+Q3g3qvvlXrfy6\nW8+th9/+zC88vXDPD7fWfvPRK7/erFdXV4f169cDgKteBkLTY//Xf/1XVFZW4o033gAAvPXWW9iz\nZw/WrFnjavPQQw9h6dKl+M53vgMAmDRpEv7hH/4B165dwz333IPdu3cjOzsbRUVFSExMxM9//nPP\nBOix64KRc9cb1edC9fwCQY9dLcJ+jt1sNqOpqcm13tTUBIvFotmmubkZZrMZFosFFosF2dnZAIBp\n06Zh3759YQ+CEEJIaGgW9qysLDQ0NKCxsRHt7e3YvHkz8vLyPNrk5eVhw4YNAICamhokJSVh6NCh\nGDZsGFJTU/H5558DAHbs2IHRo0cHTShcjylQLFw9euyR5KCGnuqerur56X1+GHm8qut5o+mxm0wm\nrF27Frm5uXA6nSgoKIDdbkdJSQkAYMGCBZg8eTLKy8ths9kQFxeHdevWufqvWbMGM2bMQHt7O4YP\nH+4RI4QQ0jXwu2LcMLIXaOTc9Ub1uVA9v0DQY1cLflcMIYT0IZQr7Op4Vvrq0WPvfj3VPV3V86PH\nbhw9b5Qr7IQQQiKDHrsbRvYCjZy73qg+F6rnFwh67GpBj50QQvoQyhV2dTwrffXosXe/nuqerur5\n0WM3jp43yhV2QgghkUGP3Q0je4FGzl1vVJ8L1fMLBD12taDHTgghfQjlCrs6npW+evTYu19PdU9X\n9fzosRtHzxvlCjshhJDIoMfuhpG9QCPnrjeqz4Xq+QWCHrta0GMnhJA+hHKFXR3PSl89euzdr6e6\np6t6fvTYjaPnjXKFnRBCSGTQY3fDyF6gkXPXG9XnQvX8AkGPXS3osRNCSB9CucKujmelrx499u7X\nU93TVT0/euzG0fNGucJOCCEkMuixu2FkL9DIueuN6nOhen6BoMeuFvTYCSGkD6FcYVfHs9JXjx57\n9+up7umqnh89duPoeaNcYSeEEBIZQT32yspKFBUVwel0orCwEEuWLPFps3jxYlRUVCA2Nhbr169H\nZmamK+Z0OpGVlQWLxYJt27b5JkCPXReMnLveqD4XqucXCHrsahG2x+50OrFo0SJUVlaivr4emzZt\nwoEDBzzalJeX4/Dhw2hoaMDrr7+OhQsXesSLi4uRkZHxzYtLCCGkq9Es7LW1tbDZbLBarYiJiUF+\nfj7Kyso82mzduhVz5swBAOTk5KC1tRWnTp0CADQ3N6O8vByFhYUhX53V8az01aPH3v16qnu6qudH\nj904et6YtIItLS1ITU11rVssFuzZsydom5aWFgwdOhQ//elPsXLlSrS1tWkmMXfuXFitVgDA2bNn\nAQAOhwPAjcEEW+/AO15XV9cpPX8HX3V1ddh6eucXSO8Gdd/8q1Z+3a3n1sNvf+YXnl6454dba7/5\n6JVfb9arq6vD+vXrAcBVLwMiGmzZskUKCwtd6xs3bpRFixZ5tJkyZYp89NFHrvWJEyfK3r17Zdu2\nbfLkk0+KiEhVVZVMmTLF7z6CpNCtABBAvBZ18tPCyLnrjepzoXp+gQg3b6OOV3W05lDTijGbzWhq\nanKtNzU1wWKxaLZpbm6G2WzG7t27sXXrVqSlpWH69OnYuXMnZs+erX2VIYQQA5OYmIKoqCiPJTEx\npfsT0boiXLlyRe644w45cuSIXL58WcaOHSv19fUebT744AN58MEHRUTk448/lpycHB+d6urqkN+x\nV1VVBcwnnFhn+sDjnUWV33cWPZmfVuxG7lUB3xX1ZH7dqRfJXDC/wLFwzw+jjjccvVDmSK/8tMq3\npsduMpmwdu1a5Obmwul0oqCgAHa7HSUlJQCABQsWYPLkySgvL4fNZkNcXBzWrVvnV4tPxRBCSPfA\n74pxw8jP2xo5d71RfS5Uzy8QfI49ON05Vn5XDOlxlPEeCekDKFfYw32OM1AsXD0+xx5JDr6xCxfO\n4/o7mapv/pVvtnVdfqo/N616fnqfH0Yer+qvrzfKFXZCCCGRQY/dDSN7garn3t3eo577SkxM8fl0\nkZCQjLa2L5TIr7ugxx4cVTx2zadiCCHuNpL7Nj7lRdRFOStGFU+NHnskOaiRn94ep+p6qp8fRh6v\n6q+vN8oVdkIIIZFBj90NI3uBquduZI9ddb3uQm+PPSEhWdd7FypAj50Q0qfhvYuuQzkrRhVPjR57\nJDmokZ/qnriRPWe9PXajnm+qvL7eKFfYCSGERAY9djeM6n0C6udOj73r9LoLvT326xhvHrRQxWPn\nO3ZCCOllKFfYVfHUjOr50WOPbF9G1lP9/KDHHpkePXZCCOnD0GN3w6jeJ6B+7vTYu06vu6DHHhx6\n7IQQQroE5Qq7Kp6aUT0/euyR7cvIeqqfH/TYI9Ojx04IIX0YeuxuGNX7BNTPnR571+l1F/TYg0OP\nnRBCSJegXGFXxVMzqudHjz2yfRlZT/Xzgx57ZHr02AkhpA8TksdeWVmJoqIiOJ1OFBYWYsmSJT5t\nFi9ejIqKCsTGxmL9+vXIzMxEU1MTZs+ejdOnTyMqKgpPPPEEFi9e7JkAPXZdUD13euxdp9dd0GMP\njioeOyQIV69eleHDh8uRI0ekvb1dxo4dK/X19R5tPvjgA3nwwQdFRKSmpkZycnJEROTEiROyf/9+\nERG5cOGCjBw50qdvCCl0G9dfEfFa1MlPC9Vz78789N6X6nrdRbh5B+pn1HnQQmtMCQnJbuO+viQk\nJEe0r0AEtWJqa2ths9lgtVoRExOD/Px8lJWVebTZunUr5syZAwDIyclBa2srTp06hWHDhmHcuHEA\ngPj4eNjtdhw/flxzf6p4akb1/OixR7YvI+upfn70dY/9xg+LCIAqAOLzC1J6eexBf0GppaUFqamp\nrnWLxYI9e/YEbdPc3IyhQ4e6tjU2NmL//v3Iycnx2cfcuXNhtVoBAGfPngUAOBwOj8EEW+/AO15X\nV9cpPX8HWHV1ddh6eucXSO8Gdd/8G5leXt6jfn+2bOvW95XIL9j8ufXQzKe36Kl+fri1DrJujPMt\nkN6NMTnc/g+vmNeWEMdbV1eH9evXA4CrXgYk2Nv9LVu2SGFhoWt948aNsmjRIo82U6ZMkY8++si1\nPnHiRPn973/vWr9w4YKMHz9efvvb33bq40R3AwN/NNQ7d9X1unNfqut1F+HmHaifUedBC60xdcVx\nFIigVozZbEZTU5NrvampCRaLRbNNc3MzzGYzAODKlSuYOnUqZs6ciUceeSTY7gghhERI0MKelZWF\nhoYGNDY2or29HZs3b0ZeXp5Hm7y8PGzYsAEAUFNTg6SkJAwdOhQigoKCAmRkZKCoqCikhFTx1Izq\n+entYauuF85r2Fv1VD8/+rrHHkos/NfDk6Aeu8lkwtq1a5Gbmwun04mCggLY7XaUlJQAABYsWIDJ\nkyejvLwcNpsNcXFxWLduHQDgv/7rv/DWW2/hrrvuQmZmJgBg+fLleOCBB0JOkBBCeoLExBS/95na\n2r7ooYxCh98V44ZRny8G1H/Wms+xd51ed9HXnmMPZ7xafbriOArUl395SgghvQzlCrsqnppRPT/V\nPXF67F2np/r5YWSPXfXxeqNcYSeEEBIZ9NjdMKr3CajvA9Nj7zq97oIeO0CPnRBCSI+gXGE3sqdG\nj7379YzsidNjj0yP4w1M0OfYCekLGPmZ5d4IX4/IoMfuhlG9T0B9H1h1j727vVEjHmfd6bGrMEdG\nOI7osRNCSB9BucJuZE+NHnv36+ntYYfji2rp0WMP3kc7FrhPbxyvXh67coWdEEJIZNBjd0MFXy9c\nVPeB6bFHlp8KqOCxJyQkd9tNVSMcR/TYieFITExBVFSUx5KYmNLTaZEexPPn5a4v3oWeKFjYjewh\ndiamVbRCKWg96Yl3V36R/kZkb/LYI5tzNc4PvT327rxHQY+dhMSNolUF73ceWjEVUCG/jkJ33333\n9Yl38yrMOTEOhvDYw/ljhXD6qOID6+3tGTW/cHMwgjfa06+vFoHOnRsXl87loKfHrtVH7z9q6s7j\nKFDuAAKOSat2GqKwq1BI9EaFwql3flo3tnpjYQ/3Rp4Kr68Wen9pV3cVdtUvwOHErhO4j/I3T0O7\nUVYdcR+PSCf8sa7yOPX2EDvjiUc6f96xUDzxcD17PTzYnhiv3seYKn8HoLrHrr+H7RnT+1iKrI8v\nynxXzI2TpBqA45ttUYE7ePSBq1+wPkDgj5p656c34X7U1M698/OnN901t+EeL+EQzmul9/Gst1VJ\nbqBCPdBCGSumKz6mdJee3n6+6uOlnjH0wrXG9Pa3VbBiwvWwe/o1DKanvMduhElU+cSiHvX6gp4K\nOaikp7zHfoNqnWM9pxeu5xxZjHrUo17f0/MkaGGvrKzEqFGjMGLECLz88st+2yxevBgjRozA2LFj\nsX///k719aVO5xj1qEc96vV2PU80C7vT6cSiRYtQWVmJ+vp6bNq0CQcOHPBoU15ejsOHD6OhoQGv\nv/46Fi5cGHJf/7TqHKMe9ahHvd6u54lmYa+trYXNZoPVakVMTAzy8/NRVlbm0Wbr1q2YM2cOACAn\nJwetra04efJkSH0JIYToj2Zhb2lpQWpqqmvdYrGgpaUlpDbHjx8P2tc/jTrHqEc96lGvt+t5IRps\n2bJFCgsLXesbN26URYsWebSZMmWKfPTRR671iRMnyt69e0Pq+80TOVy4cOHCJYwlEJp/oGQ2m9HU\n1ORab2pqgsVi0WzT3NwMi8WCK1euBO2L65lppUAIIaSTaFoxWVlZaGhoQGNjI9rb27F582bk5eV5\ntMnLy8OGDRsAADU1NUhKSsLQoUND6ksIIUR/NN+xm0wmrF27Frm5uXA6nSgoKIDdbkdJSQkAYMGC\nBZg8eTLKy8ths9kQFxeHdevWafYlhBDStfT4X54eOHAAZWVlrhurFosFeXl5QS8CBw4cwPHjx5GT\nk4P4+HjX9lWrVuHBBx9ERkYGqqursXfvXmRmZmLixIkh57Rr1y7U1tbCZDJh7ty5GDhwIC5duoQV\nK1Zg3759GD16NFJSUvD444973CDu4PLly3j33XdhNpsxadIkvP3229i9ezcyMjIwceJEbNu2Dc3N\nzYiOjkZ6ejoef/xxJCYmhpwfIYRo0aN/efryyy9j+vTpAK4/KpmTk4Nr165h+vTpWL58ecB+M2bM\nwCOPPII1a9Zg9OjRKC0tBQA8++yzeO655zBnzhz87Gc/w9KlS/H111/jhRdewMqVK310Zs+eDQCY\nMGGCa9sbb7yBv/zLv8RXX32FZ599FmvWrAEA/OQnP0FbWxuWLl2Km2++GX/7t3+LCRMm4E//9E/x\nT//0Tzhz5oxLY968eSgvL0dxcTFmzZqFLVu24O6778a//Mu/4P7778fly5dRW1uLy5cv49ixY8jJ\nyUFVVVVqs3nMAAANOklEQVTkExoBp0+f7nSfc+fOdUEm4TFp0iQsXboUM2fOxDvvvOMRe/LJJ0PW\nCWceAM5FByrNg54Ybi60norpamw2m7S3t/tsv3z5sgwfPjxgP5PJJBcuXBARkSNHjsj48ePlH//x\nH8Vut8vYsWPl4sWLEh8fL62trSIicunSJUlISJCHHnpIpkyZ4lpiY2NlypQpkpiY6NIeP368nD59\nWkRE0tPTZfTo0SIikpmZ6ZHDgAEDxOl0yr//+7/LvHnzZNCgQZKbmyvr16+XjIwMERG5cuWKDB48\nWK5cuSIiIqNHj3bpXbx4Uf7sz/5MRESOHj0qY8aMkSVLlkh6erokJSVJcnKypKeny5IlS+T8+fN+\n56G1tVXS0tJkxowZ8vbbb3vEZs2aJQUFBa7+c+fOldGjR8vMmTPl4MGDcu7cOddy9uxZuf322+W9\n996Tc+fOiYjI+fPnZf78+TJmzBiZPn26/PjHP3bNyyeffCJpaWkyfPhwSU1NFZvNJi+++KIcPnzY\nI4fa2lpxOBwyY8YMOXbsmEyaNEkSExMlKytLdu3aJX/3d38nGRkZkpCQILfccotMmDBB1q1bJ+fP\nn/c7F3PmzJHq6mr5/e9/77Hs3btX+vfvL0uWLJH3339fpkyZIo8++qh8/fXXIiKSkZHhdy4ee+wx\nj7nomIdz587Je++95xpHqHMRExMj8+fP95mHcOfi1VdfDXhM+JuHYHNx7NgxSUlJ6dQxoTUXdrtd\n/vu//7tTx4TWXGRmZspf/MVfdOqY6IvnRyj0aGFPT0+XI0eO+Gw/cuSI9OvXT8aMGeN3iYqK8mh/\n4cIF+f73vy+DBw+WsWPHioi4/u1gwIAB8vjjj8vOnTulurpaqqqqZNiwYVJdXS133HGH6wUcN26c\nq8/UqVMlNTVVRETmzp0rtbW1IiJy6NAhufnmmz30L1++LKWlpfLnf/7nctNNN8n//d//yRdffCHx\n8fFy9uxZEble2EeNGiUiIufOnZPx48e7+sfFxcmKFSvkxIkTcu3aNREROX78uCxfvlxycnL8nsT3\n3XefxMbG+i1m8fHxsnr1annppZckPT1dli9fLkePHpXVq1cLALFarR6LyWSSfv36SVpamoiIzJ8/\nX/7mb/5Gjhw5Ir/85S8lISHBlet3v/tdj7no16+fPPXUU5KamipZWVnyy1/+UlpaWiQrK0vKy8vl\nnXfeEbPZLO+9955cu3ZNduzYIcnJyfLmm2/KsWPHZNWqVfLCCy/IoUOHZNasWZKWluZ3LqKioiQ5\nOVkcDofP4n1M/P3f/718+9vfljNnzgSci6ioKImNjfWZB6vVKv369XNphToXZrNZhgwZ4jMPIhLW\nXNx6663icDj8HhMA/M6D1lzce++9YjabO3VMaM3FsGHD5OGHH+7UMaE1F/fcc4/ccccdnTom+uL5\n8eyzz0owerSwV1RUyPDhwyU3N1cKCwulsLBQcnNz5Y477pCkpCTZt2+fHDlyxGfp16+f7N+/30Or\nvb1dbrnlFteznU6n0xU7f/68jBs3TlatWiUTJ06Uffv2iYiI1WoVEZHbb7/d9QKmpaXJ8ePHRUTk\n2LFjkpycLGlpaTJhwgTXgX7vvfdKenp6wHEtW7ZM0tLSZOTIkVJSUiJ2u10KCgpk2LBhMmzYMCko\nKJCRI0fKP//zP4uIyKlTp3wuFO4EOonj4uJkwIABrnbuxcxdr+Pi1MGtt94qubm58umnn7q2Wa1W\nj4vaXXfd5TqBRET69+/v+nSVk5PjodeRw7Vr1+TDDz+UH/3oRzJ06FCJj4+XkpISvzm45y0irouc\n0+mUmJgYv/OQkZEht99+u9+YyWTyeM1FRNatWycZGRkeeu55vPLKK5KQkOAzDyIS1lyMGzdOxowZ\n4zMPDofDY7+hzsWIESNk5MiRfsfbr18/OXTokN9YoLno37+/3HbbbX5zCHRMdIyrA/e5GDVqlNx5\n550+8+A+ps7MxZ133unxhiyUY0Kk750fgY4JjzkJ2qKLuXr1quzevVt+85vfyJYtW+Tjjz+WK1eu\nyLx58+R3v/ud3z55eXly4sQJn+2XLl2SXbt2+Ww/c+aM/OEPfxARkaamJpk2bZo8+eSTYrFYNHO7\nePGi/O///q+0trbK/v375ZNPPnHt9+DBg5p9jxw54vrYdvjwYXn33Xelrq5O/vjHP8pvfvMbOXDg\ngEf7SZMmycsvvywnT550bTtx4oSsWLFC4uLi/J7Eo0aNErPZ7LHNXzH767/+a482Y8aMkWPHjsm0\nadOkqKhIvvzyS7FarWI2m2XVqlXyyiuvyO233+5x4N56660yadIk+c///E95/vnnZfHixVJdXS3P\nPfecJCcn++R25coVGTlypHzve9+TzZs3i8Vikffff19ERKqrqyU+Pt71+paWlsr3v/99V9/Y2Fi/\nc/H444/L3Xff7Xe+H374YfmP//gPn+0VFRUe7zi95yI9Pd1nHkQkrLkYNmyYzJw502ceKioqZNCg\nQVJZWdmpuZg0aZIMGjTI7zExZswYn2Mo2FykpaWJzWbzOw+BjgmtuVi9erXEx8d36pjQmouMjAyx\n2+0+8yAS+Jjoi+eHIQp7T7Ft27aQPtJ0F+fOnZNnnnnG5SEmJSVJenq6PPPMM/Lmm2/6PYmffvpp\nef755322V1RUSHJysrS1tfnEPv/8c5k6daprvbS0VCZMmCBDhgyR559/XpYtW+ZaTp06JSLXP/LO\nmjVLdu7cKY899pjrnekDDzwgr732mjz22GN+x7Rnzx757ne/K/n5+dLY2CgTJ06UhIQEyczMlLff\nfluysrJk4MCB8u1vf9t1oTx9+rS89NJLAefi448/lh07drjusXRQXl4u9fX1fmP5+flB58J9HkQk\npLn44Q9/6DEX48eP93vPKNy5OHTokNx///1+5+HcuXOya9cu+eyzz0REpKqqSlauXCk7duwQEfEb\nmzFjht95aGho8DkmcnJygs7FiRMn5Hvf+57PPLz22msybdo0v/PQMRcOh8NnLkaNGiUZGRl+j4lf\n/OIXypwfHf+6HxMzZ84MeH4Emotwz49f/epXAee2gz5b2I3Em2++GTDWYeeEut1f7OLFi65PNIH2\npaWnlV+gWLAx1dfXy/bt2z1OvuLiYjGbzfLwww/LbbfdJr/97W9dMbPZLCNHjvQbGzduXMCi/9pr\nr7m2u89Dx4Vi+/btAS8i3vmJiJSUlPjdT0VFRVh6K1eudN2g3Llzp6t4L126VHJyciQrK0ueeeYZ\nycnJkZ///Ody7733yn333RcwtnDhwpAuBh988IG8+OKLHrGOPNz7Bdrurecv5t1v+/btmnreeH86\ncmfWrFmd2q4Vu3jxokexD1UvnPzC6eOPHn+OnQQnNTXV4+sZQomF00cVvY6/Xrbb7di/fz+Ki4vx\nyCOPYMyYMYiOjsYf/vAHNDY2Ytq0aZg5cyaKiopw880348yZM4iPj/eJWSwWxMXF+eitXr0aP/vZ\nz/DAAw94bAcQsI9WTG+9Z599FqtXr0ZGRgbuu+8+/O53v8MPfvADbN++HYcOHUJLSwva29sxdOhQ\nNDc3Y+DAgfj666+RnJyMr776yif2zDPP4NVXX8Xo0aN99EwmEy5dugSn0xly7PXXX0d0dDSGDBnS\n5Xpnz56FzWbz+AqSnTt34v7778cnn3yCCRMm+MTi4+MRFRWF7OzskPsYQS8qKgpbt271e+50oMyP\nWfd17rzzzoCx5uZmv/GGhgZcvnzZbyxQn2B67e3tndZraWnRVe/LL79Ec3Ozq0hPnToVjY2NEBHc\ndNNNAACr1Yrq6mpMnToVR48ehYi4/lDNO3bu3DkcPHjQo+g3Njbi17/+NUaOHInS0lKP7UVFRQH7\naMX01isrK8OIESPw4YcfehTop59+GikpKTCZTDCZTBg+fDgGDhwIALj55psRFRXlN/bBBx8E1At0\nMdCKvfPOOzCZTN2il5KSgszMTBQWFiI6Ohoigr179+Lpp59GYWEhEhISfGLx8fGwWq34q7/6q5D7\nGEEvJEJ+b0+6lCFDhgR8Cig6OtpvbNCgQTJo0KBO9TGCnslk8pibjsdZLRaLz42j9vZ2mTVrlgDw\n+6RUR8yfXkpKisdTGB3bi4qKpH///n77aMX01tN6fDc2NlYuXrwoIr5PgAWKjRkzxvX3GP4eB+4g\n1NjYsWM1Hy/WU++uu+4K+FTb1atX/cYCbdfqYwS9UGBhVwStp4DS0tL8xubNmycTJ07sVB8j6A0Z\nMsRvkX700Uf9flXptWvXZMuWLX6flLp27ZpkZmb61Rs6dKiPXqgXiu7Q03p817vwdXDmzBn55JNP\n/Ma+9a1vyZ49e/zqhXOhGD9+vCuPrtbruCBpPdUWKBZOHyPoacHCTpTj2LFjAYu0v8dZw9U7evSo\nlJaW+t2P1oUiUExvvVAe3+0MHX+c408v0MVAK9bU1OQ3j67Q896u9VRboFg4fYyg5w/ePCWEkF5G\nj34JGCGEEP1hYSeEkF4GCzshhPQyWNgJIaSXwcJOCCG9jP8H0vP6q0ajtyEAAAAASUVORK5CYII=\n",
       "text": [
        "<matplotlib.figure.Figure at 0x7de0850>"
       ]
      }
     ],
     "prompt_number": 13
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "selected_features = np.where(feature_importances >= 0.01)[0]\n",
      "train_X = train_X[:, selected_features]\n",
      "test_X = test_X[:, selected_features]\n",
      "print train_X.shape, test_X.shape"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "(800, 15) (200, 15)\n"
       ]
      }
     ],
     "prompt_number": 14
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "svc = SVC()\n",
      "gammas = [1e-4, 3e-4, 1e-3, 3e-3, 1e-2, 3e-2, 1e-1, 3e-1, 1., 3., 10.]\n",
      "gs = GridSearchCV(svc, {'gamma': gammas}, n_jobs=-1, cv = 5, scoring='accuracy')\n",
      "gs.fit(train_X, train_y)\n",
      "print gs.best_params_\n",
      "print gs.best_score_\n",
      "\n",
      "svc = SVC(**gs.best_params_)\n",
      "svc.fit(train_X, train_y)\n",
      "print svc.score(test_X, test_y)\n"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "{'gamma': 0.1}\n",
        "0.93375\n",
        "0.905\n"
       ]
      }
     ],
     "prompt_number": 15
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "gbt = GradientBoostingClassifier(random_state=0)\n",
      "learning_rates = [1e-4, 3e-4, 1e-3, 3e-3, 1e-2, 3e-2, 1e-1, 3e-1, 1., 3., 10.]\n",
      "gs = GridSearchCV(gbt, {'learning_rate': learning_rates}, \n",
      "                  n_jobs=-1, cv = 5, scoring='accuracy')\n",
      "gs.fit(train_X, train_y)\n",
      "\n",
      "print gs.best_params_\n",
      "print gs.best_score_\n",
      "\n",
      "gbt = GradientBoostingClassifier(**gs.best_params_)\n",
      "gbt.fit(train_X, train_y)\n",
      "print gbt.score(test_X, test_y)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "{'learning_rate': 0.1}\n",
        "0.93625\n",
        "0.88"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n"
       ]
      }
     ],
     "prompt_number": 16
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "from sklearn.ensemble import RandomForestClassifier\n",
      "forest = RandomForestClassifier(n_estimators=100, )\n",
      "max_features_choices = [0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0]\n",
      "gs = GridSearchCV(forest, {'max_features': max_features_choices}, \n",
      "                  n_jobs=-1, cv = 5, scoring='accuracy')\n",
      "gs.fit(train_X, train_y)\n",
      "print gs.best_params_\n",
      "print gs.best_score_\n",
      "\n",
      "forest = RandomForestClassifier(n_estimators=100, **gs.best_params_)\n",
      "forest.fit(train_X, train_y)\n",
      "print forest.score(test_X, test_y)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "{'max_features': 0.3}\n",
        "0.935\n",
        "0.885"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n"
       ]
      }
     ],
     "prompt_number": 17
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "## benchmark 2 - single variable decision tree as discretizer"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "from sklearn.tree import DecisionTreeClassifier\n",
      "class TreeTransformer(BaseEstimator, TransformerMixin):\n",
      "    def __init__(self):\n",
      "        pass\n",
      "    def fit(self, X, y):\n",
      "        n_samples, n_features = X.shape\n",
      "        self.trees_ = []\n",
      "        for ifeat in xrange(n_features):\n",
      "            ## IT IS VERY IMPORTANT TO SET max_depth = small value to avoid overfitting\n",
      "            self.trees_.append(DecisionTreeClassifier(max_depth=1))\n",
      "            self.trees_[ifeat].fit(X[:, ifeat].reshape((-1, 1)), y)\n",
      "        return self\n",
      "    def transform(self, X):\n",
      "        n_samples, n_features = X.shape\n",
      "        assert n_features == len(self.trees_)\n",
      "        return np.concatenate([self.trees_[ifeat].predict_proba(X[:, ifeat].reshape((-1, 1)))[:, :-1]\n",
      "                       for ifeat in xrange(n_features)], \n",
      "                    axis = 1)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 18
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "tt = TreeTransformer()\n",
      "train_y = y[train_index, :]\n",
      "train_X = tt.fit_transform(X[train_index, :], train_y)\n",
      "#train_X = ss.fit_transform(train_X)\n",
      "test_y = y[test_index, :]\n",
      "test_X = tt.transform(X[test_index, :])\n",
      "#test_X = ss.transform(test_X)\n",
      "print train_X.shape, train_y.shape\n",
      "print test_X.shape, test_y.shape"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "(800, 40) (800,)\n",
        "(200, 40) (200,)\n"
       ]
      }
     ],
     "prompt_number": 19
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "svc = SVC()\n",
      "gammas = [1e-4, 3e-4, 1e-3, 3e-3, 1e-2, 3e-2, 1e-1, 3e-1, 1., 3., 10.]\n",
      "gs = GridSearchCV(svc, {'gamma': gammas}, n_jobs=-1, cv = 5, scoring='accuracy')\n",
      "gs.fit(train_X, train_y)\n",
      "print gs.best_params_\n",
      "print gs.best_score_\n",
      "\n",
      "svc = SVC(**gs.best_params_)\n",
      "svc.fit(train_X, train_y)\n",
      "print svc.score(test_X, test_y)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "{'gamma': 3.0}\n",
        "0.84125\n",
        "0.79"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n"
       ]
      }
     ],
     "prompt_number": 20
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "## benchmark 3 - extra embedding tree (unsupervised)"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "from sklearn.ensemble import RandomTreesEmbedding\n",
      "rte = RandomTreesEmbedding(n_estimators=20, max_depth=2, n_jobs=-1)\n",
      "embedded_X = rte.fit_transform(X)\n",
      "print embedded_X.shape"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "(1000, 79)\n"
       ]
      }
     ],
     "prompt_number": 14
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "embedded_X = embedded_X.toarray()\n",
      "train_X = embedded_X[train_index, :]\n",
      "test_X = embedded_X[test_index, :]\n",
      "train_y = y[train_index]\n",
      "test_y = y[test_index]\n",
      "\n",
      "print train_X.shape, test_X.shape\n",
      "print train_y.shape, test_y.shape"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "(800, 79) (200, 79)\n",
        "(800,) (200,)\n"
       ]
      }
     ],
     "prompt_number": 15
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "## feature selection\n",
      "trees = ExtraTreesClassifier(n_estimators=50, max_features=0.1, random_state=0, n_jobs=-1)\n",
      "trees.fit(train_X, train_y)\n",
      "pd.DataFrame(trees.feature_importances_).plot(kind = 'bar')"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 16,
       "text": [
        "<matplotlib.axes.AxesSubplot at 0x90f1cd0>"
       ]
      },
      {
       "metadata": {},
       "output_type": "display_data",
       "png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAEBCAYAAACUmXXrAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJztnXtcFXX+/18oIHLz4AUwD3qQOwmHU6ykresxUsRVuq6h\nuwaK+yXTzDJX+7Ym7gXxW6aV9VD3a6buZlablwpp8ytotSKm2O6mFv6CVEQslBW1DYX37w86w7nM\nmTmXOTd8Px8PyvfM5/Oe1+czn/nMOa+ZM+NHRASGYRjmpqCXpwUwDMMw7oMnfYZhmJsInvQZhmFu\nInjSZxiGuYngSZ9hGOYmgid9hmGYmwjZSb+iogLJyclISEjAypUrRcvMnz8fCQkJ0Gq1qK2tNVnX\n0dEBnU6HKVOmCMtKSkqgVquh0+mg0+lQUVHhZDMYhmEYW/CXWtnR0YF58+Zh7969GDJkCH7yk58g\nLy8PKSkpQpny8nKcOnUKdXV1OHToEObMmYPq6mph/YsvvojU1FS0tbUJy/z8/PDkk0/iySefdEGT\nGIZhGGtIftKvqalBfHw8NBoNAgICkJ+fj127dpmU2b17NwoKCgAAWVlZaG1tRXNzMwDg7NmzKC8v\nx+zZs2H+GzD+TRjDMIz7kZz0GxsbERMTI8RqtRqNjY02l3niiSfw3HPPoVcvy828/PLL0Gq1KCoq\nQmtrq1ONYBiGYWxD0t7x8/OzKYnYp/j3338fkZGR0Ol0qKqqMlk/Z84cPPvsswCApUuXYuHChdi4\ncaPD22cYhmG6kXJSJD/pDxkyBGfOnBHiM2fOQK1WS5Y5e/YshgwZgr///e/YvXs3YmNjMW3aNOzb\ntw8PP/wwACAyMhJ+fn7w8/PD7NmzUVNTIyl+2bJlICLhzzy2pYw7crAubhvr8p6cPVmXVBk5JCf9\nzMxM1NXVoaGhAe3t7di+fTvy8vJMyuTl5WHLli0AgOrqaqhUKkRHR6O0tBRnzpxBfX093nzzTdx1\n111CuaamJqH+jh07kJaWJimyoaFBMraljDtysC7352Bdns/JurwzhzUk7R1/f3+sXbsWOTk56Ojo\nQFFREVJSUrB+/XoAQHFxMSZNmoTy8nLEx8cjJCQEmzZtEs1lbNUsXrwYx44dg5+fH2JjY4V8DMMw\njIshL8Ygr7Ky0mS5eWxLGXfkYF3uz8G6PJ+TdXlXDrlp3e/HQl6Jn5+fTR4VwzAM04XcvOkTj2Ew\nv/vHPLaljDtysC7352Bdns/pi7r69+8v3Eziq3/9+vWTbb8Ykp4+wzBMT+TSpUs+7yI4eks72zsM\nw9x09IS5xVobeoS9wzAMwyiDT0z6nvb/PJmzJ+tSIgfr8nxOX9bVk7C1bT4x6TMMwzDKwJ4+wzA3\nHWJzS3h4f7S1XXLZNsPCInD58kWbyl68eBFFRUX46KOPMHDgQKxYsQLTpk0zKeOop8937zAMwwA/\nTviu+5DZ1mb73TZz585FUFAQLly4gNraWvz85z+HVqtFamqq0zp8wt7xFv/PW31JX9WlRA7W5fmc\nvqzLG7l69Sreffdd/P73v0dwcDDuvPNO3HPPPdi6datkPVvb5hOTPsMwzM3CV199BX9/f8THxwvL\ntFotvvjiC0Xys6fvIoz9QXu8PIZhXI/Y3NL1YydXzje2zWcff/wxpk6davI04j/96U944403UFlZ\n2Z2NPX3vwtgftMfLYxjm5iY0NBSXL182Wfbvf/8bYWFhiuT3CXvHW/w/R3IC3qmrp+RgXZ7P6cu6\nvJHExETcuHEDp06dEpZ9/vnnGDFihGQ9W9vmE5M+wzDMzUJISAjuv/9+PPvss7h27Ro++eQTvPfe\ne5gxY4Yi+dnTdxGm/qDvtoNheiLe7OkDXQ+EmzVrlnCffllZGfLz802zuerZOxUVFUhOTkZCQgJW\nrlwpWmb+/PlISEiAVqtFbW2tybqOjg7odDpMmTJFWHbx4kWMHz8eiYmJmDBhAlpbW+VkMAzDuJSw\nsAgAfi7768pvGxEREdixYweuXLmChoYGiwnfGSQn/Y6ODsybNw8VFRU4fvw4tm3bhhMnTpiUKS8v\nx6lTp1BXV4cNGzZgzpw5JutffPFFpKammjwGtKysDOPHj8dXX32F7OxslJWVSYr0Fv/PkZzs6bs2\nB+vyfE5f1mXM5csXLV4+ruSfq+/gk2qbMZKTfk1NDeLj46HRaBAQEID8/Hzs2rXLpMzu3btRUFAA\nAMjKykJrayuam5sBAGfPnkV5eTlmz55t8nXDuE5BQQF27txpc8MYhmEYJ5B6l+Lbb79Ns2fPFuKt\nW7fSvHnzTMpMnjyZPv30UyHOzs6mI0eOEBHRgw8+SEePHqWqqiqaPHmyUEalUgn/7uzsNImNkZHn\n1QAggH788912MExPpCcck9baINc2yfv0bX0zC5ldNCAivP/++4iMjIROp5P82mF49Zc1CgsLodFo\nAAAqlQoZGRnQ6/UAur/OeGtszdrxFn0cc3wzxz0BQ5uqqqrQ0NBgWyWpM8LBgwcpJydHiEtLS6ms\nrMykTHFxMW3btk2Ik5KSqKmpiZ5++mlSq9Wk0WgoOjqagoODacaMGSZliIjOnTtHSUlJkmcsT79d\n3pEYwif9SpMzr6d19bQcrMvzOX1RV0RExI/HqO/+hYeHi7ZNZlonSU8/MzMTdXV1aGhoQHt7O7Zv\n3468vDyTMnl5ediyZQsAoLq6GiqVCtHR0SgtLcWZM2dQX1+PN998E3fddZdQLi8vD5s3bwYAbN68\nGffee6/cuYlhGEYxLl7sumhbWVlpcrFVLnakjqtymF9ftRXZ+/T37NmDBQsWoKOjA0VFRXj66aex\nfv16AEBxcTEACHf4hISEYNOmTbjttttMcuzfvx+rVq3C7t27hQ6fOnUqTp8+DY1Gg7feegsqlcpS\nHN+nzzAMYxdy8yb/OMtF8KTPMIwn6BEvRjdcrLAW21LGHTnEclq7mOtpXT0lB+vyfE7W5Z05rOET\nkz7DMAyjDGzvuAi2dxiG8QQ9wt5hGIZhlMEnJn1v8cgcycmevmtzsC7P52Rd3pnDGj4x6TMMwzDK\nwJ6+i2BPn2EYT8CePsMwDCPgE5O+t3hkjuRkT9+1OVytKzy8v/BQwPDw/qzLRTlYl/I5rOETkz7D\neIq2tkvosukqf/y3d+Ctuhjvhz19F8Gefs/AW/ejt+piPA97+gzDMIyAT0z63uKROZKTPX3X5nCX\nLm/Yj76kyxty9mRdtpYRwycmfYZhGEYZ2NN3Eey59gy8dT96qy7G87CnzzAMwwjITvoVFRVITk5G\nQkICVq5cKVpm/vz5SEhIgFarRW1tLQDgP//5D7KyspCRkYHU1FQ8/fTTQvmSkhKo1WrodDrodDpU\nVFRIavAWj8yRnN7guXprfymRw126vGE/+pIub8jZk3XZWkYMf6mVHR0dmDdvHvbu3YshQ4bgJz/5\nCfLy8pCSkiKUKS8vx6lTp1BXV4dDhw5hzpw5qK6uRlBQECorKxEcHIwbN27gpz/9KT799FPceeed\n8PPzw5NPPoknn3zSJpEMc7MTHt5fuB8/LCzCw2oYX0bS0z948CCWL18ufBIvKysDACxZskQo88gj\nj2DcuHF46KGHAADJycnYv38/oqKihDLXrl3D2LFjsXnzZqSmpmL58uUIDQ3FwoULpcWxp894GG/Z\nj+Y6uvC8Lsb7cMrTb2xsRExMjBCr1Wo0NjbKljl79iyArm8KGRkZiIqKwrhx45CamiqUe/nll6HV\nalFUVITW1lb7WsUwDMM4hKS90/XpQh7zs4qhXu/evXHs2DH8+9//Rk5ODqqqqqDX6zFnzhw8++yz\nAIClS5di4cKF2Lhxo2juwsJCAIBGo4FKpUJGRgYAQK/XCx6W8b8djQFgzZo1yMjIUCzu8luPCdup\nqqrCsWPHsGDBAptjZ9pkLfaW/nKkP9zRX+bruqgStuGp/jIfT94yvpQcm0r2l6+ML2f7x6Cturoa\n0dHRkIUkOHjwIOXk5AhxaWkplZWVmZQpLi6mbdu2CXFSUhKdP3/eItfvfvc7eu655yyW19fX04gR\nI0S3b5BXWVlpstw8tqWMO3IYxwAIIAIqybibPa2rp+VwtS5v2Y/mOrxFl1I5WJdyOWSmdZJce/36\ndRo+fDjV19fTDz/8QFqtlo4fP25S5oMPPqDc3Fwi6jpJZGVlERHRt99+S5cuXSIiomvXrtGYMWNo\n7969RER07tw5of4LL7xA06ZNExcnI96b6T4ofbsdNzvesh/NdXiLLsb7kBsPkvaOv78/1q5di5yc\nHHR0dKCoqAgpKSlYv349AKC4uBiTJk1CeXk54uPjERISgk2bNgEAmpqaUFBQgM7OTnR2dmLGjBnI\nzs4GACxevBjHjh2Dn58fYmNjhXwMwzCMi3HTycchDPI8/XXJkRhe9PXbW/tLiRyu1uUt+9Fch7fo\nUioH61Iuh9y0zr/IZRiGuYngZ++4CG+5v5txDm/Zj3yfPmMr/OwdhmEYRsAnJn1bnjEhV8YdOcRy\nesOzUby1v5TI4S5d3rAfxXR4iy5f2Y/ekNNdOazhE5M+wzAMowzs6bsIb/GCGefwlv3Inj5jK+zp\nM4yChIf3h5+fn/AXHt7f05IYxi58YtL3Fo9MCQ/WW3T1lBzu0mXYj12PNyYAlQBIeNyxu3Sxp8+6\n7Ckjhk9M+gzDMIwysKfvIrzFC2acQ9pL71rmjn3Lnj5jK+zpMwzDMAI+Mel7i0emhAfrLbp6Sg53\n6XLWS1dKF3v6rMueMmL4xKTPMAzDKAN7+i6CPf2eAXv6jK/Bnj7DMAwj4BOTvrd4ZEp4sN6iq6fk\ncJcu9vRdm4N1KZ/DGrKTfkVFBZKTk5GQkICVK1eKlpk/fz4SEhKg1WpRW1sLAPjPf/6DrKwsZGRk\nIDU1FU8//bRQ/uLFixg/fjwSExMxYcIEtLa22iSWYRiGcRKpN6zcuHGD4uLiqL6+ntrb22XfkVtd\nXS28I5eI6OrVq0TU9a7drKws+uSTT4iIaNGiRbRy5UoiIiorK6PFixeLbl9GnlcDfodpj8B8P5rG\n7tu30jp4fDHdyI0HyU/6NTU1iI+Ph0ajQUBAAPLz87Fr1y6TMrt370ZBQQEAICsrC62trWhubgYA\nBAcHAwDa29vR0dGBiIgIizoFBQXYuXOn82cvhmEYRhbJF6M3NjYiJiZGiNVqNQ4dOiRb5uzZs4iK\nikJHRwduv/12/L//9/8wZ84cpKamAgCam5sRFRUFAIiKihJOEmIUFhYCADQaDVQqFTIyMgAAer1e\n8LCM/+1oDABr1qxBRkaGYnGX33pM2E5VVRWOHTuGBQsW2Bw70yZrsbf0lyP94Y7+Ml/XhXTsjv4y\nH0/eMr6UHJtK9pevjC9n+8egrbq6GtHR0ZBF6mvAO++8Q7NnzxbirVu30rx580zKTJ48WbBtiIiy\ns7PpyJEjJmVaW1spKytLeHGvSqUyWR8RESH5NcXTLxp2JIYXvbjaW/tLiRyu1mW+H01j941RaR38\nYvSbTZdUGZlpnSTv06+urkZJSQkqKioAACtWrECvXr2wePFiocwjjzwCvV6P/Px8AEBycjL2798v\nfJI38Pvf/x7BwcFYuHAhkpOTUVVVhejoaDQ1NWHcuHE4efKkxfb5Pn3G0/B9+u4nPLy/8PTSsLAI\nXL580cOKfAun7tPPzMxEXV0dGhoa0N7eju3btyMvL8+kTF5eHrZs2QKg6yShUqkQFRWF7777Trgr\n5/vvv8dHH30kfBXJy8vD5s2bAQCbN2/Gvffe63gLGYbpUXQ/vrr70dWMckhO+v7+/li7di1ycnKQ\nmpqKhx56CCkpKVi/fj3Wr18PAJg0aRKGDx+O+Ph4FBcX49VXXwUANDU14a677kJGRgaysrIwZcoU\nZGdnAwCWLFmCjz76CImJidi3bx+WLFkiKVKJe1bdkUMspzfcR+2t/aVEDnfpssfTd6UuZ3X40n70\nhrZ5a3/ZWkYMyQu5AJCbm4vc3FyTZcXFxSbx2rVrLeqlpaXh6NGjojn79++PvXv32iSQYRiGUQ5+\n9o6LYE+/Z+AKT98Rz/pm8vT52HEOfvYOTN9ryu80ZTwNe9aMJ/GJSd9Zf6vrwJJ+p6kt22Ff0vty\nuEuXKzx9R3Kwp+95Xb6Swxo+MekzDMMwynBTePpyHqEr7gtmX7Jn4ApP35GxwZ4+Yyty86bs3Ts3\nA90eK9DW5iddmGEYxofxCXtHGX9LPoezHiL7ku7P4S5d7Om7NgcfO8rnsIZPTPoMwzCMMrCnb8N6\nV2yT8Q3Y03c/fOw4B9+nzzAMwwj4xKSvjL8ln4M9fd/L4S5d7Om7NgcfO8rnsIZPTPoMwzCMMrCn\nb8N6V2xTCYx/XwDws8ddAXv67oc9fefg+/R7MMa/L+iK+TcGDMNI4xP2jjL+lnwOX/X0lfCXvdWX\n9BZd7Om7Noenjh1f7S9by4ghO+lXVFQgOTkZCQkJWLlypWiZ+fPnIyEhAVqtFrW1tQCAM2fOYNy4\ncbj11lsxYsQIvPTSS0L5kpISqNVq6HQ66HQ64XWMDONpDE9kHTduHD+RlemZSL1A98aNGxQXF0f1\n9fXU3t5OWq2Wjh8/blLmgw8+oNzcXCIiqq6upqysLCIiampqotraWiIiamtro8TERDpx4gQREZWU\nlNCqVaskX97747UG2TK2AOEl0uI55da7YpvKb6MrDguLEF6cHRYm/sJ5xjrm+006dmzfOjI25HT1\nJHpy29yBXJ9JftKvqalBfHw8NBoNAgICkJ+fj127dpmU2b17NwoKCgAAWVlZaG1tRXNzM6Kjo4V3\n4oaGhiIlJQWNjY3GJxuHTlL8bHxp+FntDMNIITnpNzY2IiYmRojVarXJxG2tzNmzZ03KNDQ0oLa2\nFllZWcKyl19+GVqtFkVFRcIL1K1h7FWJPRvfvIxYfDN5+ra01Vd8SU/pkutTqT629sHE3jHpiK6e\ntB+9oW3e2l+2lhFD8u6drlun5DH/1G5c78qVK3jwwQfx4osvIjQ0FAAwZ84cPPvsswCApUuXYuHC\nhdi4caNo7sLCQgBdDVKpVBbrq6qqoNfrhX8DsIiNSlvUNeWYaXTsmEk+e+Ou7XXnrKqqwrFjx0z0\nycXSei3bZLHWqI61/nE0drZ/HOkPZ/rLsYlHCvH63R9MgLa2ccJ6Q/u7sW+8mY8nV48vW2OlxpPS\nx6O3ji+ljz/Duurqarz++uuQQ/I+/erqapSUlAgXWlesWIFevXph8eLFQplHHnkEer0e+fn5AIDk\n5GTs378fUVFRuH79OiZPnozc3FwsWLBAdBsNDQ2YMmUK/vnPf1qKE7nfVIl7nJXI6ew2lcB0G13b\n6YLvcXYUufvhTeOuZfaOJ75PXxq+T985nHr2TmZmJurq6tDQ0ID29nZs374deXl5JmXy8vKwZcsW\nAF0nCZVKhaioKBARioqKkJqaajHhNzU1Cf/esWMH0tLS7G4YwzAMYz+Sk76/vz/Wrl2LnJwcpKam\n4qGHHkJKSgrWr1+P9evXAwAmTZqE4cOHIz4+HsXFxXj11VcBAJ9++in+/Oc/o7Ky0uLWzMWLFyM9\nPR1arRb79+/H6tWrJUXK2TRiZey3duTLGMdivq2zOa3lsLdttrRVCevDG3K4Spdcn8r3sbwue8eG\nIzp8eT96Q9u8tb9sLSOG7C9yc3NzkZuba7KsuLjYJF67dq1FvZ/+9Kfo7OwUzWn4ZuAKXPHqQzG6\nfVs9/xKWYRifweeeveOIX+oOj9Ud1wmkNXRtpwv2Qx2FPX3Pw56+c/Dz9BmGYRgBn5j0lfBLlfb0\nXZWTPX3P5/xxqV0xe/rO5WBPX/kc1uCnbDIMw8B91wM9DXv6NuS0po09/Z4He/qex1Oefk+5lsCe\nPsMwDCPgE5M+e/r26WBP3/kc7Om7N4e3ePo96dixhk9M+gzDMIwysKdvQ05r2tjT73mwp+95bmZP\nX4n3XvM7chmGYXwEd7z32ifsHfb07dPRk3xJ9vTt09WT9uPN7OkrocMaPjHpMwzDMMrAnr4NOa1p\nY0+/58Gevue5mT19sWPaXh18nz7DMIwD9NT3cfvEpM+evn06vMWXZE9fWhd7+tI5Pe3pO/o+bvb0\nGYZhGK9B1tOvqKjAggUL0NHRgdmzZ5u8H9fA/PnzsWfPHgQHB+P111+HTqfDmTNn8PDDD+PChQvw\n8/PDf/3Xf2H+/PkAgIsXL+Khhx7CN998A41Gg7feekv0pefs6dujoWs7XfRMr9cdsKfvebzF0+/C\nvTo87ul3dHRg3rx5qKiowPHjx7Ft2zacOHHCpEx5eTlOnTqFuro6bNiwAXPmzAEABAQEYPXq1fji\niy9QXV2NV155BSdPngQAlJWVYfz48fjqq6+QnZ2NsrIyuxrFMAzDOIbkpF9TU4P4+HhoNBoEBAQg\nPz8fu3btMimze/duFBQUAACysrLQ2tqK5uZmREdHIyMjAwAQGhqKlJQUNDY2WtQpKCjAzp07JUWy\np2+fDvb0nc/Bnr57c3ijpy8e93BPv7GxETExMUKsVquFiVuqzNmzZ03KNDQ0oLa2FllZWQCA5uZm\nREVFAQCioqLQ3Nxsk1iGYRjGOSQfw9DlL8kj7ll2ceXKFTz44IN48cUXERoaKroNqe0UFhZCo9Gg\nqqrKyPfXC+urqqqg1+uh1+vNznR6YX133L3esrw4hvzWYrEzcvd6y3WOxIZ85nH3NoxjQKx/zOsb\nt9+RWCy/vbHSnw7l+svWuAs9TBGLu8uLjz9TfWKaxdZb7z/zuqaxp/pLifGk1PHozPiyZdsGvc7G\n8u3Xw9r4Mm+v4f8NDQ14/fXX5RtCEhw8eJBycnKEuLS0lMrKykzKFBcX07Zt24Q4KSmJzp8/T0RE\n7e3tNGHCBFq9erVJnaSkJGpqaiIionPnzlFSUpLo9sXkASCAfvyzbb29dcLCIn5c1vVva9rs1aE0\nptsgQbOrt9uTMe8/6Vi5Meqsrp6Ep9rmDX1sy/iyJYcUkvZOZmYm6urq0NDQgPb2dmzfvh15eXkm\nZfLy8rBlyxYAQHV1NVQqFaKiokBEKCoqQmpqKhYsWGBRZ/PmzQCAzZs3495775U8MVmekc1j+705\n8bN817Luhx5VSt6fa09Oa2Vs+fQhX8Z+Xc5+qvaWHK7SJe/tyuWQ12Xv2HBEhy/vR0+0TX6/u6e/\nlDiGrSFp7/j7+2Pt2rXIyclBR0cHioqKkJKSgvXr1wMAiouLMWnSJJSXlyM+Ph4hISHYtGkTAODT\nTz/Fn//8Z6Snp0On0wEAVqxYgYkTJ2LJkiWYOnUqNm7cKNyyyTAMw7gefvaOSJ0upO/P5fv0eyZ8\nn77n4fv0PXifPsMwDNOz8IlJ392evu117M/Jnn4X1h5m5WldRkvtitnTdy4He/rS23Wbp88wrqL7\nYnkV2trGeVoO40GMXxEYFhbhYTU9H/b0Rep0wZ6+K/GGZ5eLwZ6++/GWtnmDDvb0GYZhGEXxiUmf\nPX37dPiCp2+01Cd0safv6hyWOdnTdzaHOD4x6TMMwzDKwJ6+SJ0u2NN3Jezps6dvwFva5g062NNn\nGIbxYbzxPbs+Mel72tM33nGmO8/+nOzpW2TxCV3s6bs6h2XOnuDpO/qeXfb0PYzxA9jMdx7DMIwv\nwZ6+SJ0upH1c8zLs6duHef+EhUWY/EDn8uWLXqGrC/b0XYm3tM0VOuzd1+7w9PkXuYxX0P1tCmhr\ns+3lPQzD2I9P2Due9vSVzMmevkUWydhbdbGnr3QOy5w9wdN3NAd7+gzDMIwisKcvUqcL9vRdibf4\nuOawp+9+vKVtvurpmz+wrq3tknP36VdUVCA5ORkJCQlYuXKlaJn58+cjISEBWq0WtbW1wvJZs2Yh\nKioKaWlpJuVLSkqgVquh0+mg0+lQUVEhJ4NhGIYRoft6mG13FkpO+h0dHZg3bx4qKipw/PhxbNu2\nDSdOnDApU15ejlOnTqGurg4bNmzAnDlzhHUzZ84UndD9/Pzw5JNPora2FrW1tZg4caKkSPb05crY\nr8s7fFzAWz1q9vTdncMyJ3v6Uutt7UNLJCf9mpoaxMfHQ6PRICAgAPn5+di1a5dJmd27d6OgoAAA\nkJWVhdbWVpw/fx4AMGbMGEREiD8f25e/jjIMw/gsJMHbb79Ns2fPFuKtW7fSvHnzTMpMnjyZPv30\nUyHOzs6mzz77TIjr6+tpxIgRJnVKSkpo2LBhlJ6eTrNmzaJLly6Jbl9MHgAC6Mc/29bbW0c6Fi9j\nr04lcESXtyDX596qS6zP5XLYu94RXb6Mt7TNFTrszWHL+LJFtxSS9+l3XVSw6cRhV705c+bg2Wef\nBQAsXboUCxcuxMaNG0XLFhYWQqPRAABUKpXF+qqqKuj1euHfYuuNIol1lusd/Ypv0CO33tm4exvG\nsZlCkf7xltje/nRXbDviep1d72v9pVx/e7Z93dsUj5UYT7Ycj6bbNq1rvXwJgAaL7YkidUY4ePAg\n5eTkCHFpaSmVlZWZlCkuLqZt27YJcVJSEp0/f16IxT7pGyO13iCvsrLSZBlQaXEWNJSBcNarNPuk\nb1rHMmd3HenY+FOA7TnNddoaS5VxRJejOpTOIdfn3qpLrM/Fc9g3Rp3V5WjbHanjmj43H8Pub5u4\nDuv70ZGccnVsGV/SdeQ/6Ut6+pmZmairq0NDQwPa29uxfft25OXlmZTJy8vDli1bAADV1dVQqVSI\nioqSPNE0NTUJ/96xY4fF3T0MwzCMOIYHQI4bN86xJ3dKnhKIqLy8nBITEykuLo5KS0uJiGjdunW0\nbt06oczcuXMpLi6O0tPT6ciRI8Ly/Px8Gjx4MAUGBpJarabXXnuNiIhmzJhBaWlplJ6eTvfcc4/J\nNwNjxORBxiMTW29vHenYNr9Pbr0thIVFCNsKC4uQ2YZyPqTcdpVArs89hSNjQS6Hvesd0eXLuKJt\njoxhV+iwN4cj40ssloJ/nCVSpwtrsXgZe3Xagn26bdOlxHaVwFt+kGMO/zjL/XjDj6K8RYctP86y\nZYxKbccnHsPgyH2v9t9XLV9HiZzmZeRi27Zjvy757Tqfw5G2eeLebCXGghJj1Jf7y/kcljndcezY\neyw5psNPa5XJAAAgAElEQVT+HMrMV+LwUzZ9CPOfWzMM49t44phme0ekTheut3fMd7j5M+RdocsW\n2N5he8edeIOt4ikd0tt0pE4PsXd6KvY+M0MJvPGdnZ6E+4O52fCJSb8ne/ru1mX86kepd3beLJ6+\ntXeYutvTFzv5eGN/uS6HZc6bydN3zXwljk9M+gzT07F+8mEYZWFPX6ROF6739H1VlxJ4i0dtXx+7\nztN3dix48WEsC3v67Om7HG/1cb1VF8P4OnxsdeMTk77SHplSPq7Snr636rKWoyd4+uIa5HUp7enb\nltP+Ouzpdy1z9FoWe/oMwzCMT3NTevpdOOfjmpdRwjv3Vl3s6bOn70rc4aXbkpM9fYZhGKbH4ROT\nvis8MiV8XFd4596qiz196dhbfFz29KVzKHE8sqfPMAzD+Azs6XuRd+6tutjT7xmevtyznjwFe/ru\n9fT5KZsMc5PQfdsi0NbmJ12Y6bHI2jsVFRVITk5GQkICVq5cKVpm/vz5SEhIgFarRW1trbB81qxZ\niIqKsngd4sWLFzF+/HgkJiZiwoQJaG1tldTAnr69deS2aUsZ+Rye9vSt/eDG3Z6+uA7LnN7g6Svh\nUbOnb68O+Rxe4+l3dHRg3rx5qKiowPHjx7Ft2zacOHHCpEx5eTlOnTqFuro6bNiwAXPmzBHWzZw5\nExUVFRZ5y8rKMH78eHz11VfIzs5GWVmZTWIZxhhveV6NK3SEh/fHuHHj+BekjPKIvEJR4O9//zvl\n5OQI8YoVK2jFihUmZYqLi+nNN98U4qSkJGpqahLi+vp6GjFihEmdpKQk4b24TU1NlJSUJLp9MXkA\nCHa8P1JsmXOxeBl7dfqSLlcg1zZHcrhChyN97okx6khb3fEuZEd0uWO/ekqHfdt0pA5ktUp+0m9s\nbERMTIwQq9VqNDY22l3GnObmZkRFRQEAoqKi0NzcLFmeYRjl8cT7HBjPI3kht+sqsTwkenXZNgxe\nqDUKCwsBABqNBiqV6selVQD0Xf+qqoJer7fqkXUvF19vyhob4gyr8Zo1a5CRkQG9Xm+0jWPdW6yq\nwrFjx7BgwQKzHFKxnG5DrDdbphe2CUCkj6qE9VL9Z6zdgKF95u21NzbvH7n+kus/c422x1Uw7T/L\n9jsfd7ffaIl5CUiNL6X7y1yno/1nX1+LjUVTHaZajSKj8SN2J5Kz46t7m64bX7Yeb7Z5+sb9UwKg\nWqSMCFJfAw4ePGhi75SWllJZWZlJmeLiYtq2bZsQG1s3RNbtHYMFdO7cOVl7p7Ky0mQZUGnx1cdQ\nBsJXnUqzr86VZl+HzOPuOtKxeA5xnaY65NrirbrM14vVsTeWa5vtOayPBUdyKNHnnhijSrTVkf5z\ntI6zuuwd50ocj0q0zbyMLePL3rbJTOskufb69es0fPhwqq+vpx9++IG0Wi0dP37cpMwHH3xAubm5\nRNR1ksjKyjJZLzbpL1q0SDh5rFixghYvXiwuTkR8dwNtX2++zLlYvIy9On1JlyuQa5sjOVyhw5E+\n98QYVaKtnsIRXfb2sRLHoznG10QA8esizul0rG1OTfpEROXl5ZSYmEhxcXFUWlpKRETr1q2jdevW\nCWXmzp1LcXFxlJ6eTkeOHBGW5+fn0+DBgykwMJDUajW99tprRETU0tJC2dnZlJCQQOPHj6dLly6J\ni3Ng0uJJnyd9JXXwpO96fHXSd2SCtm+bHpr0PYlBvC/YO+Z3QojpkGuLK3SJb9M+XebrxerYG8u1\nzfYc1seCIzmU6HNPjFEl2upI/zlapyfYO2JjwRad1nOIjy972yY36fMvchWi+15tvU/92tH4ghjQ\ndVGMYZieCz97R6Fn3MjFYt3sDc/ecSSHEijxnBO5sWDLs2aUePaOXB13jFFH+suRHK5Aiefk2DfO\nlXn2jml523Q4cjza2zYAkvuSn7LJ9Fj4PnTG3fjCu3h9YtJ3xbMsnH3eiqM5HXnWh7M6HHtuh3wO\nJZ7Z4o5nydi/D+R1OTIW3DFGPfWMG+fHgv26lBjnSjx7x7iM+CM57MshFrvt2TsMwzBMz4I9ffb0\n7c6hBO7w9F3/TgPb6rCnL01P8vTN69iXgz19hmEYRmF8YtJnT985Hezp25+DPX17crKn7+oc7Okz\nDMMwDsGePnv6dudQAvb02dM3wJ4+e/oMwzCMi/CJSZ89fed0sKdvfw729O3JyZ6+q3Owp88wDMM4\nBHv67OnbnUMJ2NNnT98Ae/rs6TMMwzAuwicmffb0ndPhrKdv7SFS7OnbE7Onr7Qu9vTltyOG7KRf\nUVGB5ORkJCQkYOXKlaJl5s+fj4SEBGi1WtTW1srWLSkpgVqthk6ng06nQ0VFhU1iGc8g/hAphmF8\nEqk3rNy4cYPi4uKovr6e2tvbZd+RW11dLbwjV6puSUkJrVq1SmrT9OO1BtFlAP34Z9t682XOxY7l\nsNY+V+uyb5uOt8VelNiGI2PBXh2u6C9XjFF39bkrUOLYcUVO+7ahzFgQy2mvTrl9KflJv6amBvHx\n8dBoNAgICEB+fj527dplUmb37t0oKCgAAGRlZaG1tRXnz5+XrdulnWEYhnEnkq9LbGxsRExMjBCr\n1WocOnRItkxjYyPOnTsnWffll1/Gli1bkJmZiVWrVkGlUolqKCwsBABoNBqjMlUA9ACA4OAwfP/9\nFQBA376hRjWruv4reGFVMMU8BoA1NsQZdsRVAI51R1VVOHbsGBYsWGBWRyqW022I9WbL9MI2AUCv\n15v5grbmtRYDa9asQUZGBvR6vUOxef/I9Zdc/xm3z3QbEF1vOjb0MMW8vc7G3e03WmJeAvaMJ2f7\ny1yn9f6Rju0pKxab6zDVahRZjB/T/nF2fHXndGR86dFNldW4qqrKxvabxuKevnH/lACoFikjgtTX\ngHfeeYdmz54txFu3bqV58+aZlJk8eTJ98sknQpydnU2fffaZZN3m5mbq7Oykzs5OeuaZZ2jWrFmi\n2zfIc+ZF1tbr2J7DNJbKYT2nAW97MbojbRNri72xLfvNvI74y+et63J8Hyg7FmzR5exYsL3P7Ruj\ntsSO1OmpL0Z3diyI5bS3bTLTOkmuPXjwIOXk5AhxaWkplZWVmZQpLi6mbdu2CXFSUhKdP3/eprpE\nRPX19TRixAhxcSYHt/G/SWSAiMemnWRbHenYsRzW2udqXfZt0/G22Isr+kuuvCM5XdFfrhijrupz\nd6DEWHBFTvu2ocxYEMtpr065fSnp6WdmZqKurg4NDQ1ob2/H9u3bkZeXZ1ImLy8PW7ZsAQBUV1dD\npVIhKipKsm5TU5NQf8eOHUhLS7Oqgd9zyjAMoyCSpwQiKi8vp8TERIqLi6PS0lIiIlq3bh2tW7dO\nKDN37lyKi4uj9PR0OnLkiGRdIqIZM2ZQWloapaen0z333EPnz58X3TaEs1il2VnNPCZie0c8h/g2\nnWubWFvsjW3Zb/b2l3kdx3MqOxZs0aXEOLetz+0bo7bEjtRhe0csh/j4srdtctO65IVcAMjNzUVu\nbq7JsuLiYpN47dq1NtcFIHwzYBiGYdyL1z97pwuDRMeeScLP3rF1m463xV7c/2wUfvYOP3vHleNL\nvI59OfjZOwzDMIzC+MikXyUT21LGHTnkc/ris3fEYyWetyK/HVc8G4WfvSOdk5+9I62rxz97h2EY\nhuk5sKfPnr5sDlvaYi/s6bOnb4A9ffb0GQWx9lhkhmFuTnxk0q+SiW0p444c8jnd7elbfyyy4zmF\nJezp2xH3DE/fde9WsF8Xe/ry2xHDRyZ9hmG8AX63gu/Dnn4P9/RdkYM9/ZvX07elT+2FPX329Hsk\n7K0zjPdgfDzebMekj0z6VTKxLWXckcN6zu4Hx1Xa6a27VpczOezxba2f9KS3w56+RRbJ2B336RuX\nV8rjd7enb3w8mlpV8rp83dOXffYOwyhBtxesR1ubn1xxhwgP7y8cvGFhES7ZBmNK9+RZhba2cZ6W\n0yNRelyzp+8R79xbddmuXQrjQQp0DdTuycHxbbjC92ZP3/05lMjprKdvGttSRhlPv/tYsOW4sC2n\nWCy1H/iTPqM4pgMZLvtkz7ge/vakLMbHhqeOC/b0Fc3hipxK5HBFTkd8Wtt1WL/Q5nhO23Xash17\nY9/09MVv0bQvhyt02bZeLK+zsXv2ozI5xJGd9CsqKpCcnIyEhASsXLlStMz8+fORkJAArVaL2tpa\n2boXL17E+PHjkZiYiAkTJqC1tVVGxTGZ2JYy7shxc+k6duyYZOyMju5PRKthOuE43zZ5nbZsx/n+\ncsV+s3+fyOdwRJecDqV1hYf3x7hx40RuFFBuPxo+iIwbN87JbSihy9YylkhO+h0dHZg3bx4qKipw\n/PhxbNu2DSdOnDApU15ejlOnTqGurg4bNmzAnDlzZOuWlZVh/Pjx+Oqrr5CdnY2ysjIZmeYnBbGT\nhFwZd+S4uXQZTtaGg+GJJ54wOxi8s22WHzLco0t+u8q3zZ62KrEfreVwRpctdbo+FCyD6QcEW7Tb\nHnd/EFnm5DaU0GVrGUskJ/2amhrEx8dDo9EgICAA+fn52LVrl0mZ3bt3o6CgAACQlZWF1tZWnD9/\nXrKucZ2CggLs3LnTJrGMd2L9YGCA7olw+fLlLrsf3Hgbjt53rsR+NM+hhC5GWSQn/cbGRsTExAix\nWq1GY2OjTWXOnTtntW5zczOioqIAAFFRUWhubpaR2SAT21LGHTlckVOJHK7ICTQ0eIMO+3PK61ZC\nR3fcPREWmE2m9ua0XsZ4G8afdt3TVvt0hYf3FzkJWM9p/cThuC7HY1/KYQWpF+i+8847NHv2bCHe\nunUrzZs3z6TM5MmT6ZNPPhHi7Oxs+uyzzyzqbtmyhR577DEiIlKpVCY5IiIiRLePrtHCf/zHf/zH\nf3b8SSF5y+aQIUNw5swZIT5z5gzUarVkmbNnz0KtVuP69esWy4cMGQKg69P9+fPnER0djaamJkRG\nRopu35F7fhmGYRjrSNo7mZmZqKurQ0NDA9rb27F9+3bk5eWZlMnLy8OWLVsAANXV1VCpVIiKipKs\nm5eXh82bNwMANm/ejHvvvdcVbWMYhmHMkPyk7+/vj7Vr1yInJwcdHR0oKipCSkoK1q9fDwAoLi7G\npEmTUF5ejvj4eISEhGDTpk2SdQFgyZIlmDp1KjZu3AiNRoO33nrLxc1kGIZhAC99DMOJEyewa9cu\n4cKvWq1GXl6ecNIwlDl37hyysrIQGhqKTz75BP3798fp06cRFBSEzz77DDqdDtnZ2Rb5P/74Y7z9\n9tsYN24c7rvvPly7dg33338/rl+/jszMTPz3f/83goKC8Oabb2LIkCG4++678Ze//AV79uzB1atX\noVar4e/vj6SkJEyfPh3h4eFu6xuGYRhn8LpJf+XKldi2bRvy8/OF6wdnzpzB9u3b8dBDD+Hpp5/G\nSy+9hFdeeQUhISFoaWlBZmYmGhsb0dHRgdOnTyM2NhY///nP8dFHH2HKlCl4++23kZycjC1btuBP\nf/oTXnnlFZw7dw7x8fGYMmUKvv76a2zZsgVhYWEIDg5G//79ERcXB39/f1y7dg0qlQqff/45Ojs7\nERAQgLNnz+IXv/gFVCoVduzYgVdffRXjxinzsKkLFy5YvcZhoKWlBQMGDFBkewDw73//GytWrMDZ\ns2fxz3/+E59//rmw7tFHH8Wrr77qkNabVaejeMO+f//997F8+XIMHDgQ3377LTo7O3H48GHodDqs\nWrUKkZGRePPNN5Gfnw+g6975hQsXoqamBmlpaVi9ejUmTpyI3NxcFBUVIS4uziU6vXXfi+k8c+YM\nli9fjqNHj2Lfvn144oknTPr06NGjuO222xAZGSnan1FRUUhPT8fUqVMxbdo05/tU8jKvB4iPj6f2\n9naL5T/88APFxcUREdGtt95KbW1tpFarqb6+noKCguj555+nq1evUq9evWjixIk0ZcoUys3NpbCw\nMAoPD6fg4GCaPHky9evXjy5cuEDJycl05coVuvXWW0mn01FGRgZ1dHTQhx9+SBEREdS7d2/Kycmh\njRs30sCBA+nWW2+lGzduUGdnJ6WmptLo0aNp8eLFFBcXR71796aIiAhKSkqixYsX06VLl4iIqLW1\nlRYvXky//OUvKT09nU6fPk1FRUW0ePFimjVrFk2bNo2SkpLoF7/4BZ08eZK+++47ioyMpJaWFmpp\naaFLly5RWloaJScn07Rp06iiooICAwOpf//+NHjwYKqsrKSamhrS6/X0y1/+kk6fPk16vZ4CAwMp\nKCiIQkJCaMCAAXTbbbfRpEmTKCkpiVQqFYWFhdGwYcOooKCAqqqqaNy4cVRYWEjPPfccBQYG0sSJ\nE6mwsJAWL15MaWlpVFhYaKKzpaWFtm/fTsOGDaOWlhb6+uuvZXUSkazWiIgIGjx4MEVHR9uk8/77\n76e6ujoqKiqiyMhIunTpkkWfyumMjY2lwMBACg8Ppz//+c8O6ZTa94MHD6a//OUvsvveXOesWbNo\n4MCBdN9999H58+fp8OHDdu97MZ3G/XnkyBGLPh00aBCtWrWKSktLqU+fPrR06VL6/PPPacWKFTR5\n8mT67rvvKCAgQBijs2bNolGjRtFnn31GL7zwAv3sZz8jf39/6tevH/Xu3ZuSkpJowYIFNHr0aK8f\no+b9effdd1NYWBgNHjyYhg8fTmFhYQ6NUb1eTy+99BINHjyYkpKSLPo0LS1N0PnLX/7Soj9jY2PJ\n39+fwsLCKDIykjIzMy369O6776bw8HDKzMyko0ePSs6xXjfpJyUlUX19vRCPGDGCRowYQUlJSRQY\nGEgjRoygPn360IgRIygwMJCIiNLS0mjChAm0YMECCgoKooyMDJo+fTrt27eP4uLiaPjw4RQZGUm7\ndu2i+Ph4IiJ64IEHaOPGjaTVaqmwsJASExOJiOjLL7+kzMxMSklJobfffpvuu+8+AkApKSn0/fff\n07Vr1ygpKYnCwsKorKyMmpqaKCUlhYiIPvzwQ5o3bx5lZWXZdED5+flRTEwM9e/fn4KDg0mj0RAA\n0mg0FBsbS7NmzaJBgwZRfX09vfDCCzRgwAC65ZZbaOHChTR48GAKDg6mmJgY2rp1K73xxhs0ZMgQ\nuv3222njxo30xhtv0LBhw2j58uV05513kk6no8cee4w6OzupV69eNHr0aBo+fDhFRERQSEgI6fV6\n0uv1FBQURMOHD6fY2Fh65plnqE+fPrRixQry8/Mz0RkYGEj+/v6k0WgoNDRUVucLL7xAWq2WysvL\nrWqNi4ujp556ih588EFasmSJrM4//OEP1K9fPyotLRUOKPM+ldNZU1NDGo2GZs2aRQEBAZSZmSnb\np+Y6jxw5IrrvCwoKqF+/fjR58mTZfW+u85lnnqHExER64YUX6J577qGxY8fave/NdRIR9erVS+hP\nvV5v0afR0dE0evRo+vbbbykgIID8/PxIo9EI+9x8jKanp9Ott94qHK8hISHCsXTy5ElKSEigQYMG\nkUqlopSUFFKpVF47Rs3786233qIpU6bQU089RTqdjlatWuXQGA0ODqZvv/2WMjIyKCYmxqJPjXUG\nBgZa9GdNTQ1lZGTQl19+STqdjvbv32/Rp2+99RZ1dnbS3r176Y477pCcY71u0t+zZw/FxcVRTk4O\nzZ49m4KCgmjUqFEUExNDmzZtovr6errjjjvogw8+oMGDBxMR0ciRI6m1tZVmzJhBAOjGjRu0atUq\nGjt2LCUlJdGwYcOETo2NjaVz584JnwoDAwNp5MiRwkAeM2YMHTt2jEpLSyk2NpYSExPppZdeoqio\nKIqIiCCVSkUDBw6kqKgoIiJqbm6mMWPGEFHXAaXX66lv3742HVDPP/885eTk0Oeff05arZaISDiR\nERGlp6dTcnKy8M0nODiYMjIyiIios7OTYmNjacCAARQVFUV6vZ4iIiIoLS1NqK/Vaun222+nhIQE\n6ujoEA7G1NRU+vLLL4mIKCEhgZKTk6mjo4OIiNRqNWm1Wtq0aROlpqZS7969iYgErYYcGRkZpNFo\nbNb5yCOPkL+/P+n1elq/fj3FxMRYaO3Tpw8RkaBVTicRUUxMDKWmptLQoUMpJibGok/ldBraQtT1\nDXL//v2yfWqu07DfxfZ9UFAQEZHsvjfX2dnZKWhNT0+nrKwsu/e9uU7jfZ+QkEBEZNGn6enpwr4P\nDw830TlixAgiIurduzetWrWKnn/+eRo2bJhJn/bt21fQSdT1oS0jI4OuX79Oe/bsoZCQEK8do+b9\nSUSCVsPx6ewY/e///m+LPh0yZAhFRERY7U/jMWrYB+Z9aoxBqzV6l5SUlDjrYylJfHw85s6di4SE\nBERGRuL8+fOYPn06Nm/ejNtuuw0qlQo5OTlQq9U4efIkHnzwQcyYMQOhoaG49957cffdd0Oj0WDU\nqFG47bbbcOjQIdxxxx1obGzEN998g8cffxxhYWEICgpCbm4uHnjgAUyePBnjx4/H8uXL8dhjjyE6\nOhpjxozBPffcg7lz50Kv1+O+++7DoEGD8LOf/QxlZWU4cuQIzp8/D61Wi8ceewwA8MYbb0Cv1yMg\nIAD79u3D1q1bcejQIcycORPr16/HwIEDMXfuXPz617/G999/j927d2Ps2LF45pln8I9//AOPPPII\nysrKEBoair///e84cOAA5s+fj2XLlkGtVuO9997D5cuXMXLkSGzatAlhYWHo7OzE6tWrQUT49NNP\nERISgoyMDDQ0NODdd9/FLbfcgn79+qG5uRmHDx/G448/jsjISPj5+eHNN99ES0sLxo4di169eiEu\nLg7Dhw9HZWUl1q9fj7i4OLz//vtYvHgxRo8ejbFjx+J//ud/cOHCBdTU1ODKlSvw9/e3SefGjRvx\n3nvvYdKkSdi1axcuXryIXr16mWi9ceMG/Pz8kJCQgE2bNuF3v/udVZ2xsbFISUnB66+/jueeew5/\n+9vfMH36dPz2t7816dMffvhBUufZs2fxxRdfoKWlBUFBQXjiiSfwzjvvSPapuc7IyEisXr0agYGB\nuH79usm+37BhA5588kls3bpVct+b61ywYAGICM8++yzOnz+PBx54APv27bNr35vrfOyxxxAYGIgP\nP/wQ//nPf/Dwww/jm2++MenT8PBw/PrXv0ZKSgoOHDiAbdu2YezYsXj88cdx4cIFFBYW4tVXX8Vt\nt92G9vZ26HQ66HQ6/PGPf0RwcDD+9a9/4dq1a0hPT8emTZvQp08fXLp0CRqNBpcvX8aBAwcs9ru9\nYzQ2NhZVVVWKj9GOjg6sWbNG6M/U1FQcPnwY33//PY4fP47IyEjs2bPHpE9///vfA4DkGL106RKm\nT5+OyspK7Ny5U9Bp6NNp06YJN54Y92ffvn3xxRdf4OTJk2hra8PXX3+NPn364P7778fGjRsRGxsr\n9KlhW/v370dlZSV+/etfW59kJU8JPYT33nuPnn76aUVztrS00KJFiwQPUqVS0S233EKzZs2ilpYW\nIiJ66qmn6G9/+xsREe3YsYN++9vf0uXLl2nPnj2CzURE9NVXX9GoUaNo5MiRFBISQiUlJcJfc3Mz\n7du3j6ZMmUIRERHUr18/mjhxIq1bt47a29vp0KFDNHbsWMrPz6eGhgYaOXIk9erVi3r37k3p6emC\nvzl37lyKjIwUtGo0Gpo6dSp98803RER0/Phx2rt3L/31r38VdBIRlZeXm+h84IEHaOfOnTRkyBAT\nrQadeXl5FBERQeHh4SY6iUhWa3V1NS1atIji4+Opb9++pFKpKCYmhmbOnEktLS1UWVlJjz76KG3c\nuJGIiCorKyk7O5t2795tsX/q6upo1KhRsjpHjBhB0dHRFjr1er3NOg12wqJFiyz2/bvvvktEJLvv\nDTqXLVsm6CQievvtt2no0KGUkZFh0aeO9GdSUpKJzgMHDtDzzz9PH374oWy8c+dOysrKosjISJO+\nPnDgAD3yyCP0s5/9jDIyMmjEiBEmOtevX09xcXGk1+ttHqOhoaGUkJBAixYtojNnztDMmTNpwoQJ\n9Jvf/IbOnTtHY8aMoZycHPrNb34jXEcx7HdrY/TChQsWx1Jubq7V/szOzqbg4GAKDg6m0NBQGj16\ntEWf9u3bl4YPHy706Zo1a+j06dMm/SO2zMDOnTtp6NChNGDAAJPl+/bto1/84hdCfyYnJ1NpaanV\nYyk7O5vCwsJIp9PR4cOHRbdl4KaY9N3Na6+9ZrHMMFFJxVevXqV//OMfNucwLyMVHz9+nBYuXEiX\nL1+mF198kRITE+mee+6hyMhIKioqEuKAgADasWOHcBIw/ip+/PhxKi0tpba2Nrp69Srt3LmTPvro\nI/rrX/8qrP/oo4/onXfeEeK9e/cKE595DrE6H3/8Mf3rX/+i8vJymjZtGg0dOpQSExNp0aJFNHjw\nYFKr1RQaGkpjx46lrKws+t3vfkfp6em0cOFCIiKLE8MHH3xAs2fPpv/93/81WW8cP/fcc7R69Wr6\n4osvrJZ59NFH6U9/+pNJnb179wrt+tWvfmXS9zNmzLDYf+bLjOOrV6/SAw88IJtDbjtS6w8cOEAx\nMTHChL5hwwbq27cvlZSU0OjRo+m+++6ziLVarRCXlpbSgQMHaMiQISY5zMskJycLJwqx9YYTSUVF\nhaDrscceE8bJlStXaODAgTRx4kRatGgRPfzww9S/f3+qrKykZcuW0bBhw2jq1Km0Z88eWrZsGU2Z\nMoWWLl1Ko0ePpscff5wuXbpEV69epenTp9PYsWOFE8WsWbOEE8fZs2epsLDQ5ERifmL57W9/S6NG\njaL58+dTa2srXblyhXJycuiuu+6i3/zmN9Ta2krh4eEUHR1Nd955J73yyisUFhZmEl+4cMGizB//\n+EeTk0B4eDgNHDiQfvrTn4quN5QxzvGHP/xB+MBmLzzpuwCDlye1zN7YmRyGST4oKIiGDh1KQ4cO\nFSbd1NRUCgoKorKyMiGOiYmhQYMGmZwEDDn69etHQ4cOtThRyMW25Lj//vspKyuLMjMzKTIykvr2\n7epHR2QAAAYoSURBVEvLli2j0aNHU58+fSgpKYmuX79O3333HfXq1YtaW1tpyZIl9JOf/IT69u0r\nemKQi7OysmjMmDEUFhZGarXaphyGT8z9+/enlJQUioyMpN69e1NkZCRFRUXR5MmTqXfv3hQVFSXE\n5mXkYiVyqFQq4a619PR00mq1wrWF0tJSuv322wWP+MqVKxQUFGQR63Q6Ib7llltEc5iXCQoKEib5\nIUOGWKwXO7EMGjSIRo0aRaWlpTR79myKiIigjz/+mJYtW0b9+vWj5ORkYVwHBQVRSkoKXb9+nYiI\n+vfvT48//jgNGzaMli5dSvfddx/Nnj3b4kRhbyyWMzAwkAYMGEBDhw4lrVZLI0aMEO76mzlzJvXu\n3ZsmTJhATz31FP3qV7+igQMHUlhYGL322mu0Y8cOmjlzJvn5+VFgYCAlJCTQqlWrLHIY1icmJtKq\nVavowoULJncXWitjKzzpO4jhriLjvz59+lCfPn0IgElsWCYXG9fx8/OTzeHn5ycZG+dMSUmhgIAA\n4RbX1atXE1HXBaHk5GTh7ietVkspKSl011130YIFCyglJYVuv/12io6Opra2NsrIyBByGJ8o5GJb\ncxjfemv4lnHt2jUKCgoyuUBluEBqOPjT0tIoNDTU4sQgF7e2tlJKSgpdvnyZbr31VptypKWl0fTp\n02nPnj0UGxtL8fHxQp+uWbOGKisryd/fn+6++25avXo1VVVVmZRZvXq1ZGye44UXXrDIsWbNGpvi\n6OhoqqqqoqSkJLpw4QKlpaXR6dOnKSkpiTIyMigtLY1aWlrou+++o6CgIIvY+KJs3759RXOYl0lN\nTaWWlhb65ptvRHOYn1jEbqE23F1H1PVAxrvuuos2btxIX375JQ0YMEC4QPvll18KF+QNJ4b09HTS\n6XQWJwp7Y7Gc1m7tfv311+ny5cuk1Wpp586d9NBDD9GAAQPohx9+oNjYWCEm6rrQ+u6779LYsWOp\nT58+ojmM1xtOHIb1YjkGDhxokkMKnvQdJDIyko4ePUr19fXC38CBA+n9998Xbg0zxB9//DENGjRI\nNlYih1jOhIQEqq+vF+52GjNmDN1xxx3CJK/X6+nw4cM0Y8YM8vPzo9TUVGpvbxfitrY2CgkJEcoT\nkcWJQi62JYf5rbcjR46kq1evElHXQWeIL126JEwkWq2WLl26RDqdjrRarcWJQS425DD835YchrvD\nsrOzKTExkW7cuEERERGUnZ0t3CM9bNgwoczRo0ctysjFSuRIS0ujmJgY+u6774T+GjZsmHBrYGxs\nLKnVatJoNDRs2DAKDAy0iFNSUqilpYUaGhqE/jLPYV7GsN5Qxny9+YmFyPIW6v/7v/+jhx9+mNRq\nNYWEhFDv3r0pJCRE+Dbi5+dHAwYMoDFjxlBeXh7V1NTQAw88QH/84x8pMzOTCgsLLU4U9sZiOc1v\n7Taf5I1PcFeuXCGi7jtvzGPDMrkcYicOsTLGOaTgSd9BZs6cSQcOHBBdlp+fb1EmPz9fNlYih1hO\nvV5PtbW1Qnz69Gk6ffq0MKmfPn2ampqaqLOzkz7++GOhvCEmIho7diz9/Oc/Jz8/PyGn8YlCLrYl\nh/mtt99//z0RkTCpG+Jvv/1WuPYxcuRI+uabb+gf//gHdXR0WJwY5GJDjsbGRtLpdDblMPCvf/2L\nVCoVPfroo6RWq+nMmTP04IMPCjERWSyzN3YmR1hYGPXu3dvkVmUiEj5NGnP16lX6+uuvLWLDBD5s\n2DCKiYkRzSFXxny9+YlF7BZqwy3Wo0aNou3bt9Phw4epqamJWltbqba2VrjQGRsbK5QfOnQoRUZG\n0pAhQ2jkyJEWJwp7Y7GcMLu1++TJk0KfXblyxSQ2YL7MeAyZr79y5YrFeuMyYicOcwxlrMGT/k2A\nYVI3x3hSlyt/+vRpOnfunFDe/EQhF9uSwzCpm+synuTNMdQxjw115GJDHfNYro5xbH53mNjdYnJl\n3JWDyHKCdwRbcsiVETvRHDt2jGpra4VJXg7DScC4vPkyZ2PjnO+8845NuqQQOzHYs97WMtbwumfv\nMAzDMK5D8nn6DMMwTM+CJ32GYZibCJ70GYZhbiJ40mcYhrmJ4EmfYRjmJuL/Azp10gTirk6MAAAA\nAElFTkSuQmCC\n",
       "text": [
        "<matplotlib.figure.Figure at 0x91140d0>"
       ]
      }
     ],
     "prompt_number": 16
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "selected_features = np.where(trees.feature_importances_ >= 0.001)[0]\n",
      "train_X = ss.fit_transform(train_X[:, selected_features])\n",
      "test_X = ss.transform(test_X[:, selected_features])"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 17
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "svc = SVC()\n",
      "gammas = [1e-4, 3e-4, 1e-3, 3e-3, 1e-2, 3e-2, 1e-1, 3e-1, 1., 3., 10.]\n",
      "gs = GridSearchCV(svc, {'gamma': gammas}, n_jobs=-1, cv = 5, scoring='accuracy')\n",
      "gs.fit(train_X, train_y)\n",
      "print gs.best_params_\n",
      "print gs.best_score_\n",
      "\n",
      "svc = SVC(**gs.best_params_)\n",
      "svc.fit(train_X, train_y)\n",
      "print svc.score(test_X, test_y)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "{'gamma': 0.01}\n",
        "0.6275\n",
        "0.685"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n"
       ]
      }
     ],
     "prompt_number": 18
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "## benchmark 4 - extra embedding tree + PCA"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "from sklearn.ensemble import RandomTreesEmbedding\n",
      "rte = RandomTreesEmbedding(n_estimators=100, max_depth=3, n_jobs=-1)\n",
      "embedded_X = rte.fit_transform(X)\n",
      "print embedded_X.shape"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "(1000, 735)\n"
       ]
      }
     ],
     "prompt_number": 30
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "from sklearn.decomposition import TruncatedSVD\n",
      "from sklearn.decomposition import PCA\n",
      "pca = PCA()\n",
      "pca.fit(embedded_X.toarray())\n",
      "pd.DataFrame(pca.explained_variance_ratio_).plot(kind = 'bar')"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 31,
       "text": [
        "<matplotlib.axes.AxesSubplot at 0x9d13b10>"
       ]
      },
      {
       "metadata": {},
       "output_type": "display_data",
       "png": "iVBORw0KGgoAAAANSUhEUgAAAYAAAAEHCAYAAACncpHfAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3X9Q03eeP/BnQgIYUCJVgiS42I2e8Wo9Vlw613bqLLVa\ntGjtjNLtD6ZCyzJlOHVnDue823X2biq9nqO30p3BbQfBTjk9d124M+bGzrl+a09g1tM7q+6ROMYm\nn4VYBQQV+fHJ5/sHzUcjkAT5kU+S52OGkc8778/n/XpFyJvP+/XJJypJkiQQEVHMUYc7ACIiCg9O\nAEREMYoTABFRjOIEQEQUozgBEBHFKE4AREQxKugEYLPZsHjxYixcuBAffvjhqH0qKiqwcOFCLFu2\nDOfPn/d7TBRFZGdn45VXXpHbOjs7sWrVKixatAgvvfQSuru7J5gGERGNV8AJQBRFlJeXw2az4fLl\ny2hoaMCVK1f8+litVjgcDtjtdhw4cABlZWV+j//zP/8zlixZApVKJbdVVVVh1apVaGtrQ15eHqqq\nqiYxJSIiCkXACaC1tRVmsxlZWVnQarUoLCxEY2OjX5+mpiYUFRUBAHJzc9Hd3Q2PxwMAcLvdsFqt\nKCkpwcPvN3t4n6KiIvzud7+b1KSIiCi4gBOAIAjIzMyUt00mEwRBCLnPtm3b8NFHH0Gt9h/G4/HA\nYDAAAAwGgzxhEBHR9NEEevDhZZtAHr2bhCRJ+Pd//3ekpaUhOzsbv//97wOOMdY4oY5PREQPhHqH\nn4ATgNFohMvlkrddLhdMJlPAPm63G0ajEb/5zW/Q1NQEq9WK+/fvo6enB2+//Tbq6+thMBjQ0dGB\n9PR0tLe3Iy0tbcwYXnjhBfl7p9OJrKwsv8cjsU1p8TCX8I/NXJjLZLWdPn16xJhjCTgB5OTkwG63\nw+l0IiMjA4cPH0ZDQ4Nfn4KCAlRXV6OwsBDNzc3Q6/VIT0/HBx98gA8++EAO6J/+6Z9QX18v71NX\nV4fKykrU1dVhw4YNY8bgdrvl7zs7O6HR+IcciW1Ki4e5hH9s5sJcJnucUATcS6PRoLq6GqtXr4Yo\niiguLobFYkFNTQ0AoLS0FPn5+bBarTCbzUhKSkJtbe2ox3p4OWfHjh3YtGkTPv30U2RlZeHIkSOP\nFTwRET0+lZJvB61SqbB371552+FwwGw2+/WJxDalxcNcwj82c2Euk9W2bdu2kGsAip8AFBweEZHi\njOd18/EWjqbRvn375O+VNJMr/a8A5sJclBSP0nLZuXMn7t27N2KcSJKcnIy///u/H/M5C4XiJwAi\nosl27969iF9dmIzL5LkEREQxJxpeW8bKYTy58W6gREQxSvFLQKwBKLtNafEwl/CPHQm5RIt9+/ZN\nKD+eARARxSjWAIgo5oz22jJrVip6e7umbMyZM2ejp6czpL6dnZ0oLi7GyZMnMWfOHOzevRuvv/66\nX5/JqAEofgmIiGg6DL/4T90fnL29oV+18/777yMxMRE3btzA+fPnsXbtWixbtgxLliyZ1JgUPwGw\nBqDsNqXFw1zCP3Yk5KJkd+/exW9/+1tcunQJOp0Ozz77LNavX49Dhw5h9+7dfn1ZAyAiiiJtbW3Q\naDR+L+rLli3DpUuXJn0s1gCIKOaM9toy/MaqqXy9Ce317Msvv8SmTZvQ3t4ut/3617/G559/jlOn\nTj04Gt8HQEQUXZKTk9HT0+PXdvv2bcycOXPSx2INIErWNJkLc2Eu0VEDWLRoEYaGhvzi/p//+R88\n9dRTI/qyBkBEFEWSkpKwceNG/OxnP8O9e/dw5swZ/Nu//RveeuutSR+LNQAiijlKfx9AV1cXtmzZ\nIr8PoKqqCoWFhX59JqMGwAmAiGJONLy2xMQbwVgDUHab0uJhLuEfOxJyiRasARAR0WMJugRks9mw\ndetWiKKIkpISVFZWjuhTUVGBEydOQKfT4eDBg8jOzsb9+/fxwgsvoL+/HwMDA1i/fr38LrZdu3bh\nk08+wdy5cwEAu3fvxpo1a0YGFwWnaUSkPNHw2jLlS0CiKKK8vBxffPEFjEYjVqxYgYKCAlgsFrmP\n1WqFw+GA3W5HS0sLysrK0NzcjMTERJw6dQo6nQ5DQ0N47rnn8NVXX+HZZ5+FSqXC9u3bsX379nGm\nTEQ0cbNnz56UT9QKp9mzZ0/4GAEngNbWVpjNZmRlZQEACgsL0djY6DcBNDU1oaioCACQm5uL7u5u\neDweGAwG6HQ6AMDAwABEUfQLONQZijUAZbcpLR7mEv6xIyGXH//4x1GRy0RrAAEnAEEQkJmZKW+b\nTCa0tLQE7eN2u2EwGCCKIpYvX46rV6+irKzM7052+/fvR319PXJycrBnzx7o9fpRY/j888+RmpoK\nYPgmSQDkZB0OBwRB8Nt+mG/70cd924IghOV4SotnIsdTWjwTOZ7S4pno8ZQWD39fpiYeQRDgcDjQ\n2RnaJaZ+pACOHj0qlZSUyNuHDh2SysvL/fqsW7dOOnPmjLydl5cnnTt3zq9Pd3e3lJubK506dUqS\nJEnyeDyS1+uVvF6vtHPnTmnLli2jjh8kPCIiesR4XjcDXgVkNBrhcrnkbZfLBZPJFLCP2+2G0Wj0\n65OSkoK1a9fiD3/4AwAgLS0NKpUKKpUKJSUlaG1tHf/MRUREExJwCSgnJwd2ux1OpxMZGRk4fPgw\nGhoa/PoUFBSguroahYWFaG5uhl6vh8FgwM2bN6HRaKDX69HX14eTJ0/i5z//OQCgvb0d8+bNAwAc\nO3YMS5cuHTMG1gCU3aa0eJhL+MdmLuHPJVQBJwCNRoPq6mqsXr0aoiiiuLgYFosFNTU1AIDS0lLk\n5+fDarXCbDYjKSkJtbW1AIZf5IuKiuD1euH1evHWW28hLy8PAFBZWYkLFy5ApVJhwYIF8vGIiGj6\n8FYQRERRhJ8HQEREQQVcAlIC1gCU3aa0eJhL+MdmLuHPJVQ8AyAiilGsARARRRHWAIiIKCjWAKJ4\nHZC5MBelxMNcpn+cUPAMgIgoRrEGQEQURVgDICKioFgDiOJ1QObCXJQSD3OZ/nFCwTMAIqIYxRoA\nEVEUYQ2AiIiCYg0gitcBmQtzUUo8zGX6xwmF4s8Atm3bFu4QiIiikuJrAABYByAiChFrAEREFJTi\nawDAgzqAktbyImEdkLkwF6XEw1ymf5xQ8AyAiChGBa0B2Gw2bN26FaIooqSkBJWVlSP6VFRU4MSJ\nE9DpdDh48CCys7Nx//59vPDCC+jv78fAwADWr1+P3bt3AwA6OzuxefNmXL9+HVlZWThy5Aj0ev3I\n4FgDICIal0mrAYiiiPLycthsNly+fBkNDQ24cuWKXx+r1QqHwwG73Y4DBw6grKwMAJCYmIhTp07h\nwoUL+N///V+cOnUKX331FQCgqqoKq1atQltbG/Ly8lBVVfU4eRIR0QQErAG0trbCbDYjKysLAFBY\nWIjGxkZYLBa5T1NTE4qKigAAubm56O7uhsfjgcFggE6nAwAMDAxAFEXMnj1b3uf06dMAgKKiIqxc\nuTLgJMAagHLblBYPcwn/2Mwl/LmEKuAEIAgCMjMz5W2TyYSWlpagfdxuNwwGA0RRxPLly3H16lWU\nlZVhyZIlACBPEABgMBjg8XgCBmmz2QAAd+/eBQA5WYfDAUEQ/LYf5tt+9HHftiAIYTme0uKZyPGU\nFs9Ejqe0eCZ6PKXFw9+XqYlHEAQ4HA50dnZi3KQAjh49KpWUlMjbhw4dksrLy/36rFu3Tjpz5oy8\nnZeXJ507d86vT3d3t5SbmyudOnVKkiRJ0uv1fo/Pnj171PEBSEFCJCKih4znNTNgDcBoNMLlcsnb\nLpcLJpMpYB+32w2j0ejXJyUlBWvXrsW5c+cADP/V39HRAQBob29HWlra+GcuIiKakIBLQDk5ObDb\n7XA6ncjIyMDhw4fR0NDg16egoADV1dUoLCxEc3Mz9Ho9DAYDbt68CY1GA71ej76+Ppw8eRI///nP\n5X3q6upQWVmJuro6bNiwIWCQrAEot01p8TCX8I/NXMKfS6gCTgAajQbV1dVYvXo1RFFEcXExLBYL\nampqAAClpaXIz8+H1WqF2WxGUlISamtrAQz/ZV9UVASv1wuv14u33noLeXl5AIAdO3Zg06ZN+PTT\nT+XLQImIaHrxXkBERFGE9wIiIqKgAi4BKQVrAMptU1o8zCX8YzOX8OcSKp4BEBHFKNYAiIiiCGsA\nREQUFGsAUbwOyFyYi1LiYS7TP04oeAZARBSjWAMgIooirAEQEVFQrAFE8Togc2EuSomHuUz/OKGI\niDOAbdu2hTsEIqKoExE1AIB1ACKiULAGQEREQUVEDQAYrgMoaS0vEtYBmQtzUUo8zGX6xwkFzwCI\niGIUawBERFGENQAiIgqKNYAoXgdkLsxFKfEwl+kfJxQ8AyAiilFBawA2mw1bt26FKIooKSlBZWXl\niD4VFRU4ceIEdDodDh48iOzsbLhcLrz99tu4ceMGVCoV3nvvPVRUVAAAdu3ahU8++QRz584FAOze\nvRtr1qwZGRxrAERE4zKeGkDAJSBRFFFeXo4vvvgCRqMRK1asQEFBASwWi9zHarXC4XDAbrejpaUF\nZWVlaG5uhlarxd69e/EXf/EXuHPnDpYvX46XXnoJixcvhkqlwvbt27F9+/aJZUpERI8t4ATQ2toK\ns9mMrKwsAEBhYSEaGxv9JoCmpiYUFRUBAHJzc9Hd3Q2Px4P09HSkp6cDAJKTk2GxWCAIAhYvXgxg\n/H/RswagzDalxcNcwj82cwl/LqEKOAEIgoDMzEx522QyoaWlJWgft9sNg8EgtzmdTpw/fx65ubly\n2/79+1FfX4+cnBzs2bMHer0+YKA2mw13794FADlZh8MBQRD8th/m2370cd+2IAhhOZ7S4pnI8ZQW\nz0SOp7R4Jno8pcXD35epiUcQBDgcDnR2dmLcpACOHj0qlZSUyNuHDh2SysvL/fqsW7dOOnPmjLyd\nl5cnnTt3Tt7u7e2Vli9fLh07dkxu83g8ktfrlbxer7Rz505py5Yto44P4LsvjTRz5uxAoRIRkTT8\nuhmqgFcBGY1GuFwuedvlcsFkMgXs43a7YTQaAQCDg4N47bXX8Oabb2LDhg1yn7S0NKhUKqhUKpSU\nlKC1tTXINDWE3t6ukCY0IiIKTcAloJycHNjtdjidTmRkZODw4cNoaGjw61NQUIDq6moUFhaiubkZ\ner0eBoMBkiShuLgYS5YswdatW/32aW9vx7x58wAAx44dw9KlS0MKtry8XDFreZGwDshcmItS4mEu\n0z9OKAJOABqNBtXV1Vi9ejVEUURxcTEsFgtqamoAAKWlpcjPz4fVaoXZbEZSUhJqa2sBAF999RU+\n++wzPP3008jOzgbw4HLPyspKXLhwASqVCgsWLJCPR0RE0ydi7gUE8L0ARETB8F5AREQUVMAlIKVh\nDUB5bUqLh7mEf2zmEv5cQsUzACKiGMUaABFRFGENgIiIgmINIIrXAZkLc1FKPMxl+scJBc8AiIhi\nFGsARERRhDUAIiIKijWAKF4HZC7MRSnxMJfpHycUPAMgIopRrAEQEUUR1gCIiCgo1gCieB2QuTAX\npcTDXKZ/nFDwDICIKEZFUA1Ag5kzZ6Kn5zE++JiIKEaMpwYQQRPAMAWHS0QUduOZACKqBgAA+/bt\n89tW0vqe0tYBmQtzUUo8zGX6xwkFawBERDEq6BKQzWbD1q1bIYoiSkpKUFlZOaJPRUUFTpw4AZ1O\nh4MHDyI7Oxsulwtvv/02bty4AZVKhffeew8VFRUAgM7OTmzevBnXr19HVlYWjhw5Ar1ePzI4LgER\nEY3LpL0PQBRFlJeXw2az4fLly2hoaMCVK1f8+litVjgcDtjtdhw4cABlZWUAAK1Wi7179+LSpUto\nbm7Gxx9/jD/+8Y8AgKqqKqxatQptbW3Iy8tDVVXV4+RJREQTELAG0NraCrPZjKysLABAYWEhGhsb\nYbFY5D5NTU0oKioCAOTm5qK7uxsejwfp6elIT08HACQnJ8NisUAQBCxevBhNTU04ffo0AKCoqAgr\nV64MeRJgDUBZbUqLh7mEf2zmEv5cQhVwAhAEAZmZmfK2yWRCS0tL0D5utxsGg0FuczqdOH/+PHJz\ncwEAHo9HftxgMMDj8YQc8MPJOhwOCILgt/1oXwAjHvdtC4Iw4vHpOJ7S4pnI8ZQWz0SOp7R4Jno8\npcXD35epiUcQBDgcDnR2PsYl8lIAR48elUpKSuTtQ4cOSeXl5X591q1bJ505c0bezsvLk86dOydv\n9/b2SsuXL5eOHTsmt+n1er9jzJ49e9TxAYz4IiKisY3ndTJgDcBoNMLlcsnbLpcLJpMpYB+32w2j\n0QgAGBwcxGuvvYY333wTGzZskPsYDAZ0dHQAANrb25GWljb+mYuIiCYk4BJQTk4O7HY7nE4nMjIy\ncPjwYTQ0NPj1KSgoQHV1NQoLC9Hc3Ay9Xg+DwQBJklBcXIwlS5Zg69atI/apq6tDZWUl6urq/CaH\nYFgDUFab0uJhLuEfm7mEP5dQBZwANBoNqqursXr1aoiiiOLiYlgsFtTU1AAASktLkZ+fD6vVCrPZ\njKSkJNTW1gIAvvrqK3z22Wd4+umnkZ2dDQDYvXs31qxZgx07dmDTpk349NNP5ctAiYhoevFWEERE\nUYSfB0BEREEFXAJSItYAlNWmtHiYS/jHZi7hzyVUPAMgIopREVYD4GcCEBEFEtWfBwCwEExENJao\n/jwAwL8OoKT1PaWtAzIX5qKUeJjL9I8TCtYAiIhiFJeAiIiiCN8HQEREQbEGEMXrgMyFuSglHuYy\n/eOEgmcAREQxijUAIqIowhoAEREFxRpAFK8DMhfmopR4mMv0jxMKngEQEcUo1gCIiKIIawBERBQU\nawBRvA7IXJiLUuJhLtM/Tih4BkBEFKOC1gBsNhu2bt0KURRRUlKCysrKEX0qKipw4sQJ6HQ6HDx4\nUP4Q+C1btuD48eNIS0vDxYsX5f67du3CJ598grlz5wJ48GHxI4IbowYAsA5ARDSaSasBiKKI8vJy\n2Gw2XL58GQ0NDbhy5YpfH6vVCofDAbvdjgMHDqCsrEx+7J133oHNZhs1wO3bt+P8+fM4f/78qC/+\nREQ0tQLWAFpbW2E2m5GVlQUAKCwsRGNjIywWi9ynqakJRUVFAIDc3Fx0d3ejo6MD6enpeP755+F0\nOkc99kT/gvfVAZS0vqe0dUDmwlyUEg9zmf5xQhFwAhAEAZmZmfK2yWRCS0tL0D6CICA9PT3gwPv3\n70d9fT1ycnKwZ88e6PX6cQfvcDggCIKcvMPhGPE4gBGP+7YFQRjx+HQcT2nxTOR4SotnIsdTWjwT\nPZ7S4uHvy9TEIwgCHA4HOjsf46NypQCOHj0qlZSUyNuHDh2SysvL/fqsW7dOOnPmjLydl5cnnTt3\nTt6+du2a9NRTT/nt4/F4JK/XK3m9Xmnnzp3Sli1bRh0fwJhfREQ00nheHwPWAIxGI1wul7ztcrlg\nMpkC9nG73TAajQEnnbS0NKhUKqhUKpSUlKC1tXU8cxYREU2CgEtAOTk5sNvtcDqdyMjIwOHDh9HQ\n0ODXp6CgANXV1SgsLERzczP0ej0MBkPAQdvb2zFv3jwAwLFjx7B06dJxB65SqbB3715Fre8pbR2Q\nuTAXpcTDXKZ/nFAEnAA0Gg2qq6uxevVqiKKI4uJiWCwW1NTUAABKS0uRn58Pq9UKs9mMpKQk1NbW\nyvu//vrrOH36NG7duoXMzEz84he/wDvvvIPKykpcuHABKpUKCxYskI9HRETTJyLvBeSj4NCJiMKC\n9wIiIqKgAi4BKd2+ffsUtb6ntHVA5sJclBIPc5n+cULBMwAiohjFGgARURRhDYCIiIJiDSCK1wGZ\nC3NRSjzMZfrHCUUEnwGoUVn5N+EOgogoYkV0DQBgHYCI6GGsARARUVARXQMAgPLycsWs7yltHZC5\nMBelxMNcpn+cUET4GYAaNTW/DncQREQRKeJrAADrAEREPqwBEBFRUBFfAwBUSEjQ4cMPP5BblLTm\nxzXNyGlTWjzMJfxjR2ouoYqCMwAJAwN94Q6CiCjiREUNAGAdgIgIYA2AiIhCEAU1AABQ+9UBlLTm\nxzXNyGlTWjzMJfxjR2ouoYqSMwAv6wBEROMUtAZgs9mwdetWiKKIkpISVFZWjuhTUVGBEydOQKfT\n4eDBg8jOzgYAbNmyBcePH0daWhouXrwo9+/s7MTmzZtx/fp1ZGVl4ciRI9Dr9SODG0cNAGAdgIho\n0moAoiiivLwcNpsNly9fRkNDA65cueLXx2q1wuFwwG6348CBAygrK5Mfe+edd2Cz2UYct6qqCqtW\nrUJbWxvy8vJQVVUVUrBERDR5AtYAWltbYTabkZWVBQAoLCxEY2MjLBaL3KepqQlFRUUAgNzcXHR3\nd6OjowPp6el4/vnn4XQ6Rxy3qakJp0+fBgAUFRVh5cqVkzIJ7Nu3D4Cy1vy4phk5bUqLh7mEf+xI\nzSVUAScAQRCQmZkpb5tMJrS0tATtIwgC0tPTxzyux+OBwWAAABgMBng8nscK/lHbtm3D+++/L287\nHA4AkJ+cR7cFQRjxuCAII/pP9vGUFs9Ejqe0eCZyPKXFM9HjKS0e/r5MTTyCIMDhcKCzsxPjFbAG\n8Jvf/AY2mw2//vXwDdc+++wztLS0YP/+/XKfV155BTt27MCzzz4LAHjxxRfxj//4j/jBD34AAHA6\nnXjllVf8agCzZ89GV1eXvJ2amjpq8OOtAQCsAxBRbJu0GoDRaITL5ZK3XS4XTCZTwD5utxtGozHg\noAaDAR0dHQCA9vZ2pKWlhRQsERFNnoBLQDk5ObDb7XA6ncjIyMDhw4fR0NDg16egoADV1dUoLCxE\nc3Mz9Hq9vLwzloKCAtTV1aGyshJ1dXXYsGHDxDMB4Hs/wLvvblHMmh/XNCOnTWnxMJfwjx2puYQq\n4BmARqNBdXU1Vq9ejSVLlmDz5s2wWCyoqalBTU0NACA/Px9PPvkkzGYzSktL8atf/Ure//XXX8df\n/uVfoq2tDZmZmaitrQUA7NixAydPnsSiRYvwn//5n9ixY8djBT8S3w9ARBSqqLkX0AMazJw5Ez09\n4y+IEBFFuvHUAKJwAhim4LSIiKbMeCaAgDWASOZ7T4BPLK4DMhfmopR4mMv0jxOKKLkXEBERjVeU\nLgGxDkBEsYmfB4Ah9Pb2Ytas1HAHQkSkWFFbAxieBLoUeX8gpcXDXMI/NnNhLqwBEBHRtInSGsAD\nCk6PiGjSsQbwkMmYRIiIolEU1wAe2LdvX0yuAzIX5qKUeJjL9I8Tiqg/AwDUqKz8m3AHQUSkOFFf\nA/BRcJpERJOGNQAiIgoqJmoAAFBeXh5z64DMhbkoJR7mMv3jhCJGzgDU+Pjjj8MdBBGRosRMDQBg\nHYCIoh9rAKPSQKXS8v5ARETfiZkaADAEAIq4P1A4x2YuzCXS2pQWTyTkEqqgZwA2mw2LFy/GwoUL\n8eGHH47ap6KiAgsXLsSyZctw/vz5oPvu2rULJpMJ2dnZyM7Ohs1me6zgiYhoAqQAhoaGpO9///vS\ntWvXpIGBAWnZsmXS5cuX/focP35cevnllyVJkqTm5mYpNzc36L67du2S9uzZE2ho6bvaxJR9ERFF\no/G8vgU8A2htbYXZbEZWVha0Wi0KCwvR2Njo16epqQlFRUUAgNzcXHR3d6OjoyPovlJYC7Ia1gKI\nKOYFrAEIgoDMzEx522QyoaWlJWgfQRDwpz/9KeC++/fvR319PXJycrBnzx7o9foJJxO6IfT23oZW\nm4CPPvJf1oqmdUDmwlyUEg9zmf5xQhHwDCDUyzDH+9d8WVkZrl27hgsXLmDevHn46U9/Oq79J4cX\nQ0MDcDgccovD4YAgCH7bjz4eaFsQhAkdb6L7h+N4SotnIsdTWjyx+PMU7flNRTyCIMBms+Hzzz/H\n559/jvEI+D6A5uZm7Nq1Sy7S7t69G2q1GpWVlXKfn/zkJ1i5ciUKCwsBAIsXL8bp06dx7dq1oPsC\ngNPpxCuvvIKLFy+ODG7Kb+WsATDE9wcQUdSYtPcB5OTkwG63w+l0YmBgAIcPH0ZBQYFfn4KCAtTX\n1wMYnjD0ej0MBkPAfdvb2+X9jx07hqVLl44rwckzBNYDiChWBawBaDQaVFdXY/Xq1RBFEcXFxbBY\nLKipqQEAlJaWIj8/H1arFWazGUlJSaitrQ24LwBUVlbiwoULUKlUWLBggXy88Jj+zw6ernGYC3NR\nSjzMZfrHCUXACQAAXn75Zbz88st+baWlpX7b1dXVIe8LQD5jUJJt234KwIv3338/3KEQEU2LmLoX\nUHAazJw5Ez09ndM8LhHR5OC9gB7bEHp7e1kTIKKYEHQJKPYMv0dApVJh7969ACJ3HTCa1jSZC3NR\nSjyRkEuoeAYwKi8AYNu2bWGOg4ho6rAGENDwCRLrAkQUKVgDmDRD8NUF+FkCRBRtWAMIyYPPEvDV\nBiJhHTCa1jSZC3NRSjyRkEuoeAYwbmrWBogoKrAG8Fh8J068jxARKct4agBcAnosQ9/9q5EnKU4E\nRBRpOAFMiG8iUEOlioPvA8d87x8AuKYZSW1Ki4e5hH/sSM0lVJwAJoX3oe/9awS8txARKRVrAFPq\nwfzK9xIQ0XQYTw2AE8C004DFYyKaKiwCK9oQhmsGagzXDIZN9L0F0b6myVyYi1LiiYRcQsUJICy8\nj2yrv/s8AgmACoDXr5BMRDQVuASkWI/OzcNXHCn4v4uIFIBLQFFhaJQ2DVQqrV8fTghE9Lg4AUSU\nkZPC8PsPgIeXj4bv8DH88ZbRsqYZTeuzzIW5REwNwGazYevWrRBFESUlJaisrBzRp6KiAidOnIBO\np8PBgweRnZ0dcN/Ozk5s3rwZ169fR1ZWFo4cOQK9Xv9YCdDD9QTJr+3jjz8OuveGDRtw9epVCILg\n1x5q23j6Kr1NafEwl/CPHam5hCrgBCCKIsrLy/HFF1/AaDRixYoVKCgogMVikftYrVY4HA7Y7Xa0\ntLSgrKwMzc3NAfetqqrCqlWr8Nd//df48MMPUVVVhaqqqsdKgCZChd/9rhGAhIsXv8aDswg80obv\n2iWkpo7uZsdzAAANnklEQVS8JXZPTw+6uroivk1p8TCX8I8dqbmEKuAE0NraCrPZjKysLABAYWEh\nGhsb/SaApqYmFBUVAQByc3PR3d2Njo4OXLt2bcx9m5qacPr0aQBAUVERVq5cyQkgLKRRvh+t7cH3\np0//P/hPCsOPXb9+fZS2b/DopDJam9st4Lnnnp1IIkT0GAJOAIIgIDMzU942mUxoaWkJ2kcQBPzp\nT38ac1+PxwODwQAAMBgM8Hg8E8+EpslYE0SgxwO3ieKQ/AeBz4MJZXrbpnvs73//+wCGl0U1Gv9f\nx4m0TcUxw9WmtHgiIZdQBdwr1MswQ7kSRZKkUY+nUqli/HJPimVXr16Vv+/t7R3x+ETapuKY4WpT\nWjyRkEsoAk4ARqMRLpdL3na5XDCZTAH7uN1umEwmDA4Ojmg3Go0Ahv/q7+joQHp6Otrb25GWljbq\n+LzEkYho6gT8RLCcnBzY7XY4nU4MDAzg8OHDKCgo8OtTUFCA+vp6AEBzczP0ej0MBkPAfQsKClBX\nVwcAqKurw4YNG6YiNyIiCiDgGYBGo0F1dTVWr14NURRRXFwMi8WCmpoaAEBpaSny8/NhtVphNpuR\nlJSE2tragPsCwI4dO7Bp0yZ8+umn8mWgREQ0vRR1K4grV65g165dOHv2LFasWAFBEHDjxg3cuXMH\n9+7dQ2pqKt577z1UVlZCq9UGPyAREY1JMRPAhx9+iH/4h3/AnTt3xuwzZ84cdHV1ISMjA7m5uXj1\n1Vexdu1a7N69GwcPHsSf/dmfyVeT3LhxA9nZ2fIbJC5duoT8/Hx4PB6sWbMG/f39OHXqFAYGBjBn\nzhxcvHgRN2/eRH5+Pr755husX78eXq8XX375JXp7ezF//nycPXsWBoMBN27cQFxcHJ544gk5NkEQ\nkJiY6Nf25Zdf4vnnn5e3L168iHnz5mHOnDlj9nnc/cbbx+v14uzZszh06BBefPFF9PX14Q9/+AN+\n+9vfQqfTISsrC3/1V3+Fv/3bv0V7eztMJhOef/55GI1G/OpXv0JcXBwMBgM2btyIzz77DB0dHZg7\ndy7y8/Px1FNP4e/+7u8wODiI733ve9i4cSNaWlpgtVrx5JNPIj8/Hz/84Q+xc+dOdHV1Yf78+di8\neTM6OjpQX18PrVaLZ555Bj/+8Y/lPosWLcJLL70Ej8cTUowff/wxtFot0tLSsHHjRhw6dGhEn08+\n+QR9fX3IzMyMmBhHe65/9rOfob+/H1lZWdi4caMiYxzPz0MkxBjqz8OaNWug1WqnNMaTJ09i/vz5\neOONN1BSUoKqqiq8+OKLePHFFwO84n5HUgiz2SzFx8dLM2fOlNRq9fBnK0bo17Jly6S4uDgJgKTX\n66XNmzdLKSkp8uM5OTlSWVmZpNFoJADSzJkzpS1btkg/+clP5NxTUlKkN954Q9JqtRIASaVSSWVl\nZdLOnTsllUrl12fGjBl+fX7xi1/I48+aNUt64403pOTkZHn8FStWSElJSWF/nvjFL35N/pfvd//d\nd98N+rqrmDOAhIQEDA4OAgCv/iEiegxxcXEQRREAkJGREfQWEQGvAppOOp0OAB77DQ1ERLHO9+If\nFxc35uX1D1PMq+2rr76KH/3oR9Dr9bh69Spqampw5coVJCQkoL+/P9zhEREpXnx8PAYGBiCKoryi\nEohiloAe1dnZiR/+8Idwu90YGhqSZzaaWhqNBiqVChqNBgMDA5AkCRqNBklJSbh//z6Ghobg9Xqh\nVqsRFxeHgYEBeL1eJCUlYXBwEHFxcRgaGpL3GxoaglqthtfrhSiK0Gq1iIuLw+DgICRJQnx8PNRq\ntTyW1+uFJElISEiQj5OQkIDExET09PTI7ygfK8a7d+8CeLCMGBcXh/7+fnms0fLQaDQQRVHRMYby\nXPvG9e2nxBiD/TxEQozj+XnwvUPX9yEtUxEjMLyEvmjRIlgsFvzxj3/EypUrkZ2djTfffDPg77ti\nJ4DR/PKXv8SuXbvQ09MjTwi+X15JkuQnxLcOFh8fL7/beHBwEPHx8fB6vRgaGu3DVkY3f/58aLVa\ndHV14c6dOxBFcczJSKVSITk5GQMDAwCAlJQUDA0Noa+vT/4h0Wq18uwsSRJ0Oh20Wi36+/vlHzjf\nD5Lvh+LRPr4fjN7eXsTHx2NoaAiDg4PQ6/Xo7++Xn49Zs2bJY3u9XgwODkKj0SAuLk7+d926dbh6\n9SoWLlyInTt3YtGiRRP6P/J4PLh58yZu3bolXxH153/+5359bt68ia6uLkiShK6uLsyePXvC407U\npUuX5NjmzJmDuXPnjjiFbmtrk3+Ru7q6YDab/a76mm7jfa4BoKurC7m5udMe68PG+1z7hPNnxPdc\nA5Cf70h4roMax4U6YZeeni5XuJX+ZbFYJKfTKUmSJF2/fl3Kzc2VEhMTpczMTEmSJKm5uVlatGiR\nNGPGDMnpdEoLFiyQtFqtZDQapRMnTkh2u11KTk6WDAaDZLVapX/913+VUlJSJLVaLZlMJulf/uVf\npIyMDHm8pKQkqb6+Xjp58qT0gx/8QDIYDNKrr74qPffcc/JVQ75/Q/1KTEwM+/M43i+NRhORcev1\nevmKr0j50mq1kl6vD3sc4/1KTU2VZs2aFfY4xvtcp6amjvn4o1dOGo1G6euvvw76mqrYM4ClS5fi\n0qVLUXNF0MOnrNPh4asBQqHRaDBjxozHvqkUESmDWq1GfHw8UlNTg14FpNgJwGAw4NtvvwUAv0lg\nPB94/LgeXlsjIooUiYmJfq+RfX19Afsr5jLQR61duxZGo1FeI5w7dy6Sk5Mxf/58uVAZ7FbSj3u7\nCL74E1Ek8hWUJUkK6Q9lxZ4BjMblcuHGjRvo6OhAd3c3gOHCkNlsxtmzZyFJEtra2qDVanHv3j0s\nXrwYx48fR2NjI7q7u6HX66HVapGZmYn/+q//Qnt7O86ePYtr167h97//Pbq7u9Hb24uVK1fiRz/6\nEZ5++mlcvnwZvb29uHz5Ms6dOwe73Y729nb09/djaGgIOp0Od+/ejZqlqofFxcVh5syZctFcrVaj\nr6/P7/IylUoFtVqNxMTEMZe41OrhvzMCTazJycm4d+/epEy+CQkJfn8k+H4h7t+/7xeT72qKscYM\ndiaYmJgY8uV2wajVaqSkpKCvrw8ajUa+yuThmEN9roP98vvedDkZz7VGo0FiYiKGhoag1WoRHx+P\nnp4ev+dErVbLf4yNdUl3KGfdk3VmrtVqodVq5ecpPj4ecXFxfh+r6HuudTod7ty5M+rzGcpznZ6e\nDo/HMymvDzqdDqIoyr9POp0O/f39uHPnjvxXv++5Tk5ORmJiItxud8BjRtQEoDSdnZ2oqqrC0aNH\n4XQ6x/xPjo+Px7x58+SrCNRqNYxGI7799lvcunVL7qdWq5GQkID4+Hjcvn171GMlJCRAFMUxr2RK\nTU2FTqdDd3d3wPsq+aSkpCA+Ph6Dg4MYGBjA9773PcTFxQEY/muip6cHTzzxhPxD58s7Pj4eOp1O\nbu/v78ft27eh0+kwMDAgx9ff3w+j0Tii3xNPPIG4uDioVKpRx/H1mzNnjtx29+5d3LlzB/PmzcP9\n+/fh8XjkyTc+Pl6++kn67nI7tVodsM03Qfi+910tpVKp5Nh8bQDGPOZo/R5u81226Hux/Pbbb5GS\nkoIZM2bIz+m333474kqYGzdujNnm9Xrl6727urqQmpqKuLg4JCQk+PWdO3fuiP19n8YXqN+j8dy8\neRMzZsxAf38/4uPjAUD+f9ZoNONq802ag4OD0Gq1SEhIgEqlGnXfYMd8OOd79+7JE6jvss3JmJzD\nISkpCXPmzMG8efOQmJgot7vdbsyZM8evraqq6rGvNuIEMEVqa2vx7rvvyp+X3NbWJl/+tn79erlf\nXV2d3OfhtjfeeAM3b95Eb28v/vu//xtDQ0NYu3YtUlNTIUkS6uvr8cwzz2DhwoUAgMbGRgDDlwH6\nLpfzHbutrQ2LFi3CsWPHAACzZs0a8+MQiSjyxMXF4aWXXsKePXv8PrM9GE4AE/TwTAxAfnMIEVE4\nJCQk4LXXXsNHH32EjIyMgH05AUzA0qVL8fXXX4c7DCKiEXzLkIEo5l5AkejGjRvIzMyEy+XyK8KE\nWoGn2DDe92QQAcMv4A/XdIDh2tijbb7l3Lt37yIpKWlcY3ACmIC1a9finXfeQXFxsXwF0ZNPPoms\nrCycOnUKycnJuHXrlnwfE99/nO+WDKIoQqPR+G1L390uYnBwELNmzcLt27f9+qrVavT39yMlJUW+\nsunOnTvyLTCGhobkqzx8t4/w3SfGVxjz3RIjOTkZv/zlL/H+++8DGP4Bev311+XvVSoVent7/YqG\nd+/exa1btzB//ny/tkf7TnabKIqYO3cu+vv7sWDBAvl57O/vxzfffCOve/raent7MW/evDHbAKC7\nuxt9fX1+baHuH2qb7//g9u3bMBqNOH78OJqbm+Vfbq1Wi76+PvT390Ov18txTKRtovuP1ua7cu6b\nb77Bf/zHf/j9TPT396O7u3tK28bTt6urC2fPnsX//d//4datW4iLi8Ps2bPlqwOB4aVb3+/JVLcB\nCNi3s7MTXq8Xc+fOxYIFC/DMM89gwYIFcLvdMJlMfs/BaG2h3PVzLFwCIiKKUYp9IxgREU0tTgBE\nRDGKEwARUYziBEBEFKM4ARARxaj/DzltH4H0Wd6zAAAAAElFTkSuQmCC\n",
       "text": [
        "<matplotlib.figure.Figure at 0x9ce68d0>"
       ]
      }
     ],
     "prompt_number": 31
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "pca = PCA(n_components=30)\n",
      "pca_embedded_X = pca.fit_transform(embedded_X.toarray())\n",
      "pca_embedded_X = ss.fit_transform(pca_embedded_X)\n",
      "\n",
      "\n",
      "train_pca_embedded_X = pca_embedded_X[train_index, :]\n",
      "test_pca_embedded_X = pca_embedded_X[test_index, :]\n",
      "train_y = y[train_index]\n",
      "test_y = y[test_index]\n",
      "\n",
      "print train_pca_embedded_X.shape, test_pca_embedded_X.shape\n",
      "print train_y.shape, test_y.shape"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "(800, 30) (200, 30)\n",
        "(800,) (200,)\n"
       ]
      }
     ],
     "prompt_number": 34
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "svc = SVC()\n",
      "gammas = [1e-4, 3e-4, 1e-3, 3e-3, 1e-2, 3e-2, 1e-1, 3e-1, 1., 3., 10.]\n",
      "gs = GridSearchCV(svc, {'gamma': gammas}, n_jobs=-1, cv = 5, scoring='accuracy')\n",
      "gs.fit(train_X, train_y)\n",
      "print gs.best_params_\n",
      "print gs.best_score_\n",
      "\n",
      "svc = SVC(**gs.best_params_)\n",
      "svc.fit(train_X, train_y)\n",
      "print svc.score(test_X, test_y)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "{'gamma': 0.01}\n",
        "0.6275\n",
        "0.685"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n"
       ]
      }
     ],
     "prompt_number": 35
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "## Visualization"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "for ylabel, c in zip(np.unique(y), ['r.', 'b+']):\n",
      "    pl.plot(sf_X[y==ylabel, 17], sf_X[y==ylabel, 12], c)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "display_data",
       "png": "iVBORw0KGgoAAAANSUhEUgAAAXAAAAD9CAYAAAClQCyNAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJztvXt4VOW59/9NSEKIgQwIEk5mVOQQxSQqxIr+5hGlCEhl\n16aiVZJaiT/tKX3fCrr3fjuTdtdixYvY3XZ3071L4qVo07eKQoFC7axpwR+yFRK1WvHASjkFFRME\nCQbC8/vjmWedZq2ZNYdkZsL9uS6uzGTW4Zkh8133+j73fT85nHMOgiAIIuvITfcACIIgiMQgAScI\ngshSSMAJgiCyFBJwgiCILIUEnCAIIkshAScIgshSUiLgfX19qKqqwqJFi1JxOIIgCMIFKRHwJ554\nAuXl5cjJyUnF4QiCIAgXJC3gBw4cwKZNm3DvvfeCaoIIgiAGjqQF/Hvf+x4ee+wx5OaSnU4QBDGQ\n5CWz88aNG3HBBRegqqoKiqLYbkO2CkEQRGLEcjWSCptffvllvPjii7joootwxx134M9//jOWLl1q\nO4hs/ef3+9M+hnN1/Nk8dhp/+v9l+/jdkJSAP/LII9i/fz/27duHZ599FnPmzMGTTz6ZzCEJgiAI\nl6TUuCa7hCAIYuBIygM34vP54PP5UnW4jIExlu4hJEU2jz+bxw7Q+NNNto/fDTncrdmS6Alyclz7\nOQRBEITAjXZS7h9BEESWQgJOEASRpZCAEwRBZCkk4ARBEFkKCThBEESWQgJOEASRpZCAEzFxaHND\nEESaIQEnYkICThCZCQk4QRBElpKyUnpicKEoeuTd2Kj/njHxjyCI9EMCTthiFepAIE0DIQjCEbJQ\nCIIgshQScCImZJkQRGZC3QgJgiAyEOpGmIVQyh5BEG4hAc8wSMAJgnALCbhLSFgJgsg0KI3QJYrS\nf5N5lHNNEEQikIBnAJRzTRBEIpCAR4EiY4IgMhkS8CikIzKmCwNBEG6hScwMgwScIAi3kIC7hISV\nIIhMgyoxCYIgMhCqxCQIghjEkIC7hAp5CILINEjAXWJMKSQIgsgEkhLwU6dOobq6GpWVlSgvL8fD\nDz+cqnFlJCTgBEFkEklPYp48eRJFRUU4c+YMrrvuOqxatQrXXXedfoIsnsS0FvL4fHpuuFNWiiy5\n78/Se4IgBj8DMolZVFQEAOjt7UVfXx9GjRqV7CEzBqNQ+3xAKCQeR7NT5O/jjdYpuicIIl6SrsQ8\ne/YsrrzySrz//vu4//77UV5enopxZQxSxAMB/Wd/QBE7QRDxkrSA5+bmoq2tDceOHcO8efOgKAqY\nRYkCBtVjjEW8ng1IW8QOGZGrKtDSIh6HQuK510u9UwiCiI2iKFDivBVPaSHPj370IwwbNgzf//73\n9RNksQcuicfXDgTM/2Id1+ix+/3iMQk+QRButDOpCPzjjz9GXl4ePB4Penp6sG3bNvilCg0CrMJt\nFdVkbY9saCNL1g5BZC5JTWIePnwYc+bMQWVlJaqrq7Fo0SLceOONqRpb2ol1N2P3uhS7wSJ6qZhc\npQlagugfkorAZ8yYgd27d6dqLBmFtDcCgfh6gScq4INF8O2gKJ4g+gfqB27B6EuHQkJ4jPnfUtSB\n1C7ykEkCRwtZEER2QAJuwc6XlimETq8PNlLxHukiQBD9Dwm4C+wEh3zd6JwLFzqCSDfUzCoKTn62\njC7PhUjyXHiPBJGtkIBHIVq/k2ivDxQDcReQiveY7s+JIAYrZKHEgYy8ZaWlJF2+brbcBWTDGAki\nG6EI3CXW7BSfTzzuL/Emj50giFhQBO4SO6Huz4k5p+iasjsIgpCQgCdIusSSsjsIgpCQgCeAmwUd\nEoGia4Ig4oEEPAGiiWkyAh5vdE2inr1kywQ0kdnQJOYA0F8TkiQA2QtNUhOpgCLwFBDL+kgk2iJx\nJggiFiTgLrGKsPG50fpQ1dRMLCYj4HR7npnQHAeRakjAXRJNwI2oqv6605fV+LM/yGQBT3Rsmfye\n3EIZRESqIQ88BvF6lV6v+CkXQA4ExFJpsqNhtLU1B5J0jSHR82bCZ0YQmQZF4FGQvb8ZE1G0quoL\nFbe06Nt5PEB3t3jc0mIW8VhRu5vI0u028dyeD4aINpuhz55IBSTgUZDiJ291jbe8Xq/zLbD19zLq\nVhSgrs4s/qqaGgHP5NvzRL3fwewZZ/v4icyABNwGO+FItIWstYcKYF7hx05oUxkdG48lx6Kq5otI\nfwtioheX/rwo0R0IMRggAbfBKBzr1wvhsH7ho2WkOB1Loqq6mDqlHcrjAvFbIrGyZYy+/LkKCTgx\nGCABj4H0tu2EWRJLPK1taGtrzdvaCWm06DOWl56QMNXXA3v3AkVFwLp1wtjvB5KpUiUIwgwJuA1G\n26OjQxfPeK0GKarGyNq4KLLcxnpOu7RDu+NGO6/1WDK90esVv6utNWfGYO9e3eOprwdaW6O+t5g4\nXBDSKeCD2VMnzk1IwBMkETFgDGhujvyd3X5u7I1oYzDub3esiN8VFYljTr0PbM3K2CePRaovCCkg\nkyd6CSIRSMBtMH7Rm5rcWRzGKFtu7yTsidgdbsr1kxKkdeuA+nooFzeBeQqTOFCY8AUBM2cCa9Yk\nfzyCICIgAbfBKJbHjrmzUKx2icQqql5vbAvEzt+WdofxuRHp1cc6FuBgb3s8IkoO2LxmwdUEYPiC\ngDVr+s1PTwayTIjBAAl4CohVXakowjqR/rPPZ/GfbY7nBuOFpr3d/kJjdyyr2PdLEZC8IGQoJODE\nYIAE3Aar3eFkTcQSPqOISu/b5xPWsFH0nUTSeGxjFWgoFCnWyVgo5A0TRHZCAh4DWRZvRyzhM74m\nKzdl5O1msQanY8tjWdMTo1k9yWZgOGW21NWlMZodoNRHgshUSMBjUFdnfh5PAYiT6HV2pmJkZvGV\nJfnRMmDcRNnx7G/05dNCBma6pAoqNCLckJSA79+/H0uXLsWHH36InJwc1NfX4zvf+U6qxpYRRCuY\nibad/J2daFovCvGOwe5cbiZHEzmXkYzrCDiIM11IwAk3JCXg+fn5WL16NSorK3HixAlcddVVmDt3\nLqZPn56q8WUEbgpn4vmyRbNl7HAj4LEmUmPt7wbZRwUQF6O0F8NkeKYLQfQ3SQl4aWkpSktLAQDF\nxcWYPn06Dh06NKgEXGaQxMrBjiVeHk/s/PBEx5dIQVGiWLswpnXCM8MzXeKFKkWJeEmZB66qKvbs\n2YPq6uqI1wKGbzljDCyL/hoVRYiW8UuUiGg1NJifp0r4+juDRF7AVFW3m+VnQqQWygY6t1EUBUqc\nPmVKBPzEiRP4yle+gieeeALFxcURrwey8C/RKlwyynazZFoWXZ9iYnfhkj8zzhMniCzGGtw2GoXF\ngaQF/PTp07jttttw1113YfHixckeLiMw9s0OhUTuNhBZCp9otNRfAj/QF47BdKHKNOizJdyQlIBz\nzvGNb3wD5eXlaLB6BFmMFOe6Oj3illE44G4VnVjH7w/6+0s/2EUlkzI/MmUcRGaTm8zOO3bswFNP\nPYVgMIiqqipUVVVhy5YtqRpbWpAVjYGAvmqN16u3Xw0EIv3fVH3Z4rUkBtrCGAh7KJ22DFlCRLaR\nVAR+3XXX4ezZs6kaS0ZgtEkqKkSPEa9XrMxTV6f3MzFun6yoGRd+iOdY6YwY++vcmRQFE0SmQ5WY\nNhjTA62Np2SU5rZft9sy9WxjsAjtuTIZTQxOSMCj4PXqHrgxMgfgqg+HG5Gz9jORROsEONgEJ53v\niVL3iGyGBNyCXd4zY0LMZQk8YwACeh8OZXETmBKI6xzGnzLTxbhavRPpFJz+EtpMEtHBcmdBnBuQ\ngFswiol8LK0TRdGFSzm4DAwAm3kSyjUPIbyLK5Eztn6VNo0U8WgtZqMxEMKTKqHNVJFMZB6CINIJ\nCXgM7Lr8BQIAGhYC9S+IPhxN+hJkTiJn53PLbY0eu1vxsApNrOeZRLSxpXPM8fSTIYhMgATcgjGC\nDoXE6jV1dea+1+J1D5TyVqDJnZ2QTD8Vu9diLeDQ3wKezLHlvIKVdF10BuO8AnFuQAJuwelLa7RV\nrGtUAs4LH0uSaQLlJGxNTfryaFbh6W8SsXikSLa06Ln0VmspHYKZDg8+k++QiOyBBNwGa2aIUyqh\nG+w6ENpFoG7TDY2RonXS086rB0SCjNtC2f4SFqc5gHMVEnAiFZCA2yC/WMYyehlBe716habcNpqf\n69YPtxLttt6I8ZhO50tFtJ8sVmtKjsnjAdra+qdAKhFIVIlsggTcAesEo92alMkc2+35JUahlo+N\neeOZsJ5BtIpS4/tR1Ug7KdrFZiCj1f48D3ntRKohAY9BqjITon1B4xEoa2qj04UkngUkUiUsbloC\nJNJLfLDYDZmU704MDkjAYxBtQtMNdtaG0zbRxmAnsuvXOx87ngUkbIWlvl4UKyWx4rtT9ozT+1HV\nwSPWBDEQkIC7wBpVxlPwkQpBMp7X+LtE29q6ap7lcsV3Y+/0lhZ94ldVdavEzk4BALYu3I5g8jIE\nGhaaLhKyIjZTvPFUMxjeA5F+SMBj4CR2yQpzIgJlPGc8VoR13LG8agCuV3w3jlfODTAGNBfUI6Be\nC/bTVignWqG8KlZqMr3XXUVg7SEALOIiYbWIBpvdQAJOpAIScCfCzapW/m0V2LuTAbi3ENx4yokI\nlDzmzp3AqVPuml/J16zHsDu26RgJrvguo/GAei0aO+qADhXo2Az2ixrt2Np73bUXaAfY1M6oFwmC\nIOwhAXdi714oIWAnJiMwexsa36rRhDNWKmF/TVZJj1iKt3Gpt1gRnbyorF8vepwbrQ7ZddF0jDhX\nfJfCrShARweAyfnwQQGb2gm2o8H++he+SLA1K7WLBHnjBOEeEnAbFEU0qwLexTF4gEWL4BsDlJYK\n2wNIPJUwXoGyVjD6/eJxRUVkeqObY7S36wVAXq/+fpK9yERctBoWIjB7G9iOlaYIPtZFgjI1CMI9\nJOA2tLUBytjbgWPvAB8Bys5CqCpQGO5ZFU9aYbR8aCC2QBlTBo2FMGVlkcLf3OycNWNXAFRX5z7V\nMG48HrBf1ERE3hRFE0TqIAG3oaEBaGjIA3AZiov1jI+WFiF4igIsXuzuWPEKVjSrQE5ayvazXq+5\n/axTkyinMRkbdMnf257bxeIVdsdPVfYNQRD2kIDbYIx2P/tM/LQubOy2t0g0nOwOp99LwTVWh1r7\ntsiI2kmMrROpbs7tNqXQeI5UQAJOENEhAbfBKHI7d+pCKcU7VXZDvPtZM1hSeU7GdD8csIi5y5RC\ngiAGFhLwGFxzjfjJmLBNZOSdysk1p7RDj8e5XaxRzI254fGOy6nNqykrJcGUQoIg+hcS8BgYI1Yp\npm695njO4WZiM5Y4x9tjxHpux7z0OFMKCYIYGEjAY5BNPqxcdNlIrPxpa3aL3NZtkZDb8xAEkXpI\nwGNgtTdkjw9X/UQSwE01ZTz7ummUJV9XVXNeeCb0Ee9vsnXcBAEAuekeQKYjsz08HpG+p6oiOpV2\ng3HizwnjhGOsycd4BDwVbW6NJGLBZDup/gwJYiChCNwl3d3miUM3PUzsovRURnzRUg4T6eRnzQl3\nc35aoIAg0gcJuAukUMkeKBUV5ucSq3C58Z/dCmU8gphoJz+rgMdjv8RznnRDFx5isJC0gN9zzz34\nwx/+gAsuuABvvPFGKsaUMdhN8Pl85nRCp54ocl+ZN248jsRtP29rBD9Q4jNY/eFsvfAQhJWkBfzr\nX/86vv3tb2Pp0qWpGE/GYfyyu6nCdBJ9Y/VkMj2uo4lPOjv52Vo5t6wCO7ExqVV9CIJwJmkBv/76\n66GmOjE6Q7DrM2KMhI3PJXYCa1yVRq5SI3urOO2XSKSdisgy0QjfVsDfHA3W4b4EP9a4+uMiNBjv\nMIhzhwHxwAMGJWGMgWXJt2bnTr0SEzBPSkqRcyOSxrdr7Gfi9zvv70aM+yM3O6X2Qn6++JmCEnwS\ncGKwoygKlDjTogZcwDMdmcEh0wX/+Efxu+5uvV2rXLzAmnZnzTqxaxrlWjCMHQCveA5AYcQmsSLi\ndHjYpgj+va8B5QXAnEVgbYUklgQRBWtw22i8BXaAslAsGIW3tBSYNk08l59lZSXQ2QkcOSKey2yU\nujpnAZdY7QnZ3dDWnjB0AGQnVwIIxP0+ks1xdhJc7cJg02Y2MoKvSfj8lC3igjhb/RKDCxJwC01N\nYtkxQBdpVQWK83rQ3j4MvlGvo/2TKzB2rBB3o79tJ5jGKDgue8LQAZCtd9+7Nh7Rc5Mm6HQOxhB3\nm9l4oWwRF/Tz/wGR2SQt4HfccQdCoRCOHj2KSZMm4Yc//CG+/vWvp2JsaUEs5iAeV1YKQWcMYJ53\n0H0MYJ+sB8YMQejIZZg2TdgqshqzpcXFWpPRMEZT//EfwIMPxt0BUIqeG48+IYulvh7Yei2wqzWm\nx01R8gBArX7PaZIW8GeeeSYV48hYpCceOlaJMqjA+Anw+i6FtyByRRuZE+4mVdBW3IzR1IMPJp21\n4RoXt+FaZL/VsNr8tGlgvtniDsFmn1QKOF0MbKivBz79VHh9v/td9As9WS2DErJQonDddXrGSMXl\nfWh/0wvcfTfQWQDAWVRkqiAQvY+3RIuEk4ymrBG1nWVi/KmNbeu1YB17wbDZ8TZcG/uuVqBDRWDm\nJmBrA+BZEvc4E4EE3Ia9e4EdO8TjWBd8sloGJ7yfGYBT9Bt+v/6zokJ/zjnnwWDk9vJ3xteM+zjt\np23T1cV5TY34GeW4dsfgnPPaWnEsv59zQH8st7eORXs+f77YYebMiHNH0NXF/eWtsbcj+p94/t/i\n2ZbICNxoZ266LyCZirUUvrPTPEFozQeXvzP+dDquIx4PlAdaI25vrZGz0zGMFo7MMTdOsjqybh1Q\nUwNs3Rr71lpbbT65W3DqAmihvl78Ry1YoK8cEot4/t/i2TZdJPIZnOOQhWLBWgoPAGVlQEcHcPPN\nkQIuHzsRkW4Xzul2yhaxTnrGErpox4n2uinDIw0r7riZQE1kkjVr+7ckYnHE8/+WDasqkc0TNyTg\nFozCJjNJFEUIuGO/7CgTRIwBTfdege73x0OFFy2bC7GzTVR4WvusGH9K4V2/HmhvB556Cnj/ff25\nHJv06CXG/a3vx/i6cZto2AniQInkYBLwmOOibBL6DBKABDwK0pJoahLP5er0zc3isQwWlJIH4D3W\nhjo0g9lEDt05IxHAtxEYvwb+u09B2VloEtJok56A6H4osWa5OOE0gWn9XaqiYFfHumUVlDdHA/n5\nokozzLlQmBPz8zkXF462Bj7n4meQJCTgUZBfuO5uPfq2jZp3/TOUzSfBZp4Uf3zWP8zbbgM+eAG4\n+G6gMLIk3pqOyJi4SCiKEHSfT/w+1oo5sbz3RKJZ4+NoVoyb47MTG/XmVuUFtlWaiVRfpqpiM63R\nezZYHKnGzjI51z6DJCEBj4K1CZUj69ZBmb0NbOtc8UXcuxdKCFAwC2plG1o6GJ66pBXv/w4YORLo\n6hJFQh6PiK4rKyPPayzLlwtHyDsCp+AkpoAyd0IM6NsZ+5d7PHqRU0JVkcZb5DmLHMcYb/Vlqio2\nUy3g1AogBmSZJA0JuAPKhUuhfFgODMlF48nl4neKQy63xyNm+KWwFhWBYbOIyLc2AA267SKzQozR\nrTFTxBqJS4/baeEIbbyKuaTfjVjLc1uPY+xdbvTY7baNS6AMt8isLfJOxJEsLUKhVgAxIMskaUjA\nHWCfvgh8/g8whIAReQgE/pf2mlUspdWh7ftAK1jxPdofptt26fJYxuPJBRnc7GsVcCcBiXY8qx9v\nXIzCep64BcpgE7iJQLVt4shOSMQmsl6EVDXS1iL6gXPRNkoxJOBO5OejGXVgRf8D3HNPVIGUAqAL\nWDFwS6u2ncxkMU5K2kavGzaA9b0ENn4vsOI5MJaiFqzG/iXr1kFRzF0DjWNpaRE55IDollhZKeYA\n5EXFKuD9iXZ8eas9ejRw6JDIE3aIxOMdk91FyFXufCwsdw2MUYRJpB4ScCfmzoX67EWiG+CNeVH9\nUeV/bwCOHtUEEh6PyYpoaTHnd0sfOSJ6VR4XkWY7gI4NaO6siSqW1ouANX9d7sf27gXr2At0hISw\nlJujYKdI2mn5uEQWl0gKeat96BCUHXmIVvKfCSiK+MyNdw0sQ8dKZDck4BY0UXz5iwhxHwIhBrz2\nK+yc9V0EAqL7XmMjhKmdnw+2+laoB/NRd6RZF8jWVpN/LZFpiFY0i8XYQvYH8xFYFTk2O4/ceHz5\n0ySyRUUATiIwfg1w8d2x/epw9Mh6qoGGh6G0eWwnNV1loKTCv5a32gsWQMEsPdsnhSiKbh8lO+Go\nKACjCTpiACABdyLcKlUBQ/cJD9r/nI9rrhcv1Y7dAk9HG7rhwcp738Mfj9wML3ZCGf81eKruQndA\nbCejYlUVhUCAeZLQRLiznDLqn6DM/g3wajFCIV2IrZkpcYnKunVg9fVga2oATyFQGKNLYkBEjwwh\noH6fFj3Kc9tNZjqOJxXVdfIikJ8PTJ0WbqKVpCVhY3GkdMIx2ybosnSi+FyHBNyCFnE1LMSvzv8Q\nytkbEBi/BovvnoZAQGROBJ4+iu4jHgRmbgIr+Ff48DYCZ38JlFcA99cAnmHa8WTULYXPTry9XgCK\n7Czng/p8G7x1lo0M2AmmNTI3EcdkEWOwTe+S43czoWoiWiRaXw9lV5Hw/KOIhrKrCEo7AwA0YgnQ\npI81YesmxWXbkXMaHqC8FawtSyZDB2MZ+zlwUSIBd8LjAUafBXw1EQU4bPWtaP7WqwjMeQjqs3no\nODIdATwA/Algi5vAlIDpUG1t5sZYgG6beL3h301eBoCBTe0ErrxWE0tjOmBjo9hPZqY4ibabfPCo\nRIke7SY+o2ZvGI6ltJmjXOzdC6WdgbVH97TZ+L1im5kzgTkPaRfSpIhyYUlEcLM+ZXAwWj6D8aJk\nJRNaImYSwaBos1pbK7pv+nzi57x54ne1teIxwHlZmfg5Mu8Y9yHIV0963NSq09jGVT6urY08p9/P\nTa1kfb7IbYNBMRY5Hmur2P5AfhZO7WlN4+eR7WqtRLw+fz73wx+7xanhs4l1Dtc4tO5NBSkbY7ws\nWyb+QObPj/999ePnkTayvIWuG+2kCNwB7xsbACwCO/g0MPt2PPSQ+KiMBTDagsTXFCDwwS8d/U5j\nBopTTrjS5kFzUSu8TTB5301NIhPEWkDkNsJLprqwv6JKLXq/4jk0bi4E5jwENBU6WyJx5o+7IlU5\nyA4LO6eFZCLOwZiTnW3zEAlAAm5BExHlcTRjBgLv3YVAQQEYq3HMD2Y3FwLM/MdvTSP0Ks1Afj4K\nL/wqADFBahRnQO8wKPF6dftFToa2tJiLa2KJRcICbhUmOPjTSvTsjejVmnJC1b0lknF+so1opm2M\ng9EGSYbBeFGyQAJuQROcg8vQAS8C49dArbg1YvLOWJRj/MKainyOb0BAeRy/yvmtWEMSwB/fy9ci\n2fXrzQIuI2tjpodVqGUL2UREwiTmVoFevtz83JrH/ICDP82iR+lZ7w3HIpNE8xyIOAkzJOAWNMHZ\ntxXq/iFg4/4O5cKzkeXyTPy0FrrILoKqCrTsXgQFw3EEY6HCC+/406j19SIQKAAgBFy7YGzYgMbd\ni4Cnn8aBviWoqxuiT3Bazukm6pbFRY0ddcCpU0BhoXmxCGvk+OGHQCgEBT7REtciTMyTomZPaa5Q\nTMl7MJJJonkORJyEhUww4jOS0lLuh58H4eP81lu1Cb1YyG2CQc59o9q5H34xATjtGe5f0cNnzjRP\nRsrHqy/5dzGhB/Cgzx9xPInrScvwCfzwi8kp67GsEzzh5/7xa8Rzm0ktuwnYeMYWDOrj4oA2roEk\nbROMBBEnbrSTInAnenvhQTcUMLCcdjQ3m/1pLZIL5zIrn82EOrMGLc8UaNt4501D4MxbUA6dQWC7\nWL2dKXoE6PEY8qoXbELg/VmiCnO9OaxPqP9IOIJWz78agYsfAgKWniu9l4CVlgK/+50YiIwkL75b\nFPvIqPzOO7XJuVhNuVylL2aS5UAQWQ4JuBNXXom2P1UCo0YBaxugLjb3BNdEdcMGsM5OAD4oQ4bA\n719i6PpXALBWeOvMh5avHztmqLR8oBXs5CpgvXnRWa3ycl2cRQlhQa5bysBu0ScJTT1XAODBB6E8\n0ApFEYUnjY0ACgFsvVb0T8FJKLO3ATU1EZWhri8mRtvkP/4DePDBAbUcok+kDsgQCKJfIAG3QVEA\n5aoNeDF0Bl2fFENdLPRu5UobD7W3V+wDBuSYjyO3s1sMIiKaLS6OKACSx1AUxJ8iFvZDmd1rNv62\nsUAnEIBozNURgjL1PmCRvviCNVPGFcaxP/iga582VX71oJ9IJc5ZSMAtaNFaYSG6TgMlJaKdKgB0\ndprT+QBAHboOwGGoxZcj9PerUatGLtBgVyWpKHo6YNRxILJSk61ZmdB708bhZuItvA1bsxLMU6iN\nKdbCErYkaJtEzZpJRfReXw9s3Cgmea+6SreT3O47yMu0icyHBNwGWa4OCJujuxsYO1b0xm5uFr/3\negH26iooeaOA4k/QcuJqVFToaX6AcwQphTlaJMvW1YuWpEVFwIrnEHhoYULZDrb+uSVbwXix0G0S\nj7B1DKdyHG8sMUtFpkaKyqKtpfw4fFg8/tOf4jvuuVCmTWQ8JOA2eL3iXygEXHIJcOYMUFysr0sJ\nhKsrT2wE2y8X6S2D4qlxXH4MMK+4EwqJBRMcC3IMAqG+tRNYyRISCTc2RCyLIaaHHEvM4khvczxX\nT7Xojpjk5Kfps5B3BgBQVSWeM+Yuqu7vyViK8Ak3JJvqsnnzZj516lQ+efJkvnLlyoRSYTKR4MLH\n+Nj8o9w3qp0DIoXO59NT5YJBrqfiFRfz2nFbeO0dn2vphtaeJatXm/uK+Er2cP/kp3hww/HIcxuP\nPXMmr73jc/tBuuh9EW/aXNz9TDh37DmRbJ8W07n6o1dHVxfnixdzfuut4nE8KY793TskzemWRPpx\no51JqeuZM2f4JZdcwvft28d7e3t5RUUFf+utt+IeREbi8/GpeIsH4eP+8lbOuY14dXXx1ef9C/fD\nzyuwx5RAZ5y5AAAgAElEQVTXLR/bNn8qW6vlfNt9Oa3NrfxXvmgv1A5fcrdNqOyItY3xM9C2dRAz\ntxcPp3MOeM52JjU/yqSxEGnBjXYmZaHs2rULkydPhjfsKyxZsgQvvPACpk+fnuyNQfopKkI3RoLN\nPAllziIxoberCNhl6F3t8aB73HQE3rsLGD9B9AxfGe4ZHoDpp6kUPz8fDErU22+lzQOlvBVqg6jo\nBF4DYGlX63AbbzdxGmGjONyia2mL1u0Nx9LGKLcLWyRu7Bq7bZz2G/AUv0yqqsyksaQSsoZSSlIC\nfvDgQUyaNEl7PnHiRLzyyisR2wUMpipjDCwbkm/XrUPhhF4oP/gzPO8VAjv3grWH16sM+7yKAihj\nb0egoACNb9XAtzMyA0ViFHB28T/Ajr0FlFSYXrPzfgHAP/lpBN5rhDL1PrD1hgyUdeugLG4ShT9R\nvgjGoiPty/Ppp+EFJBDhW9sKqlxm7f33gbIyYMQI4IrnIJLGzeOP5pfHkxro9qKTMhItRe+PcfVX\nWXy6BZQmfx1RFAWKKdKLTVICnpOTE3sjmAU80/nWt0RmGeBBx0lg8V1iEvM91OHnCAElJVBu+xlk\nEBzakQfmr4FPEY2tjL1R7IRKVQH2+R+Bjz7SMh+U8taIToemCL70NuCDF6Bc3KSl9AEQiyezAKK1\nE9HyyAHzl6cgXDHqdhIuvK8CH5QDYqCNmws1/XZKm3SdJmkYr6PAZ+qXP1PHZUessfa3wFMlriPW\n4LbRGsnZkJSAT5gwAfv379ee79+/HxMnTkzmkGnn5z8X/wCRcVJXJ4Tm52f+C8oOH9ixENjvv2Na\nZVz2BreKt6KItMO2NqC9XX+tbfh/oxJ/Rd3UnSKnu8k8BtmeFRDiVltbiEB5a8xSdiO2AnlwGRgg\nsjl6e4Hx44GtovIzpqCGv3ispA3sWEhbHcfjKUR3t75ikHG/qGPZsAGs7yWxnNqK52xbykZE65n6\n5c/UcdkRa6z9fTEarNZQukjGZD99+jS/+OKL+b59+/jnn38+aCYxg0GRdTJ0qD5H6Bv6Mq/AHh48\nbyHnqqpt6/dzHpx0t/j96NGm1zSWLdOyTnyzT3Pe1cWDPj/3r+ixnWg0TurV1kZOSMqVgdxOUmqT\ngV1dnBcU6G9q8eLo2xuRE5Wqalodx7itm8Zb2jaGCVg5SRxzHMlkfsS7Wk082/d3RkoyK+1YiTVW\nmjzNGNxoZ9LqumnTJj5lyhR+ySWX8EceeSShQWQiwSDnM8d8wH0lIrvEP/HX3Ieg6E540UXaFyq4\n4Tj3D/2JLooTJ0Yeq+K7vAz7uA9BcawrX+T+srU8OGu5eZkwmy9qNIF0+l3UbSZMEOMcMcL+YuPy\nmJzziA6NrscydSrneXliHDNm2KZSxjMOV8Sblhdr+0RFNZH9BjKlcDAurZaluNHOpAt55s+fj/nz\n5yd7mIyDMWBX+ddRF6pDG7xA5xGEcC+U3BuhHMoD2/dHMGwGK74HypCrxU5FRcD27doxNOvgs5no\ngBcsbzu6C98D690KdLwB1hECpqwFSlYDDQv7bXUX0zG8XuDgQTGJ6dCXxLS9jSeq3LIKypujoZ6Z\ngJaDc7VN3dwRMwagqVNMLADA0aNgtxRrr/db46l4bQ7r9tbPIVGrIZH9BtKioZ7i2UUmXEUyleCs\n5TwIH68Y9g7n1dV67jYg8sMLV3L/ih5R6FP0W+5v6OLBhY9FRlhdXdyXv13vz11QIH4WF2vHCpYu\n4XzUKHH8qiphswQt47E8d/pdVFzeImvHtYv+DL3GneyPqIweLY5XVGS+C7BEp9Ei8Ij3HSuydRtZ\nyuPcdJMo8JEVXCNHmj+HRK2GRPajqPicxI12koA7EAxy7hv7NvcX/VTYHhe3CAvlvIXhpehHauIj\nC3a0J/KLbrBafCPbhFgPm68L+dix2pfZP+m/Td600wISSa9C70IMTOeeNEmMqaREF1u5+EPBj3nt\nuC3xC4uqCqvJauFYLhbRBDzitVTZDNbjGJ8bhTdeUTVeGBYvTr0PTww63Ggn9UJxgDGAFe5E4Mhy\nAJ8hcEU7lEkVYC2/AG54S2Rw3H+/acFfRQGY4XZX6amGEjofABBCBerKglA/HQlvz1fRgjrgjgZg\n6zawH8wHvveCth/WroXSZL+CfdItVl3cIptSUS+8ENi/X3T1uuEGKCO+BOX4EmAYQ2PPclQcbkNg\n9jawX9S4H1dZmTgmLO/H2ua2zfW7Sp3NYD3OnXeK51VV4rNobta9onisBqN1UlPjzm/KpvREIi2Q\ngFswebAddQBUKMMXQfnG93Wv9sILoYSAZnwV6pSDCH3kQSgEND/+MVjRt1BX0AeWewzsvf8CUA1W\n0gZ8vQGB1QzKxhPAqufhveYU2M0eNHfVoPn/Ai3vfQ0oL4A6+VagoQCqKr67snmWx2NIUzT4scoD\nrSYP2e792HZHtPO2FXOzLQDAkfvBkAc28yRQUAC24wnRY7y0FOg5CYyfgMCOGqdF61193k5tbu0q\nNh398VSlp1mPk6rjxnGB0T6TbEpPJNJDJtwGZCr+FT2c19RoWRKafWHxMWU6nb/4MfPttrRKAO6f\n+ow4pp+b0u+Mz7UeJle+yAHOy4Z1cv+KHr56tbiTrq0Npw2Ge6kY+7Q4vge//WOrVWA9t29UO/ev\n6BHvXVoF4fcdnHof9zd0cd+YN6OmMbqxe+LKNDFYCv4VPTG3ySjbwaV1xbkl7ZO873MWN9pJEXg0\nCguhPGBTYm6IypQ2j95L+8T3AZyACi/q0Aw2ZDvQJ15jY97UjqOqllV6NogV5NmuVuCBVijNR+GD\nglAPAzb8Dt01NVqLW78fCOxqBTo2C5tmziIkhNOqPMrjAF4DPgECH7wFrGwFbgl/BqZFHjxQ3v0v\nKG/+jxiPTdWek92TcKaJ0VLo2ACsrIm+TSbZDi6tK9P7t9sn3aXwREZBAh4FxgDlf2+AejAf7Mqf\nAVc8B0UpBGP6F4sxIFC8CjhwAM0F1Qj0NqJu5HooF/y/UD6vQ6NaB7VgKtp2Xw2MeB/txy8BAGzZ\nIs5x5AgALEIF2qB2AHUnVyFQvguBjlkoHDFUW86spQWoqBDf16Z5v0d3x0ZgziI0PmouZ5cVoEaB\n1C4wIegXi9HTwRYPBdauNYuArLic2hl5224RFHZiI5QOJi4mcYhlwkucFRVBgQ9s5kkxb+CwDYC4\nbIe45xX6QUTl/1kgEOOi5uYCRSJ/7pAJtwGZSjDIee3YzbwM+7gffg5wXnH5mUi7IGxH+BDkvLCQ\n+xv09EF/eauebRK2Unw+867+yU9FZDgES5dw/3k/5cHhi0QGzOzTWl9y4/ljtYmtrdUf+/3cMUvG\nmPIYLF3C+ezZsW2I+fNFCuTU+7TtXLWylTbHpEki+8Z6HicbRH6e0caUgO0QV7oi5yktrLH7vIx9\n5yNwk4ZIvcQHBW60kwTcBllKX1ZmKKVHkFdgD68Y8kaksISFrAJ7NKGXX8raWs75qFHCr4af+0a/\nYXq9okL32k253z6fqPo0lJsbhV9iLWfnnGsCGJy1XJTuG7blEyfqaYHV1fZf9FgiLwmLpZMf7Vhh\naji+9h6N55k921GA4vHM3aZcxpWuyHm/lZsb50UccXOBonL4QYEb7SQLxQbGACxdCvVQPbw5fQhx\nHwCgGx509HkR2DwLmPBrsMs/Bv7P/4FyxXPAtsfRfqYSi4duga/qFBgr1DsBHrwS7E9/Aqv6FMoP\nv4+VP9fP1d4OVFaKZlWszZyBwLBZdP8rvQNqxa0IPaPbDfLWWjtHfb2wR3p7hfR1dUEBg7dsO1Av\nKgdZTzUwYQJw4IBIC+zoEAez2g1GG6KgwPmWXVoqgRgfqPG2f+xYYPhw8bikRG+MZTxPaalpXIl6\n5tGskaQqPu0yU+K1LaJsH7ONbiyrKhWZM2TDZAeZcBXJJLRb2iE/1CLvsTjM5+Vt4/NGvSKi54If\na1kgWoQ4e7aWcVI77NmIW39j1GSbGbJsGeelpdw/bKUo9lBVUQloKPqYN09vdmVdos2YmaJF+8Nf\nFbbL2M36eIcN0yPw11/nQZ/fMbI2Zp7YLZcm7zDs+rvIbTjn+jGM/2QhT00N50uXmqtQDQ2zrMQT\ngTtuG6Pi09EGsquylceyVmrGwsbmcLxjSIclQjZM2nGjnSTgTuTkaKJXW/iM6By4oof7xrwpSsDl\n7f/s2cL6mLVcF3Cs5by0NFKADNaGUcyl167ZL+ES9WCQm8TGN/t0RBqiVjUpU/zg4/4Lfsn9054R\nFyCfSAkMwidE2GKbxBTEGLfs2liifeG7usTnIV8PtwrQMO57/vlC3B08+FjjdeXBG8dy663uLZQo\nbQUiKjWtWG0kJ5vDzv9PhyWS6nNmanpnBuNGO8lCcWLECOBYuHf21ddBKVwCAPAuuAx4XnxsChjY\njkZgyhQow1eBeZ6H0u2DFyrQ2QlMnw68/bZ++2nXrOrVVWDKRijHFyAAcS8fqHoR+HODKI4J6Pt4\ny7YDooxGQ9oAygOtYAV3geXkgK29Q5zzqg1Q/54Pr+dDsOs9QPN6vbJQ2iaWXuQRuG1uFC37w+MR\nn0NdHZCTY858qa8HXn9dPC4uBo4eFY8PHBA/6+qA9eu1Q8WyN0RRT/j2f/IyBBoWRt7+9/bqj3Ny\n7Aud3L5H+Tu7Sk0j1v97J5vDLsskHT20U33OTE3vzHYy4SqSkUyYwL+Jf+e1eU9x3zU9WoBVUcF5\n7dB1mk3BzztPi7r9057hvoIdehQNPw/6/BFWgn/8Gh6c9xMRkYT7c/vhFxbKtGfMVsWs5bwWa3nF\nsHciXIiKCvFP2ilGgkHOfSV7tMWW/eWtIhLdcDyyF3nZWu6f/JTe1jVGtGSNcmtrxUSsrR1jJcqE\nJr/gAvFzyBBTX5i4MTTbsr39v+km2zuBmL1n7O5Gwr3dY75vtxHtYJ2AHKzvqx9xo50k4E6MHatZ\nIsHr/sVcLdnQxf15/yZEdMjrep/vi1u475oe4YGHKxaDG45HVNb5V/QIvzh8EdCsk2nP8NpaB9FY\nulQT2to7PtcaaDmlngWDXLN1/OPXOFsgZWsjbQGLxcA5dxR115603F963fJ8xi+29L4Zs7da3GKo\nGI2nM2Gi/cdd7ec2vTHWdi4urhkJVZXGjRvtJAvFia4uAIACH5r3+eAN/1pkKXiAPduA0GmgD2B5\nOxA4A+CDDxD6oBCYfRsCXa1QPx8H77f+L5Cfj6ahX0b35x6gvBWNjYAvbzIABjbsFbCeLUDpOAT+\nv9uB5fXCNjHO/re2AuPGQe28AV68C+/BJngnTAC7+lYoPtEHJRTSMys8HuE6sBsa0birEL6LzqBu\n1i7UFbWKJczkcevrgQPhRalHjAAee0w8tlgMAJK/BTbuD5htCOOtemsr0N2d3O27oWLUdn+DLZRw\nNooxS8OyuLMtbqsqY1lWGzYIew6IsJfk+8nINcOpz3i/QAJuQftC9/4zGuGHAgV/PzQdDw3tQeUX\nhmnbaF0Hx08AKwfYnxQEZm6CUvCvCPxbHoAaBBa3QT3mRQgMvtV/g9dzDPj0U/jyihE6cx0Y/gRl\n4t3AcQ+Qny/8aafV4sOiyqAAPQDeA9iTL4AxsSAyNmxAQHlcE4Pubo9YZ7IQYCwPSt1esHefANoN\nx927F6xvrzj+p59CqV0LpgSAK68UCy5XVQm/GnD0uF2LhdErLisz++DWL7b1yx5vSlscYhF3VWh4\nLMruEVCOix2tizs7fibJLgpRXw98/LH+3OWi4pQSOIjJhNuATCQ4ZI55GTTpIU+6m9fmP839wx7l\ntUWtmnVSO+klHtxwnFeM+gevHbuZ+4a+rDkFZdjHK6ae5MGK7+p+Ofy6H2gs2ikt1TNGjOl4EyaI\nfaQ/HH5dq8g0WjLlrZq1UlERtn2M1Z5Ll4qxjB6tn7eqitfe8bk4mIPXm9QtsN3+bjMTHDJcUm0X\nuLJCbLJOHBtrRds3kUUhjPt7PNpiE8FZyx3XV7U9L5EVuNFOisBtUBSgeci96O7zIBTO+lCOzoD3\nV1vQ9tFoeM/uReB0IzB0KLz4G9gHCpTiW6Bsvxbtn0zCYvw3vAAKcQynwqFZ6B0vlMkz0Yy16Bx+\nKbzjC4GtDSIaKioCcBLK1PvA/vgwlAW7wLbO1Vq8Bu58B+yTb6IRD4vmWCNGgP2gHsxTrBXysLFv\ng3WEwGaehPKD7wOvikhw/frwivHhdrWYswhsy0NQ2j1g+BjK8EVgN44E1q6FurhAfAB2EWyCt8Da\nLb3d/m4j0PffFz9LSnSbx3jsWBGm3esyku6pBvvjw7bta22xu5NoimGfWPd1so/c7j9yJLBnD1Bb\nC4RCYAiBle3TPr+IuwhqSzt4yYSrSCahFahcpIjoObeDA5zPG7JNK6cHOPfn/xv3e9eKnG+ZE15T\no0e6JSV6JkruHO6f8GvOb7qJ+6eJnHLrRKW/vFWL5KyRoJxo9MPPg8W32E++GSJca7vaiGPOn6/d\nAdTe8bmpjax/8lNifHZFKwkQNaqNFYHKCN3jsY0gY62CpB3T+LrMz4+VqeKEzZ2E6zuBVN/F2Hx+\ntp83TSBmJW60kyJwC5qH+Z9LoOIZeM+q2JIzH1v6xOK9dflPY3HhSwiM+Q3w8cdQUAUAUHJvBPvH\nH4CzfcDixUBTE9iSXwHvvgs2+TOsfGUiAgdnQxkzA6FwB0EZQTLmgTqzBupOvRudqop/Xi/QIheW\nyLsJ6s3fB/PYLOCwfDnw4YfAnXdC9axH5ycFps52igI0NYn5QdWzHi0ogFLwrwg9kye6HKrhycz3\n3hUTZe9tA3rD0fHXvgb84Q+p/7Bj5RrbTHwqS38DJSCeapOOB5eBAdqiExFRvYxAATEBWF+v/278\nBGBNA1xjcyfheh4g2Yk86/42n5/tWGgCcdBCAu6EnOkHcIbnIgA/UFiIllN3wnctENh5GuzzLWAI\nQYFPCPcrr4ChELjqAuDHPwZTNwC8F8rb16ETpUBREUIfXQbfqNeBU1PgKS1Ec7MQbaNOVVSIBANZ\nkOK9+B4EZrRBOVUI5X/eAxa0QnmgFcqrQsgbGwF17JehHimEFypaUACfTxyrtlYc35ydUADvFCAQ\nyEMgEL7lXvDPYJuXIzBzk7B2Rt2uD2jPnrg+OsfMjldXgZ3YGJlh44RNkQzzFIPdom8SCABoWAjU\nv2BeAs1oF6xbJ4qqOjv1i8D2PKBjAxrfqtGKmeKagMyEiUCbyV6WaWMk+hUScAtSfFSsRQgMf0cn\njmIMVHjh9RaiYqjo/816/xkK/h8E4IciXEgAgHr+1cBSBvbVC4CeHgAAw4tYPGQmWNkh4O2AabEE\nKazSt5SCyhiAOzdCOTwFKs4iULAE6DkllnnrUIGOzWC/0Bc0COz6GbBZLPLgnbMEKDQ004K7Bk3e\nyjZg61bxxR8yBDhzRrxWVRXX5+eY2cE2RkbH0YTRbTWgUcjGjBH/SkrMr7/9tnYs7SKwsgYIWDxj\n43jGjBENv4qKnLODUoE85/vvC199xIjEBJiqHc85SMAtaGXVP9qPirNtaEclzsMJqEMuAY6dRPth\nQOkejWb+G9ShGQE0IpA7BOxCFYGKNgSmPQT24neg9FQD4FDCwt7Y969Q3tuBbpyHukl/1qLD5uZI\ncdWenzoFBQyVaEPDVFWkEnaoepTsMYjPunVoYuvRPWcJlJ2FWl6412u/FibrqQYaHhY57QCwfDnq\nPEXAnT8V4jF7thCDGTOAp592/fk55iHX1wNvvCEeV1Xp0XE00THYQhGCVl8PtqsI2LXX/FpHB/DR\nRyIN0ng8tzaCcTxjxohjAREdElOK8ZyyhUAiAmycrBw2TPxHUDQ+uMkEIz4TWf3gQV6CLl5SLBZS\n8CHIfQjymXmv8WDJl7RUQFlF6UOQ+4sf47WX/EWbwDSmm/nh5/5Ln+a1ZUFTJztfyR4enLWcr37k\nZOSiBx6P2O+CX2oLPfjLW7X0Mbk2pHUtRWOpe9TSdWNKXjjFkQN6aXgCE192Jf2cc/N5jeXxlok4\nu4UybNPfrK/J9zl6tPsmUdxmAtI4nvHjtQlp/vrr/TcRKM9ZUpJcubmhalc7VrKpg/E2obJuT02s\nEsaNdpKA2yAFsGKUyEAZiaMiA2TYzZpo+xEQX47iYl6LtZqgyz4oPgTNrV2LRCtan4/z2kt3iO6D\nhSv1i8OYNzUxCS58jPuH/kS/OIx+Q8vrDQaF2JpK8MM5wLIMX5bWa31OZCvZiy7Seq8Eh83nfMkS\n/UJwcUv8ec2Wz8uUh3zli+KiEKv7XleXGFe4+6Dp3JMm6cKmquaTWl8zCnphYeT2bnOhjReu8MIS\nprbBnKdelOQ5o7TRjQvjex05MjkhjTeH3Lo95aAnjBvtJAvFgnECrv2TC8ValfCCDd0JnOqFCi9U\neMOeNwfGTQE+6ENz31Io5y2EktcGHANCYFBybwTO9sGDbnhHHgNmijtl79hPoR4pBDBWnGj4cGDK\nVGDVKuCOx8E+/hg4MxsKmFjc+GMGFh4be3UV8P77CGAjAECFF9hwHlBTg5YWYZkAIhGGMYBtfhus\n46fA0KHAP84AfWKVZaVnFthLv9QtgqlLgZqacIfC2HnNRqvE1vNWHhdvVlZ+OvnZHo+YoLRbrPjC\nC4H9+8XiEw8+aLYUrK8ZM01OnYrc3m0utNFqGTFCvNfxXwNbY1hAOdVes/GcqfCtrfniHk/iY443\nh9y6vd2kMpE6MuEqkqn4RrXzWqzl80a8zIOXf8sU9ZZhH/cXrhTd/S7/lm6VlKzmwZJbub9wpbli\nb8wYLVKWS7DV5jTzEnRpm1Sc9y6vxVrzUmrw89qi3xoG5TPZM7JRldYXXLJsmYi+L24RlsWIEWZL\np/RXPHjV/9KaXcm7Ay1y99v00Q4TcS6738uIe8gQ0eM7HBHbHs/QdMt07lnLxTFGj47sDy6PX1ys\nL4Ahm3DZWRBumkSVlopmWzfdpFlWtkvGZXpnPbv3muiY47XSrNtTDnrCuNHOhNW1tbWVl5eX89zc\nXP7aa68lNYhMJLjwMV429BCvyHtDL3AJWxHSIuEejxCOsCDzmTNNQm8S8LC3rBXdjBljKqv3l6wW\nq/FY9xk6j/uuMQiILMKpquK8rExcPMLWg3XxAa3lbU2NVjbvzwlw/yVPivdU8Tz35wR4cMgc7h/2\nqCaysVaykcU+VvxXvsiDo74sbtt9Ps7z8vT3MnGi/bGXLRPiXFrKuaqaX5dffrs1Mg2fofZ7owcc\nr11gKZEP+vzaKkNGmyoY5NkpStk45nMcN9qZsIUyY8YMPP/887jvvvtSdTOQEWgWSgjo+HwcvHgH\nZbn/ACvZDYz+B5RPboRy9nqEwBDo/i5wxe/Bnm6F0rEZbPpGsA1PQ8F3RNMpI+efD3bmT2j77Tig\n4Xbg6quBzZtRh2aRqdL7uZZ2iPHjgS1bgAULwLb/J7z+Qj29MVyEgy8+BPziF2AdG4A3Q4DyM7AZ\nR9E0NIDKLwyDcnAZgHfRiABw8Slgwil41q4G+8/7wG4vFelzShPAQ0AfoPRcD1x3HTB/PrD1WmBX\na0T2grKrCEo7g4IJWjESYLBQjh4F++Q58ctQCCgogAIfWNH/ANu3AxDFSSb27tXT8665BqjdB+3A\n0lpYsEA8Hz0aOHRIPF+3TvsMtdtzuT1j8dsFRUVirAgBVVVg6xvAFi8GdocAqCLtUztOlIyWadNE\nvnl+PvDqqyItMBOgYp7+Ic31AQkL+LRp01I5joxBSyP8t1MowOcIgaHsrArlteHw5JUhcPYHUOCD\nAiZ88TXNaHuvGL89WwOEPoPadQNaUAc1LMwMCljuX4FLLwV27EA3btI94TFjgDNhH5sXogVi1R90\n5uLjH4/D6G/sB9ZC87ZVFUBBAfxXbkBg5+NQeoZAEWdAY89yYFcAzW98iuZNw8BW3wr25D3AxacQ\nWFkIoBCBkkYob+tr+ig91UKwAPFetm8HamvBOvZC6QCYQfwUBVA+mwngXdFdcfYZAHnwePSLXmO4\nYhQA2PDdYFMOQWm/Cfj330BZK4TM+F7q6vSujgp8YJ0hsJ0rEbFKsvTPDx0y52I7+epGH1Z6wLEE\ndd06KNdsAZs+Uu+UGD6Oev7VWrWmY5qkpLNT+PKAuCDu3x9lYyJjcSvM6c69TzbMZ4wNTgsl5wae\ngz5ehn3iFnrYo9yXo2j+tD/vR7z2K8e5b9zfedmQf4htchu5D0FeW/C0nqVi7MFh8Kw55/rKMOef\nz3l1tdnbzvuRNhZjPxP/lS9y/9CfRFgtcl9fwQ59n7APLteYNFofwaCwBfiCBSJdTmZtGFYNcvKR\nI+waOc4VPcJvv/VWzseO1cclF4Xgwqkwvie5XqapO6MTbnxciyVjSqfLyYnMTjGOP/yZafZLOCtE\n69IoP087a0bul58vzlVUFPVcAw6l88WH2+yZfpwPcaOdUSPwuXPnotNQUi555JFHsGjRItcXiYCh\n1I0xBpaRHecFTU3hHvn8B+DIRTc8KIMKT88hMARFxJqXBzbmLSjPrYb37Fko8KEbwxE46xcl92eG\nAODaMZWLvg7lvIXAx2+j8dAy4KongZqvgn33ebCR9wB/+QvwyisAbtYHcuYMMGoUMG8e1BcWI/BT\nFUrhzQh1LUItjiIAP1juX8FGv4mmo3ehua9OVIT2Xgt2/utQz7scnoIr4OloQx1+BPVAOHPmUUD5\naQhevg/IyweGvARUVgL336/fFQwZAnQeBiZPBl57TY9a5W14wNAF8KmngNOnReXj3e8BY8YIq+Wj\nqwD0oREBqH98Fup1Z+CdnIdQSC/tF9FsuEryohbRx8SuaEciI+5hw0SajV10ZLVkjh/XX+PcHBXX\n14uxfjYTuO02ND5aCJRdC3SMF/c14UwWb8Bw7I7xQMfmyGjLGIkNGwa89VZktJ/O223j+KZMEfbT\nQIwhE1sQuMFt9k0K1w5VFAWKTIFzS7JXicEWgX/zmyJ4NGaHDMFpPh4HOMDF+pTYoz03/svBGRFQ\n43B/oz8AAAiZSURBVENtWz/8PHj+bebud9ar+siRnAN8NSz53fDz2twWPg/6ivW+gh3imEPmmHJ9\nZc55Gfbp+8uJ19w5ekaLpQBJ65gIn5i4KzPns/tHPB6ROWJbnAPw4JivRPzO+H6DQdGf3LiWps8n\nfprG4vNH/0+KFh3JiEj+M06kAuao2HIcmSHEIZZjM/bY9vlEVhLAuX/8msiOkm4isXTmRMvxDR8+\nsGPI1jzwDJj0daOdKRHwV199NalBZCQeDx+KHu5HgAdnfMssRlVVnJ9/viY4Pih8LA6LCkzvWvO2\n8lZaWhMyg8T4hyGtlBkzzIscDxtm+gL4S3/F/Q3hPyy5z8yZ+uOSEpEdI4tx5B+hvHgMfcRs08DP\neW6uWXiM44xlAxjF0vI+eVUV5xdcEGGN2La35ZYFJ5JZIDhsyWivn3++Psbt26Mex9qWVxubXz+2\nv7w1rnU2XY+7v+my+bsZiDFkesplBtOvAv7cc8/xiRMn8sLCQj527Fh+8803JzyIjERV+cicT4Rg\ndnVxvngx91/6lPB4u7pEytvwVdx/6dPcd00PLys6Ir7gK3pENHd/p/CepQCGjxGc/S/Rc5RVlfNx\n47i/+DHxuKtLeMiLF5ujPuM+hkq+1RVrdTGyHD/47GEevOYh4Xlv3879Ix6PLBHv6uJ8wQLuH74q\ntofb1cX5woVmD90yLuuK7dayf4n83Fx9yWOJpfXzNP4/RDmOU19v41gTXfjY1bgHgoEeQya85yxl\nQCLwVAwiU/nmNyNXejciS9tXrxbbGrdJdrkvpwKaZPaPZ5v+Xt3c0ZbJQKL9DRBEf+FGO3PCG/Yb\nOTk56OdTEARBDDrcaGfuAI2FIAiCSDEk4ARBEFkKCThBEESWQgJOEASRpZCAEwRBZCkk4ARBEFkK\nCThBEESWQgJOEASRpZCAEwRBZCkk4ARBEFkKCThBEESWQgJOEASRpZCAEwRBZCkk4ARBEFkKCThB\nEESWQgJOEASRpZCAEwRBZCkk4ARBEFkKCThBEESWQgJOEASRpZCAEwRBZCkk4ARBEFkKCThBEESW\nQgJOEASRpZCAEwRBZCkk4ARBEFkKCThBEESWkrCAP/jgg5g+fToqKirw5S9/GceOHUvluDIGRVHS\nPYSkyObxZ/PYARp/usn28bshYQH/4he/iL/97W9ob2/HlClT8JOf/CSV48oYsv2PIJvHn81jB2j8\n6Sbbx++GhAV87ty5yM0Vu1dXV+PAgQMpGxRBEAQRm5R44L/5zW+wYMGCVByKIAiCcEkO55w7vTh3\n7lx0dnZG/P6RRx7BokWLAAA//vGPsXv3bvz+97+3P0FOToqGShAEcW4RRZ4BxBDwWDQ3N+PXv/41\nXnrpJRQWFiZ6GIIgCCIB8hLdccuWLXjssccQCoVIvAmCINJAwhH4pZdeit7eXowaNQoA8IUvfAG/\n/OUvUzo4giAIwpmEJzHfffdddHR0YM+ePdizZ09U8c7mnPHf/e53uOyyyzBkyBDs3r073cNxzZYt\nWzBt2jRceumlePTRR9M9nLi45557MHbsWMyYMSPdQ0mI/fv344YbbsBll12Gyy+/HD/72c/SPaS4\nOHXqFKqrq1FZWYny8nI8/PDD6R5S3PT19aGqqkqbq8smvF4vrrjiClRVVWHWrFnRN+YDwNatW3lf\nXx/nnPMVK1bwFStWDMRpU8Lbb7/N33nnHc4Y46+99lq6h+OKM2fO8EsuuYTv27eP9/b28oqKCv7W\nW2+le1iu+ctf/sJ3797NL7/88nQPJSEOHz7M9+zZwznn/Pjx43zKlClZ9flzzvlnn33GOef89OnT\nvLq6mv/1r39N84ji4/HHH+d33nknX7RoUbqHEjder5cfPXrU1bYDUkqfzTnj06ZNw5QpU9I9jLjY\ntWsXJk+eDK/Xi/z8fCxZsgQvvPBCuoflmuuvvx4jR45M9zASprS0FJWVlQCA4uJiTJ8+HYcOHUrz\nqOKjqKgIANDb24u+vj7NKs0GDhw4gE2bNuHee++NmcWRqbgd94D3QqGc8f7n4MGDmDRpkvZ84sSJ\nOHjwYBpHdO6iqir27NmD6urqdA8lLs6ePYvKykqMHTsWN9xwA8rLy9M9JNd873vfw2OPPaYFjdlG\nTk4ObrrpJlx99dX49a9/HXXbhLNQrLjNGS8oKMCdd96ZqtOmBDdjzyYo9z4zOHHiBL7yla/giSee\nQHFxcbqHExe5ubloa2vDsWPHMG/ePCiKAsZYuocVk40bN+KCCy5AVVVV1pbS79ixA+PGjcNHH32E\nuXPnYtq0abj++uttt02ZgG/bti3q683Nzdi0aRNeeumlVJ0yZcQae7YxYcIE7N+/X3u+f/9+TJw4\nMY0jOvc4ffo0brvtNtx1111YvHhxuoeTMCUlJVi4cCFeffXVrBDwl19+GS+++CI2bdqEU6dO4dNP\nP8XSpUvx5JNPpntorhk3bhwAYMyYMfinf/on7Nq1y1HAB+QeQ+aMv/DCC1mdM54tftrVV1+Nd999\nF6qqore3F7/97W/xpS99Kd3DOmfgnOMb3/gGysvL0dDQkO7hxM3HH3+M7u5uAEBPTw+2bduGqqqq\nNI/KHY888gj279+Pffv24dlnn8WcOXOySrxPnjyJ48ePAwA+++wzbN26NWo21oAI+Le//W2cOHEC\nc+fORVVVFR544IGBOG1KeP755zFp0iTs3LkTCxcuxPz589M9pJjk5eXh5z//OebNm4fy8nLcfvvt\nmD59erqH5Zo77rgD1157Lfbu3YtJkyZh7dq16R5SXOzYsQNPPfUUgsEgqqqqUFVVhS1btqR7WK45\nfPgw5syZg8rKSlRXV2PRokW48cYb0z2shMg2O/HIkSO4/vrrtc/+lltuwRe/+EXH7ZMqpScIgiDS\nR3ZO0xIEQRAk4ARBENkKCThBEESWQgJOEASRpZCAEwRBZCkk4ARBEFnK/w8xa73D9IkvqgAAAABJ\nRU5ErkJggg==\n",
       "text": [
        "<matplotlib.figure.Figure at 0x7ada8d0>"
       ]
      }
     ],
     "prompt_number": 26
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "## Putting together\n",
      "\n",
      "- Use X, y, train_index, test_index"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "## UNSUPERVISED FEATURE LEARNING\n",
      "\n",
      "## train a sparse filter on the whole data\n",
      "sf = SparseFilter(n_features=50, n_iterations=1000)\n",
      "sf_X = sf.fit_transform(X)\n",
      "## train a PCA on the whole data\n",
      "pca = PCA(n_components=15)\n",
      "pca_X = pca.fit_transform(X)\n",
      "## combine both featues\n",
      "sf_pca_X = np.c_[sf_X, pca_X]\n",
      "## standarize the features\n",
      "ss = StandardScaler()\n",
      "norm_sf_pca_X = ss.fit_transform(sf_pca_X)\n",
      "print norm_sf_pca_X.shape"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "(1000, 65)\n"
       ]
      }
     ],
     "prompt_number": 27
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "## train and test split\n",
      "\n",
      "train_X = norm_sf_pca_X[train_index, :]\n",
      "test_X = norm_sf_pca_X[test_index, :]\n",
      "train_y = y[train_index]\n",
      "test_y = y[test_index]\n",
      "\n",
      "print train_X.shape, test_X.shape\n",
      "print train_y.shape, test_y.shape"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "(800, 65) (200, 65)\n",
        "(800,) (200,)\n"
       ]
      }
     ],
     "prompt_number": 28
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "## SVC model\n",
      "\n",
      "\n",
      "svc = SVC()\n",
      "gammas = [1e-4, 3e-4, 1e-3, 3e-3, 1e-2, 3e-2, 1e-1, 3e-1, 1., 3., 10.]\n",
      "gs = GridSearchCV(svc, {'gamma': gammas}, n_jobs=-1, cv = 5, scoring='accuracy')\n",
      "gs.fit(train_X, train_y)\n",
      "print gs.best_params_\n",
      "print gs.best_score_\n",
      "\n",
      "svc = SVC(probability=True, **gs.best_params_)\n",
      "svc.fit(train_X, train_y)\n",
      "print svc.score(test_X, test_y)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "{'gamma': 0.03}\n",
        "0.9275\n",
        "0.92"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n"
       ]
      }
     ],
     "prompt_number": 29
    },
    {
     "cell_type": "raw",
     "metadata": {},
     "source": [
      "## SGD model\n",
      "\n",
      "sgd = SGDClassifier(loss = 'log', penalty = 'elasticnet', )\n",
      "alphas = [1e-4, 3e-4, 1e-3, ]\n",
      "gs = GridSearchCV(sgd, {'alpha': alphas}, n_jobs=-1, cv = 5, scoring='accuracy')\n",
      "gs.fit(train_X, train_y)\n",
      "print gs.best_params_\n",
      "print gs.best_score_\n",
      "\n",
      "sgd = SGDClassifier(loss = 'log', penalty = 'elasticnet', **gs.best_params_)\n",
      "sgd.fit(train_X, train_y)\n",
      "print sgd.score(test_X, test_y)"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "## Extra Random Forest\n",
      "\n",
      "forest = ExtraTreesClassifier(n_estimators=100, )\n",
      "max_features_choices = [0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0]\n",
      "gs = GridSearchCV(forest, {'max_features': max_features_choices}, \n",
      "                  n_jobs=-1, cv = 5, scoring='accuracy')\n",
      "gs.fit(train_X, train_y)\n",
      "print gs.best_params_\n",
      "print gs.best_score_\n",
      "\n",
      "forest = ExtraTreesClassifier(n_estimators=100, **gs.best_params_)\n",
      "forest.fit(train_X, train_y)\n",
      "print forest.score(test_X, test_y)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "{'max_features': 0.1}\n",
        "0.92625\n",
        "0.935"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n"
       ]
      }
     ],
     "prompt_number": 30
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "pd.DataFrame(forest.feature_importances_).plot(kind = 'bar')"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 31,
       "text": [
        "<matplotlib.axes.AxesSubplot at 0x8a0ca90>"
       ]
      },
      {
       "metadata": {},
       "output_type": "display_data",
       "png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAEBCAYAAACDu+UiAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJztnX9UVNe1xzcG0PJLJAoYRhmVgUF+DCg6SbpM8KGhWoI1\nsS0m9WmEt6gNMfQlNtqXNk2aID41RlfWe9Gs1C5No1abRluRNmmZqKmGJsHmvWhb7INg8Ef8AUKI\nDYrf9weZC5e5M3OHe2fuvTP7s9YsvXvOOXufc+7dd/jeM2fCAIAYhmGYoGKE1gEwDMMw6sPJnWEY\nJgjh5M4wDBOEcHJnGIYJQji5MwzDBCGc3BmGYYIQr8m9vr6erFYrWSwWWrdunWSZlStXksViIZvN\nRk1NTYLdbDZTbm4u5efn08yZM9WLmmEYhvFIuKc3+/r6qKqqit566y1KSUmhGTNmUGlpKWVmZgpl\n6urq6PTp09Tc3EzvvvsurVixgo4fP05ERGFhYeRwOCghIcG/vWAYhmFEePzk3tjYSGlpaWQ2myki\nIoLKyspo//79ojIHDhygpUuXEhGR3W6nzs5OunDhgvA+f0eKYRgm8HhM7u3t7TRhwgTh2GQyUXt7\nu+wyYWFhNGfOHCooKKCXX35ZzbgZhmEYD3iUZcLCwmQ14u7T+dGjR+m2226jixcv0ty5c8lqtdKs\nWbOG5YNhGIYR40kZ8fjJPSUlhc6cOSMcnzlzhkwmk8cyn3zyCaWkpBAR0W233UZEROPGjaOFCxdS\nY2Oj2wCdr6eeekp0rDeb3uIxQox6i4djDM54jBCjmm16w2NyLygooObmZmptbaXe3l7as2cPlZaW\nisqUlpbSjh07iIjo+PHjFB8fT0lJSfT5559Td3c3ERH19PTQ73//e8rJyfEaUGtrq65teovHCDHq\nLR6OMTjjMUKM/mpTCo+yTHh4OL344otUXFxMfX19VF5eTpmZmbR161YiIqqsrKT58+dTXV0dpaWl\nUXR0NG3fvp2IiM6fP0/33XcfERHduHGDHnzwQbrnnntkB8YwDMMoABozNISGhgaXMnqy6S0eI8So\nt3g4xuCMxwgxqtmmt/Qd9mUhzQgLC5OlHzEMwzADeMudutt+wOFw6Nqmt3iMEKPe4uEYgzMeKVtC\nQgKFhYUZ+jX4S6Du5kEKj5o7E7zExSVQd3eHcBwbO4a6uq5oGBHDqE9HR4fhlYHhLhdnWSZE6T9h\nBo87zwMTfARDfnHXB8PJMgzDMIxydJfc9aTXGVVn9K2svuIxwphxjNr79iXGYMKXPuouuTMMwzDK\nYc09RGHNnQkFpPLL0MUEaiN3ccKVK1eovLyc3nzzTRo7diytXbuWFi9e7FJuuJo7r5ZhGCak6E/s\n/vsg090tb3XLww8/TKNGjaJPP/2Umpqa6Otf/zrZbDaaOnWqKnHoTpbRk15nZJ2RNXffbHqLR06M\ncXHiNdxxcZ7XQ4fqmOmRnp4eev311+mnP/0pRUVF0Ve/+lVasGAB7dy502M91twZJgQY+ATaQETw\nq9TAqMvf//53Cg8Pp7S0NMFms9noo48+Us0Ha+4hCmvuxofn0DtS+cV13FT36nUejhw5Qt/61rfo\n3Llzgu3ll1+m1157jRoaGsSt8Tp3hmEYYxATE0NdXV0i29WrVyk2NlY1H7pL7nrS64ysM7Lm7ptN\nb/H4EmMg5tDIY6ZH0tPT6caNG3T69GnB9pe//IWys7M91mPNnWEYRsdER0fTfffdRz/+8Y/p888/\np6NHj9JvfvMbWrJkiWo+WHMPUVivNT48h97Rq+ZO1L+p2fLly4V17rW1tVRWVuba2jA1d07uIQon\nBuPDc+gdPX+JSS5B80BVT3qdkXVG1tx9s+ktHtbc1bFJ0dV1RfKHq9V6+XPrbNbcGYZhQhyWZUIU\n/pPe+PAceicY8kvQyDIMwzCMcnSX3PWk1xlZZ2TN3Teb3uJhzV0d25gxYzT/DVSlrzFjxnjsozt0\nl9wZhmHU4sqVK9TQ0ODy0FMr23DqX7kyvAe0rLmHKKzXGh+ew9CGNXeGYZgQRHfJ3ZuWxntYe7f5\nVlZf8RhhzPQWI2vu7m2hEI87dJfcvcF7WDMMw3jHcJo764zqwONofHgOQxvW3BmGYUIQ3SV3PWnF\nRtDwWHNXx6a3eFhzV8cWCvG4Q3fJnWEYhlEOa+4hCo+j8eE5DG1Yc2cYhglBdJfc9aQVG0HDY81d\nHZve4mHNXR1bKMTjDt0ld4ZhGEY5rLmHKDyOxofnMLRRrLnX19eT1Woli8VC69atkyyzcuVKslgs\nZLPZqKmpSfReX18f5efn07333utj6AzDMMxw8Zjc+/r6qKqqiurr6+nkyZO0a9cuOnXqlKhMXV0d\nnT59mpqbm2nbtm20YsUK0fubN2+mqVOnfvkpwzt60oqNoOGx5q6OTW/xsOauji0U4nGHx+Te2NhI\naWlpZDabKSIigsrKymj//v2iMgcOHKClS5cSEZHdbqfOzk66cOECERF98sknVFdXRxUVFfznIsMw\nTCCBB/bu3YuKigrheOfOnaiqqhKVKSkpwTvvvCMcFxUV4f333wcALFq0CB988AEcDgdKSkokfXgJ\nQbI8EQa9fKvP9MPjaHx4DkMbb/Md7inxy5VSMORTOQD67W9/S4mJiZSfn+/1T4lly5aR2WwmIqL4\n+HjKy8ujwsJCIhr4M8R53I+DiAaOHQ6H2/J8LH08gPhYL/HxsbzjgfnTRzx87L9jh8NBP//5z4mI\nhHzpEU+Z/9ixYyguLhaOa2pqUFtbKypTWVmJXbt2CccZGRk4d+4c1qxZA5PJBLPZjOTkZERFRWHJ\nkiVe7z4NDQ0uZQbbSPi00uDyacVbXTVsgfLj7xgDOY7+aJNj5GtBji2Y4/GSvuFRcy8oKKDm5mZq\nbW2l3t5e2rNnD5WWlorKlJaW0o4dO4iI6Pjx4xQfH0/JyclUU1NDZ86coZaWFtq9ezf9y7/8i1CO\nYRiG8S9e17kfOnSIqqurqa+vj8rLy2nNmjW0detWIiKqrKwkIhJW1ERHR9P27dtp2rRpojbefvtt\n2rhxIx04cMA1AF7nrgk8jsaH5zC08ZY7+UtMIQqPo/HhOQxtDLdxmOsDP17b66vNt7L6iscIY6a3\nGPlacG8LhXjcobvkzjAMwyiHZZkQhcfR+PAchjaGk2UYhmEY5eguuetJKzaChseauzo2vcXDmrs6\ntlCIxx26S+4MwzCMclhzD1F4HI0Pz2Fow5o7wzBMCKK75K4nrdgIGh5r7urY9BYPa+7q2EIhHnfo\nLrkzDMMwymHNPUThcTQ+PIehDWvuDMMwIYjukruetGIjaHisuatj01s8rLmrYwuFeNyhu+TOMAzD\nKIc19xCFx9H48ByGNqy5MwzDhCC6S+560oqNoOGx5q6OTW/xsOauji0U4nGH7pI7wzAMoxzW3EMU\nHkfjE+xzGBeXQN3dHcJxbOwY6uq6omFE+oJ/Q5WRhMfR+AT7HAZ7/5RiuAeqetKKjaDhBbPmHheX\nQGFhYcIrLi5BdzEG0uberq941PejpK4x5pA1dyak6P+THETUQEQQ/YnOMIxnWJYJUYwwjkaIUUuC\nfXyCvX9KMZwswzAMwyhHd8ldT1qxETS8YNbcB1lVbS/Y5jXYrwXW3L23KYXukjvDMAyjHNbcQxQj\njKMRYtSSYB+fYO+fUlhzZxiGCUF0l9z1pMMaQcNjzV0tH/qaQ9bcRVYFdY0xh6y5MwzDMLJgzT1E\nMcI4GiFGLdFyfAbv++KvPV94/j3De8swkhhhHI0Qo5ZoOT5i3/7xy/PvGcM9UNWTDmsEDY81d7V8\n6GsOjaK5B2ZulNQ1xhyy5s4wDMPIgmWZEMUI42iEGLWEZZnQRrEsU19fT1arlSwWC61bt06yzMqV\nK8lisZDNZqOmpiYiIvrnP/9Jdrud8vLyaOrUqbRmzZphdoFhGIbxFY/Jva+vj6qqqqi+vp5OnjxJ\nu3btolOnTonK1NXV0enTp6m5uZm2bdtGK1asICKiUaNGUUNDA504cYI+/PBDamhooKNHj3oNSE86\nrBE0PNbc1fLh37lRY29693Z1YhxOPKy5ax+POzwm98bGRkpLSyOz2UwRERFUVlZG+/fvF5U5cOAA\nLV26lIiI7HY7dXZ20oULF4iIKCoqioiIent7qa+vjxISEohhQhHem54JOPDA3r17UVFRIRzv3LkT\nVVVVojIlJSV45513hOOioiK89957AIAbN27AZrMhJiYGq1atkvThJQTJ8kQY9PKtPtOPEcbRCDHK\nxR990XJ8xL794zeY5t8feBuPcE+Jv/+BhqwbhGS9W265hU6cOEFXr16l4uJicjgcVFhY6FJ/2bJl\nZDabiYgoPj6e8vLyhHLOP0PE9RxENHA8uN2h5flY+ngA8bFe4huYb2d8+ohHb+Ot1fj4qz966Z8e\njx0OB/385z8nIhLypUc8Zf5jx46huLhYOK6pqUFtba2oTGVlJXbt2iUcZ2Rk4Pz58y5tPfPMM1i/\nfr3Xu09DQ4NLmcE2Eu7mDS53c2911bAFyo+/YwzkOA63vhFilGtToy/+aHO4NrFv365huTZ/jJnW\nNjXb9JK+4VFzLygooObmZmptbaXe3l7as2cPlZaWisqUlpbSjh07iIjo+PHjFB8fT0lJSXTp0iXq\n7OwkIqJr167Rm2++Sfn5+d7vNgzDMIxivK5zP3ToEFVXV1NfXx+Vl5fTmjVraOvWrUREVFlZSUQk\nrKiJjo6m7du307Rp0+h//ud/aOnSpXTz5k26efMmLVmyhFatWuUaAK9z1wQjjKMRYpSLP/rC69xD\nG95bhpHECONohBjlwsldqQ//+TEqvLeMAdapahljYNYpa7fOWW/zqqQv/mhT6ZgFZm6U1DXutenb\nPLiiu+TOMAzDKIdlmRDFCONohBjlwrKMUh/+82NUDCfLMAzDMMrRXXLXkw5rBA2PNXe1fLDmPpx4\nWHPXPh536C65MwzDMMphzT1EMcI4GiFGubDmrtSH//wYFdbcGYZhQhDdJXc96bBG0PBYc1fLB2vu\nw4mHNXft43GHx10hGf8SF5cg2tc7NnYMdXVd0TAihmGCBdbcNUQ/mmlgfcvFCDHKhTV3pT7858eo\nsObOMAwTguguuetJhw2cZqZ2e/oaR+X11Y2HNXd1bIHyzZq79zal0F1yZxiGYZTDmruG6EczDaxv\nuRghRrmw5q7Uh//8GBXW3BmGYUIQ3SV3PemwrLkHPh6jxsiauz/9KKnLmjvDMAwTRLDmriH60UwD\n61suRohRLqy5K/XhPz9GhTV3hmGYEER3yV1POixr7oGPx6gxsubuTz9K6rLmzjAMwwQRrLlriH40\n08D6losRYpQLa+5KffjPj1FhzT2IiYtLoLCwMAoLC6O4uAStw2F0wOBzgs+L0EZ3yV1POqzeNff+\n7YIbiAiirYMD4ds3H6y5D7IqqOu9zf7zACR1XrDmrr3NX21Kwfu5MwzjN/g3C7RD15q71Ikx8MlE\naMGwOpxSTVGJ7mkEPdMIMcolUJp7P/4fM7nnnpJ+B9P8+wNDa+4Dibz/NVR6YBhGPVivDy50l9zl\nam6subuvGyjfrLkPx4+Suv69PnzV633xo/Y5rrd51aPmrrvkzjAMwyhH15q7lppiIGDN3TNGiFEu\nRtDcfYmRNXftMbTmzjAMwwwP3SV31tyV1w2Ub9bch+NHSd3AXB/ydXRfyvo/HtbcxeguuTMMwzDK\nYc1dQ1hz94wRYpQLa+7+jScUYc2dYRgmBJGV3Ovr68lqtZLFYqF169ZJllm5ciVZLBay2WzU1NRE\nRERnzpyh2bNnU1ZWFmVnZ9OWLVu8+mLNXXndQPlmzX04fpTUZc3d97qhq7l73Vumr6+Pqqqq6K23\n3qKUlBSaMWMGlZaWUmZmplCmrq6OTp8+Tc3NzfTuu+/SihUr6Pjx4xQREUGbNm2ivLw8+uyzz2j6\n9Ok0d+5cUV2jw3tnMAyjR7xq7seOHaOnn36a6uvriYiotraWiIhWr14tlPnud79Ls2fPpm9/+9tE\nRGS1Wuntt9+mpKQkUVvf+MY36JFHHqGioqKBAAyuuWupKbLmbhxYc/dvPKGIYs29vb2dJkyYIByb\nTCZqb2/3WuaTTz4RlWltbaWmpiay2+2yg2cYhvE3wbqnjldZpv/u6Z2hd5DB9T777DNatGgRbd68\nmWJiYlzqLlu2jMxmMxERXbp0iRYtWkSFhYVfvusgohNEVD2ohuPLf/vLOBwOKiwsFOlRzvovvPAC\n5eXlCccOh4NOnDhB1dXVwvHgOs5jd+0Nra80nqH98zUeoheIKG8gEpn1B3Ctr2R8pMZbSX2l4yN3\n/v19/gww/PGWMz4DNiJ/no/i/gzyrLPrQ874Duyp0/9ed/fsgd75OX/4Ut/hcFBtbS0lJycL+dIj\n8MKxY8dQXFwsHNfU1KC2tlZUprKyErt27RKOMzIycP78eQBAb28v7rnnHmzatEmy/aEhNDQ0iN4j\nAogavvyXJG1Sdf1lkxOj3DaV9qW/vmvdQPj2xTbc+kaIUa5Njb7IaTNQ56O4rG/XsD/icWcLlB8t\nzjNv6dtrcr9+/TomT56MlpYWfPHFF7DZbDh58qSozMGDBzFv3jwA/TcDu90OALh58yaWLFmC6upq\n9wF4CHBg0AHXkxcuE6EFSuJR2hdxfSV1tR9HKYwQo1z80Re1rw9f6so999SOJzZ2zKB+9h8rxajn\nmbc4vcoy4eHh9OKLL1JxcTH19fVReXk5ZWZm0tatW4mIqLKykubPn091dXWUlpZG0dHRtH37diIi\neuedd+jVV1+l3Nxcys/PJyKitWvX0te+9jVvbhmGYVwY+mM93d3yZOOQJEA3GbcMDYFlGfnx9Ndn\nWUbLGOXa1OiLnDYDdT6KywZOlvG1PbX7rdSmZpve0jd/Q5VhGCYI4b1lFMLr3P2HEWKUC69zVyee\nfvw/jkY4z3hvGYZhmBBEd8ldem2wPJvaeza428dBbozy21S3bqB8Kx2zYIoxMHMdmOtDbl3fyqob\njy8xBvt55g7dJXeGYRhGOay5K4Q1d/9hhBjlwpq7OvH0w5o7EWvuDMMwIYnukjtr7srrBso3a+7D\n8aOk7vCuD08bYwWL5j64j0M3/gr288wdXr+hyjCMsRFvjFVI3d1hkr9DYGT6+9JAzv4xrLkrJhQ1\n90D9QIlRtVAptNTcldiMorkH+/MnKUJWcw/WPZr1wMAnwf5Xd3cHjzfD6AzdJXe1NPeBBNRAzgTk\nyQdr7srqehpvX9oMJi1UL5q7Gja9a+7ubcF/nrkjKDT3YNMPGYZhlBIUmrtcmz+6KhVjbOwYWZq0\nUTV3Xn/sO6y5e29TTl13MbLm7kpQfHLXG7znNMMwWhNUmrsS/TBQumewau5qxRNMWihr7r6XY819\neG1KobvkzjAMwyiHNXeFKNGfWXP3X4x6gzV3723KqesuRrnxSC2+GCqjGuU8C9l17gwjF16jHzpI\nfUcjWNFdcmfN3dd4pGz61tz9tdfJcG3++k4Ea+6+lwuU5h6IXMGaOxNyePvCE8MwymHNXSGsuffX\n7Wc4eq37soHS3APhhzV3723KqesuRiXxuGtT77DmzjAME4LoLrmz5u5rPFI2fWvuvvoOlBYaGD9K\n6rLm7ruNNXeGYRgmiGDNXSGsuffX7Yc190D6YM19oD3W3F3hT+4MwzBBiO6SO2vuvsYjZWPN3Vdb\n4PwMr67nL1rJ88Oau+/1jay5866QIQDvd298pH4HlWE8wZq7QoyguautM7LmHngfgdDXpWysuesX\n1twZZhCD5Q3eQ4YJZnSX3Flz9zUeKVtgdEYjau798kYDSW97oJ4fdzZl4y1d358275uqudZnzd2T\nX17nzjCMDuB9gIxLSGnucn/b1BdYc/e9PS01d6kxk+tH6sG03PPHqJq7lE2pxs2auzrwb6gOgn/b\nlFECnz+MkdCdLBMIzV2p7smau2/t+VLf35q7u/YCFY/RNHfPNmk7a+6e/OpIc6+vryer1UoWi4XW\nrVsnWWblypVksVjIZrNRU1OTYF++fDklJSVRTk6O7IAYhglunA9pZ8+ezauW/Ak8cOPGDUyZMgUt\nLS3o7e2FzWbDyZMnRWUOHjyIefPmAQCOHz8Ou90uvHf48GF88MEHyM7OduvDUwhEBCIMepHqNnfE\nxo4R6sTGjlEco9y6viCur3wch+fX9/YCNT7e26SAxaPuXPvnWvBlXtU+9wZfb+JjbeIxAt7i9PjJ\nvbGxkdLS0shsNlNERASVlZXR/v37RWUOHDhAS5cuJSIiu91OnZ2ddP78eSIimjVrFo0ZY8xvQw7+\nrUXnQzSt1kjzb3wywU4o/bZpoPD4QLW9vZ0mTJggHJtMJnr33Xe9lmlvb6fk5GTZQSxbtozMZjMR\nEV26dIkWLVpEhYWFX77rIKITRFQ9qIbjy38LSYyDXHmBiPKGlBlob7CGVVhYOETTErc3sEb6BHV3\nf39Y8bzwwguUl5fntn9S8Yi/ek7U3T3bbf986Y+7+idOnKDqat/HR414hjM+Um2Xlt7nYWXLC5Lt\n+TOeQS3S8Mfbtb6/rw+p9sQxvSD2MKQ/gY3HtT2l8Qydf7nzNXTu1KjvcDiotraWkpOThXzpEU8f\n6/ft24eKigrheOfOnaiqqhKVKSkpwdGjR4XjoqIivP/++8JxS0uLT7JMQ0OD6L3+P5MaJP6EUscm\n5XfAt7icuL6UzTc/SstJxahkHN2NhdL21O73cGN0N2ZaxiOnrrv6Wtlc49H+GtbqWvDVpmabXtI3\nPL577NgxFBcXC8c1NTWora0VlamsrMSuXbuE44yMDJw/f1449jW5D32PZOhjSmzyfLvX9ZT4UVpO\nKh4l4yiFUi3UH/2Wgy9jpiQeqfFRuy/u6mtlczeOWsXtj3jkzqsvyH2OJxdv55BHzb2goICam5up\ntbWVent7ac+ePVRaWioqU1paSjt27CAiouPHj1N8fDwlJSV5alZXsJ7tGdZCPcPjo1+UXNv+mFep\n53h+xdvdoa6uDunp6ZgyZQpqamoAAC+99BJeeukloczDDz+MKVOmIDc3VyTJlJWVYfz48YiMjITJ\nZMLPfvYzr3efQP9JJ2Ub8C1lc5aVsrlv0/MnYLEfOePgLkYl46j2HPgSj5JySsdM7bGQW9fTp0OW\nZdS+XtXxI+fckzeH7sdMrs1b+vb6DdV58+bRvHnzRLbKykrR8YsvvihZd9euXd6aDxl4P25mKHxO\nGAu5208o2aZCTUJqbxm5NkB6rwq5+5Io9T0Ud/tzBGI/DSX98yUeX/o9FKmLaehWAb7OoZKxUFp3\ncH+ciSGY9pYJVDxa/cas0utVLry3DBP0BNueL4P7Y/S+BDt6/pUz3ewt4/kLQg6JGlrZ/OtHnd/K\nVH8/jUDt76F2uUDtk6JljHq7FtTvtxKbf/143hJZfjz+2FtGN5/cB+uP4i/phBasw3pGz5+UGEZP\n6EZz10ozk7JpqbkrjVGKYNLc5fp2F48RNHc9nWe+xChFsGvuSmNUAv+GKmMI9PZ9A73FwzC+osPk\n7tC5TUvfcm3604q9tem7dqm2Tb141Hlu4j1G7W1iu//6rcSmpW8pm7Q9qDV3RntYz1aHUH1uEqr9\n1iusuUvYQlVz90eMeotHT3PIMQZnPL7EqATW3EMM1ooZxtiodQ3rMLk7dG7T0rd3m/batZRNb2uf\npWxa+pZr09K3lE1L33JtWvqWsknbB18fnq5h1twZhmEMgD+fc7HmLmEzsuauN51RCh4zjjGY4/FH\njAP7JfXjPOa9ZRiGYQzMcPZPYs3dZ5uWvuXatPQttvGeQWratPQtZdPSt1yblr6lbP5q0xUdJncm\nmBj8cIh/pYhhAgdr7hI21tyDM0a9xcMxBmc8gYyR17kzDMOEGDpM7g6d27T0LdempW8pm5a+5dq0\n9C3XpqVvKZuWvuXatPQtZfNXm67oMLkzDMMwSmHNXcLGmntwxqi3eDjG4IwnkDGy5s4wDBNi6DC5\nO3Ru09K3XJuWvqVsWvqWa9PSt1yblr6lbFr6lmvT0reUzV9tuqLD5M4wDMMohTV3CRtr7sEZo97i\n4RiDM55AxsiaO8MwTIihw+Tu0LlNS99ybVr6lrJp6VuuTUvfcm1a+payaelbrk1L31I2f7Xpig6T\nO8MwDKMU1twlbKy5B2eMeouHYwzOeAIZI2vuDMMwIYYOk7tD5zYtfcu1aelbyqalb7k2LX3LtWnp\nW8qmpW+5Ni19S9n81aYrOkzuDMMwjFJYc5ewseYenDHqLR6OMTjjCWSMrLkzDMOEGDpM7g6d27T0\nLdempW8pm5a+5dq09C3XpqVvKZuWvuXatPQtZfNXm654Te719fVktVrJYrHQunXrJMusXLmSLBYL\n2Ww2ampq8qmuKyd0btNbPEaIUW/xcIzBGY8RYvRXm654TO59fX1UVVVF9fX1dPLkSdq1axedOnVK\nVKauro5Onz5Nzc3NtG3bNlqxYoXsutJ06tymt3iMEKPe4uEYgzMeI8TorzZd8ZjcGxsbKS0tjcxm\nM0VERFBZWRnt379fVObAgQO0dOlSIiKy2+3U2dlJ58+fl1WXYRiG8Q8ek3t7eztNmDBBODaZTNTe\n3i6rzNmzZ73WlaZV5zYtfcu1aelbyqalb7k2LX3LtWnpW8qmpW+5Ni19S9n81aYE8MC+fftQUVEh\nHO/cuRNVVVWiMiUlJTh69KhwXFRUhPfee09W3S+XYfKLX/ziF7+G8fJEOHkgJSWFzpw5IxyfOXOG\nTCaTxzKffPIJmUwmun79ute61B+dpxAYhmGYYeBRlikoKKDm5mZqbW2l3t5e2rNnD5WWlorKlJaW\n0o4dO4iI6Pjx4xQfH09JSUmy6jIMwzD+weMn9/DwcHrxxRepuLiY+vr6qLy8nDIzM2nr1q1ERFRZ\nWUnz58+nuro6SktLo+joaNq+fbvHugzDMIz/0Xz7gVOnTtH+/fuFh60mk4lKS0tdbgSnTp2is2fP\nkt1up5iYGCIiOnr0KH300UdUWVlJDoeD3nvvPcrPz6eioiK3/o4cOUJ79+6l2bNn08KFC+nzzz+n\n2tpa+tXmG36XAAAQEElEQVSvfkV33XUX1dbW0ujRo4XyX3zxBe3evZtSUlJozpw59Itf/IL+9Kc/\n0bhx42jUqFF07tw5GjFiBGVkZNADDzxAcXFxfhglhmEY39D0G6rr1q2jxYsXE1H/Mkq73U43b96k\nxYsX09q1a4VyW7ZsoW984xu0atUqysrKojfeeIPWrFlDjz/+OH3/+9+nH/zgB7R69Wq6du0aPf30\n07R+/Xqh7tixY4X/v/zyy/TII4/Q7t27af369bR27Vp69NFHqauriz7++GN69dVXafLkyfRf//Vf\ndPHiRSIieuihh6iuro42b95MS5YsoX379tHVq1dp27ZttGfPHmpsbKQvvviC2trayG63U0NDg9/G\n69NPP5Vd9vLly36Lw8nVq1dp9erVdNttt9Frr70meu973/ue1/py+xOIvhAp64/e+qIEI5xnZ86c\noYqKCpo2bRp1dnbSQw89RNnZ2bRkyRKX+Hfv3i38v7Ozk8rLyyknJ4ceeOABunDhgvBebm4uPfvs\ns/SPf/wj4P1xotZ5RkReHrf6mbS0NPT29rrYv/jiC0yZMkU4zsrKQnd3N0wmE1paWjB9+nQkJSXh\n+vXryM3NRUxMDDo7O1FSUoL58+cjNjYWJSUlKCkpQVhYGEpKSnDvvfdi+vTp+PTTT2G1WvHZZ58h\nKysL+fn5AIC8vDz09fVh0qRJeOihhzB27FgUFxfDZDKhq6sL169fx7hx43D9+nVkZWXh+vXryM7O\nRk9PD+666y50dHRgxYoVGDlyJOLj4zFmzBhkZGTgiSeeQEdHh9CXzs5OPPHEExg/fjx+8YtfAADa\n2tpQXl6O/Px8dHR0YNmyZcjIyMA3v/lN/PWvf8Xly5dx6dIlJCYm4vLly7h8+TI6OjqwfPlyZGdn\nIzMzE//7v/8LAPjzn/+MSZMmITIyEnFxcXj11VcF342NjSgsLMSDDz6ItrY2zJkzB7GxsRg/fjwm\nT56M2NhY3HrrrZg2bRrmz5+PjIwMoS+pqalYunQpHA4H3n//fbz//vuYPXs2li5ditGjR6OkpAT3\n3Xcfrl27hra2NiQkJAh996U/Y8eOxcKFC3H+/Hm/9SUjI8OlL+7609zcjPLyciQmJga0L3FxccjP\nz8e//du/YerUqV77E2rnWWFhIbZs2YLx48cjIyMDa9euxccff4y1a9eipKREiP/SpUuIiIgQjpcv\nX47/+I//QGVlJZ555hksWLBA6Et4eDhiY2ORmJiIgoICPP/88zh48KCuzrPU1FShL97QNLlnZGSg\npaVFZMvOzkZGRgYiIyORnZ2N7OxsjBw5EtnZ2YiMjAQAdHd3IzY2FtXV1bDZbLDZbAD6E/QDDzyA\nKVOmwOFwoKGhAeHh4Thw4AD279+PvLw8AMD999+PV155BTabDcuWLUNjYyPy8vLwt7/9DQUFBQD6\nbzBvvPEG4uLikJCQgCtXriAmJgaXLl1CdnY2rly5gqlTp+Ly5cuYPn065s6di9raWqSnp+PmzZsA\ngN/97neoqqqC3W73+UQNCwtDQkICoqKiYDabYTabQUQwm82YNGmScJK2tLQgOTkZCxYsAADcfffd\naGxshNlsxvLlyxERESGcqDabDXV1dXjttdeQkpKCX/7yl7j33nvx+OOPIz8/Hxs3bsTTTz+Nr371\nq8jPz8cjjzwi9GXEiBGYPHkyxowZg8LCQhQWFiI6OhqFhYUYNWoUAODZZ5/FnXfeiVmzZiElJQU1\nNTU+9yc9PR3PP/88FixY4Le+nD17FmFhYaK+uOvP6NGjUVNTM6y5UdKXmzdv4o477sDkyZPR1tbm\nsT+heJ5FRUXh4sWLyMvLw4QJE4T8ERYWhsjISKEvQ/uTm5uLmzdvIisrCwCQm5sr9MWZA/Lz8/H2\n22/ju9/9LsLDw5GTk4Py8nJdnGfh4eFCX7yhaXI/dOgQpkyZguLiYlRUVKCiogKRkZFISUnB9u3b\n0dLSgpaWFtx+++04ePAgxo8fL9SdMWMGFi9ejLCwMPT19QEAbty4gWeffRaxsbH44IMPAEAYDOeA\nnD17Fh0dHVi8eDEiIyMxc+ZMhIeHIzIyErNmzcKJEydEMdbU1MBsNiM9PR1bt25FZmYmbr/9dowc\nORIzZsxAeno6XnnlFVgsFly4cAGzZs0S6o4YMQKFhYX4yle+4vOJumHDBhQXFyM9PV1oz3lzAyCc\npABgtVqRk5MDALDb7QAg3MiysrJEJ2phYSG2bt0q+HHWc94gp0+fDovFgr6+PpHvqVOn4m9/+xss\nFotgs1qt6Ovrg8lkEmzbt2/HyJEjMXHiRADwuT9WqxW9vb3Izc31W1+c/UlNTRXZpPozYcIETJ06\nFRMnTgxoX5z9cfbFU39C8TwbPC8//OEPBfuGDRsQExODv/zlL4LtlltuwcaNG7FhwwakpqaK5iYn\nJ8elL9nZ2ULdvLw8HDp0CMuWLdPFeWY2myEXTZM70J+Q//SnP2Hv3r3Yt28fSkpK0NDQICrT1taG\nc+fOoaysTLBdu3YNN2/exJEjR0RlL168iDfffBOLFi3C9773PdEADqanpwcnTpxAU1MT/vznP7u0\nM5iWlhbhz6DTp09j9+7d2LdvH/bu3YtTp04BAObMmYN169bh/PnzQr309HQ89thjKCoqEmy+nKht\nbW2Ii4tDdXU1rl69KnmSAsCWLVsQExODP/zhD3jqqaewcuVKpKWl4cc//jG+853vCO0VFBTg2Wef\nxd133w2TyYTXX38dt99+O7Zs2YKZM2fijTfewD333CP0ZfLkyULdbdu24d///d9FfXn88cfx+9//\nHr/+9a9F4zVp0iSkpaUBgM/92bJlC+bMmYNJkyb5rS/nzp3DAw88gNtvv10Ut7M/r7/+umDLzc3F\noUOHkJaWFtC+AP2JITMzEwA89idUzrPB8/Lkk0/iV7/6lXCeOWlubsa8efOwaNEioT/x8fH4yU9+\nIrwuXLiALVu24K677sLcuXOFvsyePdulLzNmzEB9fT327Nmji/PMUMndn/zmN7/BmjVrAuLr8uXL\nWLVqlaC5xcfH47bbbsPy5ctF+phUQnR3ov7973/H/fffjzfeeAMzZ85EdHS0y0kK9P/5N3fuXHzz\nm99EXl4esrOzkZycjJdeekn0TOPdd9/F3XffjbKyMrS2tqKoqAhRUVGIiopCTEwM7rzzTkHfe/jh\nh5GYmCj0JSMjA8uXL8frr7+O7u5uoc2TJ0+ipqZGZHvyySexb98+lzGS25+9e/ciNTXVr31ZtWoV\nDh48iI8++ggA0NDQgPXr12PTpk0iW1FREQ4cOODSl+bmZqEvdrvdbV/27dsnqy+FhYWivsTGxsJq\ntWLq1KkYPXq0x/54Os/kJER38/LUU0+5nGdz5szx+3m2atUqoS+HDx/Ghg0b8Lvf/U5oX67NOTeJ\niYmi/jrL/ud//qeoL1/72tdEfTl8+DBWrlyJ3NzcYfXFarXiwQcfFKTnnp4e/OhHP8Kdd96JRx99\nFJ2dnYItLS0N1dXVoucmUudZUlKSy7nojqBO7nrhZz/7mYvtlVde8cnW09ODDz/80Kf23JWVYzt5\n8iQee+wxdHV1AQA2b96M9PR03HHHHZg4cSJ+/etfC7bRo0cLNidWqxVvvfWW2xuBsz8nT57Ec889\n51LuueeeE3w7bWvXrvV6Y5Gqe+TIEfz3f/83AOCPf/wj1q9fj7KyMtjtdhQUFGDVqlWw2+2YNWsW\nYmNjYTKZBNszzzyD3NxcPPbYYwCkbwIHDx7ET3/6U2zatEl44Di4nJRN6qYytNybb76JI0eOuNjf\neust0VwN/qTpZMmSJcOy9fT04P7775dV1xffcsodPnwYEyZMEBL0tm3bYLPZkJKSgjvvvBM1NTUe\nbT/5yU8E2+A2U1JSXNocWtadb6lyGzZsQH19vcjHI488ItxIP/vsMzz55JOYP38+br31Vly6dAkA\nUFFRgUcffRSpqan40Y9+hIULFwq2l156CU888QQWLlzochPo6OhAT08PHn/8cRQVFeEHP/gBOjs7\nJedjMJzcA4CUNBQI23DrO5P2qFGjhKTtXLGUl5cnrFhKTk52sW3atAmbN2/GyJEjsWDBAo83gkDY\nVq9eDbvdjq985SuihB0VFYW1a9eip6dHWG2VmZmJrq4uZGVlCbbVq1djxowZovpSNwG1bc888wxM\nJhMmTpwougFlZGQgISEBmZmZwoqwW265BUlJSUhKSvKbLSoqComJiSKb0jZHjBghrGRzJtPk5GQh\nmTpXt+Xl5Qmr2zzZgH792/mw1FObM2bMEOr7Wk7qxjJu3DjccccdqKmpERL2kSNHhBVTAISVeVar\nFUC/FOO0ZWZmCqv/PN0Ejhw5gqeeekpo0xOc3FXCubJn8GvkyJEYOXIkiEhVW1hYmIufweXclfVk\nG+onMzMTERERaGlpwbRp05CcnAxg4KFTd3c3oqOjhRVLTts999yDW2+9VXgo5elGEAib1JJZoP/C\nGvpgbPC/zv87L7qcnByPNwG1bUB/EnAut3Xa8/Ly8O1vfxuTJk0SrQibM2cONm3aJMv2/PPPy7IN\nrpucnIy0tDQUFRXhj3/847B9D7U5HA44HA4hmebk5KCtrQ0ZGRnCuZaTk4PLly8jKyvLo815fg5+\n0OmuTee/NpvN53JSNxap5dVA/8o85wcn58q8+++/H8899xwKCgoEm9VqFVbreboJOMnNzfWakzi5\nq0RiYiI++OADYYVPS0sLxo4di9/+9rcYN26c32xO+9ixY1XzbbFY0NLSIqxO6u7uxpgxY/Dggw+K\nVm/cfffd+PrXv46wsDDB1tvbi9GjR4ts7m4EgbBJLZkFgJkzZwrJ3bnaaubMmWhvb0d+fr5gs9ls\n6OjoQH5+vsebgNo2d2Vu3LiBjRs3IiYmRlgRlpqaio0bN6KoqMhvNrPZLPhWq03n+vNLly4JSTQ1\nNRVmsxmRkZHC6rbU1FSkpqZ6tQH9q3aysrK8tpmTk4PW1lbRzUFuOakbi9TyaqD/+wC33norJk2a\nJKzMmzhxIhITE5GSkiLYoqKiYLFYcOLECY83AQCiJdue4OSuEg899BAOHz4saRu8ykdtm9M+eHWB\n0jYLCwvR1NQksv3jH//AokWLREm7ra0NZ8+edVlpVFhY6KLhS90IAmGTWjIL9K9kGPpp6Nq1a7h4\n8aLwbAPoT/gff/wxPvzwQ483AbVtQP8nT2dSH2zv6OhAVlaWy4qwM2fO+N2mZpu33HKLyzJlAOjq\n6hLd5ID+ZwH/93//59U2ceJEmEwmr206bw4mk8nnclI3Fqnl1WazWVhe3dnZKazMO3fuHACIbH/9\n61/xr//6r15vAoPb9AYnd8YF59LToUgtPZVbX+pGEAibpyWzg5O4O65duyZpG1pfbRvQnwilYhxc\nVmpFWCBs/moTkE7aSpHbppJyQ5dXS11D3vB2E/ClTc03DmMYhmHUR9ONwxiGYRj/wMmdYRgmCOHk\nzjAME4RwcmcYhglCOLkzDMMEIf8PHPxapYY5GfgAAAAASUVORK5CYII=\n",
       "text": [
        "<matplotlib.figure.Figure at 0x95c0710>"
       ]
      }
     ],
     "prompt_number": 31
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "yhat_prob = (svc.predict_proba(test_X)[:, 0] + forest.predict_proba(test_X)[:, 0]) / 2.\n",
      "yhat = (yhat_prob < 0.5).astype(np.int)\n",
      "print yhat.shape\n",
      "print np.mean(yhat==test_y)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "(200,)\n",
        "0.93\n"
       ]
      }
     ],
     "prompt_number": 32
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "selected_features = np.where(forest.feature_importances_ >= 0.01)[0]\n",
      "selected_train_X = train_X[:, selected_features]\n",
      "selected_test_X = test_X[:, selected_features]"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 33
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "## SVC model\n",
      "\n",
      "\n",
      "svc = SVC()\n",
      "gammas = [1e-4, 3e-4, 1e-3, 3e-3, 1e-2, 3e-2, 1e-1, 3e-1, 1., 3., 10.]\n",
      "gs = GridSearchCV(svc, {'gamma': gammas}, n_jobs=-1, cv = 5, scoring='accuracy')\n",
      "gs.fit(selected_train_X, train_y)\n",
      "print gs.best_params_\n",
      "print gs.best_score_\n",
      "\n",
      "svc = SVC(probability = True, **gs.best_params_)\n",
      "svc.fit(selected_train_X, train_y)\n",
      "print svc.score(selected_test_X, test_y)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "{'gamma': 0.03}\n",
        "0.93375\n",
        "0.945"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n"
       ]
      }
     ],
     "prompt_number": 36
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "yhat_prob = (svc.predict_proba(selected_test_X)[:, 0] + forest.predict_proba(test_X)[:, 0]) / 2.\n",
      "yhat = (yhat_prob < 0.5).astype(np.int)\n",
      "print yhat.shape\n",
      "print np.mean(yhat==test_y)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "(200,)\n",
        "0.93\n"
       ]
      }
     ],
     "prompt_number": 38
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [],
     "language": "python",
     "metadata": {},
     "outputs": []
    }
   ],
   "metadata": {}
  }
 ]
}