{
 "metadata": {
  "name": ""
 },
 "nbformat": 3,
 "nbformat_minor": 0,
 "worksheets": [
  {
   "cells": [
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "%matplotlib inline\n",
      "import matplotlib.pyplot as plt\n",
      "import numpy as np\n",
      "\n",
      "from sklearn.datasets import load_digits\n",
      "from sklearn.cross_validation import train_test_split\n",
      "np.set_printoptions(precision=2)\n",
      "\n",
      "digits = load_digits()\n",
      "X, y = digits.data, digits.target == 3\n",
      "X_train, X_test, y_train, y_test = train_test_split(X, y)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 1
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "from sklearn.grid_search import GridSearchCV\n",
      "from sklearn.svm import SVC\n",
      "\n",
      "param_grid = {'C': 10. ** np.arange(-3, 3), 'gamma': 10. ** np.arange(-3, 3)}\n",
      "grid = GridSearchCV(SVC(), param_grid=param_grid)\n",
      "grid.fit(X_train, y_train)\n",
      "print(grid.best_params_)\n",
      "print(grid.score(X_test, y_test))"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "{'C': 10.0, 'gamma': 0.001}\n",
        "0.997777777778\n"
       ]
      }
     ],
     "prompt_number": 2
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "from sklearn.cross_validation import cross_val_score\n",
      "cross_val_score(SVC(), X_train, y_train)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 3,
       "text": [
        "array([ 0.89,  0.89,  0.89])"
       ]
      }
     ],
     "prompt_number": 3
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "from sklearn.dummy import DummyClassifier\n",
      "cross_val_score(DummyClassifier(\"most_frequent\"), X_train, y_train)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 4,
       "text": [
        "array([ 0.89,  0.89,  0.89])"
       ]
      }
     ],
     "prompt_number": 4
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "from sklearn.metrics import roc_curve, roc_auc_score\n",
      "\n",
      "for gamma in [.01, .1, 1]:\n",
      "    plt.xlabel(\"FPR\")\n",
      "    plt.ylabel(\"TPR\")\n",
      "    svm = SVC(gamma=gamma).fit(X_train, y_train)\n",
      "    decision_function = svm.decision_function(X_test)\n",
      "    fpr, tpr, _ = roc_curve(y_test, decision_function)\n",
      "    acc = svm.score(X_test, y_test)\n",
      "    auc = roc_auc_score(y_test, svm.decision_function(X_test))\n",
      "    plt.plot(fpr, tpr, label=\"acc:%.2f auc:%.2f\" % (acc, auc))\n",
      "    print()\n",
      "plt.legend(loc=\"best\")"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "()\n",
        "()"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "()"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n"
       ]
      },
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 5,
       "text": [
        "<matplotlib.legend.Legend at 0x7f814677f550>"
       ]
      },
      {
       "metadata": {},
       "output_type": "display_data",
       "png": "iVBORw0KGgoAAAANSUhEUgAAAYYAAAEPCAYAAABGP2P1AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3XlcU1feP/APiFYExAjUBagLIKBIwB21DtandRttXX4d\nBHdEtHW695ku1qUztbXLdGxpH6DuG7VVKloVFRVcUFFZrOKCqAhUrQiIILKE8/vDMSVCIITckITP\n+/XKq4ScnHtye82X7z2bmRBCgIiI6L/Mm7oBRERkWBgYiIhIBQMDERGpYGAgIiIVDAxERKSCgYGI\niFRIGhhmz56NDh06oHfv3mrLvPbaa3Bzc4NcLkdKSoqUzSEiIg1IGhhmzZqF2NhYta/v3r0bV65c\nQUZGBiIjIzF//nwpm0NERBqQNDA8++yzkMlkal/fsWMHZsyYAQAYOHAgCgsLcfv2bSmbRERE9WjS\nPobc3Fw4Ozsrnzs5OSEnJ6cJW0RERE3e+fzkihxmZmZN1BIiIgIAi6Y8uKOjI7Kzs5XPc3Jy4Ojo\nWKOcq6srMjMz9dk0IiKj5+LigitXrjT4fU2aMYwfPx7r168HAJw4cQLt2rVDhw4dapR7FBQEhOBj\n8eLFTd4GQ3nwXPBcGNq5qFBU4KODH6HDFx2w4+IO/R4/ORnC2xti7FiI3FwIIbT+g1rSjGHKlClI\nSEhAXl4enJ2dsXTpUlRUVAAAQkNDMWbMGOzevRuurq6wsrLCmjVrpGwOEZFkMvMzMfWXqWj7VFuk\nhKagk00n/Ry4vBz45BPg//4P+PJLYNo0oJG35CUNDFFRUfWWCQsLk7IJRESSEkJgw9kNeHvf2/jw\n2Q/x2sDXYG6mp5sxKSnAzJmAszOQmgp07qyTapu0j4Eazt/fv6mbYDB4Lv7Ec/EnfZ6LwoeFmL9r\nPs7ePou4aXGQd5Tr58ASZAnVmQkhDH6jnkcjlQQMv6VE1FwcyTqCab9Mw1i3sfjyhS9h2dJSPweu\nniVERtaZJZiZmUGbr3hmDEREDVChqMDHCR9jZcpK/DDuB/y1x1/1c2CJs4TqGBiIiDSUmZ+JoOgg\ntGvdDimhKeho3VE/B5aoL0GdJp/gRkRk6IQQWJe6DoNWDcIUrynYHbRbP0GhvBxYvBgYORJ4+21g\n507JgwLAjIGIqE4FpQWYv2s+fvvjNxyYfgDeHbz1c2A9ZwnVMWMgIlLjcNZh+ET4wKGNA06HnNZP\nUGiiLKE6ZgxERE+oUFRgacJSrEpZhZXjVmJsj7H6OXATZgnVGU1gqGP1biIinbmSfwVB0UFob9le\nfx3MehxxpAmjuZWUn9/ULSAiUyaEwNrUtfBb5Yeg3kHYFbhLP0EhJQXo3x84c+ZRljB9epMGBcCI\nMgYiIqkUlBYg9NdQpN9Jx8HpB9G7g/rtiHXGwLKE6owmYyAikkLC9QT4RPigo3VHnAo5pZ+gYIBZ\nQnXMGIioWapQVGBJ/BKsTl2NVeNXYYzbGOkPasBZQnUMDETU7FzJv4LAbYGwb2OP1NBUdLCuuQ+M\nzhnIiCNN8FYSETUbQgisSVkDv1V+mOY9DbsCd0kfFAxgXkJDMWMgombhcQfzhbwL+utgNqIsoTpm\nDERk8uKvx0MeLkcn605ImpMkfVAwwiyhOmYMRGSyKhQVWHRoEdalrcOq8asw2m209Ac10iyhOgYG\nIjJJGXczEBgdiKetnkbqvFQ8bfW0tAc0khFHmuCtJCIyKUIIrE5ZjcGrB2OGfAZ+nfKr9EHBwOcl\nNBQzBiIyGfml+Zi7cy4u372MQzMOwetpL2kPaEJZQnXMGIjIJMRfj4dPuA+c2johKSRJ+qBgYllC\ndcwYiMiolSvKsejQIqxPW4/VL67GKNdREh/QNLOE6hgYiMhoXb57GYHbAtHRuqN+OphNYMSRJngr\niYiMjhACK5NXYvCqwZjlMws7p+yUNigY+byEhmLGQERG5XEHc0Z+BhJmJqDX072kPWAzyRKqY8ZA\nREbj4LWDkIfL4dzWGSfnnJQ2KDSzLKE6ZgxEZPDKFeX46OBH2HB2g346mJthllAdAwMRGbRLeZcQ\nFB2ETjadkDYvDQ5WDtIdrBmMONIEbyURkUESQuCHMz9g6JqhCPYNxo6AHdIGBROel9BQzBiIyODc\nfXAXITtDcLXgKhJmJqCnQ0/pDsYsoQZmDERkUA5cPQB5uBxd23XFyTknpQ0KzBJqxYyBiAxCuaIc\nCw8uxKbfNmHNi2vwgssLEh6MWUJdGBiIqMldzLuIoOggONo4IjU0Vfq+hGY84kgTvJVERE1GCIHI\nM5EYunoo5vjOQUxAjHRBoRnPS2goZgxE1CTuPriLOTvn4FrBNRyZdQSeDp7SHYxZQoNImjHExsbC\nw8MDbm5uWL58eY3X8/LyMGrUKPj4+MDLywtr166VsjlEZCDirsZBHi6Hi8wFJ+eclC4oMEvQipkQ\nQkhRsUKhgLu7O+Li4uDo6Ij+/fsjKioKnp5/XgBLlixBWVkZPv30U+Tl5cHd3R23b9+GhYVqImNm\nZgaJmklEelRWWYaFBxdi87nNWPviWjzv8rx0B6ueJURGNsuAoO13p2QZQ1JSElxdXdG1a1e0bNkS\nAQEBiImJUSnTqVMnFBUVAQCKiopgZ2dXIygQkWm4mHcRfqv8cDn/MtLmpUkXFJglNJpk38K5ublw\ndnZWPndycsLJkydVyoSEhOC5555D586dcf/+ffz0009SNYeImsjjDuaFhxbiX8P/hbl958JMqqGh\n7EvQCckCgyb/45ctWwYfHx/Ex8cjMzMTzz//PNLS0mBjY1Oj7JIlS5Q/+/v7w9/fX4etJSIp5D3I\nw5wdc5B1LwuHZx6Wti+B8xIQHx+P+Pj4RtcjWWBwdHREdna28nl2djacnJxUyiQmJuLDDz8EALi4\nuKBbt264dOkS+vXrV6O+6oGBiAzf/sz9mBUzCwFeAdgyeQuesnhKmgMxS1B68o/mpUuXalWPZH0M\n/fr1Q0ZGBq5fv47y8nJs2bIF48ePVynj4eGBuLg4AMDt27dx6dIldO/eXaomEZEelFWW4Z1972BW\nzCysfWktvnzhS2mCAvsSJCNZxmBhYYGwsDCMHDkSCoUCwcHB8PT0REREBAAgNDQUH3zwAWbNmgW5\nXI6qqip8/vnnaN++vVRNIiKJXbhzAYHRgehi2wWp81Jh38ZemgMxS5CUZMNVdYnDVYkMmxACEWci\nsPDgQiwbsQwhfUKk6WBmX0KDaPvdybGhRNQoeQ/yELwjGNn3snF09lF42HtIcyBmCXrDtZKISGv7\nM/dDHi6Hu507jgcflyYosC9B75gxEFGDlVWW4YMDH2DL+S1Y/9J6jOg+QpoDMUtoEgwMRNQg6XfS\nEbgtEN1k3ZA2Lw12bex0fxD2JTQpBgYi0ogQAuGnw/HRoY/w6YhPMafPHGk6mJklNDkGBiKq152S\nOwjeEYzc+7k4NvsY3O3ddX8QZgkGg53PRFSnfZn74BPhA097TxwPPi5NUODeywaFGQMR1aqssgzv\nxb2HrRe2YsOEDXiu23O6PwizBIPEwEBENaTfSceUbVPgInNBamiqNB3M7EswWLyVRERKQgh8f+p7\nDFszDH8f8Hdse3mb7oMC5yUYPGYMRAQA+KPkDwTvCMbN+zel62BmlmAUmDEQEfZe2QufcB/0cuiF\nxOBE3QcFZglGhRkDUTP2sPIh3o97H1svbMWmiZswvNtw3R+EWYLRYWAgaqbO/3EegdGBcGvvhrR5\naWhvqeMl7zniyGgxMBA1M0IIfHfqOyyJX4Ll/7Mcs31n634GM7MEo8bAQNSM/FHyB2bHzMbtkttI\nDE5ED7seuj0AswSTwM5nomYi9kosfMJ90Pvp3jg2+5jugwJnL5sMZgxEJu5h5UP8Y/8/8MvFX7B5\n0mb4d/XX7QGYJZgcBgYiE3buj3MI3BYId3t3pM5L1X0HM/sSTBJvJRGZICEEwpLCMHzdcLw56E38\nNPkn3QYFzkswacwYiEzM7eLbmL1jNv4o+QOJsxPhZuem2wMwSzB5zBiITMjujN3wifCBvINc90GB\nWUKzwYyByAQ8rHyI/93/v9h+cTt+nPQj/tL1L7o9ALOEZoUZA5GR++32b+j/Q3/cKr6FtHlpug0K\nzBKaJWYMREZKCIFvk77Fxwkf44vnv8BMn5m6ncHMLKHZYmAgMkK3i29jVsws5D3Iw4k5J+Da3lV3\nlXNeQrPHW0lERmbX5V3wifBBn059cGz2Md0GBc5eJjBjIDIapRWl+N/9/4sdl3dgy+QtGNZlmO4q\nZ5ZA1TAwEBmBs7fPInBbIHo93QupoamQWcp0Vzn7EugJvJVEZMCEEFhxYgVGrB+Bdwe/ix8n/ai7\noMARR6QGMwYiA3Wr+BZmxcxCfmk+jgcf131fArMEUoMZA5EB+vXyr/CN8EXfTn1xdNZR3QUFZgmk\nAWYMRAaktKIU7+5/Fzsv79R9BzOzBNIQMwYiA3H29ln0+6Ef8h7kIW1emu6CArMEaiBmDERNrEpU\n4ZuT3+CTI5/gqxe+wjTvabqbwcwsgbQgacYQGxsLDw8PuLm5Yfny5bWWiY+Ph6+vL7y8vODv7y9l\nc4gMzs37NzFm0xj8eO5HnAg+geny6boJCswSqBHMhBBCiooVCgXc3d0RFxcHR0dH9O/fH1FRUfD0\n9FSWKSwsxJAhQ7B37144OTkhLy8P9vb2NRtpZgaJmknUZH69/CtCdoYgpE8IPhr2EVq2aKmbiqtn\nCZGRDAjNmLbfnZLdSkpKSoKrqyu6du0KAAgICEBMTIxKYNi8eTMmTZoEJycnAKg1KBCZmgcVD/Du\nvnexK2MXfpr8E57t8qxuKubsZdIRyW4l5ebmwtnZWfncyckJubm5KmUyMjKQn5+P4cOHo1+/ftiw\nYYNUzSEyCGm30tAvsh/yH+YjdV6q7oIC1zgiHZIsY9DkPmlFRQWSk5Nx4MABPHjwAH5+fhg0aBDc\n3HS8FSFRE6sSVVhxYgWWHV2Gf7/wb0z1nqq7vgRmCaRjkgUGR0dHZGdnK59nZ2crbxk95uzsDHt7\ne1haWsLS0hLDhg1DWlparYFhyZIlyp/9/f3ZUU1G4+b9m5gZMxNFZUU4Oeckusu666ZijjiiJ8TH\nxyM+Pr7xFQmJVFRUiO7du4tr166JsrIyIZfLRXp6ukqZCxcuiBEjRojKykpRUlIivLy8xPnz52vU\nJWEziSQVczFGdPiig1h0cJGoUFToptKyMiEWLRLCwUGIdeuEqKrSTb1kcrT97pQsY7CwsEBYWBhG\njhwJhUKB4OBgeHp6IiIiAgAQGhoKDw8PjBo1Ct7e3jA3N0dISAh69uwpVZOI9OZBxQO8vfdt7Lmy\nB1tf3oqhzwzVTcXMEkgPJBuuqkscrkrGJPVWKgK3BcKnow++H/s92rVu1/hK2ZdAWjC44apEzU2V\nqMJ/TvwHnx79FF+P/BpTvafqpmJmCaRnDAxEOvD7/d8xc/tMFJcX666DmVkCNREuokfUSDsu7UCf\niD4Y7DwYh2cd1k1Q4LwEakLMGIi09LiDOTYzFtte3oYhzwxpfKXMEsgAMDAQaSHlZgoCowPRt1Nf\npIamwra1rQ4qZV8CGQbeSiJqgCpRhS8Tv8QLG1/AwmcXYuPEjY0PClwJlQwMMwYiDf1+/3fM2D4D\nDyoeIGlOErrJujW+UmYJZICYMRBpYPvF7egT0QdDnYciYWZC44MCswQyYMwYiOpQUl6Ct/a+hf1X\n9yP6b9EY7Dy48ZUySyADx4yBSI3km8noG9kXDyofICU0pfFBgVkCGYkGBwYhBLZs2SJFW4gMQpWo\nwhfHvsDIjSOx6C+LsGHChsZ3MHNeAhkRtbeSiouLERERgczMTHh5eWHevHmIiYnBhx9+CFdXV/zt\nb3/TZzuJ9CK3KBczts/Aw8qHOBVyCl3bdW1chZyXQEZI7SJ6EydORNu2beHn54d9+/YhOzsbrVu3\nxjfffAMfHx/9NpKL6JEe/HLhF8zbNQ+v9n8VHzz7ASzMG9kFx72XqYlp+92pNjB4e3vj7NmzAACF\nQoFOnTohKysLlpaWjWupFhgYSEol5SV4c++biLsah00TN8HP2a9xFTJLIAOh89VVW7RoofKzo6Nj\nkwQFIikl30xG4LZADHAcgNR5qWj7VNvGVcgRR2QC1GYMLVq0QJs2bZTPS0tLlYHBzMwMRUVF+mkh\nmDGQ7j2ewfxF4hf4ZtQ3mNJ7SuMqZJZABkjnGYNCoWhUg4gMVU5RDmZsn4FyRbluOpiZJZCJUTtc\ntbS0FF9//TUWLFiAiIgIVFZW6rNdRJKIvhCNvpF94d/FH4dmHGpcUOC8BDJRajOGGTNmoFWrVhg6\ndCh2796N9PR0rFixQp9tI9KZkvISvBH7Bg5eP4iYgBgMchrUuAqZJZAJU9vH0Lt3b/z2228AgMrK\nSvTv3x8pKSl6bdxj7GOgxjj9+2kERQdhkNMgfDv628Z1MLMvgYyIzvsYLCwsav2ZyFg8nsH81fGv\n8M3obxDgFdC4CpklUDOhNmMwNzeHlZWV8jlHJZExySnKwfRfpqOyqhIbJmxAl3ZdtK+MWQIZKZ1n\nDHK5vMluHRE1xrb0bXhl9yv4+4C/4/2h76OFeYv636QOswRqhniPiExGcXkx3oh9A/HX47EjYAcG\nOg3UvjJmCdSMqQ0Md+7cwb///e9a0xAzMzO89dZbkjaMqCFO/34agdsCMdh5MFJCU2DzlI32lTFL\noGauzglu9+/f12dbiBpMUaXAF4lf4N/H/41vR3+Lv3k1YtVfZglEAOoIDB07dsTixYv12RaiBsm+\nl43p26dDUaXA6bmn8YztM9pXxiyBSIk7uJFR2pq+FX0j++L57s/j0IxD2gcFzl4mqkFtxhAXF6fP\ndhBppLi8GK/veR0JWQn4NfBXDHAcoH1lzBKIaqU2Y7Czs9NnO4jqdSr3FHwjfFGFKqSEpmgfFJgl\nENWJw1XJ4CmqFPj82Of4+sTXCBsThpd7vax9ZcwSiOrFwEAGLfteNqb9Mg0ConEdzBxxRKQxdj6T\nwfr5/M/oG9kXI11G4uD0g9oHhZQUoH9/4MyZR1nC9OkMCkR1YMZABud+2X28Hvs6jtw4gl2Bu9Df\nsb92FTFLINIKMwYyKEm5SegT2QdmMENKaIr2QYFZApHWmDGQQVBUKbD82HKsOLkCYaPD8P96/T/t\nKmKWQNRokmYMsbGx8PDwgJubG5YvX6623KlTp2BhYYHo6Ggpm0MG6sa9G3hu/XPYf3U/Toec1j4o\nMEsg0gnJAoNCocCCBQsQGxuL9PR0REVF4cKFC7WW+8c//oFRo0Zxz4Vm6KfzP6FfZD+Mdh2NuGlx\ncLZ1bnglnJdApFOS3UpKSkqCq6srunbtCgAICAhATEwMPD09Vcp9++23mDx5Mk6dOiVVU8gA3S+7\nj7/v+TsSsxMb18HMeQlEOidZxpCbmwtn5z//+nNyckJubm6NMjExMZg/fz6AR8t5k+k7mXMSvhG+\nsDC3QHJosnZBgVkCkWQkyxg0+ZJ/44038Nlnnym3n+OtJNOmqFLgs6Of4Zukb/D9mO8xqeck7Spi\nlkAkKckCg6OjI7Kzs5XPs7Oz4eTkpFLmzJkzCAh4tEF7Xl4e9uzZg5YtW2L8+PE16luyZInyZ39/\nf/j7+0vSbpJGVmEWpv0yDS3MW+B0yGnt+xI44ohIrfj4eMTHxze6HjMh0Z/plZWVcHd3x4EDB9C5\nc2cMGDAAUVFRNfoYHps1axbGjRuHiRMn1myklhtak2H48dyPeG3Pa3jb7228M/gd7fZgrp4lREYy\nSyDSgLbfnZJlDBYWFggLC8PIkSOhUCgQHBwMT09PREREAABCQ0OlOjQZiPtl97FgzwKcyDmBPUF7\n0Ldz34ZXwiyBSO8kyxh0iRmD8TmRcwJB0UF4rutz+M+o/8CqlVXDK2GWQNQoBpcxUPOkqFJg2ZFl\nCDsVpn0HM7MEoibFwEA6k1WYham/TEWrFq1wZu4ZOLV1qv9NT+KII6Imx0X0SCd+PPcj+v/QH+N6\njMP+afsbHhQ4L4HIYDBjoEYpKivCgt0LcDL3pPYdzMwSiAwKMwbS2vHs4/AJ90Fri9ZInpvc8KDA\nLIHIIDFjoAarrKrEsiPL8N2p7xA+NhwTPCc0vBJmCUQGi4GBGuR64XVMjZ6KpyyeQvLcZDi2dWxY\nBRxxRGTweCuJNLb5t83o/0N/vOj+IvZP29/woMD9EoiMAjMGqldRWRFe3f0qTuWewt6pe9GnU5+G\nVcAsgcioMGOgOiVmJ8In3AdtLNrgzNwzDQ8KzBKIjA4zBqpVZVUlPjn8Cb4//b12HczMEoiMFgMD\n1XC98DqCooNgaWGJlNAUdLZp4IghjjgiMmq8lUQqNp3dhP4/9MdEj4nYN21fw4IC5yUQmQRmDAQA\nuPfwHl7d/SrO3DyDfVP3wbeTb8MqYJZAZDKYMdCjDuYIH1i3ssaZuWcaFhSYJRCZHGYMzVhlVSX+\ndfhfCD8djoi/RuBFjxcbVgGzBCKTxMDQTF0ruIag6CBYtbJCcmhyw/sSOOKIyGTxVlIztPHsRgxY\nOQCTPCdh79S9DQsKnJdAZPKYMTQj9x7ewyu7X0HyzWTsn7YfPh19NH8zswSdaN++PQoKCpq6GWRi\nZDIZ8vPzdVYfA0MzcezGMUz9ZSpGu47Gmbln0KZlG83fzL4EnSkoKOD+5aRzZjr+I42BwcRVVlXi\nnwn/RMSZCESOi8R49/Gav5lZAlGzxMBgwq4WXEVQdBBsWtkgJTQFnWw6af5mZglEzRY7n02QEAIb\n0jZg4MqBeLnny4idGqt5UOC8BKJmjxmDiSl8WIhXdr2C1FupDe9gZpZARGDGYFKO3jgKn3AfyFrL\ncHruac2DArMEIqqGgcEEVFZVYtGhRZj802R8O/pbfDf2O81HHXFeAunR5s2b0aVLF1hbW2PChAl1\nDt1NTEzEgAED0LZtW8jlchw7dkz52q5duzB06FDIZDJ06tQJISEhKC4u1sdHUKuiogKTJ09Gt27d\nYG5ujoSEhDrL5+fnY8KECbC2tkbXrl0RFRWl8vqBAwfg4eEBKysrPPfcc7hx44aUzVcljICRNLNJ\nXLl7RQxaOUi8sOEF8XvR75q/saxMiEWLhHBwEGLdOiGqqqRrJCk152v53LlzwsbGRhw5ckQUFxeL\nwMBAERAQUGvZu3fvivbt24utW7eKqqoqsXHjRiGTyURBQYEQQojNmzeLvXv3itLSUlFQUCBGjx4t\n5s2bp8+PU0N5eblYsWKFOHr0qOjUqZNISEios3xAQIAICAgQJSUl4ujRo8LW1lacP39eCCHEnTt3\nhK2trdi6dasoKysT7777rhg0aJDautRdV9peb0ZxlTbnf0zqVFVViXWp64T95/bi6+NfC0WVQvM3\nJycL4e0txNixQuTmStdIqsGQr+VPP/1UuLi4CBsbG9GzZ0/xyy+/qLweGRkpPD09la8nJycLIYS4\nceOGmDBhgnBwcBB2dnZiwYIFtdb//vvvi6CgIOXzzMxM0apVK1FcXFyj7M6dO0XPnj1VftejRw+x\natWqWuuOjo4WvXv31uqzLV68WEydOlX5/Nq1a8LMzEwoFI/+Td29e1fMnDlTdO7cWchkMvHSSy+p\nPc5jTk5OdQaG4uJi0apVK5GRkaH83fTp08V7770nhBAiIiJCDBkyRPlaSUmJsLS0FJcuXaq1Pl0H\nBt5KMkKFDwsxZdsULD+2HHHT4vDGoDdgbqbB/0r2JVAdXF1dcfToURQVFWHx4sWYOnUqbt++DQD4\n+eefsXTpUmzYsAFFRUXYsWMH7OzsoFAo8Ne//hXdunVDVlYWcnNzERAQAAC4ceMGZDIZcnJyAADp\n6emQy+XK43Xv3h1PPfUULl++rFH7qqqqcP78+VpfS0hIgJeXl1afrb7JYdOmTcPDhw+Rnp6OP/74\nA2+++abyNZlMhsTERI3aX93ly5dhYWEBV1dX5e/kcrny850/f17lXLVp0waurq44d+5cg4+lDQYG\nI3Mk6wh8wn1g38Yep0NOQ95RXv+bAPYlGAkzM908tDF58mR07NgRAPDyyy/Dzc0NSUlJAICVK1fi\nH//4B/r27QsAcHFxwTPPPIOkpCTcvHkTX3zxBSwtLfHUU09hyJAhAIBnnnkGBQUFcHJyAgAUFxfD\n1tZW5Zht27bF/fv3a7TFz88PN2/exJYtW1BRUYF169bh6tWrePDgQY2y+/fvx/r16/Hxxx9r9dlE\nHTPRb968idjYWISHh8PW1hYWFhYYNmyY8vWCggIMHjxY7fvVKS4uRtu2bVV+Z2NjozwXtb3etm1b\nvfWjMDAYiQpFBT46+BFe3voywsaEIWxMGCxbWtb/RmYJRkUI3Ty0sX79evj6+kImk0Emk+HcuXPI\ny8sDAOTk5MDFxaXGe7Kzs9GlSxeYm9f/VWJtbY179+6p/O7evXuwsbGpUdbOzg7bt2/HV199hY4d\nO2Lv3r34n//5H2WQeezEiRMICgrCtm3bVP76bshnq0t2djbat29fI6A1lrW1NYqKilR+d+/ePWUw\nsLGxqfX12s6VFDiPwQhk5mciKDoIMksZUkJT0NG6o2Zv5LwE0lBWVhbmzp2LgwcPws/PD2ZmZvD1\n9VX+Ne3s7IwrV67UeJ+zszNu3LgBhUKBFi1a1HmMXr16IS0tTfk8MzMT5eXl6NGjR63lhw0bpvyr\nvrKyEi4uLnjnnXeUr6ekpODFF1/E2rVrMXz4cK0/m7W1tUomcuvWLZXPl5+fj3v37uk0OPTo0QOV\nlZW4cuWKMqClpaWhV69eAB6dq3Xr1inLl5SUIDMzU/m65LTqmdAzI2mmzlVVVYm1KWuF/ef24j/H\n/6N5BzNHHBksQ72Wz58/L1q3bi0uXbokKisrxerVq4WFhYWys/fnn38Wzs7O4syZM6KqqkpkZGSI\nrKwsoVAohFwuF++8844oKSkRpaWl4tixY2qP0bZtW+WopClTpogpU6aobVNycrIoLy8X9+7dE6+/\n/roYOnSo8rXffvtNPP300+Knn35q9Gfbv3+/sLe3Fzdu3BCFhYVi/PjxKp3PY8eOFYGBgaKgoECU\nl5fX2amMrqTbAAATVklEQVT88OFDUVpaKpycnMS+fftEaWmp2rIBAQFiypQpoqSkRBw5ckTY2tqK\n9PR0IcSfo5K2bdsmSktLxbvvviv8/PzU1qXuutL2ejPMq/QJhvqPSUr5D/LF337+m+j1XS+RditN\n8zdyxJFBM+Rr+cMPPxTt27cX9vb24q233hL+/v4qo4DCw8OFu7u7sLa2Fr179xapqalCiEejkl56\n6SVhZ2cn7O3txeuvvy6EECIrK0tYW1uL7OxsZR2bN28WzzzzjLCyshIvvfSScvipEELMmzdPZcjp\nlClThK2trbC1tRUBAQHizp07ytdmzZolWrRoIaytrZUPLy8vrT/bq6++Ktq1ayfc3NzEDz/8IMzN\nzZWBIT8/X8yYMUN06NBByGQyMWnSJOX7rK2txdGjR5XPu3TpIszMzIS5ubnyv1lZWUIIIT755BMx\nevRoZdn8/Hzx0ksvCSsrK9GlSxcRFRWl0ua4uDjh4eEhLC0txfDhw5X11EbXgcHsv282aGZmZs1q\nqeLDWYcx7ZdpGN9jPD5//nPN+xK4EqrBa27XMumHuutK2+uNfQwGpEJRgSXxS7A6dTVWjluJsT3G\navZG9iUQkQ5JPiopNjYWHh4ecHNzw/Lly2u8vmnTJsjlcnh7e2PIkCE4e/as1E0ySFfyr2DomqFI\nuZWC1NBUzYICRxwRkRS0ugGlocrKSuHi4iKuXbsmysvLhVwuV3auPJaYmCgKCwuFEELs2bNHDBw4\nsEY9EjezSVVVVYk1KWuE/ef2YsWJFaJK045i9iUYJVO+lqnpqLuutL3eJL2VlJSUBFdXV3Tt2hUA\nEBAQgJiYGHh6eirL+Pn5KX8eOHCgcpZkc1BQWoDQX0ORficdB6cfRO8Ovet/E/sSiEhikt5Kys3N\nhbOzs/K5k5MTcnNz1ZZftWoVxowZI2WTDEbC9QT4RPigo3VHnAo5pVlQ4OxlItIDSTOGhmxQfejQ\nIaxevVplad3qlixZovzZ398f/v7+jWxd06jewbxq/CqMcdMgEDJLICINxMfHIz4+vtH1SBoYHB0d\nkZ2drXyenZ1dY0o7AJw9exYhISGIjY2FTCarta7qgcFYZdzNQFB0EOzb2CM1NBUdrDvU/yaOOCIi\nDT35R/PSpUu1qkfSW0n9+vVDRkYGrl+/jvLycmzZsgXjx49XKXPjxg1MnDgRGzdurHOtE2MmhMDq\nlNXwW+WHad7TsCtwV/1BgSOOiKiJSJoxWFhYICwsDCNHjoRCoUBwcDA8PT0REREBAAgNDcXHH3+M\ngoICzJ8/HwDQsmVL5foopqCgtABzf52Li3kXET8zHl5Pq18aWIlZAhE1pUaMkNIbI2lmDYeuHRLO\n/3YWr+1+TZRWqF8zRYlrHJk8Y72WdWXTpk0qS2Lk5+erLXvs2DHRv39/YWNjI7y9vVWWnvj111/F\nkCFDRLt27UTHjh3FnDlzxP379/XxEeoUFxcn3N3dRZs2bepdxuIvf/mLaN26tXJJDw8PD5XXS0pK\nxPz584W9vb2wtbUVw4YNU1uXuutK2+vNKK5SY/vHVFZZJt7b/57o9GUnsfvybs3exHkJzYKxXcu6\nZOpbezZ0O84n12t6UlBQkJgyZYrIy8sTVVVVyh3zasPAYOAu510W/SL7iTGbxojbxbfrfwOzhGbF\nkK9lbu35iLZbezZ0O05/f3+xcuXKWl+7cOGCaNu2rcZZkK4DAzfq0RHx3w7mwasHY6Z8Jn6d8iue\ntnq67jdxXgIZEG7tWTtNt/bUZjvO999/Hw4ODhg6dCgSEhKUv09KSkKXLl2waNEiODg4wNvbG9HR\n0XW2U5e4iJ4O5JfmY+7Oubh89zLiZ8Sj19P1bKbBeQmkhtlS3VwHYnHDV9ScPHmy8ueXX34Zn376\nKZKSkjBu3Lhat/YEgOPHjyu39ny8i9uTW3s+pu3WnhMnTsTmzZvr3dqzrkErdX02ocHWnvn5+cq2\nP7m152MlJSVwcHCo8fnUbce5fPly9OrVC61atUJUVBTGjRuH1NRUdO/eHTk5OTh37hwmT56Mmzdv\nIjExEWPHjkXPnj3h4eGhtr26wsDQSIeuHcL07dMxyXMSNk7ciNYWret+A0ccUR20+ULXlfXr1+Pr\nr7/G9evXATz6Im/qrT3feecdvPLKKxg5cmSjt/ZU99nq0pCtPdVt16luO84BAwYof54+fTqioqKw\ne/duLFiwAJaWlmjZsiUWLlwIc3NzDBs2DMOHD8e+ffsYGAxZuaIciw4twvq09Vj94mqMch1VzxuY\nJZDh4taejd/aU5fbcXp7ewNAjWymIatJNIpWPRN6ZmjNvJR3SfSN6Cv+uvmvmnUwc8QR/ZehXcuP\ncWvPxm/t2ZDtOAsLC0VsbKwoLS0VFRUVYuPGjcLKykpkZGQIIYSoqKgQrq6u4p///KeoqKgQR48e\nFTY2Nmo7stVdV9peb4Z5lT7BUP4xVVVViR/O/CDsltuJsJNh9S+RzRFH9ARDuZZrw609G7+1Z13b\ncVbf2vOPP/5QztFo166d8PPzE3FxcSptPn/+vPDz8xNWVlaiV69eYvv27Wo/n64DA7f21FB+aT5C\ndobgSv4VbJ64uf4O5up9CZGR7EsgAIZxLZPp0fXWnhyuqoGD1w5CHi5HF9suODnnZN1BgWscEZGR\nY+dzHcoV5fjo4EfY+NtGrB6/GiNdR9b9Bo44IiITwMCgxqW8SwiMDkRnm85IDU2Fg5WD+sIccURE\nJoS3kp4ghMAPZ37AkNVDMMd3DnYE7Kg7KHD2MhGZGGYM1dx9cBchO0NwteAqDs86jJ4OPdUXZpZA\nRCaKGcN/Hbh6APJwObq264qTc07WHRSYJRCRCWv2GUO5ohwLDy7Ept82Yc2La/CCywt1FGaWQESm\nr1kHhot5FxG4LRDOts71dzBzxBERNRPN8laSEAKRZyLx7JpnMbfvXGz/23b1QYHzEoiomWl2gSHv\nQR4m/jQR35/6HodnHsa8fvPUL0zFvgQindq8eTO6dOkCa2trTJgwQWXZ6iclJiZiwIABaNu2LeRy\nOY4dO6Z8bdeuXRg6dChkMhk6deqEkJAQtctb69OBAwfg4eEBKysrPPfcc7hx44basv7+/rC0tISN\njQ1sbGzg6empdV06p9VCGnqmq2buz9wvHL9yFG/vfVs8rHioviDXOCKJGMk/OUlwa09VdW3t2dC6\n1F1X2l5vRnGVNvYf08OKh+Kdve8Ix68cxb4r++ouzJVQSUKGHBi4tecjhrC1Z0Pr0nVgMPlbSRfz\nLsJvlR8u519G6rxUPO/yfO0F2ZdAzRy39qxdU2ztqU1dOqVVONEzbZpZVVUlwk+FC7vldiL8VHjd\nS2QzSyA9qfdaBnTz0AEfHx+xY8cOIYQQL7zwgvjmm29qlElMTBQODg7Kv67rMmLECBEREaHyO0dH\nx1r3N8jLyxMymUz8+OOPory8XKxdu1aYm5vXerto3759QiaTKfcy0ET1z1ZXxvD7778Lc3NzUVhY\nWG+dwcHB4r333lP53ZAhQ8S6detqLX/y5ElRXFwsysvLxbp164SNjY24evWqVnWpu660/Yo3yYwh\n70EeJmyZgPAz4Tgy6whC+4XW/lcBswQyNLoKDVpYv349fH19IZPJIJPJcO7cuSbf2vOrr75Cx44d\nsXfv3kZv7anus9VF6q09rays0LJlS0yfPh1DhgzBrl27tKpL10wuMOzP3A+fcB+4tXfDieAT8HTw\nrL0gRxwRKT3e/vK7775Dfn4+CgoK4OXl1aCtPeuj7daed+/exfr163Hx4kWVfZIburWnus+m6dae\nDf18jdnaU5d1aUWrPEPPNGnmw4qH4q3Yt4TjV45if+Z+9QU54oiakKH+k+PWnoa1tWdD6hKCo5Jq\nlf5HuvAJ9xEvRr0o7pTcUV+QfQnUxAw1MAjBrT0NbWvPuup6kq4Dg1Fv7SmEQMSZCHx06CN88twn\nCOkTor4vgWsckQHg1p4kBV1v7Wm0ayXdKbmDOTvnIPteNo7MOgIPe4/aC3KNIyKiBjHKzud9mfvg\nE+EDdzt3HA8+XntQ4IgjIiKtGFXGUFZZhvcPvI+fzv+E9S+tx4juI2ovyCyBiEhrRhMY0u+kI3Bb\nILrLuiNtXhrs2tjVLMS+BCKiRjOawDBszTB8OuJTzOkzp/YOZmYJREQ6YTSjki7euQh3e/eaLzJL\nICPCUUkkhWY7KqnWoMAsgYyMTCard9E2ooaSyWQ6rU/SUUmxsbHw8PCAm5sbli9fXmuZ1157DW5u\nbpDL5UhJSdGsYo44IiOVn58P8WhiKR986OyRn5+v0+tUssCgUCiwYMECxMbGIj09HVFRUbhw4YJK\nmd27d+PKlSvIyMhAZGQk5s+fX3/FzXyNo/j4+KZugsHgufgTz8WfeC4aT7LAkJSUBFdXV3Tt2hUt\nW7ZEQEAAYmJiVMrs2LEDM2bMAAAMHDgQhYWFyjXSa2CWAIAXfXU8F3/iufgTz0XjSdbHkJubC2dn\nZ+VzJycnnDx5st4yOTk56NChQ80K+/dnXwIRkR5IFhg07WATQrXHXO373n6bI46IiPRBSOT48eNi\n5MiRyufLli0Tn332mUqZ0NBQERUVpXzu7u4ubt26VaMuFxcXAYAPPvjgg48GPFxcXLT6/pYsY+jX\nrx8yMjJw/fp1dO7cGVu2bEFUVJRKmfHjxyMsLAwBAQE4ceIE2rVrV+ttpNo2CCEiImlIFhgsLCwQ\nFhaGkSNHQqFQIDg4GJ6enoiIiAAAhIaGYsyYMdi9ezdcXV1hZWWFNWvWSNUcIiLSkFHMfCYiIv0x\nqGW3JZsQZ4TqOxebNm2CXC6Ht7c3hgwZgrNnzzZBK/VDk+sCAE6dOgULCwtER0frsXX6o8l5iI+P\nh6+vL7y8vODv76/fBupRfeciLy8Po0aNgo+PD7y8vLB27Vr9N1JPZs+ejQ4dOqB3795qyzT4e1Or\nngkJVFZWChcXF3Ht2jVRXl4u5HK5SE9PVymza9cu5dZ4J06cEAMHDmyKpkpOk3ORmJgoCgsLhRBC\n7Nmzp1mfi8flhg8fLsaOHSu2bt3aBC2VlibnoaCgQPTs2VO5lWb1rTBNiSbnYvHixeK9994TQjw6\nD+3btxcVFRVN0VzJHT58WCQnJ6vd2lSb702DyRh0PiHOiGlyLvz8/GBrawvg0bnIyclpiqZKTpNz\nAQDffvstJk+eDAcHhyZopfQ0OQ+bN2/GpEmT4OTkBACwt7dviqZKTpNz0alTJxQVFQEAioqKYGdn\nBwsLo1karkGeffbZOtdK0uZ702ACQ22T3XJzc+stY4pfiJqci+pWrVqFMWPG6KNpeqfpdRETE6Nc\nUsUUF6nT5DxkZGQgPz8fw4cPR79+/bBhwwZ9N1MvNDkXISEhOH/+PDp37gy5XI4VK1bou5kGQ5vv\nTYMJoTqfEGfEGvKZDh06hNWrV+PYsWMStqjpaHIu3njjDXz22WfKJYafvEZMgSbnoaKiAsnJyThw\n4AAePHgAPz8/DBo0CG5ubnpoof5oci6WLVsGHx8fxMfHIzMzE88//zzS0tJgY2OjhxYanoZ+bxpM\nYHB0dER2drbyeXZ2tjIlVlcmJycHjo6OemujvmhyLgDg7NmzCAkJQWxsrM6X3TUUmpyLM2fOICAg\nAMCjTsc9e/agZcuWGD9+vF7bKiVNzoOzszPs7e1haWkJS0tLDBs2DGlpaSYXGDQ5F4mJifjwww8B\nAC4uLujWrRsuXbqEfv366bWthkCr702d9YA0UkVFhejevbu4du2aKCsrq7fz+fjx4ybb4arJucjK\nyhIuLi7i+PHjTdRK/dDkXFQ3c+ZMsW3bNj22UD80OQ8XLlwQI0aMEJWVlaKkpER4eXmJ8+fPN1GL\npaPJuXjzzTfFkiVLhBBC3Lp1Szg6Ooq7d+82RXP14tq1axp1Pmv6vWkwGQMnxP1Jk3Px8ccfo6Cg\nQHlfvWXLlkhKSmrKZktCk3PRHGhyHjw8PDBq1Ch4e3vD3NwcISEh6NmzZxO3XPc0ORcffPABZs2a\nBblcjqqqKnz++edo3759E7dcGlOmTEFCQgLy8vLg7OyMpUuXoqKiAoD235uc4EZERCoMZlQSEREZ\nBgYGIiJSwcBAREQqGBiIiEgFAwMREalgYCAiIhUMDER1aNGiBXx9fZWPrKwsxMfHw9bWFr6+vujZ\nsyc+/vhjAKjx+4ULFzZx64m0YzAT3IgMUZs2bWqsX3/t2jUMGzYMO3fuxIMHD+Dj44Nx48bBzMxM\n+fuHDx/C19cXEyZMQN++fZuo9UTaYcZA1Aht2rRB3759a+xL3rp1a/j4+ODq1atN1DIi7TEwENWh\ntLRUeRtp0qRJNV6/e/cuTpw4AS8vL5UVLPPz85GUlGSSS1KQ6eOtJKI6WFpa1roV4pEjR9CnTx+Y\nm5vj/fffh6enJ27fvo0jR47Ax8cHGRkZmDdvHnr16tUErSZqHAYGIi08++yz2Llzp9rfX79+HcOH\nD8cbb7yhskkKkTHgrSQiCXTt2hWvv/46/vnPfzZ1U4gajBkDUR1q2+nKzMxMo9/PmzcPPXr0QE5O\nTq0bLREZKi67TUREKngriYiIVDAwEBGRCgYGIiJSwcBAREQqGBiIiEgFAwMREalgYCAiIhUMDERE\npOL/A2PPkuYDKRsfAAAAAElFTkSuQmCC\n",
       "text": [
        "<matplotlib.figure.Figure at 0x7f814677f510>"
       ]
      }
     ],
     "prompt_number": 5
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "from sklearn.metrics.scorer import SCORERS"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 6
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "SCORERS.keys()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 7,
       "text": [
        "['adjusted_rand_score',\n",
        " 'f1',\n",
        " 'mean_absolute_error',\n",
        " 'r2',\n",
        " 'recall',\n",
        " 'median_absolute_error',\n",
        " 'precision',\n",
        " 'log_loss',\n",
        " 'mean_squared_error',\n",
        " 'roc_auc',\n",
        " 'average_precision',\n",
        " 'accuracy']"
       ]
      }
     ],
     "prompt_number": 7
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "from sklearn.metrics.scorer import accuracy_scorer\n",
      "\n",
      "def my_super_scoring(est, X, y):\n",
      "\treturn accuracy_scorer(est, X, y) - np.mean(est.coef_ != 0)\n"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 8
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "from sklearn.svm import LinearSVC\n",
      "y = digits.target\n",
      "grid = GridSearchCV(LinearSVC(C=.01, dual=False), param_grid={'penalty' : ['l1', 'l2']}, scoring=my_super_scoring)\n",
      "grid.fit(X, y)\n",
      "print(grid.best_params_)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "{'penalty': 'l1'}\n"
       ]
      }
     ],
     "prompt_number": 9
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "from sklearn.metrics.scorer import make_scorer\n",
      "\n",
      "def scoring_function(y_true, y_pred):\n",
      "\treturn (np.abs(y_true - y_pred) < 3).mean()\n",
      "    \n",
      "tolerant_scoring = make_scorer(scoring_function)\n",
      "print(cross_val_score(LinearSVC(random_state=0), X, y))\n",
      "print(cross_val_score(LinearSVC(random_state=0), X, y, scoring=tolerant_scoring))"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "[ 0.88  0.94  0.88]\n",
        "[ 0.92  0.96  0.91]"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n"
       ]
      }
     ],
     "prompt_number": 10
    }
   ],
   "metadata": {}
  }
 ]
}