{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<p>Goal - show examples of cross-validation for model selection and bootstrapping for prediction error estimation.\n",
    "</p>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from sklearn.metrics import confusion_matrix, roc_auc_score\n",
    "from sklearn import linear_model\n",
    "import course_utils as bd\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')\n",
    "%matplotlib inline\n",
    "reload(bd)\n",
    "\n",
    "#Load data and downsample for a 50/50 split, then split into a train/test\n",
    "f = '/Users/briand/Desktop/ds course/datasets/ads_dataset_cut.txt'\n",
    "\n",
    "train_split = 0.8\n",
    "tdat = pd.read_csv(f, header = 0,sep = '\\t')\n",
    "moddat = bd.downSample(tdat, 'y_buy', 4)\n",
    "\n",
    "#We know the dataset is sorted so we can just split by index\n",
    "train = moddat[:int(np.floor(moddat.shape[0]*train_split))]\n",
    "test = moddat[int(np.floor(moddat.shape[0]*train_split)):]\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<p>Let's say we want to build a classifier that does ranking based on the probability of the label being 1. We can use Logistic Regression with AUC as our validation metric. We need to choose a regularization weight, but because we have limited data, we can do this via cross-validation.<br><br>\n",
    "\n",
    "The following function performs the cross-validation for various levels of C\n",
    "\n",
    "</p>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from sklearn.cross_validation import *\n",
    "\n",
    "def xValAUC(tr, lab, k, cs):\n",
    "    '''\n",
    "    Perform k-fold cross validation on logistic regression, varies C,\n",
    "    returns a dictionary where key=c,value=[auc-c1, auc-c2, ...auc-ck].\n",
    "    '''\n",
    "    cv = KFold(n = tr.shape[0], n_folds = k)\n",
    "    aucs = {}\n",
    "\n",
    "    for train_index, test_index in cv:\n",
    "        tr_f = tr.iloc[train_index]\n",
    "        va_f = tr.iloc[test_index]\n",
    "    \n",
    "        for c in cs:\n",
    "            logreg = linear_model.LogisticRegression(C = c)\n",
    "            logreg.fit(tr_f.drop(lab, 1),tr_f[lab])\n",
    "            met = roc_auc_score(va_f[lab], logreg.predict_proba(va_f.drop(lab,1))[:,1])\n",
    "\n",
    "            if (aucs.has_key(c)):\n",
    "                aucs[c].append(met)\n",
    "            else:\n",
    "                aucs[c] = [met]\n",
    "    \n",
    "    return aucs\n",
    "\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<p>Now we can run our experiment</p>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "xval_dict = {'e':[], 'mu':[], 'sig':[]}\n",
    "k = 10\n",
    "auc_cv = xValAUC(train, 'y_buy', k, [10**i for i in range(-20,20)])\n",
    "for i in range(-20,20):\n",
    "    xval_dict['e'].append(i)\n",
    "    xval_dict['mu'].append(np.array(auc_cv[10**i]).mean())\n",
    "    xval_dict['sig'].append(np.sqrt(np.array(auc_cv[10**i]).var()))\n",
    "    \n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<p>We can now load the results from above into a dataframe and begin to analyze\n",
    "\n",
    "</p>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "res = pd.DataFrame(xval_dict)\n",
    "\n",
    "#Get the confidence intervals\n",
    "res['low'] = res['mu'] - 1.96*res['sig']/np.sqrt(10)\n",
    "res['up'] = res['mu'] + 1.96*res['sig']/np.sqrt(10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<p>Now let's plot the results to get a sense of how AUC varies with regularization strength. We'll also plot the lower and upper 95% confidence bands to get a sense of the variance.\n",
    "</p>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.text.Text at 0x109e07550>"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYoAAAEZCAYAAACJjGL9AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3XmcHEX9//HX7CYhN0cSrhDYcEPCLeEUli+HCaAgN3iE\nr4gRRfgJfsGgSOQGbwEVFeUyRFBAbkiQyKGAgQABEwiQ5UjCFSAEiUKS+v3xqWZ6e3tmezYz0z3T\n7+fj0Y+Z7qmerp5s6tNd1VUFIiIiIiIiIiIiIiIiIiIiIiIiIiIiIpIxVwLn+PefBOYkTFtrxwIP\n1OlYSa0ANkw7E9I4WtLOgDS1gcA84JjQtkHAy8AhVT6W8wtYwbx5wrTdmQ4c1/NsJTIQeB+4I+az\nuEJ9EnBNaH0w8FPgJWAJ8DzwE2BItTMaYx3gCmAB8B4w2+evfx2OLXWiQCG19D4wASvEhvptFwOP\nAjfW4HiFGqRNGlBWxqFY8GwH1kqQPpynPsC9wBbAp7BAvAvwFjCmqrnsag3gH8AqwM5YwNoXWBXY\nqMbHFpEm83tgMlYQvgWsWSLdbOCA0Hov4E1gW79+A7AQeBf4G7Bl5BhBdVI78Eros+2Ax7Er3inA\ndaG0qwO3AW8AbwO3AsP9Z+cBy4Cl2JX6z/32zYGpwCKsiuvw0LGGALcAi4FH/HG6q3r6K3CK/85T\nI591d0fxZeA1KruCXwF8A3gB+30vxgJnH+w3GB1Kuybwb+LvTs4FnqzguCIiJa2GFfBvAuPLpDsT\nuDa0fgDwTGj9WGAA0BurWpkZ+qxUoOiDVcmcDLRiV+8fAmf7z9cAPgv0xaqArgduCn3vfcCXQusD\n/HePx+7It/XntYX/fIpf+gGjgFeB+8uc8wZYMFoPOJ6uBW+pQHF16Hi/L/P9cVZgdyGrASOAZylW\nr10GXBhKezLwlxLf8zBwVoXHFhEpaRpWFTW4TJqNsKv+vn79D8B3S6RdDSvwBvn1UoFiD2B+ZN+H\nKAaKqG2xq+rAfXRuoziSrgX/5cD3sED0IbBp6LPzKH9H8V3g7/79EOAjindQ0H2guAc4v8z3x1kB\n7BdaPwH79wHYCQusgRnAYSW+5zngKxUeWxqQ2iikHj6PXTlPAy4Kbb8Tq9JZAhyNVYXMBj6DVaV8\nGquyAiuEL8QaahdjjeRQbPsoZV26BoqXKLZR9McK+g7/vX/D6tjDbRjhNoENsML0ndByDNa2MBSr\nLgtXe73cTf6+iFWpgVVlTafzXddy7A4qrDcWUIJ91u3mGHGieQy+4xGsqq0dq2LbCKtKi9PTY0uD\nUaCQWlsT+DFWl/5V4Ahgd//ZOOyOYBDWboB/PRo4CPgX8KLffgwWQPbGCvKRfnupAj2wkGKbQ2CD\nUNpTsTuAMf579/TfGXxv9DtfxoLJ6qFlEPB1rP1lGbB+KP36lLYrsDF2V7HQL7v4cw3+b75M8VwD\nIyle9U/DGrErfcoomsdwML0KC+5fwILYhyW+YxpWbVfJQwQiIl1cj12xB47D7hr6lEi/NvABVhh/\nI7T9BKxNYhDWTvALOlfLXEn5NoqTsCvxQ+jcRnER9ljqKlh7xU3+e4OC+jqs+igwELv7+Lz/vt7A\njhQfxw0ay/thje3l2iguB+7CgmmwtGHVbwf6NOcDD2LBrgXYx38eNOT3wZ4iuxPYzKcZApyBBeI4\nK7CG86CNYjYWyAMjsOq3DopBPc7q2J3d1RQDz3DgR8BWZfYTEfnYwVhBGW2XuJfyHd6mYYV5+Omo\nAcDNWCE5D7vaXU4xUPyeYuHfTucqnx3o+tRTkHYdrB1iCfYE01f89waBYmessfdt7DFfsDuQ4Emp\nt3x+t/afDcWenFqMNfaeTXyg6Ou/84CYzy7DAmyQ7mJ/zu9ibQYHRtIPxhr3X6bYj+KHWEEeZwVw\nIlbV9xbwA7rWLkyjeDdXTtCPYiHFfhRnYoFSJJGx2H++ucDpMZ+vjl3BPYnVjY4KfdYBPIVdRT5a\n01yKSNQVlG7wF6maVuzKpg27PX+C4iOEgR9gVx9gt83TQp/Nw6oCRKS+2rBG+g1SzodkRC0bs8dg\ngaIDe0JjCtZAGbYFdtsPdnvfBgwLfa5GMpH6OgeYhVV3vdRNWsmJWgaK4XR+BO9Vuj598iTFMX/G\nYFcw6/l1h91hzMA6IolI7Z2JPTBwQdoZkezoVcPvTjJGzoXAz7B2iFn+dbn/bHdsoLFh2BMac8je\nKJwiIk2vloFiPvaYXWAEdlcRtoTOwyPMo/ikxQL/+ibW4D2GroHieTT4mIhIpV7A+vCkrheWmTbs\nWe+4xuxVKT5Pfzz2LDxY56FgaIYB2JAL+9FVPUb2rIZJaWcgoUlpZyChSWlnIKFJaWcggUlpZyCh\nSWlnIKFJaWcgoYrKzlreUSzDntW+G3sC6grsGesJ/vPLsU5DV2KZfprimDprURyYrRc25s89Ncyr\niIiUUMtAAdZb9M7ItnAv3X9gj8VGzaPzwGgiIpISjfVUH9PTzkBC09POQELT085AQtPTzkAC09PO\nQELT085AQtPTzoB01ShtFCIiWVJR2ak7ChERKUuBQkREylKgEBGRshQoRESkLAUKEREpS4FCRETK\nUqAQEZGyFChERKQsBQoRESlLgUJERMpSoBARkbIUKEREpCwFCpHkBgG9Y7a3YJNyFeqbHRFJQqPH\nSj0MAb6PTcu7Z8zn6wIdwELgOuAr2DwrtZ7vpVH0xmaq7IfNaNkLC65pBdaCz8+6wObYNMv7ADuk\nlJ+wAvX5XTIzwx3AWOCn2Ax3vwUuiny+OvA7YEPgP9j82c8k3Fek1tYBTsX+Lm8EdgXmxqRbgE35\nOxLYyy9nYHO67xOTfg1svuKPsP8DwfI6cGtM+k8C52L/F1qx/7et2NwHp8SkHw0cDfzbH6PFp38W\n+HNM+h2Bk4HB2F3TIP/+r8BXY9LvBpwALAbew2azXB14Cvh1TPojsQnLWn1eguUaYHxM+sOBi4EP\nQ8sy4Hbg7Jj0Y4FvU/x9guV24KyY9McCl/q8B8tibFbNx2LSjwOOwf59VmBTNffDfv9rYtIf5I/x\nrv/e4PUR4O8x6T8PnAasiV2UtAJLgQuBc2LSH479psuwf9+P/PvbgFti0q+0Wkau4A9zH2A+8E/s\nj3d2KM0PsH+kc7ArsMt8+iT7gkVF3e5LLWwH3IsVBD8EXunBdxSIv3LbFbgEu9J+G3jHLw8TX9AO\nA0YBy7ECYblf3gFejEm/GXAYMNAfYzlWwM0CJsekb8OC0RLs/+MSv7zll6gNgD2wOe9XxQLXO1ig\nmB6TvpRSv89A7Jz7hJbePi/Px6RfG6v6Wx5Z3sTu9JIet5SNgN39cRxWiH+A/Z6PxqQfCWyP/Tar\nhV4fAq6PST8cGOrz+5bPe19/rA9i0m+OXQz09ksv//oEFoySyEzZuQtwV2j9234Juw37Bwg8j0XV\nJPuCqp5k5fUpsb0V+1sUaUaZmbhoOJ2vwl7128KeBA7x78dgVyrrJdxXpCcGAQcAFwAPYFf0q8ek\nWw68Ucd8iWRWLdsokkSsC4GfATOx27iZ2H/QSqLdpND76WjOWintKuzC5J/A/VgD9SNYNYtIM2v3\nS+bsTOfqo4nA6d3sMw+rn0y6r6qeJE6pO+VR2NMuInmXmbKzF/AC1lDWB2to2SKSZlWKdcTHA1dW\nsC9k6GQlMyZgTxyJSGmZKjvHYU8vPY/dFYD9R57g3+/iP58D/AkLHOX2jcrUyUrqBmB9GbZNOyMi\nGZersjNXJyvdOo34xw9FpLNclZ25OlkpazD2lNKWaWdEpAHkquzM1clKWWcS30tWRLrKTD8KkXpy\n2OOuIiKd6I5CRKRyuqMQEZHqUaAQEZGyFChERKQsBQppZBunnQERyT41ZufX+tjIr2uknRGRBpSp\nGe5EauU72Kxpb9f3sK4A7I1NPvQk8DjwKhR00SJNKxMzHK2EzMzSJHW1ITaz2GbAovoc0rUAB2MT\naA3CJt0ahc2zXMCm0Hzcvz4GvKzgIRlWUdnZ6IWsAkU+XYlNcTmp9odyvYHPYcPcv49NeHQzFFb4\nzwvAuljA2AGbAnMHYBXgJWw+7fn+Nfp+KTblZbmlj/+u8Gv4/XJszuRgbumPYl4/KrH+kd9/IJ3n\nyh4UWfD7/jf0Gn7vQvntF/O6CjYVazC387LI+2B6V1dmga5zYkcXR3Ha1xWh98Frq89PMOd1eOlP\ncUrXYArYRaH3wfpyilOPhl+D9y2RY0bf4481KPS7R19bIr9R9DU8HW74HMPLFCi8R2kKFNLUNgMe\nxBqyF9fuMK4/8GXgW8Bz2CRb9ya/S3BrASOwIDLcv4bfD8cK+v+UWaIFcvg1WFoozindJ/I+/Bp9\nHyytWAAM5sl+L/L+fez/WThYRQNXi8/v0hKv//VpwgVq+H1QwBbKLNC1MIwuBf89Lf68wq9BAf6B\nz1d4CbYtw2Y7HBpahkTWWyhfgK8IHS+al2D5t/993w+9ht8vj/xGcb9ZK53PMbpMhELcfOeBXJWd\nurXPn/7YxFY14AaC2w/cReBeB3cjuDG1OZZIqioqOxs9ouQqKkq1uYFYo3S7X7bG2hf+ht26/yu1\nrInUlqqeROK5FmAM8GlgL4qBYbpfHobC0rRyJ1JHmSo7x2Kz180lfs7rodjc2E8ATwPHhj7rAJ4C\nZmJPuMRR1ZN0w/Xx1Um/BLcA3NPgzgO3F7h+aedOJCWZKTtbsWlM27DGl7h5rydhT5GABY1FFPt2\nzKP7zlSZOVnJEjcQ3GHg/gDubXB/B3cauE3SzplIRmRm9NgxWKDowJ4KmAIcFEmzEHscD/+6CHty\nIJCZWyPJhAR/D+4w7NHTLwP3A6OgsCsULobC3NpmT0QqdRjwm9D654FLImlasLrhBdijYeNCn72I\nVTvNAI4vcQzdUeTL/cB2pT92reCeA7d33XIk0pgyM4RHkoycgVVJtQMbAVOBbbCgsRt2xzHMb58D\nPBDzHZNC76f7RZpPK9aR7YUyaQ7D7kr/WpcciTSOdr9kzs5YQ3VgIl0btO/AAkLgXuATMd91FnBq\nzHbdUeTHpthdZgmuAO5JcAfULUcijSszbRQzgE2wxuw+wJHALZE0c4B9/Pu1sF63L1Ls4g4wANgP\nmFXDvEr2bY09BVfKgViv2Dvqkx0RqZZxwLNYo/ZEv22CX8CedLoVG4VzFnCM374hViUVPDYb7Bul\nO4r8OAc4O/4jVwD3MLjD65ojkcaVq7IzVyebc1OwNogYbm9wc6wxW0QSyFXZmauTzbkC1qAdw/0V\n3Pi65kakseWq7MzVyUoctwu4eX44cBFJJldlZ65OVuK428CdkHYuRBpMrsrOXJ2sRLltwc0H1zft\nnIg0mFyVnbk6WYly14M7Je1ciDSgXJWduTrZHFufLn1+3Bbg3gA3IJUcWS/x24CfAF/D+vqMpGSD\nu0imaOIiaSot2JSn62PzGXvuKmAuFM5NJ1usAXwS61S6cej1USCuP8c62DA1rwOvUZxiNKkCNgd1\nAZu6M2oI9htFLQJeruA4payF5T+YzjOYdnQh1nE2ajjWmx7sPIM5u18DXolJ3wfrXBue/rMVmzb0\nzZj0I7Fxv1YJLR9h/a7ipiVYzZ9DdL7rN4gfFqZf6Pi1uCAdgk2J20rxXFux3yduBII2YHOK5V0w\nl3gH1letUhWVnbUc60mkGkZiASIcJEZiPbE3qvGxC9gER09jBUrY28BfYvYpNdrBKOD7wNp+AQsW\nU4CTY9IfDvwYK7D6Y0HiQ+AK4Osx6T+JDXUTdUuJ7UcAZ2JzRQ/ARkIYDEzG7pCidgX+D+v97vzr\nCqzDbFyg2AH4pn/fQnHO7puBuOD+BeCHFOeeXu5fr/H5jNrC7/NfbF7uD7HybAXxgeIA7HeIznN9\nA/CjmPSfA36K/e7vYRcri4FrfT6jjvDn2wv7PYPld8BpMekPxH7P4FyD850C/Dwm/fZYR+UgQASB\n+iZ6Figq0uhX47qjaH6fBY7D/mN57pfAO1A4o0bHbAUOxsYmG4yNfDyjyscYhBUk/wHejfl8IHbX\nshS7g1iKFYLVshp2B9IPu2p+DxuMcwmdh/rPu97Y38CqfnkbeCkm3TrYVf9y7PcMliXYv3HW5Krs\nVBtF8zsLOK+46tb1kxENq8GxtgKuBOYDj2BBqpbjoYmkJTPDjItUw9bA9aH1U4CroRBXb72yPsIC\nxPnY9L26EBFpAvqP3PwmY6MKe242uG16+F3DsPr9a1Y+WyINLVdlZ65OVlxfcEvB9algpwHYqMS3\nY20Bk4FPk6P6WZEYuSo7c3Wy4rYB90yFOz0I3Ik1SA+sfp5EGpLaKKRpjabyCaz2wtoeRKSH9ESH\nNJLRWJ+GSihIiKykWgeKsVhnnLl0nS8bbIa7uyjOZHdsBftK/mxF5YFCRDKsFZsCtQ3rtPIE1psy\nbBJwgX8/FBtuoFfCfUFtFM2sgPVjCDU6uw5wG5fZ52pgj5rmSqQ5VFR21vKOYgxW2Hdgt/9TgIMi\naRZivR7xr4uwXqFJ9pXmtj5wKR//QbvB2OOt80qk3xi7C32sDnkTyZVaBorhdB7861W/Lew32Bg4\nC4AnKY55k2RfaW7bAE+F1rcEZkNheYn03wR+jQ2bICJVVMunnpLc2pyBVSu1YwO8TcUKiEpMCr2f\n7hdpfFvTOVCUa8geivWViKueFBErY9t7unMtA8V8YERofQR2ZxC2K8VxfF7AqhU28+m62zcwaWUz\nKpm0NTbSaKDco7EnAH/GhmgWka6m0/kiOm5E4VT0wgr/Nmx44bgG6R9TzPBaWDBYI+G+oMbsZjYH\nCw6emwbuUyXSXoJVTYlIMlUrO9fE2g+iRmGNikmMw8ZKfx6Y6LdN8AtYlcGtWPvELKz6oNy+UQoU\nzakAXIY98ea518GpnUqkOqpWdv4R2DNm+x7YeDlZoECRC25NcO+A0/hMItVRtbKz3GOGlY63UysK\nFLng9gL3QNq5EGkiVetHMajMZ73LfCZSbT0ZukNEqqRcoHgem2c2an/iJyMXqZW4QKFxykQyYFPg\nOWxqyG8AJwFXYWMvbVZ6t7pS1VMuuIfARYfmuBb4TBq5EWkCVS07+wJfwh5j/ZF/37eaB1hJChTN\nZ03sosRzBXCLwQ0JpWnDhnsZjIj0RK7KzlydbE7sD9xTXHUjwC2IpPkJcHEd8yTSbKo2cdH7kS9z\nwFvAX7FhvxdVnDWR7kWH7ogOLb46MN6nE5E6KNcgOBB78ilYBgOfAP4F/Kr2WZOcig4GGG3I/go2\n/3WpIV1EpMoqfXLkbay9otycACIrY2usp34gGihasL9BEcmw3nS+4kuT2iiaS19gKbBKcZN7HNyY\ntDIk0qSq1kZxqP+y8LAJqwNHAn+qPF8i3WrFxgH7r626VmBzrLpTRFJSbuycK+namL0IG6r29tpl\nqSLRQCZNxW0K3AmFjdLOiUiTqUvZuWOtD5CQqp6amjsE3F/SzoVIE6pa1VPUKOBo4ChgMbBDJQcS\n6YHoo7EikkEjsbkgngJmYP0o2tLMUITuKJqauwHc0aENx2HtZCKycqpWdv4DG2r828CGftu8Cr9j\nLDZT2Vysk17Ut4CZfpkFLANW8591YAFqJvBoie9XoGhqbja4rfxKAVhC8e9DRHquamXnzdgIsj8B\ndvbbKgkUrX7/NuyR2lLTmQYOBKaF1udh06KWo0DRPLYDzimuur7gloLr4zesj83DLiIrr2rzURyM\n9cSeBZwNvIjd9u+U8LvHYIGiA/gImAIcVCb9McB1kW16oik/xgDrhtY3A16Ewod+fUv0mKxIKrrr\nmf0u8DtgP2AX4EzsDuOVBN89PJLuVb8tTn/gU8CfQ9scdocxAzg+wfGksUV7YI/GLlICo1CgEElF\nJU89vQ5c4pe2BOkrubX5NPAgFpgCuwELgWHAVKytQ9NhNq+tgPCjsNHAsSXwSF1zJCJAZYEirCNB\nmvnAiND6CEoP5HYUXaudFvrXN4GbsKqJuEAxKfR+ul+ksRSIv6O4IrR+N9bOJSKVa/dL5vTCpkxt\nA/pQujF7VazHd7/Qtv4U5+weADyEVX9FqTG7OayNPXodapNyHeA0+KRIbWSq7BwHPIs1ak/02yb4\nJTAemBzZbyQWWJ7ArjInEi9TJys91pdOHTjdYHD/9mM9iUj1VVR2lnuq6JJuDnJSmc/rRWM9NSW3\nM3ApFD6Rdk5EmlRFZWe5NorHKEad6BfqSl5qKdpeISIpKhcorqxXJkQiFChEMiTJU09rAqdhjycG\nDc4O+J9aZUpybzRwZ2j9+8BvSdZ/R0RSMBX4MtaPYU/g98DFqeaoSFVgTcm9Bi7onFkA3gGGppgh\nkWZT9bLzcf8anv50RrUP0kMKFI3vQOAXxVU3DNw74IJ2sXWAN1LIl0gzq/p8FMFYO69h/6kXoKGe\npXq2Bd4Lrfv2iULwh6wxnkRSVi5Q9MYG8zsXG9r5VOyR2cHAN2ufNcmJ0cBtkfXo0B0KFCIpKhco\n5gO3YENrLMbGYWqvQ54kX0YDF0TWw9Wco9ATUCKZNRT4KnAfFjR+RnFeiqxQG0Vj6wMsxXpme+4h\ncHuG0uyIzUUhItVTk7JzXeD/YbPevQCcX4uD9IACRWMbDcwurroCuMXghqSWI5F8qFnZOQgbl+lJ\nsvMUigJFYytgbV6eGwFuYcnUIlItVS07+wFHADdi81Fchc2D3dPhyatNgaKpuP3B3ZN2LkRyoGqP\nx04G9gX+BvwB+BxWnyxSKxq6QySDygWKu7HhwJfUKS8io9HEUyJSZap6aipuJrgdQxt+g02RKiLV\nlauyM1cn22T6Yp06PdcL3AfgBoTSvA4MR0SqLVNl51hsMMG5wOkxn38LmOmXWcAyrBd4kn0hYycr\nFfkyNsCk5zYD90Lo86FYJ09NTCVSfTUpO3fDGrPH++WLCfZpxaZAbcOuHEvNmR04EJhW4b4KFI3r\nZ9iFgucOA3dz6PM9gL/XOU8ieVH1QQGvBTbECuvloe1Xd7PfGKyw7/DrU4CD6NTBqpNjsOFCerKv\nNJ7onBMa40kko5IEih2w/7SVXr0Pp/NEM68CO5VI2x/4FPC1HuwrjWk0Vt0YXv9TaF2BQiQjWhKk\neRqbE6BSlQSWTwMPYnXSle4rjWdNbJynBaFtW9E5cJyLdfAUkZQluaMYhl3ZPQr8129zwGe62W8+\nMCK0PgK7M4hzFMVqp0r3nRR6Px09h98IRgD38/EFgeuHDfw3N5QmK8PEiDSDdmo8+nd7iaU7vbAB\nBNuwq8dSDdKrAosozsddyb6682gKbjtws7pPJyJVkqmycxzwLNYwPdFvm+CXwHhsuJAk+0Zl6mSl\np9wXwcX9DYhIbVS97NwF+CfwPjbj3Qo6T12ZJgWKpuAuBndG2rkQyZGql52PAZtgneJagf8FLqz2\nQXpIgaIpuDvAhdu81MlOpLZqEiig8/SUT1T7ID2kQNEU3CvgRoY23ALsWSq1iKy0qne4+zewCjZh\n0cXAa+iKT3puXewBhQ5bdasBqwMvhdJsF1kXkRQl6UfxRZ/uROADYD3g0FpmSpraeOxvKTAKeAYK\nK/z6YCxwvFzvjIlIvCR3FB1Yz+m16dxnQaQnRgPhWey2ouvQHbOxhyZEJAOS3FF8BmvIvtuvb4fV\nIYv0RDQwRIfy0NAdIhmTJFBMwsZZesevz8QGCRSpVG/sCbpwIIgOBrgxChQiDecR/zoztO2puIQp\n0FNPjWVL4LniqiuAWwRu7VCaAp0mNBKRGqio7ExyR/EMNhdFL+xq8BI0T4D0TF9syPjA2lhbxOuh\nbQ7r2CkiDWQAcD4wwy/nYf/hs0B3FA3N7QvuvrRzIZJDuSo7c3WyzcedAu6StHMhkkNV63B3q/+y\nuM51SYYZF+nOaODhtDMhIuWVCxQ7Y3NAXEexQTsIGrqSl2oYDfw2tD4Um7xqWTrZEZFK9cKG+r4a\ne+LpXKwXbZYoYDWGVbpuci3g3ge3amjjVGBsvTIlkmM1KTtXAY4F3qLz8AtpU6DIvh2Ax+lShek2\nAhcdz2khNkSMiNRWVcvOvti4Tjdgc1KcCQyv5gFWkgJF9t0JfK3rZncQuNtDG4JqJw04KVJ7VSs7\nr8GuBM/Fhl3oibHAHGwu5NNLpGnHqraepvN81x1Yx76Z2HzdcRQosu2TwDxstNgI911wF4U2tAMP\n1iVXIlK1snMFsKTEkmSGu1ZsGtM2rKdt3LzXq2Ed+oLqhqGhz+YBa3RzDAWK7CoA92NVljHcFHCf\nD204EfhVzXMlIlDFntktwKASy+AE3z0GCxQdWE/bKcBBkTTHAH/Gnq4CawMJUzVE49oPGAZcW+Lz\n6BhP/Sh95ygiKUoyhEdPDQdeCa2/Stf2jU2wu4b7sF7fXwh95oBpfvvxtcum1Mg7wNeJfdTV9QE2\nwqolAz8AflePjIlIZZLMR9FTSW5tegPbA3tjc178A+uANRfYHViAXZVOxQqVB2K+Y1Lo/XQ6t3NI\nesrdHWwGvASF/9QrMyI51+6XHqlloJgPjAitj6BYxRR4BatuWuqX+4FtsECxwKd5E7gJq8rqLlBI\nY4jOQSEitTWdzhfRZ1Wycy2rnmZgVUtt2FMvR9J1wqO/YHcOrdgdxU7YXAT9sbYQsEEJ90MFSzOJ\nTl4kIjk2DngWa9Se6LdN8EvgW9iTT7OAk/y2DbGnpJ7ACpSJxNNTTw3J3QJO866LpCdXZWeuTrYB\n7ELnR5xLcC+C2zS04RPA6jXKk4h0lauyM1cnm2GjsQmt3sbakspwg8B9AK41tPHx7vcTkSrKVdmZ\nq5PNmH7AeGy2w/nA2cD63e/mdgL3eGhDL+ADrC1KROojV2Vnrk42Y4YDt2GdKCt4es4dB+6q0IbN\ngBeqmjMR6U7VJi6SfGrF2hnWwea0Xgd7PPndSLr5wIE9+P7oE0/RHtoikjHNEChuxob6aAm9noQ9\naRV1GbBxzPYTsb4bzZb+0kj6Vuzf/KvY02hR9wJ7YEFhIfCaf51K10DRU6OBu0PrW6FHn0UyrRkC\nxVXYAIYu9PpmibTXEj9O1RtNmn5yKH0BG05jGcXOjFFHYAM+flTi82qIdrZbgKqeRKSG1EbRUNww\ncO+C02BQwXR7AAALvUlEQVSPIumq2uixItXm2yMKCvAiDUSBQuppG9RwLSJ1pivThuH6gesAt1fa\nORGRfJWduTrZxua+B+6GtHMhIkDOys5cnWzjchuAe8teOzkG66chIvWVq7IzVyfbuNwNdkfRxcvY\nTHciUl+5KjtzdbKNyf0PuHnWRtHJqsD76IEKkTTkquzM1ck2HtcL3NPgDon5cFfKT5cqIrWjfhSS\nGV/DhgG5KeYzjfEkIgCMBeZg4xCdXiJNOzATKzSmV7iv7igyyw0D9ya4LUsk+DlwSj1zJCIfy0zZ\n2YoNzNcG9MamNd0ikmY1bBrU9fz60Ar2hQydrES5X4P7SZkEBwOj6pUbEekkM2XnLsBdofVv+yXs\na9iENz3ZFzJ0shLmdgC3ENxqaedERGJlpo1iOPBKaP1Vvy1sE2AN4D5gBvCFCvaVTHIt2LSo34FC\ntYYmF5EU1XKY8SQRqzewPbA30B/4B/Bwwn0Dk0Lvp9O5nUPq73PYv+uVKedDRIra/dIjtQwU84ER\nofUR2J1B2CvAW8BSv9yPDRz3aoJ9A5OqkFepCjcIuBA4FAor0s6NiHxsOp0vos9KJxtd9cImpGkD\n+hDfIL05MA1rvO6PTWizZcJ9QW0UGeJ6g7sE3JVp50REupWpsnMcNuXm88BEv22CXwLfwp58moVN\nYVpu36hMnWz+uBZw7eAu92M5PQhu7QQ7nkfxSTcRqb9clZ25OtlscAVwO4L7Mbj54J4A921wbQm/\noAVYgj0aLSLpqKjsbIY5s6XqXAEYAAyJLKOAo3yi64B9ofCvCr98A+Bdv4hIA2iCQOF+AxRKLHkS\nPd+49V4xS+/Q+8EUg8JyYJFf3vKv84CjgcdWYjpTDd0h0mCaIFDwT+w2qtSSJ9Hzja4vAz7yr3HL\nEj4ODoWlNcrjVihQiEgd5S0QNIPJwPi0MyGSc7kqO3N1sk1iJyDJk1EiUju5KjtzdbIiIlWSmbGe\nRESkCShQiIhIWQoUIiJSlgKF1MtI9PcmIilQY3ZjGAoswEYGFpH05arszNXJNrA/Aj9MOxMi8rFc\nlZ25OtkGdQQwG+iXdkZE5GO5KjtzdbINaC3gNayTnYhkR67KzlydbAO6ELgg7UyISBe5KjtzdbIN\nqBc2Q6GIZEumys6xwBxgLnB6zOftwGJgpl/ODH3WATzltz9a4vszdbIiIg0iM2VnKzaNaRs250Hc\nvNftwC0l9p8HrNHNMTJzsiIiDSQzYz2NwQJFBzYHwhTgoJh05SYYytvkQyIimVPLQDEceCW0/qrf\nFuaAXYEngTuALSOfTQNmAMfXLptSRWtjnetEpInUcoa7JLc2jwMjgA+AccDNwKb+s92AhcAwYCrW\n1vFAzHdMCr2f7hepr0HAPsBp2L/hRelmR0Qi2v2SOTsDd4XWJxLfoB1Wql3iLODUmO1qo0hPH+Ak\n4B5sCtWpfr13mpkSkUQyU3b2Al7AGrP7EN+YvRbFdogxWHsGQH/sKhVgAPAQsF/MMTJzsk2sF/FV\nlAXgZ8AhwOC65khEVlZFZWctq56WAScCd2NPQF2BDeUwwX9+OXAYcIJP+wFwlP9sbeDGUB7/gF25\nxgk6dIUbvi/F2kSiTqJrOwnAJU2a/hSsag/s9+mFXfGfD7wUk/43wHZYwR8sqwDbY+1IYQ44OeY7\nRKTJ1DJQANzpl7DLQ+8v80vUi8C2CY/xXuh9ECWXlUkbN+ZQs6Z/w78Gv8tHFINynF9hAWWJP9Z7\nPq3u3ESkYakAExGpXGb6UYiISBNQoBARkbIUKEREpCwFChERKUuBQkREylKgEBGRshQoRESkLAUK\nEREpS4FCRETKUqAQEZGyFChERKQsBQoRESlLgUJERMpSoBARkbJqHSjGYnNdzyV+GtR2YDEw0y/f\nrWBfERFpcK3A89hUqL2Jnwq1Hbilh/tC48xH0Z52BhJqTzsDCbWnnYGE2tPOQALtaWcgofa0M5BQ\ne9oZSCgz81GMwQr7DmxmtSnAQTHpCjHbku7bKNrTzkBC7WlnIKH2tDOQUHvaGUigPe0MJNSedgYS\nak87A7VQy0AxHHgltP4qXed7dsCu2HzMdwBbVrCviIjUQS3nzE5ya/M4MAKbl3kccDOwaQ3zJCIi\nGbIzcFdofSLdN0rPA9aoYN/nsYCkRYsWLVqSL8+TEb2AF7AG6T7EN0ivRbGNYgzWJpF0XxERaQLj\ngGex6DXRb5vgF4CvA09jgeDv2J1EuX1FRERERERW3g+A2djTUjcCq4Y+m4h10psD7Ff/rH3scOAZ\nYDmwfWh7G7CUYifDX9Q9Z52Vyidk57eMmoQ9CRf8hmNTzU1XjdJZtAN4CvsNH003K538DngdmBXa\ntgYwFXgOuAdYLYV8RcXlcxLZ+tscAdyH/R9/GjjJb8/i71l1+1J8tPdCv4A9XvsE1kmvDau2SmuY\nks2xJ7juo2ugmBW3Q0pK5TNLv2XUWcApaWeihKSdRbNgHlZgZM0nge3o/P/kYuA0//50iv/n0xSX\nz6z9ba4NbOvfD8Sq87egwt8zK//xKzUVWOHfPwKs598fBFyHddLrwP7Djql35rw5WLTOulL5zNJv\nGSeuo2YWNFpn0Sz+jg8A70S2fQa4yr+/Cji4rjmKF5dPyNZv+hp2sQLwPlYTM5wKf89GDRRhX8I6\n6wGsi932BbLaUW8kdls6Hdg93ayUlPXf8htY1eMVZOu2uZE6izpgGjADOD7lvHRnLayaB/+6Vop5\n6U5W/zbbsDugR6jw96xlh7uVNRW7bYo6A7jVv/8O8CEwucz3uCrnKyxJHqMWYPWG72BVPTcDo4Al\ntcig15N8xqnlbxlVKs/fAX4JnO3XzwF+BBxXp3x1p56/0craDVgIDMN+7znYVXLWBX0Bsiirf5sD\ngT8DJ9O1rOn298xyoNi3m8+PBfYH9g5tm48VwoH1/LZa6S6PcT70C1jP9BeATfz7WulJPuv9W0Yl\nzfNvqSzY1Vr0dxtB5zuzLFnoX98EbsKqzbIaKF7HLhxeA9YB3kg3OyWF85WVv83eWJC4BrswhQp/\nz0atehoL/B9W9/uf0PZbgKOwTnojsQI4C09zhOssh2INngAbYnl8se45ihfOZ1Z/S7A/7MBnydbD\nATOw36oN++2OJH6E5LT1Bwb59wOwp9qy9DtG3QKM9+/HUyzwsiZrf5sFrArsX8BPQ9sb5fdcKXOB\nl4h/xPQMrDFxDvCp+mftY5/F6qqXYlH7Tr/9UOwxtZnAY8ABqeSuqFQ+ITu/ZdTV2GOdT2J/4Fmr\nr26EzqIjsUbOJ7C/xyzl8zqsivZD7G/zf7Gns6aRrcc5o/n8Etn729wde/DnCTo/spvF31NERERE\nREREREREREREREREREREREREauf9ldj3RKyPxAq6jr76c6yfz5PYuDqVOBzrGHVvzGebYmOaPYf1\nv/kjsGaF3y8iIhVYmbG1tgU2oOsw3ftTHKByJ+DhCr/3LmDXmO19sQAR7py5JzZGmIiI1EgQKArY\nRFizsB61R/jtLViP/9lYz9XbsV71YdFA8StsyI7AHOJ75h7tjzWL4jwA3/N5moPNFRD2JeDK7k9J\nRESqKQgUh2KBoIBV5byEDZZ2GBYcwAr7t4FDIt8RDRS30vmOYBqwQ2Sfdf0xhmBjft1Lcb6K6ORR\ngR9hQ1eLpKJRBwUUqZbdsWHqHTaC5t+AHbEhuK/3aV7HCvEkopPWRIdv3tF/1yJs+tk/AHuU2b+7\n7SI1p0AheeeoXuGcZGj26PEKdA4mcfMCPEPXOxORulGgkLx7AGtXaMEm8NkDmwHsIaxaqoBVPbWX\n2D86NPsX/fudgXcpziIW+CfWEB1UPR2F3cWUMxmr0to/tG0P1JgtIlJT74XeX0yxMftwv62AzVYW\nNGZPpThJ1knYsNIfYncMvw5916XYo7NPEt/eABYcgsbsC0LbS7VRAGyGDQH/HHaHMRkLbCIikqIB\n/nUIVvir34LkUpanQhVJ223YhC59sHmQszr9poiIiIiIiIiIiIiIiIiIiIiIiIiIiIgk8/8BIQvY\nLml8pv4AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x10396cb90>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "fig = plt.figure()\n",
    "ax = fig.add_subplot(111)\n",
    "\n",
    "plt.plot(res['e'], res['mu'])\n",
    "plt.plot(res['e'], res['low'], 'k--')\n",
    "plt.plot(res['e'], res['up'], 'k--')\n",
    "\n",
    "plt.legend(loc = 4)\n",
    "ax.set_xlabel('log10 of C')\n",
    "ax.set_ylabel('Mean Val AUC')\n",
    "plt.title('X-validated AUC by C')\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<p>We see that AUC is definitely affected by the choice of regularization strength (here we seem to do better with less regularization, probably due to very strong signal-to-noise in the data). But its hard to see what is best in the range $1$ to $10^{30}$. We can zoom in to see this better."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYoAAAEZCAYAAACJjGL9AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3XmYHFW9//F3J5N9IYGQQAJkEEUCsiiQC7JkBOSGRUEE\nVERAvIpeQXC5YFAhFxUQEVyuXBUVAUFcrwICApEAbmySsC/BhEASCAkEErYkpH5/fE79uqamuqd7\n0j1d3f15PU893V1dXXVqpvt86yx1DpiZmZmZmZmZmZmZmZmZmZmZmZmZmZlZzvwM+Gp4vhfwSIXb\n1ttxwO39dKxKrQPe1OhEWPMY0OgEWEsbCcwHjkqsGwUsBA6r8bGisIAy5m0q3LY3s4GP9T1ZFRkJ\nrAKuy3gvK1OfCVyeeD0a+DbwJLASmAdcCGxU64Rm2BT4CbAYeAl4OKRveD8c2/qJA4XV0yrgBJSJ\njQvrzgPuBH5Xh+MV6rBtpQFlfbwfBc8uYEIF2yfTNBiYBUwB/h0F4t2BZcDUmqaypw2BvwNDgN1Q\nwHo3sAGwVZ2PbWYt5hLgSpQRLgPGl9juYeCgxOsO4Dlgp/D618ASYAVwK7Bt6hhxdVIX8FTivbcD\n/0RXvFcBv0hsOxa4FlgKPA9cA0wK730dWAu8iq7UvxvWbwPcBCxHVVxHJI61EXA18CJwRzhOb1VP\nfwY+F/b5+dR7vZUo/gN4huqu4NcBJwFPoL/veShwDkZ/g7clth0PvEx26eRrwNwqjmtmVtIYlME/\nBxxbZruvAD9PvD4IeDDx+jhgBDAIVa3cm3ivVKAYjKpkTgYGoqv31cBZ4f0NgfcBQ1EV0K+A/0vs\n9xbg+MTrEWHfx6IS+U7hvKaE968KyzBgO+Bp4LYy5zwZBaPNgI/TM+MtFSguSxzvkjL7z7IOlULG\nAJsDj1KsXvs+cG5i25OBP5TYzz+AM6s8tplZSTejqqjRZbbZCl31Dw2vrwC+XGLbMSjDGxVelwoU\newOLUp/9K8VAkbYTuqqO3UL3NooP0DPj/yFwBgpEq4GtE+99nfIlii8DfwvPNwLWUCxBQe+B4kbg\n7DL7z7IO2D/x+lPo/wPwbyiwxu4GDi+xn8eAT1R5bGtCbqOw/nA0unK+GfhGYv31qEpnJfAhVBXy\nMPBeVJXyHlRlBcqEz0UNtS+iRnIotn2UMpGegeJJim0Uw1FGvyDs91ZUx55sw0i2CUxGmekLieUo\n1LYwDlWXJau9FvaSvmNQlRqoKms23Utdb6ASVNIgFFDiz0zs5RhZ0mmM93EHqmrrQlVsW6GqtCx9\nPbY1GQcKq7fxwAWoLv2TwJHAnuG9A1CJYBRqNyA8fgg4BHgI+FdYfxQKIPuijHzLsL5Uhh5bQrHN\nITY5se3nUQlgatjvtLDPeL/pfS5EwWRsYhkFfBq1v6wFtkhsvwWlvRN4MypVLAnL7uFc49/mQorn\nGtuS4lX/zagRu9peRuk0JoPppSi4fwQFsdUl9nEzqrarphOBmVkPv0JX7LGPoVLD4BLbbwK8gjLj\nkxLrP4XaJEahdoKL6F4t8zPKt1F8Bl2JH0b3NopvoG6pQ1B7xf+F/cYZ9S9Q9VFsJCp9HB32NwjY\nlWJ33LixfBhqbC/XRvFD4AYUTOOlE1W/HRy2ORv4Cwp2A4D9wvtxQ/5g1IvseuCtYZuNgNNRIM6y\nDjWcx20UD6NAHtscVb8toBjUs4xFJbvLKAaeScC3gO3LfM7M7P87FGWU6XaJWZS/4e1mlJkne0eN\nAH6PMsn56Gr3DYqB4hKKmX8X3at8dqZnr6d4201RO8RK1IPpE2G/caDYDTX2Po+6+YJKIHFPqWUh\nvTuE98ahnlMvosbes8gOFEPDPg/KeO/7KMDG250XznkFajM4OLX9aNS4v5DifRTno4w8yzrgRFTV\ntwz4Jj1rF26mWJorJ76PYgnF+yi+ggKlWcWmox/g48BpGe+PRVdxc1H96HZh/eboB/wg8AC6IjSz\n/vETSjf4m9XUQHR104mK6HModiOMfRNdgYCKznHvi00o9v4Yia7q0p81s9rrRI30kxucDsuJejdm\nT0WBYgHqpXEVaqRMmoJKDqBg0AlsjG4imhPWr0JFWvewMKuvrwL3o+quJ3vZ1tpEvQPFJLp3w3ua\nnj1Q5lIc92cquorZLLVNJ7q79o7aJ9HMEr6COgyc0+iEWH7UO1BUMk7Ouaj3xb2oge1e1JgYGwn8\nBt0huqrWCTQzs/I66rz/RahROrY5KlUkraT7EAnzKfa2GAT8Fg3r8PuM/c/Dg4+ZmVXrCXQPTy50\noAR1ov7eWY3ZG1DsU/9x1B8edBPPZajbXyn9MbJns5jZ6ATkyMxGJyBHZjY6ATkys9EJyJGq8s56\nlyjWouqkP6EeUD9BjdInhPd/iG4c+hlK+AMUx9XZA93UdB/Fwd9moBuUzMysn9Q7UIDuGL0+tS55\np+7fUbfYtL/gIUbMzBrOGXHrmN3oBOTI7EYnIEdmNzoBOTK70QmwxnAbhZlZ9arKO12iMDOzshwo\nzMysLAcKMzMry4HCzMzKcqDIr2PpObOZmVm/c6DIpwLwFuAuNM7V7o1NjplZ82qF7rEDKD0t6Eg0\nHegT6MbE9BDtZmZ90Qp5Z8Wa/WRHosEOs2b+SxqIJrE/qZftzMwq0ex5Z1Wa+WQ3R4Mk/oTSJQoz\ns3rI1aCAlm0qmif8QuBbrH/A+zJwO2rTeGU992X11YFmcIzQoJnx8lp4bDUboouiTVPL5cCdGduP\nR9OwrumvBNZBITw284VsNw4U/W9vNMfGx4Cra7C/ASg4XICGcJ8H3BOWi4B1NThGrIAyuY3CMbOm\nytwMjfw7OLEMAhaSPadIfxgBTAjLJuFxPhrVOG1/4FBgcWp5EnixgmONArYI26bnXgFdGHwwPB+I\nfoMdwOfpPlhm7HzgoxSDycvob38O+h6lHQRsH7Z5OXxmHbqImJex/fbofxaF7daF54+WSH8XsAP6\nnw5OPF5NdsZ/BrAvsCSxzAOez9gWdPF0GGqXezCx3Exlf/+hwFjgObID7/7AS2iunGeobUDaCJiF\nOqIMBlag83wcODhj+2HAfigwPh+2fw14NSy54UDR/+4B3oWGVK+FdShIXAAMQT/8ndGXNStIDAzb\nVVry2AXNQrgFujJ8GViGMtmTM7afhH7oq9GPcHVYSv3IDwe+H/a5DFgeHm8FrsjYfqfwmQ6UScWP\ndwM/zdj+WOAHwLMoY3g2LMtKpOcplDFNBPYK5zMR+CWaTzrtUJSRb4Gm8R2KgsrZ6Ko57RSy/26l\nnIGCQkfY9whgONmZOOj/OyakezjKjArofLMCRRdwILrgKITHAej7lHWMiei7lfz/vk7p0tApZc4t\ny4dDmrcBtgvLsWge76zv0G/QVAVj0XkPQBnvVHRxknZU2H4SuuhZjs5zenieNhb9X98CbB2WycA+\n9Px9vYAuAB9HGf7YsAwrca6j0ZQLY1HJawP0P14Szjttc+BaFEReC8tdaPrauir0vkmuRTT/OfS3\nbVGm+gSar3w4yuSeBN6fsf0EYEf0o3sKBYpa6gDGZSxPAX/M2H4H1PtrDcqc4sdH0NVc2kCKV8n1\nsBO632Uh+hsur+OxrKe3o2D1Qlheo/K/fwf6fm+GLuDSwa6ASh7LUeb/WGL5G7UtrVdiCKo1GBqW\nYSh4/q0P+6oq72z2TNaBom+GAG9Dme5KlMktAJY2ME1m1n/aKu/M+5XbduiK3cwsT/Ked9ZUXk92\nF9TtdTmq5zYzy5O85p11kaeTHY4asu5CPWq+iOo/zczyJk95J9NRI+PjZN99PBbdTzAXuIPuLf29\nfRbydbI7oS6CB6IGVDOzvMpN3jkQdcfrRN0X56AW+6RvUuza9VbUV7rSz0KOTtbMrInkZirUqSiz\nX4C6MF5Fz0HtpgC3hOePosAwvsLPNkIn8HUU1MzM2kI9A8Uk1Bc+9nRYlzQX3ZwFCg6TUZ/mSj7b\nXwro7slrUV/rEdT+XgIzs9yq553ZlRRtzgW+A9yL7ry8F3ijws/2h+2Bi9Edk+cDR+KxlMyszdQz\nUCxCt5zHNqfnkAArgeMTr+ejO4aHVfDZ2MzE89lhqZXn0dg8v0MBzMysGXWFJXc6UKbfiQbIymqQ\n3oDiENsfB35WxWchPyUPM7Nmkqu88wDUSD0PmBHWnRAW0BSfj6JusL9BgaPcZ9NqcbJDQnqm1mBf\nZmbNIFeBot7W52SHo1E8nwauQ+MemZm1AweKXoxAN/A9g9oedq5piszM8s8z3PViIBpTfj9qNyeE\nmZnlVFsVn8zMaiQ3d2abmVkLcKAwM7OyHCjMzKysdgoUI2ijqf/MzEyqaZD5LvD5eiXEzKyJtFVH\noEpPtoCGBNmxjmkxM2sWDhQZtkHDlrvqyczM3WMzHYSG6WirKGpmZpVn/LOA99YzIWZmTcQlipQB\naC6JPzc6IWZm1v9clWRmVj2XKMzMrHYcKMzMrCwHCjMzK8uBwszMymr1QPEFYMNGJ8LMzEqbDjwC\nPI6mH00bB9wAzEGzzR2XeG8G8CBwP3AlMCTj8+Va7jcGVgCDq020mVmLy02P0YHAPKATGISCwZTU\nNjOBc8LzccByND1rJ/AvisHhl8CxGccod7LHAL+tOtVmZq0vN91jp6JAsQBYA1wFHJLaZgkwOjwf\njQLFWuCl8JnhKHAMBxZVefyDgD/2Id1mZtZPDgcuTrw+GvheapsBwGxgMbASOCDx3ifCuqXA5SWO\nUSoqDgKeBzapKsVmZu2hqhJFR71SQWUJOR1VSXUBWwE3ATsAE4BTUBXUi8CvgQ8DV2TsY2bi+eyw\n7I6qrp6pPtlmZi2nKyy5sxtqqI7NoGeD9nXAHonXs4BdgSOBHyfWfwT4fsYxSgWjccDO1STWzKyN\n5KYxuwNNFtSJeh5lNWZfAJwZnk8AnkbdWXdEvaCGoTkkLgU+nXGM3JysmVkTyVXeeQDwKGrUnhHW\nnRAW0JX/NcBc1A32qMRnT6XYPfZS1O6QlquTNTNrEm2Vd7bVyZqZ1UhuuseamVkLaLVA0UF9e3KZ\nmbWdVgsU+wNXNzoRZmatpNUCxUHArY1OhJmZ5UeyQaaAhgt5W2OSYmbWNNqqI1DyZLcDnkQBw8zM\nSmvbXk/xIIBtFSnNzOqtlQLFOOAPjU7E+osGaDEzs1posdJDtDVEj0D0NETfgmgXiFyVZma11mJ5\nZ3ktdLLRfhA9C9EnIJoC0VkQPR6Ws7TOzKwmWijv7F2LnGz0aYiegWhaan0Bol0hugCiRRDdC9Gp\nEG3RmHSaWYtokbyzMk1+stEgiC6C6EGI3tTLtgMh6oLoRxAtg+gvIcCM75ekmlkrafK8szpNfLLR\nhhDNguiPEI3ufftunx0M0cEQXQHRCohuCFVWu0A0rD7pNauVqADRcIgmQrSlvs/Wz6rKO5u9oTQC\nDgXmA/c1OC1ViKagoUb+AJwGhTfWY18jgPcAB6LZAd+K7ieZi/4m94XnT0GhzoE1GgSMCMvI1POh\nwGvAKyWWV6Gwtr7pA5XM2BjNf7JJeJwAjEfztK8CXk49llhXWB06GwwHNuhlGZ14PiIc67WwvF7h\n41pgHfreryuzJN8voCH6B6F5YbIe0+s6wmfXhGOurfD5YGAMMDY8lnu+DlgBrA7/h8VoVsonEkt4\nXXix1H+zMaIC+h+OL7MMIvt/2Nv/eQ36m6yp4vkbffhtR1SR/7dCoLgH+AKaArUJRNOBy1CAuKQO\n+x+MgsUOaAKo+HEoxcARB48H0d9wdJllg4x1o+geDOLHgSgjTWeqL6MfwhCUqZZa1pIZQHg1fP7V\nCpcC+rHGgSAZEDZE86k/G5ZnwuNzKINMn1f6HEcmltgaNGXvi8BLieellpdRRjIk/F/ix6EZ65Lv\nDURd2geEcxxQZonfj9O3usRj1rq14bPxIJuDKny+GngBBYAVZZ6vgMJrxT9fNAjYAk2HnF7ehP73\nySDyr3Cs9N9saJl1Q0I611a5ROg7kw4EoO/N0sTyXOLx9RL/w1L/3/gxHbgreT4NCn+hOm0XKFZQ\nvBrMsagAnIymgz0cCn/t5+OPB7anGDx2QHezr0OZW9byYon1q8i+0n6976WWqIC++MnAMQLNcljN\nMjTs8Fl6BoNngGW1K7nEVSaF1bXZn/UUxUE/HTwGUrxCT1+xl1r/RvhcRxXLAHRhsbT7Uni5rqdd\nlahQ7xJFs4uAXzU6Eb2LBkN0MUT3QdTZ6NQU+R4NszZVs2ro8eiKM207VMebBxFwbKMTUV40DqJb\nIfoDRKManRozM2oYKH4JTMtYvzdwZa0Osp4iivWFORS9DaJ/QXSOh+UwsxypWaC4p8x7D1a4j+nA\nI8DjqG4+bRxwAzAHeAA4LvHeGOA3wMPAQ8BuGZ/PaffY6M0QfRei5RAd3ejUmJml1CzvfKyP78UG\nAvOATtQyPwdID0MxEzgnPB8HLKc4lemlwPHheQfqfZOWo0ARFSB6V6hieg6isyGa1OhUmZllqNkw\n4/PQ0N1pB6Iuar2ZGvaxAPVIugo4JLXNEtTdkvC4HHVJ2wDYC/hpeG8t6oGTQ9EQiI4D7gUuAq4D\nJkPhdCgsamjSzMxqoKPMe6cA1wJHoGqoArAz8E7g4Ar2PQl4KvH6aeDfUttcDPwZ3WwzCjgyrN8S\n9UW+BHXnvAd1LX2lguP2k2g88KmwzAW+CNwIhXUNTZaZWY2VCxSPob72R6HpRSM0H/UJqE9ybyop\n2pyOqqS6UP/om1Bg6ADeAZwI3AV8G2XEZ2TsY2bi+WzqfuNdtD0KoocBvwb2hUKlbTZmZo3QFZbc\n2Q01VMdm0LNB+zpgj8TrWcAu6E7a+Yn1e6LSTVo/tVFEA8LYSjdDtBiiL6nbq5lZU6pZ3rkKWJlY\nXkK3zv8Y2KiCz3egtoxOdMdtVmP2BcCZ4fkEVD21YXh9G7B1eD4T+EbGMfohUET7QPTPsBztAczM\nrAXUNe/cEPgcqnKpxAHAo6hRe0ZYd0JYQD2drkF1/Pejaq7YjqjaaS7wO/q911O0LUTXQvQEREf4\nLmYzayH9Uhtzb38cpAJ1ONloE4h+CNFSiD6rXk1mZi2l7oFiEPkZ0ruGJxuNgOgr4Sa58yEaW7t9\nm5nlSlV5Z7leT++n5wiDY4EPoDumW0Q0EI0XdRZwO7ALFOaX/4yZWfsoFyjeQ/eoE6Eb4r4N/LGe\nieo/0f7A+Wio8sOgcGeDE2Rm1jJ2bXQCgr7OfbADRH+C6DGIDnVDtZm1mbq1UWwHfA31YCo3YGB/\n6sPJRl+F6BmITgwza5mZtZuaBootUbfW+4C7gWXovoi8qPJkoyEQvQTRhPokx8ysKdQsUPwdlRy+\niKYehO53S+dBtYFiH4j+Xp+kmJk1jZqNHvssusltArmeHKgq+wM3NjoRZmatZAyaE+JGNHzHCnqO\nANtI1ZYo/gnRHr1vZ2bW0urWmD0BOAn4G92HD2+kKk42Gg/RCjdgm5lVFyj62i20E01I1GhRjqa4\nMzNriEL1eXn6ZuqWVk2J4mcQfapuKTEzax5tdY1d4clGhTCPxFb1TY6ZWVNwoMjYbHsNF25mZtRw\nUMDv9XKQz1RzoAZzt1gzsz4qFyjuoRh10o0ezVZs2R/4QaMTYWZm/a+CgBUNg2glRGPqnxwzs6ZQ\ns6qn2HjgVGBbYFjiIPtUl66G2RO4DworGp0QM7NmVG4Ij9gVwCNovKeZ6P6Ju+uXpJpz+4SZWZ39\nMzwmpz+tNFBMR0HmceC0jPfHATcAc4AHgONS7w9E83NfU2L/lVQ9zYVo94pSa2bWHmrezvyP8Hgj\ncDDwDqCSrqYD0dwVnWie7TnAlNQ2M4FzwvNxaAa9ZHXY51CJ5uoSx+jlZKNNIXoBokqq2MzM2kXN\nRo+Nx0T6Ghoc8PPAF4AfA5+tYN9TUaBYAKwBrgIOSW2zBBgdno9GgWJteL0ZcGA4Xl9vNd8P+DMU\n1va6pZmZZSp3pb0IXcn/AngRjRzbVcW+J9F98MCn6Tny7MXAn4HFwCjgyMR7FwL/RTGQ9IXbJ8zM\n1lO5QLEtcDjwZeAy4DcoaPyjzGeSKinanI6qpLqArYCbgB2BacBS1D7R1cs+Ziaezw4LEA0A3g2c\nUVlyzcxaVhfVXej3yUTgFDTr3RPA2RV8ZjfUUB2bQc8G7euA5PwQs4Bdw/6fQjPqLQFeRsEqrUww\ninaE6PEK0mlm1m7qdtP0KOBYYC662u9NBwoqncBgshuzLwDODM8noOqpDVPbTKNPvZ6i/4Lo+xWk\n08ys3dQ0UAxD7Qa/Q1OjXoq6vFbai+gA4FHUqD0jrDshLKCeTteg4HM/cFTGPqbRp15P0U0QpRvP\nzcyshoHiSuA51Dbxfop3ZedJiZONhodhO9anIdzMrFXVLFAci6qb8qxUoPh3iG7r36SYmTWNmo31\ndOl6JqSR3C3WzMyA0iWK+yGa2r9JMTNrGs02VcR6yTjZaCJEyyEa2P/JMTNrCjUfZhx0r0NnYvuI\n7Psa8mB/YBYU3mh0QszMWkElgeLnaIjxOUAy881zoHD7hJlZP3qYvg/KV2+p4lM0AKLnIJrcmOSY\nmTWFmo0eG3sA2LRvael3OwHLofBkoxNiZtYqKql62hh4CLgTeD2si4D31itR68HVTmZmNVZJoJhZ\n70TU0P5o/CgzMzOgWz1bNAKiVRCNbFxy1ttGwLXAOxudEDNraTVvo9gduAtYhWaqWwe8VH266m4a\ncDcUVjU6IX20Hareewy4p8Q2e7N+EzmZmdXFPcBb0CRCA4GPAuc2NEVFyRLFtyE6vXFJWS/vQQMw\nfqTMNgPRxE6rUEA5D00V68BhZtWq+Z3Z8dXtfYl1c2p9kD5KBoqHINqlcUnpsxPJnia2lKGoZHEG\nmkb24Tqly8xaV1WBopL7I25DU4r+GM029wwaWXbHqpNWexFQgGhzVOKZ0IR3ZO+M/qaL+vj5Aag6\nMG1H1LD/EAomD4XlORozzss4IOv+luXAgoz1GwFboGrOpagkVet0F9AIyROA8cAQ4Ek04Vbatuhv\nOjCxDEBzqdyVsf2OwDvC84hi2u9D39W0MWgo/5fD0pfv8QB0Tlmf3QpNHDYqtdyGLjjSjgfeB7yI\n/gfxMgu4I2P74WEZgi5mhobnS8KStgeazbKA5rpZBCxGM1u+1tuJ9qMC2d+7LdB3enBqmYPOI20f\n9D+IvzcDwvMb0Jw9aQcAb0a/7TfC4zr0v/pXn8+mKOSdlamk19Mx6KROBD4LbIbmp8iTdwM3N2GQ\ngNLtEZXKChKgaWTPQxncTmhSqG3RD/2IjO3fhEoq8RcyCo8L0BS4aVOAQ4GxqeV2snvK7V5i/bUU\nZzlM2gM4C1WtTUBf6qXAJcB/Z2y/CcrslwKvhOeboECUVeo6Ef191qGMainKoC4nO1BMAQ5BP9rk\nsozsQDEZ/T0LFH+QBRQEsgLFMWhyr5Eow12DguO5wPkZ238WOBVlTkPC0oH+lmdlbL8nmoRsFbAy\nsazJ2Bb0f1wGbID+B/FSauqB04CT0N8wuVyI/qZpE4Atw/Pd0HTLE4FvAj/I2P6d4TNxQHk1rF9F\nsdt+0hCUEUeJZTCwNvHZpI8B+6HMf6PE8nHgFxnbfwbYF1idWs4hO1BsjS4K0xn/iIxtQd/drekZ\nWO6jNoGiKpVGlOHA5mRHvkaKSxRXATdC4aeNTlDOFdBV6ysZ7+0GfJLilzK+Ov0H8J2M7XdGGc8L\nqWUBUI+5ykegzH8tuupMOxxlkhPQ9/XZsFxGdsYT947LY+eHAroiH4kynxczthmDznM1yihfR5l+\nq44K+lF0YTIRmIQCQQFl2D/P2P4HqOajkFjWoIz/yozt90E3Fi9HAXJ5WFbSmn/TqkoUlXgvChAL\nwuu3U3pq0v4WaZTYaFmofsq7nRqdADMz6hD8/omuXpLF5QdqfZA+itSAHT3U6IT0ogO1FzxG6aKm\nmVl/qfl9FGuAFal1perFs0wHHkHVEadlvD8ONejMQQHouLB+c+AW4MGw/jMl9r8/8Kcq0tPfxgJ/\nBLZH1TsvNzY5Zma191Pgw8D96H6K75Fd55tlIDAPzWUxCAWDKaltZqIGIFDQWI6uwDehWFUzElV/\npT8bQTQbogMqTE9/2waVIi6k8rk/zMzqreYlipPQXcOvo9b/l4BTKtz/VBQoFqCSyVWo50jSEoo3\njY1GgWIt6jIa36+xCvVcmZhxjJ1RF7+8KaBGtnNRD5W1jU2OWV09T/ceRl7ysTxf7p+WF4cDFyde\nH41KJEkDgNmoS9lK1H84rRP1b0+P4xRBNKsWCa2ToY1OgFk/acWeQa2g1P+lqv9XueqQayjdhSqi\nsmHGK0nM6ajk0IVuSLkJ3ay0Mrw/EvgNcDKZXRnft45i//zZYelvo8ke/ypPNw6ZWfvqCkvNPYd6\nOp2KBtybljjYtAr3sRtqqI7NoGeD9nXo5qrYLCAeimMQaqguVdUVQfT2CtNSawVgL9RQ/dcGpcEs\nL1yiyKealCjK6UDVQJehgPE11FZR7T6eQFVH8e3t6QbpCyjemTsBjXu0IcqIL0MNwaVEEFXSzlJL\nA9Agfn9FPbk+jquYzBwo8qnugSJpCOq2ugwNfVCNA1CPpXmoRAFwQlhAPZ2uQWPm3I+GmgANObAO\nBZd7wzI9te9GfDmvCGk5EvXqMjMHirzql0AxFI3r9Gs0ns1X0O3zedGIL+fG1PjWd7MW4ECRT3UP\nFJeju7K/hm4WyyN/Oc3yoRl+i7NRd9HBdT7Oz9DtAJtkrP9qal0nqjlJVqEfBdyNOvQspmc7bjXq\nHijW0X2UyeSSlxnu6vXlHIIa8cfWaf9mrSbvgaITDYb5COq2Xy8jUB45F/hC6r1L6DmybyfdA8Xn\n0GCWh6IBPAcCBwHf6GN6+rWNIq/qcbLT0JfpD/S8IjCzbHnPeM5Ag5l+CbWJgi4IV9C9k87GKKCM\nC69PRVf1TwP/gTL1N5U5zjFoKPB4NIukSyhfotgABZlaTuPgQEFtT3YcGq7kKRTNzaxyec945qHM\n+y1oaPaNw/qfoOr12KdRVQ+o88wS1FNzGBpp4Q3KB4pZKBiNQvNevCPxXm+BYjqqsqplT04HCmp3\nsmPRF+LX6/3fAAASO0lEQVRCSk/MYmal9fJbjKLaLH2yJ8q049/2HIr3Zu2Lgkjsr2gECdCF49cT\n721F+RLFFiiQbB1e/x74duL93gLFh8meDXB9OFBQ25Pdoob7Mms3ec54LkZVybEvUZw2YSCqWpqK\nMu1VFKcCuB5N5hUbQvlA8SXUNhH7MJo5Me5GfzEa+y3pLRTHgctticIjmhYtbHQCzKzmhqF7ngZQ\nvFofgubY2QG1J/wK+BDK1K+hOBXAEjTdQay3ydGOCdvEx+lA06kehNpHFtLzpuUtKc7Y+Hc0+Or7\ngN9WcnJWmb5cxbjkYFZ7eS1RfAiNSL0Zmkp3PBoB4laKc5FPRZn7/WjUhdh0VNrYBk07eymlSxS7\no9LAdqnj/ByNVUd4byXwblTKmIhGvj47sZ/PoZGzDwnHHIRuWnavp/VQ7ckW0JAi7s1kVlt5zXiu\nB76Zsf4IFATiap7H0cgT6VqWL6Ig8jSqhlpH9k3H/4tuTE7bFbWPjAmvD0b3SKxA0y98A5Vwko5C\nNzivCse+Bo2b1xcOFFR/sm9Cf3jfWW1WW+2Q8UxB7Qn9Pb7c+qhJoGimE66FfYA/0x5fajNbf+9D\nV/xj0dX/1VQ3FXRLaNdAYWZWiU+gO6XnoTaITzU2OdYX1ZQMCqiRaMs6pcWsnbmUnk9uo6C6k92Y\n7pMomVnttFXG00QcKGizkzXLMf8W88mN2WZmVn8OFGZmVpYDhZmZleVAYWZmZdU7UExHkwA9DpyW\n8f441BNpDvAAcFwVn63GhymOP29m7WUBGk68PwxB0xUsQtOufp/uw4LMRkN6xLOFPpx4b3PgH2hs\nqvPp7nq6z23RMgaim1Q60cBWc9At8EkzgXPC83HoD9RR4Wehspb7QcCLaBRHM6uPPPd6mo9utu0P\nZ6IBB8egPO3vKJ+L3QIcX+KzFwEnAKNR/rdzWP8B4H/6mJ7c93qaik52Abqj8So0ImLSEvRHITwu\nR2OpVPLZSu0C/Cvs28wsNgRNLLQoLBcCg8N7twKHhed7oGE7Dgyv96U4n0XawcD30KB/y4Dv0jMw\nlBprrhONHPESGhRwS5QvngacXtkp1Uc9A8UkiuOsg0ZfTI+6eDEaencxmvDj5Co+W6l3oShuZpb0\nJXRRumNYpgJfDu/NBrrC82noYnPvxOvZZfabDAQD0BDnyZkzzwGeA/4S9hV7ANgflUZ2Bh5CM+Jd\niIJHw9QzUFRStDkdVStNBHZC9XnVTkU6M7F0Zbzv8Z3MGm8myhPSy8wqti+1bV8dBZyFrvyXAf8N\nfCS8dxvFTHwvlLnHr6ehEkeWG9AF7zg0ncFnUNqHh/dPQyWFicCP0BDi8fwW54RjzUZ54RBge+Ba\n4MpwzE/37VTpontemRu70X3IjBn0bJS+DhXrYrNQVVEln4Xeg9FQ1GA0upftzGz9NGMbxSt0b/vc\nBs0wB8rYX0UTEC1BbZ1Po7bOV4ANSxxrKKp6ehpVn38xsc8s1wMnZqwfANwObAucF/YzCE2utE2Z\n/aXlvo3ibjQfbCeq9/sAGqI36RFgv/B8AvBWVMSr5LOV6AD+kwYX28wslxajPCa2RVgHCgb3AKeg\nzHkN8Dfg8ygAPF9in68BJ6HqpjeH7e7uQ9o+gRrCHwLeFvaxJqRl+z7sL9cOAB5Ff9gZYd0JYQEV\nz65B7RP3o6Jguc+m5fkqxqyd5Pm3OB91tx+aWDpQ/f9fUT40DrUZnJX43NdRj8kvhdfxRef3yhxr\nYlgKqGZkIcWL4Q2Af08c/8NoFrs3p/YxHs3lHVdXfR9NlzoSeIzqusl6UEDa7GTNcizPv8X5qNdS\ncjkLtQF8B5UiFqMeUIMTn9sfeAO1G4Cu7N9A06iWslc43svoHokPJd4bB9yJgs0LqISSdX/HpcD7\nE683Q/dXPE/P+yt640BBm52sWY75t5hPuW+jMDOzFuBAYWZmZbVyoPgd6jllZmZtrFQ921h0/8SQ\nfkyLWTtzG0U+uY2ijL1RH+RyN7qYmVkFWjVQeNgOMzMDShef7kcDfJlZ/3DVUz75PgqyT3ZDdGNK\nR8Z7ZlYfbZXxNBEHCkqf7Ih+TYWZtVXG00QcKGizkzXLsTz/FtdRHMo7NhO4vI/7mwT8AU2G9hTF\nseuSx1tFcbrTHyXe2xcN8bEEDXYaG4MGIaz1Ra4DBW12smY5luffYlagOJO+B4pbgAvQlM07oIDR\nlTreliU+ex8aOjz+XDzJ0f8Ch/cxPeW4e6yZWR8lZ6HrQvNHzEAzz82n+0jWSSPRxEVnowEC7wN+\nQ8/pTkvlrSPQ0OH3AavR/BZTgclhP7nkBl8zM82HsxEaInx3NKna3WhY76RC6hEUFN6W2u62sP5v\nwOeAJ8P6pag0UUCBZgUatfaYWpxEvbRaiWIbfDe2WR7NJH9ToaZ9BU0OdBvwR+DIjG1WojksvoLy\nmncAhwHDEtvsjUoI26Dhy69F1VQAn0RDm/8ATbv6n8BNaO6JP6H7v/bGaipZz1ZAk4S8tUFpMWtn\neW6jWEPPfOHrwE/D8y50pZ90HnBRif1tgSZcW4pGgPgOcHOJbQeg4LJdxnubAveiiYzuRBMdTaZY\n+qgFt1GkbIWidrqoaGbtbSE9G5e3BBYkXo+lOKMcKMNeVGZ/70Ez0e0ObAzcUWLbrKqq2IVo9rzX\nKE53+iSaG3tcif1ZHySj4ifoey8GM1s/eS5RnI2mOZ2ELo73Q7PMbRve70Kljm+iTHov1L116xL7\n2wYYhWbDOxo1gG8U3tsW2AldtI5EpY2HKVY9xd6NRriOPYimSd2O7r2h1ldTdI+dDjwCPA6clvH+\nF1DR61407MZa1J8Y1APhwbD+SrLbHpIn+wvgozVJtZlVK1cZT8pQVJU0HzUe3w0cnHi/C90PcTrK\n9Beg+axLORlVO61C7RnJOazfhfK8VcCzKBhslfr8EJTnbZ5Yt09I3yKy20b6KveBYiAwD+hEUXoO\nMKXM9gdTrOfrBP5FMTj8Ejg24zPxyRbQP6VzPdJrZn2Xm4ynD7pQoGhFuW+jmIoCxQJUrLsKOKTM\n9kehUgGoWLgG1Rl2hMdS9YUAo9GdkgvWJ8FmZtZTPQPFJLpH6afDuizDUf3cb8Pr54FvoUajxai4\nWKpXAcCLqI3CzKwvmrlEVHf1DBTV/OHfgxqbVoTXWwGnoKqkiahRqFydoZlZX81GXV6thHremb2I\n7o01m6NSRZYPUqx2AtgF3dG4PLz+HfBO4IqMz85MPJ8dFjMzK+qi+3hUudEBPIFKBYMp3Zi9AQoI\nyTsbdwQeCOsKwKXApzM+6+KiWT74t5hPue/1BHAA8Chq1J4R1p1A92F5j0XdX9NOpdg99lLUcyot\nVydr1sb8W8ynpggU9RahezEmNzohZm2urTKeJuJAgU72JYp3RZpZY7RVxtNEHCjQyd7b6ESYWXtl\nPE0k9zfc9Zc/NzoBZpZbC9D0o81sJg0ex86BwsxaWTyXRa3MpPJM+0jUzf9lNH1qXzW8tNYKM9zd\n3ugEmJllWI7m1p6CBv1rWq1Qonip0Qkws1ybirraP48mK4oHGz0Y3d/1Apq1bvvEZ05DNwi/hEaD\n3QeNhj0D+ACajKi39tFZaB7sJRnvDQV+DiwLx78TzW8Bmivj1nDsG+k5N8VuqKTyQkj/tF7S0fYa\nXiQzM6CX32IEUS2WPqRrAXAfGmduLBoq6KvA29GI07uim3qPQcN8D0Kz4S0ENgn72AJ4U3h+JnBZ\nlWn4D3pWPZ0AXI0CRiGkZ1R47+/A+RTnxngpccxJKLhMD6/3C69LTXTkxmwzaw4FKNRi6cOhI+B/\n0JBCL6ApUD8EfBz4IXBX2OYy4HU0Y91aVOrYDmXWC9G0B+FUajKp0GrUrf8tFHtvrkRBaReK83ff\njqZdjR0NXAfcEF7fjObXOLAGaSrJgcLMWl1yFOuFaKDRycDnUfCIl83QPNZPoEFJZ6JSxy/C+lq6\nHPgTmn5hEfAN1GY8MaTl1cS2T1IMTpOBI1Lp3oNi6acuHCjMrNVtkXq+GAWMr6PqqHgZiSZJAwWH\nvVDGHKGMHPpW/ZX1mbXAWajU8k7UXnJMSFvW/N3xPhaiIJNM9yg0g1/dOFCYWSsroAFFJwEbAl9C\nV/E/Bj6JGroLwAjgIBQstkaN10NQddRrwBthf8+ggU4rqX4agNogBoXnQyiOWdeFGs8HoiqnNeEY\nC1FV0n+Hbfek+7StP0fTMuwfPjs07KvUXD+GG7PN8iKvv8X5qAfTg6ia5hKUuYImS7szrF+MShMj\nUQZ+B2pEXo4aneOqnQ1Ru8HzKEMv5zhgXWr5aXjvgxTn1n4G+DbFC/ct0VzcK1Gvp+/SvQF9KppO\nYTmau/sauk/pkOQhPGizkzXLMf8W88m9nszMrP4cKMzM+m4VqiJKL3s0MlHWnYu7Zvng32I+uerJ\nzMzqz4HCzMzKaoXRY82s8V7A1U959EKjE1CJ6aiv8OOoL3PaF9AYJ/cC96O7FceE98agkRcfBh5C\nIyam+YtpZla93OSdA4F56C7GQWg43Clltj8YDXAVuxQ4PjzvADbI+ExuTjYHuhqdgBzpanQCcqSr\n0QnIka5GJyBHctOYPRUFigXo9vSrgEPKbH8UGl8FFBT2ongX41rgxbqksnV0NToBOdLV6ATkSFej\nE5AjXY1OQLOqZ6CYRPdRG5+m9Hgkw9Ht9L8Nr7cEnkO32/8TuJjug2SZmVk/qWegqKZo8x40ociK\n8LoDeAdwUXh8GfhiTVNnZmYNtxvFyTVAUwhmNWgD/B8aJCu2CRrMK7YncG3G5+ZRnDzdixcvXrxU\ntswjJzrQBCCdwGBKN2ZvgEZBHJZafxsa7hc0gcg3MDOzlnMA8CiKXjPCuhPCEjsWuDLjszuiaQrn\nAr8ju9eTmZmZmZnZ+pmJelTFN+5Nb2hqGqO3GxvbyQLgPvRduLOxSel3P0VzPN+fWLchcBPwGJoE\nZ0zG51pR1t9iJu2ZV2wO3IImb3oA+ExY31bfjTOBzzU6EQ1U7Y2NrW4++gG0o72At9M9czwPODU8\nPw04t78T1SBZf4t2zSs2AXYKz0ei5oApVPHdaJVBASuZv7ZVVXtjYzto1+/D7fQc2+e9aJQDwuOh\n/Zqixsn6W0B7fjeeQReQoPkzHkb3tFX83WiVQHESavT+CS1efMpQzY2N7SBCQ8HcDXy8wWnJgwmo\nCobwOKGBacmDds4rQDUPb0dzglf83WiWQHETKkKml/cC/4vu5N4JWAJ8q0FpbJSo0QnImT3QD+EA\n4NOoCsIk7kPfrto9rxiJRr84Gc3Cl1T2u9Esw4y/u8LtfgxcU8+E5NAi1FgV2xyVKtrVkvD4HLqR\ncyqqhmhXz6I66meATYGljU1OQyXPvd3yikEoSFwO/D6sq/i70SwlinI2TTx/H90br9rB3cBbKN7Y\n+AHg6kYmqIGGA6PC8xHA/rTf9yHtanSvEuHx92W2bXXtmlcUUFXbQ8C3E+vb6rtxGeoOORedaDvW\nwWbd2NiOtkSNdnNQN8B2+1v8AlgMrEbtVh9FPcBupk26QCak/xbH0755xZ7AOvS7SHYNbtfvhpmZ\nmZmZmZmZmZmZmZmZmZmZmZmZmVm7WrUenz0R3beyjp6j1X4XDfk+Fw0nUo0j0I1RszLe2xq4DvV7\nvwf4JTC+yv2bmVkV0uPdVGMnYDI9hzU/EGXmAP8G/KPK/d4AvDNj/VAUIA5KrJsGbFfl/s3MrApx\noCgA30RDOtwHHBnWDwAuQsMy3wj8EXh/ah/pQPEDNIxK7BGy7wD+UDjW/RTnATgjpOkRNFdA0vHA\nz3o/JTMzq6U4ULwfBYICqsp5Eg2WdjgKDqDM/nngsNQ+0oHiGrqXCG4Gdk59ZmI4xkZo4qlZFOcQ\nuQV4R0Zav4WGyDbrd60wKKDZ+toTuBINs7wUuBXYFQ1Z/quwzbMoE69EenKc9PDNu4Z9LQfeAK4A\n9i7z+d7Wm9WVA4WZMvJaZc7pYd83C+vKHa9A92CSNS/Ag/QsmZj1CwcKM81X8QH0e9gYXd3fAfwV\nVUsVUNVTV4nPJzP9q4FjwvPdgBUUZxGL3YUaouOqpw+iUkw5V6IqrQMT6/bGjdlmZnX1UuL5eRQb\ns48I6wpoVrS4MfsmYN/w3mfQ8NWrUYnhR4l9/Q/qOjuX7PYGUHCIG7PPSawv1UYB8FbgetT76UEU\nPDYuc35mZtYPRoTHjVDm7/sWrO00y1SoZo1yLZrQZTBwFu09laiZmZmZmZmZmZmZmZmZmZmZmZmZ\nmZlZs/t//RFhp9En1ZEAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x109803d10>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "fig = plt.figure()\n",
    "ax = fig.add_subplot(111)\n",
    "\n",
    "res2 = res[(res['e']>-6)]\n",
    "\n",
    "best = res2['mu'].max()\n",
    "best_1sde = best - res2['sig'][(res2['mu']==best)]/np.sqrt(10)\n",
    "\n",
    "plt.plot(res2['e'], res2['mu'], label='Avg AUC')\n",
    "plt.plot(res2['e'], res2['low'], 'k--', label='Low 95%')\n",
    "plt.plot(res2['e'], res2['up'], 'k--', label='Up 95%')\n",
    "plt.plot(res2['e'], best_1sde.values[0]*np.ones(res2.shape[0]), 'r', label='best_1sde')\n",
    "\n",
    "plt.legend(loc=4)\n",
    "ax.set_xlabel('log10 of C')\n",
    "ax.set_ylabel('Mean Val AUC')\n",
    "plt.title('X-validated AUC by C')\n",
    "\n",
    "e_max = res2['e'][(res2['mu']==best)].values[0]\n",
    "e_1sde = res2['e'][(res2['mu']>=best_1sde.values[0])].values.min()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<p>So now we have 2 ways to select our regularization:\n",
    "<ul>\n",
    "    <li>Choose C with $max(AUC_{xval})$</li>\n",
    "    <li>Choose min(C) where $AUC_{xval}^C>max(AUC_{xval})-stderr(max(AUC_{xval})$</li>\n",
    "</ul><br>\n",
    "This latter decision criteria is more conservative and accounts for the fact that we have variance in our cross-validated estimates of AUC. We argue that any $C$ where $AUC_{xval}^C>max(AUC_{xval})-stderr(max(AUC_{xval})$ is statistically equivalent to the max. Therefore we take the most conservative, least complex model option.<br><br>\n",
    "\n",
    "Now that we have selected a model, let's retrain on the full training set and evaluate on the test. We'll bootstrap the testing estimation so we can get a sense of the variance.\n",
    "</p>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def testAUCBoot(test, nruns, model, lab):\n",
    "    '''\n",
    "    Samples with replacement, runs multiple eval attempts\n",
    "    returns all bootstrapped results\n",
    "    '''\n",
    "    auc_res = []; oops = 0\n",
    "    for i in range(nruns):\n",
    "        test_samp = test.iloc[np.random.randint(0, len(test), size=len(test))]\n",
    "        try:\n",
    "            auc_res.append(roc_auc_score(test_samp[lab], model.predict_proba(test_samp.drop(lab,1))[:,1]))\n",
    "        except:\n",
    "            oops += 1\n",
    "    return auc_res"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "lab = 'y_buy'\n",
    "logreg_max = linear_model.LogisticRegression(C=10**e_max)\n",
    "logreg_max.fit(train.drop(lab,1),train[lab])\n",
    "auc_max = testAUCBoot(test, 100, logreg_max, lab)\n",
    "\n",
    "logreg_1sde = linear_model.LogisticRegression(C=10**e_1sde)\n",
    "logreg_1sde.fit(train.drop(lab,1),train[lab])\n",
    "auc_1sde = testAUCBoot(test, 1000, logreg_1sde, lab)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<p>Now let's look at the distribution of AUC across the bootstrapped samples. Even though we can't use the test data for model selection, we can at least look at the test results for models built with the 2 selection criteria discussed above.\n",
    "\n",
    "</p>\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.text.Text at 0x109d1a6d0>"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEZCAYAAAB4hzlwAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3XdcVfX/B/DXBRmyh6hMUciFLBc5+Io5c2RuTQXUMk37\ntrQ0NUnN8U1zZGXDxG1alNtyYbmy1NTcEggIKoiA7PX+/eHPk1dAxr1wEV7Px+M+5J7x+bw/53jP\n+57zued8VCIiICIiAqCn6wCIiKjqYFIgIiIFkwIRESmYFIiISMGkQERECiYFIiJSMCk85SZMmIC5\nc+dqpazo6GiYm5vj4a+UAwICsGrVKq2UDQC9evXCunXrtFZeac2YMQN2dnZwcHCo9LqJnjZMClWY\nq6srTExMYGFhAWtra3To0AFffvklHr215IsvvsCMGTNKVdbBgwefuIyLiwvu378PlUoFAFCpVMrf\nZRUSEoJRo0apTdu9e3ehaRUtOjoan3zyCS5fvoy4uLhC88PDw6Gnpwdzc3OYm5vDyckJISEhWqm7\nqG1w4cIFdO/eHba2trC2tkbr1q2xZ88eJRZnZ2et1F0a48ePV9ptZGQEQ0ND5X3v3r3LXF5oaCj8\n/f1LtWxwcDAMDAxw69atQtNnzpypNi0qKgp6enooKChQpm3cuBGtW7eGubk5HBwc0KtXLxw9erTM\nMVNhTApVmEqlws6dO5Gamoro6GhMnToVCxcuxNixY8tV1pPuU8zLy9Mk1CorOjoatra2sLW1LXYZ\nR0dH3L9/H/fv38eRI0ewatUqbNu2rULi6du3L3r06IHbt2/jzp07WL58OSwsLEq9fn5+vtZiWbly\npdLu999/H8OGDVPe79q1S2v1PC49PR0//PADmjdvjvXr16vNK80XkU8++QRvvfUWZsyYgTt37iAm\nJgYTJ07E9u3bKyzmGkWoynJ1dZUDBw6oTTt58qTo6enJhQsXREQkKChIZsyYISIiCQkJ0rt3b7Gy\nshIbGxvx9/eXgoICGTlypOjp6Unt2rXFzMxMPv74Y4mMjBSVSiWrVq0SFxcX6dSpk0RFRYlKpZL8\n/HwREQkICJBp06ZJ27ZtxcLCQvr16ydJSUkiInLo0CFxcnJSi61Bgwayf/9+2bNnjxgaGoqBgYGY\nmZmJj4+PiIh06tRJvvnmGxERKSgokDlz5kiDBg2kbt26EhgYKCkpKSIiSmxr1qwRFxcXqVOnjnz0\n0UfFbqfk5GQZNWqU2NnZSYMGDWTu3LlSUFAg+/btk9q1a4uenp6YmZnJ6NGjC61bVDuGDBki8+fP\nV94fPXpUWrduLZaWltKmTRs5duyYMu/mzZvSt29fsbGxEXd3d/n6669FRIrcBomJiaJSqZR2Piot\nLU2MjY2VWM3NzSUuLk5mzZolAwcOlJEjR4qFhYWsWrVKTp48Kc8++6xYWVmJvb29TJo0SXJycpSy\nVCqVLF++XBo1aiR16tSRKVOmSEFBQbHbT0Rk1qxZMnLkSOX98ePHpV27dmJlZSXe3t4SHh6uzFu9\nerU0atRIzM3NpWHDhrJhwwa5dOmSGBkZib6+vpiZmYm1tXWxda1Zs0Y8PT1l/fr10qJFC7V5wcHB\nyv/nhx7+f8jPz5fk5GQxMzOT77///ontofJjUqjCikoKIiIuLi6ycuVKEXnwIZo5c6aIiEydOlXG\njx8veXl5kpeXJ0eOHCm2rIcftKCgIMnIyJCsrCy1D5/Ig4O4o6OjXLhwQdLT05WDk0jRB9NH6wgJ\nCZFRo0apzQ8ICJBVq1aJiMiqVavE3d1dIiMjJS0tTQYMGKAs/zCOcePGSVZWlpw9e1aMjIzk0qVL\nRW6nUaNGyYsvvihpaWkSFRUljRs3VuoJDw8vFOejHm/H1atXxdHRUQ4dOiQiInfv3hUrKytZv369\n5Ofny6ZNm8Ta2lpJjv7+/jJx4kTJzs6Wv/76S+zs7OTgwYNFboOCggJ55plnpE+fPvLTTz/JrVu3\n1GIpKtZZs2aJgYGBbNu2TUREMjMz5dSpU/L7779Lfn6+REVFSbNmzWTp0qXKOiqVSp577jm5d++e\nREdHS+PGjZVkXJxHk0JsbKzY2trKnj17RERk3759YmtrK4mJiZKWliYWFhZy9epVERG5deuW8gUl\nNDRUOnbs+MR6RESee+45mTt3rqSmpoqxsbGcOnVKmVdSUtizZ4/UqlVL+T9K2sfLR08hBwcHJCUl\nFZpuaGiI+Ph4REVFQV9fHx06dCixrJCQENSuXRtGRkaF5qlUKgQGBqJ58+YwMTHBnDlzsGXLlide\nhnpIHnzhKHb+hg0b8M4778DV1RWmpqaYP38+Nm/erHbdeNasWTAyMoKXlxe8vb1x9uzZQuXk5+fj\nu+++w/z582FqaooGDRrgnXfeUTq0SxNrXFwcrK2tYWlpiSZNmuDZZ59Vtt2uXbvQpEkTjBgxAnp6\nehg2bBiaNm2K7du3IyYmBseOHcPChQthaGgIb29vvPzyy1i7dm2R20ClUuHQoUNwdXXFO++8AwcH\nB3Tq1AnXr19/Yqzt27fHCy+8AAAwNjZGy5Yt0bZtW+jp6aFBgwYYN24cDh8+rLbOe++9BysrKzg7\nO+PNN9/Epk2bStwOD61fvx69evVCz549AQBdu3ZF69atsWvXLqhUKujp6eH8+fPIzMxEvXr10Lx5\n81Jv6+joaISHh2Pw4MEwNzdHjx49lO1VGnfv3kWdOnWgp8dDV0Xhln0KxcbGwsbGRnn/8MM4ZcoU\nuLu7o3v37nBzc8PChQtLLKukjs1H57u4uCA3NxeJiYnljPxf8fHxaNCggVrZeXl5uH37tjKtfv36\nyt8mJiZIT08vVE5iYiJyc3MLlXXz5s1Sx+Lg4IB79+4hJSUFycnJMDY2RlBQEIAHCcPFxUVt+QYN\nGiAuLg7x8fGwsbGBqalpqet2dHTEp59+iuvXr+PGjRswNTVFYGDgE+NzcnJSe3/16lX06dMH9vb2\nsLS0xPTp03H37l21ZR7fb0V1shfnxo0b2Lp1K6ytrZXX0aNHcevWLZiYmOC7777DypUr4eDggD59\n+uDKlSulLnvdunVo0aIFGjduDAAYPHgwNm7cqPSV1KpVC7m5uWrr5ObmQk9PD3p6erC1tUViYqLa\nlwfSLiaFp8wff/yBuLg4dOzYsdA8MzMzLFq0CBEREdi+fTs++eQTHDp0CACK7bwrqVMvOjpa7W8D\nAwPUqVMHpqamyMjIUObl5+cjISGh1OU6ODggKipKrexatWqhXr16T1zvcXXq1IGBgUGhsh4/kJaW\nhYUFhg8fjh07dihx3rhxQ22ZGzduwNHRUTljS0tLK7LukraBk5MTXnvtNfz999/FLl9Ux+uECRPQ\nvHlzXL9+HSkpKfjoo48KHSQf32+Ojo4lNV3h4uKCUaNG4d69e8rr/v37ePfddwEA3bt3xy+//IJb\nt26hadOmeOWVV0rVXgBYu3Ytrl27Bnt7e9jb2+PNN99EYmKi0rHt4uKiti8BIDIyUkly7dq1g5GR\nEX788cdSt4fKhkmhint4FpCamoqdO3di+PDhGDVqFDw8PNTmA8DOnTtx/fp1iAgsLCygr6+vnGbX\nq1cPERERZa57/fr1uHTpEjIyMvDBBx9g8ODBUKlUaNy4MbKysrB7927k5uZi7ty5yM7OVtatX78+\noqKiir2kMHz4cCxZsgRRUVFIS0tTfv3ypMsCRZWlr6+PIUOGYPr06UhLS8ONGzewZMkSjBw5skxt\nfSgtLQ2bN29GixYtADy4t+Lq1avYtGkT8vLy8N133+Hy5cvo06cPnJyc0L59e0ybNg3Z2dk4d+4c\nvv32W6Xux7dBcnIyZs2ahYiICBQUFCAxMRHffvst2rVrB+DBPrp79y5SU1Of2Oa0tDSYm5vDxMQE\nly9fxhdffFFomUWLFiE5ORkxMTFYvnw5hg4dWuptMHLkSOzYsQO//PIL8vPzkZWVhfDwcNy8eRN3\n7tzBtm3bkJ6eDgMDA5iamkJfX1+JPzY2ttA3/YeOHz+Of/75B3/88QfOnj2Ls2fP4u+//8ZLL72k\nXEIaMGAAdu3ahX379iE/Px9xcXGYO3cuhg8fDgCwtLTE7NmzMXHiRGzbtg0ZGRnIzc3Fnj178N57\n75W6jfQEld6LQaXm6uoqtWvXFnNzc7G0tJT27dvL559/rvZLkkc7mpcsWSKurq5iamoqTk5OMnfu\nXGW5bdu2iYuLi1hZWcnixYslMjJS9PT01DrsHp8WEBAg77//vvLroxdeeEHu3r2rLB8aGir29vZS\nt25dWbRokTRs2FDpaL5796507NhRrK2tpVWrVkp5DzuACwoKZPbs2eLs7Cx2dnYyatQoSU5OLjKO\nx9d93L1792TkyJFiZ2cnzs7OMmfOHGUbHTp0SJydnYvdxuHh4covfszMzMTW1lb69OkjERERyjJH\njhyRVq1aiaWlpbRu3VqOHj2qzIuNjZU+ffqIjY2NuLm5yZdffqnMe3wbpKenS1BQkLi6uoqZmZnU\nr19fXnrpJYmLi1PWGTNmjNja2oq1tbXExcUV2WH/66+/StOmTcXMzEz8/f3lgw8+EH9/f2W+SqWS\nTz/9VBo1aiS2trYyefLkEjtmH6/n999/l06dOomNjY3Y2dlJnz59JCYmRuLj46VTp05iaWkpVlZW\n0rlzZ+UHADk5OdK7d29lnceNHz9eBg0aVGj6yZMnxdjYWO7duyciIjt27FC2d4MGDeTdd9+VrKws\ntXU2bNggrVu3FlNTU6lfv7706dNHjh8//sQ2UumoRCpmkJ0xY8Zg165dqFu3Ls6fP682b/HixZgy\nZQoSExPVro0Tkeb09PRw/fp1NGrUSNeh0FOowi4fjR49Gnv37i00PSYmBvv27VPrGCQioqqhwpKC\nv78/rK2tC01/++238b///a+iqiWq8cr7aBIiAKhVmZVt27YNTk5O8PLyqsxqiWoUbT4Kg2qeSksK\nGRkZmDdvHvbt26dMq6DuDCIiKqdKSwoRERGIioqCt7c3gAc3YLVq1QonT55E3bp11ZZ1d3cv888n\niYhqOjc3N+UO+fKqtPsUPD09cfv2bURGRiIyMhJOTk44ffp0oYQAPEgg8v+PCKiOr1mzZuk8BraP\n7atpbasJ7dPGl+kKSwrDhw9H+/btcfXqVTg7O2P16tVq89kZRkRU9VTY5aOSHsD1zz//VFTVRERU\nTnzMhQ4EBAToOoQKxfY9vapz24Dq3z5tqLA7mjVR0ihhRERUmDaOnZV6nwIR6Z6NjQ3u3bun6zBI\nA9bW1kWOqaINPFMgqmH4+Xr6FbcPtbFv2adAREQKJgUiIlIwKRARkYJJgYhqlIsXL6JNmza6DqOQ\nyZMnY+XKlboOg0mBqKazsQFUqop7VbVxtGbOnIkpU6Yo75OSktC/f3+YmZnB1dW1xBtv58yZA2dn\nZ1hZWaFz5864ePGiMi8gIAC1a9eGubk5zM3N0axZM2Xehg0blOnm5uYwNTWFnp4ezpw5A+BBUpg3\nb16xw5lWGqmCqmhYRNXC45+viv64VaWPc1xcnNjY2Eh2drYybdiwYTJs2DBJT0+XI0eOiKWlpVy4\ncKHI9bdt2yYODg4SGRkp+fn5Mm3aNGnZsqUy/0nDxj4uNDRU3N3d1aZ169ZNvv/++xLXLe4YqY1j\nJ88UiKhKcHV1xaJFi+Dl5QVzc3OMHTsWt2/fxvPPPw9LS0t069YNycnJAIDBgwfD3t4eVlZW6NSp\nk/JtPScnB76+vlixYgWAB2NLdOjQAXPnzgUA7Nu3D61atYKhoSEAID09HWFhYZgzZw5MTEzQoUMH\n9OvXD+vWrSsyxgsXLqBjx45wdXWFnp4eRowYoXamAJR+SIDQ0FAEBgaqTQsICMCuXbtKucUqBpMC\nEVUJKpUKYWFhOHDgAK5cuYKdO3fi+eefx4IFC3Dnzh0UFBRg+fLlAIBevXrh+vXrSEhIQMuWLTFi\nxAgAgKGhIdavX48PPvgAly9fxoIFCyAimD59OgDg/PnzaNKkiVLn1atXUatWLbi7uyvTvL29ceHC\nhSJj7NKlC44fP45r164hNzcXa9aswfPPP6+2zLRp02BnZ4eOHTvi8OHDRZZz48YN/Pbbb4WSQtOm\nTXH27Nkybjnt4h3NRFRlvP7667CzswPwYEjfevXqKWOw9O/fHwcOHADwYAz4h2bNmoVly5bh/v37\nMDc3h4eHB2bMmIF+/fohMTERJ0+eVJ7KnJKSAltbW2XdtLQ0WFhYqMVgbm6O+/fvFxlf27ZtERQU\nhCZNmkBfXx8uLi5KTACwcOFCeHh4wNDQEJs2bULfvn3x119/oVGjRmrlrF27Fv/5z38KjVVvbm6u\nnA3pCs8UiKjKqFevnvJ37dq11d4bGxsjLS0NBQUFmDp1Ktzd3WFpaYmGDRtCpVIhMTFRWTYwMBDR\n0dHo1asX3NzclOnW1tZqB3wzMzOkpqaqxZCSkgJzc/Mi41uxYgUOHDiA2NhYZGdn44MPPsBzzz2H\nzMxMAA+ShqmpKQwMDBAYGIgOHTpg9+7dhcpZu3YtgoKCCk2/f/8+rKysStpMFYpJgYiqrKKuz2/c\nuBHbt2/HgQMHkJKSgsjISGWQmYdee+019OnTB3v37sXRo0eV6V5eXrh69aryvnHjxsjLy1Mbrezs\n2bNo0aJFkfHs3bsXw4cPh4ODA/T09BAUFIR79+7h0qVLpW7T0aNHER8fj0GDBhWad+nSJfj4+JS6\nrIrApEBET5X79+/DyMgINjY2SE9Px/vvv682f926dThz5gzWrFmD5cuXIygoCOnp6QCArl274vTp\n08jJyQEAmJqaYsCAAfjggw+QkZGBI0eOYMeOHRg1alSRdXt5eWHLli1KH8e6deuQl5cHd3d3pKSk\n4Oeff0ZWVhby8vKwYcMG/Pbbb+jZs6daGWvWrMGgQYNgampaqPzDhw8X6qOobEwKRDWctXXF3qdg\nbV3+2B4doVGlUkGlUiEwMBANGjSAo6MjWrRogXbt2inLRUdH46233sLatWthYmKC4cOHo3Xr1nj7\n7bcBPLg89dxzz+Gnn35Syv3888+RmZmJunXrYuTIkVi5cqVyf0F0dDTMzc0RGxsLAJgxYwaaNGkC\nLy8vWFtbY9myZfjhhx9gYWGB3NxczJw5E3Xr1oWdnR0+++wzbNu2Ta0TOysrC1u3bi3y0lF8fDwu\nXbqEF198sfwbTAv4lFSiGqamf74uXbqEoKAgnDx5UtehqJk8eTLc3d0xfvz4EpetyKekMikQ1TD8\nfD39+OhsIiKqFEwKRESkYFIgIiIFkwIRESkqNCmMGTMG9erVg6enpzJtypQpaNasGby9vTFgwACk\npKRUZAhERFQGFZoURo8ejb1796pN6969Oy5cuICzZ8+icePGmD9/fkWGQEREZVChScHf3x/Wj925\n0q1bN+jpPajWz89PuSmEiIh0T6d9Ct9++y169eqlyxCIKlxMDGBsDOjra/7avFnXranagoODMXPm\nzFIvP23aNCxbtqwCIyq77OxsNGvWTO0Bf5VJZ4/O/uijj2BoaIiXXnqpyPkhISHK3wEBAQgICKic\nwIi0LC0NcHUFinlEf6lNmgTcvauVkNTYLLTBvax72i/4/1kbWyPpvaQSl1uxYgVCQ0Px999/Y/jw\n4Vi9enWZ63r4KIzSSEhIwLp16xAREaFMO3DgACZOnIiYmBj4+fkhNDQULi4uRa4fGxuLCRMm4Nix\nYzA0NMSgQYOwdOlS6Ovrqy03e/ZshISEYP/+/XjuuecAAHl5eXjrrbfw/fffIycnBx06dMDKlSvh\n4OAAIyMjjBkzBgsWLMCiRYue2Ibw8HCEh4eXqr2lpvHYbSWIjIyUFi1aqE1bvXq1tG/fXjIzM4tc\npxLCIqo0Fy+KNG2qeTmvvSayYoXm5Tz++UJIxX7eSlt+WFiY/PTTTzJhwgQJDg4uV13BwcEyY8aM\nUi37v//9T8aNG6e8T0hIEEtLS/n+++8lOztbpkyZIs8++2yx6/fv31+Cg4MlOztbbt26JZ6enrJ8\n+XK1Za5fvy6enp7i6OgoBw4cUKYvW7ZMvL295c6dO5KVlSWBgYEyYMAAZX5MTIzUqVNHcnJyiqy7\nuGOkNo6dlX75aO/evfj444+xbds2GBsbV3b1RFRF9e/fH/369VMbBAcAEhMT0adPH1hbW8PW1hb/\n+c9/lEc5nDlzBi1btoSFhQWGDRuGrKwstXV37twJHx8fWFtbo0OHDjh//rwyb+/evejUqZPyPiws\nDC1atMDAgQNhaGiIkJAQnD17Vu1R24+6cOEChg4dCkNDQ9SrVw89e/YsNGLbpEmTsHDhQhgYGBRa\nt0ePHrCzs4ORkRGGDBmitq6TkxOsra1x/PjxMmxB7ajQpDB8+HC0b98eV65cgbOzM7799lu8/vrr\nSEtLQ7du3eDr64vXXnutIkMgoqeMPPbsnsWLF8PZ2RmJiYm4c+cO5s+fD5VKhZycHLz44ovKmAaD\nBw/GDz/8oFw+OnPmDMaOHYuvv/4aSUlJePXVV/HCCy8gNzcXQOGhOS9cuKCM8gYAJiYmcHd3x99/\n/11knD169MDGjRuRmZmJmzdvYs+ePWqPvd66dSuMjY2LfBR29+7dsWfPHsTHxyMjIwMbNmwo1L/a\nrFkznQzNWaF9Cps2bSo0bcyYMRVZJRE95R7vEzA0NER8fDyioqLg5uaGDh06AABOnDiBvLw8vPHG\nGwCAgQMHok2bNsp6X331FV599VVlWmBgIObNm4cTJ07A398fycnJaiOspaenK0OBPmRhYYG0tLQi\n4wwJCUHXrl1hYWGB/Px8BAcHo1+/fgAejPkwffp07N+/v8h1Bw4ciO3bt8PR0RH6+vrw8vLCZ599\npraMrobm5B3NRFSlPH6mMGXKFLi7u6N79+5wc3PDwoULAQBxcXFwdHRUW/bRMY9v3LiBxYsXw9ra\nWnnFxsYiLi4OgGZDc4oIevTogcGDByMjIwOJiYlISkrCe++9B+BBwhg1apRaJ/Wj7Zo8eTLu37+P\npKQkpKeno3///oXOKO7fv1/oJ/2VgUmBiKqUx88UzMzMsGjRIkRERGD79u345JNPcPDgQTg4OODm\nzZtqy964cUP528XFBdOnT8e9e/eUV1paGoYOHQrgwShqV65cUZb38PBQu1yTnp6OiIgIeHh4FIox\nMTERp06dwqRJk2BgYAAbGxsEBwcr4zEfPHgQy5cvh729Pezt7RETE4MhQ4bg448/BvCgP2P06NGw\nsrKCoaEhJk2ahJMnTyIp6d9faV26dEntclZlYVIgoiohPz9fGcoyPz8f2dnZyMvLw65du3D9+nWI\nCCwsLKCvrw99fX20a9cOtWrVwvLly5Gbm4uwsDD88ccfSnmvvPIKVq5ciZMnT0JEkJ6ejl27dimX\ng3r16oXDhw8ry/fv3x9///03wsLCkJWVhQ8//BA+Pj5o3LhxoVjr1KkDe3t7fPHFF8jPz0dycjLW\nrFmjHMQPHDigPLnhr7/+goODA7766itMnDgRwIOEtGbNGqSmpiI3Nxeff/45HB0dYWNjAwC4efMm\nkpKS8Oyzz1bY9i6Ozu5TIKKqwdrYGqoPS/fb/vKWXxpz5szB7Nmzlffr16/HrFmzYGlpiUmTJiEh\nIQHW1taYOHGi8quhsLAwvPLKK5gxYwZ69eqFgQMHKuu3atUKX3/9NSZNmoRr166hdu3a8Pf3V9YN\nDAyEj48PsrKyYGxsjDp16uCHH37ApEmTMHLkSDz77LPY/MjdgvPmzcORI0ewe/duqFQqhIWFYfLk\nyZg/fz5q1aqFLl26YMmSJQCgHNwf0tfXh7W1NUxMTAAAS5Ysweuvvw43Nzfk5eXB09MTP/74o7L8\nxo0bERwcXOhXS5WBI68RVbBLl4ABAx78q4mJE4HmzR/8qwl+vv41ffp01K1bV+msrgqys7Ph4+OD\n3377DXXq1ClymYoceY1nCkRUY3300Ue6DqEQIyMjXNL0G4QG2KdAREQKnikQPcHkXyZjz/U9GpWR\nnQ0k+LUBEKqVmIgqEpMC0ROcjj+Nd9q9Az9Hv3KXEX4uAm/HvKfFqIgqDpMCUQlcrVzhUbfwb9VL\nK9KcV2np6cH/rUREpOCZAlENY21tXeoxB6hqqsjHXzApENUwjz5KgehxvHxEREQKnilQtXT+PPDG\nG4CmN+7+5Q28/RNgrcETjBNVAHw1i4OosjApULV07RqQmws88iidcnn7LDDyP0BLDS7h3sgA5hQ9\neBdRlcOkQNWWnR3QubNmZVhHA76+QOeG5S/jUgJgGKVZHESVhX0KRESkYFIgIiIFkwIRESmYFIiI\nSMGkQERECiYFIiJSVNhPUseMGYNdu3ahbt26OH/+PIAHt9cPHToUN27cgKurK7Zs2QIrK6uKCoFq\nsPjs6/jT+UOMDNPs7rULCRegAp8TRDVHhSWF0aNH4/XXX0dgYKAybcGCBejWrRveffddLFy4EAsW\nLMCCBQsqKgSqwW5kncN940vo6f6mRuX0fqY3/JzKP5YC0dOmwpKCv78/oqKi1KZt374dhw8fBgAE\nBQUhICCASYEqTO0cF4z0GqnrMIieKpXap3D79m3Uq1cPAFCvXj3cvn27MqsnIqIS6OwxFyqV6onP\ndA8JCVH+DggIQEBAQMUHRUT0FAkPD0d4eLhWy6zUpFCvXj3cunUL9evXR3x8POrWrVvsso8mBSIi\nKuzxL8wffvihxmVW6uWjF154AWvWrAEArFmzBi+++GJlVk9ERCWosKQwfPhwtG/fHleuXIGzszNW\nr16NqVOnYt++fWjcuDEOHjyIqVOnVlT1RERUDhV2+WjTpk1FTt+/f39FVUlERBriHc1ERKRgUiAi\nIgWTAhERKZgUiIhIwaRAREQKJgUiIlIwKRARkYJJgYiIFEwKRESkYFIgIiIFkwIRESmYFIiISMGk\nQERECiYFIiJS6Gw4TqKaJCUrBevOrtOojCvGgHNBDwDFj1hIpCkmBaIK5mjhiOfdn8cv//yiUTl/\nmvwO2+z7AF7TTmBERWBSIKpgFkYWWNVvlcbleJ6YCIFoISKi4rFPgYiIFEwKRESkYFIgIiIFkwIR\nESmYFIillAAxAAAaCklEQVSISMGkQEREihKTQlpaGvLz8wEAV65cwfbt25Gbm6tRpfPnz4eHhwc8\nPT3x0ksvITs7W6PyiIhIO0pMCv/5z3+QnZ2NmzdvokePHli3bh2Cg4PLXWFUVBS+/vprnD59GufP\nn0d+fj42b95c7vKIiEh7SkwKIgITExOEhYXhtddew9atW/H333+Xu0ILCwsYGBggIyMDeXl5yMjI\ngKOjY7nLIyIi7SlVn8Lx48exYcMG9O7dGwBQUFBQ7gptbGzwzjvvwMXFBQ4ODrCyskLXrl3LXR4R\nEWlPiY+5WLp0KebPn4/+/fvDw8MDERER6Ny5c7krjIiIwNKlSxEVFQVLS0sMHjwYGzZswIgRI9SW\nCwkJUf4OCAhAQEBAueskIqqOwsPDER4ertUyVSJSqQ9T+e6777Bv3z588803AIB169bhxIkT+Oyz\nz/4NSqVCJYdF1cy7q8Ow/tx6xC0J03UoWuP53kQ0s2uOLZMn6joUqqK0cews8UzhypUrWLRoEaKi\nopCXl6dUfPDgwXJV2LRpU8yZMweZmZkwNjbG/v370bZt23KVRdVTTEoM7mXd06iMOzlR2gmGqIYp\nMSkMHjwYEyZMwMsvvwx9fX2NK/T29kZgYCBat24NPT09tGzZEuPGjdO4XKo+OnzbAaaGpjDQMyh3\nGSmpgFXmEC1GRVQzlJgUDAwMMGHCBK1W+u677+Ldd9/VaplUfeQW5OJg4EHYm9uXu4ywMGD971oM\niqiGKDEp9O3bF5999hkGDBgAIyMjZbqNjU2FBkZEhaXdB65d06wMU1PAwUE78VD1U2JSCA0NhUql\nwqJFi9SmR0ZGVlhQRFSYlTXwx29Ar42alXPjBpCTo52YqPopMSlERUVVQhhEVBIvT2DYc8BEDX+X\noVJpJx6qnkpMCmvWrIGqiP9FgYGBFRIQERHpTolJ4Y8//lCSQmZmJg4ePIiWLVsyKRARVUMlJoUV\nK1aovU9OTsbQoUMrLCAiItKdMo+nYGJiwk5mIqJqqlQ/SX2ooKAAFy9exJAhvCmISBcycjOQnJWs\nWSEGhgBMtBIPVT8lJoV33nnn34Vr1YKrqysfdU2kA04WTvjot4/w0W8faVbQlFwA6VqJiaqfEpOC\nh4cH6tSpozxoadOmTZg3b55GYyoQUdlN85+Gaf7TNC5H9SF/k0rFK7ZPISwsDHXq1IGXlxecnZ2x\nfft2tGrVClu2bMHatWsrM0aqgXJyH9xgVd6XhiPGEtVYxZ4pzJo1CydOnIC7uztOnToFPz8//Pjj\nj2p9DEQVISEBcHMD9DS8wjF2rHbiIapJik0KtWrVgru7OwCgVatWaNq0KRMCVQopAE79Cfi46ToS\nopqn2KSQkJCATz75RBmwITk5WXmvUqnw9ttvV1qQRERUOYpNCi+//DLu379f7HsiIqp+ik0Kj46R\nTERENUOZ72gmIqLqi0mBiIgUJSaFf/75p1TTiIjo6VdiUhg4cGChaYMHD66QYIiISLeK7Wi+dOkS\nLl68iJSUFISFhSk/RU1NTUVWVlZlxkhERJWk2KRw9epV7NixAykpKdixY4cy3dzcHF9//XWlBEdE\nRJWr2KTQr18/9OvXD8ePH0e7du0qMyYiItKREvsUvvjiCyQn//v89nv37mHMmDEaVZqcnIxBgwah\nWbNmaN68OU6cOKFReUREpB0lPjr73LlzsLKyUt5bW1vj9OnTGlX6xhtvoFevXvj++++Rl5eH9HQ+\n252IqCoo8UxBRJCUlKS8T0pKQn5+frkrTElJwW+//aacbdSqVQuWlpblLo+IiLSnVCOvtWvXDkOG\nDIGIYOvWrZg+fXq5K4yMjISdnR1Gjx6Ns2fPolWrVli2bBlMTDg8IBGRrpWYFAIDA9GqVSscPHgQ\nKpUKP/74I5o3b17uCvPy8nD69GmsWLECbdq0wZtvvokFCxZg9uzZass9+uylgIAABAQElLtOIqLq\nKDw8HOHh4VotUyUPn439mMzMTKxcuRLXr1+Hl5cXxowZAwMDA40rvHXrFtq1a4fIyEgAwJEjR7Bg\nwQLs3Lnz36D+f+hPqpn037XHqVdPw8fNXtehVEuqD1WQWfx8VUfaOHYW26cQFBSEU6dOwdPTE7t3\n78bkyZM1quih+vXrw9nZGVevXgUA7N+/Hx4eHlopm4iINPPEO5rPnz8P4MFYCm3atNFapZ9++ilG\njBiBnJwcuLm5YfXq1Vorm4iIyu+Jw3EW9bc2eHt7448//tBqmUREpLlij/bnzp2Dubm58j4zM1N5\n//AZSEREVL0UmxQ0uReBiIieThxkh4iIFNrtLKAabc+1PVh8fLHG5RQY34Uev68Q6QSTAmnNqfhT\nqGdWD6N9RmtUzpklH8JuQj0tRUVEZcGkQFrV0KohujbqqlEZhre0FAwRlRnP0YmISMGkQERECiYF\nIiJSMCkQEZGCSYGIiBRMCkREpGBSICIiBZMCEREpmBSIiEjBpEBERAomBSIiUjApEBGRgkmBiIgU\nfEoqaU18PHD6HJC9R7NyONJrxZvyyxSNy+jTuA86uXbSQjRUlTApkNZcuQLE3wLqNtWsnIULgXoc\nTqHibPsGdbsmaVTE0Zij2HJhC5NCNcSkQFrl5AhM0fxLKFWkM2MxpYNmRZicNMHFhIvaiYeqFPYp\nEBGRQmdJIT8/H76+vujbt6+uQiAiosfoLCksW7YMzZs3h0ql0lUIRET0GJ0khdjYWOzevRsvv/wy\nREQXIRARURF0khTeeustfPzxx9DTY5cGEVFVUum/Ptq5cyfq1q0LX19fhIeHF7tcSEiI8ndAQAAC\nAgIqPDYioqdJeHj4E4+j5VHpSeHYsWPYvn07du/ejaysLKSmpiIwMBBr165VW+7RpEBERIU9/oX5\nww8/1LjMSk8K8+bNw7x58wAAhw8fxqJFiwolBCKqWCtWaLb+r1mAHm8wrJZ0fvMaf31EVLk++AC4\nfFmzMk4nAbWdtRMPVS06TQqdOnVCp068TZ6oMmnhCgOGLAIuJWheDlU9/PkPEREpmBSIiEjBpEBE\nRAomBSIiUjApEBGRgkmBiIgUTApERKTQ+c1rRPR0Sr0P7NihWRkGBkD37gCfjVl1MCkQUZm5OAMn\nI4GvvtKsnEOHgFOngCZNtBMXaY5JgYjKrGFDoLcZ8Flvzcpp1gwoKNBOTKQdPGkjIiIFkwIRESl4\n+YgAAKfiTiE9N12jMpIRBWPU11JERKQLTAqE1OxU+H3jh/bO7TUqJ1GA1tJLS1ERkS4wKRAKpABm\nhmb4dfSvGpUzdy6QlauloIhIJ9inQERECiYFIiJSMCkQEZGCSYGIiBRMCkREpGBSICIiBZMCEREp\nmBSIiEihk6QQExODzp07w8PDAy1atMDy5ct1EQYRET1GJ3c0GxgYYMmSJfDx8UFaWhpatWqFbt26\noVmzZroIh4iI/p9OzhTq168PHx8fAICZmRmaNWuGuLg4XYRCRESP0HmfQlRUFM6cOQM/Pz9dh0JE\nVOPp9IF4aWlpGDRoEJYtWwYzMzO1eSEhIcrfAQEBCAgIqNzgiKhSpKYCycmalWFiAhgaaieep0l4\neDjCw8O1WqbOkkJubi4GDhyIkSNH4sUXXyw0/9GkQETVk6sr0KOHZmXk5gJdugDbt2slpKfK41+Y\nP/zwQ43L1ElSEBGMHTsWzZs3x5tvvqmLEKqNAilAVl6WRmVk5GZoKRqistmzR/MyDh588Nh20g6d\nJIWjR49i/fr18PLygq+vLwBg/vz56Nmzpy7CeapN3DURq86sQi09zXZlI+tGWoqIiJ5mOkkKHTt2\nREFBgS6qrnZSc1Kxut9qjPAaoetQiKga0Pmvj4iIqOpgUiAiIgWTAhERKZgUiIhIwaRAREQKnd7R\nTERPJ3Mjc3x1+iusObtGo3IM9Q1xfsJ5OFo4aiky0hSTAhGV2SivURjQbIDG5bT6qhVSs1PhCCaF\nqoJJgYjKTKVSwczQrOQFS6Cn4hXsqoZ7hIiIFEwKRESkYFIgIiIF+xR05HrSdXRf1x35kq9ROYkZ\niejftL+WoiKimo5JQUcS0hNgZWyFH4f+qFE5KpUKThZOWoqKiGo6JgUdMqplhAZWDXQdBpHOGOob\noueGnjDUL/+waZkZgMqtN4Cl2gusBmNSICKdORh4EEmZSRqVseqX4/gyPlQ7ARGTAhHpjq2JLWxN\nbDUqw84oRkvREMCkoDNx8cC5c4CXl64j0Z7bt4HXXtN1FESkCSYFHbl7FxAB1q/XdSTa5eam6wiI\nSBNMCjqkr1+9zhSI6OnHm9eIiEjBpEBERAomBSIiUuikT2Hv3r148803kZ+fj5dffhnvvfdepdR7\nIvYEpu6fWil1lSQuKQUqMdV1GEREaio9KeTn52PSpEnYv38/HB0d0aZNG7zwwgto1qxZhdd94c4F\nmBiY4N0O71Z4XU/y14m/YNDEB9/sdNZpHBUlPDwcAQEBug6jwlTn9lXntgHVv33aUOlJ4eTJk3B3\nd4erqysAYNiwYdi2bVulJAUAsDezR4BrQKXUVZzw0HD06BEA40ydhlFhqvsHrzq372lsmwp6SDP7\nCz3W9yhx2eth1+Ee615MOSos7r4YHnU9tB3iU6XSk8LNmzfh7PzvN2QnJyf8/vvvlR0GEVUTHhbt\n4XFxC94OLvmJw+tOrsOoZ0cVOW/2r7Nx5e4VJoXKrlClUlV2lQpbE1ucPtQAfb/RWQgAgCtXgH37\nAD128xNpzKiWIWIOd8WKt0pe9sqV40iJLPqMIqP5Plh1ttJydE8hqWTHjx+XHj16KO/nzZsnCxYs\nUFvGzc1NAPDFF1988VWGl5ubm8bHaJWICCpRXl4emjRpggMHDsDBwQFt27bFpk2bKq1PgYiIilfp\nl49q1aqFFStWoEePHsjPz8fYsWOZEIiIqohKP1MgIqKqq9K7Ovfu3YumTZvimWeewcKFCwvNX7Ro\nEXx9feHr6wtPT0/UqlULycnJpVq3KtCkfa6urvDy8oKvry/atm1b2aGXqKS2JSYmomfPnvDx8UGL\nFi0QGhpa6nWrAk3aV9X3HVBy++7du4f+/fvD29sbfn5+uHDhQqnXrQo0aV9V339jxoxBvXr14Onp\nWewy//3vf/HMM8/A29sbZ86cUaaXed9p3CtRBnl5eeLm5iaRkZGSk5Mj3t7ecvHixWKX37Fjh3Tp\n0qVc6+qCJu0TEXF1dZW7d+9WRqhlVpq2zZo1S6ZOnSoiIgkJCWJjYyO5ubnVZt8V1z6Rqr3vRErX\nvsmTJ8vs2bNFROTy5cvV7rNXXPtEqv7++/XXX+X06dPSokWLIufv2rVLnn/+eREROXHihPj5+YlI\n+fZdpZ4pPHrjmoGBgXLjWnE2btyI4cOHl2tdXdCkfQ9JFb2aV5q22dvbIzU1FQCQmpoKW1tb1KpV\nq9rsu+La91BV3XdA6dp36dIldO7cGQDQpEkTREVF4c6dO9Vm/xXVvoSEBGV+Vd5//v7+sLa2Lnb+\n9u3bERQUBADw8/NDcnIybt26Va59V6lJoagb127evFnkshkZGfj5558xcODAMq+rK5q0D3hwD0fX\nrl3RunVrfP311xUeb1mUpm2vvPIKLly4AAcHB3h7e2PZsmWlXlfXNGkfULX3HVC69nl7eyMsLAzA\ng4PsjRs3EBsbW232X3HtA6r+/itJce2Pi4sr876r1F8fleXGtR07dqBjx46wsrIq87q6okn7AODo\n0aOwt7dHQkICunXrhqZNm8Lf378iQi2z0rRt3rx58PHxQXh4OCIiItCtWzecPXu2EqLTnCbtMzc3\nr9L7Dihd+6ZOnYo33nhD6e/y9fWFvr5+tfnsFdc+ADhy5AgcHByq7P4rDW2d6VTqmYKjoyNiYv4d\nZDsmJgZOTk5FLrt582a1SytlWVdXNGkf8ODyBADY2dmhf//+OHnyZMUFW0aladuxY8cwePBgAICb\nmxsaNmyIK1euwMnJqVrsu+LaB1TtfQeUrn3m5ub49ttvcebMGaxduxYJCQlwc3OrNp+9otrXqFEj\nAICDgwOAqrv/SvJ4+2NjY+Hk5FS+fae9rpCS5ebmSqNGjSQyMlKys7OL7fRITk4WGxsbycjIKPO6\nuqRJ+9LT0yU1NVVERNLS0qR9+/by888/V1rsJSlN29566y0JCQkREZFbt26Jo6Oj3L17t9rsu+La\nV9X3nUjp2pecnCzZ2dkiIvLVV19JUFBQqdfVNU3a9zTsPxGRyMjIUnU0Hz9+XOloLs++q/THXOze\nvVsaN24sbm5uMm/ePBERWblypaxcuVJZJjQ0VIYPH16qdaua8rbvn3/+EW9vb/H29hYPD48q2b6S\n2paQkCB9+vQRLy8vadGihWzYsOGJ61Y15W1fREREld93IiW379ixY9K4cWNp0qSJDBw4UJKTk5+4\nblVT3vY9DZ+9YcOGib29vRgYGIiTk5OsWrWq0HFl4sSJ4ubmJl5eXnLq1Clleln3HW9eIyIiBZ/T\nSURECiYFIiJSMCkQEZGCSYGIiBRMCkREpGBSICIiBZMCaVVUVNQTH+9bFkuXLkVmZuYTl1mzZg3i\n4+O1Up82hIaG4vXXX1ebtnr1auVx6YaGhsojmt9///1Sl1vStkhMTISBgQG+/PJLtelmZmZPjG/t\n2rXw9PSEl5cXWrZsicWLF5c6JqqemBSoylq2bBkyMjKeuExoaCji4uKKnFdQUFARYZXZ6NGjcebM\nGZw5cwaOjo4IDw/HmTNnMG/evFKXUdK22Lp1K3r27IlNmzapTX/8mUCPvt+zZw+WLVuGffv24dy5\nczhx4gQsLS1LHRNVT0wKpHV5eXkYOXIkmjdvjsGDByvfcA8cOICWLVvCy8sLY8eORU5OTrHTly9f\njri4OHTu3BldunRBQUEBgoODlW+1S5cuxQ8//IA///wTI0aMQMuWLZGVlQVXV1dMnToVrVq1wtat\nW/HNN9+gbdu28PHxwaBBg5RYgoODMX78eLRp0wZNmjTBrl27ADxIMv369UPnzp3RuHFjzJ49W2nX\n+vXr4efnB19fX4wfP15JOqtXr0aTJk3g5+eHY8eOlXo7ffzxx2jbti28vb0REhICAEhPT0fv3r3h\n4+MDT09PbNmyBZ9++qnatijK5s2bMXfuXNy5c6fUTzCdP38+Fi9ejPr16wMADA0N8fLLL5c6fqqm\nKuCObKrBIiMjRaVSybFjx0REZMyYMbJo0SLJzMwUZ2dnuXbtmoiIBAYGytKlS4udLqI+8Mmff/4p\n3bp1U+pJSUkREZGAgAC1W/pdXV3l448/Vt4/OnDKjBkz5NNPPxURkaCgIOVZMdeuXRMnJyfJysqS\n1atXi729vSQlJUlmZqa0aNFC/vzzT7l48aL07dtX8vLyRERkwoQJsnbtWomLixMXFxdJTEyUnJwc\n6dChg7z++uvFbp+Hbfr5559l3LhxIiKSn58vffr0kV9//VV++OEHeeWVV5TlHz6T50mDwERHR0vT\npk1FRGTmzJmyePFiZZ6ZmZnasqGhoTJp0iQREbGxsVHKJ3qIZwqkdc7OzmjXrh0AYOTIkThy5Aiu\nXr2Khg0bwt3dHQAQFBSEX3/9tdjpj3Nzc8M///yD//73v/j5559hbm6uzJPHntQydOhQ5e/z58/D\n398fXl5e2LBhAy5evAjgwWWUIUOGAADc3d3RqFEjXL58GSqVCt27d4e1tTWMjY0xYMAAHDlyBAcO\nHMCpU6fQunVr+Pr64tChQ4iMjMTJkycREBAAW1tbGBgYYOjQoSU+wlhE8Msvv+CXX36Br68vWrVq\nhStXruD69evw9PTEvn37MHXqVBw5ckStncX57rvvMGjQIADA4MGDC11CetzT8Chs0p1KHU+BaoZH\nDzoiUuRBqLgDZ3HLW1lZ4dy5c9i7dy9WrlyJLVu2YNWqVYXqAwBTU1Pl7+DgYGzfvh2enp5Ys2YN\nwsPDSxV3UfEEBQUV6gd4fBSrkhLCo6ZNm4Zx48YVmn7mzBns2rULM2bMQJcuXTBz5swnlrNp0ybc\nvn0b69evBwDEx8cjIiICbm5uqF27NnJzc2FgYAAAuHv3Luzs7AAAHh4e+PPPP5XRyIgA9ilQBYiO\njsaJEycAPBhy1N/fXxn+MCIiAgCwbt06BAQEFDm9U6dOAB48//7h8Jd3795FXl4eBgwYgDlz5igD\nkz+6TFHS0tJQv3595ObmYv369coBXkSwdetWiAgiIiLwzz//oGnTphAR7Nu3D/fu3UNmZia2bduG\njh07okuXLvj++++V4RuTkpIQHR0NPz8/HD58GElJScjNzcXWrVtL3D4qlQo9evTAt99+i/T0dAAP\nRs5KSEhAfHw8jI2NMWLECEyePLnEdl69ehXp6emIjY1FZGQkIiMjMXXqVGzcuBEA0KlTJyVZZGZm\nYuvWrUoSmDZtGqZMmYLbt28DAHJycpRESzUXzxRIq1QqFZo0aYLPPvsMY8aMgYeHByZMmAAjIyOs\nXr0agwcPRl5eHtq2bYvx48fDwMCgyOkAMG7cOPTs2ROOjo5YsmQJRo8erXTuLliwAMC/HcYmJiZF\ndvLOmTMHfn5+sLOzg5+fH9LS0pQ4XVxc0LZtW6SmpuLLL7+EoaEhVCoV2rZti4EDByI2NhajRo1C\ny5YtAQBz585F9+7dUVBQAAMDA3z++edo27YtQkJC0K5dO1hZWcHX1/eJl2cezuvWrRsuXbqkXGYz\nNzfHunXrcP36dUyZMgV6enowMDDAypUrC22LAwcOKOVt3rwZAwYMUKtj4MCBGDZsGGbOnIlly5bh\n1VdfxfLlyyEiCAoKQseOHQEAzz//PG7fvo2uXbsqZ0Rjx44t4x6n6oaPzqYaafTo0ejbt2+hA2po\naChOnTqFTz/9VEeREekWLx8RPUKlUrEjlmo0nikQEZGCZwpERKRgUiAiIgWTAhERKZgUiIhIwaRA\nREQKJgUiIlL8H4SItJma3XSWAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x108f34550>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "r = [min(min(auc_max), min(auc_1sde)),max(max(auc_max), max(auc_1sde))]\n",
    "b = 15\n",
    "fig = plt.figure()\n",
    "frame = plt.gca()\n",
    "ax = fig.add_subplot(111)\n",
    "h1 = plt.hist(auc_max, range=r, bins=b, normed=True, histtype='step',stacked=True,label='max({})'.format(round(mean(auc_max),3)))\n",
    "h2 = plt.hist(auc_1sde,range=r, bins=b, normed=True, histtype='step',stacked=True,label='1sde({})'.format(round(mean(auc_1sde),3)))\n",
    "plt.legend()\n",
    "\n",
    "ax.set_xlabel('bootstrapped Test AUC')\n",
    "ax.set_ylabel('Pct Runs')\n",
    "plt.title('Distribution of BootStrap Test AUC')\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}