{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 222,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import statsmodels.formula.api as smf\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "from sklearn import model_selection, metrics, linear_model, decomposition, cross_decomposition\n",
    "from itertools import combinations\n",
    "import numpy as np\n",
    "import time "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 236,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>AtBat</th>\n",
       "      <th>Hits</th>\n",
       "      <th>HmRun</th>\n",
       "      <th>Runs</th>\n",
       "      <th>RBI</th>\n",
       "      <th>Walks</th>\n",
       "      <th>Years</th>\n",
       "      <th>CAtBat</th>\n",
       "      <th>CHits</th>\n",
       "      <th>CHmRun</th>\n",
       "      <th>CRuns</th>\n",
       "      <th>CRBI</th>\n",
       "      <th>CWalks</th>\n",
       "      <th>League</th>\n",
       "      <th>Division</th>\n",
       "      <th>PutOuts</th>\n",
       "      <th>Assists</th>\n",
       "      <th>Errors</th>\n",
       "      <th>Salary</th>\n",
       "      <th>NewLeague</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>315</td>\n",
       "      <td>81</td>\n",
       "      <td>7</td>\n",
       "      <td>24</td>\n",
       "      <td>38</td>\n",
       "      <td>39</td>\n",
       "      <td>14</td>\n",
       "      <td>3449</td>\n",
       "      <td>835</td>\n",
       "      <td>69</td>\n",
       "      <td>321</td>\n",
       "      <td>414</td>\n",
       "      <td>375</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>632</td>\n",
       "      <td>43</td>\n",
       "      <td>10</td>\n",
       "      <td>475.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>479</td>\n",
       "      <td>130</td>\n",
       "      <td>18</td>\n",
       "      <td>66</td>\n",
       "      <td>72</td>\n",
       "      <td>76</td>\n",
       "      <td>3</td>\n",
       "      <td>1624</td>\n",
       "      <td>457</td>\n",
       "      <td>63</td>\n",
       "      <td>224</td>\n",
       "      <td>266</td>\n",
       "      <td>263</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>880</td>\n",
       "      <td>82</td>\n",
       "      <td>14</td>\n",
       "      <td>480.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>496</td>\n",
       "      <td>141</td>\n",
       "      <td>20</td>\n",
       "      <td>65</td>\n",
       "      <td>78</td>\n",
       "      <td>37</td>\n",
       "      <td>11</td>\n",
       "      <td>5628</td>\n",
       "      <td>1575</td>\n",
       "      <td>225</td>\n",
       "      <td>828</td>\n",
       "      <td>838</td>\n",
       "      <td>354</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>200</td>\n",
       "      <td>11</td>\n",
       "      <td>3</td>\n",
       "      <td>500.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>321</td>\n",
       "      <td>87</td>\n",
       "      <td>10</td>\n",
       "      <td>39</td>\n",
       "      <td>42</td>\n",
       "      <td>30</td>\n",
       "      <td>2</td>\n",
       "      <td>396</td>\n",
       "      <td>101</td>\n",
       "      <td>12</td>\n",
       "      <td>48</td>\n",
       "      <td>46</td>\n",
       "      <td>33</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>805</td>\n",
       "      <td>40</td>\n",
       "      <td>4</td>\n",
       "      <td>91.5</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>594</td>\n",
       "      <td>169</td>\n",
       "      <td>4</td>\n",
       "      <td>74</td>\n",
       "      <td>51</td>\n",
       "      <td>35</td>\n",
       "      <td>11</td>\n",
       "      <td>4408</td>\n",
       "      <td>1133</td>\n",
       "      <td>19</td>\n",
       "      <td>501</td>\n",
       "      <td>336</td>\n",
       "      <td>194</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>282</td>\n",
       "      <td>421</td>\n",
       "      <td>25</td>\n",
       "      <td>750.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   AtBat  Hits  HmRun  Runs  RBI  Walks  Years  CAtBat  CHits  CHmRun  CRuns  \\\n",
       "1    315    81      7    24   38     39     14    3449    835      69    321   \n",
       "2    479   130     18    66   72     76      3    1624    457      63    224   \n",
       "3    496   141     20    65   78     37     11    5628   1575     225    828   \n",
       "4    321    87     10    39   42     30      2     396    101      12     48   \n",
       "5    594   169      4    74   51     35     11    4408   1133      19    501   \n",
       "\n",
       "   CRBI  CWalks  League  Division  PutOuts  Assists  Errors  Salary  NewLeague  \n",
       "1   414     375     1.0       1.0      632       43      10   475.0        1.0  \n",
       "2   266     263     0.0       1.0      880       82      14   480.0        0.0  \n",
       "3   838     354     1.0       0.0      200       11       3   500.0        1.0  \n",
       "4    46      33     1.0       0.0      805       40       4    91.5        1.0  \n",
       "5   336     194     0.0       1.0      282      421      25   750.0        0.0  "
      ]
     },
     "execution_count": 236,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hitters_dat = pd.read_csv('hitters.csv')\n",
    "hitters_dat = hitters_dat.drop(hitters_dat.columns[0], axis=1)\n",
    "hitters_dat = hitters_dat.dropna(axis=0)\n",
    "hitters_dat['League'] = hitters_dat['League'].map({'N' : 1., 'A' : 0.})\n",
    "hitters_dat['Division'] = hitters_dat['Division'].map({'W' : 1., 'E' : 0.})\n",
    "hitters_dat['NewLeague'] = hitters_dat['NewLeague'].map({'N' : 1., 'A' : 0.})\n",
    "#hitters_dat = (hitters_dat - hitters_dat.mean()) / hitters_dat.std()\n",
    "hitters_dat.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Best subset selection\n",
    "# There are 2 ^ 19 ~= 5 * 10^5 possible subsets of all features.\n",
    "# Trying them all takes ages.\n",
    "# For 4 predictors there are (19 choose 1) + .. + (19 choose 4) ~= 5 * 10^3 possible models.\n",
    "\n",
    "def n_choose_k(n, k):\n",
    "    return int(np.math.factorial(n) / (np.math.factorial(n - k) * np.math.factorial(k)))\n",
    "\n",
    "def best_subset(n_predictors, target_column, data):\n",
    "    i = 1\n",
    "    predictors = data.drop(target_column, axis=1).columns\n",
    "    top_models = []\n",
    "    while i <= n_predictors:\n",
    "        tick = time.time()\n",
    "        cmbs = list(combinations(predictors, i))\n",
    "        formulae = ['Salary ~ ' + ' + '.join(s) for s in cmbs]\n",
    "        models = [smf.ols(formula=f, data=data).fit() for f in formulae]\n",
    "        top_model = sorted(models, key=lambda m: m.rsquared, reverse=True)[0]\n",
    "        top_models.append(top_model)\n",
    "        tock = time.time()\n",
    "        print('{}/{} : {} combinations, {} seconds'.format(i, n_predictors, n_choose_k(n_predictors, i), tock-tick))\n",
    "        i += 1\n",
    "    return pd.DataFrame({'model' : top_models, 'n_predictors' : range(1, n_predictors + 1)})\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 238,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1/3 : 3 combinations, 0.07082295417785645 seconds\n",
      "2/3 : 3 combinations, 0.7477953433990479 seconds\n",
      "3/3 : 1 combinations, 5.140793085098267 seconds\n"
     ]
    }
   ],
   "source": [
    "# Select between models with different numbers of predictors using adjusted training metrics\n",
    "best = best_subset(3, 'Salary', hitters_dat)\n",
    "best['adj_R_sq'] = best['model'].map(lambda m: m.rsquared_adj)\n",
    "best['bic'] = best['model'].map(lambda m: m.bic)\n",
    "best['aic'] = best['model'].map(lambda m: m.aic)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 239,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x12211c278>"
      ]
     },
     "execution_count": 239,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA3MAAANgCAYAAACGCzFPAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzs3Xu0ZWdZJ+rfS1XCLRGQBAemEgq70QiCENIcIMiR2GjsbkOLoHC4yyF6BIkHmuaiAwU9Y2hzpI9NeyFCEugjd4iGi1yUEDoBglUhISQBuQsajYRbIKPBhPf8sWfZe9SpvfdayV577a/284yxxprzm3Ot+Va+OVfqV9+8VHcHAACAsdxq2QUAAAAwP2EOAABgQMIcAADAgIQ5AACAAQlzAAAAAxLmAAAABiTMAQAADEiYAwAAGJAwBwAAMKDdyy5gtWOOOab37t277DIAAACWYv/+/V/q7mNnWXdbhbm9e/dm3759yy4DAABgKarq87Ou6zRLAACAAQlzAAAAAxLmAAAABiTMAQAADEiYAwAAGJAwBwAAMKBt9WiCq794Xe7/nFcvuwwAgIXZ/5InLrsE4DBhZA4AAGBAwhwAAMCAhDkAAIABCXMAAAADEuYAAAAGJMwBAAAMSJgDAAAYkDAHAAAwIGEOAABgQMIcAADAgIQ5AACAAQlzAAAAAxLmAAAABiTMAQAADEiYAwAAGJAwBwAAMCBhDgAAYEDCHAAAwICEOQAAgAEJcwAAAAMS5gAAAAYkzAEAAAxImAMAABiQMAcAADAgYQ4AAGBAwhwAAMCAhDkAAIABCXMAAAADWmiYq6qzq+raqvrYIrcDAACw0yx6ZO7cJKcteBsAAAA7zkLDXHe/P8mXF7kNAACAnWjp18xV1RlVta+q9t14w/XLLgcAAGAISw9z3X1Wd5/c3Sfvvt3Ryy4HAABgCEsPcwAAAMxPmAMAABjQoh9N8NokH0zyA1X1xap66iK3BwAAsFPsXuSXd/djF/n9AAAAO5XTLAEAAAYkzAEAAAxImAMAABiQMAcAADAgYQ4AAGBAwhwAAMCAhDkAAIABCXMAAAADEuYAAAAGJMwBAAAMSJgDAAAYkDAHAAAwIGEOAABgQMIcAADAgIQ5AACAAQlzAAAAAxLmAAAABiTMAQAADEiYAwAAGJAwBwAAMCBhDgAAYEDCHAAAwICEOQAAgAEJcwAAAAMS5gAAAAYkzAEAAAxo97ILWO0H99w5+17yxGWXAQAAsO0ZmQMAABiQMAcAADAgYQ4AAGBAwhwAAMCAhDkAAIABCXMAAAADEuYAAAAGJMwBAAAMSJgDAAAYkDAHAAAwIGEOAABgQMIcAADAgIQ5AACAAQlzAAAAAxLmAAAABrR72QWs9u1rrszfvPjeyy4DAACYwwkvvGLZJexIRuYAAAAGJMwBAAAMSJgDAAAY0MxhrqpuX1W3mqa/v6pOr6ojFlcaAAAAa5lnZO79SW5TVccleXeSJyQ5dxFFAQAAsL55wlx19w1JHpnkD7r70UnutZiyAAAAWM9cYa6qHpTkcUnePrXt2vySAAAA2Mg8Ye7MJM9Pcl53X1lV35fkgsWUBQAAwHpmemh4Ve1Kcnp3n36grbs/k+SZiyoMAACAtc00MtfdNyV5yIJrAQAAYEYzjcxNPlJV5yd5Y5JvHmjs7rdselUAAACsa54wd5sk1yU5dVVbJxHmAAAAttjMYa67n7LIQgAAAJjdzHezrKo9VXVeVV07vd5cVXsWWRwAAACHNs+jCc5Jcn6S751eb53aAAAA2GLzhLlju/uc7r5xep2b5NgF1QUAAMA65glz11XV46tq1/R6fFZuiAIAAMAWmyfM/XySn03y90muSfKoJE9eQE0AAABsYJ5HE+zp7tNXN1TVKUm+sLklAQAAsJF5RuZeNmMbAAAAC7bhyFxVPSjJg5McW1XPWrXou5LsWlRhAAAArG2W0yyPTHLUtO7Rq9q/npXr5gAAANhiG4a57r4wyYVVdW53f34LagIAAGAD81wz94qquuOBmaq6U1W9awE1AQAAsIF5wtwx3f3VAzPd/ZUkd9n8kgAAANjIPGHuO1V1woGZqrpbkt78kgAAANjIPM+Z+9UkF1XVhUkqyY8kOWMhVQEAALCumcNcd7+zqk5K8sCp6Ve6+0uLKQsAAID1bHiaZVWdOL2flOSEJH83vU6Y2tb77PFVdUFVXVVVV1bVmZtRNAAAwE43y8jcs5M8LcnvHmJZJzl1nc/emOTZ3X1pVR2dZH9Vvae7r5q/VAAAAA6Y5TlzT5veHzbvl3f3NUmumaavr6qrkxyXRJgDAAC4BTYMc1X1yPWWd/dbZtlQVe1Ncr8klxzUfkamG6kcd4cjZvkqAACAHW+W0yx/anq/S5IHJ3nvNP+wJB9IsmGYq6qjkrw5KzdN+frqZd19VpKzkuQ+x93Wow4AAABmMMtplk9Jkqp6d5J7TqdOpqrumuTcjT5fVUdkJcj9yayjeAAAAKxvnoeGH38gyE3+ISt3t1xTVVWSVya5urtfejPqAwAA4BDmeWj4X1bVu5K8dpr/uSR/scFnTknyhCRXVNVlU9sLuvsd85UJAADAavM8NPwZVfXTSR46NZ3V3edt8JmLktQtqA8AAIBDmGdkLkkuTXJ9d/9FVd2uqo7u7usXURgAAABrm/mauap6WpI3JXn51HRckj9dRFEAAACsb54boDw9K9fAfT1JuvuTWXlcAQAAAFtsnjD3re7+9oGZqtqdxHPhAAAAlmCeMHdhVb0gyW2r6uFJ3pjkrYspCwAAgPXME+ael+Qfk1yR5BeSvCPJry2iKAAAANY3090sq2pXkld39+OS/PFiSwIAAGAjM43MdfdNSe5WVUcuuB4AAABmMM9z5j6T5OKqOj/JNw80dvdLN70qAAAA1jVPmPv09LpVkqMXUw4AAACzmDnMdfeLkqSqvmtltq9fWFUAAACsa+a7WVbVyVV1RZKPJrmiqi6vqvsvrjQAAADWMs9plmcn+aXu/u9JUlUPSXJOkvssojAAAADWNs9z5m46EOSSpLsvSnLj5pcEAADARuYZmbuwql6e5LVJOsnPJXlfVZ2UJN196QLqAwAA4BDmCXM/PL3/+kHt98tKuDt1UyoCAABgQ/PczfJh6y2vqid196tueUkAAABsZJ5r5jZy5iZ+FwAAAOvYzDBXm/hdAAAArGMzw1xv4ncBAACwDiNzAAAAA5r5BihVdeskP5Nk7+rPdfeLp8mLN7UyAAAA1jTPown+LMnXkuxP8q2DF3b3MzarKAAAANY3T5jb092nLawSAAAAZjbPNXMfqKp7L6wSAAAAZjbPyNxDkjy5qj6bldMsK0l3930WUhkAAABrmifM/eTCqpgcedd75YQX7lv0ZgAAAIY3c5jr7s8vshAAAABmt5nPmQMAAGCLCHMAAAADEuYAAAAGJMwBAAAMSJgDAAAYkDAHAAAwIGEOAABgQMIcAADAgIQ5AACAAQlzAAAAAxLmAAAABiTMAQAADEiYAwAAGJAwBwAAMKDdyy5gtY9f+/Gc8rJTll0GAABwGLv4ly9edgmbwsgcAADAgIQ5AACAAQlzAAAAAxLmAAAABiTMAQAADEiYAwAAGJAwBwAAMCBhDgAAYEDCHAAAwICEOQAAgAEJcwAAAAMS5gAAAAYkzAEAAAxImAMAABiQMAcAADAgYQ4AAGBAwhwAAMCAhDkAAIABCXMAAAADEuYAAAAGJMwBAAAMSJgDAAAYkDAHAAAwIGEOAABgQMIcAADAgIQ5AACAAQlzAAAAAxLmAAAABrTQMFdVt6mqD1fV5VV1ZVW9aJHbAwAA2Cl2L/j7v5Xk1O7+RlUdkeSiqvrz7v7QgrcLAABwWFtomOvuTvKNafaI6dWL3CYAAMBOsPBr5qpqV1VdluTaJO/p7ksOWn5GVe2rqn3/9I1/WnQ5AAAAh4WFh7nuvqm775tkT5IHVNUPHbT8rO4+ubtPPuKoIxZdDgAAwGFhy+5m2d1fTXJBktO2apsAAACHq0XfzfLYqrrjNH3bJA9P8vFFbhMAAGAnWPTdLO+a5FVVtSsrwfEN3f22BW8TAADgsLfou1l+NMn9FrkNAACAnWjLrpkDAABg8whzAAAAAxLmAAAABiTMAQAADEiYAwAAGJAwBwAAMCBhDgAAYEDCHAAAwICEOQAAgAEJcwAAAAMS5gAAAAYkzAEAAAxImAMAABiQMAcAADAgYQ4AAGBAwhwAAMCAhDkAAIABCXMAAAADEuYAAAAGJMwBAAAMSJgDAAAYkDAHAAAwIGEOAABgQMIcAADAgIQ5AACAAQlzAAAAA9q97AJWO/EuJ+biX7542WUAAABse0bmAAAABiTMAQAADEiYAwAAGJAwBwAAMCBhDgAAYEDCHAAAwICEOQAAgAFVdy+7hn9WVdcn+cSy64BNckySLy27CNgE9mUOF/ZlDhf25cPb3br72FlW3FYPDU/yie4+edlFwGaoqn32Zw4H9mUOF/ZlDhf2ZQ5wmiUAAMCAhDkAAIABbbcwd9ayC4BNZH/mcGFf5nBhX+ZwYV8myTa7AQoAAACz2W4jcwAAAMxAmAMAABjQUsJcVZ1WVZ+oqk9V1fMOsfzWVfX6afklVbV366uEjc2wLz+0qi6tqhur6lHLqBFmMcO+/KyquqqqPlpVf1lVd1tGnTCLGfbnX6yqK6rqsqq6qKruuYw6YSMb7cur1vuZquqq8riCHWbLw1xV7Ury+0l+Msk9kzz2ED+iT03yle7+l0n+c5Lf2doqYWMz7st/k+TJSV6ztdXB7Gbclz+S5OTuvk+SNyX5T1tbJcxmxv35Nd197+6+b1b25ZducZmwoRn35VTV0UnOTHLJ1lbIdrCMkbkHJPlUd3+mu7+d5HVJHnHQOo9I8qpp+k1JfqyqagtrhFlsuC939+e6+6NJvrOMAmFGs+zLF3T3DdPsh5Ls2eIaYVaz7M9fXzV7+yTuBsd2NMvfmZPkN7My8PE/trI4todlhLnjknxh1fwXp7ZDrtPdNyb5WpI7b0l1MLtZ9mUYwbz78lOT/PlCK4Kbb6b9uaqeXlWfzsrI3DO3qDaYx4b7clWdlOT47n77VhbG9uEGKADMrKoen+TkJC9Zdi1wS3T373f3v0jy3CS/tux6YF5VdausnCL87GXXwvIsI8z9bZLjV83vmdoOuU5V7U5yhyTXbUl1MLtZ9mUYwUz7clX96yS/muT07v7WFtUG85r3t/l1Sf79QiuCm2ejffnoJD+U5H1V9bkkD0xyvpug7CzLCHN/leQeVXX3qjoyyWOSnH/QOucnedI0/agk721PN2f7mWVfhhFsuC9X1f2SvDwrQe7aJdQIs5plf77Hqtl/m+STW1gfzGrdfbm7v9bdx3T33u7em5XrmU/v7n3LKZdl2PIwN10D94wk70pydZI3dPeVVfXiqjp9Wu2VSe5cVZ9K8qwka96KFZZlln25qv5VVX0xyaOTvLyqrlxexXBoM/4uvyTJUUneON3O3T9csC3NuD8/o6qurKrLsvL3jCet8XWwNDPuy+xwZcALAABgPG6AAgAAMCBhDgAAYEDCHAAAwICEOQAAgAEJcwAAAAMS5gA4rFXVk6vqv07Tv1hVT1xn3XOr6rPT4xcur6of27pKAWA+u5ddAABsle7+oxlWe053v6mqHpbkrCT32OgDALAMRuYAGFpV/WlV7Z8eAn3G1PaUqvrrqvpwklNWrfsbVfUfZvzqDyY5boNt/3ZVXVVVH62q/3tqu3tVfbCqrqiq36qqb9zMPxoArMvIHACj+/nu/nJV3TbJX1XV25O8KMn9k3wtyQVJPnIzvve0JH+61sKqunOSn05yYnd3Vd1xWvR7Sf6wu19dVU+/GdsFgJkYmQNgdM+sqsuTfCjJ8UmekOR93f2P3f3tJK+f8/teUlV/neQ1SX5nnfW+luR/JHllVT0yyQ1T+ylJXjtN/7c5tw0AMxPmABhWVf1okn+d5EHd/cNZGYH7+C382ud09/cneW6Ss9daqbtvTPKAJG9K8u+SvHP14ltYAwBsSJgDYGR3SPKV7r6hqk5M8sAkt03yv1bVnavqiCSPvpnf/V+T3KqqfuJQC6vqqCR36O53JPk/k/zwtOjiJI+Zph93M7cNABsS5gAY2TuT7K6qq5P8dlZOtbwmyW9k5QYmFye5+qDPzDRq1t2d5LeS/Mc1Vjk6yduq6qNJLkryrKn9zCRPr6orssENVADglqiV/1cBwOGvql6W5NLuPmcLt/mN7j5qq7YHwM5hZA6AHaGqfjPJ/5Lk/GXXAgCbwcgcADtOVf1+Vj1/bvJ7a43YVdV5Se5+UPNzu/tdi6gPAGYhzAEAAAzIaZYAAAADEuYAAAAGJMwBAAAMSJgDAAAYkDAHAAAwIGEOAABgQMIcAADAgIQ5AACAAQlzAAAAAxLmAAAABiTMAQAADEiYAwAAGJAwBwAAMCBhDgAAYEC7l13Aasccc0zv3bt32WUAAAAsxf79+7/U3cfOsu62CnN79+7Nvn37ll0GAADAUlTV52dd12mWAAAAAxLmAAAABiTMAQAADEiYAwAAGJAwBwAAMCBhDgAAYEDb6tEEV3/xutz/Oa9edhkAAMBhbP9LnrjsEjaFkTkAAIABCXMAAAADEuYAAAAGJMwBAAAMSJgDAAAYkDAHAAAwIGEOAABgQMIcAADAgIQ5AACAAQlzAAAAAxLmAAAABiTMAQAADEiYAwAAGJAwBwAAMCBhDgAAYEDCHAAAwICEOQAAgAEJcwAAAAMS5gAAAAYkzAEAAAxImAMAABiQMAcAADAgYQ4AAGBAwhwAAMCAhDkAAIABCXMAAAADEuYAAAAGtNAwV1VnV9W1VfWxRW4HAABgp1n0yNy5SU5b8DYAAAB2nIWGue5+f5IvL3IbAAAAO5Fr5gAAAAa09DBXVWdU1b6q2nfjDdcvuxwAAIAhLD3MdfdZ3X1yd5+8+3ZHL7scAACAISw9zAEAADC/RT+a4LVJPpjkB6rqi1X11EVuDwAAYKfYvcgv7+7HLvL7AQAAdiqnWQIAAAxImAMAABiQMAcAADAgYQ4AAGBAwhwAAMCAhDkAAIABCXMAAAADEuYAAAAGJMwBAAAMSJgDAAAYkDAHAAAwIGEOAABgQMIcAADAgIQ5AACAAQlzAAAAAxLmAAAABiTMAQAADEiYAwAAGJAwBwAAMCBhDgAAYEDCHAAAwICEOQAAgAEJcwAAAAMS5gAAAAYkzAEAAAxImAMAABiQMAcAADCg3csuYLUf3HPn7HvJE5ddBgAAwLZnZA4AAGBAwhwAAMCAhDkAAIABCXMAAAADEuYAAAAGJMwBAAAMSJgDAAAYkDAHAAAwIGEOAABgQMIcAADAgIQ5AACAAQlzAAAAAxLmAAAABiTMAQAADEiYAwAAGNDuZRew2revuTJ/8+J7L7sMAABgYCe88Ipll7AljMwBAAAMSJgDAAAY0MxhrqpuX1W3mqa/v6pOr6ojFlcaAAAAa5lnZO79SW5TVccleXeSJyQ5dxFFAQAAsL55wlx19w1JHpnkD7r70UnutZiyAAAAWM9cYa6qHpTkcUnePrXt2vySAAAA2Mg8Ye7MJM9Pcl53X1lV35fkgsWUBQAAwHpmes5cVe1Kcnp3n36grbs/k+SZiyoMAACAtc00MtfdNyV5yIJrAQAAYEYzjcxNPlJV5yd5Y5JvHmjs7rdselUAAACsa54wd5sk1yU5dVVbJxHmAAAAttjMYa67n7LIQgAAAJjdzHezrKo9VXVeVV07vd5cVXsWWRwAAACHNs+jCc5Jcn6S751eb53aAAAA2GLzhLlju/uc7r5xep2b5NgF1QUAAMA65glz11XV46tq1/R6fFZuiAIAAMAWmyfM/XySn03y90muSfKoJE9eQE0AAABsYJ5HE+zp7tNXN1TVKUm+sLklAQAAsJF5RuZeNmMbAAAAC7bhyFxVPSjJg5McW1XPWrXou5LsWlRhAAAArG2W0yyPTHLUtO7Rq9q/npXr5gAAANhiG4a57r4wyYVVdW53f34LagIAAGAD81wz94qquuOBmaq6U1W9awE1AQAAsIF5wtwx3f3VAzPd/ZUkd9n8kgAAANjIPGHuO1V1woGZqrpbkt78kgAAANjIPM+Z+9UkF1XVhUkqyY8kOWMhVQEAALCumcNcd7+zqk5K8sCp6Ve6+0uLKQsAAID1bHiaZVWdOL2flOSEJH83vU6Y2tb77PFVdUFVXVVVV1bVmZtRNAAAwE43y8jcs5M8LcnvHmJZJzl1nc/emOTZ3X1pVR2dZH9Vvae7r5q/VAAAAA6Y5TlzT5veHzbvl3f3NUmumaavr6qrkxyXRJgDAAC4BTYMc1X1yPWWd/dbZtlQVe1Ncr8kl8yyPgAAAGub5TTLn5re75LkwUneO80/LMkHkmwY5qrqqCRvzspNU75+0LIzMt0V87g7HDFb1QAAADvcLKdZPiVJqurdSe45nTqZqrprknM3+nxVHZGVIPcnhxrF6+6zkpyVJPc57raeWwcAADCDeR4afvyBIDf5h6zc3XJNVVVJXpnk6u5+6c2oDwAAgEOY56Hhf1lV70ry2mn+55L8xQafOSXJE5JcUVWXTW0v6O53zFcmAAAAq83z0PBnVNVPJ3no1HRWd5+3wWcuSlK3oD4AAAAOYZ6RuSS5NMn13f0XVXW7qjq6u69fRGEAAACsbeZr5qrqaUnelOTlU9NxSf50EUUBAACwvnlugPL0rFwD9/Uk6e5PZuVxBQAAAGyxecLct7r72wdmqmp3Eo8SAAAAWIJ5wtyFVfWCJLetqocneWOSty6mLAAAANYzT5h7XpJ/THJFkl9I8o4kv7aIogAAAFjfTHezrKpdSV7d3Y9L8seLLQkAAICNzDQy1903JblbVR254HoAAACYwTzPmftMkour6vwk3zzQ2N0v3fSqAAAAWNc8Ye7T0+tWSY5eTDkAAADMYuYw190vSpKq+q6V2b5+YVUBAACwrpnvZllVJ1fVFUk+muSKqrq8qu6/uNIAAABYyzynWZ6d5Je6+78nSVU9JMk5Se6ziMIAAABY2zzPmbvpQJBLku6+KMmNm18SAAAAG5lnZO7Cqnp5ktcm6SQ/l+R9VXVSknT3pQuoDwAAgEOYJ8z98PT+6we13y8r4e7UTakIAACADc1zN8uHrbe8qp7U3a+65SUBAACwkXmumdvImZv4XQAAAKxjM8NcbeJ3AQAAsI7NDHO9id8FAADAOozMAQAADGjmG6BU1a2T/EySvas/190vniYv3tTKAAAAWNM8jyb4syRfS7I/ybcOXtjdz9isogAAAFjfPGFuT3eftrBKAAAAmNk818x9oKruvbBKAAAAmNk8I3MPSfLkqvpsVk6zrCTd3fdZSGUAAACsaZ4w95MLqwIAAIC5zBzmuvvziywkSY68671ywgv3LXozAAAAw9vM58wBAACwRYQ5AACAAQlzAAAAAxLmAAAABiTMAQAADEiYAwAAGJAwBwAAMCBhDgAAYEDCHAAAwICEOQAAgAEJcwAAAAMS5gAAAAYkzAEAAAxImAMAABiQMAcAADCg3csuYLWPX/vxnPKyU5ZdBgAAMKCLf/niZZewpYzMAQAADEiYAwAAGJAwBwAAMCBhDgAAYEDCHAAAwICEOQAAgAEJcwAAAAMS5gAAAAYkzAEAAAxImAMAABiQMAcAADAgYQ4AAGBAwhwAAMCAhDkAAIABCXMAAAADEuYAAAAGJMwBAAAMSJgDAAAYkDAHAAAwIGEOAABgQMIcAADAgIQ5AACAAQlzAAAAAxLmAAAABiTMAQAADEiYAwAAGJAwBwAAMKCFhrmquk1VfbiqLq+qK6vqRYvcHgAAwE6xe8Hf/60kp3b3N6rqiCQXVdWfd/eHFrxdAACAw9pCw1x3d5JvTLNHTK9e5DYBAAB2goVfM1dVu6rqsiTXJnlPd1+y6G0CAAAc7hYe5rr7pu6+b5I9SR5QVT+0enlVnVFV+6pq3z99458WXQ4AAMBhYcvuZtndX01yQZLTDmo/q7tP7u6TjzjqiK0qBwAAYGiLvpvlsVV1x2n6tkkenuTji9wmAADATrDou1neNcmrqmpXVoLjG7r7bQveJgAAwGFv0Xez/GiS+y1yGwAAADvRll0zBwAAwOYR5gAAAAYkzAEAAAxImAMAABiQMAcAADAgYQ4AAGBAwhwAAMCAhDkAAIABCXMAAAADEuYAAAAGJMwBAAAMSJgDAAAYkDAHAAAwIGEOAABgQMIcAADAgIQ5AACAAQlzAAAAAxLmAAAABiTMAQAADEiYAwAAGJAwBwAAMCBhDgAAYEDCHAAAwICEOQAAgAEJcwAAAAMS5gAAAAYkzAEAAAxo97ILWO3Eu5yYi3/54mWXAQAAsO0ZmQMAABiQMAcAADAgYQ4AAGBAwhwAAMCAhDkAAIABCXMAAAADEuYAAAAGVN297Br+WVVdn+QTy66DQzomyZeWXQSHpG+2N/2zfemb7UvfbF/6ZvvSN9vXvH1zt+4+dpYVt9VDw5N8ortPXnYR/P9V1T59sz3pm+1N/2xf+mb70jfbl77ZvvTN9rXIvnGaJQAAwICEOQAAgAFttzB31rILYE36ZvvSN9ub/tm+9M32pW+2L32zfemb7WthfbOtboACAADAbLbbyBwAAAAz2DZhrqpOq6pPVNWnqup5y65nJ6qqz1XVFVV1WVXtm9q+u6reU1WfnN7vNLVXVf2Xqb8+WlUnLbf6w0tVnV1V11bVx1a1zd0XVfWkaf1PVtWTlvFnOdys0Te/UVV/Ox07l1XVv1m17PlT33yiqn5iVbvfvE1WVcdX1QVVdVVVXVlVZ07tjp0lW6dvHDtLVlW3qaoPV9XlU9+8aGq/e1VdMv13fn1VHTm133qa/9S0fO+q7zpkn3HzrNM351bVZ1cdN/ed2v2mbaGq2lVVH6mqt03zyzlmunvpryS7knw6yfclOTLJ5Unuuey6dtoryeeSHHNQ239K8rxp+nlJfmea/jdJ/jxJJXlgkkuWXf/h9Ery0CQnJfnYze2LJN+d5DPT+52m6Tst+882+muNvvmNJP/hEOvec/o9u3WSu0+/c7v85i2sb+6a5KRp+ugkfz31gWPioFK5AAAZd0lEQVRn+/aNY2f5fVNJjpqmj0hyyXQ8vCHJY6b2P0ryf0zTv5Tkj6bpxyR5/Xp9tuw/38ivdfrm3CSPOsT6ftO2tn+eleQ1Sd42zS/lmNkuI3MPSPKp7v5Md387yeuSPGLJNbHiEUleNU2/Ksm/X9X+6l7xoSR3rKq7LqPAw1F3vz/Jlw9qnrcvfiLJe7r7y939lSTvSXLa4qs/vK3RN2t5RJLXdfe3uvuzST6Vld87v3kL0N3XdPel0/T1Sa5OclwcO0u3Tt+sxbGzRab9/xvT7BHTq5OcmuRNU/vBx82B4+lNSX6sqipr9xk30zp9sxa/aVukqvYk+bdJXjHNV5Z0zGyXMHdcki+smv9i1v+RZzE6yburan9VnTG1fU93XzNN/32S75mm9dnWm7cv9NHWesZ0WsvZB07ji75Zmuk0lvtl5V+yHTvbyEF9kzh2lm46XeyyJNdm5S/6n07y1e6+cVpl9X/nf+6DafnXktw5+mYhDu6b7j5w3Pxf03Hzn6vq1lOb42br/D9J/mOS70zzd86SjpntEubYHh7S3Scl+ckkT6+qh65e2Ctjwm5/ug3oi23nD5P8iyT3TXJNkt9dbjk7W1UdleTNSX6lu7++epljZ7kO0TeOnW2gu2/q7vsm2ZOVkYETl1wSk4P7pqp+KMnzs9JH/yorp04+d4kl7jhV9e+SXNvd+5ddS7J9wtzfJjl+1fyeqY0t1N1/O71fm+S8rPyg/8OB0yen92un1fXZ1pu3L/TRFunuf5j+h/udJH+c/3mahL7ZYlV1RFbCwp9091umZsfONnCovnHsbC/d/dUkFyR5UFZO0ds9LVr93/mf+2Bafock10XfLNSqvjltOm25u/tbSc6J42arnZLk9Kr6XFZO9T41ye9lScfMdglzf5XkHtNdYI7MysWB5y+5ph2lqm5fVUcfmE7y40k+lpV+OHDXoycl+bNp+vwkT5zunPTAJF9bdRoTizFvX7wryY9X1Z2mU5d+fGpjkx10vehPZ+XYSVb65jHTnazunuQeST4cv3kLMV2D8MokV3f3S1ctcuws2Vp949hZvqo6tqruOE3fNsnDs3JN4wVJHjWtdvBxc+B4elSS904j3mv1GTfTGn3z8VX/OFVZuS5r9XHjN23Buvv53b2nu/dm5Tfovd39uCzpmNm98SqL1903VtUzsrJj7UpydndfueSydprvSXLeyu9Cdid5TXe/s6r+KskbquqpST6f5Gen9d+RlbsmfSrJDUmesvUlH76q6rVJfjTJMVX1xSS/nuS3M0dfdPeXq+o3s/KXnyR5cXfPeuMO1rBG3/xordwaurNyV9hfSJLuvrKq3pDkqiQ3Jnl6d980fY/fvM13SpInJLliusYkSV4Qx852sFbfPNaxs3R3TfKqqtqVlX/kf0N3v62qrkryuqr6rSQfyUoYz/T+36rqU1m5GdRjkvX7jJttrb55b1Udm5W7Vl6W5Ben9f2mLddzs4RjplaCIQAAACPZLqdZAgAAMAdhDgAAYEDCHAAAwICEOQAAgAEJcwAAAAMS5gDYkapqb1V97BDtr6iqey6jJgCYx7Z4zhwAbBfd/b8vuwYAmIWROQB2st1V9SdVdXVVvamqbldV76uqk5Okqk6rqkur6vKq+stlFwsAqxmZA2An+4EkT+3ui6vq7CS/dGBBVR2b5I+TPLS7P1tV372sIgHgUIzMAbCTfaG7L56m/98kD1m17IFJ3t/dn02S7v7yVhcHAOsR5gDYyXqDeQDYtoQ5AHayE6rqQdP0/5bkolXLPpTkoVV19yRxmiUA240wB8BO9okkT6+qq5PcKckfHljQ3f+Y5Iwkb6mqy5O8fjklAsChVbczSgAAAEZjZA4AAGBAwhwAAMCAhDkAAIABCXMAAAADEuYAAAAGJMwBAAAMSJgDAAAYkDAHAAAwIGEOAABgQMIcAADAgIQ5AACAAQlzAAAAAxLmAAAABiTMAQAADEiYAwAAGJAwBwAAMCBhDgAAYEDCHAAAwIB2L7uA1Y455pjeu3fvsssAAABYiv3793+pu4+dZd1tFeb27t2bffv2LbsMAACApaiqz8+6rtMsAQAABiTMAQAADEiYAwAAGJAwBwAAMCBhDgAAYEDCHAAAwIC21aMJrv7idbn/c1697DIAAIDD2P6XPHHZJWwKI3MAAAADEuYAAAAGJMwBAAAMSJgDAAAYkDAHAAAwIGEOAABgQMIcAADAgIQ5AACAAQlzAAAAAxLmAAAABiTMAQAADEiYAwAAGJAwBwAAMCBhDgAAYEDCHAAAwICEOQAAgAEJcwAAAAMS5gAAAAYkzAEAAAxImAMAABiQMAcAADAgYQ4AAGBAwhwAAMCAhDkAAIABCXMAAAADEuYAAAAGJMwBAAAMaKFhrqrOrqprq+pji9wOAADATrPokblzk5y24G0AAADsOAsNc939/iRfXuQ2AAAAdiLXzAEAAAxo6WGuqs6oqn1Vte/GG65fdjkAAABDWHqY6+6zuvvk7j559+2OXnY5AAAAQ1h6mAMAAGB+i340wWuTfDDJD1TVF6vqqYvcHgAAwE6xe5Ff3t2PXeT3AwAA7FROswQAABiQMAcAADAgYQ4AAGBAwhwAAMCAhDkAAIABCXMAAAADEuYAAAAGJMwBAAAMSJgDAAAYkDAHAAAwIGEOAABgQMIcAADAgIQ5AACAAQlzAAAAAxLmAAAABiTMAQAADEiYAwAAGJAwBwAAMCBhDgAAYEDCHAAAwICEOQAAgAEJcwAAAAMS5gAAAAYkzAEAAAxImAMAABiQMAcAADAgYQ4AAGBAu5ddwGo/uOfO2feSJy67DAAAgG3PyBwAAMCAhDkAAIABCXMAAAADEuYAAAAGJMwBAAAMSJgDAAAYkDAHAAAwIGEOAABgQMIcAADAgIQ5AACAAQlzAAAAAxLmAAAABiTMAQAADEiYAwAAGJAwBwAAMKDdyy5gtW9fc2X+5sX3XnYZAADAoE544RXLLmHLGJkDAAAYkDAHAAAwoJnDXFXdvqpuNU1/f1WdXlVHLK40AAAA1jLPyNz7k9ymqo5L8u4kT0hy7iKKAgAAYH3zhLnq7huSPDLJH3T3o5PcazFlAQAAsJ65wlxVPSjJ45K8fWrbtfklAQAAsJF5wtyZSZ6f5LzuvrKqvi/JBYspCwAAgPXM9Jy5qtqV5PTuPv1AW3d/JskzF1UYAAAAa5tpZK67b0rykAXXAgAAwIxmGpmbfKSqzk/yxiTfPNDY3W/Z9KoAAABY1zxh7jZJrkty6qq2TiLMAQAAbLGZw1x3P2WRhQAAADC7me9mWVV7quq8qrp2er25qvYssjgAAAAObZ5HE5yT5Pwk3zu93jq1AQAAsMXmCXPHdvc53X3j9Do3ybELqgsAAIB1zBPmrquqx1fVrun1+KzcEAUAAIAtNk+Y+/kkP5vk75Nck+RRSZ68gJoAAADYwDyPJtjT3aevbqiqU5J8YXNLAgAAYCPzjMy9bMY2AAAAFmzDkbmqelCSByc5tqqetWrRdyXZtajCAAAAWNssp1kemeSoad2jV7V/PSvXzQEAALDFNgxz3X1hkgur6tzu/vwW1AQAAMAG5rlm7hVVdccDM1V1p6p61wJqAgAAYAPzhLljuvurB2a6+ytJ7rL5JQEAALCRecLcd6rqhAMzVXW3JL35JQEAALCReZ4z96tJLqqqC5NUkh9JcsZCqgIAAGBdM4e57n5nVZ2U5IFT069095cWUxYAAADr2fA0y6o6cXo/KckJSf5uep0wta332eOr6oKquqqqrqyqMzejaAAAgJ1ulpG5Zyd5WpLfPcSyTnLqOp+9Mcmzu/vSqjo6yf6qek93XzV/qQAAABwwy3Pmnja9P2zeL+/ua5JcM01fX1VXJzkuiTAHAABwC2wY5qrqkest7+63zLKhqtqb5H5JLpllfQAAANY2y2mWPzW93yXJg5O8d5p/WJIPJNkwzFX9f+3dfbBtdVkH8O/TvRekIEQhhgFSapxhzAiJHA3HjFLRmozJksZRcpqwBLNpmgSnyZdyxpqJ3v5IMVF0fEMMNQZfMJCSRhSU95dAxEnHvBkqkDMo+PTHXsfOHO8592w8+2Xd8/nM7Dlrr7XP3s85D7/F/Z611m/VgUnel8mkKfeu2XZGhlkxjzx41+aqBgAA2OY2c5rli5Okqj6a5PHDqZOpqiOSvHVv319VuzIJcu/Y01G87j4vyXlJctyRB7hvHQAAwCZMc9Pwo1eC3OArmcxuua6qqiRvTnJrd5/7MOoDAABgD6a5afi/VNVHkrxreP78JB/by/eclOSFSW6squuGda/s7kunKxMAAIDVprlp+FlVdWqSpw2rzuvui/fyPZ9IUt9HfQAAAOzBNEfmkuQzSe7r7o9V1Q9W1UHdfd8sCgMAAGB9m75mrqp+J8lFSd44rDoyyftnURQAAAAbm2YClDMzuQbu3iTp7jsyuV0BAAAAczZNmHugu7+18qSqdiZxKwEAAIAFmCbMXVlVr0xyQFU9I8l7k/zzbMoCAABgI9OEubOT/HeSG5O8JMmlSf5kFkUBAACwsU3NZllVO5K8rbtfkORNsy0JAACAvdnUkbnufijJY6pqvxnXAwAAwCZMc5+5u5JcVVUfTPK/Kyu7+9wtrwoAAIANTRPmPjc8fiDJQbMpBwAAgM3YdJjr7tckSVX98ORp3zezqgAAANjQpmezrKoTq+rGJDckubGqrq+qn55daQAAAKxnmtMsz0/y0u7+tySpqqcmeUuS42ZRGAAAAOub5j5zD60EuSTp7k8keXDrSwIAAGBvpjkyd2VVvTHJu5J0kucn+XhVnZAk3f2ZGdQHAADAHkwT5n5q+PqqNeufmEm4O3lLKgIAAGCvppnN8uc32l5Vp3f3Bd9/SQAAAOzNNNfM7c3Lt/C9AAAA2MBWhrnawvcCAABgA1sZ5noL3wsAAIANODIHAAAwQpueAKWq9k/ya0keu/r7uvu1w+JVW1oZAAAA65rm1gQfSPKNJNcmeWDtxu4+a6uKAgAAYGPThLmjuvuUmVUCAADApk1zzdy/V9VPzqwSAAAANm2aI3NPTfJbVfX5TE6zrCTd3cfNpDIAAADWNU2Ye/bMqgAAAGAqmw5z3f2FWRaSJPsd8RP50T+9ZtYfAwAAMHpbeZ85AAAA5kSYAwAAGCFhDgAAYISEOQAAgBES5gAAAEZImAMAABghYQ4AAGCEhDkAAIAREuYAAABGSJgDAAAYIWEOAABghIQ5AACAERLmAAAARkiYAwAAGCFhDgAAYIR2LrqA1W7bfVtO+vuTFl0GAAAwMle97KpFlzB3jswBAACMkDAHAAAwQsIcAADACAlzAAAAIyTMAQAAjJAwBwAAMELCHAAAwAgJcwAAACMkzAEAAIyQMAcAADBCwhwAAMAICXMAAAAjJMwBAACMkDAHAAAwQsIcAADACAlzAAAAIyTMAQAAjJAwBwAAMELCHAAAwAgJcwAAACMkzAEAAIyQMAcAADBCwhwAAMAICXMAAAAjJMwBAACMkDAHAAAwQsIcAADACM00zFXVI6rqU1V1fVXdXFWvmeXnAQAAbBc7Z/z+DyQ5ubvvr6pdST5RVR/q7k/O+HMBAAD2aTMNc93dSe4fnu4aHj3LzwQAANgOZn7NXFXtqKrrkuxOcll3Xz3rzwQAANjXzTzMdfdD3X18kqOSPKmqnrB6e1WdUVXXVNU1377/27MuBwAAYJ8wt9ksu/vrSa5Icsqa9ed194ndfeKuA3fNqxwAAIBRm/VslodV1SOH5QOSPCPJbbP8TAAAgO1g1rNZHpHkgqrakUlwvLC7L5nxZwIAAOzzZj2b5Q1JnjjLzwAAANiO5nbNHAAAAFtHmAMAABghYQ4AAGCEhDkAAIAREuYAAABGSJgDAAAYIWEOAABghIQ5AACAERLmAAAARkiYAwAAGCFhDgAAYISEOQAAgBES5gAAAEZImAMAABghYQ4AAGCEhDkAAIAREuYAAABGSJgDAAAYIWEOAABghIQ5AACAERLmAAAARkiYAwAAGCFhDgAAYISEOQAAgBES5gAAAEZImAMAABghYQ4AAGCEdi66gNWO/ZFjc9XLrlp0GQAAAEvPkTkAAIAREuYAAABGSJgDAAAYIWEOAABghIQ5AACAERLmAAAARkiYAwAAGKHq7kXX8F1VdV+S2xddB+s6NMlXF10Ee6Q3y0tvlpfeLC+9WW76s7z0ZnlN05vHdPdhm3nhUt00PMnt3X3iootgz6rqGv1ZTnqzvPRmeenN8tKb5aY/y0tvlteseuM0SwAAgBES5gAAAEZo2cLceYsugA3pz/LSm+WlN8tLb5aX3iw3/VleerO8ZtKbpZoABQAAgM1ZtiNzAAAAbMLShLmqOqWqbq+qO6vq7EXXsx1V1d1VdWNVXVdV1wzrHlVVl1XVHcPXQ4b1VVV/N/Trhqo6YbHV71uq6vyq2l1VN61aN3Uvqur04fV3VNXpi/hZ9kXr9OfVVfWlYfxcV1XPWbXtnKE/t1fVs1att9/bQlV1dFVdUVW3VNXNVfXyYb2xswQ26I+xs2BV9Yiq+lRVXT/05jXD+mOq6urh9/yeqtpvWL//8PzOYftjV73XHnvGw7NBb95aVZ9fNW6OH9bbr81ZVe2oqs9W1SXD8/mOm+5e+CPJjiSfS/JjSfZLcn2Sxy+6ru32SHJ3kkPXrPvLJGcPy2cn+Yth+TlJPpSkkjw5ydWLrn9feiR5WpITktz0cHuR5FFJ7hq+HjIsH7Lon21feKzTn1cn+aM9vPbxwz5t/yTHDPu6HfZ7M+nLEUlOGJYPSvIfw+/f2FmCxwb9MXYW35tKcuCwvCvJ1cOYuDDJacP6NyT5vWH5pUneMCyfluQ9G/Vs0T/fmB8b9OatSZ63h9fbr82/R3+Y5J1JLhmez3XcLMuRuSclubO77+rubyV5d5LnLrgmJp6b5IJh+YIkv7pq/dt64pNJHllVRyyiwH1Rd/9rknvWrJ62F89Kcll339PdX0tyWZJTZl/9vm+d/qznuUne3d0PdPfnk9yZyT7Pfm+LdfeXu/szw/J9SW5NcmSMnaWwQX/WY+zMyTAG7h+e7hoeneTkJBcN69eOnZUxdVGSX6iqyvo942HaoDfrsV+bo6o6KskvJfnH4XllzuNmWcLckUn+c9XzL2bjHTyz0Uk+WlXXVtUZw7rDu/vLw/J/JTl8WNaz+Zu2F3o0f2cNp7Wcv3IqX/RnIYbTV56YyV+xjZ0ls6Y/ibGzcMOpYtcl2Z3JP/Q/l+Tr3f3g8JLVv+fv9mDY/o0kj47ezMTa3nT3yrh53TBu/rqq9h/WGTfz9TdJ/jjJd4bnj86cx82yhDmWw1O7+4Qkz05yZlU9bfXGnhwLNv3pEtCLpfQPSX48yfFJvpzkrxZbzvZVVQcmeV+SP+jue1dvM3YWbw/9MXaWQHc/1N3HJzkqk6MCxy64JAZre1NVT0hyTiY9+plMTp18xQJL3Jaq6peT7O7uaxdZx7KEuS8lOXrV86OGdcxRd39p+Lo7ycWZ7My/snL65PB19/ByPZu/aXuhR3PU3V8Z/of7nSRvyv+fIqE/c1RVuzIJCu/o7n8aVhs7S2JP/TF2lkt3fz3JFUmekskpejuHTat/z9/twbD94CT/E72ZqVW9OWU4bbm7+4Ekb4lxswgnJfmVqro7k9O9T07yt5nzuFmWMPfpJI8bZn/ZL5OLAj+44Jq2lar6oao6aGU5yTOT3JRJH1ZmPDo9yQeG5Q8medEwa9KTk3xj1WlMzMa0vfhIkmdW1SHDaUvPHNYxA2uuGT01k/GTTPpz2jCL1TFJHpfkU7Hf23LDtQdvTnJrd5+7apOxswTW64+xs3hVdVhVPXJYPiDJMzK5pvGKJM8bXrZ27KyMqecluXw46r1ez3iY1unNbav+QFWZXJO1etzYr81Bd5/T3Ud192Mz2Q9d3t0vyJzHzc69v2T2uvvBqjork/+odiQ5v7tvXnBZ283hSS6e7BOyM8k7u/vDVfXpJBdW1W8n+UKS3xhef2kmMybdmeSbSV48/5L3XVX1riRPT3JoVX0xyauSvD5T9KK776mqP8vkHz5J8tru3uykHWxgnf48vSZTQ3cmM8O+JEm6++aqujDJLUkeTHJmdz80vI/93tY6KckLk9w4XF+SJK+MsbMs1uvPbxo7C3dEkguqakcmf+i/sLsvqapbkry7qv48yWczCeMZvr69qu7MZDKo05KNe8bDtl5vLq+qwzKZtfK6JL87vN5+bfFekTmOm5oEQgAAAMZkWU6zBAAAYArCHAAAwAgJcwAAACMkzAEAAIyQMAcAADBCwhwADKrq0pV7OgHAsnNrAgAAgBFyZA6Abamq3l9V11bVzVV1xrDu7qo6dFh+UVXdUFXXV9XbF1stAHwvR+YA2Jaq6lHdfU9VHZDk00l+Lsm1SU5McniSi5P8bHd/deW1CywXAL7HzkUXAAAL8vtVdeqwfHSSx63adnKS93b3V5NEkANgGQlzAGw7VfX0JL+Y5Cnd/c2q+niSRyy0KACYkmvmANiODk7ytSHIHZvkyWu2X57k16vq0cnklMx5FwgAeyPMAbAdfTjJzqq6Ncnrk3xy9cbuvjnJ65JcWVXXJzl3/iUCwMZMgAIAADBCjswBAACMkDAHAAAwQsIcAADACAlzAAAAIyTMAQAAjJAwBwAAMELCHAAAwAgJcwAAACP0f116MkgtWsaCAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 1080x1080 with 3 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Plot the error metrics\n",
    "_, (ax1, ax2, ax3) = plt.subplots(nrows=3, figsize=(15,15))\n",
    "sns.barplot(y='n_predictors', x='adj_R_sq', data=best, ax=ax1, orient='h')\n",
    "sns.barplot(y='n_predictors', x='bic', data=best, ax=ax2, orient='h')\n",
    "sns.barplot(y='n_predictors', x='aic', data=best, ax=ax3, orient='h')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 249,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1/3 : 3 combinations, 0.07045507431030273 seconds\n",
      "2/3 : 3 combinations, 0.7818019390106201 seconds\n",
      "3/3 : 1 combinations, 5.664794921875 seconds\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>model</th>\n",
       "      <th>n_predictors</th>\n",
       "      <th>MSE</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>&lt;statsmodels.regression.linear_model.Regressio...</td>\n",
       "      <td>1</td>\n",
       "      <td>199054.342456</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>&lt;statsmodels.regression.linear_model.Regressio...</td>\n",
       "      <td>2</td>\n",
       "      <td>146475.652896</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>&lt;statsmodels.regression.linear_model.Regressio...</td>\n",
       "      <td>3</td>\n",
       "      <td>140876.193344</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                               model  n_predictors  \\\n",
       "0  <statsmodels.regression.linear_model.Regressio...             1   \n",
       "1  <statsmodels.regression.linear_model.Regressio...             2   \n",
       "2  <statsmodels.regression.linear_model.Regressio...             3   \n",
       "\n",
       "             MSE  \n",
       "0  199054.342456  \n",
       "1  146475.652896  \n",
       "2  140876.193344  "
      ]
     },
     "execution_count": 249,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Select between models with different numbers of predictors using the validation set approach\n",
    "train, test = model_selection.train_test_split(hitters_dat, test_size=0.2)\n",
    "best = best_subset(3, 'Salary', train)\n",
    "best['MSE'] = best['model'].map(lambda m: metrics.mean_squared_error(test['Salary'], m.predict(test)))\n",
    "best"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 251,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x114b395c0>"
      ]
     },
     "execution_count": 251,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAEKCAYAAAD+XoUoAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAEStJREFUeJzt3XuQZGV5x/Hv4+5yUVZAWa11VxiwVIJKBLYsDJcqqUjARKq8JIFSwUtJEi/B0iQlkjLR/4zRqqhJBBXRBI0SJa4Gg6gEIyboLi67XOUSvJAVBKOgUsjikz/OO0vvZGanz0y/0z28309V15x+z+lznjnd85szb7/9TmQmkqQ2PGrcBUiSlo6hL0kNMfQlqSGGviQ1xNCXpIYY+pLUEENfkhpi6EtSQwx9SWrIynEXMOiAAw7IqampcZchScvK5s2b787MNcNsO1GhPzU1xaZNm8ZdhiQtKxHx3WG3tXtHkhpi6EtSQwx9SWqIoS9JDTH0Jakhhr4kNWSihmze8IN7OOpPPz7uMiRpSW1+9+lLdiyv9CWpIYa+JDXE0Jekhhj6ktQQQ1+SGmLoS1JDDH1JaoihL0kNMfQlqSGGviQ1xNCXpIYY+pLUEENfkhpi6EtSQwx9SWqIoS9JDTH0Jakhhr4kNcTQl6SGGPqS1BBDX5IaYuhLUkMMfUlqiKEvSQ2pGvoRcX5E3BUR19Y8jiRpOLWv9C8ATqp8DEnSkKqGfmZ+DfhxzWNIkoZnn74kNWTsoR8RZ0bEpojYtOMX9427HEl6RBt76GfmeZm5ITM3rHz06nGXI0mPaGMPfUnS0qk9ZPOTwH8CT4+IH0TEa2oeT5K0eytr7jwzT6u5f0lSP3bvSFJDDH1JaoihL0kNMfQlqSGGviQ1xNCXpIYY+pLUEENfkhpi6EtSQwx9SWqIoS9JDTH0Jakhhr4kNcTQl6SGGPqS1BBDX5IaYuhLUkMMfUlqiKEvSQ0x9CWpIYa+JDXE0Jekhhj6ktQQQ1+SGrJy3AUM+rX1j2fTu08fdxmS9Ijllb4kNcTQl6SGGPqS1BBDX5IaYuhLUkMMfUlqiKEvSQ0x9CWpIYa+JDXE0Jekhhj6ktQQQ1+SGmLoS1JDhp5lMyIeA9yfmb+KiKcBhwJfzMwHR1XML7dfx/fe+axR7U5SJQe+fdu4S9AC9bnS/xqwV0SsA74EvAK4oEZRkqQ6+oR+ZOYvgBcDf5eZvws8o05ZkqQaeoV+RDwXeBnwr6VtxehLkiTV0if0zwLOBi7OzOsi4hDg8jplSZJqGOqN3IhYAZySmadMt2XmbcAf1ypMkjR6Q13pZ+ZDwLGVa5EkVdbnH6N/OyI2AhcBP59uzMzPjrwqSVIVfUJ/L+Ae4ISBtgQMfUlaJoYO/cx8Vc1CJEn1DT16JyLWR8TFEXFXuX0mItbXLE6SNFp9hmx+FNgIPKncPl/aJEnLRJ/QX5OZH83MHeV2AbCmUl2SpAr6hP49EfHyiFhRbi+ne2NXkrRM9An9VwO/B/wQ2A68FHhlhZokSZX0GbK5fvATuQARcQzw/dGWJEmqpc+V/vuHbJMkTah5r/TLzJq/AayJiDcPrHoszrIpScvKMN07ewD7lG1XD7TfS9evL0laJuYN/cy8ArgiIi7IzO8uQU2SpEr69Ol/OCL2m74TEftHxKUVapIkVdIn9A/IzJ9M38nM/wWeMPqSJEm19An9X0XEgdN3IuIgulk2JUnLRJ9x+ucAX4+IK4AAjgPOrFKVJKmKPlMr/1tEHAkcXZrelJl31ylLklTDvN07EXFo+XokcCDwP+V2YGnb3WOfHBGXR8T1EXFdRJw1iqIlSQszzJX+W4DXAu+ZZV2y63/SmmkH8JbMvDoiVgObI+KyzLy+f6mSpMUaZpz+a8vX5/XdeWZup5ucjcy8LyJuANYBhr4kjcEw0zC8eHfrh/3H6BExBRwBXDXM9pKk0Rume+eF5esT6Obg+Wq5/zzgGwzxj9EjYh/gM3Rv/t47Y92ZlFFA6/ZdNVzVkqQFGaZ751UAEfEl4LDSZUNErAUumO/xEbGKLvAvnO2vgsw8DzgP4PB1ezvuX5Iq6vPhrCdPB35xJ91onjlFRAAfAW7IzPcuoD5J0gj1+XDWV8pcO58s938f+PI8jzkGeAWwLSK2lLa3ZeYl/cqUJI1Cnw9nvSEiXgQcX5rOy8yL53nM1+k+vStJmgB9rvQBrgbuy8wvR8SjI2J1Zt5XozBJ0ugN3acfEa8F/hk4tzStA/6lRlGSpDr6vJH7ero++nsBMvNmnFpZkpaVPqH/QGb+cvpORKzEqZUlaVnpE/pXRMTbgL0j4vnARcDn65QlSaqhT+i/FfgRsA34A+AS4M9rFCVJqmOo0TsRsQL4eGa+DPhQ3ZIkSbUMdaWfmQ8BB0XEHpXrkSRV1Gec/m3AlRGxEfj5dKPTK0jS8tEn9G8tt0cBq+uUI0mqqc80DO8AiIjHdnf9JK4kLTd9PpG7ISK2AVvpJlC7JiKOqleaJGnU+nTvnA+8LjP/AyAijgU+ChxeozBJ0uj1Gaf/0HTgw84ZNHeMviRJUi19rvSviIhz6ebTT7r59P89Io4EyMyrK9QnSRqhPqH/6+XrX8xoP4Lul8AJI6lIklRNn9E7z9vd+og4IzM/tviSJEm19OnTn89ZI9yXJKmCUYa+/xZRkibcKEPfufUlacJ5pS9JDRn6jdyI2BN4CTA1+LjMfGdZvHKklUmSRq7PkM3PAT8FNgMPzFyZmW8YVVGSpDr6hP76zDypWiWSpOr6hP43IuJZmbmtVjF7rH0GB759U63dS1Lz+oT+scArI+K/6bp3gm6KZSdck6Rlok/on1ytCknSkugzDcN3axYiSapvlOP0JUkTztCXpIYY+pLUEENfkhpi6EtSQwx9SWqIoS9JDTH0Jakhhr4kNcTQl6SGGPqS1JA+E65Vd+NdN3LM+48ZdxlS0658o/8E75HMK31JaoihL0kNMfQlqSGGviQ1xNCXpIYY+pLUEENfkhpi6EtSQwx9SWqIoS9JDTH0Jakhhr4kNcTQl6SGGPqS1BBDX5IaYuhLUkMMfUlqiKEvSQ0x9CWpIYa+JDXE0Jekhhj6ktQQQ1+SGmLoS1JDqoZ+ROwVEd+MiGsi4rqIeEfN40mSdm9l5f0/AJyQmT+LiFXA1yPii5n5X5WPK0maRdXQz8wEflburiq3rHlMSdLcqvfpR8SKiNgC3AVclplX1T6mJGl21UM/Mx/KzGcD64HnRMQzB9dHxJkRsSkiNj34swdrlyNJTVuy0TuZ+RPgcuCkGe3nZeaGzNywap9VS1WOJDWp9uidNRGxX1neG3g+cGPNY0qS5lZ79M5a4GMRsYLuF8ynM/MLlY8pSZpD7dE7W4Ejah5DkjQ8P5ErSQ0x9CWpIYa+JDXE0Jekhhj6ktQQQ1+SGmLoS1JDDH1JaoihL0kNMfQlqSGGviQ1xNCXpIYY+pLUEENfkhpi6EtSQwx9SWqIoS9JDTH0Jakhhr4kNcTQl6SGGPqS1BBDX5IaYuhLUkMMfUlqyMpxFzDo0CccypVvvHLcZUjSI5ZX+pLUEENfkhpi6EtSQwx9SWqIoS9JDTH0JakhkZnjrmGniLgPuGncdczjAODucRexG9a3ONa3ONa3OAut76DMXDPMhhM1Th+4KTM3jLuI3YmITZNco/UtjvUtjvUtzlLUZ/eOJDXE0Jekhkxa6J837gKGMOk1Wt/iWN/iWN/iVK9vot7IlSTVNWlX+pKkmjJzIm7ASXTDNW8B3lr5WE8GLgeuB64DzirtfwncAWwptxcMPObsUttNwG/NVzdwMHBVaf8UsEfPGm8HtpU6NpW2xwGXATeXr/uX9gDeV461FThyYD9nlO1vBs4YaD+q7P+W8tjoUdvTB87RFuBe4E3jPH/A+cBdwLUDbdXP11zHGLK+dwM3lhouBvYr7VPA/QPn8YMLrWN33+sQ9VV/PoE9y/1byvqpHvV9aqC224EtYzx/c2XKxLwGd+5nIaE56huwArgVOATYA7gGOKzi8dZOn2RgNfAd4LDyIv+TWbY/rNS0Z3nx3lpqnrNu4NPAqWX5g8Af9azxduCAGW1/Nf2DBLwVeFdZfgHwxfJCOhq4auDFcFv5un9Znn7RfbNsG+WxJy/iufshcNA4zx9wPHAku4ZC9fM11zGGrO9EYGVZftdAfVOD283YT6865vpeh6yv+vMJvI4SysCpwKeGrW/G+vcAbx/j+ZsrUybmNbiz1oX8oI/6BjwXuHTg/tnA2Ut4/M8Bz9/Ni3yXeoBLS82z1l2elLt5+Ad6l+2GrOl2/n/o3wSsHXiR3VSWzwVOm7kdcBpw7kD7uaVtLXDjQPsu2/Ws80TgyrI81vPHjB/2pThfcx1jmPpmrHsRcOHutltIHXN9r0Oev+rP5/Rjy/LKst2sf3Xu5rwE8H3gqeM8fzOONZ0pE/UazMyJ6dNfR/ekTftBaasuIqaAI+j+tAR4Q0RsjYjzI2L/eeqbq/3xwE8yc8eM9j4S+FJEbI6IM0vbEzNze1n+IfDEBda3rizPbF+IU4FPDtyflPMHS3O+5jpGX6+mu3qbdnBEfDsiroiI4wbq7lvHYn+2aj+fOx9T1v+0bN/HccCdmXnzQNvYzt+MTJm41+CkhP5YRMQ+wGeAN2XmvcDfA08Bng1sp/uTcVyOzcwjgZOB10fE8YMrs/u1nmOprIiIPYBTgItK0ySdv10sxfla6DEi4hxgB3BhadoOHJiZRwBvBj4REY+tXccsJvb5nOE0dr3wGNv5myVTRrLfYQ1zjEkJ/Tvo3giZtr60VRMRq+ienAsz87MAmXlnZj6Umb8CPgQ8Z5765mq/B9gvIlbOaB9aZt5Rvt5F9ybfc4A7I2JtqX8t3RtbC6nvjrI8s72vk4GrM/POUuvEnL9iKc7XXMcYSkS8Evgd4GXlB5bMfCAz7ynLm+n6yZ+2wDoW/LO1RM/nzseU9fuW7YdSHvNiujd1p+sey/mbLVMWsN/qr8FJCf1vAU+NiIPL1eOpwMZaB4uIAD4C3JCZ7x1oXzuw2YuAa8vyRuDUiNgzIg4Gnkr3psqsdZcf3suBl5bHn0HXxzdsfY+JiNXTy3T95teWOs6YZZ8bgdOjczTw0/Ln3qXAiRGxf/nT/ES6vtTtwL0RcXQ5F6f3qW/ALldYk3L+BizF+ZrrGPOKiJOAPwNOycxfDLSviYgVZfkQuvN12wLrmOt7Haa+pXg+B+t+KfDV6V9+Q/pNur7unV0f4zh/c2XKAvZb/zU43xsSS3Wjezf7O3S/lc+pfKxj6f4E2srAcDTgH+iGRG0tJ3LtwGPOKbXdxMBIl7nqphvB8E264VUXAXv2qO8QupEP19AN/zqntD8e+Ard0KwvA4/Lh9/I+ttSwzZgw8C+Xl1quAV41UD7Brof4luBD9BjyGZ5/GPorsj2HWgb2/mj++WzHXiQrr/zNUtxvuY6xpD13ULXf7vL0ELgJeV53wJcDbxwoXXs7nsdor7qzyewV7l/S1l/yLD1lfYLgD+cse04zt9cmTIxr8Hpm5/IlaSGTEr3jiRpCRj6ktQQQ1+SGmLoS1JDDH1Jaoihr+ZFREbEPw7cXxkRP4qIL5T7T4yIL0TENRFxfURcUtqnIuL+iNgycDt9XN+HNIxJ+8fo0jj8HHhmROydmffTTZQ1+KnLdwKXZebfAETE4QPrbs3MZy9dqdLieKUvdS4Bfrssz5zLZS0Dk11l5tYlrEsaKUNf6vwT3dQCewGH8/Csq9B9cvIjEXF5RJwTEU8aWPeUGd07xyFNMLt3JLqr9+imxD2N7qp/cN2lZQ6Xk+gmmft2RDyzrLZ7R8uKV/rSwzYCf82uXTsAZOaPM/MTmfkKuonFjp+5jbQcGPrSw84H3pGZ2wYbI+KEiHh0WV5NN8f898ZQn7Rodu9IRXbT875vllVHAR+IiB10F0ofzsxvle6gp0TEloFtz8/M2fYhTQRn2ZSkhti9I0kNMfQlqSGGviQ1xNCXpIYY+pLUEENfkhpi6EtSQwx9SWrI/wE2f6A5uyp6/wAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Plot the error metric\n",
    "sns.barplot(y='n_predictors', x='MSE', data=best, orient='h')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 309,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1/3 : 3 combinations, 0.07561683654785156 seconds\n",
      "2/3 : 3 combinations, 0.7820789813995361 seconds\n",
      "3/3 : 1 combinations, 5.312049150466919 seconds\n",
      "1/3 : 3 combinations, 0.06390810012817383 seconds\n",
      "2/3 : 3 combinations, 0.7506320476531982 seconds\n",
      "3/3 : 1 combinations, 5.409984111785889 seconds\n",
      "1/3 : 3 combinations, 0.07043266296386719 seconds\n",
      "2/3 : 3 combinations, 0.7198760509490967 seconds\n",
      "3/3 : 1 combinations, 5.0563881397247314 seconds\n",
      "1/3 : 3 combinations, 0.06885409355163574 seconds\n",
      "2/3 : 3 combinations, 0.6995189189910889 seconds\n",
      "3/3 : 1 combinations, 4.871701955795288 seconds\n",
      "1/3 : 3 combinations, 0.06180310249328613 seconds\n",
      "2/3 : 3 combinations, 0.6945500373840332 seconds\n",
      "3/3 : 1 combinations, 5.0047760009765625 seconds\n",
      "1/3 : 3 combinations, 0.059381961822509766 seconds\n",
      "2/3 : 3 combinations, 0.8878071308135986 seconds\n",
      "3/3 : 1 combinations, 5.055046796798706 seconds\n",
      "1/3 : 3 combinations, 0.06460094451904297 seconds\n",
      "2/3 : 3 combinations, 0.6986589431762695 seconds\n",
      "3/3 : 1 combinations, 4.979748725891113 seconds\n",
      "1/3 : 3 combinations, 0.06681489944458008 seconds\n",
      "2/3 : 3 combinations, 0.7132260799407959 seconds\n",
      "3/3 : 1 combinations, 5.130325078964233 seconds\n",
      "1/3 : 3 combinations, 0.06234002113342285 seconds\n",
      "2/3 : 3 combinations, 0.6919701099395752 seconds\n",
      "3/3 : 1 combinations, 4.884754180908203 seconds\n",
      "1/3 : 3 combinations, 0.060920000076293945 seconds\n",
      "2/3 : 3 combinations, 0.7059929370880127 seconds\n",
      "3/3 : 1 combinations, 5.235835313796997 seconds\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>model</th>\n",
       "      <th>n_predictors</th>\n",
       "      <th>MSE</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>&lt;statsmodels.regression.linear_model.Regressio...</td>\n",
       "      <td>1</td>\n",
       "      <td>149077.418140</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>&lt;statsmodels.regression.linear_model.Regressio...</td>\n",
       "      <td>2</td>\n",
       "      <td>131847.483984</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>&lt;statsmodels.regression.linear_model.Regressio...</td>\n",
       "      <td>3</td>\n",
       "      <td>137281.842881</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                               model  n_predictors  \\\n",
       "0  <statsmodels.regression.linear_model.Regressio...             1   \n",
       "1  <statsmodels.regression.linear_model.Regressio...             2   \n",
       "2  <statsmodels.regression.linear_model.Regressio...             3   \n",
       "\n",
       "             MSE  \n",
       "0  149077.418140  \n",
       "1  131847.483984  \n",
       "2  137281.842881  "
      ]
     },
     "execution_count": 309,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Select between models with different numbers of predictors using cross validation\n",
    "results = pd.DataFrame()\n",
    "for train_idx, test_idx in model_selection.KFold(n_splits=10).split(hitters_dat):\n",
    "    train = hitters_dat.iloc[train_idx]\n",
    "    test = hitters_dat.iloc[test_idx]\n",
    "    best = best_subset(3, 'Salary', train)\n",
    "    mse = best['model'].map(lambda m: metrics.mean_squared_error(test['Salary'], m.predict(test)))\n",
    "    results = results.append(mse, ignore_index=True)\n",
    "    \n",
    "best['MSE'] = results.mean()\n",
    "best"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 310,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x11e11f278>"
      ]
     },
     "execution_count": 310,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXwAAAEKCAYAAAARnO4WAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAENhJREFUeJzt3XuQZGV9xvHvk12uggKCZmXFBasiAUWBrUQCWkrFiMRoaW5S3lAjqXgJRispkJQJ/qckKa+JEOVignhBUSQYRENQMUEX5H4Jd8WgXIyCaCngL3/0GbZ33GXnDH2me3i/n6quOf2e7nN+8/bMM2fec/rtVBWSpEe+X5l2AZKkpWHgS1IjDHxJaoSBL0mNMPAlqREGviQ1wsCXpEYY+JLUCANfkhqxctoFjNt5551rzZo10y5DkpaViy666M6q2mVzj5upwF+zZg3r1q2bdhmStKwkuWUhj3NIR5IaYeBLUiMMfElqhIEvSY0w8CWpEQa+JDVipi7LvPrWu9j/Lz867TIkaUlddNyrlmQ/HuFLUiMMfElqhIEvSY0w8CWpEQa+JDXCwJekRhj4ktQIA1+SGmHgS1IjDHxJaoSBL0mNMPAlqREGviQ1wsCXpEYY+JLUCANfkhph4EtSIwx8SWqEgS9JjTDwJakRBr4kNcLAl6RGGPiS1AgDX5IaMWjgJzkxye1JrhhyP5KkzRv6CP9k4JCB9yFJWoBBA7+qvgL8YMh9SJIWxjF8SWrE1AM/yRFJ1iVZd/9P7pl2OZL0iDX1wK+qE6pqbVWtXbnt9tMuR5IesaYe+JKkpTH0ZZmnAf8FPCXJrUleN+T+JEmbtnLIjVfVYUNuX5K0cA7pSFIjDHxJaoSBL0mNMPAlqREGviQ1wsCXpEYY+JLUCANfkhph4EtSIwx8SWqEgS9JjTDwJakRBr4kNcLAl6RGGPiS1AgDX5IaYeBLUiMMfElqhIEvSY0w8CWpEQa+JDXCwJekRhj4ktQIA1+SGrFy2gWM+/XVj2Xdca+adhmS9IjkEb4kNcLAl6RGGPiS1AgDX5IaYeBLUiMMfElqhIEvSY0w8CWpEQa+JDXCwJekRhj4ktQIA1+SGmHgS1IjFjxbZpJHAT+tql8k+TVgT+ALVXXfpIr5+W1X8u13Pm1Sm5OkX7LbOy6fdglT0+cI/yvA1kl2Bb4IvBI4eYiiJEmT1yfwU1U/AV4K/GNV/SGw9zBlSZImrVfgJzkAeDnwb13bismXJEkaQp/APxI4Gjijqq5Msgdw3jBlSZImbUEnbZOsAF5UVS+aa6uqG4E/H6owSdJkLegIv6oeAA4auBZJ0oD6fIj5t5KcCXwKuHeusao+M/GqJEkT1yfwtwbuAg4eayvAwJekZWDBgV9VrxmyEEnSsBZ8lU6S1UnOSHJ7d/t0ktVDFidJmpw+l2WeBJwJPKG7fb5rkyQtA30Cf5eqOqmq7u9uJwO7DFSXJGnC+gT+XUlekWRFd3sFo5O4kqRloE/gvxb4I+B7wG3AHwCHD1CTJGkAfS7LXD3+TluAJAcC35lsSZKkIfQ5wn//AtskSTNos0f43QyZvwXskuStY6sejbNlStKysZAhnS2B7brHbj/WfjejcXxJ0jKw2cCvqvOB85OcXFW3LEFNkqQB9BnD/3CSHebuJNkxyTkD1CRJGkCfwN+5qn44d6eq/g943ORLkiQNoU/g/yLJbnN3kjyJ0WyZkqRloM91+McAX0tyPhDgWcARg1QlSZq4PtMj/3uS/YBndk1vqao7hylLkjRpmx3SSbJn93U/YDfgf7vbbl3bQz33iUnOS3JVkiuTHDmJoiVJ/S3kCP9twOuBv9/IumLDT8Ca737gbVV1cZLtgYuSnFtVV/UvVZL0cCzkOvzXd1+f23fjVXUbo4nWqKp7klwN7AoY+JK0xBYytcJLH2r9Qj/EPMkaYF/gwoU8XpI0WQsZ0vm97uvjGM2p8x/d/ecCX2cBH2KeZDvg04xO9N49b90RdFf77PqYLRZWtSSpt4UM6bwGIMkXgb26YRqSrAJO3tzzk2zBKOxP3dh/A1V1AnACwD67buN1/ZI0kD5vvHriXNh3vs/oqp1NShLgI8DVVfUPi6hPkjQhfd549eVu7pzTuvt/DHxpM885EHglcHmSS7q2t1fV2f3KlCQ9XH3eePWmJC8Bnt01nVBVZ2zmOV9j9K5cSdKU9TnCB7gYuKeqvpRk2yTbV9U9QxQmSZqsBY/hJ3k9cDpwfNe0K/DZIYqSJE1en5O2b2Q0Jn83QFVdh9MjS9Ky0Sfwf1ZVP5+7k2QlTo8sSctGn8A/P8nbgW2SPA/4FPD5YcqSJE1an8A/CrgDuBz4U+Bs4K+HKEqSNHkLukonyQrgo1X1cuCfhy1JkjSEBR3hV9UDwJOSbDlwPZKkgfS5Dv9G4IIkZwL3zjU6ZYIkLQ99Av+G7vYrwPbDlCNJGkqfqRWOBUjy6NFd32ErSctJn3fark1yOXAZo8nQLk2y/3ClSZImqc+QzonAG6rqqwBJDgJOAvYZojBJ0mT1uQ7/gbmwhwdnwrx/8iVJkobQ5wj//CTHM5oPvxjNh/+fSfYDqKqLB6hPkjQhfQL/6d3Xv5nXvi+jPwAHT6QiSdIg+lyl89yHWp/k1VV1ysMvSZI0hD5j+Jtz5AS3JUmasEkGvh9lKEkzbJKB79z4kjTDPMKXpEYs+KRtkq2A3wfWjD+vqt7ZLV4w0cokSRPV57LMzwE/Ai4CfjZ/ZVW9aVJFSZImr0/gr66qQwarRJI0qD6B//UkT6uqy4cqZstVe7PbO9YNtXlJalqfwD8IODzJTYyGdMJommQnT5OkZaBP4L9gsCokSYPrM7XCLUMWIkka1iSvw5ckzTADX5IaYeBLUiMMfElqhIEvSY0w8CWpEQa+JDXCwJekRhj4ktQIA1+SGmHgS1Ij+kyeNrhrbr+GA99/4LTLkKQHXfDmR86H+XmEL0mNMPAlqREGviQ1wsCXpEYY+JLUCANfkhph4EtSIwx8SWqEgS9JjTDwJakRBr4kNcLAl6RGGPiS1AgDX5IaYeBLUiMMfElqhIEvSY0w8CWpEQa+JDXCwJekRhj4ktQIA1+SGmHgS1IjDHxJasSggZ9k6yTfSHJpkiuTHDvk/iRJm7Zy4O3/DDi4qn6cZAvga0m+UFX/PfB+JUnzDBr4VVXAj7u7W3S3GnKfkqSNG3wMP8mKJJcAtwPnVtWFQ+9TkvTLBg/8qnqgqp4BrAZ+I8lTx9cnOSLJuiTr7vvxfUOXI0nNWrKrdKrqh8B5wCHz2k+oqrVVtXaL7bZYqnIkqTlDX6WzS5IduuVtgOcB1wy5T0nSxg19lc4q4JQkKxj9cflkVZ018D4lSRsx9FU6lwH7DrkPSdLC+E5bSWqEgS9JjTDwJakRBr4kNcLAl6RGGPiS1AgDX5IaYeBLUiMMfElqhIEvSY0w8CWpEQa+JDXCwJekRhj4ktQIA1+SGmHgS1IjDHxJaoSBL0mNMPAlqREGviQ1wsCXpEYY+JLUCANfkhph4EtSI1ZOu4Bxez5uTy548wXTLkOSHpE8wpekRhj4ktQIA1+SGmHgS1IjDHxJaoSBL0mNSFVNu4YHJbkHuHbadSzQzsCd0y5igax1GMul1uVSJ1jrYj2pqnbZ3INm6jp84NqqWjvtIhYiyTprnTxrnbzlUidY69Ac0pGkRhj4ktSIWQv8E6ZdQA/WOgxrnbzlUidY66Bm6qStJGk4s3aEL0kayMwEfpJDklyb5PokRy3RPp+Y5LwkVyW5MsmRXftOSc5Ncl33dceuPUne19V4WZL9xrb16u7x1yV59Vj7/kku757zviR5mDWvSPKtJGd193dPcmG3/U8k2bJr36q7f323fs3YNo7u2q9N8vyx9om9Bkl2SHJ6kmuSXJ3kgFnt1yR/0b3+VyQ5LcnWs9KvSU5McnuSK8baBu/HTe2jZ53Hda//ZUnOSLLDYvtqMa9Hn1rH1r0tSSXZedp9OoiqmvoNWAHcAOwBbAlcCuy1BPtdBezXLW8P/A+wF/Bu4Kiu/SjgXd3yocAXgADPBC7s2ncCbuy+7tgt79it+0b32HTPfcHDrPmtwMeAs7r7nwRe1i1/CPizbvkNwIe65ZcBn+iW9+r6dytg967fV0z6NQBOAf6kW94S2GEW+xXYFbgJ2GasPw+flX4Fng3sB1wx1jZ4P25qHz3r/B1gZbf8rrE6e/dV39ejb61d+xOBc4BbgJ2n3adD3JZ0Zw/xAhwAnDN2/2jg6CnU8TngeYze/LWqa1vF6P0BAMcDh409/tpu/WHA8WPtx3dtq4Brxto3eNwi6lsNfBk4GDir+4G6c+yX6sF+7H5wD+iWV3aPy/y+nXvcJF8D4DGMQjTz2meuXxkF/ne6X9yVXb8+f5b6FVjDhkE6eD9uah996py37iXAqRvrg8311WJ+zhdTK3A68HTgZtYH/lT7dNK3WRnSmfulm3Nr17Zkun8F9wUuBB5fVbd1q74HPL5b3lSdD9V+60baF+s9wF8Bv+juPxb4YVXdv5HtP1hTt/5H3eP7fg+LsTtwB3BSRsNPH07yKGawX6vqu8DfAd8GbmPUTxcxm/06Zyn6cVP7WKzXMjraXUydi/k57yXJi4HvVtWl81bNcp/2NiuBP1VJtgM+Dbylqu4eX1ejP8dTv5QpyQuB26vqomnXsgArGf3L/E9VtS9wL6N/YR80Q/26I/BiRn+kngA8CjhkqkX1sBT9+HD3keQY4H7g1IkVNUFJtgXeDrxjqfY5rZ//WQn87zIaP5uzumsbXJItGIX9qVX1ma75+0lWdetXAbdvps6Hal+9kfbFOBB4UZKbgY8zGtZ5L7BDkrkpMsa3/2BN3frHAHct4ntYjFuBW6vqwu7+6Yz+AMxiv/42cFNV3VFV9wGfYdTXs9ivc5aiHze1j16SHA68EHh5F3KLqfMu+r8efTyZ0R/8S7vfr9XAxUl+dRG1Dt6nD8tSjyFtYjxtJaOTHruz/mTN3kuw3wAfBd4zr/04Njy58u5u+XfZ8ATON7r2nRiNWe/Y3W4CdurWzT+Bc+gE6n4O60/afooNT2a9oVt+IxuezPpkt7w3G54wu5HRybKJvgbAV4GndMt/2/XpzPUr8JvAlcC23bZOAd48S/3KL4/hD96Pm9pHzzoPAa4Cdpn3uN591ff16FvrvHU3s34Mf6p9Ounbku5sMy/AoYyukrkBOGaJ9nkQo3+rLgMu6W6HMhoD/DJwHfClsRcywAe7Gi8H1o5t67XA9d3tNWPta4Eruud8gAWcUFpA3c9hfeDv0f2AXd/9UmzVtW/d3b++W7/H2POP6eq5lrGrWyb5GgDPANZ1ffvZ7pdiJvsVOBa4ptvevzAKopnoV+A0RucW7mP0n9PrlqIfN7WPnnVez2ice+5360OL7avFvB59ap23/mbWB/7U+nSIm++0laRGzMoYviRpYAa+JDXCwJekRhj4ktQIA1+SGmHgq3nd7Ij/OnZ/ZZI7sn5G0scnOSvJpRnNrHp2174myU+TXDJ2e9W0vg9pc2btQ8ylabgXeGqSbarqp4wm0Bt/N+w7gXOr6r0ASfYZW3dDVT1j6UqVFs8jfGnkbEbvqoTRDIenja1bxdiEWFV12RLWJU2MgS+NfBx4WZKtgX0YzZo654PARzL6sJxjkjxhbN2T5w3pPGspi5b6cEhHYnTU3k2RfRijo/3xdeck2YPR3DAvAL6V5Kndaod0tGx4hC+tdyajufFPm7+iqn5QVR+rqlcC32T0qUnSsmLgS+udCBxbVZePNyY5uJsznSTbM5pO99tTqE96WBzSkTpVdSvwvo2s2h/4QJL7GR0kfbiqvtkNAT05ySVjjz2xqja2DWnqnC1TkhrhkI4kNcLAl6RGGPiS1AgDX5IaYeBLUiMMfElqhIEvSY0w8CWpEf8PK2y/6dD+e/YAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Plot the error metric\n",
    "sns.barplot(y='n_predictors', x='MSE', data=best, orient='h')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 311,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Forward selection:\n",
    "# There are 1 + p(1 + p) / 2 possible models in this space \n",
    "\n",
    "def fwd_selection(target_column, data):\n",
    "    predictors = data.drop('Salary', axis=1).columns\n",
    "    formulae_s1 = [(p, 'Salary ~ {}'.format(p)) for p in predictors]\n",
    "    models_s1 = [(p, smf.ols(formula=f, data=data).fit()) for p, f in formulae_s1]\n",
    "    predictor_s1, model_s1 = sorted(models_s1, key=lambda tup: tup[1].rsquared, reverse=True)[0]\n",
    "    predictors = predictors.drop(predictor_s1)\n",
    "    formula = 'Salary ~ {}'.format(predictor_s1)\n",
    "    step_models = [model_s1]\n",
    "    while len(predictors) > 0:\n",
    "        models = []\n",
    "        for p in predictors:\n",
    "            f = formula + ' + ' + p\n",
    "            m = smf.ols(formula=f, data=data).fit()\n",
    "            models.append((p, m))\n",
    "                    \n",
    "        predictor, model = sorted(models, key=lambda tup: tup[1].rsquared, reverse=True)[0]\n",
    "        step_models.append(model)\n",
    "        formula = formula + ' + ' + predictor\n",
    "        predictors = predictors.drop(predictor)\n",
    "        \n",
    "    return pd.DataFrame({'model': step_models, 'n_predictors' : range(1, 20)})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 312,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Select between models with different numbers of predictors using adjusted training metrics\n",
    "best = fwd_selection('Salary', hitters_dat)\n",
    "best['adj_R_sq'] = best['model'].map(lambda m: m.rsquared_adj)\n",
    "best['bic'] = best['model'].map(lambda m: m.bic)\n",
    "best['aic'] = best['model'].map(lambda m: m.aic)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 313,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x2884715f8>"
      ]
     },
     "execution_count": 313,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA3kAAANgCAYAAACRKaGGAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzs3Xu0XWV97vHvYwLKLSSQgGjAKCA91QraXY73ctGKSkG5tDCkFaSm9Va0tlZrR6nt6Rm2ansc2truQoieYlQQKJXKRavkaJGeDSIEQaFUaBAJl0C4WGPgd/7YK4ftdid7rZ291sya+/sZY4211jvnmu+TMeZI+PG+831TVUiSJEmS2uFJTQeQJEmSJM0eizxJkiRJahGLPEmSJElqEYs8SZIkSWoRizxJkiRJahGLPEmSJElqEYs8SZIkSWoRizxJkiRJahGLPEmSJElqkflNB+jW4sWLa9myZU3HkCRJkqRGXHPNNfdW1ZLpzhuaIm/ZsmWMjY01HUOSJEmSGpHk9m7OG5oib9M993PPJ/6h6RiSJAlY8pZTmo4gSdqCxp7JS7Iiyboka5rKIEmSJElt0+TCKyuBoxrsX5IkSZJap7Eir6pWA/c31b8kSZIktZFbKEiSJElSi2zXRV6S5UnGkozd9/CGpuNIkiRJ0nZvuy7yqmq0qkaqamTPXRc0HUeSJEmStnvbdZEnSZIkSepNk1sorAKuAg5KsjbJ6U1lkSRJkqS2aGwz9Ko6uam+JUmSJKmtGivyejV/yR4secspTceQJEmSpO2az+RJkiRJUosMzUjej+/5Pj/4xAeajiFJkqQBe+pbzmw6gjRUHMmTJEmSpBZppMhLsm+SryT5dpIbk5zRRA5JkiRJapumpmtuAt5dVdcm2Q24JskVVfXthvJIkiRJUis0MpJXVXdV1bWdzw8BNwFPbyKLJEmSJLVJ48/kJVkGPB+4eopjy5OMJRm77+FHBx1NkiRJkoZOo0Vekl2BzwPvrKoNk49X1WhVjVTVyJ677jz4gJIkSZI0ZBor8pLswHiBd25VXdBUDkmSJElqk6ZW1wxwNnBTVf1lExkkSZIkqY2aGsl7CfBrwBFJruu8XtNQFkmSJElqjUa2UKiqrwHp5Tc7LHkaT33LmX1KJEmSJEnt0PjqmpIkSZKk2WORJ0mSJEkt0sh0zZn44T23cuPfHNN0DEmSJInnvPXipiNIW9TU6ppPSfJvSb6V5MYkH2gihyRJkiS1TVMjeT8Cjqiqhzv75X0tyRer6hsN5ZEkSZKkVmhqdc0CHu583aHzqiaySJIkSVKbNLbwSpJ5Sa4D1gFXVNXVTWWRJEmSpLZorMirqseq6hBgKXBokudOPifJ8iRjScbWP7xx8CElSZIkacg0voVCVT0AfAU4aopjo1U1UlUji3bdcfDhJEmSJGnINLW65pIkCzufdwJeCdzcRBZJkiRJapOmVtfcB/hkknmMF5qfq6ovNJRFkiRJklqjqdU1rwee30TfkiRJktRmTY3k9WynJQfwnLde3HQMSZIkSdquNb7wiiRJkiRp9ljkSZIkSVKLDM10zYfuvYUvn/XapmNIkiRJ240jf+OSpiNoO9ToSF6SeUm+mcSVNSVJkiRpFjQ9XfMM4KaGM0iSJElSazRW5CVZCrwWOKupDJIkSZLUNk2O5P0v4D3A4w1mkCRJkqRWaaTIS3I0sK6qrpnmvOVJxpKMPfDQxgGlkyRJkqTh1dRI3kuAY5J8D/gMcESSf5h8UlWNVtVIVY0s3G3HQWeUJEmSpKHTSJFXVe+rqqVVtQw4CfiXqjqliSySJEmS1CZNr64pSZIkSZpFjW+GXlVfBb7acAxJkiRJaoXGi7xu7bb4QI78jUuajiFJkiRJ2zWna0qSJElSiwzNSN79997CZ855VdMxJEmSpNY66bTLmo6gWeBIniRJkiS1SGMjeZ098h4CHgM2VdVIU1kkSZIkqS2anq55eFXd23AGSZIkSWoNp2tKkiRJUos0WeQVcHmSa5Isn+qEJMuTjCUZe+jhjQOOJ0mSJEnDp8npmi+tqjuT7AVckeTmqlo98YSqGgVGAZ61bPdqIqQkSZIkDZPGRvKq6s7O+zrgQuDQprJIkiRJUls0UuQl2SXJbps/A78ErGkiiyRJkiS1SVPTNfcGLkyyOcOnq+rShrJIkiRJUms0UuRV1W3Awb38Zo/FB3LSaZf1KZEkSZIktYNbKEiSJElSi1jkSZIkSVKLNLmFQk/W3X8LHzv3VU3HkCRJkrQN3vEGH8Hqt8ZG8pIsTHJ+kpuT3JTkRU1lkSRJkqS2aHIk76PApVV1QpIdgZ0bzCJJkiRJrdBIkZdkd+DlwKkAVbUR2NhEFkmSJElqk6amaz4TuAc4J8k3k5zV2RRdkiRJkrQNmiry5gMvAD5RVc8HHgHeO/mkJMuTjCUZe3iDA32SJEmSNJ2miry1wNqqurrz/XzGi76fUFWjVTVSVSO7LthxoAElSZIkaRg1UuRV1Q+A/0xyUKfpSODbTWSRJEmSpDZpcnXNdwDndlbWvA04rcEskiRJktQKjRV5VXUdMNJU/5IkSZLURk2O5PVkrz0O5B1vuKzpGJIkSZK0XWtq4RVJkiRJUh8MzUje2vW38LvnH9V0DEmSJEmz6MMnXNp0hNZxJE+SJEmSWqSRIi/JQUmum/DakOSdTWSRJEmSpDZpZLpmVX0HOAQgyTzgTuDCJrJIkiRJUptsD9M1jwT+vapubzqIJEmSJA277aHIOwlYNdWBJMuTjCUZe3TDxgHHkiRJkqTh02iRl2RH4BjgvKmOV9VoVY1U1cjOC3YcbDhJkiRJGkJNj+S9Gri2qu5uOIckSZIktULTRd7JbGGqpiRJkiSpd40VeUl2AV4JXNBUBkmSJElqm0a2UACoqkeAPbs9f+miA/nwCZf2MZEkSZIkDb+mp2tKkiRJkmaRRZ4kSZIktUhj0zV7dcsDd/Dqi97RdAxJkiRJ27kvvu5jTUdoVJMLr7wryY1J1iRZleQpTWWRJEmSpLboushLskuSJ3U+PzvJMUl2mEmnSZ4O/DYwUlXPBeYBJ83kWpIkSZKkJ/QykrcaeEqnQLsc+DVg5Tb0PR/YKcl8YGfg+9twLUmSJEkSvRV5qapHgeOAv6mqE4HnzKTTqroT+DBwB3AX8GBVXT6Ta0mSJEmSntBTkZfkRcAbgEs6bfNm0mmSRcCxwDOBpwG7JDllivOWJxlLMrZxww9n0pUkSZIkzSm9FHlnAO8DLqyqG5M8C/jKDPt9BfAfVXVPVf0YuAB48eSTqmq0qkaqamTHBTvNsCtJkiRJmju62kIhyTzgmKo6ZnNbVd3G+OIpM3EH8MIkOwM/BI4ExmZ4LUmSJElSR1cjeVX1GPDS2eq0qq4GzgeuBW7o5BidretLkiRJ0lzVy2bo30xyMXAe8Mjmxqq6YCYdV9WZwJkz+a0kSZIkaWq9FHlPAe4DjpjQVow/T9d3By7cb87vXC9JkiRJ0+m6yKuq0/oZRJIkSZK07bpeXTPJ0iQXJlnXeX0+ydJ+hpMkSZIk9aaX6ZrnAJ8GTux8P6XT9srZDjWVWx64m9de8JFBdCVJkiSppS457t1NR+i7XvbJW1JV51TVps5rJbBkph0nOSPJmiQ3JnnnTK8jSZIkSXpCL0XefUlOSTKv8zqF8YVYepbkucCbgUOBg4Gjkxwwk2tJkiRJkp7QS5H3JuBXgB8AdwEnAKfOsN//BlxdVY9W1SbgSuC4GV5LkiRJktTRS5G3tKqOqaolVbVXVb0O2G+G/a4BXpZkzyQ7A68B9p3htSRJkiRJHb0UeVNtUjejjeuq6ibgz4HLgUuB64DHJp+XZHmSsSRjGx98ZPJhSZIkSdIk066umeRFwIuBJUl+Z8KhBcC8mXZcVWcDZ3f6+J/A2inOGQVGAXY/YN+aaV+SJEmSNFd0s4XCjsCunXN3m9C+gfHn8mYkyV5VtS7Jfow/j/fCmV5LkiRJkjRu2iKvqq4Erkyysqpun8W+P59kT+DHwNuq6oFZvLYkSZIkzUm9PJN3VpKFm78kWZTkspl2XFUvq6qfraqDq+rLM72OJEmSJOkJ3UzX3GzxxNG2qlqfZK8+ZJrSgQv3nhO700uSJEnStuhlJO/xzvNzACR5BuBiKJIkSZK0HellJO/9wNeSXAkEeBmwvC+ppnDr+ns5+vMrBtWdJEmSpDnmC8e/qekIs6LrIq+qLk3yAp5YBfOdVXVvf2JJkiRJkmZi2umaSX6m8/4CYD/g+53Xfp22rf12RZJ1SdZMaNsjyRVJbum8L9q2P4IkSZIkabNuRvLeDbwZ+MgUxwo4Yiu/XQl8HPjUhLb3Al+uqg8meW/n++93lVaSJEmStFXd7JP35s774b1evKpWJ1k2qflY4LDO508CX8UiT5IkSZJmxbRFXpLjtna8qi7osc+9q+quzucfAHtvpe/ldBZ32Wnxnj12I0mSJElzTzfTNX+5874X8GLgXzrfDwf+Fei1yPv/qqqSbHEbhqoaBUYBFu6/zO0aJEmSJGka3UzXPA0gyeXAz24ehUuyD+PP3PXq7iT7VNVdnWusm8E1JEmSJElT6GUz9H0nTLMEuJvx1TZ7dTHwxs7nNwL/OINrSJIkSZKm0Mtm6F9OchmwqvP9V4Evbe0HSVYxvsjK4iRrgTOBDwKfS3I6cDvwK72GliRJkiRNLVXdP+qW5PXAyztfV1fVhX1JNYWRkZEaGxsbVHeSJEmStF1Jck1VjUx3Xi8jeQDXAg9V1ZeS7Jxkt6p6aGYRJUmSJEmzretn8pK8GTgf+LtO09OBi/oRSpIkSZI0M72M5L0NOBS4GqCqbkmyV19STeHW9es5+rzzB9WdJEmSpDnmCyee0HSEWdHL6po/qqqNm78kmQ9s9YG+JCuSrEuyZkLbiUluTPJ4kmnnk0qSJEmSutdLkXdlkj8AdkrySuA84J+m+c1K4KhJbWuA44DVPfQtSZIkSepCL0Xee4F7gBuA3wT+GfjDrf2gqlYD909qu6mqvtNjTkmSJElSF7p6Ji/JPOBTVfUG4O/7G0mSJEmSNFNdjeRV1WPAM5Ls2Oc8PyHJ8iRjScY2btgwyK4lSZIkaSj1srrmbcDXk1wMPLK5sar+ctZTPXHtUWAUYOH++3e/a7skSZIkzVG9FHn/3nk9CditP3EkSZIkSdui6yKvqj4AkGTB+Nd6aLrfJFkFHAYsTrIWOJPxhVg+BiwBLklyXVW9agbZJUmSJEmTdF3kdfa0O4fOKF6SB4E3VdU1W/pNVZ28hUMX9hJSkiRJktSdXqZrrgDeWlX/ByDJSxkv+p7Xj2CTHbBoUWt2oJckSZKkfulln7zHNhd4AFX1NWDT7EeSJEmSJM1ULyN5Vyb5O2AVUMCvAl9N8gKAqrq2D/kkSZIkST3opcg7uPN+5qT25zNe9B0xK4m24Nb1G3jd+V/qZxeSJEmS5qiLTnhF0xFmTS+rax6+teNJ3lhVn5zUtgI4GlhXVc/ttH0I+GVgI+NbMpxWVQ/0GlySJEmS9NN6eSZvOmdM0bYSOGpS2xXAc6vqecB3gffNYgZJkiRJmtNms8jL5IaqWs34vngT2y6vqs0LtnwDWDqLGSRJkiRpTpvNIq9m8Js3AV+cxQySJEmSNKf1dSRvqycn72d8C4Zzt3LO8iRjScY2bnhwW/NJkiRJUut1vfBKkicDxwPLJv6uqv6k8/HrPVzrVMYXZDmyqrY4AlhVo8AowML9nz2TkUJJkiRJmlN62ULhH4EHgWuAH00+WFVv7+YiSY4C3gP8YlU92kP/kiRJkqRp9FLkLa2qyStlblWSVcBhwOIkaxnfY+99wJOBK5IAfKOqfquX60qSJEmSptZLkfevSX6uqm7o9gdVdfIUzWf30KckSZIkqQe9FHkvBU5N8h+MT9cMUJ397vrugEULWrULvSRJkiT1Qy9F3qv7lkKSJEmSNCu6LvKq6vZ+BpnObet/yImfv77JCJIkSZJa5rzjBzIxcaBmc5+8n5JkRZJ1SdZMaPvTJNcnuS7J5Ume1s8MkiRJkjSX9LXIA1YCk1fk/FBVPa+qDgG+APxRnzNIkiRJ0pzR1yKvqlYD909q2zDh6y6Am5xLkiRJ0izpZeGVWZPkz4BfZ3xz9cObyCBJkiRJbdTv6ZpTqqr3V9W+wLnA27d0XpLlScaSjP1ow/rBBZQkSZKkIdVIkTfBucDxWzpYVaNVNVJVI09esGiAsSRJkiRpOA28yEty4ISvxwI3DzqDJEmSJLVVX5/JS7IKOAxYnGQtcCbwmiQHAY8DtwO/1c8MkiRJkjSX9LXIq6qTp2g+u599SpIkSdJc1sjqmjPxrEU7tXI3ekmSJEmaTU0vvCJJkiRJmkUWeZIkSZLUIkMzXfOuB37Mn114V9MxJEmSJLXA+1+/T9MR+qavI3lJViRZl2TNFMfenaSSLO5nBkmSJEmaS/o9XXMlcNTkxiT7Ar8E3NHn/iVJkiRpTulrkVdVq4H7pzj0V8B7gOpn/5IkSZI01wx84ZUkxwJ3VtW3Bt23JEmSJLXdQBdeSbIz8AeMT9Xs5vzlwHKA3Zc8vY/JJEmSJKkdBj2Stz/wTOBbSb4HLAWuTfLUqU6uqtGqGqmqkV0W7DnAmJIkSZI0nAY6kldVNwB7bf7eKfRGqureQeaQJEmSpLbq9xYKq4CrgIOSrE1yej/7kyRJkqS5rq8jeVV18jTHl/Wzf0mSJEmaawY6XXNb7LNwh1bvSi9JkiRJs2HgWyhIkiRJkvpnaEbyHli/iYvOc30WSZIkSTP3uhMXNx2h7xzJkyRJkqQW6ffqmiuSrEuyZkLbHye5M8l1nddr+plBkiRJkuaSfo/krQSOmqL9r6rqkM7rn/ucQZIkSZLmjL4WeVW1Gri/n31IkiRJkp7Q1DN5b09yfWc656ItnZRkeZKxJGMbNtw3yHySJEmSNJSaKPI+AewPHALcBXxkSydW1WhVjVTVyIIFew4qnyRJkiQNrYEXeVV1d1U9VlWPA38PHDroDJIkSZLUVgMv8pLsM+Hr64E1WzpXkiRJktSbvm6GnmQVcBiwOMla4EzgsCSHAAV8D/jNfmaQJEmSpLmkr0VeVZ08RfPZM7nWwkXz58Tu9JIkSZK0LZpaXVOSJEmS1AcWeZIkSZLUIn2drjmbHrl3E/92zrqmY0iSJEkaMoeetlfTEQaqryN5nc3O1yVZM6n9HUluTnJjkr/oZwZJkiRJmkv6PV1zJXDUxIYkhwPHAgdX1XOAD/c5gyRJkiTNGX0t8qpqNXD/pOa3AB+sqh91znEOpiRJkiTNkiYWXnk28LIkVye5MskvNJBBkiRJklqpiYVX5gN7AC8EfgH4XJJnVVVNPjHJcmA5wFP3XDrQkJIkSZI0jJoYyVsLXFDj/g14HJhyl/OqGq2qkaoaWbjrngMNKUmSJEnDqIki7yLgcIAkzwZ2BO5tIIckSZIktU5fp2smWQUcBixOshY4E1gBrOhsq7AReONUUzUlSZIkSb3ra5FXVSdv4dAp/exXkiRJkuaqJhZemZFdFs+fczvVS5IkSVKvmngmT5IkSZLUJxZ5kiRJktQiQzNdc+PdP2bth3/QdAxJkiRJQ2Lp7z616QiN6OtIXpIVSdZ1VtLc3PbZJNd1Xt9Lcl0/M0iSJEnSXNLvkbyVwMeBT21uqKpf3fw5yUeAB/ucQZIkSZLmjH5vobA6ybKpjiUJ8CvAEf3MIEmSJElzSZMLr7wMuLuqbmkwgyRJkiS1SpNF3snAqq2dkGR5krEkY/c/fN+AYkmSJEnS8GqkyEsyHzgO+OzWzquq0aoaqaqRPXbdczDhJEmSJGmINTWS9wrg5qpa21D/kiRJktRK/d5CYRVwFXBQkrVJTu8cOolppmpKkiRJknrX79U1T95C+6n97FeSJEmS5qp+75M3a3bce4c5u2O9JEmSJHWrydU1JUmSJEmzbGhG8n5896Pc/VfXNR1DkiRJ0nZs73cd0nSExjmSJ0mSJEkt0u/VNVckWZdkzYS2Q5J8I8l1nY3OD+1nBkmSJEmaS/o9krcSOGpS218AH6iqQ4A/6nyXJEmSJM2CvhZ5VbUauH9yM7Cg83l34Pv9zCBJkiRJc0kTC6+8E7gsyYcZLzJfvKUTkywHlgMsXbTPYNJJkiRJ0hBrYuGVtwDvqqp9gXcBZ2/pxKoaraqRqhrZY5eFAwsoSZIkScOqiSLvjcAFnc/nAS68IkmSJEmzpIki7/vAL3Y+HwHc0kAGSZIkSWqlvj6Tl2QVcBiwOMla4EzgzcBHk8wH/ovOM3eSJEmSpG3X1yKvqk7ewqGf7/VaO+y9s7vXS5IkSdI0mpiuKUmSJEnqE4s8SZIkSWqRJvbJm5FN6zaw7mNXNB1DkiRJ0nZmr3e8sukI25W+juQlWZFkXZI1E9oOTnJVkhuS/FOSBf3MIEmSJElzSb+na64EjprUdhbw3qr6OeBC4Pf6nEGSJEmS5oy+FnlVtRq4f1Lzs4HVnc9XAMf3M4MkSZIkzSVNLLxyI3Bs5/OJwL4NZJAkSZKkVmqiyHsT8NYk1wC7ARu3dGKS5UnGkozd9/CDAwsoSZIkScNq4KtrVtXNwC8BJHk28NqtnDsKjAIcst+zayABJUmSJGmIDXwkL8lenfcnAX8I/O2gM0iSJElSW/V7C4VVwFXAQUnWJjkdODnJd4Gbge8D5/QzgyRJkiTNJX2drllVJ2/h0Ef72a8kSZIkzVUDfyZvpubvtcCd7CVJkiRpGqkajvVMkjwEfKfpHGqtxcC9TYdQq3mPqZ+8v9Rv3mPqN++x7jyjqpZMd9LQjOQB36mqkaZDqJ2SjHl/qZ+8x9RP3l/qN+8x9Zv32OxqYp88SZIkSVKfWORJkiRJUosMU5E32nQAtZr3l/rNe0z95P2lfvMeU795j82ioVl4RZIkSZI0vWEayZMkSZIkTcMiT5IkSZJaZLsr8pIcleQ7SW5N8t4pjj85yWc7x69OsmzwKTWsuri/Xp7k2iSbkpzQREYNty7usd9J8u0k1yf5cpJnNJFTw6mL++u3ktyQ5LokX0vys03k1PCa7h6bcN7xSSqJS96ra138HXZqkns6f4ddl+Q3msjZBttVkZdkHvDXwKuBnwVOnuIfqNOB9VV1APBXwJ8PNqWGVZf31x3AqcCnB5tObdDlPfZNYKSqngecD/zFYFNqWHV5f326qn6uqg5h/N76ywHH1BDr8h4jyW7AGcDVg02oYdbt/QV8tqoO6bzOGmjIFtmuijzgUODWqrqtqjYCnwGOnXTOscAnO5/PB45MkgFm1PCa9v6qqu9V1fXA400E1NDr5h77SlU92vn6DWDpgDNqeHVzf22Y8HUXwNXV1Itu/jsM4E8Z/5/s/zXIcBp63d5fmgXbW5H3dOA/J3xf22mb8pyq2gQ8COw5kHQadt3cX9K26PUeOx34Yl8TqU26ur+SvC3JvzM+kvfbA8qmdpj2HkvyAmDfqrpkkMHUCt3+G3l855GG85PsO5ho7bO9FXmSNCckOQUYAT7UdBa1S1X9dVXtD/w+8IdN51F7JHkS41OA3910FrXWPwHLOo80XMETs/fUo+2tyLsTmFixL+20TXlOkvnA7sB9A0mnYdfN/SVti67usSSvAN4PHFNVPxpQNg2/Xv8O+wzwur4mUttMd4/tBjwX+GqS7wEvBC528RV1adq/w6rqvgn/Lp4F/PyAsrXO9lbk/V/gwCTPTLIjcBJw8aRzLgbe2Pl8AvAv5Y7u6k4395e0Laa9x5I8H/g7xgu8dQ1k1PDq5v46cMLX1wK3DDCfht9W77GqerCqFlfVsqpaxvhzxcdU1VgzcTVkuvk7bJ8JX48BbhpgvlaZ33SAiapqU5K3A5cB84AVVXVjkj8BxqrqYuBs4H8nuRW4n/EbRJpWN/dXkl8ALgQWAb+c5ANV9ZwGY2uIdPl32IeAXYHzOmtG3VFVxzQWWkOjy/vr7Z2R4h8D63nif4pK0+ryHpNmpMv767eTHANsYvy/809tLPCQi4NgkiRJktQe29t0TUmSJEnSNrDIkyRJkqQWsciTJEmSpBaxyJMkSZKkFrHIkyRJkqQWsciTJM1JSU5N8vHO599K8utbOXdlkv9Icl2SbyU5cnBJJUnqzXa1T54kSU2oqr/t4rTfq6rzkxwOjAIHTvcDSZKa4EieJKmVklyU5JokNyZZ3mk7Lcl3k/wb8JIJ5/5xkt/t8tJXAU+fpu8PJvl2kuuTfLjT9swkVyW5Icn/SPLwDP9okiRtlSN5kqS2elNV3Z9kJ+D/JrkE+ADw88CDwFeAb87gukcBF23pYJI9gdcDP1NVlWRh59BHgU9U1aeSvG0G/UqS1BVH8iRJbfXbSb4FfAPYF/g14KtVdU9VbQQ+2+P1PpTku8CngT/fynkPAv8FnJ3kOODRTvtLgFWdz/+7x74lSeqaRZ4kqXWSHAa8AnhRVR3M+Ijdzdt42d+rqmcDvw+s2NJJVbUJOBQ4HzgauHTi4W3MIEnStCzyJElttDuwvqoeTfIzwAuBnYBfTLJnkh2AE2d47Y8DT0ryqqkOJtkV2L2q/hl4F3Bw59DXgZM6n98ww74lSZqWRZ4kqY0uBeYnuQn4IONTNu8C/pjxhVO+Dtw06TddjbJVVQH/A3jPFk7ZDfhCkuuBrwG/02k/A3hbkhuYZuEWSZK2Rcb/rZIkae5K8jHg2qo6Z4B9PlxVuw6qP0nS3OFIniRpTkvyp8B/By5uOoskSbPBkTxJkjqS/DUT9s/r+OiWRviSXAg8c1Lz71fVZf3IJ0lSNyzyJEmSJKlFnK4pSZIkSS1ikSdJkiRJLWKRJ0mSJEktYpEnSZIkSS1ikSdJkiRJLWKRJ0mSJEktYpEnSZIkSS1ikSdJkiRJLWKRJ0mSJEktYpEnSZIkSS1ikSdJkiRJLWKRJ0mSJEktYpEnSZIkSS1ikSdJkiRJLTK/6QDdWrx4cS1btqzpGJIkSZLUiGuuuebeqloy3XlDU+Qt3WUBXzz9nU3HkCRJktRSS95yStMRtirJ7d2c53RNSZIkSWqRxoq8JCuSrEuypqkMkiRJktQ2TY7krQSOarB/SZIkSWqdxoq8qloN3N9U/5IkSZJNqe1pAAAgAElEQVTURtv1M3lJlicZSzJ238Mbmo4jSZIkSdu97brIq6rRqhqpqpE9d13QdBxJkiRJ2u5t10WeJEmSJKk3FnmSJEmS1CJNbqGwCrgKOCjJ2iSnN5VFkiRJktpiflMdV9XJTfUtSZIkSW3VWJHXq/lL9mDJW05pOoYkSZIkbdd8Jk+SJEmSWmRoRvJ+fM/3+cEnPtB0DEmSJElD6qlvObPpCAPRyEhekn2TfCXJt5PcmOSMJnJIkiRJUts0NZK3CXh3VV2bZDfgmiRXVNW3G8ojSZIkSa3QyEheVd1VVdd2Pj8E3AQ8vYkskiRJktQmjS+8kmQZ8Hzg6maTSJIkSdLwa7TIS7Ir8HngnVW1YYrjy5OMJRm77+FHBx9QkiRJkoZMY0Vekh0YL/DOraoLpjqnqkaraqSqRvbcdefBBpQkSZKkIdTU6poBzgZuqqq/bCKDJEmSJLVRUyN5LwF+DTgiyXWd12sayiJJkiRJrdHIFgpV9TUgTfQtSZIkSW3W1D55PdthydPmzA71kiRJkjRTjW+hIEmSJEmaPRZ5kiRJktQiQzNd84f33MqNf3NM0zEkSZIkDaHnvPXipiMMTFNbKDwlyb8l+VaSG5N8oIkckiRJktQ2TY3k/Qg4oqoe7myK/rUkX6yqbzSUR5IkSZJaoaktFAp4uPN1h86rmsgiSZIkSW3S2MIrSeYluQ5YB1xRVVc3lUWSJEmS2qKxIq+qHquqQ4ClwKFJnjv5nCTLk4wlGVv/8MbBh5QkSZKkIdP4FgpV9QDwFeCoKY6NVtVIVY0s2nXHwYeTJEmSpCHT1OqaS5Is7HzeCXglcHMTWSRJkiSpTZpaXXMf4JNJ5jFeaH6uqr7QUBZJkiRJao2mVte8Hnh+E31LkiRJUps1NZLXs52WHDCndqmXJEmSpJlofOEVSZIkSdLsGZqRvIfuvYUvn/XapmNIkiRJGiJH/sYlTUcYOEfyJEmSJKlFGi3yksxL8s0krqwpSZIkSbOg6ZG8M4CbGs4gSZIkSa3RWJGXZCnwWuCspjJIkiRJUts0OZL3v4D3AI9v6YQky5OMJRl74KGNg0smSZIkSUOqkSIvydHAuqq6ZmvnVdVoVY1U1cjC3XYcUDpJkiRJGl5NjeS9BDgmyfeAzwBHJPmHhrJIkiRJUms0UuRV1fuqamlVLQNOAv6lqk5pIoskSZIktUnTq2tKkiRJkmZRqqrpDF0ZGRmpsbGxpmNIkiRJUiOSXFNVI9Od50ieJEmSJLWIRZ4kSZIktcj8pgN06/57b+Ez57yq6RiSJEmShshJp13WdISBa6zI62yf8BDwGLCpm7mlkiRJkqSta3ok7/CqurfhDJIkSZLUGj6TJ0mSJEkt0mSRV8DlSa5JsrzBHJIkSZLUGk1O13xpVd2ZZC/giiQ3V9XqiSd0ir/lAIv3fEoTGSVJkiRpqDQ2kldVd3be1wEXAodOcc5oVY1U1chuu+446IiSJEmSNHQaKfKS7JJkt82fgV8C1jSRRZIkSZLapKnpmnsDFybZnOHTVXVpQ1kkSZIkqTUaKfKq6jbg4Cb6liRJkqQ2a3qfvK7tsfjAOblbvSRJkiT1wn3yJEmSJKlFLPIkSZIkqUWGZrrmuvtv4WPnvqrpGJIkSZKGxDveMDcf92psJC/JwiTnJ7k5yU1JXtRUFkmSJElqiyZH8j4KXFpVJyTZEdi5wSySJEmS1AqNFHlJdgdeDpwKUFUbgY1NZJEkSZKkNmlquuYzgXuAc5J8M8lZSXZpKIskSZIktUZTRd584AXAJ6rq+cAjwHsnn5RkeZKxJGMPb3CgT5IkSZKm01SRtxZYW1VXd76fz3jR9xOqarSqRqpqZNcFOw40oCRJkiQNo0aKvKr6AfCfSQ7qNB0JfLuJLJIkSZLUJk2urvkO4NzOypq3Aac1mEWSJEmSWqGxIq+qrgNGmupfkiRJktqoyZG8nuy1x4Fzdsd6SZIkSepWUwuvSJIkSZL6YGhG8tauv4XfPf+opmNIkiRJGiIfPuHSpiMMnCN5kiRJktQijRR5SQ5Kct2E14Yk72wiiyRJkiS1SSPTNavqO8AhAEnmAXcCFzaRRZIkSZLaZHuYrnkk8O9VdXvTQSRJkiRp2G0PRd5JwKqpDiRZnmQsydijGzYOOJYkSZIkDZ9Gi7wkOwLHAOdNdbyqRqtqpKpGdl6w42DDSZIkSdIQanok79XAtVV1d8M5JEmSJKkVmi7yTmYLUzUlSZIkSb1rrMhLsgvwSuCCpjJIkiRJUts0soUCQFU9AuzZ7flLFx04J3erlyRJkqReND1dU5IkSZI0iyzyJEmSJKlFGpuu2atbHriDV1/0jqZjSJIkSRoSX3zdx5qO0IiuR/KS7JLkSZ3Pz05yTJIdZtpxkncluTHJmiSrkjxlpteSJEmSJI3rZbrmauApSZ4OXA78GrByJp12rvHbwEhVPReYB5w0k2tJkiRJkp7QS5GXqnoUOA74m6o6EXjONvQ9H9gpyXxgZ+D723AtSZIkSRI9FnlJXgS8Abik0zZvJp1W1Z3Ah4E7gLuAB6vq8plcS5IkSZL0hF6KvDOA9wEXVtWNSZ4FfGUmnSZZBBwLPBN4GrBLklOmOG95krEkYxs3/HAmXUmSJEnSnNLV6ppJ5gHHVNUxm9uq6jbGn6ubiVcA/1FV93SufwHwYuAfJp5UVaPAKMDuB+xVM+xLkiRJkuaMrkbyquox4KWz2O8dwAuT7JwkwJHATbN4fUmSJEmak3rZJ++bSS4GzgMe2dxYVRf02mlVXZ3kfOBaYBPwTTojdpIkSZKkmeulyHsKcB9wxIS2Anou8gCq6kzgzJn8VpIkSZI0tVQNx6NuIyMjNTY21nQMSZIkSWpEkmuqamS687peXTPJ0iQXJlnXeX0+ydJtiylJkiRJmk29TNc8B/g0cGLn+ymdtlfOdqip3PLA3bz2go8MoitJkiRJLXHJce9uOsLA9bJP3pKqOqeqNnVeK4ElfcolSZIkSZqBXoq8+5KckmRe53UK4wuxzEiSM5KsSXJjknfO9DqSJEmSpCf0UuS9CfgV4AfAXcAJwKkz6TTJc4E3A4cCBwNHJzlgJteSJEmSJD2hlyJvaVUdU1VLqmqvqnodsN8M+/1vwNVV9WhVbQKuBI6b4bUkSZIkSR29FHkf67KtG2uAlyXZM8nOwGuAfSeflGR5krEkYxsffOSnLiJJkiRJ+knTrq6Z5EXAi4ElSX5nwqEFwLyZdFpVNyX5c+By4BHgOuCxKc4bBUYBdj9g3+HY0E+SJEmSGtTNSN6OwK6MF4S7TXhtYPy5vBmpqrOr6uer6uXAeuC7M72WJEmSJGnctCN5VXUlcGWSlVV1+2x1nGSvqlqXZD/Gn8d74WxdW5IkSZLmql6eyTsrycLNX5IsSnLZNvT9+STfBv4JeFtVPbAN15IkSZIk0cVI3gSLJxZiVbU+yV4z7biqXjbT30qSJEmSptZLkfd4kv2q6g6AJM8ABrYYyoEL9+aS4949qO4kSZIkaSj1UuS9H/hakiuBAC8DlvcllSRJkiRpRrou8qrq0iQv4IkFUt5ZVff2J9ZPu3X9vRz9+RWD6k6SJElSC3zh+Dc1HWHgpl14JcnPdN5fAOwHfL/z2q/TtrXfrkiyLsmaCW17JLkiyS2d90Xb9keQJEmSJG3WzUjeu4E3Ax+Z4lgBR2zltyuBjwOfmtD2XuDLVfXBJO/tfP/9rtJKkiRJkraqm33y3tx5P7zXi1fV6iTLJjUfCxzW+fxJ4KtY5EmSJEnSrJi2yEty3NaOV9UFPfa5d1Xd1fn8A2DvHn8vSZIkSdqCbqZr/nLnfS/gxcC/dL4fDvwr0GuR9/9VVSXZ4jYMSZbTWcFzp8V7zrQbSZIkSZozupmueRpAksuBn908CpdkH8afuevV3Un2qaq7OtdYt5W+R4FRgIX7LxvYnnySJEmSNKymXV1zgn0nTLMEuJvx1TZ7dTHwxs7nNwL/OINrSJIkSZKm0Mtm6F9OchmwqvP9V4Evbe0HSVYxvsjK4iRrgTOBDwKfS3I6cDvwK72GliRJkiRNrZfN0N+e5PXAyztNo1V14TS/OXkLh47stl9JkiRJUvd6GckDuBZ4qKq+lGTnJLtV1UP9CDbZAYsWz8nd6iVJkiSpF10/k5fkzcD5wN91mp4OXNSPUJIkSZKkmell4ZW3AS8BNgBU1S2Mb6sgSZIkSdpO9DJd80dVtTEJAEnmAwPb1uDW9es5+rzzB9WdJEmSpJb4woknNB1hoHoZybsyyR8AOyV5JXAe8E9b+0GSFUnWJVkzoe3EJDcmeTzJyMxiS5IkSZKm0kuR917gHuAG4DeBfwb+cJrfrASOmtS2BjgOWN1D35IkSZKkLnQ1XTPJPOBTVfUG4O+7vXhVrU6ybFLbTZ1rdp9SkiRJktSVrkbyquox4BlJduxzHkmSJEnSNuhl4ZXbgK8nuRh4ZHNjVf3lrKfqSLIcWA6w0+LF/epGkiRJklqjlyLv3zuvJwG79SfOT6qqUWAUYOH++w9sJU9JkiRJGlZdF3lV9QGAJAvGv9ZDfUslSZIkSZqRrlfXTDKS5AbgeuCGJN9K8vPT/GYVcBVwUJK1SU5P8voka4EXAZckuWxb/gCSJEmSpCf0Ml1zBfDWqvo/AEleCpwDPG9LP6iqk7dw6MIe+pUkSZIkdamXIu+xzQUeQFV9LcmmPmSa0gGLFs25neolSZIkqVe9FHlXJvk7YBVQwK8CX03yAoCqurYP+SRJkiRJPeilyDu4837mpPbnM170HTEribbg1vUbeN35X+pnF5IkSZJa6qITXtF0hIHpZXXNw7d2PMkbq+qT2x5JkiRJkjRTXa+u2YUzJjckWZFkXZI1E9o+lOTmJNcnuTDJwlnMIEmSJElz2mwWeZmibSVw1KS2K4DnVtXzgO8C75vFDJIkSZI0p81mkVc/1VC1Grh/UtvlVbV5Vc5vAEtnMYMkSZIkzWn9HsmbzpuAL27xgsnyJGNJxjZueHDmySRJkiRpjuh64ZUkTwaOB5ZN/F1V/Unn49d76TjJ+4FNwLlbOqeqRoFRgIX7P/unRgolSZIkST+ply0U/hF4ELgG+NHkg1X19m4vlORU4GjgyKqyeJMkSZKkWdJLkbe0qiYvotKzJEcB7wF+saoe3dbrSZIkSZKe0Mszef+a5Od6uXiSVcBVwEFJ1iY5Hfg4sBtwRZLrkvxtL9eUJEmSJG1Zup0tmeTbwAHAfzA+XTNAdbZC6LuRkZEaGxsbRFeSJEmStN1Jck1VjUx3Xi/TNV+9DXkkSZIkSQPQdZFXVbf3M4gkSZIkadv1MpLXqNvW/5ATP3990zEkSZIkDanzjh/Ik2aNm83N0H9KkhVJ1iVZM6HtT5Nc31l05fIkT+tnBkmSJEmaS/pa5AErgcnbLnyoqp5XVYcAXwD+qM8ZJEmSJGnO6GuRV1WrgfsntW2Y8HUXwM3QJUmSJGmWNPJMXpI/A34deBA4vIkMkiRJktRG/Z6uOaWqen9V7QucC7x9S+clWZ5kLMnYjzasH1xASZIkSRpSjRR5E5wLHL+lg1U1WlUjVTXy5AWLBhhLkiRJkobTwIu8JAdO+HoscPOgM0iSJElSW/X1mbwkq4DDgMVJ1gJnAq9JchDwOHA78Fv9zCBJkiRJc0lfi7yqOnmK5rP72ackSZIkzWWNrK45E89atNOc2aFekiRJkmaq6YVXJEmSJEmzyCJPkiRJklpkaKZr3vXAj/mzC+9qOoYkSZKkIfb+1+/TdIS+6+tIXpIVSdYlWTPFsXcnqSSL+5lBkiRJkuaSfk/XXAkcNbkxyb7ALwF39Ll/SZIkSZpT+lrkVdVq4P4pDv0V8B6g+tm/JEmSJM01A194JcmxwJ1V9a1B9y1JkiRJbTfQhVeS7Az8AeNTNbs5fzmwHGD3JU/vYzJJkiRJaodBj+TtDzwT+FaS7wFLgWuTPHWqk6tqtKpGqmpklwV7DjCmJEmSJA2ngY7kVdUNwF6bv3cKvZGquneQOSRJkiSprfq9hcIq4CrgoCRrk5zez/4kSZIkaa7r60heVZ08zfFl/exfkiRJkuaagU7X3Bb7LNxhTuxOL0mSJEnbYuBbKEiSJEmS+mdoRvIeWL+Ji85zfRZJkiRJ2+Z1Jy5uOkJfOZInSZIkSS3S79U1VyT/r717j7KsrO/8//6EBuUqTTcgsTGgAvmpiS2WLLwGUbG9jHgBQ/80QWVs423UxFFIsoIm41rG8RLHrNFppWlNtBURjCHeSLz0mFG0wEaam6Bi0oiU0EBzyYjAd/44u6UsqrrrFLXPPnXq/VrrrNrn2bvO/nZ/eQ58eZ79PJlIsnlS29uTXJtkU/N6TpsxSJIkSdJi0vZI3npg1TTt76+qlc3rCy3HIEmSJEmLRqtFXlVtBLa2eQ9JkiRJ0r26eibv9Um+30znXDrTRUnWJBlPMr5t242DjE+SJEmSFqQuirwPAQ8HVgLXAe+d6cKqWltVY1U1ts8+ywYVnyRJkiQtWAMv8qrq+qq6u6ruAT4CHDXoGCRJkiRpVA28yEty0KS3LwQ2z3StJEmSJKk/rW6GnmQDcAywPMkW4HTgmCQrgQKuAV7dZgySJEmStJi0WuRV1eppms+Yy2ftu3TJyO9ML0mSJEn3V1era0qSJEmSWmCRJ0mSJEkjpNXpmvPp9hvu4jtnTnQdhiRJkqQRcNQrDug6hNa0OpLXbHY+kWTzlPY3JLkiyaVJ3t1mDJIkSZK0mLQ9XXM9sGpyQ5KnAccDj6mqRwHvaTkGSZIkSVo0Wi3yqmojsHVK82uAd1XVL5prnIMpSZIkSfOki4VXDgeekuSCJN9I8vgOYpAkSZKkkdTFwitLgP2Ao4HHA2cleVhV1dQLk6wB1gA8eNmKgQYpSZIkSQtRFyN5W4Bzquc7wD3AtLucV9XaqhqrqrF991o20CAlSZIkaSHqosj7HPA0gCSHA7sBN3QQhyRJkiSNnFanaybZABwDLE+yBTgdWAesa7ZVuBM4ebqpmpIkSZKk/rVa5FXV6hlOvazN+0qSJEnSYtXFwitzsufyJSO9K70kSZIkzYcunsmTJEmSJLVkwYzk3Xn9L9nynp91HYYkSZKkEbLiLQ/uOoR550ieJEmSJI2QVou8JOuSTDQraW5v+3SSTc3rmiSb2oxBkiRJkhaTtqdrrgf+Fvj49oaq+v3tx0neC9zScgySJEmStGi0vYXCxiSHTHcuSYCXAMe2GYMkSZIkLSZdPpP3FOD6qrpqpguSrEkynmR86203DjA0SZIkSVqYuizyVgMbdnRBVa2tqrGqGttvr2UDCkuSJEmSFq5OtlBIsgR4EfC4Lu4vSZIkSaOqq5G8ZwBXVNWWju4vSZIkSSOp7S0UNgDfAo5IsiXJKc2pk9jJVE1JkiRJUv/aXl1z9QztL2/zvpIkSZK0WHXyTN5c7Hbgrqx4y4O7DkOSJEmShlqXq2tKkiRJkubZghnJ++X1d3D9+zd1HYYkSZKkEXPgm1d2HcK8anvhlXVJJpJsntS2Msm3k2xqNjo/qs0YJEmSJGkxaXu65npg1ZS2dwPvqKqVwF807yVJkiRJ86DVIq+qNgJbpzYD+zTHDwJ+2mYMkiRJkrSYdPFM3puALyd5D70i84kdxCBJkiRJI6mL1TVfA7y5qg4G3gycMdOFSdY0z+2Nb7395oEFKEmSJEkLVRdF3snAOc3xZ4AZF16pqrVVNVZVY/vtue9AgpMkSZKkhayLIu+nwO81x8cCV3UQgyRJkiSNpFafyUuyATgGWJ5kC3A68CrgA0mWAP8XWNNmDJIkSZK0mLRa5FXV6hlOPa7N+0qSJEnSYtXF6ppzsuuBe4zcTvSSJEmSNN+6eCZPkiRJktQSizxJkiRJGiELZrrmXRPbmPjg+V2HIUmSJGkEHfCGZ3YdwrxpdSQvybokE0k2T2p7TJJvJbkkyT8m2afNGCRJkiRpMWl7uuZ6YNWUto8Cp1bV7wDnAv+15RgkSZIkadFotcirqo3A1inNhwMbm+PzgRe3GYMkSZIkLSZdLLxyKXB8c3wicHAHMUiSJEnSSOqiyHsl8NokFwJ7A3fOdGGSNUnGk4zfeNstAwtQkiRJkhaqga+uWVVXAMcBJDkceO4Orl0LrAVY+dDDayABSpIkSdICNvCRvCQHND9/A/hz4MODjkGSJEmSRlXbWyhsAL4FHJFkS5JTgNVJfgBcAfwUOLPNGCRJkiRpMWl1umZVrZ7h1AfavK8kSZIkLVYDfyZvrpYcsM9I7UIvSZIkSW1I1cJYzyTJrcCVXcehaS0Hbug6CE3L3AwvczO8zM3wMjfDy9wML3MzvOaSm9+qqv13dtGCGckDrqyqsa6D0H0lGTc3w8ncDC9zM7zMzfAyN8PL3AwvczO82sxNF/vkSZIkSZJaYpEnSZIkSSNkIRV5a7sOQDMyN8PL3AwvczO8zM3wMjfDy9wML3MzvFrLzYJZeEWSJEmStHMLaSRPkiRJkrQTQ1/kJVmV5MokVyc5tet4FqMk1yS5JMmmJONN235Jzk9yVfNzadOeJP+jydf3kxzZbfSjJcm6JBNJNk9q6zsXSU5urr8qycld/FlGzQy5eXuSa5u+synJcyadO63JzZVJnjWp3e+8eZbk4CRfS3JZkkuTvLFpt+90bAe5se90LMkDk3wnycVNbt7RtB+a5ILm7/nTSXZr2h/QvL+6OX/IpM+aNmeamx3kZn2SH0/qNyubdr/TBizJLkm+l+S85v3g+01VDe0L2AX4IfAwYDfgYuCRXce12F7ANcDyKW3vBk5tjk8F/ro5fg7wRSDA0cAFXcc/Si/gqcCRwOa55gLYD/hR83Npc7y06z/bQn/NkJu3A2+Z5tpHNt9nDwAObb7ndvE7r7XcHAQc2RzvDfygyYF9Z3hzY9/pPjcB9mqOdwUuaPrDWcBJTfuHgdc0x68FPtwcnwR8ekc56/rPt5BfO8jNeuCEaa73O23wOfpj4JPAec37gfebYR/JOwq4uqp+VFV3Ap8Cju84JvUcD3ysOf4Y8IJJ7R+vnm8D+yY5qIsAR1FVbQS2TmnuNxfPAs6vqq1VdRNwPrCq/ehH2wy5mcnxwKeq6hdV9WPganrfd37ntaCqrquqi5rjW4HLgYdg3+ncDnIzE/vOgDT//N/WvN21eRVwLHB20z6132zvT2cDT08SZs6Z5mgHuZmJ32kDlGQF8Fzgo8370EG/GfYi7yHAv096v4Udf/mrHQV8JcmFSdY0bQdW1XXN8c+AA5tjczZ4/ebCHA3W65vpMeu2TwfE3HSmmQrzWHr/59u+M0Sm5AbsO51rppxtAiboFQA/BG6uqruaSyb/Pf8qB835W4BlmJtWTM1NVW3vN+9s+s37kzygabPfDNbfAG8F7mneL6ODfjPsRZ6Gw5Or6kjg2cDrkjx18snqjSu7TOsQMBdD50PAw4GVwHXAe7sNZ3FLshfwWeBNVbVt8jn7TremyY19ZwhU1d1VtRJYQW8U4bc7DkmNqblJ8mjgNHo5ejy9KZhv6zDERSnJ84CJqrqw61iGvci7Fjh40vsVTZsGqKqubX5OAOfS+6K/fvs0zObnRHO5ORu8fnNhjgakqq5v/kV8D/AR7p1qYW4GLMmu9IqIT1TVOU2zfWcITJcb+85wqaqbga8BT6A31W9Jc2ry3/OvctCcfxBwI+amVZNys6qZ/lxV9QvgTOw3XXgS8Pwk19CbNn4s8AE66DfDXuR9FzisWZFmN3oPJH6+45gWlSR7Jtl7+zFwHLCZXh62r8J0MvAPzfHngT9sVnI6Grhl0nQotaPfXHwZOC7J0mYK1HFNm+bZlOdRX0iv70AvNyc1q2odChwGfAe/81rRPN9wBnB5Vb1v0in7Tsdmyo19p3tJ9k+yb3O8O/BMes9Mfg04oblsar/Z3p9OAL7ajJDPlDPN0Qy5uWLS/7QKvWe+Jvcbv9MGoKpOq6oVVXUIve+hr1bVS+mg3yzZ+SXdqaq7krye3j9wuwDrqurSjsNabA4Ezu19X7AE+GRVfSnJd4GzkpwC/AR4SXP9F+it4nQ1cAfwisGHPLqSbACOAZYn2QKcDryLPnJRVVuT/BW9/ygC+Muqmu2CIZrBDLk5Jr0lrIveKrWvBqiqS5OcBVwG3AW8rqrubj7H77z59yTgD4BLmmdYAP4U+84wmCk3q+07nTsI+FiSXegNCpxVVecluQz4VJL/BnyPXpFO8/PvklxNbxGqk2DHOdOczZSbrybZn94qmpuAP2qu9zute29jwP0mvWJRkiRJkjQKhn26piRJkiSpDxZ5kiRJkjRCLPIkSZIkaYRY5EmSJEnSCLHIkyRJkqQRYpEnSdIkSQ5Jsnma9o8meWQXMUmS1I+h3idPkqRhUVX/uesYJEmaDUfyJEm6ryVJPpHk8iRnJ9kjydeTjAEkWZXkoiQXJ/mXroOVJGkyR/IkSbqvI4BTqupfk6wDXrv9RJL9gY8AT62qHyfZr6sgJUmajiN5kiTd179X1b82x38PPHnSuaOBjVX1Y4Cq2jro4CRJ2hGLPEmS7qt28l6SpKFlkSdJ0n09NMkTmuP/H/jmpHPfBp6a5FAAp2tKkoaNRZ4kSfd1JfC6JJcDS4EPbT9RVT8H1gDnJLkY+HQ3IUqSNL1UOQNFkiRJkkaFI3mSJEmSNEIs8iRJkiRphFjkSZIkSdIIsciTJEmSpBFikSdJkiRJI8QiT5IkSZJGiEWeJEmSJI0QizxJkiRJGiEWeZIkSZI0QizyJEmSJGmEWORJkiRJ0gixyJMkSZKkEWKRJ0mSJEkjxCJPkiRJkkaIRZ4kSZIkjRCLPEmSJEkaIRZ5kiRJkjRCLPIkSZIkaYQs6TqA2Vq+fHkdcsghXYchSZIkSZ248MILb6iq/Xd23YIp8lbsuQ9fPOVNXYchSZIkaUTt/5qXdR3CDiX5yWyuc7qmJId0W0wAAB9FSURBVEmSJI2Qzoq8JOuSTCTZ3FUMkiRJkjRquhzJWw+s6vD+kiRJkjRyOivyqmojsLWr+0uSJEnSKBrqZ/KSrEkynmT8xtu2dR2OJEmSJA29oS7yqmptVY1V1diyvfbpOhxJkiRJGnpDXeRJkiRJkvpjkSdJkiRJI6TLLRQ2AN8CjkiyJckpXcUiSZIkSaNiSVc3rqrV/Vy/ZP/9hn4HekmSJEnqmtM1JUmSJGmEWORJkiRJ0gjpbLpmv37585/ysw+9o+swJEmSJC1AD37N6V2HMDCdjOQlOTjJ15JcluTSJG/sIg5JkiRJGjVdjeTdBfxJVV2UZG/gwiTnV9VlHcUjSZIkSSOhk5G8qrquqi5qjm8FLgce0kUskiRJkjRKOl94JckhwGOBC7qNRJIkSZIWvk6LvCR7AZ8F3lRV26Y5vybJeJLxG2+7Y/ABSpIkSdIC01mRl2RXegXeJ6rqnOmuqaq1VTVWVWPL9tpjsAFKkiRJ0gLU1eqaAc4ALq+q93URgyRJkiSNoq5G8p4E/AFwbJJNzes5HcUiSZIkSSOjky0UquqbQLq4tyRJkiSNsq72yevbrvv/5qLapV6SJEmS5qLzLRQkSZIkSfNnwYzk/cfPr+bS//n8rsOQJEmStIA86rWf7zqEgXMkT5IkSZJGSFdbKDwwyXeSXJzk0iTv6CIOSZIkSRo1XU3X/AVwbFXd1myK/s0kX6yqb3cUjyRJkiSNhK62UCjgtubtrs2ruohFkiRJkkZJZ8/kJdklySZgAji/qi7oKhZJkiRJGhWdFXlVdXdVrQRWAEclefTUa5KsSTKeZPym2+4cfJCSJEmStMB0vrpmVd0MfA1YNc25tVU1VlVjS/fabfDBSZIkSdIC09Xqmvsn2bc53h14JnBFF7FIkiRJ0ijpanXNg4CPJdmFXqF5VlWd11EskiRJkjQyulpd8/vAY7u4tyRJkiSNsq5G8vq2+/6P4FGv/XzXYUiSJEnSUOt84RVJkiRJ0vxZMCN5t95wFf/y0ed2HYYkSZKkBeDp//mfug6hM52O5DUbon8viYuuSJIkSdI86Hq65huByzuOQZIkSZJGRmdFXpIVwHOBj3YVgyRJkiSNmi5H8v4GeCtwT4cxSJIkSdJI6aTIS/I8YKKqLtzJdWuSjCcZv/nWOwcUnSRJkiQtXF2N5D0JeH6Sa4BPAccm+fupF1XV2qoaq6qxfffebdAxSpIkSdKC00mRV1WnVdWKqjoEOAn4alW9rItYJEmSJGmUdL26piRJkiRpHnW+GXpVfR34esdhSJIkSdJI6LzIm629lx+2qHetlyRJkqTZcLqmJEmSJI0QizxJkiRJGiELZrrm1huu4lNnPqvrMCRJkiQNuZNe8eWuQ+hUZ0Ves0fercDdwF1VNdZVLJIkSZI0KroeyXtaVd3QcQySJEmSNDJ8Jk+SJEmSRkiXRV4BX0lyYZI1HcYhSZIkSSOjy+maT66qa5McAJyf5Iqq2jj5gqb4WwOwfNkDu4hRkiRJkhaUzkbyqura5ucEcC5w1DTXrK2qsaoa23uv3QYdoiRJkiQtOJ0UeUn2TLL39mPgOGBzF7FIkiRJ0ijparrmgcC5SbbH8Mmq+lJHsUiSJEnSyOikyKuqHwGP6eLekiRJkjTKut4nb9b2W37Yot+5XpIkSZJ2xn3yJEmSJGmELJiRvImtV/HBTzyr6zAkSZIkDbE3vNTZf47kSZIkSdII6azIS7JvkrOTXJHk8iRP6CoWSZIkSRoVXU7X/ADwpao6IcluwB4dxiJJkiRJI6GTIi/Jg4CnAi8HqKo7gTu7iEWSJEmSRklX0zUPBX4OnJnke0k+mmTPqRclWZNkPMn4bdusASVJkiRpZ7oq8pYARwIfqqrHArcDp069qKrWVtVYVY3ttc9ug45RkiRJkhacroq8LcCWqrqgeX82vaJPkiRJknQ/dFLkVdXPgH9PckTT9HTgsi5ikSRJkqRR0uXqmm8APtGsrPkj4BUdxiJJkiRJIyFV1XUMszI2Nlbj4+NdhyFJkiRJnUhyYVWN7ey6zjZDlyRJkiTNP4s8SZIkSRohXT6T15ctN13FW85e1XUYkiRJkobUe074UtchDIVORvKSHJFk06TXtiRv6iIWSZIkSRolnYzkVdWVwEqAJLsA1wLndhGLJEmSJI2SYXgm7+nAD6vqJ10HIkmSJEkL3TAUeScBG7oOQpIkSZJGQadFXrMR+vOBz8xwfk2S8STjd2y7c7DBSZIkSdIC1PVI3rOBi6rq+ulOVtXaqhqrqrE99tltwKFJkiRJ0sLTdZG3GqdqSpIkSdK86azIS7In8EzgnK5ikCRJkqRR09lm6FV1O7Csq/tLkiRJ0ijqrMjr14qlh7mDvSRJkiTtRNfP5EmSJEmS5pFFniRJkiSNkAUzXfOqm/+NZ3/uDV2HIUmSJGkIffEFH+w6hKEx65G8JHsm+Y3m+PAkz0+y61xvnOTNSS5NsjnJhiQPnOtnSZIkSZJ6+pmuuRF4YJKHAF8B/gBYP5ebNp/xX4Cxqno0sAtw0lw+S5IkSZJ0r36KvFTVHcCLgP9ZVScCj7of914C7J5kCbAH8NP78VmSJEmSJPos8pI8AXgp8E9N2y5zuWlVXQu8B/g34Drglqr6ylw+S5IkSZJ0r36KvDcCpwHnVtWlSR4GfG0uN02yFDgeOBT4TWDPJC+b5ro1ScaTjN+57T/mcitJkiRJWlRmtbpmkl2A51fV87e3VdWP6D1XNxfPAH5cVT9vPv8c4InA30++qKrWAmsBHvSIA2qO95IkSZKkRWNWI3lVdTfw5Hm8778BRyfZI0mApwOXz+PnS5IkSdKi1M8+ed9L8nngM8Dt2xur6px+b1pVFyQ5G7gIuAv4Hs2InSRJkiRp7vop8h4I3AgcO6mtgL6LPICqOh04fS6/K0mSJEmaXqoWxqNuY2NjNT4+3nUYkiRJktSJJBdW1djOrpv16ppJViQ5N8lE8/pskhX3L0xJkiRJ0nzqZ7rmmcAngROb9y9r2p4530FN56qbr+e557x3ELeSJEmStMD804v+pOsQhkY/++TtX1VnVtVdzWs9sH9LcUmSJEmS5qCfIu/GJC9Lskvzehm9hVjmJMkbk2xOcmmSN831cyRJkiRJ9+qnyHsl8BLgZ8B1wAnAy+dy0ySPBl4FHAU8BnhekkfM5bMkSZIkSffqp8hbUVXPr6r9q+qAqnoB8NA53vf/Ay6oqjuq6i7gG8CL5vhZkiRJkqRGP0XeB2fZNhubgackWZZkD+A5wMFTL0qyJsl4kvE7b7n9Ph8iSZIkSfp1O11dM8kTgCcC+yf540mn9gF2mctNq+ryJH8NfAW4HdgE3D3NdWuBtQAPesTBC2NDP0mSJEnq0GxG8nYD9qJXEO496bWN3nN5c1JVZ1TV46rqqcBNwA/m+lmSJEmSpJ6djuRV1TeAbyRZX1U/ma8bJzmgqiaSPJTe83hHz9dnS5IkSdJi1c8zeR9Nsu/2N0mWJvny/bj3Z5NcBvwj8Lqquvl+fJYkSZIkiVmM5E2yfHIhVlU3JTlgrjeuqqf0c/1h+x7oLvaSJEmStBP9jOTd00ytBCDJbwEuhiJJkiRJQ6Sfkbw/A76Z5BtAgKcAa1qJSpIkSZI0J7Mu8qrqS0mO5N4FUt5UVTe0E9Z9XX3TDTzvs+sGdTtJkiRJC8h5L35l1yEMjZ1O10zy283PI4GHAj9tXg9t2nb0u+uSTCTZPKltvyTnJ7mq+bn0/v0RJEmSJEnbzWYk70+AVwHvneZcAcfu4HfXA38LfHxS26nAv1TVu5Kc2rx/26yilSRJkiTt0Gz2yXtV8/Np/X54VW1McsiU5uOBY5rjjwFfxyJPkiRJkubFTou8JC/a0fmqOqfPex5YVdc1xz8DDuzz9yVJkiRJM5jNdM3/1Pw8AHgi8NXm/dOA/wP0W+T9SlVVkhm3YUiyhmYFz92XL5vrbSRJkiRp0ZjNdM1XACT5CvDI7aNwSQ6i98xdv65PclBVXdd8xsQO7r0WWAuw78MPcU8+SZIkSdqJfjZDP3jSNEuA6+mtttmvzwMnN8cnA/8wh8+QJEmSJE2jn83Q/yXJl4ENzfvfB/55R7+QZAO9RVaWJ9kCnA68CzgrySnAT4CX9Bu0JEmSJGl6/WyG/vokLwSe2jStrapzd/I7q2c49fTZ3leSJEmSNHv9jOQBXATcWlX/nGSPJHtX1a1tBDbVI5Yudxd7SZIkSdqJWT+Tl+RVwNnA/2qaHgJ8ro2gJEmSJElz089I3uuAo4ALAKrqqiQHtBLVNK6+6Sae95mzB3U7SZIkSQvIeSee0HUIQ6Of1TV/UVV3bn+TZAngtgaSJEmSNET6KfK+keRPgd2TPBP4DPCPO/qFJOuSTCTZPKntxCSXJrknydjcwpYkSZIkTaefIu9U4OfAJcCrgS8Af76T31kPrJrSthl4EbCxj3tLkiRJkmZhVs/kJdkF+HhVvRT4yGw/vKo2JjlkStvlzWfOPkpJkiRJ0qzMaiSvqu4GfivJbi3HI0mSJEm6H/pZXfNHwL8m+Txw+/bGqnrfvEfVSLIGWAOw+/Llbd1GkiRJkkZGP0XeD5vXbwB7txPOr6uqtcBagH0f/nBX8pQkSZKknZh1kVdV7wBIsk/vbd3aWlSSJEmSpDmZ9eqaScaSXAJ8H7gkycVJHreT39kAfAs4IsmWJKckeWGSLcATgH9K8uX78weQJEmSJN2rn+ma64DXVtX/BkjyZOBM4Hdn+oWqWj3DqXP7uK8kSZIkaZb6KfLu3l7gAVTVN5Pc1UJM03rE0qWcd+IJg7qdJEmSJC1I/RR530jyv4ANQAG/D3w9yZEAVXVRC/FJkiRJkvrQT5H3mObn6VPaH0uv6Dt2XiKawdU3beMFZ/9zm7eQJEmStEB97oRndB3C0Ohndc2n7eh8kpOr6mNT2tYBzwMmqurRTdt/B/4TcCe9LRleUVU39xu4JEmSJOm+Zr265iy8cZq29cCqKW3nA4+uqt8FfgCcNo8xSJIkSdKiNp9FXqY2VNVGYOuUtq9U1fYFW74NrJjHGCRJkiRpUZvPIq/m8DuvBL44jzFIkiRJ0qLW6kjeDi9O/gy4C/jEDq5Zk2Q8yfid2265v/FJkiRJ0sib9cIrSR4AvBg4ZPLvVdVfNof/2sdnvZzegixPr6oZRwCrai2wFmDfhx8+l5FCSZIkSVpU+tlC4R+AW4ALgV9MPVlVr5/NhyRZBbwV+L2quqOP+0uSJEmSdqKfIm9FVU1dKXOHkmwAjgGWJ9lCb4+904AHAOcnAfh2Vf1RP58rSZIkSZpeP0Xe/0nyO1V1yWx/oapWT9N8Rh/3lCRJkiT1oZ8i78nAy5P8mN50zQDV7HfXukcs3cdd7CVJkiRpJ/op8p7dWhSSJEmSpHkx6yKvqn7SZiCSJEmSpPuvn5G8Tv3opv/gxM9+v+swJEmSJA2hz7x4IE+RLQjzuRn6fSRZl2QiyeZJbX+V5PtJNiX5SpLfbDMGSZIkSVpMWi3ygPXA1G0X/ntV/W5VrQTOA/6i5RgkSZIkadFotcirqo3A1ilt2ya93ROoNmOQJEmSpMWkk2fykrwT+EPgFuBpXcQgSZIkSaOo7ema06qqP6uqg4FPAK+f6boka5KMJxn/xbabBhegJEmSJC1QnRR5k3wCePFMJ6tqbVWNVdXYA/ZZOsCwJEmSJGlhGniRl+SwSW+PB64YdAySJEmSNKpafSYvyQbgGGB5ki3A6cBzkhwB3AP8BPijNmOQJEmSpMWk1SKvqlZP03xGm/eUJEmSpMWsk9U15+JhS3d3F3tJkiRJ2omuF16RJEmSJM2jBTOSd93Nv+Sd517XdRiSJEmShtSfvfCgrkMYCo7kSZIkSdIIabXIS7IuyUSSzdOc+5MklWR5mzFIkiRJ0mLS9kjeemDV1MYkBwPHAf/W8v0lSZIkaVFptcirqo3A1mlOvR94K1Bt3l+SJEmSFpuBP5OX5Hjg2qq6eBbXrkkynmT89m03DiA6SZIkSVrYBlrkJdkD+FPgL2ZzfVWtraqxqhrbc59l7QYnSZIkSSNg0CN5DwcOBS5Ocg2wArgoyYMHHIckSZIkjaSB7pNXVZcAB2x/3xR6Y1V1wyDjkCRJkqRR1fYWChuAbwFHJNmS5JQ27ydJkiRJi12rI3lVtXon5w+Z7WcdtO+u7mAvSZIkSTsx8NU1JUmSJEntsciTJEmSpBEy0IVX7o+bb7qLz33G9VkkSZIkTe8FJy7vOoSh0PbCK+uSTCTZPKnt7UmuTbKpeT2nzRgkSZIkaTFpe7rmemDVNO3vr6qVzesLLccgSZIkSYtGq0VeVW0EtrZ5D0mSJEnSvbpaeOX1Sb7fTOdc2lEMkiRJkjRyuijyPgQ8HFgJXAe8d6YLk6xJMp5kfNu2GwcVnyRJkiQtWAMv8qrq+qq6u6ruAT4CHLWDa9dW1VhVje2zz7LBBSlJkiRJC9TAi7wkB016+0Jg80zXSpIkSZL60+o+eUk2AMcAy5NsAU4HjkmyEijgGuDVbcYgSZIkSYtJq0VeVa2epvmMNu8pSZIkSYtZq0XefNp36RJ3sJckSZKknehqCwVJkiRJUgss8iRJkiRphCyY6Zq333AX3zlzouswJEmSJA2po15xQNchDIVWR/KSrEsykWTzlPY3JLkiyaVJ3t1mDJIkSZK0mLQ9XXM9sGpyQ5KnAccDj6mqRwHvaTkGSZIkSVo0Wi3yqmojsHVK82uAd1XVL5prnIMpSZIkSfOki4VXDgeekuSCJN9I8vgOYpAkSZKkkdTFwitLgP2Ao4HHA2cleVhV1dQLk6wB1gA8eNmKgQYpSZIkSQtRFyN5W4Bzquc7wD3AtLucV9XaqhqrqrF991o20CAlSZIkaSHqosj7HPA0gCSHA7sBN3QQhyRJkiSNnFanaybZABwDLE+yBTgdWAesa7ZVuBM4ebqpmpIkSZKk/rVa5FXV6hlOvazN+0qSJEnSYtXFwitzsufyJe5gL0mSJEk70cUzeZIkSZKkliyYkbw7r/8lW97zs67DkCRJkjTEVrzlwV2H0DlH8iRJkiRphLRa5CVZl2SiWUlze9unk2xqXtck2dRmDJIkSZK0mLQ9XXM98LfAx7c3VNXvbz9O8l7glpZjkCRJkqRFo+0tFDYmOWS6c0kCvAQ4ts0YJEmSJGkx6fKZvKcA11fVVTNdkGRNkvEk41tvu3GAoUmSJEnSwtRlkbca2LCjC6pqbVWNVdXYfnstG1BYkiRJkrRwdbKFQpIlwIuAx3Vxf0mSJEkaVV2N5D0DuKKqtnR0f0mSJEkaSW1vobAB+BZwRJItSU5pTp3ETqZqSpIkSZL61/bqmqtnaH95v5+124G7unu9JEmSJO1ElwuvSJIkSZLmmUWeJEmSJI2QTlbXnItfXn8H179/U9dhSJIkSRpiB755ZdchdK7thVfWJZlIsnlS28ok306yqdno/Kg2Y5AkSZKkxaTt6ZrrgVVT2t4NvKOqVgJ/0byXJEmSJM2DVou8qtoIbJ3aDOzTHD8I+GmbMUiSJEnSYtLFM3lvAr6c5D30iswndhCDJEmSJI2kLlbXfA3w5qo6GHgzcMZMFyZZ0zy3N7719psHFqAkSZIkLVRdFHknA+c0x58BZlx4parWVtVYVY3tt+e+AwlOkiRJkhayLoq8nwK/1xwfC1zVQQySJEmSNJJafSYvyQbgGGB5ki3A6cCrgA8kWQL8X2BNmzFIkiRJ0mLSapFXVatnOPW4Nu8rSZIkSYtVF6trzsmuB+7h7vWSJEmStBNdPJMnSZIkSWrJghnJu2tiGxMfPL/rMCRJkiQNuQPe8MyuQ+iUI3mSJEmSNEJaLfKSrEsykWTzpLbHJPlWkkuS/GOSfdqMQZIkSZIWk7ZH8tYDq6a0fRQ4tap+BzgX+K8txyBJkiRJi0arRV5VbQS2Tmk+HNjYHJ8PvLjNGCRJkiRpMenimbxLgeOb4xOBg2e6MMmaJONJxm+87ZaBBCdJkiRJC1kXRd4rgdcmuRDYG7hzpguram1VjVXV2LK9HjSwACVJkiRpoRr4FgpVdQVwHECSw4HnDjoGSZIkSRpVAx/JS3JA8/M3gD8HPjzoGCRJkiRpVLW9hcIG4FvAEUm2JDkFWJ3kB8AVwE+BM9uMQZIkSZIWk1ana1bV6hlOfaDN+0qSJEnSYjXwZ/LmaskB+3DAG57ZdRiSJEmSNNRSVV3HMCtJbgWu7DoOTWs5cEPXQWha5mZ4mZvhZn6Gl7kZXuZmeJmb4dVvbn6rqvbf2UULZiQPuLKqxroOQveVZNzcDCdzM7zMzXAzP8PL3AwvczO8zM3wais3XeyTJ0mSJElqiUWeJEmSJI2QhVTkre06AM3I3AwvczO8zM1wMz/Dy9wML3MzvMzN8GolNwtm4RVJkiRJ0s4tpJE8SZIkSdJODH2Rl2RVkiuTXJ3k1K7jWYySXJPkkiSbkow3bfslOT/JVc3PpU17kvyPJl/fT3Jkt9GPniTrkkwk2Typre98JDm5uf6qJCd38WcZNTPk5u1Jrm36z6Ykz5l07rQmN1cmedakdr/35lmSg5N8LcllSS5N8sam3b7TsR3kxr7TsSQPTPKdJBc3uXlH035okguav+dPJ9mtaX9A8/7q5vwhkz5r2pxpbnaQm/VJfjyp36xs2v1OG7AkuyT5XpLzmveD7TdVNbQvYBfgh8DDgN2Ai4FHdh3XYnsB1wDLp7S9Gzi1OT4V+Ovm+DnAF4EARwMXdB3/qL2ApwJHApvnmg9gP+BHzc+lzfHSrv9sC/01Q27eDrxlmmsf2XynPQA4tPmu28XvvdZycxBwZHO8N/CDJgf2neHNjX2n+9wE2Ks53hW4oOkPZwEnNe0fBl7THL8W+HBzfBLw6R3lrOs/30J+7SA364ETprne77TB5+iPgU8C5zXvB9pvhn0k7yjg6qr6UVXdCXwKOL7jmNRzPPCx5vhjwAsmtX+8er4N7JvkoC4CHFVVtRHYOqW533w8Czi/qrZW1U3A+cCq9qMfbTPkZibHA5+qql9U1Y+Bq+l95/m914Kquq6qLmqObwUuBx6CfadzO8jNTOw7A9L8839b83bX5lXAscDZTfvUfrO9P50NPD1JmDlnmqMd5GYmfqcNUJIVwHOBjzbvw4D7zbAXeQ8B/n3S+y3s+Itf7SjgK0kuTLKmaTuwqq5rjn8GHNgcm7Nu9JsP8zRYr2+mx6zbPh0Qc9OZZirMY+n9n2/7zhCZkhuw73SumXK2CZigVwD8ELi5qu5qLpn89/yrHDTnbwGWYW5aMTU3VbW937yz6TfvT/KAps1+M1h/A7wVuKd5v4wB95thL/I0HJ5cVUcCzwZel+Spk09Wb0zZZVqHhPkYOh8CHg6sBK4D3tttOItbkr2AzwJvqqptk8/Zd7o1TW7sO0Ogqu6uqpXACnqjCL/dcUhqTM1NkkcDp9HL0ePpTcF8W4chLkpJngdMVNWFXcYx7EXetcDBk96vaNo0QFV1bfNzAjiX3pf89dunYTY/J5rLzVk3+s2HeRqQqrq++RfxPcBHuHeqhbkZsCS70isiPlFV5zTN9p0hMF1u7DvDpapuBr4GPIHeVL8lzanJf8+/ykFz/kHAjZibVk3Kzapm+nNV1S+AM7HfdOFJwPOTXENv2vixwAcYcL8Z9iLvu8BhzWo0u9F7GPHzHce0qCTZM8ne24+B44DN9PKwfQWmk4F/aI4/D/xhs4rT0cAtk6ZCqT395uPLwHFJljZToI5r2jTPpjyT+kJ6/Qd6uTmpWVXrUOAw4Dv4vdeK5vmGM4DLq+p9k07Zdzo2U27sO91Lsn+SfZvj3YFn0ntm8mvACc1lU/vN9v50AvDVZoR8ppxpjmbIzRWT/qdV6D3zNbnf+J02AFV1WlWtqKpD6H0PfbWqXsqA+82SnV/Snaq6K8nr6f3Dtguwrqou7TisxeZA4NzedwVLgE9W1ZeSfBc4K8kpwE+AlzTXf4HeCk5XA3cArxh8yKMtyQbgGGB5ki3A6cC76CMfVbU1yV/R+48igL+sqtkuGKIZzJCbY9JbwrrorVT7aoCqujTJWcBlwF3A66rq7uZz/N6bf08C/gC4pHmGBeBPse8Mg5lys9q+07mDgI8l2YXewMBZVXVeksuATyX5b8D36BXpND//LsnV9BahOgl2nDPN2Uy5+WqS/emtorkJ+KPmer/Tuvc2Bthv0isUJUmSJEmjYNina0qSJEmS+mCRJ0mSJEkjxCJPkiRJkkaIRZ4kSZIkjRCLPEmSJEkaIRZ5kiTNQpIvbN+XSpKkYeYWCpIkSZI0QhzJkyRpiiSfS3JhkkuTrGnarkmyvDn+wyTfT3Jxkr/rNlpJkn6dI3mSJE2RZL+q2ppkd+C7wO8BFwJjwIHAucATq+qG7dd2GK4kSb9mSdcBSJI0hP5Lkhc2xwcDh006dyzwmaq6AcACT5I0bCzyJEmaJMkxwDOAJ1TVHUm+Djyw06AkSeqDz+RJkvTrHgTc1BR4vw0cPeX8V4ETkyyD3tTOQQcoSdKOWORJkvTrvgQsSXI58C7g25NPVtWlwDuBbyS5GHjf4EOUJGlmLrwiSZIkSSPEkTxJkiRJGiEWeZIkSZI0QizyJEmSJGmEWORJkiRJ0gixyJMkSZKkEWKRJ0mSJEkjxCJPkiRJkkaIRZ4kSZIkjZD/B+iyg0oS5LtbAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 1080x1080 with 3 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Plot the error metrics\n",
    "_, (ax1, ax2, ax3) = plt.subplots(nrows=3, figsize=(15,15))\n",
    "sns.barplot(y='n_predictors', x='adj_R_sq', data=best, ax=ax1, orient='h')\n",
    "sns.barplot(y='n_predictors', x='bic', data=best, ax=ax2, orient='h')\n",
    "sns.barplot(y='n_predictors', x='aic', data=best, ax=ax3, orient='h')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 314,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>model</th>\n",
       "      <th>n_predictors</th>\n",
       "      <th>MSE</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>&lt;statsmodels.regression.linear_model.Regressio...</td>\n",
       "      <td>1</td>\n",
       "      <td>135508.216515</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>&lt;statsmodels.regression.linear_model.Regressio...</td>\n",
       "      <td>2</td>\n",
       "      <td>147033.581930</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>&lt;statsmodels.regression.linear_model.Regressio...</td>\n",
       "      <td>3</td>\n",
       "      <td>147669.747336</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>&lt;statsmodels.regression.linear_model.Regressio...</td>\n",
       "      <td>4</td>\n",
       "      <td>128926.294401</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>&lt;statsmodels.regression.linear_model.Regressio...</td>\n",
       "      <td>5</td>\n",
       "      <td>127644.577719</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>&lt;statsmodels.regression.linear_model.Regressio...</td>\n",
       "      <td>6</td>\n",
       "      <td>125550.266428</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>&lt;statsmodels.regression.linear_model.Regressio...</td>\n",
       "      <td>7</td>\n",
       "      <td>121290.957269</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>&lt;statsmodels.regression.linear_model.Regressio...</td>\n",
       "      <td>8</td>\n",
       "      <td>116963.477452</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>&lt;statsmodels.regression.linear_model.Regressio...</td>\n",
       "      <td>9</td>\n",
       "      <td>120501.056237</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>&lt;statsmodels.regression.linear_model.Regressio...</td>\n",
       "      <td>10</td>\n",
       "      <td>115206.332819</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>&lt;statsmodels.regression.linear_model.Regressio...</td>\n",
       "      <td>11</td>\n",
       "      <td>121145.228980</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>&lt;statsmodels.regression.linear_model.Regressio...</td>\n",
       "      <td>12</td>\n",
       "      <td>128229.611960</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>&lt;statsmodels.regression.linear_model.Regressio...</td>\n",
       "      <td>13</td>\n",
       "      <td>128106.132018</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>&lt;statsmodels.regression.linear_model.Regressio...</td>\n",
       "      <td>14</td>\n",
       "      <td>127551.688712</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>&lt;statsmodels.regression.linear_model.Regressio...</td>\n",
       "      <td>15</td>\n",
       "      <td>128313.047544</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>&lt;statsmodels.regression.linear_model.Regressio...</td>\n",
       "      <td>16</td>\n",
       "      <td>128277.486556</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>&lt;statsmodels.regression.linear_model.Regressio...</td>\n",
       "      <td>17</td>\n",
       "      <td>128137.435745</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>&lt;statsmodels.regression.linear_model.Regressio...</td>\n",
       "      <td>18</td>\n",
       "      <td>128193.974535</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>&lt;statsmodels.regression.linear_model.Regressio...</td>\n",
       "      <td>19</td>\n",
       "      <td>128017.675903</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                model  n_predictors  \\\n",
       "0   <statsmodels.regression.linear_model.Regressio...             1   \n",
       "1   <statsmodels.regression.linear_model.Regressio...             2   \n",
       "2   <statsmodels.regression.linear_model.Regressio...             3   \n",
       "3   <statsmodels.regression.linear_model.Regressio...             4   \n",
       "4   <statsmodels.regression.linear_model.Regressio...             5   \n",
       "5   <statsmodels.regression.linear_model.Regressio...             6   \n",
       "6   <statsmodels.regression.linear_model.Regressio...             7   \n",
       "7   <statsmodels.regression.linear_model.Regressio...             8   \n",
       "8   <statsmodels.regression.linear_model.Regressio...             9   \n",
       "9   <statsmodels.regression.linear_model.Regressio...            10   \n",
       "10  <statsmodels.regression.linear_model.Regressio...            11   \n",
       "11  <statsmodels.regression.linear_model.Regressio...            12   \n",
       "12  <statsmodels.regression.linear_model.Regressio...            13   \n",
       "13  <statsmodels.regression.linear_model.Regressio...            14   \n",
       "14  <statsmodels.regression.linear_model.Regressio...            15   \n",
       "15  <statsmodels.regression.linear_model.Regressio...            16   \n",
       "16  <statsmodels.regression.linear_model.Regressio...            17   \n",
       "17  <statsmodels.regression.linear_model.Regressio...            18   \n",
       "18  <statsmodels.regression.linear_model.Regressio...            19   \n",
       "\n",
       "              MSE  \n",
       "0   135508.216515  \n",
       "1   147033.581930  \n",
       "2   147669.747336  \n",
       "3   128926.294401  \n",
       "4   127644.577719  \n",
       "5   125550.266428  \n",
       "6   121290.957269  \n",
       "7   116963.477452  \n",
       "8   120501.056237  \n",
       "9   115206.332819  \n",
       "10  121145.228980  \n",
       "11  128229.611960  \n",
       "12  128106.132018  \n",
       "13  127551.688712  \n",
       "14  128313.047544  \n",
       "15  128277.486556  \n",
       "16  128137.435745  \n",
       "17  128193.974535  \n",
       "18  128017.675903  "
      ]
     },
     "execution_count": 314,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Select between models with different numbers of predictors using the validation set approach\n",
    "train, test = model_selection.train_test_split(hitters_dat, test_size=0.2)\n",
    "best = fwd_selection('Salary', train)\n",
    "best['MSE'] = best['model'].map(lambda m: metrics.mean_squared_error(test['Salary'], m.predict(test)))\n",
    "best"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 315,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x1301f4cf8>"
      ]
     },
     "execution_count": 315,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYMAAAEKCAYAAADw2zkCAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAGxRJREFUeJzt3Xu4XFWZ5/HvT0KAhIQEEkIkaACFGaUl4GlGFBRQlNuAN2zygM2tTQ8oAzQ9Nkg/0vQ8zqOIrSgtQkMItBCFcJHBCyBtk7ZbQ59AgADBRG6GS0IgEG5jCLzzx14nqXM4J1W7Tu1adajf53nqObvWXrXXm30uK2uvvd6tiMDMzLrb23IHYGZm+bkzMDMzdwZmZubOwMzMcGdgZma4MzAzM9wZmJkZ7gzMzAx3BmZmBozKHUCjJk2aFNOnT88dhpnZiLJw4cJVETG5Xr0R0xlMnz6d3t7e3GGYmY0okh5rpF62zkDSbOAwYGVE7Fav/rpnnuOZi35YfWBmZnVMPumY3CG0XM45gznAQRnbNzOzJFtnEBHzgedytW9mZhv4biIzM+vszkDSLEm9knqffWlN7nDMzN6yOroziIhLIqInInq22XJ87nDMzN6yOrozMDOz9sjWGUiaC/wG2FXSckkn5orFzKzbZVtnEBEzc7VtZmb9jZgVyKMmb/2WXOhhZtYJPGdgZmYjZ2Tw2jNP8vRF5+YOw8ysLbY76Zy2tpdlZCBpB0m/kvSApPslnZojDjMzK+QaGawDzoiIuySNAxZKui0iHsgUj5lZV8syMoiIpyLirrT9IvAgsH2OWMzMrAMmkCVNB/YAFuSNxMyse2XtDCRtCVwHnBYRb0o+1D830SvtD9DMrEvkXIG8KUVHcFVEXD9Ynf65ica0N0Azsy6S624iAZcBD0bEP+SIwczMNsg1MvgQ8HngAEmL0uuQTLGYmXW9LLeWRsSvAZX5zKaT3972RRhmZt0i+91EZmaW34hJR/HqM8u4//uH5w7DzKyt3nvyTW1pJ9cE8uaS7pR0T0pH4aRDZmYZ5RoZ/BE4ICJeSreY/lrSzyPit5niMTPrarkmkAN4Kb3dNL0iRyxmZpZ30dkmkhYBK4HbIsLpKMzMMsnWGUTE6xExA5gG7CVpt4F1atNRrH5pbfuDNDPrEtlvLY2I54FfAQcNsm99OoqJW45uf3BmZl0i191EkyVNSNtbAAcCS3LEYmZm+e4mmgpcIWkTig7pmoi4OVMsZmZdL9fdRPdSPMPAzMw6wIhZgbzF5He1bSWemVm3yT6BbGZm+Y2YkcGLq5Zy+6WH5g7DzN5CPvoXP80dQsfI/djLTSTdLcmTx2ZmGeW+THQq8GDmGMzMul7OdBTTgEOBS3PFYGZmhZwjg+8AXwbeGKpCbTqK5190Ogozs6rkWoF8GLAyIhZurF5tOooJ45yOwsysKrlGBh8CDpf0KPAj4ABJP8wUi5lZ18vSGUTEWRExLSKmA0cB/xIRx+SIxczM8t9NZGZmHUDFQ8c6X09PT/T29uYOw8xsRJG0MCJ66tXzyMDMzEZOOornVi3lR5d/IncYZjbCHXX8LblD6EjZOoN0J9GLwOvAukaGMWZmVo3cI4P9I2JV5hjMzLqe5wzMzCxrZxDArZIWSpqVMQ4zs66X8zLRPhHxhKRtgdskLYmI+bUVUicxC2DSNpvniNHMrCtkGxlExBPp60rgBmCvQeqsz000bkvnJjIzq0quRHVjJY3r2wY+DizOEYuZmeW7TDQFuEFSXwxXR8QvMsViZtb1snQGEfEwsHuZz2w96d1eLGJmVhHfWmpmZu4MzMws/wrkhq18binfu8q5icysnFOO9uXlRmQbGUiaIGmepCWSHpS0d65YzMy6Xc6RwQXALyLis5JGA2MyxmJm1tWydAaStgI+DBwHEBFrgbU5YjEzs3yXiXYEngEul3S3pEvT4rN+JM2S1Cup96U17ivMzKqSqzMYBewJXBQRewAvA2cOrFSbjmLL8U5HYWZWlVydwXJgeUQsSO/nUXQOZmaWQZbOICKeBv4gaddU9FHggRyxmJlZ3ruJTgGuSncSPQwcnzEWM7OupojIHUNDenp6ore3N3cYZmYjiqSFjTxj3ukozMxs5KSjWL56KX8976DcYZhZBzn/s8583yq5Hm6zq6RFNa81kk7LEYuZmeV7nsFDwAwASZsAT1A8+tLMzDLohDmDjwK/j4jHcgdiZtatOqEzOAqYmzsIM7NulrUzSGsMDgeuHWL/+txErzg3kZlZZXKPDA4G7oqIFYPtrM1NNMa5iczMKpO7M5iJLxGZmWWX80lnY4EDgetzxWBmZoVsi84i4mVgm0brT5v4bi8wMTOrSMMjA0ljJb0tbe8i6XBJm1YXmpmZtUuZy0Tzgc0lbQ/cCnwemFNFUGZm1l5lLhMpIl6RdCLw/Yg4T9KiqgIbaOnzj3Pwjae0qzkzy+jnn/xe7hC6TpmRgSTtDRwN/DSVbdJsw5JOl3S/pMWS5kravNljmZnZ8JTpDE4FzgJuiIj7Je0E/KqZRtOlpv8J9ETEbhSdylHNHMvMzIavoctEKZnc4RFxeF9ZRDxM8Qd9OG1vIek1YAzw5DCOZWZmw9DQyCAiXgf2aVWjEfEEcD7wOPAU8EJE3DqwXm06irVrXm1V82ZmNkCZy0R3S7pJ0uclfbrv1UyjkiYCRwA7Am8Hxko6ZmC92nQUo8dv0UxTZmbWgDJ3E20OPAscUFMWNLeC+GPAIxHxDICk64EPAj9s4lhmZjZMDXcGEXF8C9t9HPiApDHAqxTPNPDT7s3MMimzAnmapBskrUyv6yRNa6bRiFgAzAPuAu5LcVzSzLHMzGz4FBGNVZRuA64G/jkVHQMcHREHVhRbPz09PdHb68GDmVkZkhZGRE+9emUmkCdHxOURsS695gCTm47QzMw6RpkJ5GfTHT99zx+YSTGh3BZLn1/Bodd/q13NmVkH++mnz8gdwltOmZHBCcDngKcp1gZ8Fjiu2YYlnZpSUdwv6bRmj2NmZsNXZmQwrXYFMoCkDwF/KNuopN2ALwB7AWuBX0i6OSKWlT2WmZkNX5mRwWBpBJtNLfhfgQUR8UpErAPuAJpawGZmZsNXd2SQMpV+EJgs6a9qdo2n+ayli4GvSdqGYp3BIXidgZlZNo1cJhoNbJnqjqspX0Mxb1BaRDwo6RsUD8l5GVgEvD6wnqRZwCyAzSdNbKYpMzNrQN3OICLuAO6QNCciHmtVwxFxGXAZgKT/AywfpM4lpMVoW71rh8YWRJiZWWll5gwulTSh742kiZJuabZhSdumr++gmC+4utljmZnZ8JS5m2hSRDzf9yYiVvf9QW/SdWnO4DXgi7XHNjOz9irTGbwh6R0R8TiApHdSZC1tSkTsW6b+uydM8UITM7OKlOkMzgZ+LekOQMC+pMldMzMb2cqksP6FpD2BD6Si0yJiVTVhmZlZOzWyzuC/RMSS1BHAhmcVvyNdNrprI5+dDRwGrEwPvkfS1sCPgenAo8DnImJ1vTiWrV7FYdfNrlfNzEaomz9zQu4QulojI4MzKFJHDJYlLuj/5LOB5gAXAlfWlJ0J3B4RX5d0Znr/Nw1Fa2ZmlWhkncEX0tf9yx48IuZLmj6g+Ahgv7R9BfCvuDMwM8uqkctEG80ZFBFln4E8JSKeSttPA1NKft7MzFqskctE/z193ZYiR9G/pPf7A/8BlO0M1ouIkDTk7am16Si2mLRNs82YmVkdjVwmOh5A0q3Ae/r+Vy9pKsWcQFkrJE2NiKfSMVZupO316Sgm7Dzd6SjMzCpSJh3FDjWXdwBWAO9oos2bgGPT9rHAT5o4hpmZtVCZRWe3p1xEfY+9/DPglxv7gKS5FJPFkyQtB84Bvg5cI+lE4DGKp6eZmVlGZRadfUnSp4APp6JLIuKGOp+ZOcSujzbarpmZVa/MyADgLuDFiPilpDGSxkXEi1UENtC7Jk7yohQzs4o0PGcg6QvAPODiVLQ9cGMVQZmZWXuVGRl8keIB9gsAImLpMFNYl7Js9WoOu3Zeu5ozsw5385FNPWjRhlDmbqI/RsTavjeSRlEnhbWk2ZJWSlpcU3akpPslvSGpp3zIZmbWamU6gzskfQXYQtKBwLXA/63zmTnAQQPKFlM82Wx+ibbNzKxCZTqDM4FngPuAvwR+Bvztxj4QEfOB5waUPRgRD5WM08zMKtTQnIGkTYArI+Jo4J+qDcnMzNqtoZFBRLwOvFPS6Irj6UfSLEm9knrXrlnTzqbNzLpKmbuJHgb+XdJNwMt9hRHxDy2PasOxa3IT7ezcRGZmFSnTGfw+vd4GjKsmHDMzy6FMOopzASSNL97WX3k8RG6i54DvAZOBn0paFBGfaCJ2MzNrEUU0dvUlrQm4nA2jgheAEyJiYUWx9dPT0xO9vb3taMrM7C1D0sKIqLumq8xlotnAyRHxb6mBfSg6h/c1F6KZmXWKMp3B630dAUBE/FrSugpiGtSy1Wv45LyNZsw2M2vYjZ/9WO4QOkrZFcgXS9pP0kckfR/4V0l7StpzsA8MkY7im5KWSLpX0g2SJgz3H2FmZsNTZmSwe/p6zoDyPShyFB0wyGfmABcCV9aU3QacFRHrJH0DOAv4mxJxmJlZi5W5m2j/je2XdGxEXDHgM/MlTR9QdmvN298CTj1oZpZZmctE9ZzaxGdOAH7ewhjMzKwJrewMVKqydDawDrhqI3Vq0lG8MNz4zMxsCK3sDBpOFyHpOOAw4OjYyEKHiLgkInoiomf0+K1aEKKZmQ2m7DOQN6ahkYGkg4AvAx+JiFda2L6ZmTWp4c5A0mbAZ4DptZ+LiL9Pm/8+yGcGS0dxFrAZcJskgN9GxP9oLnwzM2uFMiODn1CkoFgI/HHgzoj40iBlMwc5zmUl2jQzszYo0xlMi4iBj7Bsm3dNHO8Vg2ZmFSkzgfwfkv6kskjMzCybMiODfYDjJD1CcZlIFKms25Ko7uHVr3Lkdfe2oykze4u79jPOrzlQmc7g4LIHlzSb4hbSlRGxWyr738ARwBvASuC4iHiy7LHNzKx1Gr5MFBGPDfaq87E5wMB5hm9GxPsiYgZwM/DVciGbmVmrtXLR2ZtExHyKJ5vVltU+2X4sJRarmZlZNVq56Kxhkr4G/DnFrapDJsCTNAuYBTBm0tT2BGdm1oUqHRkMJSLOjogdKPISvWl9Qk299ekoNhs/sX0Bmpl1mSydQY2rKFY1m5lZRm3vDCS9u+btEcCSdsdgZmb9VTpnMERuokMk7Upxa+ljgPMSmZllpo1kkO4oPT090dvbmzsMM7MRRdLCiOipVy/3nIGZmXWALLeWNuOp51/jazc8lTsMMxvhzv6Ub1MfTKUjA0mzJa2UtHiQfWdICkmTqozBzMzqq/oy0RzenI4CSTsAHwcer7h9MzNrQNvTUSTfpnj05ciYvTYze4vLsc7gCOCJiLin3W2bmdng2jqBLGkM8BWKS0SN1F+fm2irydtXGJmZWXdr98hgZ2BH4B5JjwLTgLskbTdY5drcRGPHb9PGMM3MuktbRwYRcR+wbd/71CH0RMSqdsZhZmb9VX1r6VzgN8CukpZLOrHK9szMrDmVjgwiYmad/dMbPdbUCZt6sYiZWUWcjsLMzNwZmJnZCMpN9Pzqddx4reeZzaw1PnmkM+HUantuIkl/J+kJSYvS65AqYzAzs/qy5CYCvh0RM9LrZxXHYGZmdeTKTWRmZh0k1wTylyTdmy4jTRyqkqRZknol9a5Z82w74zMz6yo5OoOLKNJSzACeAr41VMXadBTjnY7CzKwybe8MImJFRLweEW8A/wTs1e4YzMysvxwprGuXEX8KeNNT0MzMrL0qXWeQchPtB0yStBw4B9hP0gyKB9s8CvxllTGYmVl9OXITXdbMsSZMHOVFImZmFXE6CjMzGznpKF5etY47L1+ZOwwze4vY6/ht61fqIm1PR5HKT5G0RNL9ks6rMgYzM6uv7ekoJO0PHAHsHhHvBc6vOAYzM6sjRzqKk4CvR8QfUx1f+zEzyyzHBPIuwL6SFki6Q9KfZojBzMxq5JhAHgVsDXwA+FPgGkk7RUQMrChpFjALYLttprU1SDOzbpJjZLAcuD4KdwJvAIMuIKjNTTRhS+cmMjOrSo7O4EZgfwBJuwCjAT/CzMwsoxzpKGYDs9PtpmuBYwe7RGRmZu2TIx0FwDFljzV20igvEjEzq4jTUZiZmTsDMzMbQbmJ1q54jeXnP507DDN7i5j219vlDqGjtD03kaQfS1qUXo9KWlRlDGZmVl/VI4M5wIXAlX0FEfFnfduSvgW8UHEMZmZWR9V3E82XNH2wfZIEfA44oMoYzMysvpwTyPsCKyJi6VAVJM2S1Cup97mXnm1jaGZm3SVnZzATmLuxCrXpKLZ2Ogozs8pkuZtI0ijg08D7c7RvZmb95RoZfAxYEhHLM7VvZmY1qr61dC7wG2BXScslnZh2HUWdS0RmZtY+WXITRcRxZY81esqmXiRiZlYRp6MwM7ORk47itRWvsOLbXqxsZq0x5fQZuUPoKDnSUcyQ9NuUjqJX0l5VxmBmZvVVfZloDnDQgLLzgHMjYgbw1fTezMwyqrQziIj5wHMDi4HxaXsr4MkqYzAzs/pyzBmcBtwi6XyKzuiDGWIwM7MaOe4mOgk4PSJ2AE4HLhuqYr/cRC8/37YAzcy6TY7O4Fjg+rR9LTDkBHK/3ERjJ7QlODOzbpSjM3gS+EjaPgAYMmupmZm1R6VzBikdxX7AJEnLgXOALwAXpGR1/w+YVWUMZmZWX5Z0FDSRrXTTKWO8SMTMrCJOR2FmZu4MzMys+jmD2cBhwMqI2C2V7Q78ANgSeBQ4OiLW1DvWupVrWPm92yqM1sy6xbanHJg7hI6TIx3FpcCZEfEnwA3A/6o4BjMzqyNHOopdgPlp+zbgM1XGYGZm9eWYM7gfOCJtHwnskCEGMzOrkaMzOAE4WdJCYBywdqiKtekonn3phbYFaGbWbdqeqC4ilgAfB5C0C3DoRupeAlwCMOMdu0RbAjQz60JtHxlI2jZ9fRvwtxR3FpmZWUZVP+lsLvAbYFdJyyWdCMyU9DtgCUWeosurjMHMzOrLlY7igirbNTOzcnI83KYpo7Yd74UiZmYVUcTImJeV9CLwUO44GjQJWJU7iAY51uqMpHgdazU6IdZ3RsTkepVGzMgAeCgienIH0QhJvY619UZSrDCy4nWs1RhJsTpRnZmZuTMwM7OR1RlckjuAEhxrNUZSrDCy4nWs1RgxsY6YCWQzM6vOSBoZmJlZRTq+M5B0kKSHJC2TdGYb291B0q8kPSDpfkmnpvKtJd0maWn6OjGVS9J3U5z3Stqz5ljHpvpLJR1bU/5+Sfelz3xXkoYZ8yaS7pZ0c3q/o6QF6fg/ljQ6lW+W3i9L+6fXHOOsVP6QpE/UlLfs+yBpgqR5kpZIelDS3h1+Xk9PPwOLJc2VtHmnnFtJsyWtlLS4pqzyczlUG03E+s30c3CvpBskTWj2fDXzPSkTa82+MySFpEmdcF5bJiI69gVsAvwe2AkYDdwDvKdNbU8F9kzb44DfAe8BzqN4OA/AmcA30vYhwM8BAR8AFqTyrYGH09eJaXti2ndnqqv02YOHGfNfAVcDN6f31wBHpe0fACel7ZOBH6Tto4Afp+33pHO8GbBjOvebtPr7AFwB/EXaHg1M6NTzCmwPPAJsUXNOj+uUcwt8GNgTWFxTVvm5HKqNJmL9ODAqbX+jJtbS56vs96RsrKl8B+AW4DFgUiec11a92tLIMH4R9wZuqXl/FnBWplh+AhxIsfBtaiqbSrH+AeBiYGZN/YfS/pnAxTXlF6eyqcCSmvJ+9ZqIbxpwO3AAcHP6IVtV84u2/lymH+a90/aoVE8Dz29fvVZ+H4CtKP64akB5p57X7YE/pF/oUencfqKTzi0wnf5/YCs/l0O1UTbWAfs+BVw12Hmod76a+XlvJlZgHrA7xSN7+zqD7Oe1Fa9Ov0zU94vYZ3kqa6s0rNwDWABMiYin0q6ngSlpe6hYN1a+fJDyZn0H+DLwRnq/DfB8RKwb5PjrY0r7X0j1y/4bmrEj8AxwuYpLWpdKGkuHnteIeAI4H3gceIriXC2kM89tn3acy6HaGI4TKP6X3Eyszfy8lyLpCOCJiLhnwK5OP68N6fTOIDtJWwLXAadFxJrafVF039lvx5J0GLAyIhbmjqUBoyiG3xdFxB7AyxTD4fU65bwCpGu2R1B0Ym8HxvLm53p3rHacy1a0IelsYB1wVUuCajFJY4CvAF9tV5vt/j3o9M7gCfo/FnNaKmsLSZtSdARXRcT1qXiFpKlp/1RgZZ1YN1Y+bZDyZnwIOFzSo8CPKC4VXQBMkNSXcqT2+OtjSvu3Ap5t4t/QjOXA8ohYkN7Po+gcOvG8AnwMeCQinomI14DrKc53J57bPu04l0O1UZqk44DDgKPTH8BmYn2W8t+TMnam+A/BPen3bBpwl6Ttmoi1Lee1tHZdj2rmRfG/yIcpvgl9k0XvbVPbAq4EvjOg/Jv0n+A5L20fSv9JpDtT+dYU18gnptcjwNZp38BJpENaEPd+bJhAvpb+E2onp+0v0n9C7Zq0/V76T9o9TDFh19LvA/BvwK5p++/SOe3I8wr8N4rndo9Jx7sCOKWTzi1vnjOo/FwO1UYTsR4EPABMHlCv9Pkq+z0pG+uAfY+yYc4g+3ltxastjQwrwGKm/ncUdxCc3cZ296EYot0LLEqvQyiuNd4OLAV+WfPNFfCPKc77gJ6aY50ALEuv42vKe4DF6TMX0sCkVgNx78eGzmCn9EO3LP2ibJbKN0/vl6X9O9V8/uwUz0PU3IXTyu8DMAPoTef2xvSL0rHnFTiX4mFMi4F/pvgD1RHnFphLMZfxGsWo68R2nMuh2mgi1mUU19X7fsd+0Oz5auZ7UibWAfsfZUNnkPW8turlFchmZtbxcwZmZtYG7gzMzMydgZmZuTMwMzPcGZiZGe4MzDYqZaf8Yc37UZKe0YbMsFMk3SzpHhUZbn+WyqdLelXSoprXn+f6d5jVM6p+FbOu9jKwm6QtIuJVimSFtSuE/x64LSIuAJD0vpp9v4+IGe0L1ax5HhmY1fczilWmUGSYnFuzbyo1Scci4t42xmXWMu4MzOr7EXCUpM2B91Fkr+3zj8BlKh6EdLakt9fs23nAZaJ92xm0WRm+TGRWR0Tcm9KYz6QYJdTuu0XSThQ5dg4G7pa0W9rty0Q2YnhkYNaYmyieazB34I6IeC4iro6IzwP/SfGULLMRxZ2BWWNmA+dGxH21hZIOSLnukTSOItXx4xniMxsWXyYya0BELAe+O8iu9wMXSlpH8Z+rSyPiP9NlpZ0lLaqpOzsiBjuGWXbOWmpmZr5MZGZm7gzMzAx3BmZmhjsDMzPDnYGZmeHOwMzMcGdgZma4MzAzM+D/A/cXKfoV3/U6AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Plot the error metric\n",
    "sns.barplot(y='n_predictors', x='MSE', data=best, orient='h')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 316,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>model</th>\n",
       "      <th>n_predictors</th>\n",
       "      <th>MSE</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>&lt;statsmodels.regression.linear_model.Regressio...</td>\n",
       "      <td>1</td>\n",
       "      <td>149077.418140</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>&lt;statsmodels.regression.linear_model.Regressio...</td>\n",
       "      <td>2</td>\n",
       "      <td>134035.444866</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>&lt;statsmodels.regression.linear_model.Regressio...</td>\n",
       "      <td>3</td>\n",
       "      <td>130954.147117</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>&lt;statsmodels.regression.linear_model.Regressio...</td>\n",
       "      <td>4</td>\n",
       "      <td>123682.462505</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>&lt;statsmodels.regression.linear_model.Regressio...</td>\n",
       "      <td>5</td>\n",
       "      <td>119248.631710</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>&lt;statsmodels.regression.linear_model.Regressio...</td>\n",
       "      <td>6</td>\n",
       "      <td>113154.213964</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>&lt;statsmodels.regression.linear_model.Regressio...</td>\n",
       "      <td>7</td>\n",
       "      <td>115580.660634</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>&lt;statsmodels.regression.linear_model.Regressio...</td>\n",
       "      <td>8</td>\n",
       "      <td>111934.205427</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>&lt;statsmodels.regression.linear_model.Regressio...</td>\n",
       "      <td>9</td>\n",
       "      <td>112854.617360</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>&lt;statsmodels.regression.linear_model.Regressio...</td>\n",
       "      <td>10</td>\n",
       "      <td>114497.887408</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>&lt;statsmodels.regression.linear_model.Regressio...</td>\n",
       "      <td>11</td>\n",
       "      <td>114714.879564</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>&lt;statsmodels.regression.linear_model.Regressio...</td>\n",
       "      <td>12</td>\n",
       "      <td>116153.597903</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>&lt;statsmodels.regression.linear_model.Regressio...</td>\n",
       "      <td>13</td>\n",
       "      <td>115401.230308</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>&lt;statsmodels.regression.linear_model.Regressio...</td>\n",
       "      <td>14</td>\n",
       "      <td>114398.000171</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>&lt;statsmodels.regression.linear_model.Regressio...</td>\n",
       "      <td>15</td>\n",
       "      <td>116273.710693</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>&lt;statsmodels.regression.linear_model.Regressio...</td>\n",
       "      <td>16</td>\n",
       "      <td>116361.112111</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>&lt;statsmodels.regression.linear_model.Regressio...</td>\n",
       "      <td>17</td>\n",
       "      <td>116308.889187</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>&lt;statsmodels.regression.linear_model.Regressio...</td>\n",
       "      <td>18</td>\n",
       "      <td>116595.910924</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>&lt;statsmodels.regression.linear_model.Regressio...</td>\n",
       "      <td>19</td>\n",
       "      <td>116599.013674</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                model  n_predictors  \\\n",
       "0   <statsmodels.regression.linear_model.Regressio...             1   \n",
       "1   <statsmodels.regression.linear_model.Regressio...             2   \n",
       "2   <statsmodels.regression.linear_model.Regressio...             3   \n",
       "3   <statsmodels.regression.linear_model.Regressio...             4   \n",
       "4   <statsmodels.regression.linear_model.Regressio...             5   \n",
       "5   <statsmodels.regression.linear_model.Regressio...             6   \n",
       "6   <statsmodels.regression.linear_model.Regressio...             7   \n",
       "7   <statsmodels.regression.linear_model.Regressio...             8   \n",
       "8   <statsmodels.regression.linear_model.Regressio...             9   \n",
       "9   <statsmodels.regression.linear_model.Regressio...            10   \n",
       "10  <statsmodels.regression.linear_model.Regressio...            11   \n",
       "11  <statsmodels.regression.linear_model.Regressio...            12   \n",
       "12  <statsmodels.regression.linear_model.Regressio...            13   \n",
       "13  <statsmodels.regression.linear_model.Regressio...            14   \n",
       "14  <statsmodels.regression.linear_model.Regressio...            15   \n",
       "15  <statsmodels.regression.linear_model.Regressio...            16   \n",
       "16  <statsmodels.regression.linear_model.Regressio...            17   \n",
       "17  <statsmodels.regression.linear_model.Regressio...            18   \n",
       "18  <statsmodels.regression.linear_model.Regressio...            19   \n",
       "\n",
       "              MSE  \n",
       "0   149077.418140  \n",
       "1   134035.444866  \n",
       "2   130954.147117  \n",
       "3   123682.462505  \n",
       "4   119248.631710  \n",
       "5   113154.213964  \n",
       "6   115580.660634  \n",
       "7   111934.205427  \n",
       "8   112854.617360  \n",
       "9   114497.887408  \n",
       "10  114714.879564  \n",
       "11  116153.597903  \n",
       "12  115401.230308  \n",
       "13  114398.000171  \n",
       "14  116273.710693  \n",
       "15  116361.112111  \n",
       "16  116308.889187  \n",
       "17  116595.910924  \n",
       "18  116599.013674  "
      ]
     },
     "execution_count": 316,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Select between models with different numbers of predictors using cross validation\n",
    "results = pd.DataFrame()\n",
    "for train_idx, test_idx in model_selection.KFold(n_splits=10).split(hitters_dat):\n",
    "    train = hitters_dat.iloc[train_idx]\n",
    "    test = hitters_dat.iloc[test_idx]\n",
    "    best = fwd_selection('Salary', train)\n",
    "    mse = best['model'].map(lambda m: metrics.mean_squared_error(test['Salary'], m.predict(test)))\n",
    "    results = results.append(mse, ignore_index=True)\n",
    "    \n",
    "best['MSE'] = results.mean()\n",
    "best"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 317,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x129eafb38>"
      ]
     },
     "execution_count": 317,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYMAAAEKCAYAAADw2zkCAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAGxJJREFUeJzt3Xu4XVV57/HvT0KAhIQEEkIkYACFU6UScJcjCgooyu2ANyx5wHKr6QHlAKXHgvSR0vN4HkWsolSEQghUiEK4SPECkVpSWw3dgQABQhOuhksSIBBuxxB4zx9z7GTtzd5Za66sucbarN/nedaz5xpzrDnezJWdkTHHHO9URGBmZt3tHbkDMDOz/NwZmJmZOwMzM3NnYGZmuDMwMzPcGZiZGe4MzMwMdwZmZoY7AzMzA0bkDqBREyZMiKlTp+YOw8xsWFmwYMGzETGxXr1h0xlMnTqV3t7e3GGYmQ0rkh5vpF62zkDSTOBwYEVE7F6v/tqVz7Py4h9VH5iZWQeZePKxbWkn55zBLODgjO2bmVmSrTOIiHnA87naNzOz9Xw3kZmZdXZnIGmGpF5Jvc+9vDp3OGZmb1sd3RlExKUR0RMRPdtsOTZ3OGZmb1sd3RmYmVl7ZOsMJM0GfgvsJmmZpJNyxWJm1u2yrTOIiOm52jYzs/6GzQrkERO3btviCzOzbuM5AzMzGz4jg9dXPsUzF5+XOwwz6zLbnXxu7hDaIsvIQNIOkn4t6QFJ90s6LUccZmZWyDUyWAucGRF3SRoDLJA0NyIeyBSPmVlXyzIyiIinI+KutP0S8CCwfY5YzMysAyaQJU0F9gTm543EzKx7Ze0MJG0JXA+cHhFvST7UPzfRq+0P0MysS+RcgbwpRUdwdUTcMFid/rmJRrU3QDOzLpLrbiIBlwMPRsTf54jBzMzWyzUy+DDwBeBASQvT69BMsZiZdb0st5ZGxG8AlfnMphPf2TWLP8zM2i373URmZpbfsElH8drKpdz/gyNyh2Fmb0PvO+Xm3CFkl2sCeXNJd0q6J6WjcNIhM7OMco0M/gAcGBEvp1tMfyPpFxHxu0zxmJl1tVwTyAG8nN5uml6RIxYzM8u76GwTSQuBFcDciHA6CjOzTLJ1BhHxRkRMA6YAe0vafWCd2nQUq15e0/4gzcy6RPZbSyPiBeDXwMGD7FuXjmL8liPbH5yZWZfIdTfRREnj0vYWwEHA4hyxmJlZvruJJgNXStqEokO6NiJuyRSLmVnXy3U30b0UzzAwM7MOMGxWIG8x8d1eJWhmVpHsE8hmZpbfsBkZvPTsEm6/7LDcYZhZB/rYn/8sdwjDXu7HXm4i6W5Jnjw2M8so92Wi04AHM8dgZtb1cqajmAIcBlyWKwYzMyvkHBl8F/gK8OZQFWrTUbzwktNRmJlVJdcK5MOBFRGxYEP1atNRjBvjdBRmZlXJNTL4MHCEpMeAHwMHSvpRpljMzLpels4gIs6OiCkRMRU4GviXiDg2RyxmZpb/biIzM+sAKh461vl6enqit7c3dxhmZsOKpAUR0VOvnkcGZmY2fNJRPP/sEn58xSdzh2FmbXb0CbfmDqErZOsM0p1ELwFvAGsbGcaYmVk1co8MDoiIZzPHYGbW9TxnYGZmWTuDAG6TtEDSjIxxmJl1vZyXifaNiCclbQvMlbQ4IubVVkidxAyACdtsniNGM7OukG1kEBFPpp8rgBuBvQepsy430ZgtnZvIzKwquRLVjZY0pm8b+ASwKEcsZmaW7zLRJOBGSX0xXBMRv8wUi5lZ18vSGUTEI8AeZT6z9YT3ePGJmVlFfGupmZm5MzAzs/wrkBu24vklfP9q5yYyG85OPcaXejtVtpGBpHGS5khaLOlBSfvkisXMrNvlHBlcCPwyIj4naSQwKmMsZmZdLUtnIGkr4CPA8QARsQZYkyMWMzPLd5loJ2AlcIWkuyVdlhaf9SNphqReSb0vr3ZfYWZWlVydwQhgL+DiiNgTeAU4a2Cl2nQUW451Ogozs6rk6gyWAcsiYn56P4eiczAzswyydAYR8Qzwe0m7paKPAQ/kiMXMzPLeTXQqcHW6k+gR4ISMsZiZdTVFRO4YGtLT0xO9vb25wzAzG1YkLWjkGfNOR2FmZsMnHcWyVUv4qzkH5w7DzFrsgs85e30nyPVwm90kLax5rZZ0eo5YzMws3/MMHgKmAUjaBHiS4tGXZmaWQSfMGXwMeDgiHs8diJlZt+qEzuBoYHbuIMzMulnWziCtMTgCuG6I/etyE73q3ERmZpXJPTI4BLgrIpYPtrM2N9Eo5yYyM6tM7s5gOr5EZGaWXc4nnY0GDgJuyBWDmZkVsi06i4hXgG0arT9l/Hu8OMXMrCINjwwkjZb0jrS9q6QjJG1aXWhmZtYuZS4TzQM2l7Q9cBvwBWBWFUGZmVl7lblMpIh4VdJJwA8i4nxJC6sKbKAlLzzBITed2q7mzKxFfvGp7+cOwRpQZmQgSfsAxwA/S2WbNNuwpDMk3S9pkaTZkjZv9lhmZrZxynQGpwFnAzdGxP2SdgZ+3Uyj6VLT/wJ6ImJ3ik7l6GaOZWZmG6+hy0QpmdwREXFEX1lEPELxD/rGtL2FpNeBUcBTG3EsMzPbCA2NDCLiDWDfVjUaEU8CFwBPAE8DL0bEbQPr1aajWLP6tVY1b2ZmA5S5THS3pJslfUHSZ/pezTQqaTxwJLAT8E5gtKRjB9arTUcxcuwWzTRlZmYNKHM30ebAc8CBNWVBcyuIPw48GhErASTdAHwI+FETxzIzs43UcGcQESe0sN0ngA9KGgW8RvFMAz/t3swskzIrkKdIulHSivS6XtKUZhqNiPnAHOAu4L4Ux6XNHMvMzDaeIqKxitJc4Brgn1LRscAxEXFQRbH109PTE729HjyYmZUhaUFE9NSrV2YCeWJEXBERa9NrFjCx6QjNzKxjlJlAfi7d8dP3/IHpFBPKbbHkheUcdsO329WcmbXYzz5zZu4QbAPKjAxOBD4PPEOxNuBzwPHNNizptJSK4n5Jpzd7HDMz23hlRgZTalcgA0j6MPD7so1K2h34IrA3sAb4paRbImJp2WOZmdnGKzMyGCz1YLPpCP8ImB8Rr0bEWuAOoKkFbGZmtvHqjgxSptIPARMl/WXNrrE0n7V0EfB1SdtQrDM4FK8zMDPLppHLRCOBLVPdMTXlqynmDUqLiAclfZPiITmvAAuBNwbWkzQDmAGw+YTxzTRlZmYNqNsZRMQdwB2SZkXE461qOCIuBy4HkPR/gWWD1LmUtBhtq3fv0NiCCDMzK63MnMFlksb1vZE0XtKtzTYsadv0c0eK+YJrmj2WmZltnDJ3E02IiBf63kTEqr5/0Jt0fZozeB34Uu2xzcysvcp0Bm9K2jEingCQ9C6KrKVNiYj9ytR/z7hJXrRiZlaRMp3BOcBvJN0BCNiPNLlrZmbDW5kU1r+UtBfwwVR0ekQ8W01YZmbWTo2sM/hvEbE4dQSw/lnFO6bLRndt4LMzgcOBFenB90jaGvgJMBV4DPh8RKyqF8fSVc9y+PUz61Uzs2Hils+emDsEq9HIyOBMitQRg2WJC/o/+WygWcBFwFU1ZWcBt0fENySdld7/dUPRmplZJRpZZ/DF9POAsgePiHmSpg4oPhLYP21fCfwr7gzMzLJq5DLRBnMGRUTZZyBPioin0/YzwKSSnzczsxZr5DLR/0g/t6XIUfQv6f0BwH8AZTuDdSIiJA15e2ptOootJmzTbDNmZlZHI5eJTgCQdBvw3r7/1UuaTDEnUNZySZMj4ul0jBUbaHtdOopxu0x1Ogozs4qUSUexQ83lHYDlwI5NtHkzcFzaPg74aRPHMDOzFiqz6Oz2lIuo77GXfwr8akMfkDSbYrJ4gqRlwLnAN4BrJZ0EPE7x9DQzM8uozKKzL0v6NPCRVHRpRNxY5zPTh9j1sUbbNTOz6pUZGQDcBbwUEb+SNErSmIh4qYrABnr3+AlepGJmVpGG5wwkfRGYA1ySirYHbqoiKDMza68yI4MvUTzAfj5ARCzZyBTWpSxdtYrDr5vTrubMrGK3HNXUgxKtImXuJvpDRKzpeyNpBHVSWEuaKWmFpEU1ZUdJul/Sm5J6yodsZmatVqYzuEPSV4EtJB0EXAf8c53PzAIOHlC2iOLJZvNKtG1mZhUq0xmcBawE7gP+Avg58Dcb+kBEzAOeH1D2YEQ8VDJOMzOrUENzBpI2Aa6KiGOAf6w2JDMza7eGRgYR8QbwLkkjK46nH0kzJPVK6l2zenU7mzYz6ypl7iZ6BPh3STcDr/QVRsTftzyq9ceuyU20i3MTmZlVpExn8HB6vQMYU004ZmaWQ5l0FOcBSBpbvK2/8niI3ETPA98HJgI/k7QwIj7ZROxmZtYiimjs6ktaE3AF60cFLwInRsSCimLrp6enJ3p7e9vRlJnZ24akBRFRd01XmctEM4FTIuLfUgP7UnQO728uRDMz6xRlOoM3+joCgIj4jaS1FcQ0qKWrVvOpORvMmG1mw9RNn/t47hC6XtkVyJdI2l/SRyX9APhXSXtJ2muwDwyRjuJbkhZLulfSjZLGbewfwszMNk6ZkcEe6ee5A8r3pMhRdOAgn5kFXARcVVM2Fzg7ItZK+iZwNvDXJeIwM7MWK3M30QEb2i/puIi4csBn5kmaOqDstpq3vwOcutDMLLMyl4nqOa2Jz5wI/KKFMZiZWRNa2RmoVGXpHGAtcPUG6tSko3hxY+MzM7MhtLIzaDhdhKTjgcOBY2IDCx0i4tKI6ImInpFjt2pBiGZmNpiyz0DekIZGBpIOBr4CfDQiXm1h+2Zm1qSGOwNJmwGfBabWfi4i/i5t/vsgnxksHcXZwGbAXEkAv4uI/9lc+GZm1gplRgY/pUhBsQD4w8CdEfHlQcqmD3Kcy0u0aWZmbVCmM5gSEQMfYdk27x4/1qsUzcwqUmYC+T8k/XFlkZiZWTZlRgb7AsdLepTiMpEoUlm3JVHdI6te46jr721HU2bWRtd91rkuO0GZzuCQsgeXNJPiFtIVEbF7Kvs/wJHAm8AK4PiIeKrssc3MrHUavkwUEY8P9qrzsVnAwHmGb0XE+yNiGnAL8LVyIZuZWau1ctHZW0TEPIonm9WW1T7ZfjQlFquZmVk1WrnorGGSvg78GcWtqkMmwJM0A5gBMGrC5PYEZ2bWhSodGQwlIs6JiB0o8hK9ZX1CTb116Sg2Gzu+fQGamXWZLJ1BjaspVjWbmVlGbe8MJL2n5u2RwOJ2x2BmZv1VOmcwRG6iQyXtRnFr6eOA8xKZmWWmDWSQ7ig9PT3R29ubOwwzs2FF0oKI6KlXL/ecgZmZdYAst5Y24+kXXufrNz6dOwwza5FzPu3bxTtJpSMDSTMlrZC0aJB9Z0oKSROqjMHMzOqr+jLRLN6ajgJJOwCfAJ6ouH0zM2tA29NRJN+hePTl8Ji9NjN7m8uxzuBI4MmIuKfdbZuZ2eDaOoEsaRTwVYpLRI3UX5ebaKuJ21cYmZlZd2v3yGAXYCfgHkmPAVOAuyRtN1jl2txEo8du08Ywzcy6S1tHBhFxH7Bt3/vUIfRExLPtjMPMzPqr+tbS2cBvgd0kLZN0UpXtmZlZcyodGUTE9Dr7pzZ6rMnjNvUiFTOzijgdhZmZuTMwM7NhlJvohVVruek6zzObvV196ihnpsmp7bmJJP2tpCclLUyvQ6uMwczM6suSmwj4TkRMS6+fVxyDmZnVkSs3kZmZdZBcE8hflnRvuow0fqhKkmZI6pXUu3r1c+2Mz8ysq+ToDC6mSEsxDXga+PZQFWvTUYx1Ogozs8q0vTOIiOUR8UZEvAn8I7B3u2MwM7P+cqSwrl1G/GngLU9BMzOz9qp0nUHKTbQ/MEHSMuBcYH9J0ygebPMY8BdVxmBmZvXlyE10eTPHGjd+hBelmJlVxOkozMxs+KSjeOXZtdx5xYrcYZhZRfY+Ydv6lawybU9HkcpPlbRY0v2Szq8yBjMzq6/t6SgkHQAcCewREe8DLqg4BjMzqyNHOoqTgW9ExB9SHV/7MTPLLMcE8q7AfpLmS7pD0p9kiMHMzGrkmEAeAWwNfBD4E+BaSTtHRAysKGkGMANgu22mtDVIM7NukmNksAy4IQp3Am8Cgy4gqM1NNG5L5yYyM6tKjs7gJuAAAEm7AiMBP8LMzCyjHOkoZgIz0+2ma4DjBrtEZGZm7ZMjHQXAsWWPNXrCCC9KMTOriNNRmJmZOwMzMxtGuYnWLH+dZRc8kzsMM6vIlL/aLncIXa3tuYkk/UTSwvR6TNLCKmMwM7P6qh4ZzAIuAq7qK4iIP+3blvRt4MWKYzAzszqqvptonqSpg+2TJODzwIFVxmBmZvXlnEDeD1geEUuGqiBphqReSb3Pv/xcG0MzM+suOTuD6cDsDVWoTUextdNRmJlVJsvdRJJGAJ8BPpCjfTMz6y/XyODjwOKIWJapfTMzq1H1raWzgd8Cu0laJumktOto6lwiMjOz9smSmygiji97rJGTNvWiFDOzijgdhZmZDZ90FK8vf5Xl3/FiZbO3q0lnTMsdQlfLkY5imqTfpXQUvZL2rjIGMzOrr+rLRLOAgweUnQ+cFxHTgK+l92ZmllGlnUFEzAOeH1gMjE3bWwFPVRmDmZnVl2PO4HTgVkkXUHRGH8oQg5mZ1chxN9HJwBkRsQNwBnD5UBX75SZ65YW2BWhm1m1ydAbHATek7euAISeQ++UmGj2uLcGZmXWjHJ3BU8BH0/aBwJBZS83MrD0qnTNI6Sj2ByZIWgacC3wRuDAlq/t/wIwqYzAzs/qypKOgiWylm04a5UUpZmYVcToKMzNzZ2BmZtXPGcwEDgdWRMTuqWwP4IfAlsBjwDERsbresdauWM2K78+tMFozy2nbUw/KHUJXy5GO4jLgrIj4Y+BG4H9XHIOZmdWRIx3FrsC8tD0X+GyVMZiZWX055gzuB45M20cBO2SIwczMauToDE4ETpG0ABgDrBmqYm06iudefrFtAZqZdZu2J6qLiMXAJwAk7QoctoG6lwKXAkzbcddoS4BmZl2o7SMDSdumn+8A/obiziIzM8uo6iedzQZ+C+wmaZmkk4Dpkv4LWEyRp+iKKmMwM7P6cqWjuLDKds3MrJwcD7dpyohtx3pRiplZRRQxPOZlJb0EPJQ7jgZNAJ7NHUQDhkuc4Fir4lhbr9PifFdETKxXadiMDICHIqIndxCNkNQ7HGIdLnGCY62KY2294RLnQE5UZ2Zm7gzMzGx4dQaX5g6ghOES63CJExxrVRxr6w2XOPsZNhPIZmZWneE0MjAzs4p0fGcg6WBJD0laKumsNra7g6RfS3pA0v2STkvlW0uaK2lJ+jk+lUvS91Kc90raq+ZYx6X6SyQdV1P+AUn3pc98T5I2It5NJN0t6Zb0fidJ89OxfyJpZCrfLL1fmvZPrTnG2an8IUmfrClv2XcgaZykOZIWS3pQ0j4dfE7PSN/9IkmzJW3eKedV0kxJKyQtqimr/DwO1UYTsX4r/R24V9KNksY1e76a+U7KxFqz70xJIWlCJ5zXlouIjn0BmwAPAzsDI4F7gPe2qe3JwF5pewzwX8B7gfMpHs4DcBbwzbR9KPALQMAHgfmpfGvgkfRzfNoen/bdmeoqffaQjYj3L4FrgFvS+2uBo9P2D4GT0/YpwA/T9tHAT9L2e9P53QzYKZ33TVr9HQBXAn+etkcC4zrxnALbA48CW9Scz+M75bwCHwH2AhbVlFV+Hodqo4lYPwGMSNvfrIm19Pkq+52UjTWV7wDcCjwOTOiE89rqV1sba+IXch/g1pr3ZwNnZ4rlp8BBFAvfJqeyyRTrHwAuAabX1H8o7Z8OXFJTfkkqmwwsrinvV69kbFOA24EDgVvSX7Rna37Z1p3H9Bd6n7Q9ItXTwHPbV6+V3wGwFcU/sBpQ3onndHvg9+kXekQ6r5/spPMKTKX/P7CVn8eh2igb64B9nwauHuw81DtfzfxdbyZWYA6wB8Wjevs6g+zntZWvTr9M1PcL2WdZKmurNLzcE5gPTIqIp9OuZ4BJaXuoWDdUvmyQ8mZ8F/gK8GZ6vw3wQkSsHeTY6+JJ+19M9cvG34ydgJXAFSouaV0maTQdeE4j4kngAuAJ4GmK87SAzjyvfdpxHodqY2OcSPG/5GZibebveimSjgSejIh7Buzq9PNaSqd3BtlJ2hK4Hjg9IlbX7ouiG896O5akw4EVEbEgZxwNGkExBL84IvYEXqEYEq/TCecUIF2zPZKiA3snMJq3Ps+7Y7XjPLaiDUnnAGuBq1sSVItJGgV8Ffhau9rM9TvQ6Z3Bk/R/LOaUVNYWkjal6AiujogbUvFySZPT/snAijqxbqh8yiDlZX0YOELSY8CPKS4VXQiMk9SXbqT22OviSfu3Ap5rIv5mLAOWRcT89H4ORefQaecU4OPAoxGxMiJeB26gONedeF77tOM8DtVGaZKOBw4Hjkn/ADYT63OU/07K2IXiPwT3pN+xKcBdkrZrIta2nNemtfu6VJkXxf8kH6H4Mvomjd7XprYFXAV8d0D5t+g/0XN+2j6M/pNJd6byrSmuk49Pr0eBrdO+gZNJh25kzPuzfgL5OvpPqp2Str9E/0m1a9P2++g/cfcIxaRdS78D4N+A3dL236bz2XHnFPjvFM/rHpWOdSVwaiedV946Z1D5eRyqjSZiPRh4AJg4oF7p81X2Oykb64B9j7F+ziD7eW3lq62NNflLeSjFnTwPA+e0sd19KYZq9wIL0+tQimuOtwNLgF/VfMkC/iHFeR/QU3OsE4Gl6XVCTXkPsCh95iIamNyqE/P+rO8Mdk5/8ZamX5bNUvnm6f3StH/nms+fk2J5iJq7cFr5HQDTgN50Xm9KvywdeU6B8ygewrQI+CeKf6A64rwCsynmMl6nGHGd1I7zOFQbTcS6lOK6et/v1g+bPV/NfCdlYh2w/zHWdwZZz2urX16BbGZmHT9nYGZmbeDOwMzM3BmYmZk7AzMzw52BmZnhzsBsg1KWyh/VvB8haaXWZ4edJOkWSfeoyHD781Q+VdJrkhbWvP4s15/DrJ4R9auYdbVXgN0lbRERr1EkK6xdKfx3wNyIuBBA0vtr9j0cEdPaF6pZ8zwyMKvv5xSrTaHINDm7Zt9kapKPRcS9bYzLrGXcGZjV92PgaEmbA++nyF7b5x+Ay1U8COkcSe+s2bfLgMtE+7UzaLMyfJnIrI6IuDelMZ9OMUqo3XerpJ0pcu0cAtwtafe025eJbNjwyMCsMTdTPN9g9sAdEfF8RFwTEV8A/pPiaVlmw4o7A7PGzATOi4j7agslHZhy3iNpDEXK4ycyxGe2UXyZyKwBEbEM+N4guz4AXCRpLcV/ri6LiP9Ml5V2kbSwpu7MiBjsGGbZOWupmZn5MpGZmbkzMDMz3BmYmRnuDMzMDHcGZmaGOwMzM8OdgZmZ4c7AzMyA/w/W5in6KnDB4gAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Plot the error metric\n",
    "sns.barplot(y='n_predictors', x='MSE', data=best, orient='h')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 334,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Backward selection\n",
    "# There are 1 + p(1 + p) / 2 possible models in this space \n",
    "\n",
    "def bwd_selection(target_column, data):\n",
    "    predictors = data.drop('Salary', axis=1).columns\n",
    "    formula_s1 = 'Salary ~ ' + ' + '.join(predictors)\n",
    "    model_s1 = smf.ols(formula=formula_s1, data=data).fit()\n",
    "    step_models = [model_s1]\n",
    "    while len(predictors) > 1:\n",
    "        models = []\n",
    "        for p in predictors:\n",
    "            f = 'Salary ~ ' + ' + '.join(predictors.drop(p))\n",
    "            m = smf.ols(formula=f, data=data).fit()\n",
    "            models.append((p, m))\n",
    "        p, m = sorted(models, key=lambda tup: tup[1].rsquared, reverse=True)[0]\n",
    "        step_models.append(m)\n",
    "        predictors = predictors.drop(p)\n",
    "    return pd.DataFrame({'model': step_models, 'n_predictors' : range(1, 20)[::-1]})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 331,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Select between models with different numbers of predictors using adjusted training metrics\n",
    "best = bwd_selection('Salary', hitters_dat)\n",
    "best['adj_R_sq'] = best['model'].map(lambda m: m.rsquared_adj)\n",
    "best['bic'] = best['model'].map(lambda m: m.bic)\n",
    "best['aic'] = best['model'].map(lambda m: m.aic)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 333,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x2a1e25710>"
      ]
     },
     "execution_count": 333,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA3kAAANgCAYAAACRKaGGAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzs3Xu0nVV97//3Ry7KPZEERYNGAempVtHu8vPaCmjFS0ERWhjSilLTeitaW6u1oxzb0zNs1fY4tLVNJaCnGFsQLJXKRavkaJWeDSIEQbFWbBQJl0C4WCP4/f2xVw7b7U72Wjt7rZn17PdrjDXWWvN51jM/GeMZCV/mfOZMVSFJkiRJ6oaHtA4gSZIkSVo4FnmSJEmS1CEWeZIkSZLUIRZ5kiRJktQhFnmSJEmS1CEWeZIkSZLUIRZ5kiRJktQhFnmSJEmS1CEWeZIkSZLUIbu2DtCvZcuW1cqVK1vHkCRJkqQmrrzyytuqavlc541Nkbdy5UomJydbx5AkSZKkJpLc1M95Y1Pk3X/rHdz6wb9rHUOSpEVj+WtPaR1BkjQPzZ7JS7ImycYk61tlkCRJkqSuabnwytnAMQ37lyRJkqTOaVbkVdU64I5W/UuSJElSF7mFgiRJkiR1yE5d5CVZlWQyyeTt92xuHUeSJEmSdno7dZFXVauraqKqJvbfe9/WcSRJkiRpp7dTF3mSJEmSpMG03EJhLfBF4LAkG5Kc1iqLJEmSJHVFs83Qq+rkVn1LkiRJUlc1K/IGtevyh7P8tae0jiFJkiRJOzWfyZMkSZKkDhmbkbwf3vpdvvfBd7aOIUmSpAE98rVntI4gLSqO5EmSJElShzQp8pIclOSzSb6a5Lokp7fIIUmSJEld02q65v3AW6rqqiT7AFcmuayqvtoojyRJkiR1QpORvKq6uaqu6n2+G7geeHSLLJIkSZLUJc2fyUuyEngqcMUsx1YlmUwyefs99406miRJkiSNnaZFXpK9gY8Db6qqzTOPV9Xqqpqoqon9995z9AElSZIkacw0K/KS7MZUgXdOVZ3fKockSZIkdUmr1TUDnAlcX1V/3iKDJEmSJHVRq5G8ZwG/ChyV5Ore60WNskiSJElSZzTZQqGqPg9kkN/stvxRPPK1ZwwpkSRJkiR1Q/PVNSVJkiRJC8ciT5IkSZI6pMl0zfn4/q3f4Lq/OrZ1DEmSJC0yT3zdha0jSANptbrmw5L8W5KvJLkuyTtb5JAkSZKkrmk1kvcD4Kiquqe3X97nk3yqqr7UKI8kSZIkdUKr1TULuKf3dbfeq1pkkSRJkqQuabbwSpJdklwNbAQuq6orWmWRJEmSpK5oVuRV1QNVdTiwAjgiyZNmnpNkVZLJJJOb7tky+pCSJEmSNGaab6FQVXcCnwWOmeXY6qqaqKqJpXvvPvpwkiRJkjRmWq2uuTzJkt7nPYDnAze0yCJJkiRJXdJqdc0DgQ8n2YWpQvMfquqTjbJIkiRJUme0Wl3zGuCpLfqWJEmSpC5rNZI3sD2WH8ITX3dh6xiSJEmStFNrvvCKJEmSJGnhWORJkiRJUoeMzXTNu2+7kc986MWtY0iSJEkL7uhfv6h1BHVI05G8JLsk+XISV9aUJEmSpAXQerrm6cD1jTNIkiRJUmc0K/KSrABeDHyoVQZJkiRJ6pqWI3n/C3gr8KOGGSRJkiSpU5oUeUleAmysqivnOG9Vkskkk3fevWVE6SRJkiRpfLUayXsWcGySbwEfA45K8nczT6qq1VU1UVUTS/bZfdQZJUmSJGnsNCnyqurtVbWiqlYCJwH/UlWntMgiSZIkSV3SenVNSZIkSdICar4ZelV9Dvhc4xiSJEmS1AnNi7x+7bPsUI7+9Ytax5AkSZKknZrTNSVJkiSpQ8ZmJO+O227kY2e9oHUMSZIkaeyc9KpLWkfQCDmSJ0mSJEkd0mwkr7dH3t3AA8D9VTXRKoskSZIkdUXr6ZpHVtVtjTNIkiRJUmc4XVOSJEmSOqRlkVfApUmuTLJqthOSrEoymWTy7nu2jDieJEmSJI2fltM1n11V30lyAHBZkhuqat30E6pqNbAa4PEr96sWISVJkiRpnDQbyauq7/TeNwIXAEe0yiJJkiRJXdGkyEuyV5J9tn4GfhFY3yKLJEmSJHVJq+majwAuSLI1w0er6uJGWSRJkiSpM5oUeVX1TeApg/zm4csO5aRXXTKkRJIkSZLUDW6hIEmSJEkdYpEnSZIkSR3ScguFgWy840bef84LWseQJEmSBLzxFT5KtbNqNpKXZEmS85LckOT6JM9olUWSJEmSuqLlSN77gIur6oQkuwN7NswiSZIkSZ3QpMhLsh/w88CpAFW1BdjSIoskSZIkdUmr6ZqPA24Fzkry5SQf6m2KLkmSJEnaAa2KvF2BpwEfrKqnAvcCb5t5UpJVSSaTTN6z2YE+SZIkSZpLqyJvA7Chqq7ofT+PqaLvx1TV6qqaqKqJvffdfaQBJUmSJGkcNSnyqup7wH8mOazXdDTw1RZZJEmSJKlLWq6u+UbgnN7Kmt8EXtUwiyRJkiR1QrMir6quBiZa9S9JkiRJXdRyJG8gBzz8UN74iktax5AkSZKknVqrhVckSZIkSUMwNiN5GzbdyO+cd0zrGJIkSZIW2HtOuLh1hE5xJE+SJEmSOqRJkZfksCRXT3ttTvKmFlkkSZIkqUuaTNesqq8BhwMk2QX4DnBBiyySJEmS1CU7w3TNo4F/r6qbWgeRJEmSpHG3MxR5JwFrZzuQZFWSySST923eMuJYkiRJkjR+mhZ5SXYHjgXOne14Va2uqomqmthz391HG06SJEmSxlDrkbwXAldV1S2Nc0iSJElSJ7Qu8k5mG1M1JUmSJEmDa1bkJdkLeD5wfqsMkiRJktQ1TbZQAKiqe4H9+z1/xdJDec8JFw8xkSRJkiSNv9bTNSVJkiRJC8giT5IkSZI6pNl0zUHdeOe3eeEn3tg6hiRJkqSd3Kde+v7WEZpqufDKm5Ncl2R9krVJHtYqiyRJkiR1Rd9FXpK9kjyk9/kJSY5Nstt8Ok3yaOC3gImqehKwC3DSfK4lSZIkSXrQICN564CH9Qq0S4FfBc7egb53BfZIsiuwJ/DdHbiWJEmSJInBirxU1X3A8cBfVdWJwBPn02lVfQd4D/Bt4Gbgrqq6dD7XkiRJkiQ9aKAiL8kzgFcAF/XadplPp0mWAscBjwMeBeyV5JRZzluVZDLJ5JbN359PV5IkSZK0qAxS5J0OvB24oKquS/J44LPz7Pd5wH9U1a1V9UPgfOCZM0+qqtVVNVFVE7vvu8c8u5IkSZKkxaOvLRSS7AIcW1XHbm2rqm8ytXjKfHwbeHqSPYHvA0cDk/O8liRJkiSpp6+RvKp6AHj2QnVaVVcA5wFXAdf2cqxeqOtLkiRJ0mI1yGboX05yIXAucO/Wxqo6fz4dV9UZwBnz+a0kSZIkaXaDFHkPA24HjprWVkw9Tzd0hy55zKLfuV6SJEmS5tJ3kVdVrxpmEEmSJEnSjut7dc0kK5JckGRj7/XxJCuGGU6SJEmSNJhBpmueBXwUOLH3/ZRe2/MXOtRsbrzzFl58/ntH0ZUkSZKkjrro+Le0jjB0g+yTt7yqzqqq+3uvs4Hl8+04yelJ1ie5Lsmb5nsdSZIkSdKDBinybk9ySpJdeq9TmFqIZWBJngS8BjgCeArwkiSHzOdakiRJkqQHDVLkvRr4ZeB7wM3ACcCp8+z3vwFXVNV9VXU/cDlw/DyvJUmSJEnqGaTIW1FVx1bV8qo6oKpeCjxmnv2uB56TZP8kewIvAg6a57UkSZIkST2DFHmzbVI3r43rqup64E+BS4GLgauBB2ael2RVkskkk1vuunfmYUmSJEnSDHOurpnkGcAzgeVJfnvaoX2BXebbcVWdCZzZ6+N/AhtmOWc1sBpgv0MOqvn2JUmSJEmLRT9bKOwO7N07d59p7ZuZei5vXpIcUFUbkzyGqefxnj7fa0mSJEmSpsxZ5FXV5cDlSc6uqpsWsO+PJ9kf+CHw+qq6cwGvLUmSJEmL0iDP5H0oyZKtX5IsTXLJfDuuqudU1U9X1VOq6jPzvY4kSZIk6UH9TNfcatn00baq2pTkgCFkmtWhSx6xKHanlyRJkqQdMchI3o96z88BkOSxgIuhSJIkSdJOZJCRvHcAn09yORDgOcCqoaSaxTc23cZLPr5mVN1JkiRJWmQ++fJXt46wIPou8qrq4iRP48FVMN9UVbcNJ5YkSZIkaT7mnK6Z5Kd6708DHgN8t/d6TK9te79dk2RjkvXT2h6e5LIkN/bel+7YH0GSJEmStFU/I3lvAV4DvHeWYwUctZ3fng18APjItLa3AZ+pqncleVvv++/1lVaSJEmStF397JP3mt77kYNevKrWJVk5o/k44Lm9zx8GPodFniRJkiQtiDmLvCTHb+94VZ0/YJ+PqKqbe5+/BzxiO32vore4yx7L9h+wG0mSJElafPqZrvlLvfcDgGcC/9L7fiTwr8CgRd7/U1WVZJvbMFTVamA1wJKDV7pdgyRJkiTNoZ/pmq8CSHIp8NNbR+GSHMjUM3eDuiXJgVV1c+8aG+dxDUmSJEnSLAbZDP2gadMsAW5harXNQV0IvLL3+ZXAP87jGpIkSZKkWQyyGfpnklwCrO19/xXg09v7QZK1TC2ysizJBuAM4F3APyQ5DbgJ+OVBQ0uSJEmSZpeq/h91S/Iy4Od7X9dV1QVDSTWLiYmJmpycHFV3kiRJkrRTSXJlVU3Mdd4gI3kAVwF3V9Wnk+yZZJ+qunt+ESVJkiRJC63vZ/KSvAY4D/ibXtOjgU8MI5QkSZIkaX4GGcl7PXAEcAVAVd2Y5IChpJrFNzZt4iXnnjeq7iRJkiQtMp888YTWERbEIKtr/qCqtmz9kmRXYLsP9CVZk2RjkvXT2k5Mcl2SHyWZcz6pJEmSJKl/gxR5lyf5fWCPJM8HzgX+aY7fnA0cM6NtPXA8sG6AviVJkiRJfRikyHsbcCtwLfAbwD8Df7C9H1TVOuCOGW3XV9XXBswpSZIkSepDX8/kJdkF+EhVvQL42+FGkiRJkiTNV18jeVX1APDYJLsPOc+PSbIqyWSSyS2bN4+ya0mSJEkaS4OsrvlN4AtJLgTu3dpYVX++4KkevPZqYDXAkoMP7n/XdkmSJElapAYp8v6993oIsM9w4kiSJEmSdkTfRV5VvRMgyb5TX+vuuX6TZC3wXGBZkg3AGUwtxPJ+YDlwUZKrq+oF88guSZIkSZqh7yKvt6fdWfRG8ZLcBby6qq7c1m+q6uRtHLpgkJCSJEmSpP4MMl1zDfC6qvo/AEmezVTR9+RhBJvpkKVLO7MDvSRJkiQNyyD75D2wtcADqKrPA/cvfCRJkiRJ0nwNMpJ3eZK/AdYCBfwK8LkkTwOoqquGkE+SJEmSNIBBiryn9N7PmNH+VKaKvqMWJNE2fGPTZl563qeH2YUkSZKkReoTJzyvdYQFM8jqmkdu73iSV1bVh2e0rQFeAmysqif12t4N/BKwhaktGV5VVXcOGlySJEmS9JMGeSZvLqfP0nY2cMyMtsuAJ1XVk4GvA29fwAySJEmStKgtZJGXmQ1VtY6pffGmt11aVVsXbPkSsGIBM0iSJEnSoraQRV7N4zevBj61gBkkSZIkaVEb6kjedk9O3sHUFgznbOecVUkmk0xu2XzXjuaTJEmSpM7re+GVJA8FXg6snP67qvqj3scvDHCtU5lakOXoqtrmCGBVrQZWAyw5+AnzGSmUJEmSpEVlkC0U/hG4C7gS+MHMg1X1hn4ukuQY4K3AL1TVfQP0L0mSJEmawyBF3oqqmrlS5nYlWQs8F1iWZANTe+y9HXgocFkSgC9V1W8Ocl1JkiRJ0uwGKfL+NcnPVNW1/f6gqk6epfnMAfqUJEmSJA1gkCLv2cCpSf6DqemaAaq3393QHbJ0307tQi9JkiRJwzBIkffCoaWQJEmSJC2Ivou8qrppmEHm8s1N3+fEj1/TMoIkSZKkjjn35SOZmDhSC7lP3k9IsibJxiTrp7X9cZJrklyd5NIkjxpmBkmSJElaTIZa5AFnAzNX5Hx3VT25qg4HPgn84ZAzSJIkSdKiMdQir6rWAXfMaNs87etegJucS5IkSdICGWThlQWT5E+AX2Nqc/UjW2SQJEmSpC4a9nTNWVXVO6rqIOAc4A3bOi/JqiSTSSZ/sHnT6AJKkiRJ0phqUuRNcw7w8m0drKrVVTVRVRMP3XfpCGNJkiRJ0ngaeZGX5NBpX48Dbhh1BkmSJEnqqqE+k5dkLfBcYFmSDcAZwIuSHAb8CLgJ+M1hZpAkSZKkxWSoRV5VnTxL85nD7FOSJEmSFrMmq2vOx+OX7tHJ3eglSZIkaSG1XnhFkiRJkrSALPIkSZIkqUPGZrrmzXf+kD+54ObWMSRJkiR1wDtedmDrCEMz1JG8JGuSbEyyfpZjb0lSSZYNM4MkSZIkLSbDnq55NnDMzMYkBwG/CHx7yP1LkiRJ0qIy1CKvqtYBd8xy6C+AtwI1zP4lSZIkabEZ+cIrSY4DvlNVXxl135IkSZLUdSNdeCXJnsDvMzVVs5/zVwGrAPZb/ughJpMkSZKkbhj1SN7BwOOAryT5FrACuCrJI2c7uapWV9VEVU3ste/+I4wpSZIkSeNppCN5VXUtcMDW771Cb6KqbhtlDkmSJEnqqmFvobAW+CJwWJINSU4bZn+SJEmStNgNdSSvqk6e4/jKYfYvSZIkSYvNSKdr7ogDl+zW6V3pJUmSJGkhjHwLBUmSJEnS8IzNSN6dm+7nE+e6PoskSZKk+XvpictaRxg6R/IkSZIkqUOGvbrmmiQbk6yf1vbfk3wnydW914uGmUGSJEmSFpNhj+SdDRwzS/tfVNXhvdc/DzmDJEmSJC0aQy3yqmodcMcw+5AkSZIkPajVM3lvSHJNbzrn0m2dlGRVkskkk5s33z7KfJIkSZI0lloUeR8EDgYOB24G3rutE6tqdVVNVNXEvvvuP6p8kiRJkjS2Rl7kVdUtVfVAVf0I+FvgiFFnkCRJkqSuGnmRl+TAaV9fBqzf1rmSJEmSpMEMdTP0JGuB5wLLkmwAzgCem+RwoIBvAb8xzAySJEmStJgMtcirqpNnaT5zPtdasnTXRbE7vSRJkiTtiFara0qSJEmShsAiT5IkSZI6ZKjTNRfSvbfdz7+dtbF1DEmSJElj5ohXHdA6wkgNdSSvt9n5xiTrZ7S/MckNSa5L8mfDzCBJkiRJi8mwp2ueDRwzvSHJkcBxwFOq6onAe4acQZIkSZIWjaEWeVW1DrhjRvNrgXdV1Q965zgHU5IkSZIWSIuFV54APCfJFUkuT/JzDTJIkiRJUie1WHhlV+DhwNOBnwP+Icnjq6pmnphkFbAK4JH7rxhpSEmSJEkaRy1G8jYA59eUfwN+BMy6y3lVra6qiaqaWLL3/iMNKUmSJEnjqEWR9wngSIAkTwB2B25rkEOSJEmSOmeo0zWTrAWeCyxLsgE4A1gDrOltq7AFeOVsUzUlSZIkSYMbapFXVSdv49Apw+xXkiRJkharFguvzMtey3ZddDvVS5IkSdKgWjyTJ0mSJEkaEos8SZIkSeqQsZmuueWWH7LhPd9rHUOSJEnSmFjxO49sHaGJoY7kJVmTZGNvJc2tbX+f5Ore61tJrh5mBkmSJElaTIY9knc28AHgI1sbqupXtn5O8l7griFnkCRJkqRFY9hbKKxLsnK2Y0kC/DJw1DAzSJIkSdJi0nLhlecAt1TVjQ0zSJIkSVKntCzyTgbWbu+EJKuSTCaZvOOe20cUS5IkSZLGV5MiL8muwPHA32/vvKpaXVUTVTXx8L33H004SZIkSRpjrUbyngfcUFUbGvUvSZIkSZ007C0U1gJfBA5LsiHJab1DJzHHVE1JkiRJ0uCGvbrmydtoP3WY/UqSJEnSYjXsffIWzO6P2G3R7lgvSZIkSf1qubqmJEmSJGmBjc1I3g9vuY9b/uLq1jEkSZIk7cQe8ebDW0dozpE8SZIkSeqQYa+uuSbJxiTrp7UdnuRLSa7ubXR+xDAzSJIkSdJiMuyRvLOBY2a0/Rnwzqo6HPjD3ndJkiRJ0gIYapFXVeuAO2Y2A/v2Pu8HfHeYGSRJkiRpMWmx8MqbgEuSvIepIvOZ2zoxySpgFcCKpQeOJp0kSZIkjbEWC6+8FnhzVR0EvBk4c1snVtXqqpqoqomH77VkZAElSZIkaVy1KPJeCZzf+3wu4MIrkiRJkrRAWhR53wV+off5KODGBhkkSZIkqZOG+kxekrXAc4FlSTYAZwCvAd6XZFfgv+g9cydJkiRJ2nFDLfKq6uRtHPrZQa+12yP2dPd6SZIkSZpDi+makiRJkqQhsciTJEmSpA5psU/evNy/cTMb339Z6xiSJEmSdjIHvPH5rSPsVIY6kpdkTZKNSdZPa3tKki8muTbJPyXZd5gZJEmSJGkxGfZ0zbOBY2a0fQh4W1X9DHAB8LtDziBJkiRJi8ZQi7yqWgfcMaP5CcC63ufLgJcPM4MkSZIkLSYtFl65Djiu9/lE4KAGGSRJkiSpk1oUea8GXpfkSmAfYMu2TkyyKslkksnb77lrZAElSZIkaVyNfHXNqroB+EWAJE8AXrydc1cDqwEOf8wTaiQBJUmSJGmMjXwkL8kBvfeHAH8A/PWoM0iSJElSVw17C4W1wBeBw5JsSHIacHKSrwM3AN8FzhpmBkmSJElaTIY6XbOqTt7GofcNs19JkiRJWqxG/kzefO16wL7uZC9JkiRJc0jVeKxnkuRu4Gutc6izlgG3tQ6hTvMe0zB5f2nYvMc0bN5j/XlsVS2f66SxGckDvlZVE61DqJuSTHp/aZi8xzRM3l8aNu8xDZv32MJqsU+eJEmSJGlILPIkSZIkqUPGqchb3TqAOs37S8PmPaZh8v7SsHmPadi8xxbQ2Cy8IkmSJEma2ziN5EmSJEmS5mCRJ0mSJEkdstMVeUmOSfK1JN9I8rZZjj80yd/3jl+RZOXoU2pc9XF//XySq5Lcn+SEFhk13vq4x347yVeTXJPkM0ke2yKnxlMf99dvJrk2ydVJPp/kp1vk1Pia6x6bdt7Lk1QSl7xX3/r4O+zUJLf2/g67Osmvt8jZBTtVkZdkF+AvgRcCPw2cPMs/UKcBm6rqEOAvgD8dbUqNqz7vr28DpwIfHW06dUGf99iXgYmqejJwHvBno02pcdXn/fXRqvqZqjqcqXvrz0ccU2Osz3uMJPsApwNXjDahxlm/9xfw91V1eO/1oZGG7JCdqsgDjgC+UVXfrKotwMeA42accxzw4d7n84Cjk2SEGTW+5ry/qupbVXUN8KMWATX2+rnHPltV9/W+fglYMeKMGl/93F+bp33dC3B1NQ2in/8OA/hjpv4n+3+NMpzGXr/3lxbAzlbkPRr4z2nfN/TaZj2nqu4H7gL2H0k6jbt+7i9pRwx6j50GfGqoidQlfd1fSV6f5N+ZGsn7rRFlUzfMeY8leRpwUFVdNMpg6oR+/418ee+RhvOSHDSaaN2zsxV5krQoJDkFmADe3TqLuqWq/rKqDgZ+D/iD1nnUHUkewtQU4Le0zqLO+idgZe+Rhst4cPaeBrSzFXnfAaZX7Ct6bbOek2RXYD/g9pGk07jr5/6SdkRf91iS5wHvAI6tqh+MKJvG36B/h30MeOlQE6lr5rrH9gGeBHwuybeApwMXuviK+jTn32FVdfu0fxc/BPzsiLJ1zs5W5P1f4NAkj0uyO3AScOGMcy4EXtn7fALwL+WO7upPP/eXtCPmvMeSPBX4G6YKvI0NMmp89XN/HTrt64uBG0eYT+Nvu/dYVd1VVcuqamVVrWTqueJjq2qyTVyNmX7+Djtw2tdjgetHmK9Tdm0dYLqquj/JG4BLgF2ANVV1XZI/Aiar6kLgTOB/J/kGcAdTN4g0p37uryQ/B1wALAV+Kck7q+qJDWNrjPT5d9i7gb2Bc3trRn27qo5tFlpjo8/76w29keIfApt48H+KSnPq8x6T5qXP++u3khwL3M/Uf+ef2izwmIuDYJIkSZLUHTvbdE1JkiRJ0g6wyJMkSZKkDrHIkyRJkqQOsciTJEmSpA6xyJMkSZKkDrHIkyQtSklOTfKB3uffTPJr2zn37CT/keTqJF9JcvTokkqSNJidap88SZJaqKq/7uO0362q85IcCawGDp3rB5IkteBIniSpk5J8IsmVSa5LsqrX9qokX0/yb8Czpp3735P8Tp+X/iLw6Dn6fleSrya5Jsl7em2PS/LFJNcm+R9J7pnnH02SpO1yJE+S1FWvrqo7kuwB/N8kFwHvBH4WuAv4LPDleVz3GOAT2zqYZH/gZcBPVVUlWdI79D7gg1X1kSSvn0e/kiT1xZE8SVJX/VaSrwBfAg4CfhX4XFXdWlVbgL8f8HrvTvJ14KPAn27nvLuA/wLOTHI8cF+v/VnA2t7n/z1g35Ik9c0iT5LUOUmeCzwPeEZVPYWpEbsbdvCyv1tVTwB+D1izrZOq6n7gCOA84CXAxdMP72AGSZLmZJEnSeqi/YBNVXVfkp8Cng7sAfxCkv2T7AacOM9rfwB4SJIXzHYwyd7AflX1z8Cbgaf0Dn0BOKn3+RXz7FuSpDlZ5EmSuuhiYNck1wPvYmrK5s3Af2dq4ZQvANfP+E1fo2xVVcD/AN66jVP2AT6Z5Brg88Bv99pPB16f5FrmWLhFkqQdkal/qyRJWrySvB+4qqrOGmGf91TV3qPqT5K0eDiSJ0la1JL8MfD/ARe2ziJJ0kJwJE+SpJ4kf8m0/fN63retEb4kFwCPm9H8e1V1yTDySZLUD4s8SZIkSeoQp2tKkiRJUodY5EmSJElSh1jkSZIkSVKHWORJkiRJUodY5EmSJElSh1jkSZIkSVKHWORJkiRJUodY5EmSJElSh1jkSZIkSVKHWORJkiRJUodY5EmSJElSh1jkSZIkSVKHWORJkiRJUodY5EmSJElSh+zaOkC/li1bVitXrmwdQ5IkSZKauPLKK2+rquVznTc2Rd6KvfblU6e9qXUMSZIkSR21/LWntI6wXUlu6uc8p2tKkiRJUoc0K/KSrEmyMcn6VhkkSZIkqWtajuSdDRzTsH9JkiRJ6pxmRV5VrQPuaNW/JEny3e01AAAgAElEQVSSJHXRTv1MXpJVSSaTTN5+z+bWcSRJkiRpp7dTF3lVtbqqJqpqYv+9920dR5IkSZJ2ejt1kSdJkiRJGoxFniRJkiR1SMstFNYCXwQOS7IhyWmtskiSJElSV+zaquOqOrlV35IkSZLUVc2KvEHtuvzhLH/tKa1jSJIkSdJOzWfyJEmSJKlDxmYk74e3fpfvffCdrWNIkiRJGmOPfO0ZrSMMXZORvCQHJflskq8muS7J6S1ySJIkSVLXtBrJux94S1VdlWQf4Mokl1XVVxvlkSRJkqROaDKSV1U3V9VVvc93A9cDj26RRZIkSZK6pPnCK0lWAk8FrmibRJIkSZLGX9MiL8newMeBN1XV5lmOr0oymWTy9nvuG31ASZIkSRozzYq8JLsxVeCdU1Xnz3ZOVa2uqomqmth/7z1HG1CSJEmSxlCr1TUDnAlcX1V/3iKDJEmSJHVRq5G8ZwG/ChyV5Ore60WNskiSJElSZzTZQqGqPg+kRd+SJEmS1GWt9skb2G7LH7UodqeXJEmSpB3RfAsFSZIkSdLCsciTJEmSpA4Zm+ma37/1G1z3V8e2jiFJkiRpDD3xdRe2jjAyrbZQeFiSf0vylSTXJXlnixySJEmS1DWtRvJ+ABxVVff0NkX/fJJPVdWXGuWRJEmSpE5otYVCAff0vu7We1WLLJIkSZLUJc0WXkmyS5KrgY3AZVV1RasskiRJktQVzYq8qnqgqg4HVgBHJHnSzHOSrEoymWRy0z1bRh9SkiRJksZM8y0UqupO4LPAMbMcW11VE1U1sXTv3UcfTpIkSZLGTKvVNZcnWdL7vAfwfOCGFlkkSZIkqUtara55IPDhJLswVWj+Q1V9slEWSZIkSeqMVqtrXgM8tUXfkiRJktRlrUbyBrbH8kMW1S71kiRJkjQfzRdekSRJkiQtnLEZybv7thv5zIde3DqGJEmSpDFz9K9f1DrCSDmSJ0mSJEkd0rTIS7JLki8ncWVNSZIkSVoArUfyTgeub5xBkiRJkjqjWZGXZAXwYuBDrTJIkiRJUte0HMn7X8BbgR9t64Qkq5JMJpm88+4to0smSZIkSWOqSZGX5CXAxqq6cnvnVdXqqpqoqokl++w+onSSJEmSNL5ajeQ9Czg2ybeAjwFHJfm7RlkkSZIkqTOaFHlV9faqWlFVK4GTgH+pqlNaZJEkSZKkLmm9uqYkSZIkaQGlqlpn6MvExERNTk62jiFJkiRJTSS5sqom5jrPkTxJkiRJ6hCLPEmSJEnqkF1bB+jXHbfdyMfOekHrGJIkSZLGyEmvuqR1hJFrVuT1tk+4G3gAuL+fuaWSJEmSpO1rPZJ3ZFXd1jiDJEmSJHWGz+RJkiRJUoe0LPIKuDTJlUlWNcwhSZIkSZ3Rcrrms6vqO0kOAC5LckNVrZt+Qq/4WwWwbP+HtcgoSZIkSWOl2UheVX2n974RuAA4YpZzVlfVRFVN7LP37qOOKEmSJEljp0mRl2SvJPts/Qz8IrC+RRZJkiRJ6pJW0zUfAVyQZGuGj1bVxY2ySJIkSVJnNCnyquqbwFNa9C1JkiRJXdZ6n7y+PXzZoYtyt3pJkiRJGoT75EmSJElSh1jkSZIkSVKHjM10zY133Mj7z3lB6xiSJEmSxsgbX7H4HvlqNpKXZEmS85LckOT6JM9olUWSJEmSuqLlSN77gIur6oQkuwN7NswiSZIkSZ3QpMhLsh/w88CpAFW1BdjSIoskSZIkdUmr6ZqPA24Fzkry5SQfSrJXoyySJEmS1BmtirxdgacBH6yqpwL3Am+beVKSVUkmk0zes9mBPkmSJEmaS6sibwOwoaqu6H0/j6mi78dU1eqqmqiqib333X2kASVJkiRpHDUp8qrqe8B/Jjms13Q08NUWWSRJkiSpS1qurvlG4JzeyprfBF7VMIskSZIkdUKzIq+qrgYmWvUvSZIkSV3UciRvIAc8/NBFuVu9JEmSJA2i1cIrkiRJkqQhGJuRvA2bbuR3zjumdQxJkiRJY+I9J1zcOkITjuRJkiRJUoc0KfKSHJbk6mmvzUne1CKLJEmSJHVJk+maVfU14HCAJLsA3wEuaJFFkiRJkrpkZ5iueTTw71V1U+sgkiRJkjTudoYi7yRg7WwHkqxKMplk8r7NW0YcS5IkSZLGT9MiL8nuwLHAubMdr6rVVTVRVRN77rv7aMNJkiRJ0hhqPZL3QuCqqrqlcQ5JkiRJ6oTWRd7JbGOqpiRJkiRpcM2KvCR7Ac8Hzm+VQZIkSZK6pskWCgBVdS+wf7/nr1h66KLdsV6SJEmS+tV6uqYkSZIkaQFZ5EmSJElShzSbrjmoG+/8Ni/8xBtbx5AkSZI0Jj710ve3jtBE3yN5SfZK8pDe5yckOTbJbvPtOMmbk1yXZH2StUkeNt9rSZIkSZKmDDJdcx3wsCSPBi4FfhU4ez6d9q7xW8BEVT0J2AU4aT7XkiRJkiQ9aJAiL1V1H3A88FdVdSLwxB3oe1dgjyS7AnsC392Ba0mSJEmSGLDIS/IM4BXARb22XebTaVV9B3gP8G3gZuCuqrp0PteSJEmSJD1okCLvdODtwAVVdV2SxwOfnU+nSZYCxwGPAx4F7JXklFnOW5VkMsnkls3fn09XkiRJkrSo9LW6ZpJdgGOr6titbVX1Taaeq5uP5wH/UVW39q5/PvBM4O+mn1RVq4HVAPsdckDNsy9JkiRJWjT6GsmrqgeAZy9gv98Gnp5kzyQBjgauX8DrS5IkSdKiNMg+eV9OciFwLnDv1saqOn/QTqvqiiTnAVcB9wNfpjdiJ0mSJEmav0GKvIcBtwNHTWsrYOAiD6CqzgDOmM9vJUmSJEmzS9V4POo2MTFRk5OTrWNIkiRJUhNJrqyqibnO63t1zSQrklyQZGPv9fEkK3YspiRJkiRpIQ0yXfMs4KPAib3vp/Tanr/QoWZz45238OLz3zuKriRJkiR1wEXHv6V1hCYG2SdveVWdVVX3915nA8uHlEuSJEmSNA+DFHm3JzklyS691ylMLcQyL0lOT7I+yXVJ3jTf60iSJEmSHjRIkfdq4JeB7wE3AycAp86n0yRPAl4DHAE8BXhJkkPmcy1JkiRJ0oMGKfJWVNWxVbW8qg6oqpcCj5lnv/8NuKKq7quq+4HLgePneS1JkiRJUs8gRd77+2zrx3rgOUn2T7In8CLgoJknJVmVZDLJ5Ja77v2Ji0iSJEmSftycq2smeQbwTGB5kt+edmhfYJf5dFpV1yf5U+BS4F7gauCBWc5bDawG2O+Qg8ZjQz9JkiRJaqifkbzdgb2ZKgj3mfbazNRzefNSVWdW1c9W1c8Dm4Cvz/dakiRJkqQpc47kVdXlwOVJzq6qmxaq4yQHVNXGJI9h6nm8py/UtSVJkiRpsRrkmbwPJVmy9UuSpUku2YG+P57kq8A/Aa+vqjt34FqSJEmSJPoYyZtm2fRCrKo2JTlgvh1X1XPm+1tJkiRJ0uwGKfJ+lOQxVfVtgCSPBUa2GMqhSx7BRce/ZVTdSZIkSdJYGqTIewfw+SSXAwGeA6waSipJkiRJ0rz0XeRV1cVJnsaDC6S8qapuG06sn/SNTbfxko+vGVV3kiRJkjrgky9/desIIzfnwitJfqr3/jTgMcB3e6/H9Nq299s1STYmWT+t7eFJLktyY+996Y79ESRJkiRJW/UzkvcW4DXAe2c5VsBR2/nt2cAHgI9Ma3sb8JmqeleSt/W+/15faSVJkiRJ29XPPnmv6b0fOejFq2pdkpUzmo8Dntv7/GHgc1jkSZIkSdKCmLPIS3L89o5X1fkD9vmIqrq59/l7wCMG/L0kSZIkaRv6ma75S733A4BnAv/S+34k8K/AoEXe/1NVlWSb2zAkWUVvBc89lu0/324kSZIkadHoZ7rmqwCSXAr89NZRuCQHMvXM3aBuSXJgVd3cu8bG7fS9GlgNsOTglSPbk0+SJEmSxtWcq2tOc9C0aZYAtzC12uagLgRe2fv8SuAf53ENSZIkSdIsBtkM/TNJLgHW9r7/CvDp7f0gyVqmFllZlmQDcAbwLuAfkpwG3AT88qChJUmSJEmzG2Qz9DckeRnw872m1VV1wRy/OXkbh47ut19JkiRJUv8GGckDuAq4u6o+nWTPJPtU1d3DCDbTIUuXLcrd6iVJkiRpEH0/k5fkNcB5wN/0mh4NfGIYoSRJkiRJ8zPIwiuvB54FbAaoqhuZ2lZBkiRJkrSTGGS65g+qaksSAJLsCoxsW4NvbNrES849b1TdSZIkSeqIT554QusIIzXISN7lSX4f2CPJ84FzgX/a3g+SrEmyMcn6aW0nJrkuyY+STMwvtiRJkiRpNoMUeW8DbgWuBX4D+GfgD+b4zdnAMTPa1gPHA+sG6FuSJEmS1Ie+pmsm2QX4SFW9Avjbfi9eVeuSrJzRdn3vmv2nlCRJkiT1pa+RvKp6AHhskt2HnEeSJEmStAMGWXjlm8AXklwI3Lu1sar+fMFT9SRZBawC2GPZsmF1I0mSJEmdMUiR9++910OAfYYT58dV1WpgNcCSgw8e2UqekiRJkjSu+i7yquqdAEn2nfpadw8tlSRJkiRpXvpeXTPJRJJrgWuAa5N8JcnPzvGbtcAXgcOSbEhyWpKXJdkAPAO4KMklO/IHkCRJkiQ9aJDpmmuA11XV/wFI8mzgLODJ2/pBVZ28jUMXDNCvJEmSJKlPgxR5D2wt8ACq6vNJ7h9CplkdsnTpotupXpIkSZIGNUiRd3mSvwHWAgX8CvC5JE8DqKqrhpBPkiRJkjSAQYq8p/Tez5jR/lSmir6jFiTRNnxj02Zeet6nh9mFJEmSpI76xAnPax1hZAZZXfPI7R1P8sqq+vCOR5IkSZIkzVffq2v24fSZDUnWJNmYZP20tncnuSHJNUkuSLJkATNIkiRJ0qK2kEVeZmk7GzhmRttlwJOq6snA14G3L2AGSZIkSVrUFrLIq59oqFoH3DGj7dKq2roq55eAFQuYQZIkSZIWtWGP5M3l1cCntnnBZFWSySSTWzbfNf9kkiRJkrRI9L3wSpKHAi8HVk7/XVX9Ue/jFwbpOMk7gPuBc7Z1TlWtBlYDLDn4CT8xUihJkiRJ+nGDbKHwj8BdwJXAD2YerKo39HuhJKcCLwGOriqLN0mSJElaIIMUeSuqauYiKgNLcgzwVuAXquq+Hb2eJEmSJOlBgzyT969JfmaQiydZC3wROCzJhiSnAR8A9gEuS3J1kr8e5JqSJEmSpG1Lv7Mlk3wVOAT4D6amawao3lYIQzcxMVGTk5Oj6EqSJEmSdjpJrqyqibnOG2S65gt3II8kSZIkaQT6LvKq6qZhBpEkSZIk7bhBRvKa+uam73Pix69pHUOSJEnSmDr35SN50qy5hdwM/SckWZNkY5L109r+OMk1vUVXLk3yqGFmkCRJkqTFZKhFHnA2MHPbhXdX1ZOr6nDgk8AfDjmDJEmSJC0aQy3yqmodcMeMts3Tvu4FuBm6JEmSJC2QJs/kJfkT4NeAu4AjW2SQJEmSpC4a9nTNWVXVO6rqIOAc4A3bOi/JqiSTSSZ/sHnT6AJKkiRJ0phqUuRNcw7w8m0drKrVVTVRVRMP3XfpCGNJkiRJ0ngaeZGX5NBpX48Dbhh1BkmSJEnqqqE+k5dkLfBcYFmSDcAZwIuSHAb8CLgJ+M1hZpAkSZKkxWSoRV5VnTxL85nD7FOSJEmSFrMmq2vOx+OX7rFodqiXJEmSpPlqvfCKJEmSJGkBWeRJkiRJUoeMzXTNm+/8IX9ywc2tY0iSJEkaY+942YGtIwzdUEfykqxJsjHJ+lmOvSVJJVk2zAySJEmStJgMe7rm2cAxMxuTHAT8IvDtIfcvSZIkSYvKUIu8qloH3DHLob8A3grUMPuXJEmSpMVm5AuvJDkO+E5VfWXUfUuSJElS14104ZUkewK/z9RUzX7OXwWsAthv+aOHmEySJEmSumHUI3kHA48DvpLkW8AK4Kokj5zt5KpaXVUTVTWx1777jzCmJEmSJI2nkY7kVdW1wAFbv/cKvYmqum2UOSRJkiSpq4a9hcJa4IvAYUk2JDltmP1JkiRJ0mI31JG8qjp5juMrh9m/JEmSJC02I52uuSMOXLLbotidXpIkSZJ2xMi3UJAkSZIkDc/YjOTduel+PnGu67NIkiRJ2jEvPXFZ6whD5UieJEmSJHXIsFfXXJNk4//f3r1HWVbWd/5/f0KDcu+mG5DYGFCB/NTEFksWXoOo2F5GvIChf5qgMrbxNmriKCRZQZNxLeN4iWPW6LTStCbaigjGEG8kXnrMKFpgI81NUDFpRFpooLlkQOA7f5zdUhZV3XWK2mefOvV+rXVW7fPsXWd/u788B748z36eJJsmtL0jybVJNjav57YZgyRJkiQtJG2P5K0DVk7R/oGqWtG8vthyDJIkSZK0YLRa5FXVBmBrm/eQJEmSJN2nq2fy3pDkB810ziXTXZRkdZLxJOPbtt04yPgkSZIkaV7qosj7MPAIYAVwHfC+6S6sqjVVNVZVY/vss3RQ8UmSJEnSvDXwIq+qrq+qe6rqXuCjwFGDjkGSJEmSRtXAi7wkB014+yJg03TXSpIkSZL60+pm6EnWA8cAy5JsBk4HjkmyAijgGuA1bcYgSZIkSQtJq0VeVa2aovmM2XzW4iWLRn5nekmSJEl6oLpaXVOSJEmS1AKLPEmSJEkaIa1O15xLt99wN989c0vXYUiSJEkaAUe98oCuQ2hNqyN5zWbnW5JsmtT+xiRXJLk0yXvajEGSJEmSFpK2p2uuA1ZObEjydOB44LFV9WjgvS3HIEmSJEkLRqtFXlVtALZOan4t8O6qurO5xjmYkiRJkjRHulh45XDgqUkuSPLNJE/oIAZJkiRJGkldLLyyCNgPOBp4AnBWkodXVU2+MMlqYDXAQ5YuH2iQkiRJkjQfdTGStxk4p3q+C9wLTLnLeVWtqaqxqhpbvNfSgQYpSZIkSfNRF0Xe54GnAyQ5HNgNuKGDOCRJkiRp5LQ6XTPJeuAYYFmSzcDpwFpgbbOtwl3AyVNN1ZQkSZIk9a/VIq+qVk1z6uVt3leSJEmSFqouFl6ZlT2XLRrpXeklSZIkaS508UyeJEmSJKkl82Yk767rf8nm9/686zAkSZIkjYjlb31I1yG0wpE8SZIkSRohrRZ5SdYm2dKspLm97TNJNjava5JsbDMGSZIkSVpI2p6uuQ74W+AT2xuq6ve3Hyd5H3BLyzFIkiRJ0oLR9hYKG5IcMtW5JAFeChzbZgySJEmStJB0+UzeU4Hrq+qq6S5IsjrJeJLxrbfdOMDQJEmSJGl+6rLIWwWs39EFVbWmqsaqamy/vZYOKCxJkiRJmr862UIhySLgxcDju7i/JEmSJI2qrkbynglcUVWbO7q/JEmSJI2ktrdQWA98GzgiyeYkpzSnTmInUzUlSZIkSf1re3XNVdO0v6LN+0qSJEnSQtXJM3mzsduBu7L8rQ/pOgxJkiRJGmpdrq4pSZIkSZpj82Yk75fX38H1H9jYdRiSJEmSRsiBb1nRdQhzru2FV9Ym2ZJk04S2FUm+k2Rjs9H5UW3GIEmSJEkLSdvTNdcBKye1vQd4Z1WtAP6ieS9JkiRJmgOtFnlVtQHYOrkZ2Kc53hf4WZsxSJIkSdJC0sUzeW8GvpLkvfSKzCd1EIMkSZIkjaQuVtd8LfCWqjoYeAtwxnQXJlndPLc3vvX2mwcWoCRJkiTNV10UeScD5zTHnwWmXXilqtZU1VhVje235+KBBCdJkiRJ81kXRd7PgN9rjo8FruogBkmSJEkaSa0+k5dkPXAMsCzJZuB04NXAB5MsAv4vsLrNGCRJkiRpIWm1yKuqVdOcenyb95UkSZKkhaqL1TVnZdcD9xjJ3eglSZIkaS518UyeJEmSJKklFnmSJEmSNELmzXTNu7dsY8uHzu86DEmSJEkj5oA3PqvrEOZUqyN5SdYm2ZJk04S2xyb5dpJLkvxjkn3ajEGSJEmSFpK2p2uuA1ZOavsYcGpV/Q5wLvBfW45BkiRJkhaMVou8qtoAbJ3UfDiwoTk+H3hJmzFIkiRJ0kLSxcIrlwLHN8cnAgd3EIMkSZIkjaQuirxXAa9LciGwN3DXdBcmWZ1kPMn4jbfdMrAAJUmSJGm+GvjqmlV1BXAcQJLDgeft4No1wBqAFQ87vAYSoCRJkiTNYwMfyUtyQPPzN4A/Bz4y6BgkSZIkaVS1vYXCeuDbwBFJNic5BViV5IfAFcDPgDPbjEGSJEmSFpJWp2tW1appTn2wzftKkiRJ0kI18GfyZmvRAfuM3E70kiRJkjTXUjU/1jNJcitwZddxaErLgBu6DkJTMjfDy9wML3MzvMzN8DI3w8vcDK/Z5Oa3qmr/nV00b0bygCuraqzrIHR/ScbNzXAyN8PL3AwvczO8zM3wMjfDy9wMrzZz08U+eZIkSZKklljkSZIkSdIImU9F3pquA9C0zM3wMjfDy9wML3MzvMzN8DI3w8vcDK/WcjNvFl6RJEmSJO3cfBrJkyRJkiTtxNAXeUlWJrkyydVJTu06noUoyTVJLkmyMcl407ZfkvOTXNX8XNK0J8n/aPL1gyRHdhv9aEmyNsmWJJsmtPWdiyQnN9dfleTkLv4so2aa3LwjybVN39mY5LkTzp3W5ObKJM+e0O533hxLcnCSrye5LMmlSd7UtNt3OraD3Nh3OpbkwUm+m+TiJjfvbNoPTXJB8/f8mSS7Ne0Pat5f3Zw/ZMJnTZkzzc4OcrMuyU8m9JsVTbvfaQOUZJck309yXvO+mz5TVUP7AnYBfgQ8HNgNuBh4VNdxLbQXcA2wbFLbe4BTm+NTgb9ujp8LfAkIcDRwQdfxj9ILeBpwJLBptrkA9gN+3Pxc0hwv6frPNt9f0+TmHcBbp7j2Uc332YOAQ5vvuV38zmstNwcBRzbHewM/bHJg3xne3Nh3us9NgL2a412BC5r+cBZwUtP+EeC1zfHrgI80xycBn9lRzrr+883n1w5ysw44YYrr/U4bbH7+GPgUcF7zvpM+M+wjeUcBV1fVj6vqLuDTwPEdx6Se44GPN8cfB144of0T1fMdYHGSg7oIcBRV1QZg66TmfnPxbOD8qtpaVTcB5wMr249+tE2Tm+kcD3y6qu6sqp8AV9P7vvM7rwVVdV1VXdQc3wpcDjwU+07ndpCb6dh3BqT55/+25u2uzauAY4Gzm/bJ/WZ7fzobeEaSMH3ONEs7yM10/E4bkCTLgecBH2veh476zLAXeQ8F/n3C+83s+Mtf7Sjgq0kuTLK6aTuwqq5rjn8OHNgcm7PB6zcX5miw3tBMj1m7fTog5qYzzXSYx9H7P9/2nSEyKTdg3+lcM+1sI7CFXgHwI+Dmqrq7uWTi3/OvctCcvwVYirlpxeTcVNX2fvOupt98IMmDmjb7zeD8DfA24N7m/VI66jPDXuRpODylqo4EngO8PsnTJp6s3tiyy7QOAXMxdD4MPAJYAVwHvK/bcBa2JHsBnwPeXFXbJp6z73RritzYd4ZAVd1TVSuA5fRGEn6745DUmJybJI8BTqOXoyfQm4L59g5DXHCSPB/YUlUXdh0LDH+Rdy1w8IT3y5s2DVBVXdv83AKcS++L/vrt0zCbn1uay83Z4PWbC3M0IFV1ffMv4nuBj3LfdAtzM2BJdqVXRHyyqs5pmu07Q2Cq3Nh3hktV3Qx8HXgival+i5pTE/+ef5WD5vy+wI2Ym1ZNyM3KZvpzVdWdwJnYbwbtycALklxDb8r4scAH6ajPDHuR9z3gsGZVmt3oPZT4hY5jWlCS7Jlk7+3HwHHAJnp52L4K08nAPzTHXwD+sFnJ6WjglgnTodSOfnPxFeC4JEuaKVDHNW2aY5OeR30Rvb4Dvdyc1KysdShwGPBd/M5rRfOMwxnA5VX1/gmn7Dsdmy439p3uJdk/yeLmeHfgWfSemfw6cEJz2eR+s70/nQB8rRkhny5nmqVpcnPFhP9pFXrPfU3sN36ntayqTquq5VV1CL3voK9V1cvoqM8s2vkl3amqu5O8gd4/cLsAa6vq0o7DWmgOBM7tfV+wCPhUVX05yfeAs5KcAvwUeGlz/RfpreJ0NXAH8MrBhzy6kqwHjgGWJdkMnA68mz5yUVVbk/wVvf8oAvjLqprpgiGaxjS5OSa9JayL3iq1rwGoqkuTnAVcBtwNvL6q7mk+x++8ufdk4A+AS5pnWAD+FPvOMJguN6vsO507CPh4kl3oDQqcVVXnJbkM+HSS/wZ8n16RTvPz75JcTW8RqpNgxznTrE2Xm68l2Z/eKpobgT9qrvc7rVtvp4M+k17BKEmSJEkaBcM+XVOSJEmS1AeLPEmSJEkaIRZ5kiRJkjRCLPIkSZIkaYRY5EmSJEnSCLHIkyRpgiSHJNk0RfvHkjyqi5gkSerHUO+TJ0nSsKiq/9x1DJIkzYQjeZIk3d+iJJ9McnmSs5PskeQbScYAkqxMclGSi5P8S9fBSpI0kSN5kiTd3xHAKVX1r0nWAq/bfiLJ/sBHgadV1U+S7NdVkJIkTcWRPEmS7u/fq+pfm+O/B54y4dzRwIaq+glAVW0ddHCSJO2IRZ4kSfdXO3kvSdLQssiTJOn+Hpbkic3x/w98a8K57wBPS3IogNM1JUnDxiJPkqT7uxJ4fZLLgSXAh7efqKpfAKuBc5JcDHymmxAlSZpaqpyBIkmSJEmjwpE8SZIkSRohFnmSJEmSNEIs8iRJkiRphFjkSZIkSdIIsciTJEmSpBFikSdJkiRJI8QiT5IkSZJGiEWeJEmSJI0QizxJkiRJGiEWeZIkSZI0QizyJEmSJGmEWORJkiRJ0gixyJMkSZKkEWKRJ0mSJEkjxCJPkiRJkkaIRZ4kSZIkjRCLPEmSJEkaIRZ5kiRJkjRCFnUdwEwtW7asDjnkkK7DkCRJkqROXHjhhTdU1f47u27eFHnL99yHL53y5q7DkCRJkjSi9n/ty7sOYYeS/HQm1zldU4n+kisAAB9GSURBVJIkSZJGSGdFXpK1SbYk2dRVDJIkSZI0arocyVsHrOzw/pIkSZI0cjor8qpqA7C1q/tLkiRJ0iga6mfykqxOMp5k/MbbtnUdjiRJkiQNvaEu8qpqTVWNVdXY0r326TocSZIkSRp6Q13kSZIkSZL6Y5EnSZIkSSOkyy0U1gPfBo5IsjnJKV3FIkmSJEmjYlFXN66qVf1cv2j//YZ+B3pJkiRJ6prTNSVJkiRphFjkSZIkSdII6Wy6Zr9++Yuf8fMPv7PrMCRJkiTNUw957eldhzAQnYzkJTk4ydeTXJbk0iRv6iIOSZIkSRo1XY3k3Q38SVVdlGRv4MIk51fVZR3FI0mSJEkjoZORvKq6rqouao5vBS4HHtpFLJIkSZI0SjpfeCXJIcDjgAu6jUSSJEmS5r9Oi7wkewGfA95cVdumOL86yXiS8Rtvu2PwAUqSJEnSPNNZkZdkV3oF3ier6pyprqmqNVU1VlVjS/faY7ABSpIkSdI81NXqmgHOAC6vqvd3EYMkSZIkjaKuRvKeDPwBcGySjc3ruR3FIkmSJEkjo5MtFKrqW0C6uLckSZIkjbKu9snr2677/+aC2aFekiRJkmar8y0UJEmSJElzZ96M5P3HL67m0v/5gq7DkCRJkjSPPPp1X+g6hIFzJE+SJEmSRkhXWyg8OMl3k1yc5NIk7+wiDkmSJEkaNV1N17wTOLaqbms2Rf9Wki9V1Xc6ikeSJEmSRkJXWygUcFvzdtfmVV3EIkmSJEmjpLNn8pLskmQjsAU4v6ou6CoWSZIkSRoVnRV5VXVPVa0AlgNHJXnM5GuSrE4ynmT8ptvuGnyQkiRJkjTPdL66ZlXdDHwdWDnFuTVVNVZVY0v22m3wwUmSJEnSPNPV6pr7J1ncHO8OPAu4ootYJEmSJGmUdLW65kHAx5PsQq/QPKuqzusoFkmSJEkaGV2trvkD4HFd3FuSJEmSRllXI3l9233/R/Lo132h6zAkSZIkaah1vvCKJEmSJGnuzJuRvFtvuIp/+djzug5DkiRJ0jzxjP/8T12H0IlOR/KaDdG/n8RFVyRJkiRpDnQ9XfNNwOUdxyBJkiRJI6OzIi/JcuB5wMe6ikGSJEmSRk2XI3l/A7wNuLfDGCRJkiRppHRS5CV5PrClqi7cyXWrk4wnGb/51rsGFJ0kSZIkzV9djeQ9GXhBkmuATwPHJvn7yRdV1ZqqGquqscV77zboGCVJkiRp3umkyKuq06pqeVUdApwEfK2qXt5FLJIkSZI0SrpeXVOSJEmSNIc63wy9qr4BfKPjMCRJkiRpJHRe5M3U3ssOW7A71kuSJEnSTDldU5IkSZJGiEWeJEmSJI2QeTNdc+sNV/HpM5/ddRiSJEmShtxJr/xK1yF0qrMir9kj71bgHuDuqhrrKhZJkiRJGhVdj+Q9vapu6DgGSZIkSRoZPpMnSZIkSSOkyyKvgK8muTDJ6g7jkCRJkqSR0eV0zadU1bVJDgDOT3JFVW2YeEFT/K0GWLb0wV3EKEmSJEnzSmcjeVV1bfNzC3AucNQU16ypqrGqGtt7r90GHaIkSZIkzTudFHlJ9kyy9/Zj4DhgUxexSJIkSdIo6Wq65oHAuUm2x/CpqvpyR7FIkiRJ0sjopMirqh8Dj+3i3pIkSZI0yrreJ2/G9lt22ILfuV6SJEmSdsZ98iRJkiRphMybkbwtW6/iQ598dtdhSJIkSRpib3yZs/8cyZMkSZKkEdJZkZdkcZKzk1yR5PIkT+wqFkmSJEkaFV1O1/wg8OWqOiHJbsAeHcYiSZIkSSOhkyIvyb7A04BXAFTVXcBdXcQiSZIkSaOkq+mahwK/AM5M8v0kH0uy5+SLkqxOMp5k/LZt1oCSJEmStDNdFXmLgCOBD1fV44DbgVMnX1RVa6pqrKrG9tpnt0HHKEmSJEnzTldF3mZgc1Vd0Lw/m17RJ0mSJEl6ADop8qrq58C/JzmiaXoGcFkXsUiSJEnSKOlydc03Ap9sVtb8MfDKDmORJEmSpJGQquo6hhkZGxur8fHxrsOQJEmSpE4kubCqxnZ2XWeboUuSJEmS5p5FniRJkiSNkC6fyevL5puu4q1nr+w6DEmSJElD6r0nfLnrEIZCJyN5SY5IsnHCa1uSN3cRiyRJkiSNkk5G8qrqSmAFQJJdgGuBc7uIRZIkSZJGyTA8k/cM4EdV9dOuA5EkSZKk+W4YiryTgPVdByFJkiRJo6DTIq/ZCP0FwGenOb86yXiS8Tu23TXY4CRJkiRpHup6JO85wEVVdf1UJ6tqTVWNVdXYHvvsNuDQJEmSJGn+6brIW4VTNSVJkiRpznRW5CXZE3gWcE5XMUiSJEnSqOlsM/Squh1Y2tX9JUmSJGkUdVbk9Wv5ksPcwV6SJEmSdqLrZ/IkSZIkSXPIIk+SJEmSRsi8ma551c3/xnM+/8auw5AkSZI0hL70wg91HcLQmPFIXpI9k/xGc3x4khck2XW2N07yliSXJtmUZH2SB8/2syRJkiRJPf1M19wAPDjJQ4GvAn8ArJvNTZvP+C/AWFU9BtgFOGk2nyVJkiRJuk8/RV6q6g7gxcD/rKoTgUc/gHsvAnZPsgjYA/jZA/gsSZIkSRJ9FnlJngi8DPinpm2X2dy0qq4F3gv8G3AdcEtVfXU2nyVJkiRJuk8/Rd6bgNOAc6vq0iQPB74+m5smWQIcDxwK/CawZ5KXT3Hd6iTjScbv2vYfs7mVJEmSJC0oM1pdM8kuwAuq6gXb26rqx/Seq5uNZwI/qapfNJ9/DvAk4O8nXlRVa4A1APs+8oCa5b0kSZIkacGY0UheVd0DPGUO7/tvwNFJ9kgS4BnA5XP4+ZIkSZK0IPWzT973k3wB+Cxw+/bGqjqn35tW1QVJzgYuAu4Gvk8zYidJkiRJmr1+irwHAzcCx05oK6DvIg+gqk4HTp/N70qSJEmSppaq+fGo29jYWI2Pj3cdhiRJkiR1IsmFVTW2s+tmvLpmkuVJzk2ypXl9LsnyBxamJEmSJGku9TNd80zgU8CJzfuXN23PmuugpnLVzdfzvHPeN4hbSZIkSZpH/unFf9J1CEOln33y9q+qM6vq7ua1Dti/pbgkSZIkSbPQT5F3Y5KXJ9mleb2c3kIss5LkTUk2Jbk0yZtn+zmSJEmSpPv0U+S9Cngp8HPgOuAE4BWzuWmSxwCvBo4CHgs8P8kjZ/NZkiRJkqT79FPkLa+qF1TV/lV1QFW9EHjYLO/7/wEXVNUdVXU38E3gxbP8LEmSJElSo58i70MzbJuJTcBTkyxNsgfwXODgyRclWZ1kPMn4Xbfcfr8PkSRJkiT9up2urpnkicCTgP2T/PGEU/sAu8zmplV1eZK/Br4K3A5sBO6Z4ro1wBqAfR958PzY0E+SJEmSOjSTkbzdgL3oFYR7T3hto/dc3qxU1RlV9fiqehpwE/DD2X6WJEmSJKlnpyN5VfVN4JtJ1lXVT+fqxkkOqKotSR5G73m8o+fqsyVJkiRpoernmbyPJVm8/U2SJUm+8gDu/bkklwH/CLy+qm5+AJ8lSZIkSWIGI3kTLJtYiFXVTUkOmO2Nq+qp/Vx/2OID3clekiRJknain5G8e5uplQAk+S3AxVAkSZIkaYj0M5L3Z8C3knwTCPBUYHUrUUmSJEmSZmXGRV5VfTnJkdy3QMqbq+qGdsK6v6tvuoHnf27toG4nSZIkaZ447yWv6jqEobLT6ZpJfrv5eSTwMOBnzethTduOfndtki1JNk1o2y/J+Umuan4ueWB/BEmSJEnSdjMZyfsT4NXA+6Y4V8CxO/jddcDfAp+Y0HYq8C9V9e4kpzbv3z6jaCVJkiRJOzSTffJe3fx8er8fXlUbkhwyqfl44Jjm+OPAN7DIkyRJkqQ5sdMiL8mLd3S+qs7p854HVtV1zfHPgQP7/H1JkiRJ0jRmMl3zPzU/DwCeBHytef904P8A/RZ5v1JVlWTabRiSrKZZwXP3ZUtnextJkiRJWjBmMl3zlQBJvgo8avsoXJKD6D1z16/rkxxUVdc1n7FlB/deA6wBWPyIQ9yTT5IkSZJ2op/N0A+eMM0S4Hp6q2326wvAyc3xycA/zOIzJEmSJElT6Gcz9H9J8hVgffP+94F/3tEvJFlPb5GVZUk2A6cD7wbOSnIK8FPgpf0GLUmSJEmaWj+bob8hyYuApzVNa6rq3J38zqppTj1jpveVJEmSJM1cPyN5ABcBt1bVPyfZI8neVXVrG4FN9sgly9zJXpIkSZJ2YsbP5CV5NXA28L+apocCn28jKEmSJEnS7PQzkvd64CjgAoCquirJAa1ENYWrb7qJ53/27EHdTpIkSdI8cd6JJ3QdwlDpZ3XNO6vqru1vkiwC3NZAkiRJkoZIP0XeN5P8KbB7kmcBnwX+cUe/kGRtki1JNk1oOzHJpUnuTTI2u7AlSZIkSVPpp8g7FfgFcAnwGuCLwJ/v5HfWASsntW0CXgxs6OPekiRJkqQZmNEzeUl2AT5RVS8DPjrTD6+qDUkOmdR2efOZM49SkiRJkjQjMxrJq6p7gN9KslvL8UiSJEmSHoB+Vtf8MfCvSb4A3L69sareP+dRNZKsBlYD7L5sWVu3kSRJkqSR0U+R96Pm9RvA3u2E8+uqag2wBmDxIx7hSp6SJEmStBMzLvKq6p0ASfbpva1bW4tKkiRJkjQrM15dM8lYkkuAHwCXJLk4yeN38jvrgW8DRyTZnOSUJC9Kshl4IvBPSb7yQP4AkiRJkqT79DNdcy3wuqr63wBJngKcCfzudL9QVaumOXVuH/eVJEmSJM1QP0XePdsLPICq+laSu1uIaUqPXLKE8048YVC3kyRJkqR5qZ8i75tJ/hewHijg94FvJDkSoKouaiE+SZIkSVIf+inyHtv8PH1S++PoFX3HzklE07j6pm288Ox/bvMWkiRJkuapz5/wzK5DGBr9rK759B2dT3JyVX18Utta4PnAlqp6TNP234H/BNxFb0uGV1bVzf0GLkmSJEm6vxmvrjkDb5qibR2wclLb+cBjqup3gR8Cp81hDJIkSZK0oM1lkZfJDVW1Adg6qe2rVbV9wZbvAMvnMAZJkiRJWtDmssirWfzOq4AvzWEMkiRJkrSgtTqSt8OLkz8D7gY+uYNrVicZTzJ+17ZbHmh8kiRJkjTyZrzwSpIHAS8BDpn4e1X1l83hv/bxWa+gtyDLM6pq2hHAqloDrAFY/IjDZzNSKEmSJEkLSj9bKPwDcAtwIXDn5JNV9YaZfEiSlcDbgN+rqjv6uL8kSZIkaSf6KfKWV9XklTJ3KMl64BhgWZLN9PbYOw14EHB+EoDvVNUf9fO5kiRJkqSp9VPk/Z8kv1NVl8z0F6pq1RTNZ/RxT0mSJElSH/op8p4CvCLJT+hN1wxQzX53rXvkkn3cxV6SJEmSdqKfIu85rUUhSZIkSZoTMy7yquqnbQYiSZIkSXrg+hnJ69SPb/oPTvzcD7oOQ5IkSdIQ+uxLBvIU2bwwl5uh30+StUm2JNk0oe2vkvwgycYkX03ym23GIEmSJEkLSatFHrAOmLztwn+vqt+tqhXAecBftByDJEmSJC0YrRZ5VbUB2DqpbduEt3sC1WYMkiRJkrSQdPJMXpJ3AX8I3AI8vYsYJEmSJGkUtT1dc0pV9WdVdTDwSeAN012XZHWS8STjd267aXABSpIkSdI81UmRN8EngZdMd7Kq1lTVWFWNPWifJQMMS5IkSZLmp4EXeUkOm/D2eOCKQccgSZIkSaOq1WfykqwHjgGWJdkMnA48N8kRwL3AT4E/ajMGSZIkSVpIWi3yqmrVFM1ntHlPSZIkSVrIOlldczYevmR3d7GXJEmSpJ3oeuEVSZIkSdIcmjcjedfd/Evede51XYchSZIkaQj92YsO6jqEoeFIniRJkiSNkFaLvCRrk2xJsmmKc3+SpJIsazMGSZIkSVpI2h7JWwesnNyY5GDgOODfWr6/JEmSJC0orRZ5VbUB2DrFqQ8AbwOqzftLkiRJ0kIz8GfykhwPXFtVF8/g2tVJxpOM377txgFEJ0mSJEnz20CLvCR7AH8K/MVMrq+qNVU1VlVje+6ztN3gJEmSJGkEDHok7xHAocDFSa4BlgMXJXnIgOOQJEmSpJE00H3yquoS4IDt75tCb6yqbhhkHJIkSZI0qtreQmE98G3giCSbk5zS5v0kSZIkaaFrdSSvqlbt5PwhM/2sgxbv6i72kiRJkrQTA19dU5IkSZLUHos8SZIkSRohA1145YG4+aa7+fxnXZ9FkiRJ0tReeOKyrkMYCm0vvLI2yZYkmya0vSPJtUk2Nq/nthmDJEmSJC0kbU/XXAesnKL9A1W1onl9seUYJEmSJGnBaLXIq6oNwNY27yFJkiRJuk9XC6+8IckPmumcSzqKQZIkSZJGThdF3oeBRwArgOuA9013YZLVScaTjG/bduOg4pMkSZKkeWvgRV5VXV9V91TVvcBHgaN2cO2aqhqrqrF99lk6uCAlSZIkaZ4aeJGX5KAJb18EbJruWkmSJElSf1rdJy/JeuAYYFmSzcDpwDFJVgAFXAO8ps0YJEmSJGkhabXIq6pVUzSf0eY9JUmSJGkha7XIm0uLlyxyB3tJkiRJ2omutlCQJEmSJLXAIk+SJEmSRsi8ma55+w13890zt3QdhiRJkqQhddQrD+g6hKHQ6khekrVJtiTZNKn9jUmuSHJpkve0GYMkSZIkLSRtT9dcB6yc2JDk6cDxwGOr6tHAe1uOQZIkSZIWjFaLvKraAGyd1Pxa4N1VdWdzjXMwJUmSJGmOdLHwyuHAU5NckOSbSZ7QQQySJEmSNJK6WHhlEbAfcDTwBOCsJA+vqpp8YZLVwGqAhyxdPtAgJUmSJGk+6mIkbzNwTvV8F7gXmHKX86paU1VjVTW2eK+lAw1SkiRJkuajLoq8zwNPB0hyOLAbcEMHcUiSJEnSyGl1umaS9cAxwLIkm4HTgbXA2mZbhbuAk6eaqilJkiRJ6l+rRV5VrZrm1MvbvK8kSZIkLVRdLLwyK3suW+QO9pIkSZK0E108kydJkiRJasm8Gcm76/pfsvm9P+86DEmSJElDavlbH9J1CEPBkTxJkiRJGiGtFnlJ1ibZ0qykub3tM0k2Nq9rkmxsMwZJkiRJWkjanq65Dvhb4BPbG6rq97cfJ3kfcEvLMUiSJEnSgtH2Fgobkhwy1bkkAV4KHNtmDJIkSZK0kHT5TN5Tgeur6qrpLkiyOsl4kvGtt904wNAkSZIkaX7qsshbBazf0QVVtaaqxqpqbL+9lg4oLEmSJEmavzrZQiHJIuDFwOO7uL8kSZIkjaquRvKeCVxRVZs7ur8kSZIkjaS2t1BYD3wbOCLJ5iSnNKdOYidTNSVJkiRJ/Wt7dc1V07S/ot/P2u3AXd3BXpIkSZJ2osuFVyRJkiRJc8wiT5IkSZJGSCera87GL6+/g+s/sLHrMCRJkiQNsQPfsqLrEDrX9sIra5NsSbJpQtuKJN9JsrHZ6PyoNmOQJEmSpIWk7ema64CVk9reA7yzqlYAf9G8lyRJkiTNgVaLvKraAGyd3Azs0xzvC/yszRgkSZIkaSHp4pm8NwNfSfJeekXmkzqIQZIkSZJGUhera74WeEtVHQy8BThjuguTrG6e2xvfevvNAwtQkiRJkuarLoq8k4FzmuPPAtMuvFJVa6pqrKrG9ttz8UCCkyRJkqT5rIsi72fA7zXHxwJXdRCDJEmSJI2kVp/JS7IeOAZYlmQzcDrwauCDSRYB/xdY3WYMkiRJkrSQtFrkVdWqaU49vs37SpIkSdJC1cXqmrOy64F7uHu9JEmSJO1EF8/kSZIkSZJaMm9G8u7eso0tHzq/6zAkSZIkDbED3visrkPonCN5kiRJkjRCWi3ykqxNsiXJpgltj03y7SSXJPnHJPu0GYMkSZIkLSRtj+StA1ZOavsYcGpV/Q5wLvBfW45BkiRJkhaMVou8qtoAbJ3UfDiwoTk+H3hJmzFIkiRJ0kLSxTN5lwLHN8cnAgdPd2GS1UnGk4zfeNstAwlOkiRJkuazLoq8VwGvS3IhsDdw13QXVtWaqhqrqrGle+07sAAlSZIkab4a+BYKVXUFcBxAksOB5w06BkmSJEkaVQMfyUtyQPPzN4A/Bz4y6BgkSZIkaVS1vYXCeuDbwBFJNic5BViV5IfAFcDPgDPbjEGSJEmSFpJWp2tW1appTn2wzftKkiRJ0kI18GfyZmvRAftwwBuf1XUYkiRJkjTUUlVdxzAjSW4Fruw6Dk1pGXBD10FoSuZmeJmb4WVuhpv5GV7mZniZm+HVb25+q6r239lF82YkD7iyqsa6DkL3l2Tc3AwnczO8zM3wMjfDzfwML3MzvMzN8GorN13skydJkiRJaolFniRJkiSNkPlU5K3pOgBNy9wML3MzvMzN8DI3w838DC9zM7zMzfBqJTfzZuEVSZIkSdLOzaeRPEmSJEnSTgx9kZdkZZIrk1yd5NSu41mIklyT5JIkG5OMN237JTk/yVXNzyVNe5L8jyZfP0hyZLfRj54ka5NsSbJpQlvf+UhycnP9VUlO7uLPMmqmyc07klzb9J+NSZ474dxpTW6uTPLsCe1+782xJAcn+XqSy5JcmuRNTbt9p2M7yI19p2NJHpzku0kubnLzzqb90CQXNH/Pn0myW9P+oOb91c35QyZ81pQ50+zsIDfrkvxkQr9Z0bT7nTZgSXZJ8v0k5zXvB9tvqmpoX8AuwI+AhwO7ARcDj+o6roX2Aq4Blk1qew9wanN8KvDXzfFzgS8BAY4GLug6/lF7AU8DjgQ2zTYfwH7Aj5ufS5rjJV3/2eb7a5rcvAN46xTXPqr5TnsQcGjzXbeL33ut5eYg4MjmeG/gh00O7DvDmxv7Tve5CbBXc7wrcEHTH84CTmraPwK8tjl+HfCR5vgk4DM7ylnXf775/NpBbtYBJ0xxvd9pg8/RHwOfAs5r3g+03wz7SN5RwNVV9eOqugv4NHB8xzGp53jg483xx4EXTmj/RPV8B1ic5KAuAhxVVbUB2Dqpud98PBs4v6q2VtVNwPnAyvajH23T5GY6xwOfrqo7q+onwNX0vvP83mtBVV1XVRc1x7cClwMPxb7TuR3kZjr2nQFp/vm/rXm7a/Mq4Fjg7KZ9cr/Z3p/OBp6RJEyfM83SDnIzHb/TBijJcuB5wMea92HA/WbYi7yHAv8+4f1mdvzFr3YU8NUkFyZZ3bQdWFXXNcc/Bw5sjs1ZN/rNh3karDc002PWbp8OiLnpTDMV5nH0/s+3fWeITMoN2Hc610w52whsoVcA/Ai4uarubi6Z+Pf8qxw0528BlmJuWjE5N1W1vd+8q+k3H0jyoKbNfjNYfwO8Dbi3eb+UAfebYS/yNByeUlVHAs8BXp/kaRNPVm9M2WVah4T5GDofBh4BrACuA97XbTgLW5K9gM8Bb66qbRPP2Xe6NUVu7DtDoKruqaoVwHJ6owi/3XFIakzOTZLHAKfRy9ET6E3BfHuHIS5ISZ4PbKmqC7uMY9iLvGuBgye8X960aYCq6trm5xbgXHpf8tdvn4bZ/NzSXG7OutFvPszTgFTV9c2/iO8FPsp9Uy3MzYAl2ZVeEfHJqjqnabbvDIGpcmPfGS5VdTPwdeCJ9Kb6LWpOTfx7/lUOmvP7Ajdiblo1ITcrm+nPVVV3Amdiv+nCk4EXJLmG3rTxY4EPMuB+M+xF3veAw5rVaHaj9zDiFzqOaUFJsmeSvbcfA8cBm+jlYfsKTCcD/9AcfwH4w2YVp6OBWyZMhVJ7+s3HV4DjkixppkAd17Rpjk16JvVF9PoP9HJzUrOq1qHAYcB38XuvFc3zDWcAl1fV+yecsu90bLrc2He6l2T/JIub492BZ9F7ZvLrwAnNZZP7zfb+dALwtWaEfLqcaZamyc0VE/6nVeg98zWx3/idNgBVdVpVLa+qQ+h9D32tql7GgPvNop1f0p2qujvJG+j9w7YLsLaqLu04rIXmQODc3ncFi4BPVdWXk3wPOCvJKcBPgZc213+R3gpOVwN3AK8cfMijLcl64BhgWZLNwOnAu+kjH1W1Nclf0fuPIoC/rKqZLhiiaUyTm2PSW8K66K1U+xqAqro0yVnAZcDdwOur6p7mc/zem3tPBv4AuKR5hgXgT7HvDIPpcrPKvtO5g4CPJ9mF3sDAWVV1XpLLgE8n+W/A9+kV6TQ//y7J1fQWoToJdpwzzdp0uflakv3praK5Efij5nq/07r3dgbYb9IrFCVJkiRJo2DYp2tKkiRJkvpgkSdJkiRJI8QiT5IkSZJGiEWeJEmSJI0QizxJkiRJGiEWeZIkzUCSL27fl0qSpGHmFgqSJEmSNEIcyZMkaZIkn09yYZJLk6xu2q5Jsqw5/sMkP0hycZK/6zZaSZJ+nSN5kiRNkmS/qtqaZHfge8DvARcCY8CBwLnAk6rqhu3XdhiuJEm/ZlHXAUiSNIT+S5IXNccHA4dNOHcs8NmqugHAAk+SNGws8iRJmiDJMcAzgSdW1R1JvgE8uNOgJEnqg8/kSZL06/YFbmoKvN8Gjp50/mvAiUmWQm9q56ADlCRpRyzyJEn6dV8GFiW5HHg38J2JJ6vqUuBdwDeTXAy8f/AhSpI0PRdekSRJkqQR4kieJEmSJI0QizxJkiRJGiEWeZIkSZI0QizyJEmSJGmEWORJkiRJ0gixyJMkSZKkEWKRJ0mSJEkjxCJPkiRJkkbI/wMoq4NKy/ND8QAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 1080x1080 with 3 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Plot the error metrics\n",
    "_, (ax1, ax2, ax3) = plt.subplots(nrows=3, figsize=(15,15))\n",
    "sns.barplot(y='n_predictors', x='adj_R_sq', data=best, ax=ax1, orient='h')\n",
    "sns.barplot(y='n_predictors', x='bic', data=best, ax=ax2, orient='h')\n",
    "sns.barplot(y='n_predictors', x='aic', data=best, ax=ax3, orient='h')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 336,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>model</th>\n",
       "      <th>n_predictors</th>\n",
       "      <th>MSE</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>&lt;statsmodels.regression.linear_model.Regressio...</td>\n",
       "      <td>19</td>\n",
       "      <td>133352.083840</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>&lt;statsmodels.regression.linear_model.Regressio...</td>\n",
       "      <td>18</td>\n",
       "      <td>133284.236773</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>&lt;statsmodels.regression.linear_model.Regressio...</td>\n",
       "      <td>17</td>\n",
       "      <td>133233.527027</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>&lt;statsmodels.regression.linear_model.Regressio...</td>\n",
       "      <td>16</td>\n",
       "      <td>133792.636968</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>&lt;statsmodels.regression.linear_model.Regressio...</td>\n",
       "      <td>15</td>\n",
       "      <td>133156.193247</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>&lt;statsmodels.regression.linear_model.Regressio...</td>\n",
       "      <td>14</td>\n",
       "      <td>135624.311929</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>&lt;statsmodels.regression.linear_model.Regressio...</td>\n",
       "      <td>13</td>\n",
       "      <td>135391.355291</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>&lt;statsmodels.regression.linear_model.Regressio...</td>\n",
       "      <td>12</td>\n",
       "      <td>135395.399373</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>&lt;statsmodels.regression.linear_model.Regressio...</td>\n",
       "      <td>11</td>\n",
       "      <td>133589.596734</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>&lt;statsmodels.regression.linear_model.Regressio...</td>\n",
       "      <td>10</td>\n",
       "      <td>132451.355454</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>&lt;statsmodels.regression.linear_model.Regressio...</td>\n",
       "      <td>9</td>\n",
       "      <td>133550.524887</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>&lt;statsmodels.regression.linear_model.Regressio...</td>\n",
       "      <td>8</td>\n",
       "      <td>130130.135009</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>&lt;statsmodels.regression.linear_model.Regressio...</td>\n",
       "      <td>7</td>\n",
       "      <td>139175.543438</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>&lt;statsmodels.regression.linear_model.Regressio...</td>\n",
       "      <td>6</td>\n",
       "      <td>145552.220218</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>&lt;statsmodels.regression.linear_model.Regressio...</td>\n",
       "      <td>5</td>\n",
       "      <td>155594.110409</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>&lt;statsmodels.regression.linear_model.Regressio...</td>\n",
       "      <td>4</td>\n",
       "      <td>160845.201250</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>&lt;statsmodels.regression.linear_model.Regressio...</td>\n",
       "      <td>3</td>\n",
       "      <td>182224.634993</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>&lt;statsmodels.regression.linear_model.Regressio...</td>\n",
       "      <td>2</td>\n",
       "      <td>185567.253911</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>&lt;statsmodels.regression.linear_model.Regressio...</td>\n",
       "      <td>1</td>\n",
       "      <td>236588.256708</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                model  n_predictors  \\\n",
       "0   <statsmodels.regression.linear_model.Regressio...            19   \n",
       "1   <statsmodels.regression.linear_model.Regressio...            18   \n",
       "2   <statsmodels.regression.linear_model.Regressio...            17   \n",
       "3   <statsmodels.regression.linear_model.Regressio...            16   \n",
       "4   <statsmodels.regression.linear_model.Regressio...            15   \n",
       "5   <statsmodels.regression.linear_model.Regressio...            14   \n",
       "6   <statsmodels.regression.linear_model.Regressio...            13   \n",
       "7   <statsmodels.regression.linear_model.Regressio...            12   \n",
       "8   <statsmodels.regression.linear_model.Regressio...            11   \n",
       "9   <statsmodels.regression.linear_model.Regressio...            10   \n",
       "10  <statsmodels.regression.linear_model.Regressio...             9   \n",
       "11  <statsmodels.regression.linear_model.Regressio...             8   \n",
       "12  <statsmodels.regression.linear_model.Regressio...             7   \n",
       "13  <statsmodels.regression.linear_model.Regressio...             6   \n",
       "14  <statsmodels.regression.linear_model.Regressio...             5   \n",
       "15  <statsmodels.regression.linear_model.Regressio...             4   \n",
       "16  <statsmodels.regression.linear_model.Regressio...             3   \n",
       "17  <statsmodels.regression.linear_model.Regressio...             2   \n",
       "18  <statsmodels.regression.linear_model.Regressio...             1   \n",
       "\n",
       "              MSE  \n",
       "0   133352.083840  \n",
       "1   133284.236773  \n",
       "2   133233.527027  \n",
       "3   133792.636968  \n",
       "4   133156.193247  \n",
       "5   135624.311929  \n",
       "6   135391.355291  \n",
       "7   135395.399373  \n",
       "8   133589.596734  \n",
       "9   132451.355454  \n",
       "10  133550.524887  \n",
       "11  130130.135009  \n",
       "12  139175.543438  \n",
       "13  145552.220218  \n",
       "14  155594.110409  \n",
       "15  160845.201250  \n",
       "16  182224.634993  \n",
       "17  185567.253911  \n",
       "18  236588.256708  "
      ]
     },
     "execution_count": 336,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Select between models with different numbers of predictors using the validation set approach\n",
    "train, test = model_selection.train_test_split(hitters_dat, test_size=0.2)\n",
    "best = bwd_selection('Salary', train)\n",
    "best['MSE'] = best['model'].map(lambda m: metrics.mean_squared_error(test['Salary'], m.predict(test)))\n",
    "best"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 338,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x12ce60828>"
      ]
     },
     "execution_count": 338,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYMAAAEKCAYAAADw2zkCAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAGexJREFUeJzt3XuYXVWZ5/HvT0KA3EggIUYCBhCYsWmNWM2DCggoyG3AC9jkAeU2pgeVAUenG5p+Gu1+nEdt1LGlRRBCtMWoXIIMXiAqTcZuDV3BEBIJJtABE0KKkIRwG0PgnT/2KnKqqKpzSe29TnF+n+c5T+2z9jp7v3vnVK2stfd6tyICMzPrbK/LHYCZmeXnxsDMzNwYmJmZGwMzM8ONgZmZ4cbAzMxwY2BmZrgxMDMz3BiYmRkwKncAjZo8eXLMmDEjdxhmZiPK4sWLN0TElHr1RkxjMGPGDLq7u3OHYWY2okh6tJF62RoDSXOAU4CeiDikXv1tT27kyau/W35gZmZtZMqFZ1eyn5zXDOYCJ2Tcv5mZJdkag4hYCGzMtX8zM9vOdxOZmVl7NwaSZkvqltT91LNbcodjZvaa1daNQURcGxFdEdG157gJucMxM3vNauvGwMzMqpGtMZA0D/g1cLCkNZIuyBWLmVmnyzbPICJm5dq3mZn1NWJmII+askdlky/MzDqNrxmYmdnI6Rm8+OTjPHH153KHYWYlev2FV+QOoWNl6RlI2kfS3ZJ+J2m5pItzxGFmZoVcPYNtwKcj4j5J44HFkhZExO8yxWNm1tGy9AwiYl1E3JeWnwEeBPbOEYuZmbXBBWRJM4C3AYvyRmJm1rmyNgaSxgG3AJdExKuSD/XNTfR89QGamXWInDOQd6ZoCG6MiFsHqtM3N9GYagM0M+sgue4mEnA98GBEfCVHDGZmtl2unsG7gI8Ax0pakl4nZYrFzKzjZbm1NCJ+BaiZz+w85Q2ekGJmVpLsdxOZmVl+IyYdxQtPrmL5N07NHYaZDZM/+fjtuUOwGrkuIO8q6V5J96d0FE46ZGaWUa6ewR+BYyPi2XSL6a8k/TQifpMpHjOzjpbrAnIAz6a3O6dX5IjFzMzyTjrbSdISoAdYEBFOR2Fmlkm2xiAiXoqImcB04DBJh/SvU5uOYtOzW6sP0sysQ2S/tTQiNgN3AycMsO6VdBSTxo2uPjgzsw6R626iKZImpuXdgOOAFTliMTOzfHcTTQO+LWknigbphxFxR6ZYzMw6Xq67iZZSPMPAzMzawIiZgbzblDd5xqKZWUmyX0A2M7P8RkzP4JkNK/nFdSfnDsPsNes9//XHuUOwjHI/9nInSb+V5IvHZmYZ5R4muhh4MHMMZmYdL2c6iunAycB1uWIwM7NCzp7B/wb+Enh5sAq16Sg2P+N0FGZmZck1A/kUoCciFg9VrzYdxcTxTkdhZlaWXD2DdwGnSloNfB84VtJ3M8ViZtbxsjQGEXFZREyPiBnAmcAvI+LsHLGYmVn+u4nMzKwNqHjoWPvr6uqK7u7u3GGYmY0okhZHRFe9eu4ZmJnZyElHsXHDSr5/w/tyh2E2op153p25Q7A2la0xSHcSPQO8BGxrpBtjZmblyN0zOCYiNmSOwcys4/magZmZZW0MArhL0mJJszPGYWbW8XIOEx0REWsl7QUskLQiIhbWVkiNxGyAyXvumiNGM7OOkK1nEBFr088eYD5w2AB1XslNNH6ccxOZmZUlV6K6sZLG9y4DxwPLcsRiZmb5hommAvMl9cbwvYj4WaZYzMw6XpbGICIeAd7azGf2mHygJ8yYmZXEt5aamZkbAzMzyz8DuWE9G1fy9Rudm8gM4KKzPGRqwytbz0DSREk3S1oh6UFJ78gVi5lZp8vZM/ga8LOIOF3SaGBMxljMzDpalsZA0u7AUcC5ABGxFdiaIxYzM8s3TLQf8CRwg6TfSrouTT7rQ9JsSd2Sup/d4rbCzKwsuRqDUcChwNUR8TbgOeDS/pVq01GMm+B0FGZmZcnVGKwB1kTEovT+ZorGwczMMsjSGETEE8AfJB2cit4D/C5HLGZmlvduoouAG9OdRI8A52WMxcysoykicsfQkK6uruju7s4dhpnZiCJpcSPPmHc6CjMzGznpKNZsWslnbj4hdxhmpbvydGdzt+rlerjNwZKW1Ly2SLokRyxmZpbveQYPATMBJO0ErKV49KWZmWXQDtcM3gM8HBGP5g7EzKxTtUNjcCYwL3cQZmadLGtjkOYYnArcNMj6V3ITPe/cRGZmpcndMzgRuC8i1g+0sjY30RjnJjIzK03uxmAWHiIyM8su55POxgLHAbfmisHMzArZJp1FxHPAno3Wnz7pQE/GMTMrScM9A0ljJb0uLR8k6VRJO5cXmpmZVaWZYaKFwK6S9gbuAj4CzC0jKDMzq1Yzw0SKiOclXQB8IyK+JGlJWYH1t3LzY5x420VV7c5s2Pz0/V/PHYJZXc30DCTpHcBZwI9T2U6t7ljSpyQtl7RM0jxJu7a6LTMz2zHNNAYXA5cB8yNiuaT9gbtb2WkaavrvQFdEHELRqJzZyrbMzGzHNTRMlJLJnRoRp/aWRcQjFH/Qd2Tfu0l6ERgDPL4D2zIzsx3QUM8gIl4CjhiunUbEWuBK4DFgHfB0RNzVv15tOoqtW14Yrt2bmVk/zQwT/VbS7ZI+IumDva9WdippEnAasB/wBmCspLP716tNRzF6wm6t7MrMzBrQzN1EuwJPAcfWlAWtzSB+L/AfEfEkgKRbgXcC321hW2ZmtoMabgwi4rxh3O9jwOGSxgAvUDzTwE+7NzPLpJkZyNMlzZfUk163SJreyk4jYhFwM3Af8ECK49pWtmVmZjtOEdFYRWkB8D3gn1PR2cBZEXFcSbH10dXVFd3d7jyYmTVD0uKI6KpXr5kLyFMi4oaI2JZec4EpLUdoZmZto5kLyE+lO356nz8wi+KCciVWbl7Pybd+uardmZXqxx/8dO4QzPpopmdwPvBh4AmKuQGnA+e2umNJF6dUFMslXdLqdszMbMc10zOYXjsDGUDSu4A/NLtTSYcAHwMOA7YCP5N0R0SsanZbZma245rpGQyUerHVdIz/GVgUEc9HxDbgHqClCWxmZrbj6vYMUqbSdwJTJP2PmlUTaD1r6TLg85L2pJhncBKeZ2Bmlk0jw0SjgXGp7via8i0U1w2aFhEPSvoixUNyngOWAC/1rydpNjAbYNfJk1rZlZmZNaBuYxAR9wD3SJobEY8O144j4nrgegBJ/wtYM0Cda0mT0XZ/0z6NTYgwM7OmNXPN4DpJE3vfSJok6c5Wdyxpr/RzX4rrBd9rdVtmZrZjmrmbaHJEbO59ExGbev+gt+iWdM3gReATtds2M7NqNdMYvCxp34h4DEDSGymylrYkIo5spv6BE6d6oo6ZWUmaaQwuB34l6R5AwJGki7tmZjayNZPC+meSDgUOT0WXRMSGcsIyM7MqNTLP4D9FxIrUEMD2ZxXvm4aN7hvis3OAU4Ce9OB7JO0B/ACYAawGPhwRm+rFsWrTBk65ZU69amZt7Y4PnZ87BLMBNdIz+DRF6oiBssQFfZ981t9c4CrgOzVllwK/iIgvSLo0vf+rhqI1M7NSNDLP4GPp5zHNbjwiFkqa0a/4NODotPxt4F9wY2BmllUjw0RD5gyKiGafgTw1Ital5SeAqU1+3szMhlkjw0T/Jf3ciyJH0S/T+2OAfwOabQxeEREhadDbU2vTUew2ec9Wd2NmZnU0Mkx0HoCku4A39/6vXtI0imsCzVovaVpErEvb6Bli36+ko5h4wAynozAzK0kz6Sj2qRneAVgP7NvCPm8HzknL5wA/amEbZmY2jJqZdPaLlIuo97GXfw78fKgPSJpHcbF4sqQ1wBXAF4AfSroAeJTi6WlmZpZRM5POPinpA8BRqejaiJhf5zOzBln1nkb3a2Zm5WumZwBwH/BMRPxc0hhJ4yPimTIC6+9NkyZ7wo6ZWUkavmYg6WPAzcA1qWhv4LYygjIzs2o10zP4BMUD7BcBRMTKHUxh3ZRVmzZxyk03V7U7s1LdcUZLDwk0K00zdxP9MSK29r6RNIo6KawlzZHUI2lZTdkZkpZLellSV/Mhm5nZcGumMbhH0l8Du0k6DrgJ+D91PjMXOKFf2TKKJ5stbGLfZmZWomYag0uBJ4EHgL8AfgL8zVAfiIiFwMZ+ZQ9GxENNxmlmZiVq6JqBpJ2A70TEWcC3yg3JzMyq1lDPICJeAt4oaXTJ8fQhabakbkndW7dsqXLXZmYdpZm7iR4B/lXS7cBzvYUR8ZVhj2r7tmtyEx3g3ERmZiVppjF4OL1eB4wvJxwzM8uhmXQUnwOQNKF4W3/m8SC5iTYCXwemAD+WtCQi3tdC7GZmNkwU0djoS5oTcAPbewVPA+dHxOKSYuujq6sruru7q9iVmdlrhqTFEVF3Tlczw0RzgI9HxP9NOziConF4S2shmplZu2imMXiptyEAiIhfSdpWQkwDWrVpC++/eciM2WYjzm2nvzd3CGZA8zOQr5F0tKR3S/oG8C+SDpV06EAfGCQdxT9IWiFpqaT5kibu6EGYmdmOaaZn8Nb084p+5W+jyFF07ACfmQtcBXynpmwBcFlEbJP0ReAy4K+aiMPMzIZZM3cTHTPUeknnRMS3+31moaQZ/cruqnn7G8DpG83MMmtmmKiei1v4zPnAT4cxBjMza8FwNgZqqrJ0ObANuHGIOjXpKJ7e0fjMzGwQw9kYNJwuQtK5wCnAWTHERIeIuDYiuiKia/SE3YchRDMzG0izz0AeSkM9A0knAH8JvDsinh/G/ZuZWYsabgwk7QJ8CJhR+7mI+Lu0+K8DfGagdBSXAbsACyQB/CYi/ltr4ZuZ2XBopmfwI4oUFIuBP/ZfGRGfHKBs1gDbub6JfZqZWQWaaQymR0T/R1hW5k2TJni2pplZSZq5gPxvkv60tEjMzCybZnoGRwDnSvoPimEiUaSyriRR3SObXuCMW5ZWsSuzytz0Ied5tPbQTGNwYrMblzSH4hbSnog4JJX9PXAa8DLQA5wbEY83u20zMxs+DQ8TRcSjA73qfGwu0P86wz9ExFsiYiZwB/C3zYVsZmbDbTgnnb1KRCykeLJZbVntk+3H0sRkNTMzK8dwTjprmKTPAx+luFV10AR4kmYDswHGTJ5WTXBmZh2o1J7BYCLi8ojYhyIv0avmJ9TUeyUdxS4TJlUXoJlZh8nSGNS4kWJWs5mZZVR5YyDpwJq3pwErqo7BzMz6KvWawSC5iU6SdDDFraWPAs5LZGaWmYbIIN1Wurq6oru7O3cYZmYjiqTFEdFVr17uawZmZtYGstxa2op1m1/k8/PX5Q7DbNhd/gHfNm35ldozkDRHUo+kZQOs+7SkkDS5zBjMzKy+soeJ5vLqdBRI2gc4Hnis5P2bmVkDKk9HkXyV4tGXI+PqtZnZa1yOeQanAWsj4v6q921mZgOr9AKypDHAX1MMETVS/5XcRLtP2bvEyMzMOlvVPYMDgP2A+yWtBqYD90l6/UCVa3MTjZ2wZ4Vhmpl1lkp7BhHxALBX7/vUIHRFxIYq4zAzs77KvrV0HvBr4GBJayRdUOb+zMysNaX2DCJiVp31Mxrd1rSJO3tyjplZSZyOwszM3BiYmdkIyk20edM2brvJ15lt5Hv/Gc7AYu2n8txEkj4raa2kJel1UpkxmJlZfVlyEwFfjYiZ6fWTkmMwM7M6cuUmMjOzNpLrAvInJS1Nw0iTBqskabakbkndW7Y8VWV8ZmYdJUdjcDVFWoqZwDrgy4NVrE1HMcHpKMzMSlN5YxAR6yPipYh4GfgWcFjVMZiZWV85UljXTiP+APCqp6CZmVm1Sp1nkHITHQ1MlrQGuAI4WtJMigfbrAb+oswYzMysvhy5ia5vZVsTJ43yZB0zs5I4HYWZmY2cdBTPbdjGvTf05A7DbFgcdt5e9SuZVajydBSp/CJJKyQtl/SlMmMwM7P6Kk9HIekY4DTgrRHxJ8CVJcdgZmZ15EhHcSHwhYj4Y6rjsR8zs8xyXEA+CDhS0iJJ90j6swwxmJlZjRwXkEcBewCHA38G/FDS/hER/StKmg3MBnj9ntMrDdLMrJPk6BmsAW6Nwr3Ay8CAEwhqcxNNHOfcRGZmZcnRGNwGHAMg6SBgNOBHmJmZZZQjHcUcYE663XQrcM5AQ0RmZladHOkoAM5udltjJ4/yRB0zs5I4HYWZmbkxMDOzEZSbaOv6F1lz5RO5wzDbYdM/8/rcIZi9SuW5iST9QNKS9FotaUmZMZiZWX1l9wzmAlcB3+ktiIg/712W9GXg6ZJjMDOzOsq+m2ihpBkDrZMk4MPAsWXGYGZm9eW8gHwksD4iVg5WQdJsSd2Sujc++1SFoZmZdZacjcEsYN5QFWrTUezhdBRmZqXJcjeRpFHAB4G359i/mZn1latn8F5gRUSsybR/MzOrUfatpfOAXwMHS1oj6YK06kzqDBGZmVl1suQmiohzm93W6Kk7e7KOmVlJnI7CzMxGTjqKF9c/z/qverKyjXxTPzUzdwhmr5IjHcVMSb9J6Si6JR1WZgxmZlZf2cNEc4ET+pV9CfhcRMwE/ja9NzOzjEptDCJiIbCxfzEwIS3vDjxeZgxmZlZfjmsGlwB3SrqSojF6Z4YYzMysRo67iS4EPhUR+wCfAq4frGKf3ETPba4sQDOzTpOjMTgHuDUt3wQMegG5T26isRMrCc7MrBPlaAweB96dlo8FBs1aamZm1Sj1mkFKR3E0MFnSGuAK4GPA11Kyuv8HzC4zBjMzqy9LOgpayFa689QxnqxjZlYSp6MwMzM3BmZmVv41gznAKUBPRBySyt4KfBMYB6wGzoqILfW2ta1nCz1fX1BitGbV2eui43KHYNZHjnQU1wGXRsSfAvOB/1lyDGZmVkeOdBQHAQvT8gLgQ2XGYGZm9eW4ZrAcOC0tnwHskyEGMzOrkaMxOB/4uKTFwHhg62AVa9NRPPXs05UFaGbWaSpPVBcRK4DjASQdBJw8RN1rgWsBZu57UFQSoJlZB6q8ZyBpr/TzdcDfUNxZZGZmGZX9pLN5wK+BgyWtkXQBMEvS74EVFHmKbigzBjMzqy9XOoqvlblfMzNrTo6H27Rk1F4TPFHHzKwkihgZ12UlPQM8lDuONjEZ2JA7iDbi87Gdz8V2PheFN0bElHqVRkzPAHgoIrpyB9EOJHX7XGzn87Gdz8V2PhfNcaI6MzNzY2BmZiOrMbg2dwBtxOeiL5+P7XwutvO5aMKIuYBsZmblGUk9AzMzK0nbNwaSTpD0kKRVki7NHc9wkrRa0gOSlkjqTmV7SFogaWX6OSmVS9I/pvOwVNKhNds5J9VfKemcmvK3p+2vSp9V9Uc5OElzJPVIWlZTVvrxD7aPnAY5F5+VtDZ9P5ZIOqlm3WXpuB6S9L6a8gF/XyTtJ2lRKv+BpNGpfJf0flVaP6OaIx6cpH0k3S3pd5KWS7o4lXfkd6MyEdG2L2An4GFgf2A0cD/w5txxDePxrQYm9yv7EsXDfwAuBb6Ylk8CfgoIOBxYlMr3AB5JPyel5Ulp3b2prtJnT8x9zP2O9SjgUGBZlcc/2D7a8Fx8FvjMAHXfnH4XdgH2S78jOw31+wL8EDgzLX8TuDAtfxz4Zlo+E/hBG5yLacChaXk88Pt0zB353ajsvOcOoM6X4h3AnTXvLwMuyx3XMB7fal7dGDwETEvL0yjmVwBcA8zqXw+YBVxTU35NKpsGrKgp71OvXV7AjH5/AEs//sH2kfs1wLn4LAM3Bn1+D4A70+/KgL8v6Q/eBmBUKn+lXu9n0/KoVE+5z0W/4/0RcFwnfzeqeLX7MNHewB9q3q9JZa8VAdwlabGk2alsakSsS8tPAFPT8mDnYqjyNQOUt7sqjn+wfbSjT6ahjzk1QxbNnos9gc0Rsa1feZ9tpfVPp/ptIQ1bvQ1YhL8bpWr3xuC17oiIOBQ4EfiEpKNqV0bx35OOvd2riuNv83N8NXAAMBNYB3w5bzjVkjQOuAW4JCK21K7zd2P4tXtjsJa+j8WcnspeEyJibfrZA8wHDgPWS5oGkH72pOqDnYuhyqcPUN7uqjj+wfbRViJifUS8FBEvA9+i+H5A8+fiKWCipFH9yvtsK63fPdXPStLOFA3BjRFxayr2d6NE7d4Y/DtwYLoTYjTFBa7bM8c0LCSNlTS+d5ni6W/LKI6v966HcyjGS0nlH013ThwOPJ26s3cCx0ualIYRjqcYD14HbJF0eLpT4qM122pnVRz/YPtoK71/lJIPUHw/oIj/zHQn0H7AgRQXRAf8fUn/w70bOD19vv957T0XpwO/TPWzSf9e1wMPRsRXalb5u1Gm3Bct6r0o7hT4PcVdEpfnjmcYj2t/irs97geW9x4bxXjtL4CVwM+BPVK5gH9K5+EBoKtmW+cDq9LrvJryLoo/IA8DV9F+FwbnUQx/vEgxbntBFcc/2D7a8Fz8czrWpRR/pKbV1L88HddD1NwlNtjvS/q+3ZvO0U3ALql81/R+VVq/fxuciyMohmeWAkvS66RO/W5U9fIMZDMza/thIjMzq4AbAzMzc2NgZmZuDMzMDDcGZmaGGwOzIUkKSd+teT9K0pOS7kjvp0q6Q9L9KcvmT1L5DEkvaHvG0SWSPprrOMzqGVW/illHew44RNJuEfECRcK02pncfwcsiIivAUh6S826hyNiZnWhmrXOPQOz+n4CnJyWZ1FMEOs1jZqkZxGxtMK4zIaNGwOz+r5Pkf5hV+AtFBk0e/0TcH16GMvlkt5Qs+6AfsNER1YZtFkzPExkVkdELE2plGdR9BJq190paX/gBIrss7+VdEha7WEiGzHcMzBrzO3AlfQdIgIgIjZGxPci4iMUyeKO6l/HrN25MTBrzBzgcxHxQG2hpGMljUnL4ymeP/BYhvjMdoiHicwaEBFrgH8cYNXbgaskbaP4z9V1EfHvaVjpAElLaurOiYiBtmGWnbOWmpmZh4nMzMyNgZmZ4cbAzMxwY2BmZrgxMDMz3BiYmRluDMzMDDcGZmYG/H+WOjNbiYuhMgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Plot the error metric\n",
    "sns.barplot(y='n_predictors', x='MSE', data=best, orient='h')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 339,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>model</th>\n",
       "      <th>n_predictors</th>\n",
       "      <th>MSE</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>&lt;statsmodels.regression.linear_model.Regressio...</td>\n",
       "      <td>19</td>\n",
       "      <td>116599.013674</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>&lt;statsmodels.regression.linear_model.Regressio...</td>\n",
       "      <td>18</td>\n",
       "      <td>116263.927026</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>&lt;statsmodels.regression.linear_model.Regressio...</td>\n",
       "      <td>17</td>\n",
       "      <td>115939.235073</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>&lt;statsmodels.regression.linear_model.Regressio...</td>\n",
       "      <td>16</td>\n",
       "      <td>115616.664737</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>&lt;statsmodels.regression.linear_model.Regressio...</td>\n",
       "      <td>15</td>\n",
       "      <td>115373.619649</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>&lt;statsmodels.regression.linear_model.Regressio...</td>\n",
       "      <td>14</td>\n",
       "      <td>114599.797519</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>&lt;statsmodels.regression.linear_model.Regressio...</td>\n",
       "      <td>13</td>\n",
       "      <td>115082.285922</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>&lt;statsmodels.regression.linear_model.Regressio...</td>\n",
       "      <td>12</td>\n",
       "      <td>112655.258048</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>&lt;statsmodels.regression.linear_model.Regressio...</td>\n",
       "      <td>11</td>\n",
       "      <td>110741.661641</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>&lt;statsmodels.regression.linear_model.Regressio...</td>\n",
       "      <td>10</td>\n",
       "      <td>110090.954229</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>&lt;statsmodels.regression.linear_model.Regressio...</td>\n",
       "      <td>9</td>\n",
       "      <td>112044.513156</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>&lt;statsmodels.regression.linear_model.Regressio...</td>\n",
       "      <td>8</td>\n",
       "      <td>108997.685343</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>&lt;statsmodels.regression.linear_model.Regressio...</td>\n",
       "      <td>7</td>\n",
       "      <td>115301.385440</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>&lt;statsmodels.regression.linear_model.Regressio...</td>\n",
       "      <td>6</td>\n",
       "      <td>118540.196656</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>&lt;statsmodels.regression.linear_model.Regressio...</td>\n",
       "      <td>5</td>\n",
       "      <td>122845.644224</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>&lt;statsmodels.regression.linear_model.Regressio...</td>\n",
       "      <td>4</td>\n",
       "      <td>128180.461938</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>&lt;statsmodels.regression.linear_model.Regressio...</td>\n",
       "      <td>3</td>\n",
       "      <td>136141.808701</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>&lt;statsmodels.regression.linear_model.Regressio...</td>\n",
       "      <td>2</td>\n",
       "      <td>137656.003772</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>&lt;statsmodels.regression.linear_model.Regressio...</td>\n",
       "      <td>1</td>\n",
       "      <td>148995.519722</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                model  n_predictors  \\\n",
       "0   <statsmodels.regression.linear_model.Regressio...            19   \n",
       "1   <statsmodels.regression.linear_model.Regressio...            18   \n",
       "2   <statsmodels.regression.linear_model.Regressio...            17   \n",
       "3   <statsmodels.regression.linear_model.Regressio...            16   \n",
       "4   <statsmodels.regression.linear_model.Regressio...            15   \n",
       "5   <statsmodels.regression.linear_model.Regressio...            14   \n",
       "6   <statsmodels.regression.linear_model.Regressio...            13   \n",
       "7   <statsmodels.regression.linear_model.Regressio...            12   \n",
       "8   <statsmodels.regression.linear_model.Regressio...            11   \n",
       "9   <statsmodels.regression.linear_model.Regressio...            10   \n",
       "10  <statsmodels.regression.linear_model.Regressio...             9   \n",
       "11  <statsmodels.regression.linear_model.Regressio...             8   \n",
       "12  <statsmodels.regression.linear_model.Regressio...             7   \n",
       "13  <statsmodels.regression.linear_model.Regressio...             6   \n",
       "14  <statsmodels.regression.linear_model.Regressio...             5   \n",
       "15  <statsmodels.regression.linear_model.Regressio...             4   \n",
       "16  <statsmodels.regression.linear_model.Regressio...             3   \n",
       "17  <statsmodels.regression.linear_model.Regressio...             2   \n",
       "18  <statsmodels.regression.linear_model.Regressio...             1   \n",
       "\n",
       "              MSE  \n",
       "0   116599.013674  \n",
       "1   116263.927026  \n",
       "2   115939.235073  \n",
       "3   115616.664737  \n",
       "4   115373.619649  \n",
       "5   114599.797519  \n",
       "6   115082.285922  \n",
       "7   112655.258048  \n",
       "8   110741.661641  \n",
       "9   110090.954229  \n",
       "10  112044.513156  \n",
       "11  108997.685343  \n",
       "12  115301.385440  \n",
       "13  118540.196656  \n",
       "14  122845.644224  \n",
       "15  128180.461938  \n",
       "16  136141.808701  \n",
       "17  137656.003772  \n",
       "18  148995.519722  "
      ]
     },
     "execution_count": 339,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Select between models with different numbers of predictors using cross validation\n",
    "results = pd.DataFrame()\n",
    "for train_idx, test_idx in model_selection.KFold(n_splits=10).split(hitters_dat):\n",
    "    train = hitters_dat.iloc[train_idx]\n",
    "    test = hitters_dat.iloc[test_idx]\n",
    "    best = bwd_selection('Salary', train)\n",
    "    mse = best['model'].map(lambda m: metrics.mean_squared_error(test['Salary'], m.predict(test)))\n",
    "    results = results.append(mse, ignore_index=True)\n",
    "    \n",
    "best['MSE'] = results.mean()\n",
    "best"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 340,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x1d903e9e8>"
      ]
     },
     "execution_count": 340,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYMAAAEKCAYAAADw2zkCAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAGxRJREFUeJzt3Xu4XVV57/HvT0KAhIQEEkIkYACFU6UScJcjCgooyu2ANyx5wHKr6QHlAKXHgvSR0vN4HkWsolSEQghUiEK4SPECkVpSWw3dgQABQhOuhlsCBMLtGALv+WOOHdbe7J215tprrrE26/d5nvXsucYca443c2VnZMwxxzsVEZiZWXd7R+4AzMwsP3cGZmbmzsDMzNwZmJkZ7gzMzAx3BmZmhjsDMzPDnYGZmeHOwMzMgFG5A2jUpEmTYvr06bnDMDMbURYtWvRMREyuV2/EdAbTp0+nt7c3dxhmZiOKpEcbqZetM5A0GzgUWBkRu9arv27Vc6y68EfVB2Zm1kEmn3h0W9rJOWcwBzgwY/tmZpZk6wwiYgHwXK72zczsTb6byMzMOrszkDRLUq+k3mdfWpM7HDOzt62O7gwi4uKI6ImInq02H587HDOzt62O7gzMzKw9snUGkuYCvwV2kbRC0gm5YjEz63bZ1hlExMxcbZuZWX8jZgXyqMlbtm3xhZlZt/GcgZmZjZyRwWurnuCpC8/JHYaZ2XrbnHh27hBaJsvIQNJ2kn4t6T5J90o6JUccZmZWyDUyWAecHhF3SBoHLJI0PyLuyxSPmVlXyzIyiIgnI+KOtP0icD+wbY5YzMysAyaQJU0HdgcW5o3EzKx7Ze0MJG0OXAucGhFvST7UPzfRK+0P0MysS+RcgbwxRUdwZURcN1id/rmJxrQ3QDOzLpLrbiIBlwL3R8Tf54jBzMzelGtk8GHgC8D+khan18GZYjEz63pZbi2NiN8AKvOZjSe/8221wMPMrJNkv5vIzMzyGzHpKF5dtZx7f3BY7jDMrMu976Qbc4dQiVwTyJtKul3SXSkdhZMOmZlllGtk8Adg/4h4Kd1i+htJv4iI32WKx8ysq+WaQA7gpfR24/SKHLGYmVneRWcbSVoMrATmR4TTUZiZZZKtM4iI1yNiBjAN2FPSrgPr1KajWP3S2vYHaWbWJbLfWhoRzwO/Bg4cZN/6dBQTNx/d/uDMzLpErruJJkuakLY3Aw4AluaIxczM8t1NNBW4XNJGFB3S1RFxU6ZYzMy6Xq67ie6meIaBmZl1gBGzAnmzye9+2678MzPLLfsEspmZ5TdiRgYvPrOMWy85JHcYZjaCfezPf5Y7hI6V+7GXG0m6U5Inj83MMsp9megU4P7MMZiZdb2c6SimAYcAl+SKwczMCjlHBt8FvgK8MVSF2nQUz7/odBRmZlXJtQL5UGBlRCzaUL3adBQTxjkdhZlZVXKNDD4MHCbpEeDHwP6SfpQpFjOzrpelM4iIMyNiWkRMB44E/iUijs4Ri5mZ5b+byMzMOoCKh451vp6enujt7c0dhpnZiCJpUUT01KvnkYGZmY2cdBTPPbOMH1/2ydxhmFmHOfK4m3OH8LaQrTNIdxK9CLwOrGtkGGNmZtXIPTLYLyKeyRyDmVnX85yBmZll7QwCuEXSIkmzMsZhZtb1cl4m2jsiHpe0NTBf0tKIWFBbIXUSswAmbbVpjhjNzLpCtpFBRDyefq4Ergf2HKTO+txE4zZ3biIzs6rkSlQ3VtK4vm3gE8CSHLGYmVm+y0RTgOsl9cVwVUT8MlMsZmZdL0tnEBEPAbuV+cyWk97jxSVmZhXxraVmZubOwMzM8q9AbtjK55bx/Sudm8ism5x8lC8Nt0u2kYGkCZLmSVoq6X5Je+WKxcys2+UcGZwP/DIiPidpNDAmYyxmZl0tS2cgaQvgI8CxABGxFlibIxYzM8t3mWgHYBVwmaQ7JV2SFp/1I2mWpF5JvS+tcV9hZlaVXJ3BKGAP4MKI2B14GThjYKXadBSbj3c6CjOzquTqDFYAKyJiYXo/j6JzMDOzDLJ0BhHxFPB7Sbukoo8B9+WIxczM8t5NdDJwZbqT6CHguIyxmJl1NUVE7hga0tPTE729vbnDMDMbUSQtauQZ805HYWZmIycdxYrVy/ireQfmDsPMWuy8zzl7fSfI9XCbXSQtrnmtkXRqjljMzCzf8wweAGYASNoIeJzi0ZdmZpZBJ8wZfAx4MCIezR2ImVm36oTO4Ehgbu4gzMy6WdbOIK0xOAy4Zoj963MTveLcRGZmlck9MjgIuCMinh5sZ21uojHOTWRmVpncncFMfInIzCy7nE86GwscAFyXKwYzMytkW3QWES8DWzVaf9rE93hxiplZRRoeGUgaK+kdaXtnSYdJ2ri60MzMrF3KXCZaAGwqaVvgFuALwJwqgjIzs/Yqc5lIEfGKpBOAH0TEuZIWVxXYQMuef4yDbji5Xc2ZWUm/+NT3c4dgw1BmZCBJewFHAT9LZRs127Ck0yTdK2mJpLmSNm32WGZmNjxlOoNTgDOB6yPiXkk7Ar9uptF0qel/AT0RsStFp3JkM8cyM7Pha+gyUUomd1hEHNZXFhEPUfyDPpy2N5P0GjAGeGIYxzIzs2FoaGQQEa8De7eq0Yh4HDgPeAx4EnghIm4ZWK82HcXaNa+2qnkzMxugzGWiOyXdKOkLkj7T92qmUUkTgcOBHYB3AmMlHT2wXm06itHjN2umKTMza0CZu4k2BZ4F9q8pC5pbQfxx4OGIWAUg6TrgQ8CPmjiWmZkNU8OdQUQc18J2HwM+KGkM8CrFMw38tHszs0zKrECeJul6SSvT61pJ05ppNCIWAvOAO4B7UhwXN3MsMzMbPkVEYxWl+cBVwD+loqOBoyLigIpi66enpyd6ez14MDMrQ9KiiOipV6/MBPLkiLgsItal1xxgctMRmplZxygzgfxsuuOn7/kDMykmlNti2fNPc8h1325Xc2bWQj/7zOm5Q7A6yowMjgc+DzxFsTbgc8CxzTYs6ZSUiuJeSac2exwzMxu+MiODabUrkAEkfRj4fdlGJe0KfBHYE1gL/FLSTRGxvOyxzMxs+MqMDAZLSdhsmsI/AhZGxCsRsQ64DWhqAZuZmQ1f3ZFBylT6IWCypL+s2TWe5rOWLgG+LmkrinUGB+N1BmZm2TRymWg0sHmqO66mfA3FvEFpEXG/pG9SPCTnZWAx8PrAepJmAbMANp00sZmmzMysAXU7g4i4DbhN0pyIeLRVDUfEpcClAJL+L7BikDoXkxajbfHu7RpbEGFmZqWVmTO4RNKEvjeSJkq6udmGJW2dfm5PMV9wVbPHMjOz4SlzN9GkiHi+701ErO77B71J16Y5g9eAL9Ue28zM2qtMZ/CGpO0j4jEASe+iyFralIjYp0z990yY4oUrZmYVKdMZnAX8RtJtgIB9SJO7ZmY2spVJYf1LSXsAH0xFp0bEM9WEZWZm7dTIOoP/FhFLU0cAbz6rePt02eiODXx2NnAosDI9+B5JWwI/AaYDjwCfj4jV9eJYvvoZDr12dr1qZpbZTZ89PncI1oRGRganU6SOGCxLXND/yWcDzQEuAK6oKTsDuDUiviHpjPT+rxuK1szMKtHIOoMvpp/7lT14RCyQNH1A8eHAvmn7cuBfcWdgZpZVI5eJNpgzKCLKPgN5SkQ8mbafAqaU/LyZmbVYI5eJ/kf6uTVFjqJ/Se/3A/4DKNsZrBcRIWnI21Nr01FsNmmrZpsxM7M6GrlMdByApFuA9/b9r17SVIo5gbKeljQ1Ip5Mx1i5gbbXp6OYsNN0p6MwM6tImXQU29Vc3gF4Gti+iTZvBI5J28cAP23iGGZm1kJlFp3dmnIR9T328k+BX23oA5LmUkwWT5K0Ajgb+AZwtaQTgEcpnp5mZmYZlVl09mVJnwY+kooujojr63xm5hC7PtZou2ZmVr0yIwOAO4AXI+JXksZIGhcRL1YR2EDvnjjJi1nMzCrS8JyBpC8C84CLUtG2wA1VBGVmZu1VZmTwJYoH2C8EiIhlw0xhXcry1as59Jp57WrOzIbppiOaehCiZVLmbqI/RMTavjeSRlEnhbWk2ZJWSlpSU3aEpHslvSGpp3zIZmbWamU6g9skfRXYTNIBwDXAP9f5zBzgwAFlSyiebLagRNtmZlahMp3BGcAq4B7gL4CfA3+zoQ9ExALguQFl90fEAyXjNDOzCjU0ZyBpI+CKiDgK+MdqQzIzs3ZraGQQEa8D75I0uuJ4+pE0S1KvpN61a9a0s2kzs65S5m6ih4B/l3Qj8HJfYUT8fcujevPYNbmJdnJuIjOzipTpDB5Mr3cA46oJx8zMciiTjuIcAEnji7f1Vx4PkZvoOeD7wGTgZ5IWR8Qnm4jdzMxaRBGNXX1JawIu481RwQvA8RGxqKLY+unp6Yne3t52NGVm9rYhaVFE1F3TVeYy0WzgpIj4t9TA3hSdw/ubC9HMzDpFmc7g9b6OACAifiNpXQUxDWr56jV8at4GM2abWQe74XMfzx2CbUDZFcgXSdpX0kcl/QD4V0l7SNpjsA8MkY7iW5KWSrpb0vWSJgz3D2FmZsNTZmSwW/p59oDy3SlyFO0/yGfmABcAV9SUzQfOjIh1kr4JnAn8dYk4zMysxcrcTbTfhvZLOiYiLh/wmQWSpg8ou6Xm7e8ApzY0M8uszGWiek5p4jPHA79oYQxmZtaEVnYGKlVZOgtYB1y5gTo16SheGG58ZmY2hFZ2Bg2ni5B0LHAocFRsYKFDRFwcET0R0TN6/BYtCNHMzAZT9hnIG9LQyEDSgcBXgI9GxCstbN/MzJrUcGcgaRPgs8D02s9FxN+lzX8f5DODpaM4E9gEmC8J4HcR8T+bC9/MzFqhzMjgpxQpKBYBfxi4MyK+PEjZzEGOc2mJNs3MrA3KdAbTImLgIyzb5t0Tx3sFo5lZRcpMIP+HpD+uLBIzM8umzMhgb+BYSQ9TXCYSRSrrtiSqe2j1qxxx7d3taMrM2uSazzrPZaco0xkcVPbgkmZT3EK6MiJ2TWX/BzgceANYCRwbEU+UPbaZmbVOw5eJIuLRwV51PjYHGDjP8K2IeH9EzABuAr5WLmQzM2u1Vi46e4uIWEDxZLPaston24+lxGI1MzOrRisXnTVM0teBP6O4VXXIBHiSZgGzAMZMmtqe4MzMulClI4OhRMRZEbEdRV6it6xPqKm3Ph3FJuMnti9AM7Muk6UzqHElxapmMzPLqO2dgaT31Lw9HFja7hjMzKy/SucMhshNdLCkXShuLX0UcF4iM7PMtIEM0h2lp6cnent7c4dhZjaiSFoUET316uWeMzAzsw6Q5dbSZjz5/Gt8/fonc4dhZi1y1qd9u3gnqXRkIGm2pJWSlgyy73RJIWlSlTGYmVl9VV8mmsNb01EgaTvgE8BjFbdvZmYNaHs6iuQ7FI++HBmz12Zmb3M51hkcDjweEXe1u20zMxtcWyeQJY0BvkpxiaiR+utzE20xedsKIzMz627tHhnsBOwA3CXpEWAacIekbQarXJubaOz4rdoYpplZd2nryCAi7gG27nufOoSeiHimnXGYmVl/Vd9aOhf4LbCLpBWSTqiyPTMza06lI4OImFln//RGjzV1wsZepGJmVhGnozAzM3cGZmY2gnITPb96HTdc43lms7ebTx3hjDSdoO25iST9raTHJS1Or4OrjMHMzOrLkpsI+E5EzEivn1ccg5mZ1ZErN5GZmXWQXBPIX5Z0d7qMNHGoSpJmSeqV1LtmzbPtjM/MrKvk6AwupEhLMQN4Evj2UBVr01GMdzoKM7PKtL0ziIinI+L1iHgD+Edgz3bHYGZm/eVIYV27jPjTwFuegmZmZu1V6TqDlJtoX2CSpBXA2cC+kmZQPNjmEeAvqozBzMzqy5Gb6NJmjjVh4igvTjEzq4jTUZiZ2chJR/HyM+u4/bKVucMwsxbb87it61eyyrU9HUUqP1nSUkn3Sjq3yhjMzKy+tqejkLQfcDiwW0S8Dziv4hjMzKyOHOkoTgS+ERF/SHV87cfMLLMcE8g7A/tIWijpNkl/kiEGMzOrkWMCeRSwJfBB4E+AqyXtGBExsKKkWcAsgG22mtbWIM3MukmOkcEK4Loo3A68AQy6gKA2N9GEzZ2byMysKjk6gxuA/QAk7QyMBvwIMzOzjHKko5gNzE63m64FjhnsEpGZmbVPjnQUAEeXPdbYSaO8OMXMrCJOR2FmZu4MzMxsBOUmWvv0a6w476ncYZhZBab91Ta5Q+h6bc9NJOknkhan1yOSFlcZg5mZ1Vf1yGAOcAFwRV9BRPxp37akbwMvVByDmZnVUfXdRAskTR9snyQBnwf2rzIGMzOrL+cE8j7A0xGxbKgKkmZJ6pXU+9xLz7YxNDOz7pKzM5gJzN1Qhdp0FFs6HYWZWWWy3E0kaRTwGeADOdo3M7P+co0MPg4sjYgVmdo3M7MaVd9aOhf4LbCLpBWSTki7jqTOJSIzM2ufLLmJIuLYsscaPWVjL0wxM6uI01GYmdnISUfx2tOv8PR3vFjZ7O1qymkzcofQ1XKko5gh6XcpHUWvpD2rjMHMzOqr+jLRHODAAWXnAudExAzga+m9mZllVGlnEBELgOcGFgPj0/YWwBNVxmBmZvXlmDM4FbhZ0nkUndGHMsRgZmY1ctxNdCJwWkRsB5wGXDpUxX65iV5+vm0Bmpl1mxydwTHAdWn7GmDICeR+uYnGTmhLcGZm3ShHZ/AE8NG0vT8wZNZSMzNrj0rnDFI6in2BSZJWAGcDXwTOT8nq/h8wq8oYzMysvizpKGgiW+nGU8Z4UYqZWUWcjsLMzNwZmJlZ9XMGs4FDgZURsWsq2w34IbA58AhwVESsqXesdSvXsPL78yuM1sxy2vrkA3KH0NVypKO4BDgjIv4YuB743xXHYGZmdeRIR7EzsCBtzwc+W2UMZmZWX445g3uBw9P2EcB2GWIwM7MaOTqD44GTJC0CxgFrh6pYm47i2ZdeaFuAZmbdpu2J6iJiKfAJAEk7A4dsoO7FwMUAM7bfOdoSoJlZF2r7yEDS1unnO4C/obizyMzMMqr6SWdzgd8Cu0haIekEYKak/wKWUuQpuqzKGMzMrL5c6SjOr7JdMzMrJ8fDbZoyauvxXpRiZlYRRYyMeVlJLwIP5I6jQZOAZ3IH0YCREic41qo41tbrtDjfFRGT61UaMSMD4IGI6MkdRCMk9Y6EWEdKnOBYq+JYW2+kxDmQE9WZmZk7AzMzG1mdwcW5AyhhpMQ6UuIEx1oVx9p6IyXOfkbMBLKZmVVnJI0MzMysIh3fGUg6UNIDkpZLOqON7W4n6deS7pN0r6RTUvmWkuZLWpZ+TkzlkvS9FOfdkvaoOdYxqf4yScfUlH9A0j3pM9+TpGHEu5GkOyXdlN7vIGlhOvZPJI1O5Zuk98vT/uk1xzgzlT8g6ZM15S37DiRNkDRP0lJJ90vaq4PP6Wnpu18iaa6kTTvlvEqaLWmlpCU1ZZWfx6HaaCLWb6W/A3dLul7ShGbPVzPfSZlYa/adLikkTeqE89pyEdGxL2Aj4EFgR2A0cBfw3ja1PRXYI22PA/4LeC9wLsXDeQDOAL6Ztg8GfgEI+CCwMJVvCTyUfk5M2xPTvttTXaXPHjSMeP8SuAq4Kb2/Gjgybf8QODFtnwT8MG0fCfwkbb83nd9NgB3Sed+o1d8BcDnw52l7NDChE88psC3wMLBZzfk8tlPOK/ARYA9gSU1Z5edxqDaaiPUTwKi0/c2aWEufr7LfSdlYU/l2wM3Ao8CkTjivrX61tbEmfiH3Am6ueX8mcGamWH4KHECx8G1qKptKsf4B4CJgZk39B9L+mcBFNeUXpbKpwNKa8n71SsY2DbgV2B+4Kf1Fe6bml239eUx/ofdK26NSPQ08t331WvkdAFtQ/AOrAeWdeE63BX6ffqFHpfP6yU46r8B0+v8DW/l5HKqNsrEO2Pdp4MrBzkO989XM3/VmYgXmAbtRPKq3rzPIfl5b+er0y0R9v5B9VqSytkrDy92BhcCUiHgy7XoKmJK2h4p1Q+UrBilvxneBrwBvpPdbAc9HxLpBjr0+nrT/hVS/bPzN2AFYBVym4pLWJZLG0oHnNCIeB84DHgOepDhPi+jM89qnHedxqDaG43iK/yU3E2szf9dLkXQ48HhE3DVgV6ef11I6vTPITtLmwLXAqRGxpnZfFN141tuxJB0KrIyIRTnjaNAoiiH4hRGxO/AyxZB4vU44pwDpmu3hFB3YO4GxvPV53h2rHeexFW1IOgtYB1zZkqBaTNIY4KvA19rVZq7fgU7vDB6n/2Mxp6WytpC0MUVHcGVEXJeKn5Y0Ne2fCqysE+uGyqcNUl7Wh4HDJD0C/JjiUtH5wARJfelGao+9Pp60fwvg2Sbib8YKYEVELEzv51F0Dp12TgE+DjwcEasi4jXgOopz3YnntU87zuNQbZQm6VjgUOCo9A9gM7E+S/nvpIydKP5DcFf6HZsG3CFpmyZibct5bVq7r0uVeVH8T/Ihii+jb9LofW1qW8AVwHcHlH+L/hM956btQ+g/mXR7Kt+S4jr5xPR6GNgy7Rs4mXTwMGPelzcnkK+h/6TaSWn7S/SfVLs6bb+P/hN3D1FM2rX0OwD+Ddglbf9tOp8dd06B/07xvO4x6ViXAyd30nnlrXMGlZ/HodpoItYDgfuAyQPqlT5fZb+TsrEO2PcIb84ZZD+vrXy1tbEmfykPpriT50HgrDa2uzfFUO1uYHF6HUxxzfFWYBnwq5ovWcA/pDjvAXpqjnU8sDy9jqsp7wGWpM9cQAOTW3Vi3pc3O4Md01+85emXZZNUvml6vzzt37Hm82elWB6g5i6cVn4HwAygN53XG9IvS0eeU+AciocwLQH+ieIfqI44r8BcirmM1yhGXCe04zwO1UYTsS6nuK7e97v1w2bPVzPfSZlYB+x/hDc7g6zntdUvr0A2M7OOnzMwM7M2cGdgZmbuDMzMzJ2BmZnhzsDMzHBnYLZBKUvlj2rej5K0Sm9mh50i6SZJd6nIcPvzVD5d0quSFte8/izXn8OsnlH1q5h1tZeBXSVtFhGvUiQrrF0p/HfA/Ig4H0DS+2v2PRgRM9oXqlnzPDIwq+/nFKtNocg0Obdm31Rqko9FxN1tjMusZdwZmNX3Y+BISZsC76fIXtvnH4BLVTwI6SxJ76zZt9OAy0T7tDNoszJ8mcisjoi4O6Uxn0kxSqjdd7OkHSly7RwE3Clp17Tbl4lsxPDIwKwxN1I832DuwB0R8VxEXBURXwD+k+JpWWYjijsDs8bMBs6JiHtqCyXtn3LeI2kcRcrjxzLEZzYsvkxk1oCIWAF8b5BdHwAukLSO4j9Xl0TEf6bLSjtJWlxTd3ZEDHYMs+yctdTMzHyZyMzM3BmYmRnuDMzMDHcGZmaGOwMzM8OdgZmZ4c7AzMxwZ2BmZsD/B5yiKfpDaBtMAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Plot the error metric\n",
    "sns.barplot(y='n_predictors', x='MSE', data=best, orient='h')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 382,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x1e3e4f8d0>"
      ]
     },
     "execution_count": 382,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZsAAAEKCAYAAADEovgeAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3XuYXFd55/vvu+vWF92ltixLli3AhpjggNUQExgwVwMh4wAmmOHiEB48BDiZOfMQMJNDgCHnPJAzIeFyAjEnBBtyTLgMBwebeEBA9ByDAQl8kQ02AuNYRtbNsm5d3VV77/f8sdcu7S51qyV1VXdV9+/zPP101apd1Xt3dffba613vcvcHRERkW6K5vsERERk4VOwERGRrlOwERGRrlOwERGRrlOwERGRrlOwERGRrlOwERGRrlOwERGRrlOwERGRrivP9wn0ijVr1vj5558/36chItJXtm/fvt/dR2Y6TsEmOP/889m2bdt8n4aISF8xswdP5TgNo4mISNcp2IiISNd1LdiY2WfMbK+Z7Si0vdrM7jGz1MxGC+2vM7M7Ch+pmT01PLbZzO42s51m9jEzs9C+ysy+aWY/D59XhnYLx+00s7vM7JJuXaOIiJyabvZsPgu8pK1tB/BKYGux0d3/0d2f6u5PBd4APODud4SHPwm8BbggfOSveS2wxd0vALaE+wAvLRx7TXi+iIjMo64FG3ffCjza1vZTd79vhqe+FvgCgJmtA5a5++2ebbxzA/D74bgrgOvD7evb2m/wzO3AivA6IiIyT3oxG+01ZAEDYD2wq/DYrtAGsNbdd4fbjwBrC895aIrn7KaNmV1D1vth48aNnTh3EZGe12wmHGs2mWg6zdRJUmegUmJkSY0osq58zZ4KNmb228CYu++Y8eACd3czO+0tR939OuA6gNHRUW1ZKiILQrOZcKTRJDJIC3/ZIoMogvGmc6ges//IBH/65bvYdbDOhpWDfPoNm3ni2cu6EnB6KtgAVwE3Fu4/DGwo3N8Q2gD2mNk6d98dhsn2Fp5z7jTPERHpe2nqHJlokKR+QkCplODIREo5MuLCA/l9T8GBXY/Wee/XdrDrYB2AXQfrvOVz2/nq257FyNJax8+5Z4KNmUXAHwD/Lm8LgeSwmV0K/AB4I/Dx8PBNwNXAh8LnrxXa32FmXwB+GzhUGG4TEekbU/VQIoN6M8VhyoBSbzipZwHJC0Eov5+4Y8BQtdQKNLldB+s04qQr19K1YGNmNwKXAWvMbBfwPrKEgY8DI8DNZnaHu18envIc4CF3/2XbS72NLLNtEPhG+IAsyHzRzN4MPEgWqABuAV4G7ATGgDd1/OJERDoon0MpBpQogqNT9FA8PR54pgoo+bHmWQ8ml9+PzIgMxhoJG1YOTgo4G1YOUi2XunKN5q6pCsjmbFSuRkS6Ke+pAK3eSrmUzaHkQaI43JU6mDE5oLi3gojRFlAAMwufJz8vv7/l3t287LfO6dicjZltd/fRmY7rmWE0EZGFIp9TcfdWL6RcynoqMHn4yxtZwPC23kriU/dQypG1jp0qoHzrnt28+Cnrpp2z2bxpDbfc+WuuuGQ9y2plvnDNpaSpU1tM2WgiIv2mOFkPUIpgrJFSah/+akw9/JXPobR6K4Xhrql6KKXIGG/G1CqlKQPK5k1r+J937+b3nraeaskmJQ9US8bAihqrnryOw/WEcmSsXVKjUunO0FmRgo2IyGkozq8UJ+sh+2M/1gg9k/b5lOLwV6G3ks+htPdWttw7dQ/FceIUkomEVcOVEwLKuStrrBpax9HxLJiMDFepVtv+1A938jtyahRsRESmEccpRyYa2DTzK8XJeggT9ImDnfrwVz6H0j5nc7IeykA5ot5IORICylltvZMVQ934bsyOgo2ISFCcwM/nWE46v1LorUAWYEphzqN9+Gu8mTBUywJC8TWLcyj58cXhrqPTBJT56J3MhoKNiCxK7XMt+WJImDwcdrL5lWJvBQg9ICdJU4Zr5cnDXzFMNBOWDVYwaPVW1rfNoUwKKn0WUE5GwUZEFo1GI+ZoM55yriVfDAnTD4e1z68UJ+tbr9N0molzuB6zZkml9ZoD5Yhm7NQbKWmoRbYmz/5aQEFlOgo2IrJgFYfFij2XKedaCg3TDYe1z6+0T9YDLB8oEVdKTMQph+oJlVLEWUtqlMuLe69KBRsRWTDS1DnaaBAnPmldC0C9mHo8xVxLnmoMWYA5NNZkxXCZSik66fzKlJP1Q91PJe43CjYi0teK9cMaiU+azJ/UeykEmKnmWvLFkPnj1UrE0fGYlcNVhirRyedXFsEw2Gwp2IhI38nnXoq9lzR0YqbLFCsGmKnmWoqpxgaMLKkw0XQasTPeSFsr7FcNd2eF/UKnYCMiPS9f7+K0zb0Uei9J6tn8SnhOe+/lJw8eYPOm1dnzpphrmWox5Ioh/YnsFH0nRaQn5QEmiuBYY+a5l1Jkkybzi+taAM4fWcb2Bw7wrAtGgKnnWlZorqVrFGxEpGdMFWBOyBybZu7FydbAlErZMFf7upb1K2qsGlo9KbisGlZwmSsKNiIyr/IAUyrMv0zas+Ukcy/F3kszyfZoGaiUWFIrTbmuZd3ywa5VNZaTU7ARkTmXr94381ZJmGMNn3Jy/6RzL4Xey0AZhsP6liPjWt/SaxRsRGTONBoxY3FMvZmV4G/Ex0vC5Cv2oT1zLAs+ZtmDxbkX9V76h4KNiHRVvg4mzyJLU1ol+IslYUqFAFEMMI3EcXeGqhHlyKace5mL/VhkdhRsRKTj8mGyyLyVppxnkSXpiQGmuGK/PcAMlCPSFCbilHqaUilFrF06oOGxPqNgIyId02wmjMXN1k6VzdhPmOifKsBUStGkFfsKMAuPgo2IzEoxm2w8rLhvDZO1bVnsnqUo5yX4iwFm0or9NFuxr/mXhUPBRkTOSN6Laa2HCRuLFedhisUt8yyyZkKrBP9UJWHOWT6gALMAqU8qIqel2Uw4Mj7OvrEJjo6nJAkkCTRTJ0mdUmSUzChFxrfv3U2pBKXS8SyypbUSq4aqDFZKHKonJA5nLx1g4+ph1i5ToFmo1LMRkRm1D5XlczHF9TD5xmLFnSqLxS2VRba4KdiIyLTiOGWs2eBo+1BZmIsprofJNxYr7lQ5VXHLalV/dhYjvesicoI4TqnHDY5MpJNKxySe1R/L52KK62GKG4slbTtVrl+uTLLFrmvvvpl9xsz2mtmOQturzeweM0vNbLTt+IvN7Pvh8bvNbCC0vzbcv8vM/sXM1oT2VWb2TTP7efi8MrSbmX3MzHaG51zSrWsUWWiazYRD9XEerU9wqJ7Nx8Spk3j2EbXNxTSSrBeTL7Z8UdhYbCJOOWtJjfNWD3POikEFGulqgsBngZe0te0AXglsLTaaWRn4PPBWd38ycBnQDO0fBZ7n7hcDdwHvCE+7Ftji7hcAW8J9gJcCF4SPa4BPdvSqRBag4qT/kfGURpyt7k+8MOFvWZCplo9vNLakVmJJtUwzdo6MJ5gZ65cPsn7lkOZjZJKuDaO5+1YzO7+t7adAq8ZRwYuBu9z9znDcgXBchaxa0rCZHQCWATvDc64gC0oA1wPfBd4d2m9wdwduN7MVZrbO3Xd38PJEFoRGI2Y8jSdN+uebkOWLL6cbKtNcjJyOXvnpuBBwM7sVGAG+4O5/6e5NM/tj4G7gGPBz4O3hOWsLAeQRYG24vR54qPDau0LbCcHGzK4h6/2wcePGzl6RSA9rNhPGk2ZrTqY46Z9vQpav7m8ktErHrF9RY1UYKitHxjnLBtSDkVPSKwOpZeDZwOvC51eY2QtCz+aPgacB55ANo72n/cmhF+Pt7TNx9+vcfdTdR0dGRmZz/iJ9odGIORyGyw6HOZl8fUwUhsoO17Oimfnq/sFKpKEymbVeCTa7gK3uvt/dx4BbgEuApwK4+y9CQPki8DvhOXvMbB1A+Lw3tD8MnFt47Q2hTWTRyudk9tcbHGmb+G+f9K9WIh4ba7JquMKKwSqN2BlrJJSjiHNXDmnCX85Ir/zE3Ao8xcyGQlLAc4F7yYLERWaWdzteBPw03L4JuDrcvhr4WqH9jSEr7VLgkOZrZLGK45SjU/VkChP/7ZP+eZCpN1Im4pRqKWLDyiHWqoyMzELX5mzM7EayCfw1ZrYLeB/wKPBxsnmZm83sDne/3N0PmtlHgB+RDYfd4u43h9f5ALDVzJrAg8Afhi/xIeCLZvbm0P4Hof0W4GVkiQRjwJu6dY0ivSpfjDken7jaP18jk9cqm27S/yxN+ksHmftpT3UsSKOjo75t27b5Pg2RWUlTZ6zR4PBEQpot+j9htf+We3fz4qes48DRJr/ad5hnXTBCvZESp64yMnLazGy7u4/OdFyvDKOJyCykqXN4fIL9x8Z5rJ5kczLJ5In/nzx4gFKJ1nDZOStqXHJeVqsM4JxlA5r0l65RsBHpc41GzJ4jdQ7XkxMXYxYm/vOqy/lK/6MKMjKHNCAr0qfy+mWH6+mUizEP15usWVqZVHm5WHVZa2RkLinYiPSZ4ryMp9mcTL7dcnExZp7CfE5YiJlP/J+9pEatpl99mVv6iRPpI41GzGMTzeNbL7tP2m7ZYNJWy55CvZGSps5ApcTIkprSl2VeKNiI9IFmM6GeNDlcz1LM8q2X8wWZ482YKIoYayQMVErUSmUasZOmTq1cYmS5gozMLyUIiPSwNHWOjk9k1ZjraSvDrLgg03HiFJpxysqhKiUzxuOUCOOc5YNajCk9QcFGpEflWWaHQipzMz2eYXa43iSKfNKq/8FKibHG8Qyzc1aqrIz0Dg2jifSYPAHgsXqCt638b5+XWbus2kpjLkfGuqU1rfqXnqR/e0R6SLE30755WTZkBmONhFo5ahXJTFJnoFxi3fJBBRrpWQo2Ij0gTZ1j4xPsPdYgDhWZ8yAz3kxaK/+/c+8jrNC8jPQh/RskMs/ydOZmWP2PZXXMUneSNMWBiWbSWi9Tb2jITPqPflJF5kmaOvVGg4P1bFI/780AjDcTKuWIZpLVNhuslI+vlymXGFmqVGbpLwo2IvOg2Uw4ON5o9WbMjicA5L2ZNE1ZMVSlGWf7ypQjY51KzEif0pyNyBxKU+fI+AR7jk7QiH1SOnOeANBMsoyzwUqZRpxOSgBQoJF+pWAjMkfiOJ2caRbK/2eJzT4pAWCwUmJCCQCygCjYiMyBLKV5PMs0S07szYw1EpYPlnlRSAAAWDtc08JMWTD0UyzSZePjcZbSnB5fNzNVb6ZkRpw6lShi7dIBBgY0pSoLh36aRbokjlPG4waHQk2zcimrzny43mR4oMR4M53UmylHxtrhmoKMLEj6qRbpgomJmAPjDTydXNNsvBlTrURMNBOWDVZIwn405cg4a0lNCQCyYCnYiHTY+HjM/nqjVdcsTwIYbybEKRiwfLBKnLrWzciioTkbkQ7JS84cqDcm1TVrT2keCJlmBqxdUlOmmSwKCjYiHdBsJuw5UuexenJ82MyMnzx44ISU5qSQBKDtmWWx0E+6yCxNTMTsG5s8bHa43mTFcJnzR5bxq32HeaGSAGSRU89G5Azlu2juHztx2Ky438wl560mCUkAI8NVBRpZlBRsRM5APmx2qJ5M2g4gHza77f69LNN+MyItXQs2ZvYZM9trZjsKba82s3vMLDWz0bbjLzaz74fH7zazgdBeNbPrzOx+M/uZmb0qtNfM7J/MbKeZ/cDMzi+81ntC+31mdnm3rlEWp2YzYc/RCeLCVs2OE6fJpGGzRkgCOGtYSQAi3ezZfBZ4SVvbDuCVwNZio5mVgc8Db3X3JwOXAc3w8J8Be939QuAi4F9D+5uBg+7+BOCvgQ+H17oIuAp4cvj6f2tmXVm8kKbOviMTPHxwjH1HJkhT78aXkR7SbCbsPTrRGjaLzChF2VxNM3EGKxFPKwybrRnSsJkIdDFBwN23Fnsboe2nAGYn/If3YuAud78zHHeg8NgfAU8K7SmwP7RfAbw/3P4y8AnLXvgK4AvuPgE8YGY7gWcA3+/EdeXS1LlvzxHecsM2dh2ss2HlIJ9+4yhPXLtU/8EuUHkiQFLYd2bLvbt53kVnA7B0oEySMml+RsNmIplembO5EHAzu9XMfmxm7wIwsxXh8Q+G9i+Z2drQth54CMDdY+AQsLrYHuwKbScws2vMbJuZbdu3b99pnfCBY41WoAHYdbDOW27YxoFjjdN6HekP4+NZoMmLaObDZnla81C1TBoCTZ7WrEAjclyvBJsy8GzgdeHzK8zsBaF9A/A9d7+ErHfy3zv1Rd39OncfdffRkZGR03puI05agSa362CdRpx06vSkB+QZZ/lCzazszORhsxcWKjWfvUSVmkWm0iu/EbuAre6+393HgFuAS4ADwBjwP8JxXwrtAA8D50Jrzmd5OL7VHmwIbR1VLZfYsHJwUtuGlYNUy6pttVBM2n+mkAhwuJ5NJy4dKDMQFmnmw2ZapCkytV4JNrcCTzGzoRA4ngvc6+4O/DNZwgDAC4B7w+2bgKvD7SuBb4fjbwKuCtlqm4ALgB92+oRXD1f59BtHWwEnn7NZPVzt9JeSeRDHKY+E/WfyjLPD9SapO2bGsfGEOKE1f6P5GZGT69pvh5ndSBYk1pjZLuB9wKPAx4ER4GYzu8PdL3f3g2b2EeBHgAO3uPvN4aXeDXzOzP4G2Ae8KbT/fWjfGV73KgB3v8fMvkgWlGLg7e7e8bGtKDKeuHYpX33bs2jECdVyidXDVSUHLADFjDMMIjPGm8kJ1ZoTVWsWOWWWdQZkdHTUt23bNt+nIfOs0cg2Osu3bIYs4+xZF55FOYKBSpkk9VagWbt0QPMzsqiZ2XZ3H53pOP2WiAQTE2FHzSkyzm67f28r0KSFigAKNCKnRoPMs5SmzoFjDQ2l9bl8Dc3xjDPjWL2JmbUyzhpxmvVmltSUCCBymvQbMwta2LkwTEzErWKaeY9mvJkQRRGNZsqywXIrEWDNkDLORM6ExgBmQQs7+19rsWZbxlm+0dnyoQqph9RmBRqRM6bfnFnQws7+NjERtxZrlkvGobEmlbIp40ykC9SzmQUt7Oxf+dBZvoYmT212dwarZeKEVm9n7dIBBRqRWVKwmQUt7OxPxaGzfFfNbKgMlg1WicxI3alExtlKbRbpCA2jzYIWdvaf8fHJQ2ffuidbQ5OmKaWo1No6QGtoRDpLv0mzFEXGyNIa61cOMbK0pkDTw/JAUxw6K+6qGZnh7gyUI87RGhqRjtJvkywKxUBTHDpbUot4wUXZGhp3qJSNs5ZqV02RTlOwmSXt1tn7WkNnIdB8+97drWSAUpT9CjhgBiPDCjQi3aA5m1nQos7el6c3N0MK87fu2d0aOrv8Kee0ys8MlksaBhXpIvVsZkGLOntbo3E8vTmv3Lx50xq2P7Cf54ehMwNWDZdZu1w9GpFuUrCZBS3q7F159ebi0JlDa47GPWuPIhiqKlVdpNsUbGZBizp7UyvQhFpn375396TKzWZgQLUUsXbpoHo0InNAwWYWtKiz9zSbyaRAo6Ezkd6gBIFZ0KLO3hLHaWuHzbyoZqVsraGzVENnIvNGPRtZEOI45ZEj45OqN1fKpvRmkR5x0mBjZq8v3H5W22Pv6NZJ9Ys89fkVf3sbz/rwd3jF397GfXuOaK3NHEtTzwJNcmKgGayWMaxVgka1zkTmx0y/df+lcPvjbY/9UYfPpe8o9bk37DsyUdhhEwUakR4005yNTXN7qvuLjlKf59/4eMx4nLR22Dxcj6lVSkRmkwLN2iU1BRqReTTTb59Pc3uq+4uOUp/nV16GJg809UaMmXFsPMGw1g6bZw1XtR+NyDybKdg8yczuMrO7C7fz+0+cg/PraUp9nj/tFZzzQNOMnWWD2VbOpRBoqlUlXYrMt5l+C39jTs6iTyn1eX40GpMrON9696951oVn0WimkwLNyJACjUivOGnPxt0fLH4AR4FLgDXh/qIXRcbq4SrVcolGnHDgWEPZaF0Uxyn7jh0PNMXqAEtDoClHxprBKrWaAo1Irzjpb6OZfR241t13mNk64MfANuDxZnadu//NXJxkL1Pl57mTr6VJ2gJNXh3AQ6BZPVhlYECBRqSXzDRns8ndd4TbbwK+6e6/B/w2M6Q+m9lnzGyvme0otL3azO4xs9TMRtuOv9jMvh8ev9vMBtoev6nttVaZ2TfN7Ofh88rQbmb2MTPbGeaXLpnxuzALSn+eG+1raRRoRPrLTMGmWbj9AuAWAHc/AqQzPPezwEva2nYArwS2FhvNrAx8Hniruz8ZuKz4tc3slWRDeEXXAlvc/QJgS7gP8FLggvBxDfDJGc5zVpT+PDeKa2na653lgWb5YEmBRqRHzRRsHjKz/8XMXkE2V/MvAGY2CFRO9kR33wo82tb2U3e/b4rDXwzc5e53huMOuHsSvtYSssWlf9H2nCuA68Pt64HfL7Tf4JnbgRVhCLArlP7cfRMTk9fSHJuItVWASJ+ZKdi8GXgy8IfAa9z9sdB+KfAPHTyPCwE3s1vN7Mdm9q7CYx8E/goYa3vOWnffHW4/AqwNt9cDDxWO2xXaukLpz92Vb4BWXEszWC2x/0iTOEmJE8eBNUPaZVOkl510zMHd9wJvnaL9O8B3OnwezwaeThZUtpjZduAA8Hh3/1/N7PyTnKeb2WmngJnZNWRDbWzcuPEMTlvpz91U3ACtVo44Vm+21tKcvXyA1J2KFm2K9IWZstFuOtnj7v7vO3Qeu4Ct7r4/fN1byIbtjgKjZvYrsnM9y8y+6+6XAXvMbJ277w7DZHvDaz0MnFt47Q2hbarzvw64DmB0dFT5yj0kjtPWvjRRpLU0Iv1upt/SZ5INSd0I/IDu1UO7FXiXmQ0BDeC5wF+7+82ECf7Qs/l6CDQANwFXAx8Kn79WaH+HmX2BLGvuUGG4reOU+twdewsJAVsKa2meH/alyTPPtJZGpD/MNGdzNvBfgd8EPgq8CNjv7v/q7v96siea2Y3A94EnmtkuM3uzmb3CzHaRBbGbzexWAHc/CHwE+BFwB/DjEGhO5kPAi8zs58ALw33IMuZ+CewEPg28bYbXmRWlPnfexERMM01b2wVccv7qEzLPlinzTKSvzDRnk5BloP2LmdWA1wLfNbMPuPsnZnjua6d56KvTHP95svTn6V7vV2RBL79/gCwdu/04B95+snPrJKU+d1YxIeDQWLYvTdnh2ReuJU5SUofIYFiZZyJ9ZcZ/DUOQ+V2yQHM+8DGmCRiLUZ76XAw4Sn0+M81m0koIqJZsyn1pqmFfGg1RivSXmXbqvIFsKOwS4APu/nR3/6C7Tznhvhgp9bkz0tTZezSbp4nMuP62B6iWoxM2QNO+NCL9ybJRp2keNEuBY+Fu8UAjG7Fa1sVzm1Ojo6O+bdu2M3puHKfsPTpBM0mplCLO0h/E07bn0DjjcUJkxxMCvn7HLq4c3UgpMqrliJEhJQSI9Boz2+7uozMdN9Ocjf5iziBNnZ/vO6pstFkoVgg4NHY8IeB1z9w0KSFAgUakfymYzJKy0WYnTwiIzChFUCkbtbK1EgKaiZO4KyFApM/pX8VZUjbamWst3EwdAw7XY2qVEpHZpISAtUtUikak36lnM0sqxHnm8oWbkRn/z+2/wsw4Np5gWGvhpkrRiCwM6tnMUp6N9tffvI9XbT6X1cNVzlpaY+XgSYtiL3rFhZtb7t3Ny5+6ga9se4grRzfSTNJWQoBK0YgsDPpNnqUoMi4YWcJ/euGF/MfPbVeSwCloNpNJCzeVECCy8GkYrQMO1putQANKEjiZOE7Zc3SCZupEpoQAkcVCwaYDlCRwaopbO7cv3KyVI6LIGKhErFOFAJEFR8GmA5QkcGr2HT1eyfnbYZ7mxtsfZKKZ0kxSymHLACUEiCw8CjYdoJI1M2s2E8abyQmVnF/3zE1UyxG1csRyzdOILFj6ze6APEngi//xmZNK1mgoKJOmzp6jE5TMSHHiNGWgEk2q5Gyq5CyyoCnYdIBK1pzcvrCeplbJtnYerJY4NBazZkmJ1KFajli3VMFZZCHTMFoHqGTN9BqN43XPHj3awMxoxs7Zy7MkgFolYsVQSfM0IgucejYdoGy0qaWps+9YVvfscL3Z2p+mVinRLGyEtkTDZyILnoJNB1TLJV580Vm8avO5rBis8Fi9yVe2P7Tos9H2HZkgTrOdKeI0peyRNkITWaQUbDpg5WCFP3nBhbz188crCHzq9ZsXdcma4rYBn/veA7xq9FwOjcVUSyVSdyrlSBuhiSwi+k3vgIP1ZivQQDaE9tbPb+dgvTnPZzY/8nI0kRnjzaRV92zFUAUHapWIlZqnEVlU1LPpAM3ZHJenOefbBhybiFk2WOb1z9xE6t7at0ZpziKLi3o2HaAKAsflVQLybQMGqyX2H2kSJylx4jiwZkhpziKLjXo2HbB6uMoNf/QMHjwwxlC1xFgj4bzVQ4uugkCxSkD7tgH58JnK0YgsTgo2HTIRp7z3azsmLepcTIpVAg7VtW2AiEymYbQO0KLOQpHNkhGnqbYNEJFJFGw6YLEnCMRx2ho+e/Rog8FqicP1hJKBO1TLxvLBsuZpRBaxro1pmNlngJcDe939N0Pbq4H3A78BPMPdtxWOvxj4O2AZkAJPJwuGXwIeDyTAP7v7teH4GnADsBk4ALzG3X8VHnsP8ObwnD9x91u7dZ1wPEFgZEmNt172eFYMVhhrJAxWF8fcxN4jEydUCVg5XMUBBwxYWlOvRmQx6+YA+meBT5AFhNwO4JVkQaXFzMrA54E3uPudZrYaaAI14L+7+3fMrApsMbOXuvs3yILJQXd/gpldBXwYeI2ZXQRcBTwZOAf4lpld6O5d62bkCQJ7Do/zp1++a9K8zYrB6oL+j35iIqaZpoCqBIjI9Lo2jObuW4FH29p+6u73TXH4i4G73P3OcNwBd0/cfczdvxPaGsCPgQ3hOVcA14fbXwZeYGYW2r/g7hPu/gCwE3hGhy9vkigylgyUW4EGFse8Tb54sxQdT3M+XE9IEqcRp0RmqhIgIkDvzNlcCLiZ3WpmPzazd7UfYGYrgN8DtoSm9cBDAO4eA4eA1cX2YFdo66pmnC6qeZs8+6yZOpHB7/7W+hOqBCwfiJTmLCJA76Q+l4E3bhdyAAAUjklEQVRnk83TjJENl2139y3QGma7EfiYu/+yU1/UzK4BrgHYuHHjrF5rsRXjbC3ejIzrb3uA1156Hq9++nlEBqmj7DMRmaRXgs0uYKu77wcws1uASzjei7kO+Lm7/03hOQ8D5wK7QjBaTpYokLfnNoS2E7j7deG1GR0d9dlcwGIqxjnV4s0bb3+QK0c3ghkDYfGmhs9EJNcrfw1uBZ5iZkMhcDwXuBfAzP6CLJD857bn3ARcHW5fCXzb3T20X2VmNTPbBFwA/LDbF7BYinGmqbP36PHss+LizWo5olaOWK7FmyLSppupzzcClwFrzGwX8D6yhIGPAyPAzWZ2h7tf7u4HzewjwI/IsmVvcfebzWwD8GfAz4AfZ/P/fMLd/2/g74HPmdnO8LpXAbj7PWb2RbJgFQNv72YmWq4RJ4wsqfHel1/UGkb71Hd/seDmbPYdnbxHzUAlai3eTB3MVGRTRE5kWWdARkdHfdu2bTMfOI1Hj01w3yNHJqU+/59XXswTz17KquFaB890/jSbCb8+PH7CHjVrllRJHSrliHVLa0oKEFlEwvz6jPW5emUYre/FqZ+Q+vynX76r1Qvod3n2Wb5FgLLPROR0aGC9Q6ZLfW7G6TydUWftD9lnZnC4nu1Ro+wzETlVCjYdspBL1qSpUw/ZZ5//fjZ8dmw8YdlghTRUCThrWNlnIjI9BZsOWcgla/YVtg7Ih8+uHN1IM0mpliOWDZaoVvWjJCLT07+iHbJQS9a01tQUtg549dPPoxQZqUPqzhINn4nIDPTvaAc143RBpT/HcdpKCsi3Dsiyz0qk2jpARE6Dgk0HDVZLvOslTzwh/bkf523S1HnkyHgrKSCv6FzcOgC0dYCInBoNo3XQQkp/zrPP2is6H98QLWKdtg4QkVOknk0HTTeM1m/pz8Xss8NtSQH5mpplgyWtqRGRU6Zg00ELZRgtzz5L8VZJmuKaGiUFiMjp0jBaBy2EYbRi9lm9EWv4TEQ6Qj2bDsqrCDzt3BWthZ2P1Zt4nwSbYkmaR482qJSNZuycvXyA1J1KZBo+E5EzomDTQfkGalf/zibe/ZXjQ2l/94bNjPRBb6BYkibPPqtVSjQLFZ01fCYiZ0LDaB20erjK//a7F3H99x7gvS+/iH+65lLe+/KL+Oi37u/5hZ3FpABln4lIp6ln00FRZNTK0Qk9mw+/6mLStLcz0qYrSaPsMxHpBPVsOixxWoEGsiSBd3/lLpIenrZRSRoR6Tb1bDrM3adca9Orm9S1JwWoJI2IdIOCTYf121qbfVMkBagkjYh0moJNh+VrbYq9m7FGgtN7PZs4TrPhs8I+NZN7Ndk2z+rViMhsKdh0WF6y5p2XP3Fy+vPrN7NysLf+cO8Nw2cqSSMi3aZg02HVcok/ecEFrfTnfN7mo1vu539/xcWMLK3N9ykCWVJAM8ky5FSSRkS6TcGmw1YPV7lw7ZKeTn/O96kpRcbnvqfhMxHpPgWbDosiw8xO6Nlc/70HeP+//835Pr1J+9RUyzbl8NnSWqThMxHpKAWbLigZU/ZsSj3QUSjuU3P9bQ/w2kvPmzR8lrgzrOEzEekwBZsuSJye7NlMtU/Njbc/yJWjG8GMgZAUUC5rra+IdJaCTRf0as9G+9SIyHzp2r+wZvYZM9trZjsKba82s3vMLDWz0bbjLzaz74fH7zazgdC+OdzfaWYfMzML7avM7Jtm9vPweWVot3DcTjO7y8wu6dY1TicvWTOypMbfvWEzf/Xq36IRp4RTnxfap0ZE5lM3x0s+C7ykrW0H8Epga7HRzMrA54G3uvuTgcuAZnj4k8BbgAvCR/6a1wJb3P0CYEu4D/DSwrHXhOfPqbxkzTsvfyIf/Pq9vOa623nv13aw78gE6TzsbdNeksbs+D41UWTUKhErhrSmRkS6p2vDaO6+1czOb2v7KTDVf/gvBu5y9zvDcQfCceuAZe5+e7h/A/D7wDeAK8iCEsD1wHeBd4f2GzwrRna7ma0ws3XuvruzVzi9XltrM1VJGu1TIyJzqVfmbC4E3MxuBUaAL7j7XwLrgV2F43aFNoC1hQDyCLA23F4PPDTFc+Ys2PTSWhuVpBGRXtArwaYMPBt4OjAGbDGz7cChU3myu7uZnfb4lJldQzbUxsaNG0/36dPqpbU2KkkjIr2gV3JcdwFb3X2/u48BtwCXAA8DGwrHbQhtAHvCMFs+3LY3tD8MnDvNcyZx9+vcfdTdR0dGRjp2MXA8I+0r2x/isXqT1cNVrn3pb1Cdw5S0RiMOQ2WufWpEZF71SrC5FXiKmQ2FZIHnAveGYbLDZnZpyEJ7I/C18JybgKvD7avb2t8YstIuBQ7N5XxNLl9rc/XvbOKDX7+XKz/1fd74mR+y5/DcJAk0mwl7jzW0zbOI9ISuDaOZ2Y1kE/hrzGwX8D7gUeDjZPMyN5vZHe5+ubsfNLOPAD8i20blFne/ObzU28gy2wbJEgO+Edo/BHzRzN4MPAj8QWi/BXgZsJNsSO5N3brGk3F3XrX53HlJEsizz1SSRkR6hfXqDpJzbXR01Ldt29ax19t3ZIK9R8ZpJikHjzUZqpYYaySsHK6wbtkAa5cPduxrtdt7eJyxxvGkgNdeeh5xQmvxZqVsrF82qEoBIjJrZrbd3UdnOk5/bbpk9XCV1cNV6o2EG3/4II/VmywdKLNsoEK5i/M205Wkyf+pGKhErBwqK9CIyJzqlWy0BSeKjCR1/uG2B05Igf67129m5VB30o33HclK0hyqN6mUjbIzqSSNKylAROaBgk0XJXM8bzMxETMeJ1TLUWvx5mC1jGEk7lQj42wlBYjIPNBYShcNVEpsWDnI2573BKql7FtdLUW87XlP6PjizmYzYf9Yo1WSJs8+SxKnEadEBksHVNFZROaH/vJ00ZrhGquHq7T3I4wpS/acsTz7rJl6a02Nu7NyuIqTpfcBLK1p+ExE5oeCTRdFkWHAWFuSwNKBCp0cydp3JEtzjkxrakSkNynYdFkzJAnkQ2lJ6jx4YIzH6s2OLO5sNLJ5mlJkfPve3bz8qRv4yraHWDFUaa2pWTtc1ZoaEZlXCjZdlrrzpmdtOmEo7dhEzGP1xqxeu1gl4HC9ySXnr2b7A/t53TM3US1H1MoRywdL1GrKAxGR+aVg02UDlRLnrhpkrJFw+y/2sWq4ynmrh1g1XOP4bMrpi+O0VSWgFFmr9tmzL1xLnKQ0EydxZ1hpziLSAxRsumzNcA0wvvuzPbxq9Fx2Hayz78gEO/ce5cCxJnF8+llpaeo8cmS8FWhuvfvXDNXKJ8zTnL1EWweISG9QsOmyKDLS1PkPl57PgaONSYkCJTMOjZ/+UFqeEFAcPrvt/r2sDTtvZlUCSlSrGj4Tkd6gv0ZzYKBaYqKZ8g+3PcB/efGFlKNSa0X/sUbCytRPuQdSTAg4XKgSkA+fpZ7VQNPwmYj0EvVs5sCa4RqpO29/3hNoxM6XfvQgSepEBpEZRxun1ruZmIhbCQHO8fU0g9UytXLU6tWsWzao4TMR6SkKNnMgiozBSolVwzVuvvNhXnvpea1FnePNlINj8YxzNxMTMfvGGq3hs3ojnqJKgLF2SU1VAkSk5+iv0hxZsyTr3Vz9rE0crsf8f/fvoVyKKJeMshljzel7NxMTMfsLgebWu39NFEU0Y+fsME+j9TQi0ssUbOZIFBmVUkTq8ONfHWDzpjV8+97dRGY0U+fIeMr4eDzpOXGccmR8POvRpCcmBCwdrJC6U45M62lEpKcp2Myhs5bUSFLn+RetY/sD+9m8aQ13/tsBBsoRDuw5NsGeQ+MhyEyw99g4h+vppMyzStlOWE+Taj2NiPQ4/Ss8h8rliEopopGkPP+iddz5bwd40jkrOFRvsHK4SjmKKEWw79g4aZplqyWeBZrxZkK1EpGmqbYNEJG+o57NHDtrSY1KZKTuPO281TSaKUsGysSpE4fhtDiBOM0qAESW9Wiy4TJYNlglsuz5lRBolBAgIr1Of6XmWLkcsWaoSiXs5LlssEKaGkkCjdiJQ5mZUmSULCuuWa1EuDulKCJOvDV/s1aBRkT6hP5SzYNarczqwSrlEHASDx/p8SBTiiCKnM2b1nDb/XtbPRp3Z7Bc4pzlgwo0ItI3NGczTwYGyqwG9tcbeKjHaZZViU7SlJSsN7N2WZUXPXkdjTilHGXraJR1JiL9Rv8az6OBgTJrBquUSlAqQbVsONBMnGacMlwt04yzYbNKFLF26YACjYj0Jf3lmmcDA2XOKkccmWjgwPKBEnGlxEScMtZIqJQi1i/X3IyI9DcFmx5QLkesLA/M92mIiHSN/l0WEZGu61qwMbPPmNleM9tRaHu1md1jZqmZjRbazzezupndET4+VXjstWZ2t5ndZWb/YmZrQvsqM/ummf08fF4Z2s3MPmZmO8NzLunWNYqIyKnpZs/ms8BL2tp2AK8Etk5x/C/c/anh460AZlYGPgo8z90vBu4C3hGOvxbY4u4XAFvCfYCXAheEj2uAT3bsikRE5Ix0Ldi4+1bg0ba2n7r7fafxMhY+hi2ryb8M+HV47Arg+nD7euD3C+03eOZ2YIWZrTvDyxARkQ7opTmbTWb2EzP7VzP7dwDu3gT+GLibLMhcBPx9OH6tu+8Otx8B1obb64GHCq+7K7SJiMg86ZVstN3ARnc/YGabgf/XzJ4M1MmCzdOAXwIfB94D/EXxye7uZuan+0XN7BqyoTaAo2Z2Or2uojXA/jN8bq/RtfSmhXItC+U6QNeSO+9UDuqJYOPuE8BEuL3dzH4BXEg2hIa7/wLAzL7I8bmZPWa2zt13h2GyvaH9YeDcwstvCG1Tfd3rgOtme/5mts3dR2c+svfpWnrTQrmWhXIdoGs5XT0xjGZmI2ZWCrcfRza5/0uyIHGRmY2EQ18E/DTcvgm4Oty+Gvhaof2NISvtUuBQYbhNRETmQdd6NmZ2I3AZsMbMdgHvI0sY+DgwAtxsZne4++XAc4D/ZmZNIAXe6u6Phtf5ALA1PPYg8IfhS3wI+KKZvTm0/0FovwV4GbATGAPe1K1rFBGRU9O1YOPur53moa9OcexXgK9M8zqfAj41RfsB4AVTtDvw9tM62dmb9VBcD9G19KaFci0L5TpA13JazP2059VFREROS0/M2YiIyMKmYDNLZvYSM7svlMe5duZn9BYz+1UoB3SHmW0LbVOWAuo105RE6rsyRtNcx/vN7OFCCaeXFR57T7iO+8zs8vk566mZ2blm9h0zuzeUpvpPob2v3peTXEffvS9mNmBmPzSzO8O1fCC0bzKzH4Rz/iczq4b2Wri/Mzx+fkdOxN31cYYfQAn4BfA4oArcCVw03+d1mtfwK2BNW9tfAteG29cCH57v85zm3J8DXALsmOncyZJGvkGWTn8p8IP5Pv8ZruP9wDunOPai8HNWAzaFn7/SfF9D4fzWAZeE20uB+8M599X7cpLr6Lv3JXxvl4TbFeAH4Xv9ReCq0P4p4I/D7bcBnwq3rwL+qRPnoZ7N7DwD2Onuv3T3BvAFsnI5/W66UkA9xacoiUQfljGa5jqmcwXwBXefcPcHyLIun9G1kztN7r7b3X8cbh8hW6qwnj57X05yHdPp2fclfG+PhruV8OHA84Evh/b29yR/r74MvMDMbLbnoWAzOwuhNI4D/9PMtoeKCjB9KaB+sJDKGL0jDC19pjCU2TfXEYZfnkb2n3Tfvi9t1wF9+L6YWcnM7iBb/P5Nsp7XY+4eh0OK59u6lvD4IWD1bM9BwUae7e6XkFXLfruZPaf4oGd96b5MWezncyerVv544Klk5Zz+an5P5/SY2RKy5Qz/2d0PFx/rp/dliuvoy/fF3RN3fypZRZVnAE+a63NQsJmdUy6N06vc/eHweS/ZGqhnEEoBAdjkUkD9YLpz76v3yt33hD8QKfBpjg/J9Px1mFmF7A/0P7r7/wjNffe+THUd/fy+ALj7Y8B3gGeSDVnmay2L59u6lvD4cuDAbL+2gs3s/Ai4IGR1VMkm026a53M6ZWY2bGZL89vAi8n2HJquFFA/WBBljNrmLV5B9r5Adh1XhYyhTWSlnX441+c3nTC2//fAT939I4WH+up9me46+vF9sawc2Ipwe5DjZb++A1wZDmt/T/L36krg26E3OjvznSnR7x9k2TT3k42B/tl8n89pnvvjyDJo7gTuyc+fbHx2C/Bz4FvAqvk+12nO/0ayoYwm2Zjzm6c7d7KMnP8rvE93A6Pzff4zXMfnwnneFX751xWO/7NwHfcBL53v82+7lmeTDZHdBdwRPl7Wb+/LSa6j794X4GLgJ+GcdwB/HtofRxYQdwJfAmqhfSDc3xkef1wnzkMVBEREpOs0jCYiIl2nYCMiIl2nYCMiIl2nYCMiIl2nYCMiIl2nYCPSRWZ2dOajTul13m9m7zyF4z5rZlfOdJzIXFOwERGRrlOwEZkDZrbEzLaY2Y8t2z/oitB+vpn9LPRI7jezfzSzF5rZbWHvl2Ll4N8ys++H9reE55uZfSLsofIt4KzC1/xzM/uRme0ws+s6UblX5Ewp2IjMjXHgFZ4VPX0e8FeFP/5PICvo+KTw8R/IVrC/E/ivhde4mKws/DOBPzezc8hKpjyRbD+VNwK/Uzj+E+7+dHf/TWAQeHmXrk1kRuWZDxGRDjDg/whVtVOyMu55mf0H3P1uADO7B9ji7m5mdwPnF17ja+5eB+pm9h2yIpDPAW509wT4tZl9u3D888zsXcAQsIqsJNE/d+0KRU5CwUZkbrwOGAE2u3vTzH5FVoMKYKJwXFq4nzL5d7S9ttS0tabMbAD4W7JaYw+Z2fsLX09kzmkYTWRuLAf2hkDzPOC8M3iNK8J+8quBy8iqjm8FXhM2x1pHNkQHxwPL/rAnizLUZF6pZyMyN/4R+OcwNLYN+NkZvMZdZGXh1wAfdPdfm9lXyeZx7gX+Dfg+ZPuWmNmnyar8PkIWmETmjao+i4hI12kYTUREuk7BRkREuk7BRkREuk7BRkREuk7BRkREuk7BRkREuk7BRkREuk7BRkREuu7/B1FrWuBVhsoFAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Ridge Regression - Cross-Validation with test MSE\n",
    "\n",
    "results = pd.DataFrame()\n",
    "alpha_range = np.linspace(10**-3, 3 * (10**2), num=1000)\n",
    "for train_idx, test_idx in model_selection.KFold(n_splits=10).split(hitters_dat):\n",
    "    train, test = hitters_dat.iloc[train_idx], hitters_dat.iloc[test_idx]\n",
    "    errors = []\n",
    "    for alpha in alpha_range:\n",
    "        reg = linear_model.Ridge(alpha=alpha)\n",
    "        reg.fit(train.drop('Salary', axis=1), train['Salary'])\n",
    "        error = metrics.mean_squared_error(test['Salary'], reg.predict(test.drop('Salary', axis=1)))\n",
    "        errors.append(error)\n",
    "    results = pd.concat([results, pd.DataFrame(errors, index=alpha_range).T], axis=0, ignore_index=True)\n",
    "\n",
    "df = pd.DataFrame({'lambda' : alpha_range, 'MSE' : results.mean()})\n",
    "sns.scatterplot(x='lambda', y='MSE', data=df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 384,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x1e77340f0>"
      ]
     },
     "execution_count": 384,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZsAAAEKCAYAAADEovgeAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3Xt4XNV96P3vb2Y0Gt1lXSwbW2ATOxCTkhSrxEBPXwgNl4QGCNBAE3AJB4eX8rbvobmQ9qE0T87bkzQnSZsLCeQEMEkOSQonB04gIRyg5TmAk8gNF4NDUDAgGYOutm6jue3f+8dee7w1ki1b0kij0e/zPPNoZu21R3t7bP9mrfVba4mqYowxxhRTZLEvwBhjTPmzYGOMMaboLNgYY4wpOgs2xhhjis6CjTHGmKKzYGOMMaboLNgYY4wpOgs2xhhjis6CjTHGmKKLLfYFlIqWlhZdt27dYl+GMcYsKTt37uxX1daZ6lmwcdatW0dnZ+diX4YxxiwpIvLakdSzbjRjjDFFZ8HGGGNM0VmwMcYYU3QWbIwxxhSdBRtjjDFFZ9loxhizTHmeMjCWJp3NEY9Faa6JE4lIUX6XBRtjjFlmPE8Znkizd2iCj39vJz1DSdauqOLbV3VwQltdUQKOdaMZY8wy4nnKqwNjDI5l8oEGoGcoybV3dzIwli7K77WWjTHGLCP9oyleGxinLhHLB5pAz1CSdDZXlN9rLRtjjFkGPE85kEyRzOSojkcZGEuzdkXVpDprV1QRj0WL8vst2BhjTJnLZj1e6R9lOJnDU2U8neO+nd184ZKT8wFn7YoqbrtyM8018aJcg3WjGWNMGctmPfYNT9A9mOS45mreGp6gqaaCq89Yz51P7uHmCzbRXBOnta6SNQ1VRctGs5aNMcaUqUwmx77hCVJZj+p4lJyn3PnkHhSIRSJ85vx3sKG1lrb6BGsaqojFihcSLNgYY0wZSqezvOECTVRgPJ3j3s7XueG9G7n18S4Gx9MMjKWJRYVVdZVFDTRg3WjGGFNWPE8ZSaUZmcjRN5KiuSZO/2iappoKzjyxjQef3csnzz2RaESIxyK01VZSUVGcpIAwCzbGGFMmslmPVwfHqIxFyeQ8BsbSqEI85o/DxCIRLj/1ODyFimiEVXWJordoAtaNZowxZSAYn+keTJLJeeQ85b6d3VS4QJPOKi21cTyFeCzC6vqFCzRQxGAjIneISK+I7AqVfVFEfiMiz4nIj0Wk0ZU3i8jjIjIqIl8veJ/NIvK8iHSJyFdFRFx5k4g8IiIvu58rXLm4el3u95xSrHs0xphSEB6fCRIBwuMz4+kcLXX+umd1iRhrGoubDDCdYv62u4DzCsoeAd6pqicDvwU+48ongJuBT0zzPt8ErgU2ukfwnjcBj6rqRuBR9xrg/FDdbe58Y4wpO8FEzd6xNH0jqUmJAB9415r8+ExbfYKKaITVdZWsrE8ULb35cIoWbFT1CWCwoOznqpp1L3cAa135mKr+H/ygkyciq4F6Vd2hqgrcDVzkDl8IbHfPtxeU362+HUCjex9jjCkb4Yma6aw/PlOYCHBpx7GTEgHi8cUbpl/MMZuPAT+doc4aoCf0useVAbSp6j73/E2gLXRO9yHOMcaYJc3zlP3jqcOOzwSJABGRfCLAQmScHc6iBBsR+VsgC3x/Pt7PtXp0FtexTUQ6RaSzr69vPi7FGGOKJmjN7B/PlvT4zHQW/ApE5M+BC4CPuCBxOHtxXW3OWlcG8FbQPeZ+9obOaT/EOZOo6u2q2qGqHa2trUd1H8YYs1AKWzNZT0t6fGY6CxpsROQ84FPAB1V1fKb6rptsWES2uCy0q4D73eEHgK3u+daC8qtcVtoW4ECou80YY5aU6VozUaGkx2emU7SrEZF7gDOBFhHpAW7Bzz6rBB5xGcw7VPU6V/9VoB6Ii8hFwDmq+iJwPX5mWxX+GE8wzvN54Ecicg3wGvCnrvwh4P1AFzAOXF2sezTGmGIJVgIYTuboHkzS3lSdb830j6ZLYqLm0ZCZe7KWh46ODu3s7FzsyzDGGLJZjzeGk6hCJqcMjKZYWVc5KcjA4k/UBBCRnaraMVO90mpnGWPMMhZuzaSzyv7xNM018SmtmXRWWd1QiSJ4qtRURGmprSyZ8ZnpWLAxxphF5nnK8ESa4YlsvjUTEfJrmzXVVOTrBq2ZVFaJx6ToWwPMl9K/QmOMKWPhBIB0VvMrAXjKlLXNmmvirGpIlFxa85FYGldpjDFlZrp05qA10z+aJuflpsydySlUxiK0r6guqbTmI2HBxhhjFlAQZKZLZw63ZiYyHlERbr7gJI5prKIiGuGY+sSSCzIBCzbGGLMAgkUzgyBTODlzutZMdWUUT5WqiijHNFQt+pIzc2EJAsYYU0RBhtmBpL8GcTBnpnByJsBExiMWiXDzBSfhqRKNyILtpFlsFmyMMaYIslmPAxNpxtK5fIYZkA8yM6UzVy2BdOajYcHGGGPmiecp+5NpIqL0jWaoiETIev58maaaOMLBIFMO6cxHw4KNMcbMQTbrMTieJhaFwbEMOc+jprKCHtddFmSYNVbHGRqbGmSW2uTM2Sqv0GmMMUUWZJP1j0wwODbBnsExhsZTjKb8NcyikSjpaTLMsl6Ohmr/+314zsxSTmc+GhZsjDFmBkGAGRyboGf/OPuTGQbH04ylPHryAUapjkeJCOQ8nZJh9uWf/5ZkOsfK+gSrGxN4yyTIBKwbzRhjphEsIeOpMprK4akiSH6gf+/QBMc1V+cDTDAe4ync2/k6l3S0k0znpmSY5Twtmwyzo2HBxhhjOBhcUhmPiphMG2D2j6doqokD5HfJDAJMMB6T83J84F1ruK+zmz/bso54LELOUz/DLB6lpaY8x2RmYsHGGLNsBQEmm1PGMzkmMjlUlURFbNoAEwz0g9+KeWy334LJufGYZDrHcDJHe1OCK09fT85TVCnZPWYWUtHuXkTuEJFeEdkVKvuiiPxGRJ4TkR+LSGPo2GdEpEtEXhKRc0Pl57myLhG5KVS+XkR+4cp/KCJxV17pXne54+uKdY/GmKUnm/UYGJ2gf8Qff+kd8Qf301ll79AE0UiUvpEUESGfSebp5IH+WBTWrEhw5olt3NfZTaIiRmN1nFVuPCadVeJRfzzmmCW0WGYxFfNP4C7gvIKyR4B3qurJwG/xd+5ERDYBlwMnuXNuFZGoiESBbwDnA5uAK1xdgC8AX1HVDcAQcI0rvwYYcuVfcfWMMctYEGAGxybYO5xkYCw9KcAEi2AG4y+HCjDBQP+B8QyN1XGOb6nhytPXIwAKrXUJ1q6o5tjmmmUx6H80ihZsVPUJYLCg7OeqmnUvdwBr3fMLgR+oakpV9+Bv6Xyqe3Sp6iuqmgZ+AFwo/p7S7wXudedvBy4Kvdd29/xe4GxX3xhT5sJpyW8dSNI3MjnAjKU8MtMEmCBFORh/OVSAWdWQ4JY/OYnG6jjZnNJcU2nB5Qgt5pjNx4Afuudr8INPoMeVAXQXlL8HaAb2hwJXuP6a4BxVzYrIAVe/f75vwBizOMJjLTlPiUSEaIT8oH4666GqiAhVFbF8gDmuuXpSgMnpwRn99VVR1qxITEpV/ouzNrC6sYpb/uQksp6SzSltNv4yK4sSbETkb4Es8P3F+P2h69gGbAM49thjF/NSjDEzCM/ULwwqlRXRKWnJ7U3VdA+OTwowOc8PQIUBJpyi3FQbJxYRCzDzbMH/9ETkz4ELgI+oqrrivUB7qNpaV3ao8gGgUURiBeWT3ssdb3D1p1DV21W1Q1U7Wltb53hnxphi8FsxqfxM/ZEJf6wlmyM/oJ/NMWlQPxh7CQJM0EV2b+fr+cH9uNszJpNVVtYnaKtP0FBdQSbnocDqhiqOa66xAf55sqB/giJyHvAp4IOqOh469ABwucskWw9sBH4J/ArY6DLP4vhJBA+4IPU4cKk7fytwf+i9trrnlwKPhYKaMWaJyGY9htyM/eFkLj9Tf7qgUjioH4y9FAaYM09s454dr1FbGaOtIMBEIsIxDVUc21TDyjobf5lvxUx9vgd4GjhBRHpE5Brg60Ad8IiIPCMi3wJQ1ReAHwEvAj8D/kJVc25M5gbgYWA38CNXF+DTwI0i0oU/JvMdV/4doNmV3wjk06WNMaUtvCzM3uEkw64VE6w1dqigMl1acs7LTQkw6132WDanRERYGWSPWYApOrEv/b6Ojg7t7Oxc7MswZlkK7/0ykclRVREjlfXyS/PnPKVnyF9F+R9/tpsbz3k7iYropDGb4WSWrz/2cn5QX13igOcpGU/xPCVRxqsqLxYR2amqHTPVsxUEjDGL4lB7v4SzxoIZ+8FaY9NligVBpTYemzSov7K20sZaSoh9EsaYBRV0lb3SP0rfyASjbuXk8LyXYFA/6BoL1hpLVMSmnevSWpegpS7BmhXVNqhfouzTMMYsCM9TDiT9ILN/PHvIvV/Cg/pBK0ZV+ehp6wHyqcgWVJYW+5SMMUUVzio7kPSDTNY79N4v4UH9lto4f+daMdGIv12yBZilycZsjDFFkc16jKTSDKdyoOQnXAatmMPt/bLOZY1lckpVRZSWBhvUX+rs64ExZl6FJ2GOurXIgrkxQZDpH52690tjdZyVoXkv8WW0i+VyYC0bY8y88DxlJJXmQDJLRISeweSUrLJgg7GA7f2yfNgnaoyZk/CYzP7x7KRJmIVZZQ3V/vfbdFZpromzqiFByvZ+WRbsUzXGzEoQZMIz/YPusmDQvzCrLJnOsdJtMOYpVFpX2bJhwcYYc8Q8TxkcS7F/fGLaMZlgKZl7O18/ZFZZ0FVme8AsLxZsjDGHFQSYwbGJKRMxMzlvUpApnIR5TGMVHz3NzyqrsK6yZc0+cWPMFME2yv0j4QCTmzIRs3BMZrpJmMH8mDZrxSxrFmyMMcDBABPeRnlygJk6EXO6mf43X2CTMM1U9jfAmGVsugAz5sZh9g5NTJrpXzgR02b6m6NhfxOMWWZmCjDBOEzhTP/pJmLamIw5UsXcPO0OEekVkV2hsstE5AUR8USkI1QeF5E7ReR5EXlWRM4MHdvsyrtE5KsiIq68SUQeEZGX3c8VrlxcvS4ReU5ETinWPRqzFAQD/P0jRxZgwtsohwNMMEdmOJljZV2cK0+3MRlz5Ir59eMu4LyCsl3Ah4AnCsqvBVDV3wPeB3xJRIJr+6Y7vtE9gve8CXhUVTcCj3JwR87zQ3W3ufONWVamyyAbHD+yABPeRjke84PHcDJHfcLfSnl1Y4K0TcQ0R6lof0NU9QlgsKBst6q+NE31TcBjrk4vsB/oEJHVQL2q7lB/S9G7gYvcORcC293z7QXld6tvB9Do3seYsjZ9ivLBAf4jDTDhbZTb6hMHA0zO36TsmIYqmyNjjlqpfB15FvigiMREZD2wGWgH1gA9oXo9rgygTVX3uedvAm3u+Rqg+xDnGFNWslmP3uGJQwaYcAbZkQaY9W7F5WxOiYiwsi7B2hXVHNtUw8o6CzBmdkol2NyBHxQ6gX8CngJyR3qya/Xo0f5SEdkmIp0i0tnX13e0pxuzKMID/HsGxxgaTx0ywIQH+I8mwFjrxcy3klj1WVWzwH8KXovIU8BvgSFgbajqWmCve/6WiKxW1X2um6zXle/FbxVNd07h770duB2go6PjqIOVMQslm/U4MJFGBEZSOVKZHFUVMXoGk7Q3VU8KMMLBABOssly4KdnWM9ZTn6ig3QWYREWEYxossJjiKYmWjYhUi0iNe/4+IKuqL7pusmER2eKy0K4C7nenPQBsdc+3FpRf5bLStgAHQt1txiwZR5KiPF0L5lAZZPUJa8GYxVO0lo2I3AOcCbSISA9wC37CwNeAVuBBEXlGVc8FVgIPi4iH3wq5MvRW1+NntlUBP3UPgM8DPxKRa4DXgD915Q8B7we6gHHg6iLdojHz7lAtmCDABPvDBOMv07VgGqpjJNO5/F4x9YkKsp6SzloLxiwe8Yc7TEdHh3Z2di72ZZhlyN/ZMo2nOinApLIe3YPjHNdcDUDfSIqmmjgiQvfgODt+18clHe2oKiJCMu0v89/elACErKd4npKoiNJSa9sqm+IQkZ2q2jFTvZLoRjNmOQpvOtY7kjrqOTBnntjGfZ3dJCpiNFbHWRWaA2NdZKbUWLAxZoF5nnIgmZq06dhs5sAE4y8CoNAapChbgDElyIKNMQsom/V4pX+U4WRu0qZjs50DY60Xs1RYsDFmgWSzHvuGJ+ieZtMxCzCm3FmwMWYBZDI59g1PkDrEpmMWYEy5s2BjTJGl01necIEmmA9TuOlYMp1jnQUYU8Ys2BhTRJlMjt6xNH0jKaJycMLldJuOqcKquoQFGFOWSmK5GmPKUSaT463RFOmsx8BYGlXyS/bHIhEuP/U4kmmPeCzCmgZbpt+UN/vbbUwRZDI53hieIJ31yHnKfTu7qXCBJp1VWmrjeArxWITV9QkLNKbs2d9wY+ZZNuvx1miKvpEUOU+5t/N1bnjvRm59vIvxdI6WujiRiFCXiLHGNh4zy4R1oxkzjzxPeeNAkqynDIyleeSFfXzgXWt48Nm9fPLcE4lGhIpohJU1ceJx++dnlg/7SmXMPOofTdHrWjT37ezmj05o48Fn93Jpx7FEI0I8FqGtttICjVl2LNgYM0+yWY9kJsfAWDrfdbb9qT2csq6ZA8kM0YjQVltJRUV0sS/VmAVnX6+MmQfB6gCe+i2araevn9R1FrRoLNCY5cpaNsbMUTBOk8p6vHlggqvPWD+pRQNQWxm1QGOWNQs2xsxRME4TFbjzyT1UxaNccepxNFZVMDKRxVOlPhFf7Ms0ZlEVLdiIyB0i0isiu0Jll4nICyLiiUhHqLxCRLaLyPMisltEPhM6dp6IvCQiXSJyU6h8vYj8wpX/UETirrzSve5yx9cV6x6N8TzNj9P0j6a5+oz13Pp4F+mcRzQiHNdcTUtt3FYDMMteMVs2dwHnFZTtAj4EPFFQfhlQqaq/B2wGPi4i60QkCnwDOB/YBFwhIpvcOV8AvqKqG4Ah4BpXfg0w5Mq/4uoZUxT9o6n8OE1FTKguaNUA1qoxhiIGG1V9AhgsKNutqi9NVx2oEZEYUAWkgWHgVKBLVV9R1TTwA+BCERHgvcC97vztwEXu+YXuNe742a6+MfMqyD4LxmmCSZvrWqpZ1ZDgxNV1rGuusVaNMZTOmM29wBiwD3gd+K+qOgisAbpD9XpcWTOwX1WzBeWEz3HHD7j6U4jINhHpFJHOvr6++b0jU9aCpABPddI4TUSEV/vHyeQ82upsMU1jAqUSbE4FcsAxwHrgr0Xk+GL/UlW9XVU7VLWjtbW12L/OlJEgKSDcqrFxGmMOrVSCzZ8BP1PVjKr2Ak8CHcBeoD1Ub60rGwAaXbdbuJzwOe54g6tvzLwIJwVY9pkxR6ZUgs3r+GMwiEgNsAX4DfArYKPLPIsDlwMPqKoCjwOXuvO3Ave75w+417jjj7n6xsyLcFLA1tMnt2qOb61hXZON0xhTqJipz/cATwMniEiPiFwjIheLSA9wGvCgiDzsqn8DqBWRF/ADzJ2q+pwbc7kBeBjYDfxIVV9w53wauFFEuvDHZL7jyr8DNLvyG4F8urQxc1WYFLD9qT1csrmd5po4TTVxGqpitoqzMdMQ+9Lv6+jo0M7OzsW+DFPCPE/pGRonp8p/eWg315+1gaGxDNXxKOPpHO1NVRzfUmutGrOsiMhOVe2Yqd5hv4KJyEdDz88oOHbD7C/PmKWnf8ySAoyZrZna+zeGnn+t4NjH5vlajClpE5YUYMyszRRs5BDPp3ttTNnyPCWCWFKAMbM0U7DRQzyf7rUxZat/NMU+SwowZtZm+hdyoog8JyLPh54Hr09YgOszZtEFGWj/8NBu6z4zZpZm2jztHQtyFcaUqPCyNH2jKT77wItcd+bbqMbfm6auMmbdZ8YcgcO2bFT1tfADGAVOAVrca2PKWnhZmi9eejJ9oyk+/t2d/PW/PEuiIkKFdZ8Zc0QO27IRkZ8AN6nqLhFZDfw70Am8TURuV9V/WoiLNGYxBN1nA2Np7tvZzfVnbeBzF74zP6+mta6SxirrQjPmSMz0tWy9qgabn10NPKKqfwK8B0t9NmUs3H1mGWjGzN1MYzaZ0POzgW8DqOqIiHhFuypjFlnQfZbJeVx9xnrufNIy0IyZi5n+tXSLyP8jIhfjj9X8DEBEqoCKYl+cMYvBVnU2Zv7NFGyuAU4C/hz4sKrud+VbgDuLeF3GLBpb1dmY+XfYbjS3t8x105Q/jr/EvzFlpXBVZ+s+M2Z+zJSN9sDhjqvqB+f3coxZPIVbPV9/1gauOPU4quNRRiayNFZXWPeZMbM0U4LAaUA3cA/wC2w9NFPGCpMCbn28i0s2t1MXiXFcczVNNRXWfWbMLM3UH7AK+BvgncA/A+8D+lX131T13w53oojcISK9IrIrVHaZiLwgIp6IdITKPyIiz4Qenoi82x3bLCLPi0iXiHxVRMSVN4nIIyLysvu5wpWLq9flltY5ZTZ/MGZ5saQAY4prphUEcqr6M1Xdip8U0AX86xHuZXMXcF5B2S7gQ8ATBb/n+6r6blV9N3AlsEdVn3GHvwlcC2x0j+A9bwIeVdWNwKMc3JHz/FDdbe58Yw7LkgKMKa4ZRzpFpFJEPgR8D/gL4KvAj2c6T1WfAAYLynar6ksznHoF8AP3u1cD9aq6Q/0tRe8GLnL1LgS2u+fbC8rvVt8OoNG9jzHTsq2ejSm+mXbqvBt4Gn+OzWdV9Q9U9XOqureI1/Rh/DEigDVAT+hYjysDaFPVfe75m0Bb6JzuQ5xjzCSFSQHWfWZMccz0de2j+N1RfwU8JSLD7jEiIsPzfTEi8h5gPLREzhFxrZ6j3l9HRLaJSKeIdPb19R3t6aYMhBfatK2ejSmemcZsIqpa5x71oUedqtYX4Xou52CrBmAvsDb0eq0rA3gr6B5zP3tD57Qf4pxJVPV2Ve1Q1Y7W1tZ5uHyzlFhSgDELp2Q6okUkAvwpbrwGwHWTDYvIFpeFdhVwvzv8ALDVPd9aUH6Vy0rbAhwIdbcZk2dJAcYsnJnm2cyaiNwDnAm0iEgPcAt+wsDXgFbgQRF5RlXPdaf8EdCtqq8UvNX1+JltVcBP3QPg88CPROQa4DX8QAXwEPB+/My5cfzVqo2ZxFYKMGZhiT/cYTo6OrSzs3OxL8MsgGzWY69LCvgvD+3m+rM2MDSWye9T095UxfEttdaqMeYIiMhOVe2YqZ59dTPLSjbrsW94gj5LCjBmQVmwmQPPU/pGUuwdGqdvJIXnWSuxlAVpzqmsZ0kBxiwwCzaz5HnKS2+NcPGtT3LGFx7n4luf5KW3RizglLAgzTkqWFKAMQvMgs0sDYylufbuTnqGkgD0DCW59u5OBsbSi3xlZjrhNOf+0bStFGDMArN/WbOUzubygSbQM5Qknc0t0hWZwxkYS+fTnCtiQnVB9xlg3WfGFJEFm1mKx6KsXVE1qWztiirisegiXZE5nFQ2NykhYDydY11LNasaEpy4uo51zdZ9ZkwxWbCZpeaaON++qiMfcNauqOLbV3XQXGPfjktNNusBTEoIiIjwav84mZxHW13CAo0xRWbzbJzZzLPxPGVgLE06myMei9JcYymzpcbzlJ6hcUZSWfaPZ6aM09QnYrTUJRb7Mo1ZsmyezQKIRITWukrWrKimta7SAk0JCjLQeoaS+UDTWFXBwFiaz/90N2qbzxqzIIq2XI0xiy2cgRakOn/6vufoGUqydkUVt1252bo9jVkgFmxM2SpcaHP7U3u4+YJNNNfE/RZpQ5W1Ro1ZINaNZsqS7b5pTGmxf22m7Njum8aUHgs2puzY7pvGlB4LNqas2O6bxpSmogUbEblDRHpFZFeo7DIReUFEPBHpKKh/sog87Y4/LyIJV77Zve4Ska+6HTsRkSYReUREXnY/V7hycfW6ROQ5ETmlWPdoSk//mO2+aUwpKmbL5i7gvIKyXcCHgCfChSISA74HXKeqJ+Hv8Jlxh78JXAtsdI/gPW8CHlXVjcCj7jXA+aG629z5ZpmYsKQAY0pS0f7lqeoT+NtAh8t2q+pL01Q/B3hOVZ919QZUNSciq4F6Vd2h/lIHdwMXuXMuBLa759sLyu9W3w6g0b2PKXPZrIcg1n1mTAkqla95bwdURB4WkX8XkU+58jVAT6hejysDaFPVfe75m0Bb6JzuQ5xjylSQgWZJAcaUplIJNjHgD4GPuJ8Xi8jZR3qya/Uc9SJvIrJNRDpFpLOvr+9oTzclJMhA+4eHdlurxpgSVCrBpgd4QlX7VXUceAg4BdgLrA3VW+vKAN4Kusfcz15XvhdoP8Q5k6jq7araoaodra2t83YzZmEFEzgHxtL0jab47AMvks55+eNNtkCqMYuuVILNw8DviUi1Sxb4v4AXXTfZsIhscVloVwH3u3MeALa651sLyq9yWWlbgAOh7rYF5XlK30iKvUPj9I2kbMvoIghP4LxvZzdfuORk+kZTfPy7O/nrf3mWlXWVNFZZq8aYxVa0tdFE5B78rLIWEekBbsFPGPga0Ao8KCLPqOq5qjokIl8GfoXfHfaQqj7o3up6/My2KuCn7gHweeBHInIN8Brwp678IeD9QBcwDlxdrHuEQ28z4HnKS2+N5LeODva7OaGtzr5lz6Og+yyT87j6jPXc+eTB9c+aauI01VTYn7cxJcD2s3Fmu5/NoQLKwFiai299ctLW0WtXVPHj68+gta5yvi9/WcpmPfYeSPKbN0e4b2c315+1gaGxDNXxKOPpHO1NVRzfUmvBxpgiOtL9bGzV5zkYGEvnAw1Az1CSa+/u5MfXn0E6m6NnKMnvtzdy3Zlvo7Gqgv3JDJ7nzfCu5khksx77hiemTOC8ZHM7dZEYx7fW2KrOxpQQCzZzEASUsJ6hJOlsjopYhHM2rZx2D5VW24Z4ToJxmnRO6RuZyHef2QROY0qXBZs5iMeirF1RRWttZb71Mp7OURWPAnDT+e/gqjt+Oanl8/Hv7rSutDkKxmmaa+Lc+eQerj9rA1ecehzV8SgjE1kaqyss1dmYEmPBZg6aa+Lc/bFTeWt4gk/e+9ykcZv6RIwDyQyttZXcfMGmfDfat/71d6SzucW+9CWIXjC1AAAXF0lEQVQrvNCmKvkJnEH32XHN1ZYUYEwJsn6GOYhEhNpELB9o4OC4jYiQzSmfOu8EPveTF/nw7Tv43E9e5FPnnZBv+ZijF959syImVBdM4ASsVWNMCbJgM0eZrDftuE1UYFVDYkog+uS9z5G1+TazUrj75q2PdzGezrGupZpVDQlOXF3HumZb1dmYUmTdaHMUjNsUpjhHIhEi3vQZaWrB5qgV7r4ZjNNERHi1f5z2piraV1RboDGmRFnLZo6aa+J8+6oO1q6oAsiP2ayoqkBEOGfTSj5x7uSutP6xtK0mcJRs901jljZr2cxRJCJsbK3lRx8/jUzOoyIaobUmzst9o3zlkZfyGWnhRIHe4RSrG9I01VhG2pEIJwUEkzct+8yYpcWCzRx5nvJy3+ikVQT++398T/71/33mBlprK/nEuSdMnm/z0c00Vtm38SMRTgqwyZvGLE0WbOZoulUEekdS+de9Iyn+8uyNbH9qz6QU6H9+9Lf8fxefbPNtZlCYFGCTN41Zmuxf6RxNt4rAwFg6P4bzrX/9HRtW1rD19PWTxm22nr7elq6ZQWFSgO1TY8zSZS2bOZpuFYGICLdduZmPf3cnv+7eTyanfPq+5yaN24ync/i7JphDKVzR2SZvGrN0WbCZo+lWEThn00o++8GT+OG2LeQUQA85btNSW2n/YU4jvCGaJQUYs/RZsJmjYBWBq+54Lj+nZuvp67nsth2TEgZs3ObI2YrOxpSfoo3ZiMgdItIrIrtCZZeJyAsi4olIR6h8nYgkReQZ9/hW6NhmEXleRLpE5Ktux05EpElEHhGRl93PFa5cXL0uEXlORE4p1j0GwqsIXHfm2/KtF/ATBv7zgy+ysc3GbY5EME6Tynr5pIDtT1lSgDFLXTH/xd4FnFdQtgv4EPDENPV/p6rvdo/rQuXfBK4FNrpH8J43AY+q6kbgUfca4PxQ3W3u/KKKx6Kcs2klt125mY0ra6ckDPz8xV4EyY/b3HblZr502btIZz0btykQjNNEBUsKMKaMFC3YqOoT+NtAh8t2q+pLR/oeIrIaqFfVHepvKXo3cJE7fCGw3T3fXlB+t/p2AI3ufYpmRVUFf3n22/ncT17k5d7RfCZaYO2KKjw9OG4TtG5uvn8XfSMpW03ACY/T9I+mbaUAY8pIKfVFrBeRX4vIv4nIf3Bla4CeUJ0eVwbQpqr73PM3gbbQOd2HOKcohpIZrvveTnqGknzrX3/Hly5715Tlayorovzl2RuntG56R1LsT6aLeXlLQrDFs63obEx5KpUEgX3Asao6ICKbgf8pIicd6cmqqiJy1M0DEdmG39XGsccee7Sn5xXOtamsiPC5C99JdTzKeDpHZSxCU1Wc9S01tprANDxP2Tc8QV9BmvPVZ6xnXUs1ERHisQhttsOpMUtWSQQbVU0BKfd8p4j8Dng7sBdYG6q61pUBvCUiq1V1n+sm63Xle4H2Q5xT+HtvB24H6OjomHVfVniuzT9eejJX3/WrKatA//j6M6iujFpW2jT6R1Okst6UNGdb0dmY8lESwUZEWoFBVc2JyPH4g/uvqOqgiAyLyBbgF8BVwNfcaQ8AW4HPu5/3h8pvEJEfAO8BDoS624oiPNdmPD11RYGeoSTpbI7VDVW8va2Wraevn9Sy+cIlJy/brLRgnCYqWJqzMWWsmKnP9wBPAyeISI+IXCMiF4tID3Aa8KCIPOyq/xHwnIg8A9wLXKeqQXLB9cB/A7qA3wE/deWfB94nIi8Df+xeAzwEvOLqf9udX1TBXJs7n9xDfSI2bYJAPBYlEhFELCstEB6nCRICLM3ZmPIkfpKX6ejo0M7Ozlmfv3donD39Y9z99KtTWi63XbmZd6yqJxIR9g6Nc8N///W04zbvWF2/bL7Be56yd3+St4YnyOQ8qt1W2YNjmfxYV3tTFce31C6bPxNjliIR2amqHTPVK4lutHIQj0VZ1VDFz1/spW8kPWlMpqnm4OB/PDY5Ky2/x81IitWNy2ePm+nGaTJZZV1zNZ5CPBZhdb0lBBhTLizYzJPmmjipTI61K6r4dfd+Pv7dnYDfhfbDbVsm1VvuWWmHG6cRgda6SlbXJ6z7zJgyYv+a50kkIlRVRvjipSdPmmPzxUtPpsp1EQX1CrPSfrhtCzdfsIl/fvS3DIyV95wbG6cxZnmyMRtnrmM24I9DvDowxmsD4/lxh+Oaq1nXXDOpteJ5yr4DSV4dGJ/yH219IkZLXWKut1OSbJzGmPJjYzaLIBIR1jXXUJeoIJ3NEY9Faa6Z2i0WZKVtf2rP1GSCj26mqab8th3wPOWtkQkbpzFmmbJgM88iETmiyZmqyiWb25fFBM+gxReJiI3TGLNMWbBZJMGqA8thgmf/aIrXBsZZ11KdH6e580kbpzFmObFgs0iaa+Kksjk+/t2dZb1ddCaTI5nJUR2P8uaBCarjUeJEbddNY5YZCzaLJBiXKOftojOZHG+4HTfH0znu+eVrNk5jzDJlwWYRVZbxBM9s1uOt0VR+Jeemmor8as42TmPM8mP/yhdReILnLR/cRDx68OMYHEsv2U3VslmPfcMTpF3m2Z1P7kGBWCTCZ85/Bxtaa2mrT7CmocoCjTHLhLVsFlEwwfNv3v8OkukcN9+/K9+V9sVLT6apJr7kWjfBpM1MTlG3EZplnhljLNgsspaaStIZj8u/vWNKokAmu7Sy0sKboDXXxPnBr17nhvdu5OuPvZzPPGutq+QYCzTGLDsWbBZZJCJ46LSJAt/66GZal8julIWTNlXhzBPbePDZvXzy3BOJRoSKaIT6qigVFdGZ39AYU1bs62UJSFRMv1baV5fIWmnBpM1U1stP2qyICdXxKFve1srgWJrXBsbJ5DzqKi3F2ZjlyIJNCWipqczv4Hnfzm72JzM018S56fx3IJR+kkAwaVNDi2ve+ngX4+kc61qqWdWQ4MTVdVPWiDPGLB/F3KnzDhHpFZFdobLLROQFEfFEZMrCbSJyrIiMisgnQmXnichLItIlIjeFyteLyC9c+Q9FJO7KK93rLnd8XbHucb4UrpX2uZ+8yKXfepqr7vglbw2nSjYrzfOUA8nUpEmbcdeiueLU44iI8Gq/36JpWyLdgcaY4ihmy+Yu4LyCsl3Ah4AnDnHOlzm47TMiEgW+AZwPbAKuEJFN7vAXgK+o6gZgCLjGlV8DDLnyr7h6JS9YK61wy+jekRT7k6XXlZbNerzSP8pwMpeftBmkOKfdpM2W2ko2rKxlXZO1aIxZ7ooWbFT1CWCwoGy3qr40XX0RuQjYA7wQKj4V6FLVV1Q1DfwAuFD89VzeC9zr6m0HLnLPL3SvccfPliWw/kuwQvRSmHMTZJ11DybJ5DzePDAxadLm4HiagbE0sahYirMxBiiRMRsRqQU+DXy24NAaoDv0useVNQP7VTVbUD7pHHf8gKs/3e/dJiKdItLZ19c3H7cya801cVbWVU6ac/Ph23dw8/276Cuh1k0466w6HiXnqU3aNMbMqFRSn/8ev0tsdCEbIap6O3A7+JunLdgvnkYkIhzTUIWIcEWJzrnJZj1eHRwjFo0QFRhP53hs9+S5NMGkzVV1lRZojDF5pRJs3gNcKiL/CDQCnohMADuB9lC9tcBeYABoFJGYa70E5bif7UCPiMSABle/5MViEdTNubnlg5sYGsvkj42ksrR6umhjH5lMjjdHUnQPJvNbBTTVVEyZSxOPRWirrbS5NMaYSUriq6eq/gdVXaeq64B/Av5BVb8O/ArY6DLP4sDlwAPq72X9OHCpe4utwP3u+QPuNe74Y7qE9r5OVBxcvuaeX77G/mSGukSMiAjDEwvfleZ5yvBEijeGD3adBVln4HedXe4yzyqiEVbVJSzQGGOmKGbq8z3A08AJItIjIteIyMUi0gOcBjwoIg8f7j1cq+UG4GFgN/AjVQ0SCD4N3CgiXfhjMt9x5d8Bml35jcBNLCEtNZWsbkhw55N78vNuBsbSDI6lOZDMkl3A7rRMJkfP/nGGkzn6RlL5rrNw1llLbXzSVgHWdWaMmY4soS/9RdXR0aGdnZ2LfRkAvD44xu59I9y3s5vrz9rA0FiG6niU8XSO9qYqjm+pLWp3mucpo+k0Q2NZMjklIvBy7yhN1XHiMckHnPB6Z5YMYMzyJCI7VXXKvMlC9r9DCUpU+GnQV5+xfkG704JJmuHWTEQg52l+CRqwrDNjzNGz/yFKUEtNJSvrKlm1QN1pnqfsH/eDzP7xLOms5vei8RTu7fQzzoIlaFrq4kQiQl0ixppGCzTGmJnZ/xIlKEiDjohwyeZ2tj+1h+vP2kBTdZzmmjiZnD+pcq4BJwgyr/SP5oNMYWsm6+X4wLvW5DPO2uoTVEQjrK6rZKVt52yMOUIWbEpULBahKtSdJkA8Jrw6MM7AaIqu3lFeHRyb1coCQXdZEGS6B5NkPX9sprA18+Wf/xZV5aOnrScei5CoiLKqLkE8XipZ88aYpcCCTQlrqT3YnZbO+uuP3fPL10hURNiwspaKaIR9B5JH3MIJj8kcSB4MMtXxKFEBT5m2NVOXqEAEVtdVcox1mxljZsH+1yhh4e60lto4dz65hxvPeTsiwvee3kM665HKerw5MkE6nZ32PYKusv6RiYIxmYNBZjydo380Tc7LWWvGGFMUFmxKXNCdllO4ZHM7sUiUB5/dywfetYYvPvwbRiYyRCNC71ia1wbG6HVjOeEA0zuSYjSVmzQmEw4yTTUVxGPCRMYjKsLNF5xEY3UcETimPmGtGWPMnNlX1SWgpbaSVNajuSZORODSjmP54sO/4cZz3k5EhN7hCdJZpb0pgQi8MZxkIpMjHo2S9ZS9QxO0N1Xnx2Qaq+MMjflBJpDOKqsbKlEET5WqiigttZWWAGCMmRf2dXUJiET8pfpb6yrxFKIRybdyDoz73Wf1VVGGJ7KMuRbM3qGJ/KD/dGMyDdX+94x0VmmuibOqIUFOoTIWoX1FtWWaGWPmlQWbJSIWi7CmoYp4zF/sMmjltNTGGRzLEI1EDxlgphuTSaZzrKxPsLoxgWdBxhhTZNaNtoTEYhHWNlYzmk6T8yrJ5Py05+p4dFKAyYUCTH1VlDUrEiTTOSYyHrFIhJsvOAlPlZyntkKzMWZBWLBZYiIRoT5RSVU0xr6RCcAPLN4MAWZlfQIBsp76YzLxKC01NiZjjFkYFmyWqIqKKGsbqxlJpWlvqiLn5Q4bYDI5j0RFlGPqrJvMGLPwLNgsYZGI0FBVSV1lnP3JNJ6n5BKKYgHGGFNaLNiUgUhEaKqpXOzLMMaYQyrm5ml3iEiviOwKlV0mIi+IiCciHaHyU0XkGfd4VkQuDh07T0ReEpEuEbkpVL5eRH7hyn/odvJERCrd6y53fF2x7tEYY8yRKWbq813AeQVlu4APAU9MU96hqu9259wmIjERiQLfAM4HNgFXiMgmd84XgK+o6gZgCLjGlV8DDLnyr7h6xhhjFlHRgo2qPgEMFpTtVtWXpqk77raABkgAwVLGpwJdqvqKqqaBHwAXiogA7wXudfW2Axe55xe617jjZ7v6xhhjFknJTOoUkfeIyAvA88B1LvisAbpD1XpcWTOwPxSggnLC57jjB1z96X7nNhHpFJHOvr6++b4lY4wxTskEG1X9haqeBPwB8BkRSSzA77xdVTtUtaO1tbXYv84YY5atkstGU9XdIjIKvBPYC7SHDq91ZQNAo4jEXOslKCd0To+IxIAGV/+wdu7c2S8ir83ysluA/lmeu1TZPS8Pds/Lw1zu+bgjqVQSwUZE1gPdqpoVkeOAE4FXgf3ARnd8L3A58GeqqiLyOHAp/jjOVuB+93YPuNdPu+OPqeqM21mq6qybNiLSqaodM9csH3bPy4Pd8/KwEPdctGAjIvcAZwItItID3IKfMPA1oBV4UESeUdVzgT8EbhKRDOAB16tqv3ufG4CHgShwh6q+4H7Fp4EfiMh/Bn4NfMeVfwf4roh0ud93ebHu0RhjzJGRI/jSb2Zg34SWB7vn5cHuuThKJkFgibt9sS9gEdg9Lw92z8tD0e/ZWjbGGGOKzlo2xhhjis6CzRwdau22pUxE2kXkcRF50a1l91euvElEHhGRl93PFa5cROSr7s/gORE5ZXHvYPZEJCoivxaRn7jXZb0Gn4g0isi9IvIbEdktIqeV++csIv/J/b3eJSL3iEii3D5nmX5tyqP+XEVkq6v/sohsncs1WbCZAzn82m1LWRb4a1XdBGwB/sLd103Ao6q6EXjUvQb//je6xzbgmwt/yfPmr4DdodflvgbfPwM/U9UTgXfh33vZfs4isgb4S/y1GN+Jn+V6OeX3Od/F1LUpj+pzFZEm/Czi9+AvHXZLEKBmRVXtMcsHcBrwcOj1Z4DPLPZ1FeE+7wfeB7wErHZlq4GX3PPbgCtC9fP1ltIDf3Lwo/jr7v0EEPyJbrHCzxs/Hf809zzm6sli38NR3m8DsKfwusv5c+bgclZN7nP7CXBuOX7OwDpg12w/V+AK4LZQ+aR6R/uwls3cHGrttrLhug1+H/gF0Kaq+9yhN4E297xc/hz+CfgU/lwvmKc1+ErYeqAPuNN1Hf43EamhjD9nVd0L/FfgdWAf/ue2k/L+nANH+7nO6+dtwcYckojUAvcB/6+qDoePqf9Vp2xSGUXkAqBXVXcu9rUsoBhwCvBNVf19YIyDXStAWX7OK/BXhl8PHAPUMLW7qewtxudqwWZuDrV225InIhX4geb7qvo/XPFbIrLaHV8N9LrycvhzOAP4oIi8ir8E0nvxxzMaxV9jD6Zfgw85ijX4SkwP0KOqv3Cv78UPPuX8Of8xsEdV+1Q1A/wP/M++nD/nwNF+rvP6eVuwmZtf4dZuc9krl+OvzbakiYjgL/uzW1W/HDoUrDsHU9eju8pltWwBDoSa60uCqn5GVdeq6jr8z/ExVf0IEKzBB9OvwQdHsQZfKVHVN4FuETnBFZ0NvEgZf8743WdbRKTa/T0P7rlsP+eQo/1cHwbOEZEVrkV4jiubncUexFrqD+D9wG+B3wF/u9jXM0/39If4TezngGfc4/34fdWPAi8D/xtocvUFPyvvd/j7EXUs9j3M8f7PBH7inh8P/BLoAv4FqHTlCfe6yx0/frGve5b3+m6g033W/xNYUe6fM/BZ4Df4OwR/F6gst88ZuAd/TCqD34K9ZjafK/Axd+9dwNVzuSZbQcAYY0zRWTeaMcaYorNgY4wxpugs2BhjjCk6CzbGGGOKzoKNMcaYorNgY0wRicjoPL3P34vIJ46g3l0iculM9YxZaBZsjDHGFJ0FG2MWgIjUisijIvLvIvK8iFzoyte5vWTuEpHfisj3ReSPReRJt4fIqaG3eZeIPO3Kr3Xni4h8Xfw9lf43sDL0O/9ORH7l9m253c2YN2ZRWLAxZmFMABer6inAWcCXQv/5bwC+BJzoHn+Gv4rDJ4C/Cb3Hyfhrtp0G/J2IHANcDJyAv5/SVcDpofpfV9U/UH/flirggiLdmzEzis1cxRgzDwT4BxH5I/wtDNZwcIn3Par6PICIvIC/wZWKyPP4e5IE7lfVJJAUkcfxN7T6I+AeVc0Bb4jIY6H6Z4nIp4Bq/P1bXgD+V9Hu0JjDsGBjzML4CNAKbFbVjFtdOuGOpUL1vNBrj8n/RgvXljrkWlMikgBuxV/nqltE/j70+4xZcNaNZszCaMDfLycjImcBx83iPS4UkYSINOMvFvor4AngwyISdcvGn+XqBoGl3+1LZBlqZlFZy8aYhfF94H+5rrFO/FWHj9Zz+EvhtwCfU9U3ROTH+OM4L+Ivn/80gKruF5Fv469s/CZ+YDJm0diqz8YYY4rOutGMMcYUnQUbY4wxRWfBxhhjTNFZsDHGGFN0FmyMMcYUnQUbY4wxRWfBxhhjTNFZsDHGGFN0/z+47FRvIql9qAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# LASSO Regression - Cross-Validation with test MSE\n",
    "\n",
    "results = pd.DataFrame()\n",
    "alpha_range = np.linspace(10**-2, 10**3, num=500)\n",
    "for train_idx, test_idx in model_selection.KFold(n_splits=10).split(hitters_dat):\n",
    "    train, test = hitters_dat.iloc[train_idx], hitters_dat.iloc[test_idx]\n",
    "    errors = []\n",
    "    for alpha in alpha_range:\n",
    "        reg = linear_model.Lasso(alpha=alpha, max_iter=10000)\n",
    "        reg.fit(train.drop('Salary', axis=1), train['Salary'])\n",
    "        error = metrics.mean_squared_error(test['Salary'], reg.predict(test.drop('Salary', axis=1)))\n",
    "        errors.append(error)\n",
    "    results = pd.concat([results, pd.DataFrame(errors, index=alpha_range).T], axis=0, ignore_index=True)\n",
    "    \n",
    "\n",
    "df = pd.DataFrame({'lambda' : alpha_range, 'MSE' : results.mean()})\n",
    "sns.scatterplot(x='lambda', y='MSE', data=df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 387,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x1f0dd6160>"
      ]
     },
     "execution_count": 387,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZsAAAEKCAYAAADEovgeAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAHiBJREFUeJzt3X+0VlW97/H3J0hTOwrGFg3owFG0iFuGXKRrdTxiCOYRKi25lqgkw/yReeqW1h1RmmNo2fFkt2iYoFge0UiTTEVCO557biAbf/DDH7lDjc3lx05QK28a+r1/rLlPy+2zf61nz/0QfF5jPGOvNdecc821edifvdYz91qKCMzMzHJ6Q6MHYGZmuz6HjZmZZeewMTOz7Bw2ZmaWncPGzMyyc9iYmVl2DhszM8vOYWNmZtk5bMzMLLuBjR7AzmLIkCExcuTIRg/DzOyvyqpVq34XEU3d1XPYJCNHjqS5ubnRwzAz+6si6Zme1PNlNDMzy85hY2Zm2TlszMwsO4eNmZll57AxM7PsHDZmZpadw8bMzLJz2JiZWXb+o84O2ub+qFK7pk9/oo9HYma26/CZjZmZZeewMTOz7Bw2ZmaWncPGzMyyc9iYmVl2DhszM8vOYWNmZtk5bMzMLDuHjZmZZeewMTOz7Bw2ZmaWXbawkTRf0lZJa2ts+5ykkDQkrUvS1ZJaJK2WNK5Ud6akJ9NrZqn8CElrUpurJSmV7y9paaq/VNLgXMdoZmY9k/PM5npgSsdCSSOAycBvS8VTgdHpNRuYm+ruD8wBjgQmAHNK4TEXOKvUrn1fFwHLImI0sCytm5lZA2ULm4i4H9hWY9NVwBeAKJVNA26IwnJgkKSDgOOApRGxLSK2A0uBKWnbvhGxPCICuAGYXuprQVpeUCo3M7MG6dfPbCRNAzZGxCMdNg0DNpTWW1NZV+WtNcoBhkbEprS8GRjaxXhmS2qW1NzW1tbbwzEzsx7qt7CRtDfwJeAr/bXPdNYTXWy/JiLGR8T4pqam/hqWmdlupz/PbA4GRgGPSHoaGA48KOlAYCMwolR3eCrrqnx4jXKALekyG+nr1j4/EjMz65V+C5uIWBMRB0TEyIgYSXHpa1xEbAYWA6elWWkTgefTpbAlwGRJg9PEgMnAkrTtBUkT0yy004Db064WA+2z1maWys3MrEFyTn2+CfgVcJikVkmzuqh+J7AeaAF+AJwDEBHbgEuBlel1SSoj1bk2tfkNcFcqvxz4oKQngWPTupmZNdDAXB1HxIxuto8sLQdwbif15gPza5Q3A2NrlD8LTOrlcM3MLKNsYWP1W/e9Eyu1e+c5i/t4JGZm9XHYZLB57tcqtTvw03P6eCRmZjsH3xvNzMyyc9iYmVl2DhszM8vOYWNmZtk5bMzMLDuHjZmZZeepz9avvnPjcZXanX/qkj4eiZn1J5/ZmJlZdg4bMzPLzmFjZmbZ+TMb65GF11X7rOWUM/xZi5n5zMbMzPqBw8bMzLJz2JiZWXYOGzMzy84TBHYDy679UKV2kz718z4eiZntrrKFjaT5wAnA1ogYm8q+Cfwj8DLwG+CMiHgubbsYmAW8AnwmIpak8inAt4EBwLURcXkqHwUsBN4CrAI+GREvS9oTuAE4AngW+HhEPJ3rOK3/fX7RlMptrzzp7v9cnvrT8yv1cdf071Tev9nuKudltOuBjj8VlgJjI+JdwK+BiwEkjQFOAd6Z2nxP0gBJA4DvAlOBMcCMVBfgCuCqiDgE2E4RVKSv21P5VamemZk1ULawiYj7gW0dyu6JiB1pdTkwPC1PAxZGxEsR8RTQAkxIr5aIWB8RL1OcyUyTJOAYYFFqvwCYXuprQVpeBExK9c3MrEEa+ZnNmcDNaXkYRfi0a01lABs6lB9JcensuVJwlesPa28TETskPZ/q/67jACTNBmYDvO1tb6vzcGx39aFbv1Wp3c8/8rk+Hgmc8ONF3Veq4Y6TT+rjkZi9VkNmo0n6MrADuLER+28XEddExPiIGN/U1NTIoZiZ7dL6/cxG0ukUEwcmRUSk4o3AiFK14amMTsqfBQZJGpjObsr12/tqlTQQ2C/VNzOrywPXba3UbsIZB7xmvfXKzZX6Gf75Ayu12xn065lNmln2BeDEiHixtGkxcIqkPdMss9HAA8BKYLSkUZL2oJhEsDiF1H1A+7n/TOD2Ul8z0/JJwL2lUDMzswbIOfX5JuBoYIikVmAOxeyzPYGl6TP75RFxdkSsk3QL8CjF5bVzI+KV1M95wBKKqc/zI2Jd2sUXgYWSvg48BMxL5fOAH0pqoZigcEquYzTrKyf8ZH6ldnd89Mw+Hklh+qJfVGr305OO7eOR9I2f/vh1H9n22PSTh/ThSHZf2cImImbUKJ5Xo6y9/mXAZTXK7wTurFG+nmK2WsfyPwEn92qwZrbTuuy2TZXaffnDB/XxSHZNW7+ztFK7A87/YK/q+w4CZpbFyT9ZXandjz/6rj4eya5py1UPV2o39MLD+3gkPeN7o5mZWXYOGzMzy85hY2Zm2TlszMwsO4eNmZll57AxM7PsHDZmZpadw8bMzLJz2JiZWXYOGzMzy85hY2Zm2TlszMwsO4eNmZll57AxM7PsHDZmZpadw8bMzLJz2JiZWXYOGzMzyy5b2EiaL2mrpLWlsv0lLZX0ZPo6OJVL0tWSWiStljSu1GZmqv+kpJml8iMkrUltrpakrvZhZmaNk/PM5npgSoeyi4BlETEaWJbWAaYCo9NrNjAXiuAA5gBHAhOAOaXwmAucVWo3pZt9mJlZg2QLm4i4H9jWoXgasCAtLwCml8pviMJyYJCkg4DjgKURsS0itgNLgSlp274RsTwiArihQ1+19mFmZg3S35/ZDI2ITWl5MzA0LQ8DNpTqtaayrspba5R3tY/XkTRbUrOk5ra2tgqHY2ZmPdGwCQLpjCQauY+IuCYixkfE+KamppxDMTPbrfV32GxJl8BIX7em8o3AiFK94amsq/LhNcq72oeZmTVIf4fNYqB9RtlM4PZS+WlpVtpE4Pl0KWwJMFnS4DQxYDKwJG17QdLENAvttA591dqHmZk1yMBcHUu6CTgaGCKplWJW2eXALZJmAc8AH0vV7wSOB1qAF4EzACJim6RLgZWp3iUR0T7p4ByKGW97AXelF13sw8zMGiRb2ETEjE42TapRN4BzO+lnPjC/RnkzMLZG+bO19mFmZo3jOwiYmVl2DhszM8vOYWNmZtk5bMzMLDuHjZmZZeewMTOz7Bw2ZmaWncPGzMyyc9iYmVl2DhszM8vOYWNmZtk5bMzMLDuHjZmZZeewMTOz7Bw2ZmaWXZdhI+kTpeWjOmw7L9egzMxs19Ldmc0/lZa/02HbmX08FjMz20V1FzbqZLnWupmZWU3dhU10slxrvcckXShpnaS1km6S9CZJoyStkNQi6WZJe6S6e6b1lrR9ZKmfi1P5E5KOK5VPSWUtki6qOk4zM+sb3YXN2yWtlrSmtNy+fliVHUoaBnwGGB8RY4EBwCnAFcBVEXEIsB2YlZrMAran8qtSPSSNSe3eCUwBvidpgKQBwHeBqcAYYEaqa2ZmDTKwm+3vyLjfvST9Gdgb2AQcA/z3tH0B8FVgLjAtLQMsAv6XJKXyhRHxEvCUpBZgQqrXEhHrASQtTHUfzXQsZmbWjS7PbCLimfIL+AMwDhiS1nstIjYCVwK/pQiZ54FVwHMRsSNVawWGpeVhwIbUdkeq/5ZyeYc2nZW/jqTZkpolNbe1tVU5HDMz64Hupj7fIWlsWj4IWEsxC+2Hkj5bZYeSBlOcaYwC3grsQ3EZrN9FxDURMT4ixjc1NTViCGZmu4XuPrMZFRFr0/IZwNKI+EfgSKpPfT4WeCoi2iLiz8CtwFHAIEntl/WGAxvT8kZgBEDavh/wbLm8Q5vOys3MrEG6C5s/l5YnAXcCRMTvgVcr7vO3wERJe6fPXiZRfJ5yH3BSqjMTuD0tL07rpO33RkSk8lPSbLVRwGjgAWAlMDrNbtuDYhLB4opjNTOzPtDdBIENks6n+NxjHHA3gKS9gDdW2WFErJC0CHgQ2AE8BFwD/BxYKOnrqWxeajKP4rJdC7CNIjyIiHWSbqEIqh3AuRHxShrfecASiplu8yNiXZWxmplZ3+gubGYBl1Bc+vp4RDyXyicC11XdaUTMAeZ0KF7PX2aTlev+CTi5k34uAy6rUX4n6SzMzMwar8uwiYitwNk1yu+juOxlZmbWrS7DRlKXn3VExIl9OxwzM9sVdXcZ7b0Uf7NyE7AC3w/NzMwq6C5sDgQ+CMyg+Ov+nwM3+QN3MzPrje7uIPBKRNwdETMpJgW0AL/0s2zMzKw3ujuzQdKewIcozm5GAlcDt+UdlpmZ7Uq6myBwAzCWYhrx10p3EzAzM+ux7s5sPgH8EbgA+EzxB/9AMVEgImLfjGMzM7NdRHd/Z9Pd7WzMzMy65TAxM7PsHDZmZpadw8bMzLJz2JiZWXYOGzMzy85hY2Zm2TlszMwsO4eNmZll57AxM7PsHDZmZpZdQ8JG0iBJiyQ9LukxSe+VtL+kpZKeTF8Hp7qSdLWkFkmrJY0r9TMz1X9S0sxS+RGS1qQ2V6t0UzczM+t/jTqz+TZwd0S8HXg38BhwEbAsIkYDy9I6wFRgdHrNBuYCSNofmAMcCUwA5rQHVKpzVqndlH44JjMz60S/h42k/YAPAPMAIuLliHgOmAYsSNUWANPT8jTghigsBwZJOgg4DlgaEdsiYjuwFJiStu0bEcsjIoAbSn2ZmVkDNOLMZhTQBlwn6SFJ10raBxgaEZtSnc3A0LQ8DNhQat+ayroqb61R/jqSZktqltTc1tZW52GZmVlnGhE2A4FxwNyIeA/F83IuKldIZySReyARcU1EjI+I8U1NTbl3Z2a222pE2LQCrRGxIq0vogifLekSGOnr1rR9IzCi1H54KuuqfHiNcjMza5B+D5uI2AxskHRYKpoEPAosBtpnlM0Ebk/Li4HT0qy0icDz6XLbEmCypMFpYsBkYEna9oKkiWkW2mmlvszMrAG6eyx0LucDN0raA1gPnEERfLdImgU8A3ws1b0TOB5oAV5MdYmIbZIuBVamepdExLa0fA5wPbAXcFd6mZlZgzQkbCLiYWB8jU2TatQN4NxO+pkPzK9R3gyMrXOYZmbWR3wHATMzy85hY2Zm2TlszMwsO4eNmZll57AxM7PsHDZmZpadw8bMzLJz2JiZWXYOGzMzy85hY2Zm2TlszMwsO4eNmZll57AxM7PsHDZmZpadw8bMzLJz2JiZWXYOGzMzy85hY2Zm2TUsbCQNkPSQpDvS+ihJKyS1SLpZ0h6pfM+03pK2jyz1cXEqf0LScaXyKamsRdJF/X1sZmb2Wo08s7kAeKy0fgVwVUQcAmwHZqXyWcD2VH5VqoekMcApwDuBKcD3UoANAL4LTAXGADNSXTMza5CGhI2k4cCHgGvTuoBjgEWpygJgelqeltZJ2yel+tOAhRHxUkQ8BbQAE9KrJSLWR8TLwMJU18zMGqRRZzb/AnwBeDWtvwV4LiJ2pPVWYFhaHgZsAEjbn0/1/7O8Q5vOyl9H0mxJzZKa29ra6j0mMzPrRL+HjaQTgK0Rsaq/991RRFwTEeMjYnxTU1Ojh2Nmtssa2IB9HgWcKOl44E3AvsC3gUGSBqazl+HAxlR/IzACaJU0ENgPeLZU3q7cprNyMzNrgH4/s4mIiyNieESMpPiA/96IOBW4DzgpVZsJ3J6WF6d10vZ7IyJS+SlpttooYDTwALASGJ1mt+2R9rG4Hw7NzMw60Ygzm858EVgo6evAQ8C8VD4P+KGkFmAbRXgQEesk3QI8CuwAzo2IVwAknQcsAQYA8yNiXb8eiZmZvUZDwyYifgn8Mi2vp5hJ1rHOn4CTO2l/GXBZjfI7gTv7cKhmZlYH30HAzMyyc9iYmVl2DhszM8vOYWNmZtk5bMzMLDuHjZmZZeewMTOz7Bw2ZmaWncPGzMyyc9iYmVl2DhszM8vOYWNmZtk5bMzMLDuHjZmZZeewMTOz7Bw2ZmaWncPGzMyyc9iYmVl2DhszM8uu38NG0ghJ90l6VNI6SRek8v0lLZX0ZPo6OJVL0tWSWiStljSu1NfMVP9JSTNL5UdIWpPaXC1J/X2cZmb2F404s9kBfC4ixgATgXMljQEuApZFxGhgWVoHmAqMTq/ZwFwowgmYAxwJTADmtAdUqnNWqd2UfjguMzPrRL+HTURsiogH0/LvgceAYcA0YEGqtgCYnpanATdEYTkwSNJBwHHA0ojYFhHbgaXAlLRt34hYHhEB3FDqy8zMGqChn9lIGgm8B1gBDI2ITWnTZmBoWh4GbCg1a01lXZW31iivtf/ZkpolNbe1tdV1LGZm1rmGhY2kNwM/AT4bES+Ut6Uzksg9hoi4JiLGR8T4pqam3LszM9ttNSRsJL2RImhujIhbU/GWdAmM9HVrKt8IjCg1H57KuiofXqPczMwapBGz0QTMAx6LiH8ubVoMtM8omwncXio/Lc1Kmwg8ny63LQEmSxqcJgZMBpakbS9Impj2dVqpLzMza4CBDdjnUcAngTWSHk5lXwIuB26RNAt4BvhY2nYncDzQArwInAEQEdskXQqsTPUuiYhtafkc4HpgL+Cu9DIzswbp97CJiP8NdPZ3L5Nq1A/g3E76mg/Mr1HeDIytY5hmZtaHfAcBMzPLzmFjZmbZOWzMzCw7h42ZmWXnsDEzs+wcNmZmlp3DxszMsnPYmJlZdg4bMzPLzmFjZmbZOWzMzCw7h42ZmWXnsDEzs+wcNmZmlp3DxszMsnPYmJlZdg4bMzPLzmFjZmbZ7bJhI2mKpCcktUi6qNHjMTPbne2SYSNpAPBdYCowBpghaUxjR2VmtvvaJcMGmAC0RMT6iHgZWAhMa/CYzMx2W4qIRo+hz0k6CZgSEZ9K658EjoyI8zrUmw3MTquHAU900/UQ4Hd1Dq8v+tjZ+vFY8vazM42lr/rxWPL2059j+duIaOquo4F9MJi/WhFxDXBNT+tLao6I8fXssy/62Nn68Vjy9rMzjaWv+vFY8vazM42l3a56GW0jMKK0PjyVmZlZA+yqYbMSGC1plKQ9gFOAxQ0ek5nZbmuXvIwWETsknQcsAQYA8yNiXR903eNLbpn72Nn68Vjy9rMzjaWv+vFY8vazM40F2EUnCJiZ2c5lV72MZmZmOxGHjZmZZeew6QFJ8yVtlbS2jj5GSLpP0qOS1km6oGI/b5L0gKRHUj9fq2NMAyQ9JOmOOvp4WtIaSQ9Laq7YxyBJiyQ9LukxSe+t0MdhaQztrxckfbbieC5M39u1km6S9KYKfVyQ2q/rzThqvdck7S9pqaQn09fBFfs5OY3nVUndTmftpI9vpn+n1ZJukzSoYj+Xpj4elnSPpLdW6ae07XOSQtKQCmP5qqSNpffO8VXHIun89P1ZJ+kbFcZyc2kcT0t6uMpYJB0uaXn7/0tJEyr2825Jv0r/x38mad/u+ulURPjVzQv4ADAOWFtHHwcB49Ly3wC/BsZU6EfAm9PyG4EVwMSKY/on4F+BO+o4rqeBIXV+fxcAn0rLewCD6uxvALCZ4o/Nett2GPAUsFdavwU4vZd9jAXWAntTTML5BXBI1fca8A3gorR8EXBFxX7eQfHHy78ExlfsYzIwMC1fUcdY9i0tfwb4fpV+UvkIislAz3T3XuxkLF8FPt/Lf+Na/fxD+rfeM60fUOV4Stu/BXyl4ljuAaam5eOBX1bsZyXw92n5TODS3nyfyi+f2fRARNwPbKuzj00R8WBa/j3wGMUPtt72ExHxh7T6xvTq9SwPScOBDwHX9rZtX5K0H8WbfB5ARLwcEc/V2e0k4DcR8UzF9gOBvSQNpAiM/9vL9u8AVkTEixGxA/g34CM9adjJe20aRSCTvk6v0k9EPBYR3d0lo7s+7knHBLCc4m/YqvTzQml1H3rwHu7i/+FVwBfq7KNXOunn08DlEfFSqrO16lgkCfgYcFPFsQTQfhayHz14D3fSz6HA/Wl5KfDR7vrpjMOmASSNBN5DcVZSpf2AdHq9FVgaEVX6+ReK/6CvVhlDSQD3SFql4vY/vTUKaAOuS5f0rpW0T51jOoUe/CetJSI2AlcCvwU2Ac9HxD297GYt8H5Jb5G0N8VvliO6adOVoRGxKS1vBobW0VdfOhO4q2pjSZdJ2gCcCnylYh/TgI0R8UjVcSTnpct683tymbITh1L8u6+Q9G+S/msd43k/sCUinqzY/rPAN9P390rg4or9rOMv95U8mTrexw6bfibpzcBPgM92+O2uxyLilYg4nOK3ygmSxvZyDCcAWyNiVZX9d/C+iBhHcYftcyV9oJftB1Kcus+NiPcAf6S4VFSJij/iPRH4ccX2gyn+c40C3grsI+kTvekjIh6juMR0D3A38DDwSpXx1Og7qHAm29ckfRnYAdxYtY+I+HJEjEh9nNdd/Rpj2Bv4EhWDqmQucDBwOMUvGN+q2M9AYH9gIvA/gFvSGUoVM6j4C1PyaeDC9P29kHTloIIzgXMkraK4/P9y1QE5bPqRpDdSBM2NEXFrvf2ly033AVN62fQo4ERJT1PcEfsYST+qOIaN6etW4DaKO273RivQWjo7W0QRPlVNBR6MiC0V2x8LPBURbRHxZ+BW4L/1tpOImBcRR0TEB4DtFJ/RVbVF0kEA6WuXl2dyk3Q6cAJwagq/et1ItcszB1P8UvBIei8PBx6UdGBvOomILekXuFeBH9D793C7VuDWdKn7AYqrBl1OWKglXb79CHBzxXEAzKR470Lxi1elY4qIxyNickQcQRF+v6k6IIdNP0m/4cwDHouIf66jn6b2GUCS9gI+CDzemz4i4uKIGB4RIykuOd0bEb367T3tfx9Jf9O+TPHhca9m7EXEZmCDpMNS0STg0d6OpaTe3wh/C0yUtHf6N5tE8flar0g6IH19G8UPjn+tY0yLKX54kL7eXkdfdZE0heLy64kR8WId/YwurU6jl+9hgIhYExEHRMTI9F5upZiEs7mXYzmotPphevkeLvkpxSQBJB1KMdmlyp2XjwUej4jWiuOA4jOav0/LxwCVLseV3sdvAP4n8P3KI6o6s2B3elH88NoE/JniDT2rQh/vo7j8sZrissrDwPEV+nkX8FDqZy09mK3STX9HU3E2GvB3wCPptQ74csV+Dgea0zH9FBhcsZ99gGeB/er8nnyN4offWuCHpNlFvezj3ylC8xFgUj3vNeAtwDKKHxi/APav2M+H0/JLwBZgSYU+WoANpfdwT2aR1ernJ+n7uxr4GTCsSj8dtj9N97PRao3lh8CaNJbFwEEVj2kP4EfpuB4EjqlyPMD1wNl1vmfeB6xK778VwBEV+7mA4qz818DlpLvOVHn5djVmZpadL6OZmVl2DhszM8vOYWNmZtk5bMzMLDuHjZmZZeewMbPXkHS0pF7/IatZVxw2ZtbR0VS4a4JZVxw2ZjVIGqni2To/SM8muSfdsaFW3UMk/ULFM4YelHSwCt9U8UybNZI+nuoenW7SeLuk9ZIul3SqimcUrZF0cKp3vaTvp2eR/Drdz679eUbXpboPSWr/i/XTJd0q6W4Vz735Rml8k9MzSR6U9ON0f772ZxF9LZWvkfT2dJPYs4ELVTwL5f0qnoOzNh3f/ZhVMLDRAzDbiY0GZkTEWZJuobh/V617yN1IcWv521Q8aO0NFLeoORx4N8X9sVaWflC/m+IxBNuA9cC1ETFBxQP1zqe4Yy/ASIp7Wh0M3CfpEOBcivtx/hdJb6e44/ahqf7hFHcTfwl4QtJ3gP9HcZuRYyPij5K+SPEco0tSm99FxDhJ51A80+VTkr4P/CEirgSQtAY4LiI2qgcPSzOrxWc2Zp17KiLan5S4iuKH/2uke8MNi4jbACLiT1HcM+x9wE1R3OBxC8UzbdpvOb8yiucbvURxY8P2Rxis6bCPWyLi1ShuM78eeHvq90dpX49TPDCsPWyWRcTzEfEnilvl/C3FHYjHAP+h4rEUM1N5u/abNdY8vuQ/gOslnUXxYDqzXvOZjVnnXiotvwLUvIxWZ7+vltZf5bX/JzveS6q7e0t1HO9Aiie7Lo2IGd20aa//OhFxtqQjKR62t0rSERHxbDdjMXsNn9mY1SGKp662SpoOIGnP9JyVfwc+ruJBd00UTyN9oJfdnyzpDelznL8Dnkj9npr2dSjwtlTemeXAUekSXPudug/toj7A7ymeXUJqc3BErIiIr1A86K6eB8HZbsphY1a/TwKfkbQa+D/AgRTP9llNcdfde4EvRC9vfU/xuIMHKJ6GeXa6PPY94A3pc5SbgdPT5biaIqINOB24KY3vVxSX47ryM+DD7RMEKJ74uEbS2nR89T4V03ZDvuuz2U5I0vUUj35Y1OixmPUFn9mYmVl2PrMx6yFJ36V4pHbZtyPiukaMx+yvicPGzMyy82U0MzPLzmFjZmbZOWzMzCw7h42ZmWXnsDEzs+z+PwyEwlV0jjc3AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# PCR Regression - Cross-Validation with test MSE\n",
    "\n",
    "results = pd.DataFrame()\n",
    "predictors = hitters_dat.drop('Salary', axis=1).columns\n",
    "c_range = range(1, len(predictors) + 1)\n",
    "for train_idx, test_idx in model_selection.KFold(n_splits=10).split(hitters_dat):\n",
    "    train, test = hitters_dat.iloc[train_idx], hitters_dat.iloc[test_idx]\n",
    "    errors = []\n",
    "    for c in c_range:\n",
    "        pca = decomposition.PCA(n_components=c)\n",
    "        X = pca.fit_transform(train.drop('Salary', axis=1))\n",
    "        reg = linear_model.LinearRegression()\n",
    "        reg.fit(X, train['Salary'])\n",
    "        \n",
    "        test_X = pca.transform(test.drop('Salary', axis=1))\n",
    "        error = metrics.mean_squared_error(test['Salary'], reg.predict(test_X))\n",
    "        #error = metrics.mean_squared_error(test['Salary'], reg.predict(test_X))\n",
    "        errors.append(error)\n",
    "    results = pd.concat([results, pd.DataFrame(errors, index=c_range).T], axis=0, ignore_index=True)\n",
    "\n",
    "df = pd.DataFrame({'n_components' : c_range, 'MSE' : results.mean()})\n",
    "sns.barplot(x='n_components', y='MSE', data=df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 388,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x1e7ba3128>"
      ]
     },
     "execution_count": 388,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZsAAAEKCAYAAADEovgeAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAHIlJREFUeJzt3XuYHVWd7vHvSwIIOECA5mISJxGCGhhFyIR4UIdDEBJkSFRQGJQAkTzIndGjIOcREXkeUBxGOBCfDAn3IYRwixgIkYvMmWNCEi65EJA2XNI5uTQkgCMHMPA7f9Rq3TS7e3fX7tU7dN7P8+ynq1atWrWqs9NvV+3VqxQRmJmZ5bRFoztgZmZ9n8PGzMyyc9iYmVl2DhszM8vOYWNmZtk5bMzMLDuHjZmZZeewMTOz7Bw2ZmaWXf9Gd2BTscsuu8SQIUMa3Q0zsw+URYsWvRwRTbXqOWySIUOGsHDhwkZ3w8zsA0XSi12p59toZmaWncPGzMyyc9iYmVl2DhszM8vOYWNmZtk5bMzMLDuHjZmZZeewMTOz7PxHne20Tr651H5N3/5GD/fEzKzvyBY2kqYBRwLrImLfVPYz4B+Bt4E/ACdFxKtp2/nAROAd4KyImJPKxwC/APoB10bEpal8KDAd2BlYBHwzIt6WtDVwI3AA8Arw9Yh4Idd5bi6mX3d4qf2OPWlOD/fEzD6Ict5Gux4Y065sLrBvRHwK+D1wPoCk4cCxwD5pn2sk9ZPUD7gaGAsMB45LdQEuA66IiL2ADRRBRfq6IZVfkeqZmVkDZbuyiYhHJQ1pV/ZAxeo84Oi0PA6YHhFvAc9LagZGpm3NEbECQNJ0YJyk5cAhwD+lOjcAPwImp7Z+lMpnAv9LkiIieuzkalgz+aJS++3+7Qt7uCebnqtuKXeFdObxvkIy+yBr5Gc2JwO3peWBFOHTpiWVAaxsV34gxa2zVyNiY5X6A9v2iYiNkl5L9V9u3wFJk4BJAB/96EfrPJ2et+yao0rtt89ps3q4J2Zm9WlI2Ei6ANgI3NKI47eJiCnAFIARI0b02pVPb3vw2i+V2m/0t37dwz0xs81Vr4eNpBMpBg6Mrri1tQoYXFFtUCqjg/JXgB0l9U9XN5X129pqkdQf2CHVtz7iuzPbfxTYdZcffX8P9sTMuqpX/84mjSz7HnBURLxRsWkWcKykrdMos2HAY8ACYJikoZK2ohhEMCuF1MP89TOfCcA9FW1NSMtHAw/15uc1Zmb2fjmHPt8KHAzsIqkFuJBi9NnWwFxJAPMi4tSIWCZpBvA0xe210yPindTOGcAciqHP0yJiWTrE94Hpkn4CPAFMTeVTgZvSIIP1FAFlZmYNlHM02nFViqdWKWurfwlwSZXy2cDsKuUr+OuItcryN4FjutVZMzPLyjMImFkWx9yxuNR+t3/1Uz3cE+vMuqvmltpv1zO/2K36DhuzOn3pzp+X2u/XX/nOX5aPvGNaqTbu/erJ71k/8vaZ5do55uj3rI+f+ZtS7dx99KGl9svt7tvf95cPXTb+mF3+svzYdetKtTHypF1LH7+vcNjYZmvs3WeW2u++8Vf1cE9sc9Ny+ZpS+w367u5/WV57xZOl2tjt3P1K7Vcvh42ZbdIuuWt1qf0u+PIePdwTq4cfMWBmZtk5bMzMLDuHjZmZZeewMTOz7Bw2ZmaWncPGzMyyc9iYmVl2DhszM8vOYWNmZtk5bMzMLDuHjZmZZeewMTOz7Bw2ZmaWncPGzMyyc9iYmVl2DhszM8vOYWNmZtk5bMzMLDuHjZmZZeewMTOz7Bw2ZmaWXbawkTRN0jpJSyvKdpI0V9Jz6euAVC5JV0pqlrRY0v4V+0xI9Z+TNKGi/ABJS9I+V0pSZ8cwM7PGyXllcz0wpl3ZecCDETEMeDCtA4wFhqXXJGAyFMEBXAgcCIwELqwIj8nAKRX7jalxDDMza5BsYRMRjwLr2xWPA25IyzcA4yvKb4zCPGBHSXsAhwNzI2J9RGwA5gJj0rbtI2JeRARwY7u2qh3DzMwapLc/s9ktIlan5TXAbml5ILCyol5LKuusvKVKeWfHeB9JkyQtlLSwtbW1xOmYmVlXNGyAQLoiiUYeIyKmRMSIiBjR1NSUsytmZpu13g6btekWGOnrulS+ChhcUW9QKuusfFCV8s6OYWZmDdLbYTMLaBtRNgG4p6L8hDQqbRTwWroVNgc4TNKANDDgMGBO2va6pFFpFNoJ7dqqdgwzM2uQ/rkalnQrcDCwi6QWilFllwIzJE0EXgS+lqrPBo4AmoE3gJMAImK9pIuBBanejyOibdDBaRQj3rYB7ksvOjmGmZk1SLawiYjjOtg0ukrdAE7voJ1pwLQq5QuBfauUv1LtGGZm1jieQcDMzLJz2JiZWXYOGzMzy85hY2Zm2TlszMwsO4eNmZll57AxM7PsHDZmZpadw8bMzLJz2JiZWXYOGzMzy85hY2Zm2TlszMwsO4eNmZll57AxM7PsHDZmZpadw8bMzLJz2JiZWXYOGzMzy85hY2Zm2TlszMwsO4eNmZll57AxM7PsHDZmZpadw8bMzLJrSNhIOlfSMklLJd0q6UOShkqaL6lZ0m2Stkp1t07rzWn7kIp2zk/lz0o6vKJ8TCprlnRe75+hmZlV6vWwkTQQOAsYERH7Av2AY4HLgCsiYi9gAzAx7TIR2JDKr0j1kDQ87bcPMAa4RlI/Sf2Aq4GxwHDguFTXzMwapFG30foD20jqD2wLrAYOAWam7TcA49PyuLRO2j5aklL59Ih4KyKeB5qBkenVHBErIuJtYHqqa2ZmDdLrYRMRq4DLgZcoQuY1YBHwakRsTNVagIFpeSCwMu27MdXfubK83T4dlb+PpEmSFkpa2NraWv/JmZlZVY24jTaA4kpjKPARYDuK22C9LiKmRMSIiBjR1NTUiC6YmW0WGnEb7VDg+YhojYg/A3cCBwE7pttqAIOAVWl5FTAYIG3fAXilsrzdPh2Vm5lZgzQibF4CRknaNn32Mhp4GngYODrVmQDck5ZnpXXS9ociIlL5sWm02lBgGPAYsAAYlka3bUUxiGBWL5yXmZl1oH/tKj0rIuZLmgk8DmwEngCmAL8Gpkv6SSqbmnaZCtwkqRlYTxEeRMQySTMogmojcHpEvAMg6QxgDsVIt2kRsay3zs/MzN6v18MGICIuBC5sV7yCYiRZ+7pvAsd00M4lwCVVymcDs+vvqZmZ9QTPIGBmZtk5bMzMLDuHjZmZZeewMTOz7Bw2ZmaWncPGzMyyc9iYmVl2DhszM8vOYWNmZtk5bMzMLDuHjZmZZeewMTOz7Bw2ZmaWXadhI+kbFcsHtdt2Rq5OmZlZ31LryuafK5avarft5B7ui5mZ9VG1wkYdLFdbNzMzq6pW2EQHy9XWzczMqqr1pM5PSFpMcRWzZ1omrX8sa8/MzKzPqBU2n+yVXpiZWZ/WadhExIuV65J2Br4AvBQRi3J2zMzM+o5aQ5/vlbRvWt4DWEoxCu0mSef0Qv/MzKwPqDVAYGhELE3LJwFzI+IfgQPx0GczM+uiWmHz54rl0cBsgIj4I/Burk6ZmVnfUmuAwEpJZwItwP7A/QCStgG2zNw3MzPrI2pd2UwE9gFOBL4eEa+m8lHAdRn7ZWZmfUinYRMR6yLi1IgYFxEPVJQ/HBGXlz2opB0lzZT0jKTlkj4raSdJcyU9l74OSHUl6UpJzZIWS9q/op0Jqf5zkiZUlB8gaUna50pJnu3AzKyBOr2NJmlWZ9sj4qiSx/0FcH9EHC1pK2Bb4AfAgxFxqaTzgPOA7wNjgWHpdSAwGThQ0k7AhcAIitkMFkmaFREbUp1TgPkUnzONAe4r2VczM6tTrc9sPgusBG6l+MFd9xWCpB0o/lbnRICIeBt4W9I44OBU7QbgEYqwGQfcGBEBzEtXRXukunMjYn1qdy4wRtIjwPYRMS+V3wiMx2FjZtYwtT6z2Z3iimNfiquRLwIvR8RvI+K3JY85FGgFrpP0hKRrJW0H7BYRq1OdNcBuaXkgReC1aUllnZW3VCl/H0mTJC2UtLC1tbXk6ZiZWS21PrN5JyLuj4gJFIMCmoFH6nyWTX+KkW2TI+IzwJ8obplVHjfohYk+I2JKRIyIiBFNTU25D2dmttmq+aROSVtL+gpwM3A6cCVwVx3HbAFaImJ+Wp9JET5r0+2xttkK1qXtq4DBFfsPSmWdlQ+qUm5mZg1Sa7qaG4HfUYTBRRHx9xFxcUSU/uEdEWso/n7n46loNPA0MAtoG1E2AbgnLc8CTkij0kYBr6XbbXOAwyQNSCPXDgPmpG2vSxqVRqGdUNGWmZk1QK0BAt+guM11NnBWxQhiUdzt2r7kcc8Ebkkj0VZQTIWzBTBD0kTgReBrqe5s4AiKW3hvpLpExHpJFwMLUr0ftw0WAE4Drge2oRgY4MEBZmYNVGvW55q32cqIiCcphiy3N7pK3aC4fVetnWnAtCrlCykGNZiZ2SYgS5iYmZlVctiYmVl2DhszM8vOYWNmZtk5bMzMLDuHjZmZZeewMTOz7Bw2ZmaWncPGzMyyc9iYmVl2DhszM8vOYWNmZtk5bMzMLDuHjZmZZeewMTOz7Bw2ZmaWncPGzMyyc9iYmVl2DhszM8vOYWNmZtk5bMzMLDuHjZmZZeewMTOz7Bw2ZmaWXcPCRlI/SU9IujetD5U0X1KzpNskbZXKt07rzWn7kIo2zk/lz0o6vKJ8TCprlnReb5+bmZm9VyOvbM4GllesXwZcERF7ARuAial8IrAhlV+R6iFpOHAssA8wBrgmBVg/4GpgLDAcOC7VNTOzBmlI2EgaBHwJuDatCzgEmJmq3ACMT8vj0jpp++hUfxwwPSLeiojngWZgZHo1R8SKiHgbmJ7qmplZgzTqyuZfge8B76b1nYFXI2JjWm8BBqblgcBKgLT9tVT/L+Xt9umo3MzMGqTXw0bSkcC6iFjU28eu0pdJkhZKWtja2tro7piZ9VmNuLI5CDhK0gsUt7gOAX4B7Cipf6ozCFiVllcBgwHS9h2AVyrL2+3TUfn7RMSUiBgRESOamprqPzMzM6uq18MmIs6PiEERMYTiA/6HIuJ44GHg6FRtAnBPWp6V1knbH4qISOXHptFqQ4FhwGPAAmBYGt22VTrGrF44NTMz60D/2lV6zfeB6ZJ+AjwBTE3lU4GbJDUD6ynCg4hYJmkG8DSwETg9It4BkHQGMAfoB0yLiGW9eiZmZvYeDQ2biHgEeCQtr6AYSda+zpvAMR3sfwlwSZXy2cDsHuyqmZnVwTMImJlZdg4bMzPLzmFjZmbZOWzMzCw7h42ZmWXnsDEzs+wcNmZmlp3DxszMsnPYmJlZdg4bMzPLzmFjZmbZOWzMzCw7h42ZmWXnsDEzs+wcNmZmlp3DxszMsnPYmJlZdg4bMzPLzmFjZmbZOWzMzCw7h42ZmWXnsDEzs+wcNmZmlp3DxszMsnPYmJlZdr0eNpIGS3pY0tOSlkk6O5XvJGmupOfS1wGpXJKulNQsabGk/SvampDqPydpQkX5AZKWpH2ulKTePk8zM/urRlzZbAS+ExHDgVHA6ZKGA+cBD0bEMODBtA4wFhiWXpOAyVCEE3AhcCAwEriwLaBSnVMq9hvTC+dlZmYd6PWwiYjVEfF4Wv4jsBwYCIwDbkjVbgDGp+VxwI1RmAfsKGkP4HBgbkSsj4gNwFxgTNq2fUTMi4gAbqxoy8zMGqChn9lIGgJ8BpgP7BYRq9OmNcBuaXkgsLJit5ZU1ll5S5VyMzNrkIaFjaQPA3cA50TE65Xb0hVJ9EIfJklaKGlha2tr7sOZmW22GhI2krakCJpbIuLOVLw23QIjfV2XylcBgyt2H5TKOisfVKX8fSJiSkSMiIgRTU1N9Z2UmZl1qBGj0QRMBZZHxL9UbJoFtI0omwDcU1F+QhqVNgp4Ld1umwMcJmlAGhhwGDAnbXtd0qh0rBMq2jIzswbo34BjHgR8E1gi6clU9gPgUmCGpInAi8DX0rbZwBFAM/AGcBJARKyXdDGwINX7cUSsT8unAdcD2wD3pZeZmTVIr4dNRPxvoKO/exldpX4Ap3fQ1jRgWpXyhcC+dXTTzMx6kGcQMDOz7Bw2ZmaWncPGzMyyc9iYmVl2DhszM8vOYWNmZtk5bMzMLDuHjZmZZeewMTOz7Bw2ZmaWncPGzMyyc9iYmVl2DhszM8vOYWNmZtk5bMzMLDuHjZmZZeewMTOz7Bw2ZmaWncPGzMyyc9iYmVl2DhszM8vOYWNmZtk5bMzMLDuHjZmZZeewMTOz7Pps2EgaI+lZSc2Szmt0f8zMNmd9Mmwk9QOuBsYCw4HjJA1vbK/MzDZffTJsgJFAc0SsiIi3genAuAb3ycxss6WIaHQfepyko4ExEfGttP5N4MCIOKNdvUnApLT6ceDZGk3vArxcZ/d6oo1NrR33JW87m1Jfeqod9yVvO73Zl7+NiKZaDfXvgc58YEXEFGBKV+tLWhgRI+o5Zk+0sam1477kbWdT6ktPteO+5G1nU+pLm756G20VMLhifVAqMzOzBuirYbMAGCZpqKStgGOBWQ3uk5nZZqtP3kaLiI2SzgDmAP2AaRGxrAea7vItt8xtbGrtuC9529mU+tJT7bgvedvZlPoC9NEBAmZmtmnpq7fRzMxsE+KwMTOz7Bw2XSBpmqR1kpbW0cZgSQ9LelrSMklnl2znQ5Iek/RUaueiOvrUT9ITku6to40XJC2R9KSkhSXb2FHSTEnPSFou6bMl2vh46kPb63VJ55Tsz7npe7tU0q2SPlSijbPT/su6049q7zVJO0maK+m59HVAyXaOSf15V1LN4awdtPGz9O+0WNJdknYs2c7FqY0nJT0g6SNl2qnY9h1JIWmXEn35kaRVFe+dI8r2RdKZ6fuzTNJPS/Tltop+vCDpyTJ9kbSfpHlt/y8ljSzZzqcl/S79H/+VpO1rtdOhiPCrxgv4ArA/sLSONvYA9k/LfwP8Hhheoh0BH07LWwLzgVEl+/TPwL8D99ZxXi8Au9T5/b0B+FZa3grYsc72+gFrKP7YrLv7DgSeB7ZJ6zOAE7vZxr7AUmBbikE4vwH2KvteA34KnJeWzwMuK9nOJyn+ePkRYETJNg4D+qfly+roy/YVy2cBvyzTTiofTDEY6MVa78UO+vIj4Lvd/Deu1s5/T//WW6f1XcucT8X2nwM/LNmXB4CxafkI4JGS7SwA/iEtnwxc3J3vU+XLVzZdEBGPAuvrbGN1RDyelv8ILKf4wdbddiIi/iutbple3R7lIWkQ8CXg2u7u25Mk7UDxJp8KEBFvR8SrdTY7GvhDRLxYcv/+wDaS+lMExv/t5v6fBOZHxBsRsRH4LfCVruzYwXttHEUgk76OL9NORCyPiFqzZNRq44F0TgDzKP6GrUw7r1esbkcX3sOd/D+8AvhenW10SwftfBu4NCLeSnXWle2LJAFfA24t2ZcA2q5CdqAL7+EO2tkbeDQtzwW+WqudjjhsGkDSEOAzFFclZfbvly6v1wFzI6JMO/9K8R/03TJ9qBDAA5IWqZj+p7uGAq3AdemW3rWStquzT8fShf+k1UTEKuBy4CVgNfBaRDzQzWaWAp+XtLOkbSl+sxxcY5/O7BYRq9PyGmC3OtrqSScD95XdWdIlklYCxwM/LNnGOGBVRDxVth/JGem23rSu3KbswN4U/+7zJf1W0t/X0Z/PA2sj4rmS+58D/Cx9fy8Hzi/ZzjL+Oq/kMdTxPnbY9DJJHwbuAM5p99tdl0XEOxGxH8VvlSMl7dvNPhwJrIuIRWWO387nImJ/ihm2T5f0hW7u35/i0n1yRHwG+BPFraJSVPwR71HA7SX3H0Dxn2so8BFgO0nf6E4bEbGc4hbTA8D9wJPAO2X6U6XtoMSVbE+TdAGwEbilbBsRcUFEDE5tnFGrfpU+bAv8gJJBVWEysCewH8UvGD8v2U5/YCdgFPA/gBnpCqWM4yj5C1PybeDc9P09l3TnoISTgdMkLaK4/f922Q45bHqRpC0pguaWiLiz3vbS7aaHgTHd3PUg4ChJL1DMiH2IpJtL9mFV+roOuItixu3uaAFaKq7OZlKET1ljgccjYm3J/Q8Fno+I1oj4M3An8N+620hETI2IAyLiC8AGis/oyloraQ+A9LXT2zO5SToROBI4PoVfvW6h3O2ZPSl+KXgqvZcHAY9L2r07jUTE2vQL3LvAv9H993CbFuDOdKv7MYq7Bp0OWKgm3b79CnBbyX4ATKB470Lxi1epc4qIZyLisIg4gCL8/lC2Qw6bXpJ+w5kKLI+If6mjnaa2EUCStgG+CDzTnTYi4vyIGBQRQyhuOT0UEd367T0dfztJf9O2TPHhcbdG7EXEGmClpI+notHA093tS4V6fyN8CRgladv0bzaa4vO1bpG0a/r6UYofHP9eR59mUfzwIH29p4626iJpDMXt16Mi4o062hlWsTqObr6HASJiSUTsGhFD0nu5hWIQzppu9mWPitUv0833cIW7KQYJIGlvisEuZWZePhR4JiJaSvYDis9o/iEtHwKUuh1X8T7eAvifwC9L96jsyILN6UXxw2s18GeKN/TEEm18juL2x2KK2ypPAkeUaOdTwBOpnaV0YbRKjfYOpuRoNOBjwFPptQy4oGQ7+wEL0zndDQwo2c52wCvADnV+Ty6i+OG3FLiJNLqom238B0VoPgWMrue9BuwMPEjxA+M3wE4l2/lyWn4LWAvMKdFGM7Cy4j3clVFk1dq5I31/FwO/AgaWaafd9heoPRqtWl9uApakvswC9ih5TlsBN6fzehw4pMz5ANcDp9b5nvkcsCi9/+YDB5Rs52yKq/LfA5eSZp0p8/J0NWZmlp1vo5mZWXYOGzMzy85hY2Zm2TlszMwsO4eNmZll57Axs/eQdLCkbv8hq1lnHDZm1t7BlJg1wawzDhuzKiQNUfFsnX9LzyZ5IM3YUK3uXpJ+o+IZQ49L2lOFn6l4ps0SSV9PdQ9OkzTeI2mFpEslHa/iGUVLJO2Z6l0v6ZfpWSS/T/PZtT3P6LpU9wlJbX+xfqKkOyXdr+K5Nz+t6N9h6Zkkj0u6Pc3P1/YsootS+RJJn0iTxJ4KnKviWSifV/EcnKXp/B7FrIT+je6A2SZsGHBcRJwiaQbF/F3V5pC7hWJq+btUPGhtC4opavYDPk0xP9aCih/Un6Z4DMF6YAVwbUSMVPFAvTMpZuwFGEIxp9WewMOS9gJOp5iP8+8kfYJixu29U/39KGYTfwt4VtJVwP+jmGbk0Ij4k6TvUzzH6Mdpn5cjYn9Jp1E80+Vbkn4J/FdEXA4gaQlweESsUhcelmZWja9szDr2fES0PSlxEcUP//dIc8MNjIi7ACLizSjmDPsccGsUEzyupXimTduU8wuieL7RWxQTG7Y9wmBJu2PMiIh3o5hmfgXwidTuzelYz1A8MKwtbB6MiNci4k2KqXL+lmIG4uHAf6p4LMWEVN6mbbLGqueX/CdwvaRTKB5MZ9ZtvrIx69hbFcvvAFVvo9XZ7rsV6+/y3v+T7eeSqjW3VPv+9qd4suvciDiuxj5t9d8nIk6VdCDFw/YWSTogIl6p0Rez9/CVjVkdonjqaouk8QCStk7PWfkP4OsqHnTXRPE00se62fwxkrZIn+N8DHg2tXt8OtbewEdTeUfmAQelW3BtM3Xv3Ul9gD9SPLuEtM+eETE/In5I8aC7eh4EZ5sph41Z/b4JnCVpMfB/gN0pnu2zmGLW3YeA70U3p76neNzBYxRPwzw13R67BtgifY5yG3Biuh1XVUS0AicCt6b+/Y7idlxnfgV8uW2AAMUTH5dIWprOr96nYtpmyLM+m22CJF1P8eiHmY3ui1lP8JWNmZll5ysbsy6SdDXFI7Ur/SIirmtEf8w+SBw2ZmaWnW+jmZlZdg4bMzPLzmFjZmbZOWzMzCw7h42ZmWX3/wHTwP9U+vjIjgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# PLS Regression - Cross-Validation with test MSE\n",
    "\n",
    "results = pd.DataFrame()\n",
    "predictors = hitters_dat.drop('Salary', axis=1).columns\n",
    "c_range = range(1, len(predictors) + 1)\n",
    "for train_idx, test_idx in model_selection.KFold(n_splits=10).split(hitters_dat):\n",
    "    train, test = hitters_dat.iloc[train_idx], hitters_dat.iloc[test_idx]\n",
    "    errors = []\n",
    "    for c in c_range:\n",
    "        reg = cross_decomposition.PLSRegression(n_components=c)\n",
    "        reg.fit(train.drop('Salary', axis=1), train['Salary'])\n",
    "        error = metrics.mean_squared_error(test['Salary'], reg.predict(test.drop('Salary', axis=1)))\n",
    "        errors.append(error)\n",
    "    results = pd.concat([results, pd.DataFrame(errors, index=c_range).T], axis=0, ignore_index=True)\n",
    "\n",
    "df = pd.DataFrame({'n_components' : c_range, 'MSE' : results.mean()})\n",
    "sns.barplot(x='n_components', y='MSE', data=df)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}