{
"cells": [
{
"cell_type": "code",
"execution_count": 402,
"metadata": {},
"outputs": [],
"source": [
"import pandas as pd\n",
"import statsmodels.formula.api as smf\n",
"import matplotlib.pyplot as plt\n",
"import seaborn as sns\n",
"from sklearn import model_selection, metrics, linear_model, decomposition, cross_decomposition, datasets\n",
"from itertools import combinations\n",
"import numpy as np\n",
"import time "
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"8. In this exercise, we will generate simulated data, and will then use this data to perform best subset selection.\n",
"\n",
"\n",
"(a) Use the rnorm() function to generate a predictor X of length n = 100, as well as a noise vector ε of length n = 100."
]
},
{
"cell_type": "code",
"execution_count": 167,
"metadata": {},
"outputs": [],
"source": [
"x = np.random.normal(0,1, 100)\n",
"eps = np.random.normal(0,1, 100)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"(b) Generate a response vector Y of length n = 100 according to the model\n",
"Y = β0 +β1X +β2X2 +β3X3 +ε, where β0, β1, β2, and β3 are constants of your choice."
]
},
{
"cell_type": "code",
"execution_count": 168,
"metadata": {},
"outputs": [],
"source": [
"coeffs = np.array([1,2,3,4])\n",
"design_matrix = pd.DataFrame({ 'c' : np.repeat(1, 100),\n",
" 'x' : x, \\\n",
" 'x2' : x ** 2, \\\n",
" 'x3' : x ** 3})\n",
"y = (design_matrix @ coeffs.reshape(-1, 1)) + eps.reshape(-1, 1)"
]
},
{
"cell_type": "code",
"execution_count": 177,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"<matplotlib.axes._subplots.AxesSubplot at 0x11905d630>"
]
},
"execution_count": 177,
"metadata": {},
"output_type": "execute_result"
},
{
"data": {
"image/png": "iVBORw0KGgoAAAANSUhEUgAAAXwAAAD8CAYAAAB0IB+mAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAH0RJREFUeJzt3X903HWd7/Hne2YyyTRNm5CmpTbF9rrYa6wt0Ihoz3VZuyoqUoGCKFB+bQtWV9f1B3CRg3tYz4JF0T1uhVZ+VavCLfTAcS9CF+F6ZOVHClihpVBBbLA/QkjbNJ1mksz7/pGZ6Uw606aZSSbN9/U4h9OZ73xnvp+B8son7+/nh7k7IiIy9oXK3QARERkZCnwRkYBQ4IuIBIQCX0QkIBT4IiIBocAXEQkIBb6ISEAo8EVEAkKBLyISEJFyNyDbpEmTfMaMGeVuhojIMWXDhg1vuXvDkc4bVYE/Y8YMWlpayt0MEZFjipm9MZjzVNIREQkIBb6ISEAo8EVEAkKBLyISEAp8EZGAGFWjdEREgiaZdNq7EiR6+4hGwtRXRwmFbFiupcAXESmTZNLZsrOTJatbaO2I01gXY9XiZmZNqRmW0FdJR0SkTNq7EpmwB2jtiLNkdQvtXYlhuZ4CX0SkTBK9fZmwT2vtiJPo7RuW6ynwRUTKJBoJ01gXyznWWBcjGgkPy/UU+CIiZVJfHWXV4uZM6Kdr+PXV0WG5nm7aioiUSShkzJpSw7pl8zVKR0RkrAuFjIaaypG51ohcRUREyk6BLyISEAp8EZGAUOCLiASEAl9EJCAU+CIiATHowDezO81sl5m9mHVsuZm9bGYbzWydmdVmvXatmW01sy1m9vFSN1xERI7O0fTw7wbOGHBsPTDb3ecArwDXAphZE3AB8N7Ue1aY2fDMFRYRkUEZdOC7+2+Btwcce9Tde1NPnwIaU48XAr909253fx3YCpxagvaKiMgQlbKGfznwcOrxNGBb1mutqWMiIlImJQl8M7sO6AXWDOG9S82sxcxa2traStEcERHJo+jAN7NLgTOBC93dU4ffBKZnndaYOnYId1/p7s3u3tzQ0FBsc0REpICiAt/MzgC+CZzl7vuzXnoIuMDMKs1sJnAi8Ewx1xIRkeIMerVMM/sFcDowycxagRvoH5VTCaw3M4Cn3P0qd3/JzO4DNtFf6vmiuw/PFi4iIjIodrAKU37Nzc3e0tJS7maIiBxTzGyDuzcf6TzNtBURCQgFvohIQCjwRUQCQoEvIhIQCnwRkYBQ4IuIBIQCX0QkIBT4IiIBocAXEQkIBb6ISEAo8EVEAkKBLyISEAp8EZGAUOCLiASEAl9EJCAU+CIiAaHAFxEJiEEHvpndaWa7zOzFrGPHmdl6M3s19Wdd6riZ2b+b2VYz22hmpwxH40VEZPCOpod/N3DGgGPXAI+5+4nAY6nnAJ+gf+PyE4GlwI+La6aIiBRr0IHv7r8F3h5weCFwT+rxPcBnso6v9n5PAbVmNrXYxoqIyNAVW8Of4u7bU493AFNSj6cB27LOa00dExGRMinZTVt3d8CP9n1mttTMWsyspa2trVTNERGRAYoN/J3pUk3qz12p428C07POa0wdO4S7r3T3ZndvbmhoKLI5IiJSSLGB/xBwSerxJcCDWccXp0brnAbsySr9iIhIGUQGe6KZ/QI4HZhkZq3ADcBNwH1mdgXwBnB+6vT/C3wS2ArsBy4rYZtFRGQIBh347v65Ai8tyHOuA18caqNERKT0NNNWRCQgFPgiIgGhwBcRCQgFvohIQAz6pq2ISBAlk057V4JEbx/RSJj66iihkJW7WUOiwBcRKSCZdLbs7GTJ6hZaO+I01sVYtbiZWVNqjsnQV0lHRKSA9q5EJuwBWjviLFndQntXoswtGxoFvohIAYnevkzYp7V2xEn09pWpRcVR4IuIFBCNhGmsi+Uca6yLEY2Ey9Si4ijwRUQKqK+Osmpxcyb00zX8+upomVs2NLppKyJSQChkzJpSw7pl8zVKR0RkrAuFjIaaynI3oyRU0hERCQgFvohIQCjwRUQCQoEvIhIQCnwRkYAoSeCb2VfN7CUze9HMfmFmVWY208yeNrOtZnavmR2bA1dFRMaIogPfzKYBXwaa3X02EAYuAG4GbnX3vwE6gCuKvZaIiAxdqUo6ESBmZhFgHLAd+AiwNvX6PcBnSnQtEREZgqID393fBG4B/kJ/0O8BNgC73b03dVorMK3Ya4mIyNCVoqRTBywEZgLvAKqBM47i/UvNrMXMWtra2optjoiIFFCKks7fA6+7e5u79wAPAPOB2lSJB6AReDPfm919pbs3u3tzQ0NDCZojIiL5lCLw/wKcZmbjzMyABcAm4HFgUeqcS4AHS3AtEREZolLU8J+m/+bsc8AfU5+5Erga+Gcz2wrUA3cUey0RERm6kqyW6e43ADcMOPwacGopPl9ERIqnmbYiIgGhwBcRCQhtgCIigZBMOu1diTGxc9VQKfBFZMxLJp0tOztZsrqF1o54Zm/aWVNqAhX6KumIyJjX3pXIhD1Aa0ecJatbaO9KlLllI0uBLyJjXqK3LxP2aa0dcRK9fWVqUXko8EVkzItGwjTWxXKONdbFiEbCZWpReSjwRWTMq6+Osmpxcyb00zX8+upgbdOhm7YiMuaFQsasKTWsWzZfo3RERMa6UMhoqKksdzPKSiUdEZGAUOCLiASEAl9EJCAU+CIiAaHAFxEJCAW+iEhAKPBFRAKiJIFvZrVmttbMXjazzWb2QTM7zszWm9mrqT/rSnEtEREZmlL18H8I/Nrd/ycwF9gMXAM85u4nAo+lnouISJkUHfhmNhH4MKlNyt094e67gYXAPanT7gE+U+y1RERk6ErRw58JtAF3mdnzZvYTM6sGprj79tQ5O4ApJbiWiIgMUSkCPwKcAvzY3U8GuhhQvnF3Bzzfm81sqZm1mFlLW1tbCZojIiL5lCLwW4FWd3869Xwt/T8AdprZVIDUn7vyvdndV7p7s7s3NzQ0lKA5IiKST9GB7+47gG1mNit1aAGwCXgIuCR17BLgwWKvJSIiQ1eq5ZH/EVhjZlHgNeAy+n+Y3GdmVwBvAOeX6FoiIjIEJQl8d38BaM7z0oJSfL6IiBRPM21FRAJCgS8iEhAKfBGRgFDgi4gEhAJfRCQgFPgiIgGhwBcRCQgFvohIQCjwRUQCQoEvIhIQCnwRkYBQ4IuIBIQCX0QkIBT4IiIBocAXEQkIBb6ISEAo8EVEAqJkgW9mYTN73sx+lXo+08yeNrOtZnZvavtDEREpk1L28L8CbM56fjNwq7v/DdABXFHCa4mIyFEqSeCbWSPwKeAnqecGfARYmzrlHuAzpbiWiIgMTal6+D8AvgkkU8/rgd3u3pt63gpMy/dGM1tqZi1m1tLW1lai5oiIyEBFB76ZnQnscvcNQ3m/u69092Z3b25oaCi2OSIiUkCkBJ8xHzjLzD4JVAETgB8CtWYWSfXyG4E3S3AtEREZoqJ7+O5+rbs3uvsM4ALgN+5+IfA4sCh12iXAg8VeS0REhm44x+FfDfyzmW2lv6Z/xzBeS0REjqAUJZ0Md38CeCL1+DXg1FJ+voiIDJ1m2oqIBIQCX0QkIBT4IiIBUdIavogEWzLptHclSPT2EY2Eqa+OEgpZuZslKQp8ESmJZNLZsrOTJatbaO2I01gXY9XiZmZNqVHojxIq6YjIkCWTTltnN2927GfH3gPcun4LrR1xAFo74ixZ3UJ7V6LMrZQ09fBFZEh6e5Ns2dXJlT/dkOnR//iieVw2fybf/fUWnt+2m9aOOInevnI3VVLUwxeRo9bbm2RH5wHiiT6uP7OJk6fX0toR5ws/24CZ8fWPz+Lk6bU01sWIRsLlbq6kqIcvIkclmfRDevY3nzuHWx7p79VPGh/l0rue5caFszl+YhX11dr7aLRQD19EBiVdr2/dvZ9de7tpGF8J9Nfqr75/I1ed/i4a62KEzWjtiPOuyeN1w3aUUQ9fRI4o3wic7F59a0ec+uooyxfNYcfeAzTWxYhVhBX2o4x6+CJyWMmks2PvgUzYQ26vHqCxLkbtuCjjKyPc9eTrrFrcrFLOKKQevogUlO7Zd3X3ZsI+rbUjTm2sgsa6GLddNI8JVRFqKsN85+w5mnA1SqmHLyIFtXclMmPpG+tiOa811sVorIuxbtl8mqZOYPKEKqZMjNFQU6mwH6UU+CJSUKK3j9aOOLc98SduPndOJvTTs2inKuCPKSrpiEjGwLVwKiIhGutiPL9tN7c8soXrz2yivjrKO2pjHD+hSkF/jFEPX0SAg/X6s1c8yfybH+fsFU+y70AvqxY3Z0L/xl9toroyorA/Rpm7F/cBZtOB1cAUwIGV7v5DMzsOuBeYAfwZON/dOw73Wc3Nzd7S0lJUe0RkaNo6uzl7xZM5N2cb62I89KX59CXRCpijmJltcPfmI51Xih5+L/A1d28CTgO+aGZNwDXAY+5+IvBY6rmIjDLpCVX7E/lH4sQTfTTUVDKtbpzq9ce4ogPf3be7+3Opx53AZmAasBC4J3XaPcBnir2WiBQve4XLt7u6M2Wcl3d05h2Jo7Vwxo6S1vDNbAZwMvA0MMXdt6de2kF/ySffe5aaWYuZtbS1tZWyOSKSkkw6b3f1h/y2jv38qW0fX/r58/xh257MhKpCI3E0gWrsKNkoHTMbD9wP/JO77zU7+Gufu7uZ5b1Z4O4rgZXQX8MvVXtEpF8y6fy5vYudew/wjbUbM0sjfO+8uYyLhjNlnOyROO85voZYNKJ6/RhTkh6+mVXQH/Zr3P2B1OGdZjY19fpUYFcpriUihWWXa9o6uzPDLN9o358Je+ivzX/t//yBqopwThknPRInFo2oXj8GFR341t+VvwPY7O7fz3rpIeCS1ONLgAeLvZaIFJY9rPJLP3+eF9/cw1/e3k9Pbx+Txkfz3pDd193L8kUq4wRFKUo684GLgT+a2QupY/8buAm4z8yuAN4Azi/BtUSkgN3xBDv2HODHF56CA8vWPHdwJ6oLT+FjTZN5dNPBX7Qb62Ls3t/DO+vH8cCyD9HTm9SwyzGu6MB3998Bhf52LCj280XkyJJJZ/vuA1z/4IvcdM77uOaBP+aUb76w5jl+dsUH2LS9M/ND4PaL5zF1YhW1MQV8UGhpBZExoL0rwZU/69+B6viJVXnLNyGDe5eeRp9DVUWISdWq0QeNAl9kDEgvcgYQNqOxLnbIjNlwyJhWN65cTZRRQGvpiByDBo7GSS9yBrBj74FDbsQuXzSHWFQTqIKu6LV0Sklr6YgUlh5imUwmeasrkbOJ+OrLT6W7N8mS1S00jK/k22c18XZXD+OiYfYn+nhn/Thm1FerhDNGDXYtHQW+yDEge0/Z689s4sZfbTrsImexaJjepGvkTUAMNvBVwxc5BqR3nkpvK1hokTPV6OVwFPgio0h22abPoS+ZJGRGJGQ0jK+ktSPO7nhP3puyWuRMjkQ3bUVGgfTiZpu37+W6dRvZ2tbF+bf/nv/13Sf47MqneO2tLr59VhMnT6/VImcyZKrhi5RJdm++s7sXMHbvTzAxVsFldz97SA/+xoWzAbjs7mf5WNNkvvWpJsIhU41eVMMXGc2yb8IuXzQHILO42dqrPpi3Rj8uGmZqbYwnr/47hbwMiQJfZBj09ibZta+bnr4kFeEQk8dXEomEMr36eE8vO/YcoGF8JcdPqOLiO5/JhHx7VyJvjX5/oo9YRZiGmspyfS05xinwRYqQTDpvdXVzoKePsBmxaJiaaIQtu/Zx1c8OjpO/7aJ5zJo8nq1vdWVG2zTWxbj53DkkISfc0zX6q+8/uHb98kVzmDKhSnV6KYpq+CKDkO6ZZ2/kDWTKMg3jK/nyghM5oX4cVZEQn1351CE99HuXnpb3+D2Xn8olWT18gI81TeaGT7+XpEPIIBYNa5EzKUg1fJESSPfg93f38fpbXfz7Y6/Stq+7f1TM+Ggm7L/+8Vlcff9GGsZX8oMLTspbg+9Net7j+7t7ufX8uXz1vj9kevRf/egspk6MKeClpBT4Mirk60Gnw+5wrw3m/UNtRywaZufe7kNKMLc8soUlq1v4+ZIP0NoR5/ozmzJh//WPz+K1tq68NfhIKP+iZsdVR4lFw1qTXoadAl/KLj1i5db1Wzh33nTqq6PEE728Y2L/OPO/7omzq7Ob9q4E92/YxjWfeA8Tx0U4kEjS5051ZZhdexM5wbxqcTNTJlQST/RhZoQNQqEQdbEKOuI9BX+wpEs0rR1x7rr0/Vz/4Is568pfff9Grj+ziSt/uiGzKmV65mt28A+swd920Twmj69k1eLmQ9qpnryMFAW+FOwdDzw+MCzrYhXs7e4hnuijz52qivCQ1lhv70pw6/otXPKhmTkhufLieUQjIS6969mcjbd7+vp4ZcfBDbnzBfOS1S3cuHA2l939bKZn/tstO/n0SY05N1NXLW5m1pQaQiHLWb4AyNngOy29tEFjXYxYNMyqxc3s2HMgJ/hbO+KZzcBrYxVMq4txfE0VkUiIWVNqWLdsftG/iYgMxbAHvpmdAfwQCAM/cfebhvuacqiBo0kqwiHcHQsZ+w70sjh10zAdgic2jOfVtn2ZAPxY02T+ccG7+UJWWN512fvZ3ZXIqT3ffvE8Zk2uIRIJFbx2LBpmQmUFb8cTHOjpI2TGdZ9q4sKfPJ0T2kt/uoEbF84+ZOPtuy87lW+s3XDEYB6XWg443TO/69L350xoSv9geGDZh5hcU5WzpjxQcAmD/Yk+Vi1upjYWpTYWZfKEKLddNI+2zu7M+c9v282VP91AY12MdcvmZ/59hEKmYZVSNsO6tIKZhYH/AD4BNAGfM7Om4bzmWDJwzfNksvCIqoHn9vT08dfdcd5o72L77jh/fquLc1b8Nx9OTdV/o72L7XsP8K11f2Tn3v7x4CdPr+X6M5vo6u5lZ+cBbl2/JRN2586bngl76A/L1rfjmbBPH7vypxv46554pq3pMsk5K/6br/ziBV7ZuY898R7e3BPnW+v+yIe/+wQXrHyKvfEeGsbnBmF2aGcfCxt5gzlbY12M3fGe3PeFLO8PhgM9SQCikXDO59z2xJ8OWVf+9ovmMXf6xMxvBaGQUV9dRdPUCcydPpHbL5qnJQ9k1BruHv6pwFZ3fw3AzH4JLAQ2DfN1j3kD68nZPe+BNWgg59x8vfHli+ZkFt9q7egP6lvOm8tl82dyoCfJLefNxQxuengzj27alSmftHUmeH7b7rwrNBbqWe/q7CYWjdBQU5kpk2SPZMm+AZr+/C+seS5TgklL96azNdbFiIRDOT3vdDCnSzzp7/vdX2/JeV9f0vPvBJWqqNRXR3Nq7G37upkyoWpQN1NDIeO46kpqY1GVbGTUGu7F06YB27Ket6aOZZjZUjNrMbOWtra2YW7OsWNgPbm1I86t67ewZVcnZ694kvk3P87ZK55ky85Odsdzz83XG//G2o1cdfq7Mp/f2hFnck0l4ysjXP/giyz4/v9j8Z3PcMmHZnLy9NpM+eRrH3s3kL8XvT/Rl7dnna77w8Gt9646/V2ZsE9f/+r7D7aptSPOCfXjcnrHt54/l7rqipxjt100j0iYnMXD2vZ1E4uGuemc97H2qg9yz+WnEouGadvXnXnfzefOYW3LX1hx4SkFd4IKhSxTY3/y6r9j3bL5zKivZnJNFdPqxtFQc+T7E+mSzWDPFxlJZb9p6+4rgZXQP/GqzM0ZNQbWk6E/yNO7HMHBGvS9S0/LObfQeum1sYrM88a6GBXhEF9Y81zBUSitHXGmpkbK3L9hG7ddNC/nhue0uqpDxo9/77y53PG71zjlhP71YdJlkiO1qbEuRltnN/dd+UHcnYpIiH0Hernp4c1cf2YT9dVRjquOctsTf+Ksk97B6t//mZvOeR9Ta2P8pX0///LQJtr2dbPiwlO4+eHNtHUmuHHhbP5HQzUV4RBhgxOnvIvunj5uXDg7sxPUlAlV1MYOllxUY5exbLgD/01getbzxtQxOYJ0UGaHZH11NG9o9jk55x7uZmP68fJFcwiFOGIIRyMhfvO1v+XVXftoef0t1vzDB2hLD5Fs2cal82fyy6Wn0d2TZPueOHf87jW++tFZmVJTukySHskysE3pti5fNIfJNf3ryqR7xZOqne+cPYdEbx99Sedf/3MTj27axe54gi8veDdX/WxDZobr9z97EpVhY1xlmH89+30FSzDJpFNVEVHJRQJpWJdWMLMI8AqwgP6gfxb4vLu/lO98La1wUL4a/s//4QN8PmskC/SH5gPLPkT7vsRha/g/vvAUDvQkSbqzP9HH9OP6w/zzqw79vPQWessXzSESCjGtLoa75x2amQ72w016Siadjng323d35/yGcPvF8ziuOkoy6UdcOuBIQ0QV3BJko2ZPWzP7JPAD+odl3unu3yl07lgK/KOd+Znv/GTSc1ZcbKiOHrL4VnocOeSGbiTkvLBtL7XjKpgYq+DeZ97glBn11FdHmVxTiRnc+KtNh4x9X3HhKRjw1r4Ex1VXUFNVUbLNr0s1G1ZEco2awD8aYyXwC42wSQ/lG+z5lZFQ3vHxg+nZZn9muuwxc1I14yr7J0cBOVvppevm4RB0dScJa8EukWOGAr8Mstc6/9Ou/oW2nt+2GyAzASffDcG2zm7OXvFk3h2OBg5TLPQZh2uPetQiY5tWyxxh+XrUt5w/l+274zz0wl9Z0DSF/Yle2jo5JHjzjcgpNOkoPdxxMDTiRESyKfBL5HATjFZceAo/+s2rmQlNA8s7+UbkFJp0FI3k/hAQERms4Z54FQjJpJPo7eN7583lu4vmHDLBaNma5zh33vTM8yWrW2jvSmTenx66OHBK/jsHTETSNH0RKYZ6+EXItznGdZ96zxEnPQ0szWTP8Bw43FHT9EWkVBT4h3GkTTkGjqy5+dw59PQlC04wyn4+sDRTqN6uGryIlIpKOgWkA33gujXpVSDzrXVz9f0b6Ut6zjov6fVf7t+wLfNcpRkRKQf18AvIF+hLVrdkhkUWGllTEQ5xx+9ey6wJk54V+p2z53DDp1WaEZHyUeAXUCjQ07X3w42s+epHZ+WsCQMqzYhI+amkU8DAzTAgt/aeb2TNwM0xRERGE820LWAwyyNoJquIjAaaaVukQkMlswNdM1lF5FiiwD8MBbqIjCWBCnyVYEQkyAIT+Mmk8+f2Lt5o35/Z3u6d9eNKtta7iMhoF4jATyadt/Z1s3PvAa5/8MXMTdjli+ZQO66C46pVthGRsa+oYZlmttzMXjazjWa2zsxqs1671sy2mtkWM/t48U0dmvRom65EL99Ym7uo2TfWbiSeGPxywyIix7Jix+GvB2a7+xz69669FsDMmoALgPcCZwArzKws6/qmZ8z2Jb3gBuAiIkFQVOC7+6Pu3pt6+hTQmHq8EPilu3e7++vAVuDUYq41VOkZszv2HMg7kaqqQnPPRCQYSpl2lwMPpx5PA7ZlvdaaOlZyyaTT1tnNmx37aevszixulpaeMfu9R1/he+fNPWR9+Umq34tIQBzxpq2Z/RdwfJ6XrnP3B1PnXAf0AmuOtgFmthRYCnDCCScc1XsHMxs2vQTCktUt3PTwy9y4cHbOZt4aoSMiQVH00gpmdilwJbDA3fenjl0L4O7/lnr+CPBtd//94T7raJdWKLT598CNvjX+XkTGshFZWsHMzgC+CfxtOuxTHgJ+bmbfB94BnAg8U8y18jnSipZpmjErIlL8OPwfAZXAejMDeMrdr3L3l8zsPmAT/aWeL7p7ycc/FlqiWBt9i4gc6pheLXMwNXwRkbEuEKtlDmZFSxER6XdMBz6oPi8iMliadSQiEhAKfBGRgFDgi4gEhAJfRCQgFPgiIgExqsbhm1kb8MYQ3z4JeKuEzTlW6HsHi753cBzNd36nuzcc6aRRFfjFMLOWwUw8GGv0vYNF3zs4huM7q6QjIhIQCnwRkYAYS4G/stwNKBN972DR9w6Okn/nMVPDFxGRwxtLPXwRETmMMRX4ZrbczF42s41mts7MasvdppFgZueZ2UtmljSzMT2SwczOMLMtZrbVzK4pd3tGipndaWa7zOzFcrdlpJjZdDN73Mw2pf5+f6XcbRoJZlZlZs+Y2R9S3/tfSvXZYyrwgfXAbHefA7wCXFvm9oyUF4FzgN+WuyHDyczCwH8AnwCagM+ZWVN5WzVi7gbOKHcjRlgv8DV3bwJOA74YkP/e3cBH3H0ucBJwhpmdVooPHlOB7+6Puntv6ulTQGM52zNS3H2zu28pdztGwKnAVnd/zd0TwC+BhWVu04hw998Cb5e7HSPJ3be7+3Opx53AZmBaeVs1/LzfvtTTitQ/JbnZOqYCf4DLgYfL3QgpqWnAtqznrQQgAATMbAZwMvB0eVsyMswsbGYvALuA9e5eku99zG2AYmb/BRyf56Xr3P3B1DnX0f/r4JqRbNtwGsz3FhmLzGw8cD/wT+6+t9ztGQmpPcBPSt2HXGdms9296Ps3x1zgu/vfH+51M7sUOBNY4GNozOmRvndAvAlMz3remDomY5SZVdAf9mvc/YFyt2ekuftuM3uc/vs3RQf+mCrpmNkZwDeBs9x9f7nbIyX3LHCimc00syhwAfBQmdskw8TMDLgD2Ozu3y93e0aKmTWkRxiaWQz4KPByKT57TAU+8COgBlhvZi+Y2W3lbtBIMLOzzawV+CDwn2b2SLnbNBxSN+S/BDxC/w28+9z9pfK2amSY2S+A3wOzzKzVzK4od5tGwHzgYuAjqf+fXzCzT5a7USNgKvC4mW2kv5Oz3t1/VYoP1kxbEZGAGGs9fBERKUCBLyISEAp8EZGAUOCLiASEAl9EJCAU+CIiAaHAFxEJCAW+iEhA/H9++7INARdTbgAAAABJRU5ErkJggg==\n",
"text/plain": [
"<Figure size 432x288 with 1 Axes>"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"sns.scatterplot(x=x, y=y.T.values[0])"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"(c) Use the regsubsets() function to perform best subset selection in order to choose the best model containing the predictors X, X2, . . . , X10. What is the best model obtained according to Cp, BIC, and adjusted R2? Show some plots to provide evidence for your answer, and report the coefficients of the best model ob- tained. Note you will need to use the data.frame() function to create a single data set containing both X and Y ."
]
},
{
"cell_type": "code",
"execution_count": 418,
"metadata": {},
"outputs": [],
"source": [
"def n_choose_k(n, k):\n",
" return int(np.math.factorial(n) / (np.math.factorial(n - k) * np.math.factorial(k)))\n",
"\n",
"def best_subset(n_predictors, target_column, data):\n",
" i = 1\n",
" predictors = data.drop(target_column, axis=1).columns\n",
" top_models = []\n",
" while i <= n_predictors:\n",
" tick = time.time()\n",
" cmbs = list(combinations(predictors, i))\n",
" formulae = ['{} ~ '.format(target_column) + ' + '.join(s) for s in cmbs]\n",
" models = [smf.ols(formula=f, data=data).fit() for f in formulae]\n",
" top_model = sorted(models, key=lambda m: m.rsquared, reverse=True)[0]\n",
" top_models.append(top_model)\n",
" tock = time.time()\n",
" print('{}/{} : {} combinations, {} seconds'.format(i, n_predictors, n_choose_k(n_predictors, i), tock-tick))\n",
" i += 1\n",
" return pd.DataFrame({'model' : top_models, 'n_predictors' : range(1, n_predictors + 1)})"
]
},
{
"cell_type": "code",
"execution_count": 198,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"1/10 : 10 combinations, 0.03826308250427246 seconds\n",
"2/10 : 45 combinations, 0.18390679359436035 seconds\n",
"3/10 : 120 combinations, 0.6283471584320068 seconds\n",
"4/10 : 210 combinations, 1.3881821632385254 seconds\n",
"5/10 : 252 combinations, 1.7857210636138916 seconds\n",
"6/10 : 210 combinations, 1.7825541496276855 seconds\n",
"7/10 : 120 combinations, 1.0421671867370605 seconds\n",
"8/10 : 45 combinations, 0.45599985122680664 seconds\n",
"9/10 : 10 combinations, 0.1031639575958252 seconds\n",
"10/10 : 1 combinations, 0.01294708251953125 seconds\n"
]
}
],
"source": [
"data = pd.DataFrame({'x' : x, \\\n",
" 'x2' : x ** 2,\\\n",
" 'x3' : x ** 3,\\\n",
" 'x4' : x ** 4,\\\n",
" 'x5' : x ** 5,\\\n",
" 'x6' : x ** 6,\\\n",
" 'x7' : x ** 7,\\\n",
" 'x8' : x ** 8,\\\n",
" 'x9' : x ** 9,\\\n",
" 'x10' : x ** 10,\\\n",
" 'y' : y.T.values[0]})\n",
"\n",
"# 2 ^ 10 = 1024 possible subsets\n",
"best = best_subset(10, 'y', data)"
]
},
{
"cell_type": "code",
"execution_count": 180,
"metadata": {},
"outputs": [],
"source": [
"# Select between models with different numbers of predictors using adjusted training metrics\n",
"best['adj_R_sq'] = best['model'].map(lambda m: m.rsquared_adj)\n",
"best['bic'] = best['model'].map(lambda m: m.bic)\n",
"best['aic'] = best['model'].map(lambda m: m.aic)"
]
},
{
"cell_type": "code",
"execution_count": 181,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"<matplotlib.axes._subplots.AxesSubplot at 0x119317908>"
]
},
"execution_count": 181,
"metadata": {},
"output_type": "execute_result"
},
{
"data": {
"image/png": "iVBORw0KGgoAAAANSUhEUgAAA3kAAANgCAYAAACRKaGGAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzs3X20pnVdL/73xxkIQQQE9Ecgjp0MMRO1vfih+JCghWZYhqZH8yGOc34rNSxPHc2WT+e4Vh3T6tjDr0kRLaMUwdDKhxQhUbE9iAoMpOITSI7Ko7BSwM/vj33PaX7jwOx7z733tfc1r9da99r3dd3X3Nfbtb7O5j3f6/pe1d0BAABgHO42dAAAAABmR8kDAAAYESUPAABgRJQ8AACAEVHyAAAARkTJAwAAGBElDwAAYESUPAAAgBFR8gAAAEZk/dABFuuQQw7pDRs2DB0DAABgEJs3b/5Wdx+6q+PWTMnbsGFD5ufnh44BAAAwiKr6ymKOc7kmAADAiKyZmbwtV387P/mbbx86BgAAMIDNr3/O0BHWDDN5AAAAI6LkAQAAjIiSBwAAMCJKHgAAwIgoeQAAACMySMmrqtOramtVXTrE+QEAAMZqqJm8M5KcNNC5AQAARmuQktfdFyS5bohzAwAAjNmqvievqjZW1XxVzd9+681DxwEAAFj1VnXJ6+5N3T3X3XPr991/6DgAAACr3qoueQAAAExHyQMAABiRoR6hcGaSTyQ5qqqurqpTh8gBAAAwNuuHOGl3P3OI8wIAAIydyzUBAABGRMkDAAAYESUPAABgRJQ8AACAERlk4ZWlOPqIgzP/+ucMHQMAAGBVM5MHAAAwIkoeAADAiCh5AAAAI6LkAQAAjMiaWXjle9delq++9ieGjgEAAIzEka/83NARloWZPAAAgBFR8gAAAEZEyQMAABgRJQ8AAGBElDwAAIARGaTkVdV9q+q8qrq8qi6rqtOGyAEAADA2Qz1C4fYkL+3ui6tq/ySbq+pD3X35QHkAAABGYZCZvO6+trsvnry/OcmWJIcPkQUAAGBMBr8nr6o2JHlYkot28tnGqpqvqvnrbrljpaMBAACsOYOWvKq6R5J3J3lJd9+04+fdvam757p77l77rVv5gAAAAGvMYCWvqvbKQsF7R3efPVQOAACAMRlqdc1K8pYkW7r7jUNkAAAAGKOhZvKOT/LLSU6oqksmrycNlAUAAGA0BnmEQnd/LEkNcW4AAIAxG3x1TQAAAGZHyQMAABgRJQ8AAGBElDwAAIARGWThlaXY+7Afz5GvnB86BgAAwKpmJg8AAGBElDwAAIARUfIAAABGRMkDAAAYkTWz8MoVW6/I8W86fugYAADASF344guHjjATZvIAAABGRMkDAAAYESUPAABgRJQ8AACAEVHyAAAARmSQkldV+1TVp6rqM1V1WVW9ZogcAAAAYzPUIxS+m+SE7v5OVe2V5GNV9Y/d/cmB8gAAAIzCICWvuzvJdyabe01ePUQWAACAMRnsnryqWldVlyTZmuRD3X3RTo7ZWFXzVTV/23duW/mQAAAAa8xgJa+77+juhyY5IsmxVfXgnRyzqbvnuntur3vstfIhAQAA1pjBV9fs7huSnJfkpKGzAAAArHVDra55aFUdOHl/9yRPSHLFEFkAAADGZKjVNQ9L8raqWpeFovnO7n7fQFkAAABGY6jVNT+b5GFDnBsAAGDMBr8nDwAAgNlR8gAAAEZEyQMAABgRJQ8AAGBEhlpdc2oPvPcDc+GLLxw6BgAAwKpmJg8AAGBElDwAAIARUfIAAABGRMkDAAAYkTWz8MrNV16Z8x/z2KFjAAAAI/XYC84fOsJMmMkDAAAYESUPAABgRJQ8AACAEVHyAAAARkTJAwAAGJFBS15VrauqT1fV+4bMAQAAMBZDz+SdlmTLwBkAAABGY7CSV1VHJPnZJG8eKgMAAMDYDDmT94dJfivJ9+/sgKraWFXzVTV/4223rVwyAACANWqQkldVT06ytbs339Vx3b2pu+e6e+6AvfZaoXQAAABr11AzeccnObmqvpzkb5KcUFV/NVAWAACA0Rik5HX3y7v7iO7ekOQZST7S3c8eIgsAAMCYDL26JgAAADO0fugA3f3RJB8dOAYAAMAomMkDAAAYESUPAABgRBZd8qpqv6q62+T9j1XVyVXluQYAAACryDQzeRck2aeqDk/ywSS/nOSM5QgFAADA0kyz8Ep1961VdWqSP+3u/1VVlyxXsB3tf9RReewF56/U6QAAANakaWbyqqoekeRZSf5+sm/d7CMBAACwVNOUvNOSvDzJOd19WVX9SJLzlicWAAAAS7GoyzWral2Sk7v75G37uvuqJL+2XMEAAACY3qJm8rr7jiSPWuYsAAAA7KZpFl75dFWdm+RdSW7ZtrO7z555qp3YevWN+eOXvnclTgUAAOyBXvSGnxs6wkxMU/L2SfLtJCdst6+TrEjJAwAAYNcWXfK6+/nLGQQAAIDdt+jVNavqiKo6p6q2Tl7vrqojljMcAAAA05nmEQpvTXJukh+evN472QcAAMAqMU3JO7S739rdt09eZyQ5dJlyAQAAsATTLLzy7ap6dpIzJ9vPzMJCLEtSVV9OcnOSO5Lc3t1zS/0uAAAAFkxT8n4lyZuS/EEWVtX8eJLn7eb5H9fd39rN7wAAAGBimpJ3RHefvP2Oqjo+yddmGwkAAIClmuaevDctct9idZIPVtXmqtq4swOqamNVzVfV/HduvXE3TgUAALBn2OVMXlU9IskjkxxaVb+x3Uf3TLJuN879qO6+pqruneRDVXVFd1+w/QHdvSnJpiQ58v96QO/GuQAAAPYIi5nJ2zvJPbJQCPff7nVTklOWeuLuvmbyc2uSc5Icu9TvAgAAYMEuZ/K6+/wk51fVGd39lVmctKr2S3K37r558v6nk7x2Ft8NAACwJ5vmnrw3V9WB2zaq6qCq+sASz3ufJB+rqs8k+VSSv+/u9y/xuwAAAJiYZnXNQ7r7hm0b3X395H66qXX3VUmOWcqfBQAA4M5NM5P3/ao6cttGVd0vCytkAgAAsEpMM5P3iixcYnl+kkry6CQ7ffQBAAAAw1h0yevu91fVw5McN9n1ku7+1vLEAgAAYCl2eblmVT1w8vPhSY5M8vXJ68jJPgAAAFaJxczkvTTJC5K8YSefdZITZproTtz7iAPyojf83EqcCgAAYM1azHPyXjD5+bjljwMAAMDu2GXJq6qn3tXn3X327OIAAACwOxZzuea2ayTvneSRST4y2X5cko8nUfIAAABWicVcrvn8JKmqDyZ5UHdfO9k+LMkZy5oOAACAqUzznLz7bit4E9/IwmqbK+LaL30xr3v2KSt1OgAAYA/zir86a+gIMzFNyftwVX0gyZmT7V9K8k+zjwQAAMBSTfMw9BdV1S8kecxk16buPmd5YgEAALAU08zkJcnFSW7u7n+qqn2rav/uvnk5ggEAADC9uy32wKp6QZKzkvz5ZNfhSd6zHKEAAABYmkWXvCQvTHJ8kpuSpLs/n4XHKgAAALBKTFPyvtvd39u2UVXrk/RST1xVB1bVWVV1RVVtqapHLPW7AAAAWDDNPXnnV9VvJ7l7VT0hya8mee9unPuPkry/u0+pqr2T7Lsb3wUAAECmm8l7WZJvJvlckv+a5B+S/M5STlpVB2Rhlc63JEl3f6+7b1jKdwEAAPAfFjWTV1Xrkry9u5+V5C9mcN77Z6EwvrWqjkmyOclp3X3LDufdmGRjkhyw791ncFoAAIBxW9RMXnffkeR+k8sqZ2F9kocn+bPufliSW7IwU7jjeTd191x3z+23zw/N6NQAAADjNc09eVclubCqzs1CKUuSdPcbl3Deq5Nc3d0XTbbPyk5KHgAAANOZpuR9cfK6W5L9d+ek3f1vVfW1qjqqu69McmKSy3fnOwEAAJii5HX3a5Kkqu65sNk37+a5X5zkHZNLQK9K8vzd/D4AAIA93qJLXlXNJXlrJrN4VXVjkl/p7s1LOXF3X5Jkbil/FgAAgJ2b5nLN05P8anf/c5JU1aOyUPoeshzBAAAAmN40z8m7Y1vBS5Lu/liS22cfCQAAgKWaZibv/Kr68yRnJukkv5Tko1X18CTp7ouXIR8AAABTmKbkHTP5+aod9j8sC6XvhJkkAgAAYMmqu2fzRVXP7e63zeTLdmJubq7n5+eX6+sBAABWtara3N27XLxymnvyduW0GX4XAAAASzDLklcz/C4AAACWYJYlbzbXfQIAALBkZvIAAABGZNGra1bVDyX5xSQbtv9z3f3aydsLZ5psB/9+7c3Z8rqPLOcpAACAPdjRrxjHAwOmeYTC3yW5McnmJN/d8cPuftGsQgEAALA005S8I7r7pGVLAgAAwG6b5p68j1fVTyxbEgAAAHbbNDN5j0ryvKr6UhYu16wk3d0PWZZkAAAATG2akvfEZUsBAADATCy65HX3V2Z10qo6KsnfbrfrR5K8srv/cFbnAAAA2BNNM5M3M919ZZKHJklVrUtyTZJzhsgCAAAwJrN8GPpSnZjki7OcKQQAANhTrYaS94wkZ+7sg6raWFXzVTV/3S03rHAsAACAtWfQkldVeyc5Ocm7dvZ5d2/q7rnunrvXfgeubDgAAIA1aOiZvCcmubi7vzFwDgAAgFEYuuQ9M3dyqSYAAADTG6zkVdV+SZ6Q5OyhMgAAAIzNII9QSJLuviXJwUOdHwAAYIyGvlwTAACAGVLyAAAARkTJAwAAGBElDwAAYEQGW3hlWvsctn+OfsUJQ8cAAABY1czkAQAAjIiSBwAAMCJKHgAAwIgoeQAAACOyZhZe+frXv55Xv/rVQ8cAAABGaix9w0weAADAiCh5AAAAI6LkAQAAjIiSBwAAMCJKHgAAwIgMVvKq6ter6rKqurSqzqyqfYbKAgAAMBaDlLyqOjzJryWZ6+4HJ1mX5BlDZAEAABiTIS/XXJ/k7lW1Psm+Sb4+YBYAAIBRGKTkdfc1SX4/yVeTXJvkxu7+4I7HVdXGqpqvqvlbb711pWMCAACsOUNdrnlQkqckuX+SH06yX1U9e8fjuntTd89199y+++670jEBAADWnKEu13x8ki919ze7+7YkZyd55EBZAAAARmOokvfVJMdV1b5VVUlOTLJloCwAAACjMdQ9eRclOSvJxUk+N8mxaYgsAAAAY7J+qBN396uSvGqo8wMAAIzRkI9QAAAAYMaUPAAAgBFR8gAAAEZEyQMAABiR6u6hMyzK3Nxcz8/PDx0DAABgEFW1ubvndnWcmTwAAIARUfIAAABGRMkDAAAYESUPAABgRNYPHWCxrr9+S975rmOHjgEAAIzU05/2qaEjzISZPAAAgBFR8gAAAEZEyQMAABgRJQ8AAGBElDwAAIARGazkVdVpVXVpVV1WVS8ZKgcAAMCYDFLyqurBSV6Q5NgkxyR5clX96BBZAAAAxmSombyjk1zU3bd29+1Jzk/y1IGyAAAAjMZQJe/SJI+uqoOrat8kT0py3x0PqqqNVTVfVfM33XT7iocEAABYa9YPcdLu3lJVv5fkg0luSXJJkjt2ctymJJuS5D/9p/16RUMCAACsQYMtvNLdb+nun+zuxyS5Psm/DpUFAABgLAaZyUuSqrp3d2+tqiOzcD/ecUNlAQAAGIvBSl6Sd1fVwUluS/LC7r5hwCwAAACjMFjJ6+5HD3VuAACAsRrsnjwAAABmT8kDAAAYESUPAABgRJQ8AACAERlydc2pHHTQ0Xn60z41dAwAAIBVzUweAADAiCh5AAAAI6LkAQAAjIiSBwAAMCJrZuGVy6+/Kcec9YGhYwAAACP1mVN+ZugIM2EmDwAAYESUPAAAgBFR8gAAAEZEyQMAABgRJQ8AAGBElrXkVdXpVbW1qi7dbt+9qupDVfX5yc+DljMDAADAnmS5Z/LOSHLSDvteluTD3f2AJB+ebAMAADADy1ryuvuCJNftsPspSd42ef+2JD+/nBkAAAD2JEPck3ef7r528v7fktznzg6sqo1VNV9V87ffdOPKpAMAAFjDBl14pbs7Sd/F55u6e66759bf84AVTAYAALA2DVHyvlFVhyXJ5OfWATIAAACM0hAl79wkz528f26SvxsgAwAAwCgt9yMUzkzyiSRHVdXVVXVqkt9N8oSq+nySx0+2AQAAmIH1y/nl3f3MO/noxOU8LwAAwJ5q0IVXAAAAmC0lDwAAYESUPAAAgBFR8gAAAEZkWRdemaUHHXTPzJ/yM0PHAAAAWNXM5AEAAIxIdffQGRalqm5OcuXQOdgjHZLkW0OHYI9l/DEk448hGX8MZTWPvft196G7OmjNXK6Z5Mrunhs6BHueqpo39hiK8ceQjD+GZPwxlDGMPZdrAgAAjIiSBwAAMCJrqeRtGjoAeyxjjyEZfwzJ+GNIxh9DWfNjb80svAIAAMCuraWZPAAAAHZByQMAABiRVVXyquqkqrqyqr5QVS/byec/VFV/O/n8oqrasPIpGatFjL/fqKrLq+qzVfXhqrrfEDkZp12Nv+2O+8Wq6qpa00s7s7osZvxV1dMnfwdeVlV/vdIZGadF/O49sqrOq6pPT37/PmmInIxTVZ1eVVur6tI7+byq6n9Pxudnq+rhK51xqVZNyauqdUn+JMkTkzwoyTOr6kE7HHZqkuu7+0eT/EGS31vZlIzVIsffp5PMdfdDkpyV5H+tbErGapHjL1W1f5LTkly0sgkZs8WMv6p6QJKXJzm+u388yUtWPCijs8i/+34nyTu7+2FJnpHkT1c2JSN3RpKT7uLzJyZ5wOS1McmfrUCmmVg1JS/JsUm+0N1Xdff3kvxNkqfscMxTkrxt8v6sJCdWVa1gRsZrl+Ovu8/r7lsnm59McsQKZ2S8FvP3X5L8jyz849a/r2Q4Rm8x4+8FSf6ku69Pku7eusIZGafFjL1Ocs/J+wOSfH0F8zFy3X1Bkuvu4pCnJHl7L/hkkgOr6rCVSbd7VlPJOzzJ17bbvnqyb6fHdPftSW5McvCKpGPsFjP+tndqkn9c1kTsSXY5/iaXiNy3u/9+JYOxR1jM338/luTHqurCqvpkVd3Vv3zDYi1m7L06ybOr6uok/5DkxSsTDZJM/9+Hq8b6oQPAWlNVz04yl+SxQ2dhz1BVd0vyxiTPGzgKe671Wbhc6aeycBXDBVX1E919w6Cp2BM8M8kZ3f2GqnpEkr+sqgd39/eHDgar2WqaybsmyX232z5ism+nx1TV+ixM2397RdIxdosZf6mqxyd5RZKTu/u7K5SN8dvV+Ns/yYOTfLSqvpzkuCTnWnyFGVnM339XJzm3u2/r7i8l+dcslD7YHYsZe6cmeWeSdPcnkuyT5JAVSQeL/O/D1Wg1lbx/SfKAqrp/Ve2dhZtrz93hmHOTPHfy/pQkH2lPc2c2djn+quphSf48CwXP/SjM0l2Ov+6+sbsP6e4N3b0hC/eEntzd88PEZWQW8/v3PVmYxUtVHZKFyzevWsmQjNJixt5Xk5yYJFV1dBZK3jdXNCV7snOTPGeyyuZxSW7s7muHDrUYq+Zyze6+vapelOQDSdYlOb27L6uq1yaZ7+5zk7wlC9P0X8jCTZLPGC4xY7LI8ff6JPdI8q7Jej9f7e6TBwvNaCxy/MGyWOT4+0CSn66qy5PckeQ3u9uVNOyWRY69lyb5i6r69SwswvI8/8DPrFTVmVn4B6xDJvd9virJXknS3f9vFu4DfVKSLyS5Ncnzh0k6vfL/EwAAgPFYTZdrAgAAsJuUPAAAgBFR8gAAAEZEyQMAABgRJQ8AAGBElDwA9khV9byq+uPJ+/+nqp5zF8eeUVVfqqpLquozVXXiyiUFgOmsmufkAcBQJs9D2pXf7O6zqupxSTYlecAyxwKAJTGTB8AoVdV7qmpzVV1WVRsn+55fVf9aVZ9Kcvx2x766qv7bIr/6E0kO38W5f7eqLq+qz1bV70/23b+qPlFVn6uq/1lV31ni/zQAuEtm8gAYq1/p7uuq6u5J/qWq/j7Ja5L8ZJIbk5yX5NNL+N6Tkrznzj6sqoOT/EKSB3Z3V9WBk4/+KMmfdffbq+qFSzgvACyKmTwAxurXquozST6Z5L5JfjnJR7v7m939vSR/O+X3vb6q/jXJXyf5vbs47sYk/57kLVX11CS3TvYfn+TMyfu/nPLcALBoSh4Ao1NVP5Xk8Uke0d3HZGHG7ord/Nrf7O4fS/Lfk5x+Zwd19+1Jjk1yVpInJ3n/9h/vZgYA2CUlD4AxOiDJ9d19a1U9MMlxSe6e5LFVdXBV7ZXkaUv87j9Ocreq+pmdfVhV90hyQHf/Q5JfT3LM5KMLkzxj8v5ZSzw3AOySkgfAGL0/yfqq2pLkd7Nwyea1SV6dhYVTLkyyZYc/s6hZtu7uJP8zyW/dySH7J3lfVX02yceS/MZk/2lJXlhVn8suFm4BgN1RC7+rAGDPVVVvSnJxd791Bc/5ne6+x0qdD4A9h5k8APZoVfU/kvzfSc4dOgsAzIKZPACYqKo/yXbPz5v4ozub4auqc5Lcf4fd/727P7Ac+QBgMZQ8AACAEXG5JgAAwIgoeQAAACOi5AEAAIyIkgcAADAiSh4AAMCIKHkAAAAjouQBAACMiJIHAAAwIkoeAADAiCh5AAAAI6LkAQAAjIiSBwAAMCJKHgAAwIgoeQAAACOyfugAi3XIIYf0hg0bho4BAAAwiM2bN3+ruw/d1XFrpuRt2LAh8/PzQ8cAAAAYRFV9ZTHHuVwTAABgRNbMTN6Wq7+dn/zNtw8dAwAAGKnNr3/O0BFmwkweAADAiCh5AAAAI6LkAQAAjIiSBwAAMCKDlLyqOr2qtlbVpUOcHwAAYKyGmsk7I8lJA50bAABgtAYped19QZLrhjg3AADAmLknDwAAYERWdcmrqo1VNV9V87ffevPQcQAAAFa9VV3yuntTd89199z6ffcfOg4AAMCqt6pLHgAAANMZ6hEKZyb5RJKjqurqqjp1iBwAAABjs36Ik3b3M4c4LwAAwNi5XBMAAGBElDwAAIARUfIAAABGRMkDAAAYESUPAABgRAZZXXMpjj7i4My//jlDxwAAAFjVzOQBAACMiJIHAAAwIkoeAADAiCh5AAAAI7JmFl753rWX5auv/YmhYwAArBpHvvJzQ0cAViEzeQAAACOi5AEAAIyIkgcAADAiSh4AAMCIDFLyquq+VXVeVV1eVZdV1WlD5AAAABiboVbXvD3JS7v74qraP8nmqvpQd18+UB4AAIBRGGQmr7uv7e6LJ+9vTrIlyeFDZAEAABiTwe/Jq6oNSR6W5KJhkwAAAKx9g5a8qrpHkncneUl337STzzdW1XxVzV93yx0rHxAAAGCNGazkVdVeWSh47+jus3d2THdv6u657p67137rVjYgAADAGjTU6pqV5C1JtnT3G4fIAAAAMEZDzeQdn+SXk5xQVZdMXk8aKAsAAMBoDPIIhe7+WJIa4twAAABjNvjqmgAAAMyOkgcAADAiSh4AAMCIKHkAAAAjouQBAACMyCCray7F3of9eI585fzQMQAAAFY1M3kAAAAjouQBAACMiJIHAAAwIkoeAADAiKyZhVeu2HpFjn/T8UPHAGAVu/DFFw4dAQAGZyYPAABgRJQ8AACAEVHyAAAARkTJAwAAGJFBSl5V7VNVn6qqz1TVZVX1miFyAAAAjM1Qq2t+N8kJ3f2dqtoryceq6h+7+5MD5QEAABiFQUped3eS70w295q8eogsAAAAYzLYPXlVta6qLkmyNcmHuvuiobIAAACMxWAlr7vv6O6HJjkiybFV9eAdj6mqjVU1X1Xzt33ntpUPCQAAsMYMvrpmd9+Q5LwkJ+3ks03dPdfdc3vdY6+VDwcAALDGDLW65qFVdeDk/d2TPCHJFUNkAQAAGJOhVtc8LMnbqmpdFormO7v7fQNlAQAAGI2hVtf8bJKHDXFuAACAMRv8njwAAABmR8kDAAAYESUPAABgRJQ8AACAEVHyAAAARmSoRyhM7YH3fmAufPGFQ8cAAABY1czkAQAAjIiSBwAAMCJKHgAAwIgoeQAAACOyZhZeufnKK3P+Yx47dAwA1qDHXnD+0BEAYMWYyQMAABgRJQ8AAGBElDwAAIARUfIAAABGZNCSV1XrqurTVfW+IXMAAACMxdAzeacl2TJwBgAAgNEYrORV1RFJfjbJm4fKAAAAMDZDzuT9YZLfSvL9ATMAAACMyiAlr6qenGRrd2/exXEbq2q+quZvvO22FUoHAACwdg01k3d8kpOr6stJ/ibJCVX1Vzse1N2bunuuu+cO2Guvlc4IAACw5gxS8rr75d19RHdvSPKMJB/p7mcPkQUAAGBMhl5dEwAAgBlaP3SA7v5oko8OHAMAAGAUzOQBAACMiJIHAAAwIosueVW1X1XdbfL+x6rq5Kqy5CUAAMAqMs1M3gVJ9qmqw5N8MMkvJzljOUIBAACwNNOUvOruW5M8NcmfdvfTkvz48sQCAABgKaZZXbOq6hFJnpXk1Mm+dbOPtHP7H3VUHnvB+St1OgAAgDVpmpm805K8PMk53X1ZVf1IkvOWJxYAAABLsaiZvKpal+Tk7j55277uvirJry1XMAAAAKa3qJm87r4jyaOWOQsAAAC7aZp78j5dVecmeVeSW7bt7O6zZ54KAACAJZmm5O2T5NtJTthuXydZkZK39eob88cvfe9KnAqAPcSL3vBzQ0cAgJlbdMnr7ucvZxAAAAB236JX16yqI6rqnKraOnm9u6qOWM5wAAAATGeaRyi8Ncm5SX548nrvZB8AAACrxDQl79Dufmt33z55nZHk0GXKBQAAwBJMs/DKt6vq2UnOnGw/MwsLsSxJVX05yc1J7khye3fPLfW7AAAAWDBNyfuVJG9K8gdZWFXz40met5vnf1x3f2s3vwMAAICJaUreEd198vY7qur4JF+bbSQAAACWapp78t60yH2L1Uk+WFWbq2rjbnwPAAAAE7ucyauqRyR5ZJJDq+o3tvvonknW7ca5H9Xd11TVvZN8qKqu6O4Ldjj3xiQbk+Sg/a3xAgAAsCuLmcnbO8k9slAI99/udVOSU5Z64u6+ZvJza5Jzkhy7k2M2dfdcd8/dY98DlnoqAACAPcYuZ/K6+/wk51fVGd39lVmctKr2S3K37r6DEfLVAAAgAElEQVR58v6nk7x2Ft8NAACwJ5vmnrw3V9WB2zaq6qCq+sASz3ufJB+rqs8k+VSSv+/u9y/xuwAAAJiYZnXNQ7r7hm0b3X395H66qXX3VUmOWcqfBQAA4M5NM5P3/ao6cttGVd0vCytkAgAAsEpMM5P3iixcYnl+kkry6ExWvgQAAGB1WHTJ6+73V9XDkxw32fWS7v7W8sQCAABgKXZ5uWZVPXDy8+FJjkzy9cnryMk+AAAAVonFzOS9NMkLkrxhJ591khNmmggAAIAlq+61sXbK3Nxcz8/PDx0DAABgEFW1ubvndnXcLmfyquqpd/V5d589TTAAAACWz2Iu1/y5yc97J3lkko9Mth+X5ONJlDwAAIBVYpclr7ufnyRV9cEkD+ruayfbhyU5Y1nTAQAAMJVpnpN3320Fb+IbWVhtc0Vc+6Uv5nXPPmWlTgfAHugVf3XW0BEAYLdNU/I+XFUfSHLmZPuXkvzT7CMBAACwVNM8DP1FVfULSR4z2bWpu89ZnlgAAAAsxTQzeUlycZKbu/ufqmrfqtq/u29ejmAAAABM726LPbCqXpDkrCR/Ptl1eJL3LEcoAAAAlmbRJS/JC5Mcn+SmJOnuz2fhsQoAAACsEtOUvO929/e2bVTV+iS91BNX1YFVdVZVXVFVW6rqEUv9LgAAABZMc0/e+VX120nuXlVPSPKrSd67G+f+oyTv7+5TqmrvJPvuxncBAACQ6WbyXpbkm0k+l+S/JvmHJL+zlJNW1QFZWKXzLUnS3d/r7huW8l0AAAD8h0XN5FXVuiRv7+5nJfmLGZz3/lkojG+tqmOSbE5yWnffMoPvBgAA2GMtaiavu+9Icr/JZZWzsD7Jw5P8WXc/LMktWZgp/P+pqo1VNV9V87f8+3dndGoAAIDxmuaevKuSXFhV52ahlCVJuvuNSzjv1Umu7u6LJttnZSclr7s3JdmUJIcffNCSF3kBAADYU0xT8r44ed0tyf67c9Lu/req+lpVHdXdVyY5Mcnlu/OdAAAATFHyuvs1SVJV91zY7Jt389wvTvKOySWgVyV5/m5+HwAAwB5v0SWvquaSvDWTWbyqujHJr3T35qWcuLsvSTK3lD8LAADAzk1zuebpSX61u/85SarqUVkofQ9ZjmAAAABMb5rn5N2xreAlSXd/LMnts48EAADAUk0zk3d+Vf15kjOTdJJfSvLRqnp4knT3xcuQDwAAgClMU/KOmfx81Q77H5aF0nfCTBIBAACwZNOsrvm4u/q8qp7b3W/b/UgAAAAsVXXP5hnjVXVxdz98Jl+2E3Nzcz0/P79cXw8AALCqVdXm7t7lEwqmWXhll+ec4XcBAACwBLMsebOZEgQAAGDJzOQBAACMyKIXXqmqH0ryi0k2bP/nuvu1k7cXzjTZDv792puz5XUfWc5TAMBOHf0KC0gDsHZM8wiFv0tyY5LNSb6744fd/aJZhQIAAGBppil5R3T3ScuWBAAAgN02zT15H6+qn1i2JAAAAOy2aWbyHpXkeVX1pSxcrllJursfsizJAAAAmNo0Je+Jy5YCAACAmVh0yevur8zqpFV1VJK/3W7XjyR5ZXf/4azOAQAAsCeaZiZvZrr7yiQPTZKqWpfkmiTnDJEFAABgTGb5MPSlOjHJF2c5UwgAALCnWg0l7xlJzhw6BAAAwBgMWvKqau8kJyd51518vrGq5qtq/rpbbljZcAAAAGvQ0DN5T0xycXd/Y2cfdvem7p7r7rl77XfgCkcDAABYe4Yuec+MSzUBAABmZrCSV1X7JXlCkrOHygAAADA2gzxCIUm6+5YkBw91fgAAgDEa+nJNAAAAZkjJAwAAGBElDwAAYESUPAAAgBFR8gAAAEZksNU1p7XPYfvn6FecMHQMAACAVc1MHgAAwIgoeQAAACOi5AEAAIzImrkn7+tf/3pe/epXDx0DAPw+AmBVM5MHAAAwIkoeAADAiCh5AAAAI6LkAQAAjIiSBwAAMCKDlbyq+vWquqyqLq2qM6tqn6GyAAAAjMUgJa+qDk/ya0nmuvvBSdYlecYQWQAAAMZkyMs11ye5e1WtT7Jvkq8PmAUAAGAUBil53X1Nkt9P8tUk1ya5sbs/OEQWAACAMRnqcs2Dkjwlyf2T/HCS/arq2Ts5bmNVzVfV/K233rrSMQEAANacoS7XfHySL3X3N7v7tiRnJ3nkjgd196bunuvuuX333XfFQwIAAKw1Q5W8ryY5rqr2rapKcmKSLQNlAQAAGI2h7sm7KMlZSS5O8rlJjk1DZAEAABiT9UOduLtfleRVQ50fAABgjIZ8hAIAAAAzpuQBAACMiJIHAAAwIkoeAADAiCh5AAAAI1LdPXSGRZmbm+v5+fmhYwAAAAyiqjZ399yujjOTBwAAMCJKHgAAwIgoeQAAACOyfugAi3X99VvyzncdO3QMALhTT3/ap4aOAABm8gAAAMZEyQMAABgRJQ8AAGBElDwAAIARUfIAAABGZLCSV1WnVdWlVXVZVb1kqBwAAABjMkjJq6oHJ3lBkmOTHJPkyVX1o0NkAQAAGJOhZvKOTnJRd9/a3bcnOT/JUwfKAgAAMBpDlbxLkzy6qg6uqn2TPCnJfQfKAgAAMBrrhzhpd2+pqt9L8sEktyS5JMkdOx5XVRuTbEySQw7Ze0UzAgAArEWDLbzS3W/p7p/s7sckuT7Jv+7kmE3dPdfdc/e85yB9FAAAYE0ZrDlV1b27e2tVHZmF+/GOGyoLAADAWAw5Pfbuqjo4yW1JXtjdNwyYBQAAYBQGK3nd/eihzg0AADBWg92TBwAAwOwpeQAAACOi5AEAAIyIkgcAADAiSh4AAMCIrJknjB900NF5+tM+NXQMAACAVc1MHgAAwIgoeQAAACOi5AEAAIzImrkn7/Lrb8oxZ31g6BgAsCSfOeVnho4AwB7CTB4AAMCIKHkAAAAjouQBAACMiJIHAAAwIkoeAADAiCxryauq06tqa1Vdut2+e1XVh6rq85OfBy1nBgAAgD3Jcs/knZHkpB32vSzJh7v7AUk+PNkGAABgBpa15HX3BUmu22H3U5K8bfL+bUl+fjkzAAAA7EmGuCfvPt197eT9vyW5zwAZAAAARmnQhVe6u5P0nX1eVRurar6q5m+/6cYVTAYAALA2DVHyvlFVhyXJ5OfWOzuwuzd191x3z62/5wErFhAAAGCtGqLknZvkuZP3z03ydwNkAAAAGKXlfoTCmUk+keSoqrq6qk5N8rtJnlBVn0/y+Mk2AAAAM7B+Ob+8u595Jx+duJznBQAA2FMNuvAKAAAAs6XkAQAAjIiSBwAAMCJKHgAAwIgoeQAAACOyrKtrztKDDrpn5k/5maFjAAAArGpm8gAAAEakunvoDItSVTcnuXLoHKx5hyT51tAhWPOMI2bBOGIWjCNmwThaO+7X3Yfu6qA1c7lmkiu7e27oEKxtVTVvHLG7jCNmwThiFowjZsE4Gh+XawIAAIyIkgcAADAia6nkbRo6AKNgHDELxhGzYBwxC8YRs2AcjcyaWXgFAACAXVtLM3kAAADswqoveVV1UlVdWVVfqKqXDZ2H1auqTq+qrVV16Xb77lVVH6qqz09+HjTZX1X1vyfj6rNV9fDhkrOaVNV9q+q8qrq8qi6rqtMm+40lFq2q9qmqT1XVZybj6DWT/fevqosm4+Vvq2rvyf4fmmx/YfL5hiHzs7pU1bqq+nRVvW+ybRwxlar6clV9rqouqar5yT6/10ZsVZe8qlqX5E+SPDHJg5I8s6oeNGwqVrEzkpy0w76XJflwdz8gyYcn28nCmHrA5LUxyZ+tUEZWv9uTvLS7H5TkuCQvnPy9Yywxje8mOaG7j0ny0CQnVdVxSX4vyR90948muT7JqZPjT01y/WT/H0yOg21OS7Jlu23jiKV4XHc/dLtHJfi9NmKruuQlOTbJF7r7qu7+XpK/SfKUgTOxSnX3BUmu22H3U5K8bfL+bUl+frv9b+8Fn0xyYFUdtjJJWc26+9ruvnjy/uYs/IfV4TGWmMJkPHxnsrnX5NVJTkhy1mT/juNo2/g6K8mJVVUrFJdVrKqOSPKzSd482a4YR8yG32sjttpL3uFJvrbd9tWTfbBY9+nuayfv/y3JfSbvjS12aXKp08OSXBRjiSlNLrG7JMnWJB9K8sUkN3T37ZNDth8r/2ccTT6/McnBK5uYVeoPk/xWku9Ptg+OccT0OskHq2pzVW2c7PN7bcTWDx0AVkp3d1VZTpZFqap7JHl3kpd0903b/2O4scRidPcdSR5aVQcmOSfJAweOxBpTVU9OsrW7N1fVTw2dhzXtUd19TVXdO8mHquqK7T/0e218VvtM3jVJ7rvd9hGTfbBY39h2icHk59bJfmOLO1VVe2Wh4L2ju8+e7DaWWJLuviHJeUkekYXLnrb9A+v2Y+X/jKPJ5wck+fYKR2X1OT7JyVX15SzcsnJCkj+KccSUuvuayc+tWfhHp2Pj99qorfaS9y9JHjBZRWrvJM9Icu7AmVhbzk3y3Mn75yb5u+32P2eygtRxSW7c7pIF9mCT+1fekmRLd79xu4+MJRatqg6dzOClqu6e5AlZuL/zvCSnTA7bcRxtG1+nJPlIe5DtHq+7X97dR3T3hiz8N9BHuvtZMY6YQlXtV1X7b3uf5KeTXBq/10Zt1T8MvaqelIXr0dclOb27XzdwJFapqjozyU8lOSTJN5K8Ksl7krwzyZFJvpLk6d193eQ/5P84C6tx3prk+d09P0RuVpeqelSSf07yufzHPTC/nYX78owlFqWqHpKFhQzWZeEfVN/Z3a+tqh/JwozMvZJ8Osmzu/u7VbVPkr/Mwj2g1yV5RndfNUx6VqPJ5Zr/rbufbBwxjcl4OWeyuT7JX3f366rq4Pi9NlqrvuQBAACweKv9ck0AAACmoOQBAACMiJIHAAAwIkoeAADAiCh5AAAAI6LkAcB2qmpDVV26k/1vrqoHDZEJAKaxfugAALAWdPd/GToDACyGmTwA+EHrq+odVbWlqs6qqn2r6qNVNZckVXVSVV1cVZ+pqg8PHRYAtmcmDwB+0FFJTu3uC6vq9CS/uu2Dqjo0yV8keUx3f6mq7jVUSADYGTN5APCDvtbdF07e/1WSR2332XFJLujuLyVJd1+30uEA4K4oeQDwg3oX2wCwail5APCDjqyqR0ze/+ckH9vus08meUxV3T9JXK4JwGqj5AHAD7oyyQur/r/27j3YsrMsE/jz0p0YEkIIuVAxIUZLhovIzVNUIFwkgEbEoExAKBBEip4quYQZZiyUKRCqqNJyxAvOWPZwCSgTgZAAOg4kkpAIaPB0SCShyQDhlhBoIRdCUkIS3/njrNamp2OffXqfs/ZZ+f2qdp291l691/PHV3366W+tb9XOJEcm+ePdH3T3PybZluTcqroiybvHiQgA+1bdrkABAACYCjN5AAAAE6LkAQAATIiSBwAAMCFKHgAAwIQoeQAAABOi5AEAAEyIkgcAADAhSh4AAMCEKHkAAAATouQBAABMiJIHAAAwIUoeAADAhCh5AAAAE6LkAQAATIiSBwAAMCFKHgAAwIQoeQAAABOi5AEAAEzI1rEDrNbRRx/dJ5100tgxAAAARrFjx45vdvcx+ztu05S8k046KcvLy2PHAAAAGEVVfXk1x22akrfz2m/lJ/7LO8eOAQAATNSO33nB2BHmwj15AAAAE6LkAQAATIiSBwAAMCFKHgAAwIQoeQAAABMySsmrqrdV1a6qunKM8wMAAEzVWDN5ZyU5baRzAwAATNYoJa+7L0lywxjnBgAAmDL35AEAAEzIQpe8qtpWVctVtXzHbbeMHQcAAGDhLXTJ6+7t3b3U3UtbDz187DgAAAALb6FLHgAAALMZ6xEKZyf52yQPrKprq+rFY+QAAACYmq1jnLS7nzvGeQEAAKbO5ZoAAAATouQBAABMiJIHAAAwIUoeAADAhCh5AAAAEzLK6ppr8eATjsry77xg7BgAAAALzUweAADAhCh5AAAAE6LkAQAATMimuSfve9dfla+84cfHjgEAsOFOfO2nx44AbCJm8gAAACZEyQMAAJgQJQ8AAGBClDwAAIAJUfIAAAAmZJSSV1X3r6qLquozVXVVVZ05Rg4AAICpGesRCnckeVV3X1ZVhyfZUVUXdPdnRsoDAAAwCaPM5HX39d192fD+liQ7kxw/RhYAAIApGf2evKo6Kckjk1w6bhIAAIDNb9SSV1X3SvK+JK/s7m/v4/NtVbVcVcs33HrnxgcEAADYZEYreVV1UFYK3ru6+9x9HdPd27t7qbuX7nvYlo0NCAAAsAmNtbpmJXlrkp3d/aYxMgAAAEzRWDN5pyT5pSSnVtXlw+tpI2UBAACYjFEeodDdH0tSY5wbAABgykZfXRMAAID5UfIAAAAmRMkDAACYECUPAABgQpQ8AACACRlldc21OPi4H8uJr10eOwYAAMBCM5MHAAAwIUoeAADAhCh5AAAAE7Jp7sn77K7P5pQ3nzJ2DAA20Mdf/vGxIwDApmMmDwAAYEKUPAAAgAlR8gAAACZEyQMAAJgQJQ8AAGBCRil5VXVIVX2yqq6oqquq6vVj5AAAAJiasR6h8N0kp3b3d6rqoCQfq6r/091/N1IeAACASRil5HV3J/nOsHnQ8OoxsgAAAEzJaPfkVdWWqro8ya4kF3T3pWNlAQAAmIrRSl5339ndj0hyQpJHV9VD9z6mqrZV1XJVLd/+nds3PiQAAMAmM/rqmt19U5KLkpy2j8+2d/dSdy8ddK+DNj4cAADAJjPW6prHVNV9hvf3TPLUJJ8dIwsAAMCUjLW65nFJ3lFVW7JSNN/T3X85UhYAAIDJGGt1zX9I8sgxzg0AADBlo9+TBwAAwPwoeQAAABOi5AEAAEyIkgcAADAhSh4AAMCEjPUIhZk96NgH5eMv//jYMQAAABaamTwAAIAJUfIAAAAmRMkDAACYkE1zT94tV1+di5/wxLFjADCSJ15y8dgRAGBTMJMHAAAwIUoeAADAhCh5AAAAE6LkAQAATIiSBwAAMCGjlryq2lJVn6qqvxwzBwAAwFSMPZN3ZpKdI2cAAACYjNFKXlWdkORnk7xlrAwAAABTM+ZM3u8n+bUk/zxiBgAAgEkZpeRV1dOT7OruHfs5bltVLVfV8s23375B6QAAADavsWbyTklyelV9KcmfJzm1qv5s74O6e3t3L3X30hEHHbTRGQEAADadUUped/96d5/Q3ScleU6SC7v7+WNkAQAAmJKxV9cEAABgjraOHaC7P5rkoyPHAAAAmAQzeQAAABOi5AEAAEzIqkteVR1WVfcY3v+7qjq9qix5CQAAsEBmmcm7JMkhVXV8kvOT/FKSs9YjFAAAAGszS8mr7r4tyTOT/I/uflaSH1ufWAAAAKzFLKtrVlU9Jsnzkrx42Ldl/pH27fAHPjBPvOTijTodAADApjTLTN6ZSX49yXndfVVV/UiSi9YnFgAAAGuxqpm8qtqS5PTuPn33vu6+Jskr1isYAAAAs1vVTF5335nkceucBQAAgAM0yz15n6qqDyZ5b5Jbd+/s7nPnnmofdl17c/7oVX+xEacCYIG97Hd/buwIALDQZil5hyT5VpJT99jXSTak5AEAALB/qy553f2i9QwCAADAgVv16ppVdUJVnVdVu4bX+6rqhPUMBwAAwGxmeYTC25N8MMkPDq+/GPYBAACwIGYpecd099u7+47hdVaSY9YpFwAAAGswy8Ir36qq5yc5e9h+blYWYlmTqvpSkluS3Jnkju5eWut3AQAAsGKWkvcrSd6c5PeysqrmJ5L88gGe/0nd/c0D/A4AAAAGs5S8E7r79D13VNUpSb4630gAAACs1Sz35L15lftWq5OcX1U7qmrbAXwPAAAAg/3O5FXVY5I8NskxVfWf9vjo3km2HMC5H9fd11XVsUkuqKrPdvcle517W5JtSXLk4dZ4AQAA2J/VzOQdnOReWSmEh+/x+naSM9Z64u6+bvi5K8l5SR69j2O2d/dSdy/d69Aj1noqAACAu439zuR198VJLq6qs7r7y/M4aVUdluQe3X3L8P6nkrxhHt8NAABwdzbLPXlvqar77N6oqiOr6sNrPO/9knysqq5I8skk/7u7P7TG7wIAAGAwy+qaR3f3Tbs3uvvG4X66mXX3NUkevpY/CwAAwF2bZSbvn6vqxN0bVfVDWVkhEwAAgAUxy0zea7JyieXFSSrJ4zOsfAkAAMBiWHXJ6+4PVdWjkpw87Hpld39zfWIBAACwFvu9XLOqHjT8fFSSE5N8bXidOOwDAABgQaxmJu9VSV6S5Hf38VknOXWuie7CsScckZf97s9txKkAAAA2rdU8J+8lw88nrX8cAAAADsR+S15VPfPf+ry7z51fHAAAAA7Eai7X3H2N5LFJHpvkwmH7SUk+kUTJAwAAWBCruVzzRUlSVecneUh3Xz9sH5fkrHVNBwAAwExmeU7e/XcXvME3srLa5oa4/otfyBuff8ZGnQ6ABfWaPztn7AgAsNBmKXkfqaoPJzl72P7FJH89/0gAAACs1SwPQ39ZVf1CkicMu7Z393nrEwsAAIC1mGUmL0kuS3JLd/91VR1aVYd39y3rEQwAAIDZ3WO1B1bVS5Kck+RPhl3HJ3n/eoQCAABgbVZd8pK8NMkpSb6dJN39uaw8VgEAAIAFMUvJ+253f2/3RlVtTdJrPXFV3aeqzqmqz1bVzqp6zFq/CwAAgBWz3JN3cVX9RpJ7VtVTk/xqkr84gHP/QZIPdfcZVXVwkkMP4LsAAADIbDN5r07yj0k+neQ/JPmrJP91LSetqiOyskrnW5Oku7/X3Tet5bsAAAD4V6uayauqLUne2d3PS/I/53DeH85KYXx7VT08yY4kZ3b3rXP4bgAAgLutVc3kdfedSX5ouKxyHrYmeVSSP+7uRya5NSszhd+nqrZV1XJVLd/6T9+d06kBAACma5Z78q5J8vGq+mBWSlmSpLvftIbzXpvk2u6+dNg+J/soed29Pcn2JDn+qCPXvMgLAADA3cUsJe8Lw+seSQ4/kJN299er6qtV9cDuvjrJk5N85kC+EwAAgBlKXne/Pkmq6t4rm33LAZ775UneNVwCek2SFx3g9wEAANztrbrkVdVSkrdnmMWrqpuT/Ep371jLibv78iRLa/mzAAAA7Nssl2u+LcmvdvffJElVPS4rpe9h6xEMAACA2c3ynLw7dxe8JOnujyW5Y/6RAAAAWKtZZvIurqo/SXJ2kk7yi0k+WlWPSpLuvmwd8gEAADCDWUrew4efr9tr/yOzUvpOnUsiAAAA1qy65/P4uap6YXe/Yy5ftg9LS0u9vLy8Xl8PAACw0KpqR3fvd/HKWe7J258z5/hdAAAArME8S17N8bsAAABYg3mWvPlc9wkAAMCamckDAACYkFWvrllVP5Dk3yc5ac8/191vGN5+fK7J9vJP19+SnW+8cD1PAcAm8+DXWNgZAPY2yyMUPpDk5iQ7knx37w+7+2XzCgUAAMDazFLyTuju09YtCQAAAAdslnvyPlFVP75uSQAAADhgs8zkPS7JL1fVF7NyuWYl6e5+2LokAwAAYGazlLyfWbcUAAAAzMWqS153f3leJ62qByZ59x67fiTJa7v79+d1DgAAgLujWWby5qa7r07yiCSpqi1Jrkty3hhZAAAApmSeD0Nfqycn+cI8ZwoBAADurhah5D0nydljhwAAAJiCUUteVR2c5PQk772Lz7dV1XJVLd9w600bGw4AAGATGnsm72eSXNbd39jXh929vbuXunvpvofdZ4OjAQAAbD5jl7znxqWaAAAAczNayauqw5I8Ncm5Y2UAAACYmlEeoZAk3X1rkqPGOj8AAMAUjX25JgAAAHOk5AEAAEyIkgcAADAhSh4AAMCEjLbwyqwOOe7wPPg1p44dAwAAYKGZyQMAAJgQJQ8AAGBClDwAAIAJUfIAAAAmZNMsvPK1r30tv/mbvzl2DAAWiN8LAPD/M5MHAAAwIUoeAADAhCh5AAAAE6LkAQAATIiSBwAAMCGjlbyq+o9VdVVVXVlVZ1fVIWNlAQAAmIpRSl5VHZ/kFUmWuvuhSbYkec4YWQAAAKZkzMs1tya5Z1VtTXJokq+NmAUAAGASRil53X1dkv+W5CtJrk9yc3efP0YWAACAKRnrcs0jkzwjyQ8n+cEkh1XV8/dx3LaqWq6q5dtuu22jYwIAAGw6Y12u+ZQkX+zuf+zu25Ocm+Sxex/U3du7e6m7lw499NANDwkAALDZjFXyvpLk5Ko6tKoqyZOT7BwpCwAAwGSMdU/epUnOSXJZkk8PObaPkQUAAGBKto514u5+XZLXjXV+AACAKRrzEQoAAADMmZIHAAAwIUoeAADAhCh5AAAAE1LdPXaGVVlaWurl5eWxYwAAAIyiqnZ099L+jjOTBwAAMCFKHgAAwIQoeQAAABOi5AEAAEzI1rEDrNaNN+7Me9776LFjALCAnv2sT44dAQAWhpk8AACACVHyAAAAJkTJAwAAmBAlDwAAYEKUPAAAgAkZreRV1ZlVdWVVXVVVrxwrBwAAwJSMUvKq6qFJXpLk0UkenuTpVfWjY2QBAACYkrFm8h6c5NLuvq2770hycZJnjpQFAABgMsYqeVcmeXxVHVVVhyZ5WpL7j5QFAABgMraOcdLu3llVv53k/CS3Jrk8yZ17H1dV25JsS5Kjjz54QzMCAABsRqMtvNLdb+3un+juJyS5Mcn/3ccx27t7qbuX7n3vUfooAADApjJac6qqY7t7V1WdmJX78U4eKwsAAMBUjDk99r6qOirJ7Ule2t03jZgFAABgEkYred39+LHODQAAMFWj3ZMHAADA/Cl5AAAAE6LkAQAATIiSBwAAMCGb5uFzRx754Dz7WZ8cOwYAAMBCM5MHAAAwIUoeAADAhCh5AAAAE6LkAQAATMimWXjlMzd+Ow8/58NjxwBgwV1xxk+PHQEARmUmDwAAYEKUPLl1AYMAAAb8SURBVAAAgAlR8gAAACZEyQMAAJgQJQ8AAGBC1rXkVdXbqmpXVV25x777VtUFVfW54eeR65kBAADg7mS9Z/LOSnLaXvteneQj3f2AJB8ZtgEAAJiDdS153X1Jkhv22v2MJO8Y3r8jyc+vZwYAAIC7kzHuybtfd18/vP96kvuNkAEAAGCSRl14pbs7Sd/V51W1raqWq2r5jm/fvIHJAAAANqcxSt43quq4JBl+7rqrA7t7e3cvdffS1nsfsWEBAQAANqsxSt4Hk7xweP/CJB8YIQMAAMAkrfcjFM5O8rdJHlhV11bVi5P8VpKnVtXnkjxl2AYAAGAOtq7nl3f3c+/ioyev53kBAADurkZdeAUAAID5UvIAAAAmRMkDAACYECUPAABgQtZ14ZV5esiR987yGT89dgwAAICFZiYPAABgQpQ8AACACanuHjvDqlTVLUmuHjsHk3B0km+OHYLJMJ6YF2OJeTGWmCfjabH8UHcfs7+DNs09eUmu7u6lsUOw+VXVsrHEvBhPzIuxxLwYS8yT8bQ5uVwTAABgQpQ8AACACdlMJW/72AGYDGOJeTKemBdjiXkxlpgn42kT2jQLrwAAALB/m2kmDwAAgP1Y+JJXVadV1dVV9fmqevXYeVh8VfW2qtpVVVfuse++VXVBVX1u+HnksL+q6g+H8fUPVfWo8ZKzaKrq/lV1UVV9pqquqqozh/3GEzOpqkOq6pNVdcUwll4/7P/hqrp0GDPvrqqDh/0/MGx/fvj8pDHzs3iqaktVfaqq/nLYNpZYk6r6UlV9uqour6rlYZ/fc5vcQpe8qtqS5L8n+ZkkD0ny3Kp6yLip2ATOSnLaXvteneQj3f2AJB8ZtpOVsfWA4bUtyR9vUEY2hzuSvKq7H5Lk5CQvHf4OMp6Y1XeTnNrdD0/yiCSnVdXJSX47ye91948muTHJi4fjX5zkxmH/7w3HwZ7OTLJzj21jiQPxpO5+xB6PSvB7bpNb6JKX5NFJPt/d13T395L8eZJnjJyJBdfdlyS5Ya/dz0jyjuH9O5L8/B7739kr/i7JfarquI1JyqLr7uu7+7Lh/S1Z+QfV8TGemNEwJr4zbB40vDrJqUnOGfbvPZZ2j7Fzkjy5qmqD4rLgquqEJD+b5C3DdsVYYr78ntvkFr3kHZ/kq3tsXzvsg1ndr7uvH95/Pcn9hvfGGKsyXOL0yCSXxnhiDYbL6y5PsivJBUm+kOSm7r5jOGTP8fIvY2n4/OYkR21sYhbY7yf5tST/PGwfFWOJtesk51fVjqraNuzze26T2zp2ANho3d1VZVlZVq2q7pXkfUle2d3f3vM/wY0nVqu770zyiKq6T5Lzkjxo5EhsQlX19CS7untHVf3k2HmYhMd193VVdWySC6rqs3t+6Pfc5rToM3nXJbn/HtsnDPtgVt/YfTnB8HPXsN8Y499UVQdlpeC9q7vPHXYbT6xZd9+U5KIkj8nKpU67/8N1z/HyL2Np+PyIJN/a4KgsplOSnF5VX8rKbSynJvmDGEusUXdfN/zclZX/gHp0/J7b9Ba95P19kgcMK0YdnOQ5ST44ciY2pw8meeHw/oVJPrDH/hcMq0WdnOTmPS5P4G5uuG/lrUl2dveb9vjIeGImVXXMMIOXqrpnkqdm5R7Pi5KcMRy291jaPcbOSHJhe7AtSbr717v7hO4+KSv/Lrqwu58XY4k1qKrDqurw3e+T/FSSK+P33Ka38A9Dr6qnZeXa8y1J3tbdbxw5Eguuqs5O8pNJjk7yjSSvS/L+JO9JcmKSLyd5dnffMPwj/o+yshrnbUle1N3LY+Rm8VTV45L8TZJP51/vffmNrNyXZzyxalX1sKwsXrAlK//B+p7ufkNV/UhWZmPum+RTSZ7f3d+tqkOS/GlW7gO9IclzuvuacdKzqIbLNf9zdz/dWGIthnFz3rC5Ncn/6u43VtVR8XtuU1v4kgcAAMDqLfrlmgAAAMxAyQMAAJgQJQ8AAGBClDwAAIAJUfIAAAAmRMkDgFWoqr/a/aw7AFhkHqEAAAAwIWbyAGAvVfX+qtpRVVdV1bZh35eq6ujh/Quq6h+q6oqq+tNx0wLA9zOTBwB7qar7dvcNVXXPJH+f5IlJdiRZSnK/JOcleWx3f3P3sSPGBYDvs3XsAACwgF5RVb8wvL9/kgfs8dmpSd7b3d9MEgUPgEWj5AHAHqrqJ5M8Jcljuvu2qvpokkNGDQUAM3BPHgB8vyOS3DgUvAclOXmvzy9M8qyqOipZubRzowMCwL9FyQOA7/ehJFurameS30ryd3t+2N1XJXljkour6ookb9r4iABw1yy8AgAAMCFm8gAAACZEyQMAAJgQJQ8AAGBClDwAAIAJUfIAAAAmRMkDAACYECUPAABgQpQ8AACACfl/1qOqXD8wa0kAAAAASUVORK5CYII=\n",
"text/plain": [
"<Figure size 1080x1080 with 3 Axes>"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"# Plot the error metrics\n",
"_, (ax1, ax2, ax3) = plt.subplots(nrows=3, figsize=(15,15))\n",
"sns.barplot(y='n_predictors', x='adj_R_sq', data=best, ax=ax1, orient='h')\n",
"sns.barplot(y='n_predictors', x='bic', data=best, ax=ax2, orient='h')\n",
"sns.barplot(y='n_predictors', x='aic', data=best, ax=ax3, orient='h')"
]
},
{
"cell_type": "code",
"execution_count": 182,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
" OLS Regression Results \n",
"==============================================================================\n",
"Dep. Variable: y R-squared: 0.998\n",
"Model: OLS Adj. R-squared: 0.998\n",
"Method: Least Squares F-statistic: 1.680e+04\n",
"Date: Mon, 01 Oct 2018 Prob (F-statistic): 1.93e-130\n",
"Time: 11:28:39 Log-Likelihood: -126.13\n",
"No. Observations: 100 AIC: 260.3\n",
"Df Residuals: 96 BIC: 270.7\n",
"Df Model: 3 \n",
"Covariance Type: nonrobust \n",
"==============================================================================\n",
" coef std err t P>|t| [0.025 0.975]\n",
"------------------------------------------------------------------------------\n",
"Intercept 1.0229 0.117 8.713 0.000 0.790 1.256\n",
"x 2.2893 0.139 16.474 0.000 2.013 2.565\n",
"x2 3.0189 0.077 38.971 0.000 2.865 3.173\n",
"x3 3.9099 0.044 88.854 0.000 3.823 3.997\n",
"==============================================================================\n",
"Omnibus: 0.465 Durbin-Watson: 1.885\n",
"Prob(Omnibus): 0.792 Jarque-Bera (JB): 0.386\n",
"Skew: 0.150 Prob(JB): 0.825\n",
"Kurtosis: 2.951 Cond. No. 7.10\n",
"==============================================================================\n",
"\n",
"Warnings:\n",
"[1] Standard Errors assume that the covariance matrix of the errors is correctly specified.\n",
" OLS Regression Results \n",
"==============================================================================\n",
"Dep. Variable: y R-squared: 0.998\n",
"Model: OLS Adj. R-squared: 0.998\n",
"Method: Least Squares F-statistic: 1.680e+04\n",
"Date: Mon, 01 Oct 2018 Prob (F-statistic): 1.93e-130\n",
"Time: 11:28:39 Log-Likelihood: -126.13\n",
"No. Observations: 100 AIC: 260.3\n",
"Df Residuals: 96 BIC: 270.7\n",
"Df Model: 3 \n",
"Covariance Type: nonrobust \n",
"==============================================================================\n",
" coef std err t P>|t| [0.025 0.975]\n",
"------------------------------------------------------------------------------\n",
"Intercept 1.0229 0.117 8.713 0.000 0.790 1.256\n",
"x 2.2893 0.139 16.474 0.000 2.013 2.565\n",
"x2 3.0189 0.077 38.971 0.000 2.865 3.173\n",
"x3 3.9099 0.044 88.854 0.000 3.823 3.997\n",
"==============================================================================\n",
"Omnibus: 0.465 Durbin-Watson: 1.885\n",
"Prob(Omnibus): 0.792 Jarque-Bera (JB): 0.386\n",
"Skew: 0.150 Prob(JB): 0.825\n",
"Kurtosis: 2.951 Cond. No. 7.10\n",
"==============================================================================\n",
"\n",
"Warnings:\n",
"[1] Standard Errors assume that the covariance matrix of the errors is correctly specified.\n",
" OLS Regression Results \n",
"==============================================================================\n",
"Dep. Variable: y R-squared: 0.998\n",
"Model: OLS Adj. R-squared: 0.998\n",
"Method: Least Squares F-statistic: 1.680e+04\n",
"Date: Mon, 01 Oct 2018 Prob (F-statistic): 1.93e-130\n",
"Time: 11:28:39 Log-Likelihood: -126.13\n",
"No. Observations: 100 AIC: 260.3\n",
"Df Residuals: 96 BIC: 270.7\n",
"Df Model: 3 \n",
"Covariance Type: nonrobust \n",
"==============================================================================\n",
" coef std err t P>|t| [0.025 0.975]\n",
"------------------------------------------------------------------------------\n",
"Intercept 1.0229 0.117 8.713 0.000 0.790 1.256\n",
"x 2.2893 0.139 16.474 0.000 2.013 2.565\n",
"x2 3.0189 0.077 38.971 0.000 2.865 3.173\n",
"x3 3.9099 0.044 88.854 0.000 3.823 3.997\n",
"==============================================================================\n",
"Omnibus: 0.465 Durbin-Watson: 1.885\n",
"Prob(Omnibus): 0.792 Jarque-Bera (JB): 0.386\n",
"Skew: 0.150 Prob(JB): 0.825\n",
"Kurtosis: 2.951 Cond. No. 7.10\n",
"==============================================================================\n",
"\n",
"Warnings:\n",
"[1] Standard Errors assume that the covariance matrix of the errors is correctly specified.\n"
]
}
],
"source": [
"print(best.sort_values('adj_R_sq', ascending=False)['model'].iloc[0].summary())\n",
"print(best.sort_values('bic', ascending=True)['model'].iloc[0].summary())\n",
"print(best.sort_values('aic', ascending=True)['model'].iloc[0].summary())"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"(d) Repeat (c), using forward stepwise selection and also using backwards stepwise selection. How does your answer compare to the results in (c)?"
]
},
{
"cell_type": "code",
"execution_count": 236,
"metadata": {},
"outputs": [],
"source": [
"# Forward selection:\n",
"# There are 1 + 10(1 + 10) / 2 = 56 possible models in this space \n",
"\n",
"def fwd_selection(target_column, data):\n",
" predictors = data.drop('y', axis=1).columns\n",
" formulae_s1 = [(p, 'y ~ {}'.format(p)) for p in predictors]\n",
" models_s1 = [(p, smf.ols(formula=f, data=data).fit()) for p, f in formulae_s1]\n",
" predictor_s1, model_s1 = sorted(models_s1, key=lambda tup: tup[1].rsquared, reverse=True)[0]\n",
" predictors = predictors.drop(predictor_s1)\n",
" formula = 'y ~ {}'.format(predictor_s1)\n",
" step_models = [model_s1]\n",
" while len(predictors) > 0:\n",
" models = []\n",
" for p in predictors:\n",
" f = formula + ' + ' + p\n",
" m = smf.ols(formula=f, data=data).fit()\n",
" models.append((p, m))\n",
" \n",
" predictor, model = sorted(models, key=lambda tup: tup[1].rsquared, reverse=True)[0]\n",
" step_models.append(model)\n",
" formula = formula + ' + ' + predictor\n",
" predictors = predictors.drop(predictor)\n",
" \n",
" return pd.DataFrame({'model': step_models, 'n_predictors' : range(1,data.shape[1])})"
]
},
{
"cell_type": "code",
"execution_count": 184,
"metadata": {},
"outputs": [],
"source": [
"# Select between models with different numbers of predictors using adjusted training metrics\n",
"best = fwd_selection('y', data)\n",
"best['adj_R_sq'] = best['model'].map(lambda m: m.rsquared_adj)\n",
"best['bic'] = best['model'].map(lambda m: m.bic)\n",
"best['aic'] = best['model'].map(lambda m: m.aic)"
]
},
{
"cell_type": "code",
"execution_count": 185,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"<matplotlib.axes._subplots.AxesSubplot at 0x118d6fda0>"
]
},
"execution_count": 185,
"metadata": {},
"output_type": "execute_result"
},
{
"data": {
"image/png": "iVBORw0KGgoAAAANSUhEUgAAA3kAAANgCAYAAACRKaGGAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzs3X20pnVdL/73xxkIQQQE9Ecgjp0MMRO1vfih+JCghWZYhqY/zYf4Oee3UsPy1NFs+XSOa9UxrY49nCZFtIxSBEMrH1KERMX2ICowkIpPIDkqj8JKAT+/P/Y9pznjwOx7z733tfc1r9da99r3dd3X3Nfbtb7O5j3f6/pe1d0BAABgHO42dAAAAABmR8kDAAAYESUPAABgRJQ8AACAEVHyAAAARkTJAwAAGBElDwAAYESUPAAAgBFR8gAAAEZk/dABFuuQQw7pDRs2DB0DAABgEJs3b/5Wdx+6q+PWTMnbsGFD5ufnh44BAAAwiKr6ymKOc7kmAADAiKyZmbwtV387P/mbbx86BgAAMIDNr3/O0BHWDDN5AAAAI6LkAQAAjIiSBwAAMCJKHgAAwIgoeQAAACMySMmrqtOramtVXTrE+QEAAMZqqJm8M5KcNNC5AQAARmuQktfdFyS5bohzAwAAjNmqvievqjZW1XxVzd9+681DxwEAAFj1VnXJ6+5N3T3X3XPr991/6DgAAACr3qoueQAAAExHyQMAABiRoR6hcGaSTyQ5qqqurqpTh8gBAAAwNuuHOGl3P3OI8wIAAIydyzUBAABGRMkDAAAYESUPAABgRJQ8AACAERlk4ZWlOPqIgzP/+ucMHQMAAGBVM5MHAAAwIkoeAADAiCh5AAAAI6LkAQAAjMiaWXjle9delq++9ieGjgEAAIzEka/83NARloWZPAAAgBFR8gAAAEZEyQMAABgRJQ8AAGBElDwAAIARGaTkVdV9q+q8qrq8qi6rqtOGyAEAADA2Qz1C4fYkL+3ui6tq/ySbq+pD3X35QHkAAABGYZCZvO6+trsvnry/OcmWJIcPkQUAAGBMBr8nr6o2JHlYkot28tnGqpqvqvnrbrljpaMBAACsOYOWvKq6R5J3J3lJd9+04+fdvam757p77l77rVv5gAAAAGvMYCWvqvbKQsF7R3efPVQOAACAMRlqdc1K8pYkW7r7jUNkAAAAGKOhZvKOT/LLSU6oqksmrycNlAUAAGA0BnmEQnd/LEkNcW4AAIAxG3x1TQAAAGZHyQMAABgRJQ8AAGBElDwAAIARGWThlaXY+7Afz5GvnB86BgAAwKpmJg8AAGBElDwAAIARUfIAAABGRMkDAAAYkTWz8MoVW6/I8W86fugYAADASF344guHjjATZvIAAABGRMkDAAAYESUPAABgRJQ8AACAEVHyAAAARmSQkldV+1TVp6rqM1V1WVW9ZogcAAAAYzPUIxS+m+SE7v5OVe2V5GNV9Y/d/cmB8gAAAIzCICWvuzvJdyabe01ePUQWAACAMRnsnryqWldVlyTZmuRD3X3RTo7ZWFXzVTV/23duW/mQAAAAa8xgJa+77+juhyY5IsmxVfXgnRyzqbvnuntur3vstfIhAQAA1pjBV9fs7huSnJfkpKGzAAAArHVDra55aFUdOHl/9yRPSHLFEFkAAADGZKjVNQ9L8raqWpeFovnO7n7fQFkAAABGY6jVNT+b5GFDnBsAAGDMBr8nDwAAgNlR8gAAAEZEyQMAABgRJQ8AAGBEhlpdc2oPvPcDc+GLLxw6BgAAwKpmJg8AAGBElDwAAIARUfIAAABGRMkDAAAYkTWz8MrNV16Z8x/z2KFjAAAAI/XYC84fOsJMmMkDAAAYESUPAABgRJQ8AACAEVHyAAAARkTJAwAAGJFBS15VrauqT1fV+4bMAQAAMBZDz+SdlmTLwBkAAABGY7CSV1VHJPnZJG8eKgMAAMDYDDmT94dJfivJ9+/sgKraWFXzVTV/4223rVwyAACANWqQkldVT06ytbs339Vx3b2pu+e6e+6AvfZaoXQAAABr11AzeccnObmqvpzkb5KcUFV/NVAWAACA0Rik5HX3y7v7iO7ekOQZST7S3c8eIgsAAMCYDL26JgAAADO0fugA3f3RJB8dOAYAAMAomMkDAAAYESUPAABgRBZd8qpqv6q62+T9j1XVyVXluQYAAACryDQzeRck2aeqDk/ywSS/nOSM5QgFAADA0kyz8Ep1961VdWqSP+3u/1FVlyxXsB3tf9RReewF56/U6QAAANakaWbyqqoekeRZSf5+sm/d7CMBAACwVNOUvNOSvDzJOd19WVX9SJLzlicWAAAAS7GoyzWral2Sk7v75G37uvuqJL+2XMEAAACY3qJm8rr7jiSPWuYsAAAA7KZpFl75dFWdm+RdSW7ZtrO7z555qp3YevWN+eOXvnclTgUAAOyBXvSGnxs6wkxMU/L2SfLtJCdst6+TrEjJAwAAYNcWXfK6+/nLGQQAAIDdt+jVNavqiKo6p6q2Tl7vrqojljMcAAAA05nmEQpvTXJukh+evN472QcAAMAqMU3JO7S739rdt09eZyQ5dJlyAQAAsATTLLzy7ap6dpIzJ9vPzMJCLEtSVV9OcnOSO5Lc3t1zS/0uAAAAFkxT8n4lyZuS/EEWVtX8eJLn7eb5H9fd39rN7wAAAGBimpJ3RHefvP2Oqjo+yddmGwkAAIClmuaevDctct9idZIPVtXmqtq4swOqamNVzVfV/HduvXE3TgUAALBn2OVMXlU9IskjkxxaVb+x3Uf3TLJuN879qO6+pqruneRDVXVFd1+w/QHdvSnJpiQ58v96QO/GuQAAAPYIi5nJ2zvJPbJQCPff7nVTklOWeuLuvmbyc2uSc5Icu9TvAgAAYMEuZ/K6+/wk51fVGd39lVmctKr2S3K37r558v6nk7x2Ft8NAACwJ5vmnrw3V9WB2zaq6qCq+sASz3ufJB+rqs8k+VSSv+/u9y/xuwAAAJiYZnXNQ7r7hm0b3X395H66qXX3VUmOWcqfBQAA4M5NM5P3/ao6cttGVd0vCytkAgAAsEpMM5P3iixcYnl+kkry6CQ7ffQBAAAAw1h0yevu91fVw5McN9n1ku7+1vLEAgAAYCl2eblmVT1w8vPhSY5M8vXJ68jJPgAAAFaJxczkvTTJC5K8YSefdZITZproTtz7iAPyojf83EqcCgAAYM1azHPyXjD5+bjljwMAAMDu2GXJq6qn3tXn3X327OIAAACwOxZzuea2ayTvneSRST4y2X5cko8nUfIAAABWicVcrvn8JKmqDyZ5UHdfO9k+LMkZy5oOAACAqUzznLz7bit4E9/IwmqbK+LaL30xr3v2KSt1OgAAYA/zir86a+gIMzFNyftwVX0gyZmT7V9K8k+zjwQAAMBSTfMw9BdV1S8kecxk16buPmd5YgEAALAU08zkJcnFSW7u7n+qqn2rav/uvnk5ggEAADC9uy32wKp6QZKzkvz5ZNfhSd6zHKEAAABYmkWXvCQvTHJ8kpuSpLs/n4XHKgAAALBKTFPyvtvd39u2UVXrk/RST1xVB1bVWVV1RVVtqapHLPW7AAAAWDDNPXnnV9VvJ7l7VT0hya8mee9unPuPkry/u0+pqr2T7Lsb3wUAAECmm8l7WZJvJvlckv+c5B+S/M5STlpVB2Rhlc63JEl3f6+7b1jKdwEAAPAfFjWTV1Xrkry9u5+V5C9mcN77Z6EwvrWqjkmyOclp3X3LDufdmGRjkhyw791ncFoAAIBxW9RMXnffkeR+k8sqZ2F9kocn+bPufliSW7IwU7jjeTd191x3z+23zw/N6NQAAADjNc09eVclubCqzs1CKUuSdPcbl3Deq5Nc3d0XTbbPyk5KHgAAANOZpuR9cfK6W5L9d+ek3f1vVfW1qjqqu69McmKSy3fnOwEAAJii5HX3a5Kkqu65sNk37+a5X5zkHZNLQK9K8vzd/D4AAIA93qJLXlXNJXlrJrN4VXVjkl/p7s1LOXF3X5Jkbil/FgAAgJ2b5nLN05P8anf/c5JU1aOyUPoeshzBAAAAmN40z8m7Y1vBS5Lu/liS22cfCQAAgKWaZibv/Kr68yRnJukkv5Tko1X18CTp7ouXIR8AAABTmKbkHTP5+aod9j8sC6XvhJkkAgAAYMmqu2fzRVXP7e63zeTLdmJubq7n5+eX6+sBAABWtara3N27XLxymnvyduW0GX4XAAAASzDLklcz/C4AAACWYJYlbzbXfQIAALBkZvIAAABGZNGra1bVDyX5xSQbtv9z3f3aydsLZ5psB/9+7c3Z8rqPLOcpAACAPdjRrxjHAwOmeYTC3yW5McnmJN/d8cPuftGsQgEAALA005S8I7r7pGVLAgAAwG6b5p68j1fVTyxbEgAAAHbbNDN5j0ryvKr6UhYu16wk3d0PWZZkAAAATG2akvfEZUsBAADATCy65HX3V2Z10qo6KsnfbrfrR5K8srv/cFbnAAAA2BNNM5M3M919ZZKHJklVrUtyTZJzhsgCAAAwJrN8GPpSnZjki7OcKQQAANhTrYaS94wkZ+7sg6raWFXzVTV/3S03rHAsAACAtWfQkldVeyc5Ocm7dvZ5d2/q7rnunrvXfgeubDgAAIA1aOiZvCcmubi7vzFwDgAAgFEYuuQ9M3dyqSYAAADTG6zkVdV+SZ6Q5OyhMgAAAIzNII9QSJLuviXJwUOdHwAAYIyGvlwTAACAGVLyAAAARkTJAwAAGBElDwAAYEQGW3hlWvsctn+OfsUJQ8cAAABY1czkAQAAjIiSBwAAMCJKHgAAwIgoeQAAACOyZhZe+frXv55Xv/rVQ8cAAABGaix9w0weAADAiCh5AAAAI6LkAQAAjIiSBwAAMCJKHgAAwIgMVvKq6ter6rKqurSqzqyqfYbKAgAAMBaDlLyqOjzJryWZ6+4HJ1mX5BlDZAEAABiTIS/XXJ/k7lW1Psm+Sb4+YBYAAIBRGKTkdfc1SX4/yVeTXJvkxu7+4I7HVdXGqpqvqvlbb711pWMCAACsOUNdrnlQkqckuX+SH06yX1U9e8fjuntTd89199y+++670jEBAADWnKEu13x8ki919ze7+7YkZyd55EBZAAAARmOokvfVJMdV1b5VVUlOTLJloCwAAACjMdQ9eRclOSvJxUk+N8mxaYgsAAAAY7J+qBN396uSvGqo8wMAAIzRkI9QAAAAYMaUPAAAgBFR8gAAAEZEyQMAABiR6u6hMyzK3Nxcz8/PDx0DAABgEFW1ubvndnWcmTwAAIARUfIAAABGRMkDAAAYESUPAABgRNYPHWCxrr9+S975rmOHjgEAAIzU05/2qaEjzISZPAAAgBFR8gAAAEZEyQMAABgRJQ8AAGBElDwAAIARGazkVdVpVXVpVV1WVS8ZKgcAAMCYDFLyqurBSV6Q5NgkxyR5clX96BBZAAAAxmSombyjk1zU3bd29+1Jzk/y1IGyAAAAjMZQJe/SJI+uqoOrat8kT0py3x0PqqqNVTVfVfM33XT7iocEAABYa9YPcdLu3lJVv5fkg0luSXJJkjt2ctymJJuS5D/9p/16RUMCAACsQYMtvNLdb+nun+zuxyS5Psm/DpUFAABgLAaZyUuSqrp3d2+tqiOzcD/ecUNlAQAAGIvBSl6Sd1fVwUluS/LC7r5hwCwAAACjMFjJ6+5HD3VuAACAsRrsnjwAAABmT8kDAAAYESUPAABgRJQ8AACAERlydc2pHHTQ0Xn60z41dAwAAIBVzUweAADAiCh5AAAAI6LkAQAAjIiSBwAAMCJrZuGVy6+/Kcec9YGhYwAAACP1mVN+ZugIM2EmDwAAYESUPAAAgBFR8gAAAEZEyQMAABgRJQ8AAGBElrXkVdXpVbW1qi7dbt+9qupDVfX5yc+DljMDAADAnmS5Z/LOSHLSDvteluTD3f2AJB+ebAMAADADy1ryuvuCJNftsPspSd42ef+2JD+/nBkAAAD2JEPck3ef7r528v7fktznzg6sqo1VNV9V87ffdOPKpAMAAFjDBl14pbs7Sd/F55u6e66759bf84AVTAYAALA2DVHyvlFVhyXJ5OfWATIAAACM0hAl79wkz528f26SvxsgAwAAwCgt9yMUzkzyiSRHVdXVVXVqkt9N8oSq+nySx0+2AQAAmIH1y/nl3f3MO/noxOU8LwAAwJ5q0IVXAAAAmC0lDwAAYESUPAAAgBFR8gAAAEZkWRdemaUHHXTPzJ/yM0PHAAAAWNXM5AEAAIxIdffQGRalqm5OcuXQOdgjHZLkW0OHYI9l/DEk448hGX8MZTWPvft196G7OmjNXK6Z5Mrunhs6BHueqpo39hiK8ceQjD+GZPwxlDGMPZdrAgAAjIiSBwAAMCJrqeRtGjoAeyxjjyEZfwzJ+GNIxh9DWfNjb80svAIAAMCuraWZPAAAAHZByQMAABiRVVXyquqkqrqyqr5QVS/byec/VFV/O/n8oqrasPIpGatFjL/fqKrLq+qzVfXhqrrfEDkZp12Nv+2O+8Wq6qpa00s7s7osZvxV1dMnfwdeVlV/vdIZGadF/O49sqrOq6pPT37/PmmInIxTVZ1eVVur6tI7+byq6n9Oxudnq+rhK51xqVZNyauqdUn+JMkTkzwoyTOr6kE7HHZqkuu7+0eT/EGS31vZlIzVIsffp5PMdfdDkpyV5H+sbErGapHjL1W1f5LTkly0sgkZs8WMv6p6QJKXJzm+u388yUtWPCijs8i/+34nyTu7+2FJnpHkT1c2JSN3RpKT7uLzJyZ5wOS1McmfrUCmmVg1JS/JsUm+0N1Xdff3kvxNkqfscMxTkrxt8v6sJCdWVa1gRsZrl+Ovu8/r7lsnm59McsQKZ2S8FvP3X5L8tyz849a/r2Q4Rm8x4+8FSf6ku69Pku7eusIZGafFjL1Ocs/J+wOSfH0F8zFy3X1Bkuvu4pCnJHl7L/hkkgOr6rCVSbd7VlPJOzzJ17bbvnqyb6fHdPftSW5McvCKpGPsFjP+tndqkn9c1kTsSXY5/iaXiNy3u/9+JYOxR1jM338/luTHqurCqvpkVd3Vv3zDYi1m7L06ybOr6uok/5DkxSsTDZJM/9+Hq8b6oQPAWlNVz04yl+SxQ2dhz1BVd0vyxiTPGzgKe671Wbhc6aeycBXDBVX1E919w6Cp2BM8M8kZ3f2GqnpEkr+sqgd39/eHDgar2WqaybsmyX232z5ism+nx1TV+ixM2397RdIxdosZf6mqxyd5RZKTu/u7K5SN8dvV+Ns/yYOTfLSqvpzkuCTnWnyFGVnM339XJzm3u2/r7i8l+dcslD7YHYsZe6cmeWeSdPcnkuyT5JAVSQeL/O/D1Wg1lbx/SfKAqrp/Ve2dhZtrz93hmHOTPHfy/pQkH2lPc2c2djn+quphSf48CwXP/SjM0l2Ov+6+sbsP6e4N3b0hC/eEntzd88PEZWQW8/v3PVmYxUtVHZKFyzevWsmQjNJixt5Xk5yYJFV1dBZK3jdXNCV7snOTPGeyyuZxSW7s7muHDrUYq+Zyze6+vapelOQDSdYlOb27L6uq1yaZ7+5zk7wlC9P0X8jCTZLPGC4xY7LI8ff6JPdI8q7Jej9f7e6TBwvNaCxy/MGyWOT4+0CSn66qy5PckeQ3u9uVNOyWRY69lyb5i6r69SwswvI8/8DPrFTVmVn4B6xDJvd9virJXknS3f8rC/eBPinJF5LcmuT5wySdXvn/CQAAwHispss1AQAA2E1KHgAAwIgoeQAAACOi5AEAAIyIkgcAADAiSh4Ae6Sqel5V/fHk/f9XVc+5i2PPqKovVdUlVfWZqjpx5ZICwHRWzXPyAGAok+ch7cpvdvdZVfW4JJuSPGCZYwHAkpjJA2CUquo9VbW5qi6rqo2Tfc+vqn+tqk8lOX67Y19dVf9lkV/9iSSH7+Lcv1tVl1fVZ6vq9yf77l9Vn6iqz1XVf6+q7yzxfxoA3CUzeQCM1a9093VVdfck/1JVf5/kNUl+MsmNSc5L8uklfO9JSd5zZx9W1cFJfiHJA7u7q+rAyUd/lOTPuvvtVfXCJZwXABbFTB4AY/VrVfWZJJ9Mct8kv5zko939ze7+XpK/nfL7Xl9V/5rkr5P83l0cd2OSf0/ylqp6apJbJ/uPT3Lm5P1fTnluAFg0JQ+A0amqn0ry+CSP6O5jsjBjd8Vufu1vdvePJfmvSU6/s4O6+/YkxyY5K8mTk7x/+493MwMA7JKSB8AYHZDk+u6+taoemOS4JHdP8tiqOriq9krytCV+9x8nuVtV/czOPqyqeyQ5oLv/IcmvJzlm8tGFSZ4xef+sJZ4bAHZJyQNgjN6fZH1VbUnyu1m4ZPPaJK/OwsIpFybZssOfWdQsW3d3kv+e5Lfu5JD9k7yvqj6b5GNJfmOy/7QkL6yqz2UXC7cAwO6ohd9VALDnqqo3Jbm4u9+6guf8TnffY6XOB8Cew0weAHu0qvpvSf7vJOcOnQUAZsFMHgBMVNWfZLvn50380Z3N8FXVOUnuv8Pu/9rdH1iOfACwGEoeAADAiLhcEwAAYESUPAAAgBFR8gAAAEZEyQMAABgRJQ8AAGBElDwAAIARUfIAAABGRMkDAAAYESUPAABgRJQ8AACAEVHyAAAARkTJAwAAGBElDwAAYESUPAAAgBFZP3SAxTrkkEN6w4YNQ8cAAAAYxObNm7/V3Yfu6rg1U/I2bNiQ+fn5oWMAAAAMoqq+spjjXK4JAAAwImtmJm/L1d/OT/7m24eOAQAAjNTm1z9n6AgzYSYPAABgRJQ8AACAEVHyAAAARkTJAwAAGJFBSl5VnV5VW6vq0iHODwAAMFZDzeSdkeSkgc4NAAAwWoOUvO6+IMl1Q5wbAABgzNyTBwAAMCKruuRV1caqmq+q+dtvvXnoOAAAAKveqi553b2pu+e6e279vvsPHQcAAGDVW9UlDwAAgOkM9QiFM5N8IslRVXV1VZ06RA4AAICxWT/ESbv7mUOcFwAAYOxcrgkAADAiSh4AAMCIKHkAAAAjouQBAACMiJIHAAAwIoOsrrkURx9xcOZf/5yhYwAAAKxqZvIAAABGRMkDAAAYESUPAABgRJQ8AACAEVkzC69879rL8tXX/sTQMQAAVo0jX/m5oSMAq5CZPAAAgBFR8gAAAEZEyQMAABgRJQ8AAGBEBil5VXXfqjqvqi6vqsuq6rQhcgAAAIzNUKtr3p7kpd19cVXtn2RzVX2ouy8fKA8AAMAoDDKT193XdvfFk/c3J9mS5PAhsgAAAIzJ4PfkVdWGJA9LctGwSQAAANa+QUteVd0jybuTvKS7b9rJ5xurar6q5q+75Y6VDwgAALDGDFbyqmqvLBS8d3T32Ts7prs3dfdcd8/da791KxsQAABgDRpqdc1K8pYkW7r7jUNkAAAAGKOhZvKOT/LLSU6oqksmrycNlAUAAGA0BnmEQnd/LEkNcW4AAIAxG3x1TQAAAGZHyQMAABgRJQ8AAGBElDwAAIARUfIAAABGZJDVNZdi78N+PEe+cn7oGAAAAKuamTwAAIARUfIAAABGRMkDAAAYESUPAABgRNbMwitXbL0ix7/p+KFjALCKXfjiC4eOAACDM5MHAAAwIkoeAADAiCh5AAAAI6LkAQAAjMggJa+q9qmqT1XVZ6rqsqp6zRA5AAAAxmao1TW/m+SE7v5OVe2V5GNV9Y/d/cmB8gAAAIzCICWvuzvJdyabe01ePUQWAACAMRnsnryqWldVlyTZmuRD3X3RUFkAAADGYrCS1913dPdDkxyR5NiqevCOx1TVxqqar6r5275z28qHBAAAWGMGX12zu29Icl6Sk3by2abunuvuub3usdfKhwMAAFhjhlpd89CqOnDy/u5JnpDkiiGyAAAAjMlQq2seluRtVbUuC0Xznd39voGyAAAAjMZQq2t+NsnDhjg3AADAmA1+Tx4AAACzo+QBAACMiJIHAAAwIkoeAADAiCh5AAAAIzLUIxSm9sB7PzAXvvjCoWMAAACsambyAAAARkTJAwAAGBElDwAAYESUPAAAgBFZMwuv3HzllTn/MY8dOgYAa9BjLzh/6AgAsGLM5AEAAIyIkgcAADAiSh4AAMCIKHkAAAAjMmjJq6p1VfXpqnrfkDkAAADGYuiZvNOSbBk4AwAAwGgMVvKq6ogkP5vkzUNlAAAAGJshZ/L+MMlvJfn+gBkAAABGZZCSV1VPTrK1uzfv4riNVTVfVfM33nbbCqUDAABYu4aayTs+yclV9eUkf5PkhKr6qx0P6u5N3T3X3XMH7LXXSmcEAABYcwYped398u4+ors3JHlGko9097OHyAIAADAmQ6+uCQAAwAytHzpAd380yUcHjgEAADAKZvIAAABGRMkDAAAYkUWXvKrar6ruNnn/Y1V1clVZ8hIAAGAVmWYm74Ik+1TV4Uk+mOSXk5yxHKEAAABYmmlKXnX3rUmemuRPu/tpSX58eWIBAACwFNOsrllV9Ygkz0py6mTfutlH2rn9jzoqj73g/JU6HQAAwJo0zUzeaUlenuSc7r6sqn4kyXnLEwsAAIClWNRMXlWtS3Jyd5+8bV93X5Xk15YrGAAAANNb1Exed9+R5FHLnAUAAIDdNM09eZ+uqnOTvCvJLdt2dvfZM08FAADAkkxT8vZJ8u0kJ2y3r5OsSMnbevWN+eOXvnclTgXAHuRFb/i5oSMAwEwtuuR19/OXMwgAAAC7b9Gra1bVEVV1TlVtnbzeXVVHLGc4AAAApjPNIxTemuTcJD88eb13sg8AAIBVYpqSd2h3v7W7b5+8zkhy6DLlAgAAYAmmWXjl21X17CRnTrafmYWFWJakqr6c5OYkdyS5vbvnlvpdAAAALJim5P1Kkjcl+YMsrKr58STP283zP667v7Wb3wEAAMDENCXviO4+efsdVXV8kq/NNhIAAABLNc09eW9a5L7F6iQfrKrNVbVxN74HAACAiV3O5FXVI5I8MsmhVfUb2310zyTrduPcj+rua6rq3kk+VFVXdPcFO5x7Y5KNSXLQ/tZ4AQAA2JXFzOTtneQeWSiE+2/3uinJKUs9cXdfM/m5Nck5SY7dyTGbunuuu+fuse8BSz0VAADAHmOXM3ndfX6S86vqjO7+yixOWlX7Jblbd9+L9038AAAgAElEQVQ8ef/TSV47i+8GAADYk01zT96bq+rAbRtVdVBVfWCJ571Pko9V1WeSfCrJ33f3+5f4XQAAAExMs7rmId19w7aN7r5+cj/d1Lr7qiTHLOXPAgAAcOemmcn7flUduW2jqu6XhRUyAQAAWCWmmcl7RRYusTw/SSV5dCYrXwIAALA6LLrkdff7q+rhSY6b7HpJd39reWIBAACwFLu8XLOqHjj5+fAkRyb5+uR15GQfAAAAq8RiZvJemuQFSd6wk886yQkzTQQAAMCSVffaWDtlbm6u5+fnh44BAAAwiKra3N1zuzpulzN5VfXUu/q8u8+eJhgAAADLZzGXa/7c5Oe9kzwyyUcm249L8vEkSh4AAMAqscuS193PT5Kq+mCSB3X3tZPtw5KcsazpAAAAmMo0z8m777aCN/GNLKy2uSKu/dIX87pnn7JSpwNgD/eKvzpr6AgAsCTTlLwPV9UHkpw52f6lJP80+0gAAAAs1TQPQ39RVf1CksdMdm3q7nOWJxYAAABLMc1MXpJcnOTm7v6nqtq3qvbv7puXIxgAAADTu9tiD6yqFyQ5K8mfT3YdnuQ9yxEKAACApVl0yUvywiTHJ7kpSbr781l4rAIAAACrxDQl77vd/b1tG1W1Pkkv9cRVdWBVnVVVV1TVlqp6xFK/CwAAgAXT3JN3flX9dpK7V9UTkvxqkvfuxrn/KMn7u/uUqto7yb678V0AAABkupm8lyX5ZpLPJfnPSf4hye8s5aRVdUAWVul8S5J09/e6+4alfBcAAAD/YVEzeVW1Lsnbu/tZSf5iBue9fxYK41ur6pgkm5Oc1t23zOC7AQAA9liLmsnr7juS3G9yWeUsrE/y8CR/1t0PS3JLFmYK/w9VtbGq5qtq/pZ//+6MTg0AADBe09yTd1WSC6vq3CyUsiRJd79xCee9OsnV3X3RZPus7KTkdfemJJuS5PCDD1ryIi8AAAB7imlK3hcnr7sl2X93Ttrd/1ZVX6uqo7r7yiQnJrl8d74TAACAKUped78mSarqngubffNunvvFSd4xuQT0qiTP383vAwAA2OMtuuRV1VySt2Yyi1dVNyb5le7evJQTd/clSeaW8mcBAADYuWku1zw9ya929z8nSVU9Kgul7yHLEQwAAIDpTfOcvDu2Fbwk6e6PJbl99pEAAABYqmlm8s6vqj9PcmaSTvJLST5aVQ9Pku6+eBnyAQAAMIVpSt4xk5+v2mH/w7JQ+k6YSSIAAACWbJrVNR93V59X1XO7+227HwkAAIClqu7ZPGO8qi7u7ofP5Mt2Ym5urufn55fr6wEAAFa1qtrc3bt8QsE0C6/s8pwz/C4AAACWYJYlbzZTggAAACyZmTwAAIARWfTCK1X1Q0l+McmG7f9cd7928vbCmSbbwb9fe3O2vO4jy3kKALhTR7/CItIArA3TPELh75LcmGRzku/u+GF3v2hWoQAAAFiaaUreEd190rIlAQAAYLdNc0/ex6vqJ5YtCQAAALttmpm8RyV5XlV9KQuXa1aS7u6HLEsyAAAApjZNyXvisqUAAABgJhZd8rr7K7M6aVUdleRvt9v1I0le2d1/OKtzAAAA7Immmcmbme6+MslDk6Sq1iW5Jsk5Q2QBAAAYk1k+DH2pTkzyxVnOFAIAAOypVkPJe0aSM4cOAQAAMAaDlryq2jvJyUnedSefb6yq+aqav+6WG1Y2HAAAwBo09EzeE5Nc3N3f2NmH3b2pu+e6e+5e+x24wtEAAADWnqFL3jPjUk0AAICZGazkVdV+SZ6Q5OyhMgAAAIzNII9QSJLuviXJwUOdHwAAYIyGvlwTAACAGVLyAAAARkTJAwAAGBElDwAAYESUPAAAgBEZbHXNae1z2P45+hUnDB0DAABgVTOTBwAAMCJKHgAAwIgoeQAAACOyZu7J+/rXv55Xv/rVQ8cAgP+D300ArDZm8gAAAEZEyQMAABgRJQ8AAGBElDwAAIARUfIAAABGZLCSV1W/XlWXVdWlVXVmVe0zVBYAAICxGKTkVdXhSX4tyVx3PzjJuiTPGCILAADAmAx5ueb6JHevqvVJ9k3y9QGzAAAAjMIgJa+7r0ny+0m+muTaJDd29weHyAIAADAmQ12ueVCSpyS5f5IfTrJfVT17J8dtrKr5qpq/9dZbVzomAADAmjPU5ZqPT/Kl7v5md9+W5Owkj9zxoO7e1N1z3T237777rnhIAACAtWaokvfVJMdV1b5VVUlOTLJloCwAAACjMdQ9eRclOSvJxUk+N8mxaYgsAAAAY7J+qBN396uSvGqo8wMAAIzRkI9QAAAAYMaUPAAAgBFR8gAAAEZEyQMAABgRJQ8AAGBEqruHzrAoc3NzPT8/P3QMAACAQVTV5u6e29VxZvIAAABGRMkDAAAYESUPAABgRNYPHWCxrr9+S975rmOHjgEAu/T0p31q6AgA7MHM5AEAAIyIkgcAADAiSh4AAMCIKHkAAAAjouQBAACMyGAlr6pOq6pLq+qyqnrJUDkAAADGZJCSV1UPTvKCJMcmOSbJk6vqR4fIAgAAMCZDzeQdneSi7r61u29Pcn6Spw6UBQAAYDSGKnmXJnl0VR1cVfsmeVKS+w6UBQAAYDTWD3HS7t5SVb+X5INJbklySZI7djyuqjYm2Zgkhxyy94pmBAAAWIsGW3ilu9/S3T/Z3Y9Jcn2Sf93JMZu6e6675+55z0H6KAAAwJoyWHOqqnt399aqOjIL9+MdN1QWAACAsRhyeuzdVXVwktuSvLC7bxgwCwAAwCgMVvK6+9FDnRsAAGCsBrsnDwAAgNlT8gAAAEZEyQMAABgRJQ8AAGBElDwAAIARWTNPGD/ooKPz9Kd9augYAAAAq5qZPAAAgBFR8gAAAEZEyQMAABiRNXNP3uXX35RjzvrA0DEAYEk+c8rPDB0BgD2EmTwAAIARUfIAAABGRMkDAAAYESUPAABgRJQ8AACAEVnWkldVp1fV1qq6dLt996qqD1XV5yc/D1rODAAAAHuS5Z7JOyPJSTvse1mSD3f3A5J8eLINAADADCxryevuC5Jct8PupyR52+T925L8/HJmAAAA2JMMcU/efbr72sn7f0tynwEyAAAAjNKgC690dyfpO/u8qjZW1XxVzd9+040rmAwAAGBtGqLkfaOqDkuSyc+td3Zgd2/q7rnunlt/zwNWLCAAAMBaNUTJOzfJcyfvn5vk7wbIAAAAMErL/QiFM5N8IslRVXV1VZ2a5HeTPKGqPp/k8ZNtAAAAZmD9cn55dz/zTj46cTnPCwAAsKcadOEVAAAAZkvJAwAAGBElDwAAYESUPAAAgBFR8gAAAEZkWVfXnKUHHXTPzJ/yM0PHAAAAWNXM5AEAAIxIdffQGRalqm5OcuXQOVjzDknyraFDsOYZR8yCccQsGEfMgnG0dtyvuw/d1UFr5nLNJFd299zQIVjbqmreOGJ3GUfMgnHELBhHzIJxND4u1wQAABgRJQ8AAGBE1lLJ2zR0AEbBOGIWjCNmwThiFowjZsE4Gpk1s/AKAAAAu7aWZvIAAADYhVVf8qrqpKq6sqq+UFUvGzoPq1dVnV5VW6vq0u323auqPlRVn5/8PGiyv6rqf07G1Wer6uHDJWc1qar7VtV5VXV5VV1WVadN9htLLFpV7VNVn6qqz0zG0Wsm++9fVRdNxsvfVtXek/0/NNn+wuTzDUPmZ3WpqnVV9emqet9k2zhiKlX15ar6XFVdUlXzk31+r43Yqi55VbUuyZ8keWKSByV5ZlU9aNhUrGJnJDlph30vS/Lh7n5Akg9PtpOFMfWAyWtjkj9boYysfrcneWl3PyjJcUleOPl7x1hiGt9NckJ3H5PkoUlOqqrjkvxekj/o7h9Ncn2SUyfHn5rk+sn+P5gcB9uclmTLdtvGEUvxuO5+6HaPSvB7bcRWdclLcmySL3T3Vd39vSR/k+QpA2dileruC5Jct8PupyR52+T925L8/Hb7394LPpnkwKo6bGWSspp197XdffHk/c1Z+A+rw2MsMYXJePjOZHOvyauTnJDkrMn+HcfRtvF1VpITq6pWKC6rWFUdkeRnk7x5sl0xjpgNv9dGbLWXvMOTfG277asn+2Cx7tPd107e/1uS+0zeG1vs0uRSp4cluSjGElOaXGJ3SZKtST6U5ItJbuju2yeHbD9W/vc4mnx+Y5KDVzYxq9QfJvmtJN+fbB8c44jpdZIPVtXmqto42ef32oitHzoArJTu7qqynCyLUlX3SPLuJC/p7pu2/8dwY4nF6O47kjy0qg5Mck6SBw4ciTWmqp6cZGt3b66qnxo6D2vao7r7mqq6d5IPVdUV23/o99r4rPaZvGuS3He77SMm+2CxvrHtEoPJz62T/cYWd6qq9spCwXtHd5892W0ssSTdfUOS85I8IguXPW37B9btx8r/HkeTzw9I8u0Vjsrqc3ySk6vqy1m4ZeWEJH8U44gpdfc1k59bs/CPTsfG77VRW+0l71+SPGCyitTeSZ6R5NyBM7G2nJvkuZP3z03yd9vtf85kBanjkty43SUL7MEm96+8JcmW7n7jdh8ZSyxaVR06mcFLVd09yROycH/neUlOmRy24zjaNr5OSfKR9iDbPV53v7y7j+juDVn4b6CPdPezYhwxharar6r23/Y+yU8nuTR+r43aqn8YelU9KQvXo69Lcnp3v27gSKxSVXVmkp9KckiSbyR5VZL3JHlnkiOTfCXJ07v7usl/yP9xFlbjvDXJ87t7fojcrC5V9agk/5zkc/mPe2B+Owv35RlLLEpVPSQLCxmsy8I/qL6zu19bVT+ShRmZeyX5dJJnd/d3q2qfJH+ZhXtAr0vyjO6+apj0rEaTyzX/S3c/2ThiGpPxcs5kc32Sv+7u11XVwfF7bbRWfckDAABg8Vb75ZoAAABMQckDAAAYESUPAABgRJQ8AACAEVHyAAAARkTJA4DtVNWGqrp0J/vfXFUPGiITAExj/dABAGAt6O7/d+gMALAYZvIA4Aetr6p3VNWWqjqrqvatqo9W1VySVNVJVXVxVX2mqj48dFgA2J6ZPAD4QUclObW7L6yq05P86rYPqurQJH+R5DHd/aWqutdQIQFgZ8zkAcAP+lp3Xzh5/1dJHrXdZ8cluaC7v5Qk3X3dSocDgLui5AHAD+pdbAPAqqXkAcAPOrKqHjF5//8k+dh2n30yyWOq6v5J4nJNAFYbJQ8AftCVSV5Y/3979x5s2VmWCfx56U4MCSGEJFCYEAMlw0Xk5ikqEOQSQANiUCYoFAgiRU+VgGGGGQtlSi5VVGk5Ag7OUPZwCSgTgZBwcRwgckkENXg6EEhoMlwETYi0kAshKYHEd/44q7Xp6thnn97nrL1Xfr+qXWevtVfv9fzxVZ9++lvrW1W7kxyb5I17P+juf0yyI8n5VXVZkneOExEADqy6XYECAAAwFWbyAAAAJkTJAwAAmBAlDwAAYEKUPAAAgAlR8gAAACZEyQMAAJgQJQ8AAGBClDwAAIAJUfIAAAAmRMkDAACYECUPAABgQpQ8AACACVHyAAAAJkTJAwAAmBAlDwAAYEKUPAAAgAlR8gAAACZEyQMAAJiQ7WMHWK/jjz++TznllLFjAAAAjGLXrl3f7O4TDnbc0pS8U045Jaurq2PHAAAAGEVVfW09xy1Nydt91bfyE//l7WPHAAAAJmrX7z5n7Ahz4Z48AACACVHyAAAAJkTJAwAAmBAlDwAAYEKUPAAAgAkZpeRV1Vuqak9VXT7G+QEAAKZqrJm8c5KcMdK5AQAAJmuUktfdFye5doxzAwAATJl78gAAACZkoUteVe2oqtWqWr3l5hvHjgMAALDwFrrkdffO7l7p7pXtRx49dhwAAICFt9AlDwAAgNmM9QiFc5P8VZL7VtVVVfX8MXIAAABMzfYxTtrdzxzjvAAAAFPnck0AAIAJUfIAAAAmRMkDAACYECUPAABgQpQ8AACACRlldc2NuP9Jx2X1d58zdgwAAICFZiYPAABgQpQ8AACACVHyAAAAJmRp7sn73jVX5O9e/eNjxwAA2HIn/9bnxo4ALBEzeQAAABOi5AEAAEyIkgcAADAhSh4AAMCEKHkAAAATMkrJq6p7VtXHqurzVXVFVZ09Rg4AAICpGesRCrckeWl3X1pVRyfZVVUXdvfnR8oDAAAwCaPM5HX3Nd196fD+xiS7k5w4RhYAAIApGf2evKo6JclDk1wybhIAAIDlN2rJq6o7JXlPkpd097cP8PmOqlqtqtVrb7p16wMCAAAsmdFKXlUdlrWC947uPv9Ax3T3zu5e6e6Vux61bWsDAgAALKGxVtesJG9Osru7XztGBgAAgCkaaybvtCS/lOT0qvrM8HrySFkAAAAmY5RHKHT3J5LUGOcGAACYstFX1wQAAGB+lDwAAIAJUfIAAAAmRMkDAACYECUPAABgQkZZXXMjDr/Hj+Xk31odOwYAAMBCM5MHAAAwIUoeAADAhCh5AAAAE7I09+R9Yc8XctobThs7BgBb6JMv/uTYEQBg6ZjJAwAAmBAlDwAAYEKUPAAAgAlR8gAAACZEyQMAAJiQUUpeVR1RVZ+qqsuq6oqqetUYOQAAAKZmrEcofDfJ6d39nao6LMknqur/dvdfj5QHAABgEkYped3dSb4zbB42vHqMLAAAAFMy2j15VbWtqj6TZE+SC7v7krGyAAAATMVoJa+7b+3uhyQ5KcnDq+qB+x9TVTuqarWqVr//ne9vfUgAAIAlM/rqmt19fZKPJTnjAJ/t7O6V7l457E6HbX04AACAJTPW6ponVNVdhvd3TPLEJF8YIwsAAMCUjLW65j2SvK2qtmWtaL6ru/90pCwAAACTMdbqmp9N8tAxzg0AADBlo9+TBwAAwPwoeQAAABOi5AEAAEyIkgcAADAhSh4AAMCEjPUIhZnd7273yydf/MmxYwAAACw0M3kAAAATouQBAABMiJIHAAAwIUtzT96NV16Zix79mLFjADCSx1x80dgRAGApmMkDAACYECUPAABgQpQ8AACACVHyAAAAJkTJAwAAmJBRS15VbauqT1fVn46ZAwAAYCrGnsk7O8nukTMAAABMxmglr6pOSvIzSd40VgYAAICpGXMm7/VJfj3JP4+YAQAAYFJGKXlV9ZQke7p710GO21FVq1W1esP3v79F6QAAAJbXWDN5pyU5s6q+muRPkpxeVX+8/0HdvbO7V7p75ZjDDtvqjAAAAEtnlJLX3b/R3Sd19ylJnpHko9397DGyAAAATMnYq2sCAAAwR9vHDtDdH0/y8ZFjAAAATIKZPAAAgAlR8gAAACZk3SWvqo6qqjsM7/9dVZ1ZVZa8BAAAWCCzzORdnOSIqjoxyYeT/FKSczYjFAAAABszS8mr7r45ydOS/M/ufnqSH9ucWAAAAGzELKtrVlU9Ismzkjx/2Ldt/pEO7Oj73jePufiirTodAADAUpplJu/sJL+R5ILuvqKq7p3kY5sTCwAAgI1Y10xeVW1LcmZ3n7l3X3d/JcmvbVYwAAAAZreumbzuvjXJozY5CwAAAIdolnvyPl1V70/y7iQ37d3Z3efPPdUB7LnqhvzBSz+wFacCYMG96Pd+duwIALCwZil5RyT5VpLT99nXSbak5AEAAHBw6y553f28zQwCAADAoVv36ppVdVJVXVBVe4bXe6rqpM0MBwAAwGxmeYTCW5O8P8kPD68PDPsAAABYELOUvBO6+63dfcvwOifJCZuUCwAAgA2YZeGVb1XVs5OcO2w/M2sLsWxIVX01yY1Jbk1yS3evbPS7AAAAWDNLyfuVJG9I8rqsrar5l0l++RDP/7ju/uYhfgcAAACDWUreSd195r47quq0JH8/30gAAABs1Cz35L1hnfvWq5N8uKp2VdWOQ/geAAAABgedyauqRyR5ZJITquo/7fPRnZNsO4RzP6q7r66quyW5sKq+0N0X73fuHUl2JMmxR1vjBQAA4GDWM5N3eJI7Za0QHr3P69tJztroibv76uHnniQXJHn4AY7Z2d0r3b1ypyOP2eipAAAAbjcOOpPX3Rcluaiqzunur83jpFV1VJI7dPeNw/ufSvLqeXw3AADA7dks9+S9qarusnejqo6tqg9t8Lx3T/KJqrosyaeS/J/u/uAGvwsAAIDBLKtrHt/d1+/d6O7rhvvpZtbdX0ny4I38WQAAAG7bLDN5/1xVJ+/dqKofydoKmQAAACyIWWbyXp61SywvSlJJfjLDypcAAAAshnWXvO7+YFU9LMmpw66XdPc3NycWAAAAG3HQyzWr6n7Dz4clOTnJ14fXycM+AAAAFsR6ZvJemuQFSX7vAJ91ktPnmug23O2kY/Ki3/vZrTgVAADA0lrPc/JeMPx83ObHAQAA4FActORV1dP+rc+7+/z5xQEAAOBQrOdyzb3XSN4tySOTfHTYflySv0yi5AEAACyI9Vyu+bwkqaoPJ3lAd18zbN8jyTmbmg4AAICZzPKcvHvuLXiDb2Rttc0tcc3ffjmvefZZW3U6AJbAy//4vLEjAMDCmaXkfaSqPpTk3GH7F5P8+fwjAQAAsFGzPAz9RVX180kePeza2d0XbE4sAAAANmKWmbwkuTTJjd3951V1ZFUd3d03bkYwAAAAZneH9R5YVS9Icl6SPxx2nZjkvZsRCgAAgI1Zd8lL8sIkpyX5dpJ09xez9lgFAAAAFsQsJe+73f29vRtVtT1Jb/TEVXWXqjqvqr5QVbur6hEb/S4AAADWzHJP3kVV9ZtJ7lhVT0zyq0k+cAjn/v0kH+zus6rq8CRHHsJ3AQAAkNlm8l6W5B+TfC7Jf0jyZ0n+60ZOWlXHZG2VzjcnSXd/r7uv38h3AQAA8K/WNZNXVduSvL27n5Xkf83hvPfKWmF8a1U9OMmuJGd3901z+G4AAIDbrXXN5HX3rUl+ZLisch62J3lYkjd290OT3JS1mcIfUFU7qmq1qlZv+qfvzunUAAAA0zXLPXlfSfLJqnp/1kpZkqS7X7uB816V5KruvmTYPi8HKHndvTPJziQ58bhjN7zICwAAwO3FLCXvy8PrDkmOPpSTdvc/VNXfV9V9u/vKJI9P8vlD+U4AAABmKHnd/aokqao7r232jYd47hcnecdwCehXkjzvEL8PAADgdm/dJa+qVpK8NcMsXlXdkORXunvXRk7c3Z9JsrKRPwsAAMCBzXK55luS/Gp3/0WSVNWjslb6HrQZwQAAAJjdLM/Ju3VvwUuS7v5EklvmHwkAAICNmmUm76Kq+sMk5ybpJL+Y5ONV9bAk6e5LNyEfAAAAM5il5D14+PmK/fY/NGul7/S5JAIAAGDDqns+j5+rqud299vm8mUHsLKy0qurq5v19QAAAAutqnZ190EXr5zlnryDOXuO3wUAAMAGzLPk1Ry/CwAAgA2YZ8mbz3WfAAAAbJiZPAAAgAlZ9+qaVfVDSf59klP2/XPd/erh7Sfnmmw//3TNjdn9mo9u5ikAWDL3f7mFnQFgf7M8QuF9SW5IsivJd/f/sLtfNK9QAAAAbMwsJe+k7j5j05IAAABwyGa5J+8vq+rHNy0JAAAAh2yWmbxHJfnlqvrbrF2uWUm6ux+0KckAAACY2Swl70mblgIAAIC5WHfJ6+6vzeukVXXfJO/cZ9e9k/xWd79+XucAAAC4PZplJm9uuvvKJA9JkqraluTqJBeMkQUAAGBK5vkw9I16fJIvz3OmEAAA4PZqEUreM5KcO3YIAACAKRi15FXV4UnOTPLu2/h8R1WtVtXqtTddv7XhAAAAltDYM3lPSnJpd3/jQB92987uXunulbsedZctjgYAALB8xi55z4xLNQEAAOZmtJJXVUcleWKS88fKAAAAMDWjPEIhSbr7piTHjXV+AACAKRr7ck0AAADmSMkDAACYECUPAABgQpQ8AACACRlt4ZVZHXGPo3P/l58+dgwAAICFZiYPAABgQpQ8AACACVHyAAAAJkTJAwAAmJClWXjl61//el75yleOHQOABeV3BACsMZMHAAAwIUoeAADAhCh5AAAAE6LkAQAATIiSBwAAMCGjlbyq+o9VdUVVXV5V51bVEWNlAQAAmIpRSl5VnZjk15KsdPcDk2xL8owxsgAAAEzJmJdrbk9yx6ranuTIJF8fMQsAAMAkjFLyuvvqJP8tyd8luSbJDd394TGyAAAATMlYl2sem+SpSe6V5IeTHFVVzz7AcTuqarWqVm+++eatjgkAALB0xrpc8wlJ/ra7/7G7v5/k/CSP3P+g7t7Z3SvdvXLkkUdueUgAAIBlM1bJ+7skp1bVkVVVSR6fZPdIWQAAACZjrHvyLklyXpJLk3xuyLFzjCwAAABTsn2sE3f3K5K8YqzzAwAATNGYj1AAAABgzpQ8AACACVHyAAAAJkTJAwAAmJDq7rEzrMvKykqvrq6OHQMAAGAUVbWru1cOdpyZPAAAgAlR8gAAACZEyQMAAJgQJQ8AAGBCto8dYL2uu2533vXuh48dA4AF9gtP/9TYEQBgdGbyAAAAJkTJAwAAmBAlDwAAYEKUPAAAgAlR8gAAACZktJJXVWdX1eVVdUVVvWSsHAAAAFMySsmrqgcmeUGShyd5cJKnVNWPjpEFAABgSsaaybt/kku6++buviXJRUmeNlIWAACAyRir5F2e5Cer6riqOjLJk5Pcc6QsAAAAk7F9jJN29+6q+p0kH05yU5LPJLl1/+OqakeSHUly/PGHb2lGAACAZTTawivd/ebu/onufnSS65L8vwMcs7O7V7p75c53HqWPAgAALJXRmlNV3a2791TVyVm7H+/UsbIAAABMxZjTY++pquOSfD/JC7v7+hGzAAAATMJoJa+7f3KscwMAAEzVaPfkAQAAMH9KHgAAwIQoeQAAABOi5AEAAEzI0jx87thj759fePqnxo4BAACw0MzkAQAATIiSBwAAMCFKHgAAwIQoeQAAABOyNAuvfP66b+fB531o7BgALLjLzvrpsSMAwKjM5AEAALX4aX0AAAb+SURBVEyIkgcAADAhSh4AAMCEKHkAAAATouQBAABMyKaWvKp6S1XtqarL99l316q6sKq+OPw8djMzAAAA3J5s9kzeOUnO2G/fy5J8pLvvk+QjwzYAAABzsKklr7svTnLtfrufmuRtw/u3Jfm5zcwAAABwezLGPXl37+5rhvf/kOTuI2QAAACYpFEXXunuTtK39XlV7aiq1apaveXbN2xhMgAAgOU0Rsn7RlXdI0mGn3tu68Du3tndK929sv3Ox2xZQAAAgGU1Rsl7f5LnDu+fm+R9I2QAAACYpM1+hMK5Sf4qyX2r6qqqen6S307yxKr6YpInDNsAAADMwfbN/PLufuZtfPT4zTwvAADA7dWoC68AAAAwX0oeAADAhCh5AAAAE6LkAQAATMimLrwyTw849s5ZPeunx44BAACw0MzkAQAATIiSBwAAMCHV3WNnWJequjHJlWPnYBKOT/LNsUMwGcYT82IsMS/GEvNkPC2WH+nuEw520NLck5fkyu5eGTsEy6+qVo0l5sV4Yl6MJebFWGKejKfl5HJNAACACVHyAAAAJmSZSt7OsQMwGcYS82Q8MS/GEvNiLDFPxtMSWpqFVwAAADi4ZZrJAwAA4CAWvuRV1RlVdWVVfamqXjZ2HhZfVb2lqvZU1eX77LtrVV1YVV8cfh477K+q+u/D+PpsVT1svOQsmqq6Z1V9rKo+X1VXVNXZw37jiZlU1RFV9amqumwYS68a9t+rqi4Zxsw7q+rwYf8PDdtfGj4/Zcz8LJ6q2lZVn66qPx22jSU2pKq+WlWfq6rPVNXqsM/vuSW30CWvqrYl+R9JnpTkAUmeWVUPGDcVS+CcJGfst+9lST7S3fdJ8pFhO1kbW/cZXjuSvHGLMrIcbkny0u5+QJJTk7xw+DvIeGJW301yenc/OMlDkpxRVacm+Z0kr+vuH01yXZLnD8c/P8l1w/7XDcfBvs5OsnufbWOJQ/G47n7IPo9K8HtuyS10yUvy8CRf6u6vdPf3kvxJkqeOnIkF190XJ7l2v91PTfK24f3bkvzcPvvf3mv+OsldquoeW5OURdfd13T3pcP7G7P2D6oTYzwxo2FMfGfYPGx4dZLTk5w37N9/LO0dY+cleXxV1RbFZcFV1UlJfibJm4btirHEfPk9t+QWveSdmOTv99m+atgHs7p7d18zvP+HJHcf3htjrMtwidNDk1wS44kNGC6v+0ySPUkuTPLlJNd39y3DIfuOl38ZS8PnNyQ5bmsTs8Ben+TXk/zzsH1cjCU2rpN8uKp2VdWOYZ/fc0tu+9gBYKt1d1eVZWVZt6q6U5L3JHlJd3973/8EN55Yr+6+NclDquouSS5Icr+RI7GEquopSfZ0966qeuzYeZiER3X31VV1tyQXVtUX9v3Q77nltOgzeVcnuec+2ycN+2BW39h7OcHwc8+w3xjj31RVh2Wt4L2ju88fdhtPbFh3X5/kY0kekbVLnfb+h+u+4+VfxtLw+TFJvrXFUVlMpyU5s6q+mrXbWE5P8vsxltig7r56+Lkna/8B9fD4Pbf0Fr3k/U2S+wwrRh2e5BlJ3j9yJpbT+5M8d3j/3CTv22f/c4bVok5NcsM+lydwOzfct/LmJLu7+7X7fGQ8MZOqOmGYwUtV3THJE7N2j+fHkpw1HLb/WNo7xs5K8tH2YFuSdPdvdPdJ3X1K1v5d9NHuflaMJTagqo6qqqP3vk/yU0kuj99zS2/hH4ZeVU/O2rXn25K8pbtfM3IkFlxVnZvksUmOT/KNJK9I8t4k70pycpKvJfmF7r52+Ef8H2RtNc6bkzyvu1fHyM3iqapHJfmLJJ/Lv9778ptZuy/PeGLdqupBWVu8YFvW/oP1Xd396qq6d9ZmY+6a5NNJnt3d362qI5L8UdbuA702yTO6+yvjpGdRDZdr/ufufoqxxEYM4+aCYXN7kv/d3a+pquPi99xSW/iSBwAAwPot+uWaAAAAzEDJAwAAmBAlDwAAYEKUPAAAgAlR8gAAACZEyQOAdaiqP9v7rDsAWGQeoQAAADAhZvIAYD9V9d6q2lVVV1TVjmHfV6vq+OH9c6rqs1V1WVX90bhpAeAHmckDgP1U1V27+9qqumOSv0nymCS7kqwkuXuSC5I8sru/uffYEeMCwA/YPnYAAFhAv1ZVPz+8v2eS++zz2elJ3t3d30wSBQ+ARaPkAcA+quqxSZ6Q5BHdfXNVfTzJEaOGAoAZuCcPAH7QMUmuGwre/ZKcut/nH03y9Ko6Llm7tHOrAwLAv0XJA4Af9MEk26tqd5LfTvLX+37Y3VckeU2Si6rqsiSv3fqIAHDbLLwCAAAwIWbyAAAAJkTJAwAAmBAlDwAAYEKUPAAAgAlR8gAAACZEyQMAAJgQJQ8AAGBClDwAAIAJ+f9ig6pcWhwJnwAAAABJRU5ErkJggg==\n",
"text/plain": [
"<Figure size 1080x1080 with 3 Axes>"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"# Plot the error metrics\n",
"_, (ax1, ax2, ax3) = plt.subplots(nrows=3, figsize=(15,15))\n",
"sns.barplot(y='n_predictors', x='adj_R_sq', data=best, ax=ax1, orient='h')\n",
"sns.barplot(y='n_predictors', x='bic', data=best, ax=ax2, orient='h')\n",
"sns.barplot(y='n_predictors', x='aic', data=best, ax=ax3, orient='h')"
]
},
{
"cell_type": "code",
"execution_count": 186,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
" OLS Regression Results \n",
"==============================================================================\n",
"Dep. Variable: y R-squared: 0.998\n",
"Model: OLS Adj. R-squared: 0.998\n",
"Method: Least Squares F-statistic: 1.680e+04\n",
"Date: Mon, 01 Oct 2018 Prob (F-statistic): 1.93e-130\n",
"Time: 11:29:24 Log-Likelihood: -126.13\n",
"No. Observations: 100 AIC: 260.3\n",
"Df Residuals: 96 BIC: 270.7\n",
"Df Model: 3 \n",
"Covariance Type: nonrobust \n",
"==============================================================================\n",
" coef std err t P>|t| [0.025 0.975]\n",
"------------------------------------------------------------------------------\n",
"Intercept 1.0229 0.117 8.713 0.000 0.790 1.256\n",
"x3 3.9099 0.044 88.854 0.000 3.823 3.997\n",
"x2 3.0189 0.077 38.971 0.000 2.865 3.173\n",
"x 2.2893 0.139 16.474 0.000 2.013 2.565\n",
"==============================================================================\n",
"Omnibus: 0.465 Durbin-Watson: 1.885\n",
"Prob(Omnibus): 0.792 Jarque-Bera (JB): 0.386\n",
"Skew: 0.150 Prob(JB): 0.825\n",
"Kurtosis: 2.951 Cond. No. 7.10\n",
"==============================================================================\n",
"\n",
"Warnings:\n",
"[1] Standard Errors assume that the covariance matrix of the errors is correctly specified.\n",
" OLS Regression Results \n",
"==============================================================================\n",
"Dep. Variable: y R-squared: 0.998\n",
"Model: OLS Adj. R-squared: 0.998\n",
"Method: Least Squares F-statistic: 1.680e+04\n",
"Date: Mon, 01 Oct 2018 Prob (F-statistic): 1.93e-130\n",
"Time: 11:29:24 Log-Likelihood: -126.13\n",
"No. Observations: 100 AIC: 260.3\n",
"Df Residuals: 96 BIC: 270.7\n",
"Df Model: 3 \n",
"Covariance Type: nonrobust \n",
"==============================================================================\n",
" coef std err t P>|t| [0.025 0.975]\n",
"------------------------------------------------------------------------------\n",
"Intercept 1.0229 0.117 8.713 0.000 0.790 1.256\n",
"x3 3.9099 0.044 88.854 0.000 3.823 3.997\n",
"x2 3.0189 0.077 38.971 0.000 2.865 3.173\n",
"x 2.2893 0.139 16.474 0.000 2.013 2.565\n",
"==============================================================================\n",
"Omnibus: 0.465 Durbin-Watson: 1.885\n",
"Prob(Omnibus): 0.792 Jarque-Bera (JB): 0.386\n",
"Skew: 0.150 Prob(JB): 0.825\n",
"Kurtosis: 2.951 Cond. No. 7.10\n",
"==============================================================================\n",
"\n",
"Warnings:\n",
"[1] Standard Errors assume that the covariance matrix of the errors is correctly specified.\n",
" OLS Regression Results \n",
"==============================================================================\n",
"Dep. Variable: y R-squared: 0.998\n",
"Model: OLS Adj. R-squared: 0.998\n",
"Method: Least Squares F-statistic: 1.680e+04\n",
"Date: Mon, 01 Oct 2018 Prob (F-statistic): 1.93e-130\n",
"Time: 11:29:24 Log-Likelihood: -126.13\n",
"No. Observations: 100 AIC: 260.3\n",
"Df Residuals: 96 BIC: 270.7\n",
"Df Model: 3 \n",
"Covariance Type: nonrobust \n",
"==============================================================================\n",
" coef std err t P>|t| [0.025 0.975]\n",
"------------------------------------------------------------------------------\n",
"Intercept 1.0229 0.117 8.713 0.000 0.790 1.256\n",
"x3 3.9099 0.044 88.854 0.000 3.823 3.997\n",
"x2 3.0189 0.077 38.971 0.000 2.865 3.173\n",
"x 2.2893 0.139 16.474 0.000 2.013 2.565\n",
"==============================================================================\n",
"Omnibus: 0.465 Durbin-Watson: 1.885\n",
"Prob(Omnibus): 0.792 Jarque-Bera (JB): 0.386\n",
"Skew: 0.150 Prob(JB): 0.825\n",
"Kurtosis: 2.951 Cond. No. 7.10\n",
"==============================================================================\n",
"\n",
"Warnings:\n",
"[1] Standard Errors assume that the covariance matrix of the errors is correctly specified.\n"
]
}
],
"source": [
"print(best.sort_values('adj_R_sq', ascending=False)['model'].iloc[0].summary())\n",
"print(best.sort_values('bic', ascending=True)['model'].iloc[0].summary())\n",
"print(best.sort_values('aic', ascending=True)['model'].iloc[0].summary())"
]
},
{
"cell_type": "code",
"execution_count": 187,
"metadata": {},
"outputs": [],
"source": [
"# Backward selection\n",
"# There are 1 + p(1 + p) / 2 possible models in this space \n",
"\n",
"def bwd_selection(target_column, data):\n",
" predictors = data.drop('y', axis=1).columns\n",
" formula_s1 = 'y ~ ' + ' + '.join(predictors)\n",
" model_s1 = smf.ols(formula=formula_s1, data=data).fit()\n",
" step_models = [model_s1]\n",
" while len(predictors) > 1:\n",
" models = []\n",
" for p in predictors:\n",
" f = 'y ~ ' + ' + '.join(predictors.drop(p))\n",
" m = smf.ols(formula=f, data=data).fit()\n",
" models.append((p, m))\n",
" p, m = sorted(models, key=lambda tup: tup[1].rsquared, reverse=True)[0]\n",
" step_models.append(m)\n",
" predictors = predictors.drop(p)\n",
" return pd.DataFrame({'model': step_models, 'n_predictors' : range(1, 11)[::-1]})"
]
},
{
"cell_type": "code",
"execution_count": 188,
"metadata": {},
"outputs": [],
"source": [
"# Select between models with different numbers of predictors using adjusted training metrics\n",
"best = bwd_selection('y', data)\n",
"best['adj_R_sq'] = best['model'].map(lambda m: m.rsquared_adj)\n",
"best['bic'] = best['model'].map(lambda m: m.bic)\n",
"best['aic'] = best['model'].map(lambda m: m.aic)"
]
},
{
"cell_type": "code",
"execution_count": 189,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"<matplotlib.axes._subplots.AxesSubplot at 0x119863b70>"
]
},
"execution_count": 189,
"metadata": {},
"output_type": "execute_result"
},
{
"data": {
"image/png": "iVBORw0KGgoAAAANSUhEUgAAA3kAAANgCAYAAACRKaGGAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzs3X20pnVdL/73xxkIQQQE9Ecgjp0MMRO1vfih+JCghWZYhqZH8yGOc34rNSxPHc2WT+e4Vh3T6tjDr0kRLaMUwdDKhxQhUbE9iAoMpOITSI7Ko7BSwM/vj33PaX7jwOx7z733tfc1r9da99r3dd3X3Nfbtb7O5j3f6/pe1d0BAABgHO42dAAAAABmR8kDAAAYESUPAABgRJQ8AACAEVHyAAAARkTJAwAAGBElDwAAYESUPAAAgBFR8gAAAEZk/dABFuuQQw7pDRs2DB0DAABgEJs3b/5Wdx+6q+PWTMnbsGFD5ufnh44BAAAwiKr6ymKOc7kmAADAiKyZmbwtV387P/mbbx86BgAAMIDNr3/O0BHWDDN5AAAAI6LkAQAAjIiSBwAAMCJKHgAAwIgoeQAAACMySMmrqtOramtVXTrE+QEAAMZqqJm8M5KcNNC5AQAARmuQktfdFyS5bohzAwAAjNmqvievqjZW1XxVzd9+681DxwEAAFj1VnXJ6+5N3T3X3XPr991/6DgAAACr3qoueQAAAExHyQMAABiRoR6hcGaSTyQ5qqqurqpTh8gBAAAwNuuHOGl3P3OI8wIAAIydyzUBAABGRMkDAAAYESUPAABgRJQ8AACAERlk4ZWlOPqIgzP/+ucMHQMAAGBVM5MHAAAwIkoeAADAiCh5AAAAI6LkAQAAjMiaWXjle9delq++9ieGjgEAAIzEka/83NARloWZPAAAgBFR8gAAAEZEyQMAABgRJQ8AAGBElDwAAIARGaTkVdV9q+q8qrq8qi6rqtOGyAEAADA2Qz1C4fYkL+3ui6tq/ySbq+pD3X35QHkAAABGYZCZvO6+trsvnry/OcmWJIcPkQUAAGBMBr8nr6o2JHlYkot28tnGqpqvqvnrbrljpaMBAACsOYOWvKq6R5J3J3lJd9+04+fdvam757p77l77rVv5gAAAAGvMYCWvqvbKQsF7R3efPVQOAACAMRlqdc1K8pYkW7r7jUNkAAAAGKOhZvKOT/LLSU6oqksmrycNlAUAAGA0BnmEQnd/LEkNcW4AAIAxG3x1TQAAAGZHyQMAABgRJQ8AAGBElDwAAIARGWThlaXY+7Afz5GvnB86BgAAwKpmJg8AAGBElDwAAIARUfIAAABGRMkDAAAYkTWz8MoVW6/I8W86fugYAADASF344guHjjATZvIAAABGRMkDAAAYESUPAABgRJQ8AACAEVHyAAAARmSQkldV+1TVp6rqM1V1WVW9ZogcAAAAYzPUIxS+m+SE7v5OVe2V5GNV9Y/d/cmB8gAAAIzCICWvuzvJdyabe01ePUQWAACAMRnsnryqWldVlyTZmuRD3X3RTo7ZWFXzVTV/23duW/mQAAAAa8xgJa+77+juhyY5IsmxVfXgnRyzqbvnuntur3vstfIhAQAA1pjBV9fs7huSnJfkpKGzAAAArHVDra55aFUdOHl/9yRPSHLFEFkAAADGZKjVNQ9L8raqWpeFovnO7n7fQFkAAABGY6jVNT+b5GFDnBsAAGDMBr8nDwAAgNlR8gAAAEZEyQMAABgRJQ8AAGBEhlpdc2oPvPcDc+GLLxw6BgAAwKpmJg8AAGBElDwAAIARUfIAAABGRMkDAAAYkTWz8MrNV16Z8x/z2KFjAAAAI/XYC84fOsJMmMkDAAAYESUPAABgRJQ8AACAEVHyAAAARkTJAwAAGJFBS15VrauqT1fV+4bMAQAAMBZDz+SdlmTLwBkAAABGY7CSV1VHJPnZJG8eKgMAAMDYDDmT94dJfivJ9+/sgKraWFXzVTV/4223rVwyAACANWqQkldVT06ytbs339Vx3b2pu+e6e+6AvfZaoXQAAABr11AzeccnObmqvpzkb5KcUFV/NVAWAACA0Rik5HX3y7v7iO7ekOQZST7S3c8eIgsAAMCYDL26JgAAADO0fugA3f3RJB8dOAYAAMAomMkDAAAYESUPAABgRBZd8qpqv6q62+T9j1XVyVXluQYAAACryDQzeRck2aeqDk/ywSS/nOSM5QgFAADA0kyz8Ep1961VdWqSP+3u/1VVlyxXsB3tf9RReewF56/U6QAAANakaWbyqqoekeRZSf5+sm/d7CMBAACwVNOUvNOSvDzJOd19WVX9SJLzlicWAAAAS7GoyzWral2Sk7v75G37uvuqJL+2XMEAAACY3qJm8rr7jiSPWuYsAAAA7KZpFl75dFWdm+RdSW7ZtrO7z555qp3YevWN+eOXvnclTgUAAOyBXvSGnxs6wkxMU/L2SfLtJCdst6+TrEjJAwAAYNcWXfK6+/nLGQQAAIDdt+jVNavqiKo6p6q2Tl7vrqojljMcAAAA05nmEQpvTXJukh+evN472QcAAMAqMU3JO7S739rdt09eZyQ5dJlyAQAAsATTLLzy7ap6dpIzJ9vPzMJCLEtSVV9OcnOSO5Lc3t1zS/0uAAAAFkxT8n4lyZuS/EEWVtX8eJLn7eb5H9fd39rN7wAAAGBimpJ3RHefvP2Oqjo+yddmGwkAAIClmuaevDctct9idZIPVtXmqtq4swOqamNVzVfV/HduvXE3TgUAALBn2OVMXlU9IskjkxxaVb+x3Uf3TLJuN879qO6+pqruneRDVXVFd1+w/QHdvSnJpiQ58v96QO/GuQAAAPYIi5nJ2zvJPbJQCPff7nVTklOWeuLuvmbyc2uSc5Icu9TvAgAAYMEuZ/K6+/wk51fVGd39lVmctKr2S3K37r558v6nk7x2Ft8NAACwJ5vmnrw3V9WB2zaq6qCq+sASz3ufJB+rqs8k+VSSv+/u9y/xuwAAAJiYZnXNQ7r7hm0b3X395H66qXX3VUmOWcqfBQAA4M5NM5P3/ao6cttGVd0vCytkAgAAsEpMM5P3iixcYnl+kkry6CQ7ffQBAAAAw1h0yevu91fVw5McN9n1ku7+1vLEAgAAYCl2eblmVT1w8vPhSY5M8vXJ68jJPgAAAFaJxczkvTTJC5K8YSefdZITZproTtz7iAPyojf83EqcCgAAYM1azHPyXjD5+bjljwMAAMDu2GXJq6qn3tXn3X327OIAAACwOxZzuea2ayTvneSRST4y2X5cko8nUfIAAABWicVcrvn8JKmqDyZ5UHdfO9k+LMkZy5oOAACAqUzznLz7bit4E9/IwmqbK+LaL30xr3v2KSt1OgAAYA/zir86a+gIMzFNyftwVX0gyZmT7V9K8k+zjwQAAMBSTfMw9BdV1S8kecxk16buPmd5YgEAALAU08zkJcnFSW7u7n+qqn2rav/uvnk5ggEAADC9uy32wKp6QZKzkvz5ZNfhSd6zHKEAAABYmkWXvCQvTHJ8kpuSpLs/n4XHKgAAALBKTFPyvtvd39u2UVXrk/RST1xVB1bVWVV1RVVtqapHLPW7AAAAWDDNPXnnV9VvJ7l7VT0hya8mee9unPuPkry/u0+pqr2T7Lsb3wUAAECmm8l7WZJvJvlckv+a5B+S/M5STlpVB2Rhlc63JEl3f6+7b1jKdwEAAPAfFjWTV1Xrkry9u5+V5C9mcN77Z6EwvrWqjkmyOclp3X3LDufdmGRjkhyw791ncFoAAIBxW9RMXnffkeR+k8sqZ2F9kocn+bPufliSW7IwU7jjeTd191x3z+23zw/N6NQAAADjNc09eVclubCqzs1CKUuSdPcbl3Deq5Nc3d0XTbbPyk5KHgAAANOZpuR9cfK6W5L9d+ek3f1vVfW1qjqqu69McmKSy3fnOwEAAJii5HX3a5Kkqu65sNk37+a5X5zkHZNLQK9K8vzd/D4AAIA93qJLXlXNJXlrJrN4VXVjkl/p7s1LOXF3X5Jkbil/FgAAgJ2b5nLN05P8anf/c5JU1aOyUPoeshzBAAAAmN40z8m7Y1vBS5Lu/liS22cfCQAAgKWaZibv/Kr68yRnJukkv5Tko1X18CTp7ouXIR8AAABTmKbkHTP5+aod9j8sC6XvhJkkAgAAYMmqu2fzRVXP7e63zeTLdmJubq7n5+eX6+sBAABWtara3N27XLxymnvyduW0GX4XAAAASzDLklcz/C4AAACWYJYlbzbXfQIAALBkZvIAAABGZNGra1bVDyX5xSQbtv9z3f3aydsLZ5psB/9+7c3Z8rqPLOcpAACAPdjRrxjHAwOmeYTC3yW5McnmJN/d8cPuftGsQgEAALA005S8I7r7pGVLAgAAwG6b5p68j1fVTyxbEgAAAHbbNDN5j0ryvKr6UhYu16wk3d0PWZZkAAAATG2akvfEZUsBAADATCy65HX3V2Z10qo6KsnfbrfrR5K8srv/cFbnAAAA2BNNM5M3M919ZZKHJklVrUtyTZJzhsgCAAAwJrN8GPpSnZjki7OcKQQAANhTrYaS94wkZ+7sg6raWFXzVTV/3S03rHAsAACAtWfQkldVeyc5Ocm7dvZ5d2/q7rnunrvXfgeubDgAAIA1aOiZvCcmubi7vzFwDgAAgFEYuuQ9M3dyqSYAAADTG6zkVdV+SZ6Q5OyhMgAAAIzNII9QSJLuviXJwUOdHwAAYIyGvlwTAACAGVLyAAAARkTJAwAAGBElDwAAYEQGW3hlWvsctn+OfsUJQ8cAAABY1czkAQAAjIiSBwAAMCJKHgAAwIgoeQAAACOyZhZe+frXv55Xv/rVQ8cAAABGaix9w0weAADAiCh5AAAAI6LkAQAAjIiSBwAAMCJKHgAAwIgMVvKq6ter6rKqurSqzqyqfYbKAgAAMBaDlLyqOjzJryWZ6+4HJ1mX5BlDZAEAABiTIS/XXJ/k7lW1Psm+Sb4+YBYAAIBRGKTkdfc1SX4/yVeTXJvkxu7+4I7HVdXGqpqvqvlbb711pWMCAACsOUNdrnlQkqckuX+SH06yX1U9e8fjuntTd89199y+++670jEBAADWnKEu13x8ki919ze7+7YkZyd55EBZAAAARmOokvfVJMdV1b5VVUlOTLJloCwAAACjMdQ9eRclOSvJxUk+N8mxaYgsAAAAY7J+qBN396uSvGqo8wMAAIzRkI9QAAAAYMaUPAAAgBFR8gAAAEZEyQMAABiR6u6hMyzK3Nxcz8/PDx0DAABgEFW1ubvndnWcmTwAAIARUfIAAABGRMkDAAAYESUPAABgRNYPHWCxrr9+S975rmOHjgEAAIzU05/2qaEjzISZPAAAgBFR8gAAAEZEyQMAABgRJQ8AAGBElDwAAIARGazkVdVpVXVpVV1WVS8ZKgcAAMCYDFLyqurBSV6Q5NgkxyR5clX96BBZAAAAxmSombyjk1zU3bd29+1Jzk/y1IGyAAAAjMZQJe/SJI+uqoOrat8kT0py3x0PqqqNVTVfVfM33XT7iocEAABYa9YPcdLu3lJVv5fkg0luSXJJkjt2ctymJJuS5D/9p/16RUMCAACsQYMtvNLdb+nun+zuxyS5Psm/DpUFAABgLAaZyUuSqrp3d2+tqiOzcD/ecUNlAQAAGIvBSl6Sd1fVwUluS/LC7r5hwCwAAACjMFjJ6+5HD3VuAACAsRrsnjwAAABmT8kDAAAYESUPAABgRJQ8AACAERlydc2pHHTQ0Xn60z41dAwAAIBVzUweAADAiCh5AAAAI6LkAQAAjIiSBwAAMCJrZuGVy6+/Kcec9YGhYwAAACP1mVN+ZugIM2EmDwAAYESUPAAAgBFR8gAAAEZEyQMAABgRJQ8AAGBElrXkVdXpVbW1qi7dbt+9qupDVfX5yc+DljMDAADAnmS5Z/LOSHLSDvteluTD3f2AJB+ebAMAADADy1ryuvuCJNftsPspSd42ef+2JD+/nBkAAAD2JEPck3ef7r528v7fktznzg6sqo1VNV9V87ffdOPKpAMAAFjDBl14pbs7Sd/F55u6e66759bf84AVTAYAALA2DVHyvlFVhyXJ5OfWATIAAACM0hAl79wkz528f26SvxsgAwAAwCgt9yMUzkzyiSRHVdXVVXVqkt9N8oSq+nySx0+2AQAAmIH1y/nl3f3MO/noxOU8LwAAwJ5q0IVXAAAAmC0lDwAAYESUPAAAgBFR8gAAAEZkWRdemaUHHXTPzJ/yM0PHAAAAWNXM5AEAAIxIdffQGRalqm5OcuXQOdgjHZLkW0OHYI9l/DEk448hGX8MZTWPvft196G7OmjNXK6Z5Mrunhs6BHueqpo39hiK8ceQjD+GZPwxlDGMPZdrAgAAjIiSBwAAMCJrqeRtGjoAeyxjjyEZfwzJ+GNIxh9DWfNjb80svAIAAMCuraWZPAAAAHZByQMAABiRVVXyquqkqrqyqr5QVS/byec/VFV/O/n8oqrasPIpGatFjL/fqKrLq+qzVfXhqrrfEDkZp12Nv+2O+8Wq6qpa00s7s7osZvxV1dMnfwdeVlV/vdIZGadF/O49sqrOq6pPT37/PmmInIxTVZ1eVVur6tI7+byq6n9Pxudnq+rhK51xqVZNyauqdUn+JMkTkzwoyTOr6kE7HHZqkuu7+0eT/EGS31vZlIzVIsffp5PMdfdDkpyV5H+tbErGapHjL1W1f5LTkly0sgkZs8WMv6p6QJKXJzm+u388yUtWPCijs8i/+34nyTu7+2FJnpHkT1c2JSN3RpKT7uLzJyZ5wOS1McmfrUCmmVg1JS/JsUm+0N1Xdff3kvxNkqfscMxTkrxt8v6sJCdWVa1gRsZrl+Ovu8/r7lsnm59McsQKZ2S8FvP3X5L8jyz849a/r2Q4Rm8x4+8FSf6ku69Pku7eusIZGafFjL1Ocs/J+wOSfH0F8zFy3X1Bkuvu4pCnJHl7L/hkkgOr6rCVSbd7VlPJOzzJ17bbvnqyb6fHdPftSW5McvCKpGPsFjP+tndqkn9c1kTsSXY5/iaXiNy3u/9+JYOxR1jM338/luTHqurCqvpkVd3Vv3zDYi1m7L06ybOr6uok/5DkxSsTDZJM/9+Hq8b6oQPAWlNVz04yl+SxQ2dhz1BVd0vyxiTPGzgKe671Wbhc6aeycBXDBVX1E919w6Cp2BM8M8kZ3f2GqnpEkr+sqgd39/eHDgar2WqaybsmyX232z5ism+nx1TV+ixM2397RdIxdosZf6mqxyd5RZKTu/u7K5SN8dvV+Ns/yYOTfLSqvpzkuCTnWnyFGVnM339XJzm3u2/r7i8l+dcslD7YHYsZe6cmeWeSdPcnkuyT5JAVSQeL/O/D1Wg1lbx/SfKAqrp/Ve2dhZtrz93hmHOTPHfy/pQkH2lPc2c2djn+quphSf48CwXP/SjM0l2Ov+6+sbsP6e4N3b0hC/eEntzd88PEZWQW8/v3PVmYxUtVHZKFyzevWsmQjNJixt5Xk5yYJFV1dBZK3jdXNCV7snOTPGeyyuZxSW7s7muHDrUYq+Zyze6+vapelOQDSdYlOb27L6uq1yaZ7+5zk7wlC9P0X8jCTZLPGC4xY7LI8ff6JPdI8q7Jej9f7e6TBwvNaCxy/MGyWOT4+0CSn66qy5PckeQ3u9uVNOyWRY69lyb5i6r69SwswvI8/8DPrFTVmVn4B6xDJvd9virJXknS3f9vFu4DfVKSLyS5Ncnzh0k6vfL/EwAAgPFYTZdrAgAAsJuUPAAAgBFR8gAAAEZEyQMAABgRJQ8AAGBElDwA9khV9byq+uPJ+/+nqp5zF8eeUVVfqqpLquozVXXiyiUFgOmsmufkAcBQJs9D2pXf7O6zqupxSTYlecAyxwKAJTGTB8AoVdV7qmpzVV1WVRsn+55fVf9aVZ9Kcvx2x766qv7bIr/6E0kO38W5f7eqLq+qz1bV70/23b+qPlFVn6uq/1lV31ni/zQAuEtm8gAYq1/p7uuq6u5J/qWq/j7Ja5L8ZJIbk5yX5NNL+N6Tkrznzj6sqoOT/EKSB3Z3V9WBk4/+KMmfdffbq+qFSzgvACyKmTwAxurXquozST6Z5L5JfjnJR7v7m939vSR/O+X3vb6q/jXJXyf5vbs47sYk/57kLVX11CS3TvYfn+TMyfu/nPLcALBoSh4Ao1NVP5Xk8Uke0d3HZGHG7ord/Nrf7O4fS/Lfk5x+Zwd19+1Jjk1yVpInJ3n/9h/vZgYA2CUlD4AxOiDJ9d19a1U9MMlxSe6e5LFVdXBV7ZXkaUv87j9Ocreq+pmdfVhV90hyQHf/Q5JfT3LM5KMLkzxj8v5ZSzw3AOySkgfAGL0/yfqq2pLkd7Nwyea1SV6dhYVTLkyyZYc/s6hZtu7uJP8zyW/dySH7J3lfVX02yceS/MZk/2lJXlhVn8suFm4BgN1RC7+rAGDPVVVvSnJxd791Bc/5ne6+x0qdD4A9h5k8APZoVfU/kvzfSc4dOgsAzIKZPACYqKo/yXbPz5v4ozub4auqc5Lcf4fd/727P7Ac+QBgMZQ8AACAEXG5JgAAwIgoeQAAACOi5AEAAIyIkgcAADAiSh4AAMCIKHkAAAAjouQBAACMiJIHAAAwIkoeAADAiCh5AAAAI6LkAQAAjIiSBwAAMCJKHgAAwIgoeQAAACOyfugAi3XIIYf0hg0bho4BAAAwiM2bN3+ruw/d1XFrpuRt2LAh8/PzQ8cAAAAYRFV9ZTHHuVwTAABgRNbMTN6Wq7+dn/zNtw8dAwAAGKnNr3/O0BFmwkweAADAiCh5AAAAI6LkAQAAjIiSBwAAMCKDlLyqOr2qtlbVpUOcHwAAYKyGmsk7I8lJA50bAABgtAYped19QZLrhjg3AADAmLknDwAAYERWdcmrqo1VNV9V87ffevPQcQAAAFa9VV3yuntTd89199z6ffcfOg4AAMCqt6pLHgAAANMZ6hEKZyb5RJKjqurqqjp1iBwAAABjs36Ik3b3M4c4LwAAwNi5XBMAAGBElDwAAIARUfIAAABGRMkDAAAYESUPAABgRAZZXXMpjj7i4My//jlDxwAAAFjVzOQBAACMiJIHAAAwIkoeAADAiCh5AAAAI7JmFl753rWX5auv/YmhYwAArBpHvvJzQ0cAViEzeQAAACOi5AEAAIyIkgcAADAiSh4AAMCIDFLyquq+VXVeVV1eVZdV1WlD5AAAABiboVbXvD3JS7v74qraP8nmqvpQd18+UB4AAIBRGGQmr7uv7e6LJ+9vTrIlyeFDZAEAABiTwe/Jq6oNSR6W5KJhkwAAAKx9g5a8qrpHkncneUl337STzzdW1XxVzV93yx0rHxAAAGCNGazkVdVeWSh47+jus3d2THdv6u657p67137rVjYgAADAGjTU6pqV5C1JtnT3G4fIAAAAMEZDzeQdn+SXk5xQVZdMXk8aKAsAAMBoDPIIhe7+WJIa4twAAABjNvjqmgAAAMyOkgcAADAiSh4AAMCIKHkAAAAjouQBAACMyCCray7F3of9eI585fzQMQAAAFY1M3kAAAAjouQBAACMiJIHAAAwIkoeAADAiKyZhVeu2HpFjn/T8UPHAGAVu/DFFw4dAQAGZyYPAABgRJQ8AACAEVHyAAAARkTJAwAAGJFBSl5V7VNVn6qqz1TVZVX1miFyAAAAjM1Qq2t+N8kJ3f2dqtoryceq6h+7+5MD5QEAABiFQUped3eS70w295q8eogsAAAAYzLYPXlVta6qLkmyNcmHuvuiobIAAACMxWAlr7vv6O6HJjkiybFV9eAdj6mqjVU1X1Xzt33ntpUPCQAAsMYMvrpmd9+Q5LwkJ+3ks03dPdfdc3vdY6+VDwcAALDGDLW65qFVdeDk/d2TPCHJFUNkAQAAGJOhVtc8LMnbqmpdFormO7v7fQNlAQAAGI2hVtf8bJKHDXFuAACAMRv8njwAAABmR8kDAAAYESUPAABgRJQ8AACAEVHyAAAARmSoRyhM7YH3fmAufPGFQ8cAAABY1czkAQAAjIiSBwAAMCJKHgAAwIgoeQAAACOyZhZeufnKK3P+Yx47dAwA1qjHXnD+0BEAYEWYyQMAABgRJQ8AAGBElDwAAIARUfIAAABGZNCSV1XrqurTVfW+IXMAAACMxdAzeacl2TJwBgAAgNEYrORV1RFJfjbJm4fKAAAAMDZDzuT9YZLfSvL9ATMAAACMyiAlr6qenGRrd2/exXEbq2q+quZvvO22FUoHAACwdg01k3d8kpOr6stJ/ibJCVX1Vzse1N2bunuuu+cO2Guvlc4IAACw5gxS8rr75d19RHdvSPKMJB/p7mcPkQUAAGBMhl5dEwAAgBlaP3SA7v5oko8OHAMAAGAUzOQBAACMiJIHAAAwIosueVW1X1XdbfL+x6rq5Kqy5CUAAMAqMs1M3gVJ9qmqw5N8MMkvJzljOUIBAACwNNOUvOruW5M8NcmfdvfTkvz48sQCAABgKaZZXbOq6hFJnpXk1Mm+dbOPtHP7H3VUHnvB+St1OgAAgDVpmpm805K8PMk53X1ZVf1IkvOWJxYAAABLsaiZvKpal+Tk7j55277uvirJry1XMAAAAKa3qJm87r4jyaOWOQsAAAC7aZp78j5dVecmeVeSW7bt7O6zZ54KAACAJZmm5O2T5NtJTthuXydZkZK39eob88cvfe9KnAqAPcSL3vBzQ0cAgJlbdMnr7ucvZxAAAAB236JX16yqI6rqnKraOnm9u6qOWM5wAAAATGeaRyi8Ncm5SX548nrvZB8AAACrxDQl79Dufmt33z55nZHk0GXKBQAAwBJMs/DKt6vq2UnOnGw/MwsLsSxJVX05yc1J7khye3fPLfW7AAAAWDBNyfuVJG9K8gdZWFXz40met5vnf1x3f2s3vwMAAICJaUreEd198vY7qur4JF+bbSQAAACWapp78t60yH2L1Uk+WFWbq2rjbnwPAAAAE7ucyauqRyR5ZJJDq+o3tvvonknW7ca5H9Xd11TVvZN8qKqu6O4Ldjj3xiQbk+Sg/a3xAgAAsCuLmcnbO8k9slAI99/udVOSU5Z64u6+ZvJza5Jzkhy7k2M2dfdcd8/dY98DlnoqAACAPcYuZ/K6+/wk51fVGd39lVmctKr2S3K37r5EkELxAAAgAElEQVR58v6nk7x2Ft8NAACwJ5vmnrw3V9WB2zaq6qCq+sASz3ufJB+rqs8k+VSSv+/u9y/xuwAAAJiYZnXNQ7r7hm0b3X395H66qXX3VUmOWcqfBQAA4M5NM5P3/ao6cttGVd0vCytkAgAAsEpMM5P3iixcYnl+kkry6ExWvgQAAGB1WHTJ6+73V9XDkxw32fWS7v7W8sQCAABgKXZ5uWZVPXDy8+FJjkzy9cnryMk+AAAAVonFzOS9NMkLkrxhJ591khNmmggAAIAlq+61sXbK3Nxcz8/PDx0DAABgEFW1ubvndnXcLmfyquqpd/V5d589TTAAAACWz2Iu1/y5yc97J3lkko9Mth+X5ONJlDwAAIBVYpclr7ufnyRV9cEkD+ruayfbhyU5Y1nTAQAAMJVpnpN3320Fb+IbWVhtc0Vc+6Uv5nXPPmWlTgfAHuwVf3XW0BEAYMmmKXkfrqoPJDlzsv1LSf5p9pEAAABYqmkehv6iqvqFJI+Z7NrU3ecsTywAAACWYpqZvCS5OMnN3f1PVbVvVe3f3TcvRzAAAACmd7fFHlhVL0hyVpI/n+w6PMl7liMUAAAAS7PokpfkhUmOT3JTknT357PwWAUAAABWiWlK3ne7+3vbNqpqfZJe6omr6sCqOquqrqiqLVX1iKV+FwAAAAumuSfv/Kr67SR3r6onJPnVJO/djXP/UZL3d/cpVbV3kn1347sAAADIdDN5L0vyzSSfS/Jfk/xDkt9Zykmr6oAsrNL5liTp7u919w1L+S4AAAD+w6Jm8qpqXZK3d/ezkvzFDM57/ywUxrdW1TFJNic5rbtvmcF3AwAA7LEWNZPX3Xckud/ksspZWJ/k4Un+rLsfluSWLMwU/v9U1caqmq+q+Vv+/bszOjUAAMB4TXNP3lVJLqyqc7NQypIk3f3GJZz36iRXd/dFk+2zspOS192bkmxKksMPPmjJi7wAAADsKaYpeV+cvO6WZP/dOWl3/1tVfa2qjuruK5OcmOTy3flOAAAApih53f2aJKmqey5s9s27ee4XJ3nH5BLQq5I8fze/DwAAYI+36JJXVXNJ3prJLF5V3ZjkV7p781JO3N2XJJlbyp8FAABg56a5XPP0JL/a3f+cJFX1qCyUvocsRzAAAACmN81z8u7YVvCSpLs/luT22UcCAABgqaaZyTu/qv48yZlJOskvJfloVT08Sbr74mXIBwAAwBSmKXnHTH6+aof9D8tC6TthJokAAABYsmlW13zcXX1eVc/t7rftfiQAAACWqrpn84zxqrq4ux8+ky/bibm5uZ6fn1+urwcAAFjVqmpzd+/yCQXTLLyyy3PO8LsAAABYglmWvNlMCQIAALBkZvIAAABGZNELr1TVDyX5xSQbtv9z3f3aydsLZ5psB/9+7c3Z8rqPLOcpAOAuHf0KC0kDsPpN8wiFv0tyY5LNSb6744fd/aJZhQIAAGBppil5R3T3ScuWBAAAgN02zT15H6+qn1i2JAAAAOy2aWbyHpXkeVX1pSxcrllJursfsizJAAAAmNo0Je+Jy5YCAACAmVh0yevur8zqpFV1VJK/3W7XjyR5ZXf/4azOAQAAsCeaZiZvZrr7yiQPTZKqWpfkmiTnDJEFAABgTGb5MPSlOjHJF2c5UwgAALCnWg0l7xlJzhw6BAAAwBgMWvKqau8kJyd51518vrGq5qtq/rpbbljZcAAAAGvQ0DN5T0xycXd/Y2cfdvem7p7r7rl77XfgCkcDAABYe4Yuec+MSzUBAABmZrCSV1X7JXlCkrOHygAAADA2gzxCIUm6+5YkBw91fgAAgDEa+nJNAAAAZkjJAwAAGBElDwAAYESUPAAAgBFR8gAAAEZksNU1p7XPYfvn6FecMHQMAACAVc1MHgAAwIgoeQAAACOi5AEAAIzImrkn7+tf/3pe/epXDx0DAPw+AmBVM5MHAAAwIkoeAADAiCh5AAAAI6LkAQAAjIiSBwAAMCKDlbyq+vWquqyqLq2qM6tqn6GyAAAAjMUgJa+qDk/ya0nmuvvBSdYlecYQWQAAAMZkyMs11ye5e1WtT7Jvkq8PmAUAAGAUBil53X1Nkt9P8tUk1ya5sbs/OEQWAACAMRnqcs2Dkjwlyf2T/HCS/arq2Ts5bmNVzVfV/K233rrSMQEAANacoS7XfHySL3X3N7v7tiRnJ3nkjgd196bunuvuuX333XfFQwIAAKw1Q5W8ryY5rqr2rapKcmKSLQNlAQAAGI2h7sm7KMlZSS5O8rlJjk1DZAEAABiT9UOduLtfleRVQ50fAABgjIZ8hAIAAAAzpuQBAACMiJIHAAAwIkoeAADAiCh5AAAAI1LdPXSGRZmbm+v5+fmhYwAAAAyiqjZ399yujjOTBwAAMCJKHgAAwIgoeQAAACOyfugAi3X99VvyzncdO3QMALhTT3/ap4aOAABm8gAAAMZEyQMAABgRJQ8AAGBElDwAAIARUfIAAABGZLCSV1WnVdWlVXVZVb1kqBwAAABjMkjJq6oHJ3lBkmOTHJPkyVX1o0NkAQAAGJOhZvKOTnJRd9/a3bcnOT/JUwfKAgAAMBpDlbxLkzy6qg6uqn2TPCnJfQfKAgAAMBrrhzhpd2+pqt9L8sEktyS5JMkdOx5XVRuTbEySQw7Ze0UzAgAArEWDLbzS3W/p7p/s7sckuT7Jv+7kmE3dPdfdc/e85yB9FAAAYE0ZrDlV1b27e2tVHZmF+/GOGyoLAADAWAw5Pfbuqjo4yW1JXtjdNwyYBQAAYBQGK3nd/eihzg0AADBWg92TBwAAwOwpeQAAACOi5AEAAIyIkgcAADAiSh4AAMCIrJknjB900NF5+tM+NXQMAACAVc1MHgAAwIgoeQAAACOi5AEAAIzImrkn7/Lrb8oxZ31g6BgAsCSfOeVnho4AwB7CTB4AAMCIKHkAAAAjouQBAACMiJIHAAAwIkoeAADAiCxryauq06tqa1Vdut2+e1XVh6rq85OfBy1nBgAAgD3Jcs/knZHkpB32vSzJh7v7AUk+PNkGAABgBpa15HX3BUmu22H3U5K8bfL+bUl+fjkzAAAA7EmGuCfvPt197eT9vyW5zwAZAAAARmnQhVe6u5P0nX1eVRurar6q5m+/6cYVTAYAALA2DVHyvlFVhyXJ5OfWOzuwuzd191x3z62/5wErFhAAAGCtGqLknZvkuZP3z03ydwNkAAAAGKXlfoTCmUk+keSoqrq6qk5N8rtJnlBVn0/y+Mk2AAAAM7B+Ob+8u595Jx+duJznBQAA2FMNuvAKAAAAs6XkAQAAjIiSBwAAMCJKHgAAwIgoeQAAACOyrKtrztKDDrpn5k/5maFjAAAArGpm8gAAAEakunvoDItSVTcnuXLoHKx5hyT51tAhWPOMI2bBOGIWjCNmwThaO+7X3Yfu6qA1c7lmkiu7e27oEKxtVTVvHLG7jCNmwThiFowjZsE4Gh+XawIAAIyIkgcAADAia6nkbRo6AKNgHDELxhGzYBwxC8YRs2AcjcyaWXgFAACAXVtLM3kAAADswqoveVV1UlVdWVVfqKqXDZ2H1auqTq+qrVV16Xb77lVVH6qqz09+HjTZX1X1vyfj6rNV9fDhkrOaVNV9q+q8qrq8qi6rqtMm+40lFq2q9qmqT1XVZybj6DWT/fevqosm4+Vvq2rvyf4fmmx/YfL5hiHzs7pU1bqq+nRVvW+ybRwxlar6clV9rqouqar5yT6/10ZsVZe8qlqX5E+SPDHJg5I8s6oeNGwqVrEzkpy0w76XJflwdz8gyYcn28nCmHrA5LUxyZ+tUEZWv9uTvLS7H5TkuCQvnPy9Yywxje8mOaG7j0ny0CQnVdVxSX4vyR90948muT7JqZPjT01y/WT/H0yOg21OS7Jlu23jiKV4XHc/dLtHJfi9NmKruuQlOTbJF7r7qu7+XpK/SfKUgTOxSnX3BUmu22H3U5K8bfL+bUl+frv9b+8Fn0xyYFUdtjJJWc26+9ruvnjy/uYs/IfV4TGWmMJkPHxnsrnX5NVJTkhy1mT/juNo2/g6K8mJVVUrFJdVrKqOSPKzSd482a4YR8yG32sjttpL3uFJvrbd9tWTfbBY9+nuayfv/y3JfSbvjS12aXKp08OSXBRjiSlNLrG7JMnWJB9K8sUkN3T37ZNDth8r/2ccTT6/McnBK5uYVeoPk/xWku9Ptg+OccT0OskHq2pzVW2c7PN7bcTWDx0AVkp3d1VZTpZFqap7JHl3kpd0903b/2O4scRidPcdSR5aVQcmOSfJAweOxBpTVU9OsrW7N1fVTw2dhzXtUd19TVXdO8mHquqK7T/0e218VvtM3jVJ7rvd9hGTfbBY39h2icHk59bJfmOLO1VVe2Wh4L2ju8+e7DaWWJLuviHJeUkekYXLnrb9A+v2Y+X/jKPJ5wck+fYKR2X1OT7JyVX15SzcsnJCkj+KccSUuvuayc+tWfhHp2Pj99qorfaS9y9JHjBZRWrvJM9Icu7AmVhbzk3y3Mn75yb5u+32P2eygtRxSW7c7pIF9mCT+1fekmRLd79xu4+MJRatqg6dzOClqu6e5AlZuL/zvCSnTA7bcRxtG1+nJPlIe5DtHq+7X97dR3T3hiz8N9BHuvtZMY6YQlXtV1X7b3uf5KeTXBq/10Zt1T8MvaqelIXr0dclOb27XzdwJFapqjozyU8lOSTJN5K8Ksl7krwzyZFJvpLk6d193eQ/5P84C6tx3prk+d09P0RuVpeqelSSf07yufzHPTC/nYX78owlFqWqHpKFhQzWZeEfVN/Z3a+tqh/JwozMvZJ8Osmzu/u7VbVPkr/Mwj2g1yV5RndfNUx6VqPJ5Zr/rbufbBwxjcl4OWeyuT7JX3f366rq4Pi9NlqrvuQBAACweKv9ck0AAACmoOQBAACMiJIHAAAwIkoeAADAiCh5AAAAI6LkAcB2qmpDVV26k/1vrqoHDZEJAKaxfugAALAWdPd/GToDACyGmTwA+EHrq+odVbWlqs6qqn2r6qNVNZckVXVSVV1cVZ+pqg8PHRYAtmcmDwB+0FFJTu3uC6vq9CS/uu2Dqjo0yV8keUx3f6mq7jVUSADYGTN5APCDvtbdF07e/1WSR2332XFJLujuLyVJd1+30uEA4K4oeQDwg3oX2wCwail5APCDjqyqR0ze/+ckH9vus08meUxV3T9JXK4JwGqj5AHAD7oyyQur/r/27j3YsrMsE/jz0p0YEkIIuVAxIUZLJoBoAE9RgYBIAI2IQZmgUCCIFD1VCoQZZiyUKVCqqNJyxAvOWPZwSVAmAiEBdBxMJCGRqMHTIZGEJoOEW0KghVwISQlJfOePs1qbno599ul9ztpn5fer2nX2Wnv1Xs8fX/Xpp7+1vlU7kxyZ5A92f9Dd/5hkW5ILquqaJO8eJyIA7Ft1uwIFAABgKszkAQAATIiSBwAAMCFKHgAAwIQoeQAAABOi5AEAAEyIkgcAADAhSh4AAMCEKHkAAAATouQBAABMiJIHAAAwIUoeAADAhCh5AAAAE6LkAQAATIiSBwAAMCFKHgAAwIQoeQAAABOi5AEAAEyIkgcAADAhW8cOsFpHH310n3TSSWPHAAAAGMWOHTu+2t3H7O+4TVPyTjrppCwvL48dAwAAYBRV9fnVHLdpSt7OG7+WH/wv7xw7BgAAMFE7fvPFY0eYC/fkAQAATIiSBwAAMCFKHgAAwIQoeQAAABOi5AEAAEzIKCWvqt5eVbuq6toxzg8AADBVY83knZPkjJHODQAAMFmjlLzuvjzJLWOcGwAAYMrckwcAADAhC13yqmpbVS1X1fI9d90xdhwAAICFt9Alr7u3d/dSdy9tPfTwseMAAAAsvIUueQAAAMxmrEconJfkb5KcXFU3VtXLxsgBAAAwNVvHOGl3v2CM8wIAAEydyzUBAAAmRMkDAACYECUPAABgQpQ8AACACVHyAAAAJmSU1TXX4lEnHJXl33zx2DEAAAAWmpk8AACACVHyAAAAJkTJAwAAmJBNc0/et26+Ll944/ePHQMAYMOd+PpPjB0B2ETM5AEAAEyIkgcAADAhSh4AAMCEKHkAAAATouQBAABMyCglr6oeXlWXVtUnq+q6qjp7jBwAAABTM9YjFO5J8pruvqqqDk+yo6ou7u5PjpQHAABgEkaZyevum7v7quH9HUl2Jjl+jCwAAABTMvo9eVV1UpLHJbly3CQAAACb36glr6oelOR9SV7d3V/fx+fbqmq5qpZvufPejQ8IAACwyYxW8qrqoKwUvHd19wX7Oqa7t3f3UncvPfSwLRsbEAAAYBMaa3XNSvK2JDu7+81jZAAAAJiisWbyTkvys0lOr6qrh9ezRsoCAAAwGaM8QqG7P5qkxjg3AADAlI2+uiYAAADzo+QBAABMiJIHAAAwIUoeAADAhCh5AAAAEzLK6pprcfBx35cTX788dgwAAICFZiYPAABgQpQ8AACACVHyAAAAJmTT3JP3qV2fymlvOW3sGABsoCteecXYEQBg0zGTBwAAMCFKHgAAwIQoeQAAABOi5AEAAEyIkgcAADAho5S8qjqkqj5WVddU1XVV9Wtj5AAAAJiasR6h8M0kp3f3N6rqoCQfrar/091/O1IeAACASRil5HV3J/nGsHnQ8OoxsgAAAEzJaPfkVdWWqro6ya4kF3f3lWNlAQAAmIrRSl5339vdj01yQpInVNVj9j6mqrZV1XJVLd/9jbs3PiQAAMAmM/rqmt19W5JLk5yxj8+2d/dSdy8d9KCDNj4cAADAJjPW6prHVNVDhvcPTPLMJJ8aIwsAAMCUjLW65nFJzq2qLVkpmu/p7j8bKQsAAMBkjLW65t8nedwY5wYAAJiy0e/JAwAAYH6UPAAAgAlR8gAAACZEyQMAAJgQJQ8AAGBCxnqEwsweeewjc8Urrxg7BgAAwEIzkwcAADAhSh4AAMCEKHkAAAATsmnuybvj+utz2Q89dewYAIzkqZdfNnYEANgUzOQBAABMiJIHAAAwIUoeAADAhCh5AAAAE6LkAQAATMioJa+qtlTVx6vqz8bMAQAAMBVjz+SdnWTnyBkAAAAmY7SSV1UnJPnxJG8dKwMAAMDUjDmT9ztJfinJP4+YAQAAYFJGKXlV9ewku7p7x36O21ZVy1W1fPvdd29QOgAAgM1rrJm805KcWVWfS/InSU6vqj/e+6Du3t7dS929dMRBB210RgAAgE1nlJLX3b/c3Sd090lJnp/kku5+0RhZAAAApmTs1TUBAACYo61jB+jujyT5yMgxAAAAJsFMHgAAwIQoeQAAABOy6pJXVYdV1QOG9/+uqs6sKkteAgAALJBZZvIuT3JIVR2f5KIkP5vknPUIBQAAwNrMUvKqu+9K8twk/6O7n5fk+9YnFgAAAGsxy+qaVVVPTPLCJC8b9m2Zf6R9O/zkk/PUyy/bqNMBAABsSrPM5J2d5JeTXNjd11XV9yS5dH1iAQAAsBarmsmrqi1JzuzuM3fv6+4bkrxqvYIBAAAwu1XN5HX3vUmevM5ZAAAAOECz3JP38ar6YJL3Jrlz987uvmDuqfZh14235/df86cbcSoAFtgrfusnxo4AAAttlpJ3SJKvJTl9j32dZENKHgAAAPu36pLX3S9dzyAAAAAcuFWvrllVJ1TVhVW1a3i9r6pOWM9wAAAAzGaWRyi8I8kHk3zn8PrTYR8AAAALYpaSd0x3v6O77xle5yQ5Zp1yAQAAsAazLLzytap6UZLzhu0XZGUhljWpqs8luSPJvUnu6e6ltX4XAAAAK2YpeT+f5C1Jfjsrq2r+dZKfO8DzP627v3qA3wEAAMBglpJ3QnefueeOqjotyRfnGwkAAIC1muWevLesct9qdZKLqmpHVW07gO8BAABgsN+ZvKp6YpInJTmmqv7THh89OMmWAzj3k7v7pqo6NsnFVfWp7r58r3NvS7ItSY483BovAAAA+7OambyDkzwoK4Xw8D1eX09y1lpP3N03DT93JbkwyRP2ccz27l7q7qUHHXrEWk8FAABwv7HfmbzuvizJZVV1Tnd/fh4nrarDkjygu+8Y3v9IkjfO47sBAADuz2a5J++tVfWQ3RtVdWRV/cUaz/uwJB+tqmuSfCzJ/+7uD63xuwAAABjMsrrm0d192+6N7r51uJ9uZt19Q5JT1vJnAQAAuG+zzOT9c1WduHujqr4rKytkAgAAsCBmmcl7XVYusbwsSSV5SoaVLwEAAFgMqy553f2hqnp8klOHXa/u7q+uTywAAADWYr+Xa1bVI4efj09yYpIvDa8Th30AAAAsiNXM5L0mycuT/NY+Puskp8810X049oQj8orf+omNOBUAAMCmtZrn5L18+Pm09Y8DAADAgdhvyauq5/5bn3f3BfOLAwAAwIFYzeWau6+RPDbJk5JcMmw/LclfJ1HyAAAAFsRqLtd8aZJU1UVJHt3dNw/bxyU5Z13TAQAAMJNZnpP38N0Fb/CVrKy2uSFu/uxn8qYXnbVRpwNgwb3uj88fOwIALKRZSt6Hq+ovkpw3bP9Mkr+cfyQAAADWapaHob+iqn4qyQ8Nu7Z394XrEwsAAIC1mGUmL0muSnJHd/9lVR1aVYd39x3rEQwAAIDZPWC1B1bVy5Ocn+QPh13HJ3n/eoQCAABgbVZd8pL8YpLTknw9Sbr701l5rAIAAAALYpaS983u/tbujaramqTXeuKqekhVnV9Vn6qqnVX1xLV+FwAAACtmuSfvsqr6lSQPrKpnJvmFJH96AOf+3SQf6u6zqurgJIcewHcBAACQ2WbyXpvkH5N8Isl/SPLnSf7rWk5aVUdkZZXOtyVJd3+ru29by3cBAADwr1Y1k1dVW5K8s7tfmOR/zuG8352VwviOqjolyY4kZ3f3nXP4bgAAgPutVc3kdfe9Sb5ruKxyHrYmeXySP+juxyW5Myszhd+mqrZV1XJVLd/5T9+c06kBAACma5Z78m5IckVVfTArpSxJ0t1vXsN5b0xyY3dfOWyfn32UvO7enmR7khx/1JFrXuQFAADg/mKWkveZ4fWAJIcfyEm7+8tV9cWqOrm7r0/y9CSfPJDvBAAAYIaS192/liRV9eCVzb7jAM/9yiTvGi4BvSHJSw/w+wAAAO73Vl3yqmopyTsyzOJV1e1Jfr67d6zlxN19dZKltfxZAAAA9m2WyzXfnuQXuvuvkqSqnpyV0vcD6xEMAACA2c3ynLx7dxe8JOnujya5Z/6RAAAAWKtZZvIuq6o/THJekk7yM0k+UlWPT5Luvmod8gEAADCDWUreKcPPN+y1/3FZKX2nzyURAAAAa1bd83n8XFW9pLvPncuX7cPS0lIvLy+v19cDAAAstKra0d37Xbxylnvy9ufsOX4XAAAAazDPkldz/C4AAADWYJ4lbz7XfQIAALBmZvIAAAAmZNWra1bVdyT590lO2vPPdfcbh7dXzDXZXv7p5juy802XrOcpANikHvU6CzwDwG6zPELhA0luT7IjyTf3/rC7XzGvUAAAAKzNLCXvhO4+Y92SAAAAcMBmuSfvr6vq+9ctCQAAAAdslpm8Jyf5uar6bFYu16wk3d0/sC7JAAAAmNksJe/H1i0FAAAAc7Hqktfdn5/XSavq5CTv3mPX9yR5fXf/zrzOAQAAcH80y0ze3HT39UkemyRVtSXJTUkuHCMLAADAlMzzYehr9fQkn5nnTCEAAMD91SKUvOcnOW/sEAAAAFMwasmrqoOTnJnkvffx+baqWq6q5VvuvG1jwwEAAGxCY8/k/ViSq7r7K/v6sLu3d/dSdy899LCHbHA0AACAzWfskveCuFQTAABgbkYreVV1WJJnJrlgrAwAAABTM8ojFJKku+9MctRY5wcAAJiisS/XBAAAYI6UPAAAgAlR8gAAACZEyQMAAJiQ0RZemdUhxx2eR73u9LFjAAAALDQzeQAAABOi5AEAAEyIkgcAADAhSh4AAMCEbJqFV770pS/lV3/1V8eOAcAC8XsBAP5/ZvIAAAAmRMkDAACYECUPAABgQpQ8AACACVHyAAAAJmS0kldV/7Gqrquqa6vqvKo6ZKwsAAAAUzFKyauq45O8KslSdz8myZYkzx8jCwAAwJSMebnm1iQPrKqtSQ5N8qURswAAAEzCKCWvu29K8t+SfCHJzUlu7+6LxsgCAAAwJWNdrnlkkuck+e4k35nksKp60T6O21ZVy1W1fNddd210TAAAgE1nrMs1n5Hks939j919d5ILkjxp74O6e3t3L3X30qGHHrrhIQEAADabsUreF5KcWlWHVlUleXqSnSNlAQAAmIyx7sm7Msn5Sa5K8okhx/YxsgAAAEzJ1rFO3N1vSPKGsc4PAAAwRWM+QgEAAIA5U/IAAAAmRMkDAACYECUPAABgQqq7x86wKktLS728vDx2DAAAgFFU1Y7uXtrfcWbyAAAAJkTJAwAAmBAlDwAAYEKUPAAAgAnZOnaA1br11p15z3ufMHYMABbQTz/vY2NHAICFYSYPAABgQpQ8AACACVHyAAAAJkTJAwAAmBAlDwAAYEJGK3lVdXZVXVtV11XVq8fKAQAAMCWjlLyqekySlyd5QpJTkjy7qr53jCwAAABTMtZM3qOSXNndd3X3PUkuS/LckbIAAABMxlgl79okT6mqo6rq0CTPSvLwkbIAAABMxtYxTtrdO6vqN5JclOTOJFcnuXfv46pqW5JtSXL00QdvaEYAAIDNaLSFV7r7bd39g939Q0luTfJ/93HM9u5e6u6lBz94lD4KAACwqYzWnKrq2O7eVVUnZuV+vFPHygIAADAVY06Pva+qjkpyd5Jf7O7bRswCAAAwCaOVvO5+yljnBgAAmKrR7skDAABg/pQ8AACACVHyAAAAJkTJAwAAmJBN8/C5I498VH76eR8bOwYAAMBCM5MHAAAwIUoeAADAhCh5AAAAE6LkAQAATMimWXjlk7d+Paec/xdjxwBgwV1z1o+OHQEARmUmDwAAYEKUPK8LRFMAAAb9SURBVAAAgAlR8gAAACZEyQMAAJgQJQ8AAGBC1rXkVdXbq2pXVV27x76HVtXFVfXp4eeR65kBAADg/mS9Z/LOSXLGXvtem+TD3f2IJB8etgEAAJiDdS153X15klv22v2cJOcO789N8pPrmQEAAOD+ZIx78h7W3TcP77+c5GEjZAAAAJikURde6e5O0vf1eVVtq6rlqlq+5+u3b2AyAACAzWmMkveVqjouSYafu+7rwO7e3t1L3b209cFHbFhAAACAzWqMkvfBJC8Z3r8kyQdGyAAAADBJ6/0IhfOS/E2Sk6vqxqp6WZJfT/LMqvp0kmcM2wAAAMzB1vX88u5+wX189PT1PC8AAMD91agLrwAAADBfSh4AAMCEKHkAAAATouQBAABMyLouvDJPjz7ywVk+60fHjgEAALDQzOQBAABMiJIHAAAwIdXdY2dYlaq6I8n1Y+dgEo5O8tWxQzAZxhPzYiwxL8YS82Q8LZbv6u5j9nfQprknL8n13b00dgg2v6paNpaYF+OJeTGWmBdjiXkynjYnl2sCAABMiJIHAAAwIZup5G0fOwCTYSwxT8YT82IsMS/GEvNkPG1Cm2bhFQAAAPZvM83kAQAAsB8LX/Kq6oyqur6q/qGqXjt2HhZfVb29qnZV1bV77HtoVV1cVZ8efh457K+q+r1hfP19VT1+vOQsmqp6eFVdWlWfrKrrqursYb/xxEyq6pCq+lhVXTOMpV8b9n93VV05jJl3V9XBw/7vGLb/Yfj8pDHzs3iqaktVfbyq/mzYNpZYk6r6XFV9oqqurqrlYZ/fc5vcQpe8qtqS5L8n+bEkj07ygqp69Lip2ATOSXLGXvtem+TD3f2IJB8etpOVsfWI4bUtyR9sUEY2h3uSvKa7H53k1CS/OPwdZDwxq28mOb27T0ny2CRnVNWpSX4jyW939/cmuTXJy4bjX5bk1mH/bw/HwZ7OTrJzj21jiQPxtO5+7B6PSvB7bpNb6JKX5AlJ/qG7b+jubyX5kyTPGTkTC667L09yy167n5Pk3OH9uUl+co/97+wVf5vkIVV13MYkZdF1983dfdXw/o6s/IPq+BhPzGgYE98YNg8aXp3k9CTnD/v3Hku7x9j5SZ5eVbVBcVlwVXVCkh9P8tZhu2IsMV9+z21yi17yjk/yxT22bxz2wawe1t03D++/nORhw3tjjFUZLnF6XJIrYzyxBsPldVcn2ZXk4iSfSXJbd98zHLLnePmXsTR8fnuSozY2MQvsd5L8UpJ/HraPirHE2nWSi6pqR1VtG/b5PbfJbR07AGy07u6qsqwsq1ZVD0ryviSv7u6v7/mf4MYTq9Xd9yZ5bFU9JMmFSR45ciQ2oap6dpJd3b2jqn547DxMwpO7+6aqOjbJxVX1qT0/9Htuc1r0mbybkjx8j+0Thn0wq6/svpxg+Llr2G+M8W+qqoOyUvDe1d0XDLuNJ9asu29LcmmSJ2blUqfd/+G653j5l7E0fH5Ekq9tcFQW02lJzqyqz2XlNpbTk/xujCXWqLtvGn7uysp/QD0hfs9teote8v4uySOGFaMOTvL8JB8cOROb0weTvGR4/5IkH9hj/4uH1aJOTXL7HpcncD833LfytiQ7u/vNe3xkPDGTqjpmmMFLVT0wyTOzco/npUnOGg7beyztHmNnJbmkPdiWJN39y919QneflJV/F13S3S+MscQaVNVhVXX47vdJfiTJtfF7btNb+IehV9WzsnLt+ZYkb+/uN40ciQVXVecl+eEkRyf5SpI3JHl/kvckOTHJ55P8dHffMvwj/vezshrnXUle2t3LY+Rm8VTVk5P8VZJP5F/vffmVrNyXZzyxalX1A1lZvGBLVv6D9T3d/caq+p6szMY8NMnHk7you79ZVYck+aOs3Ad6S5Lnd/cN46RnUQ2Xa/7n7n62scRaDOPmwmFza5L/1d1vqqqj4vfcprbwJQ8AAIDVW/TLNQEAAJiBkgcAADAhSh4AAMCEKHkAAAATouQBAABMiJIHAKtQVX+++1l3ALDIPEIBAABgQszkAcBequr9VbWjqq6rqm3Dvs9V1dHD+xdX1d9X1TVV9UfjpgWAb2cmDwD2UlUP7e5bquqBSf4uyVOT7EiylORhSS5M8qTu/uruY0eMCwDfZuvYAQBgAb2qqn5qeP/wJI/Y47PTk7y3u7+aJAoeAItGyQOAPVTVDyd5RpIndvddVfWRJIeMGgoAZuCePAD4dkckuXUoeI9Mcupen1+S5HlVdVSycmnnRgcEgH+LkgcA3+5DSbZW1c4kv57kb/f8sLuvS/KmJJdV1TVJ3rzxEQHgvll4BQAAYELM5AEAAEyIkgcAADAhSh4AAMCEKHkAAAATouQBAABMiJIHAAAwIUoeAADAhCh5AAAAE/L/AKSLqlye4+yMAAAAAElFTkSuQmCC\n",
"text/plain": [
"<Figure size 1080x1080 with 3 Axes>"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"# Plot the error metrics\n",
"_, (ax1, ax2, ax3) = plt.subplots(nrows=3, figsize=(15,15))\n",
"sns.barplot(y='n_predictors', x='adj_R_sq', data=best, ax=ax1, orient='h')\n",
"sns.barplot(y='n_predictors', x='bic', data=best, ax=ax2, orient='h')\n",
"sns.barplot(y='n_predictors', x='aic', data=best, ax=ax3, orient='h')"
]
},
{
"cell_type": "code",
"execution_count": 190,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
" OLS Regression Results \n",
"==============================================================================\n",
"Dep. Variable: y R-squared: 0.998\n",
"Model: OLS Adj. R-squared: 0.998\n",
"Method: Least Squares F-statistic: 1.680e+04\n",
"Date: Mon, 01 Oct 2018 Prob (F-statistic): 1.93e-130\n",
"Time: 11:29:50 Log-Likelihood: -126.13\n",
"No. Observations: 100 AIC: 260.3\n",
"Df Residuals: 96 BIC: 270.7\n",
"Df Model: 3 \n",
"Covariance Type: nonrobust \n",
"==============================================================================\n",
" coef std err t P>|t| [0.025 0.975]\n",
"------------------------------------------------------------------------------\n",
"Intercept 1.0229 0.117 8.713 0.000 0.790 1.256\n",
"x 2.2893 0.139 16.474 0.000 2.013 2.565\n",
"x2 3.0189 0.077 38.971 0.000 2.865 3.173\n",
"x3 3.9099 0.044 88.854 0.000 3.823 3.997\n",
"==============================================================================\n",
"Omnibus: 0.465 Durbin-Watson: 1.885\n",
"Prob(Omnibus): 0.792 Jarque-Bera (JB): 0.386\n",
"Skew: 0.150 Prob(JB): 0.825\n",
"Kurtosis: 2.951 Cond. No. 7.10\n",
"==============================================================================\n",
"\n",
"Warnings:\n",
"[1] Standard Errors assume that the covariance matrix of the errors is correctly specified.\n",
" OLS Regression Results \n",
"==============================================================================\n",
"Dep. Variable: y R-squared: 0.998\n",
"Model: OLS Adj. R-squared: 0.998\n",
"Method: Least Squares F-statistic: 1.680e+04\n",
"Date: Mon, 01 Oct 2018 Prob (F-statistic): 1.93e-130\n",
"Time: 11:29:50 Log-Likelihood: -126.13\n",
"No. Observations: 100 AIC: 260.3\n",
"Df Residuals: 96 BIC: 270.7\n",
"Df Model: 3 \n",
"Covariance Type: nonrobust \n",
"==============================================================================\n",
" coef std err t P>|t| [0.025 0.975]\n",
"------------------------------------------------------------------------------\n",
"Intercept 1.0229 0.117 8.713 0.000 0.790 1.256\n",
"x 2.2893 0.139 16.474 0.000 2.013 2.565\n",
"x2 3.0189 0.077 38.971 0.000 2.865 3.173\n",
"x3 3.9099 0.044 88.854 0.000 3.823 3.997\n",
"==============================================================================\n",
"Omnibus: 0.465 Durbin-Watson: 1.885\n",
"Prob(Omnibus): 0.792 Jarque-Bera (JB): 0.386\n",
"Skew: 0.150 Prob(JB): 0.825\n",
"Kurtosis: 2.951 Cond. No. 7.10\n",
"==============================================================================\n",
"\n",
"Warnings:\n",
"[1] Standard Errors assume that the covariance matrix of the errors is correctly specified.\n",
" OLS Regression Results \n",
"==============================================================================\n",
"Dep. Variable: y R-squared: 0.998\n",
"Model: OLS Adj. R-squared: 0.998\n",
"Method: Least Squares F-statistic: 1.680e+04\n",
"Date: Mon, 01 Oct 2018 Prob (F-statistic): 1.93e-130\n",
"Time: 11:29:50 Log-Likelihood: -126.13\n",
"No. Observations: 100 AIC: 260.3\n",
"Df Residuals: 96 BIC: 270.7\n",
"Df Model: 3 \n",
"Covariance Type: nonrobust \n",
"==============================================================================\n",
" coef std err t P>|t| [0.025 0.975]\n",
"------------------------------------------------------------------------------\n",
"Intercept 1.0229 0.117 8.713 0.000 0.790 1.256\n",
"x 2.2893 0.139 16.474 0.000 2.013 2.565\n",
"x2 3.0189 0.077 38.971 0.000 2.865 3.173\n",
"x3 3.9099 0.044 88.854 0.000 3.823 3.997\n",
"==============================================================================\n",
"Omnibus: 0.465 Durbin-Watson: 1.885\n",
"Prob(Omnibus): 0.792 Jarque-Bera (JB): 0.386\n",
"Skew: 0.150 Prob(JB): 0.825\n",
"Kurtosis: 2.951 Cond. No. 7.10\n",
"==============================================================================\n",
"\n",
"Warnings:\n",
"[1] Standard Errors assume that the covariance matrix of the errors is correctly specified.\n"
]
}
],
"source": [
"print(best.sort_values('adj_R_sq', ascending=False)['model'].iloc[0].summary())\n",
"print(best.sort_values('bic', ascending=True)['model'].iloc[0].summary())\n",
"print(best.sort_values('aic', ascending=True)['model'].iloc[0].summary())"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"(e) Now fit a lasso model to the simulated data, again using X,X2, . . . , X 10 as predictors. Use cross-validation to select the optimal value of λ. Create plots of the cross-validation error as a function of λ. Report the resulting coefficient estimates, and discuss the results obtained."
]
},
{
"cell_type": "code",
"execution_count": 192,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"<matplotlib.axes._subplots.AxesSubplot at 0x11999a240>"
]
},
"execution_count": 192,
"metadata": {},
"output_type": "execute_result"
},
{
"data": {
"image/png": "iVBORw0KGgoAAAANSUhEUgAAAY4AAAEKCAYAAAAFJbKyAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3X+UnVV97/H3JxMmDBDMCCFiEhpLIzblKuAISLosiGKkXGMqVvQq0VoCDVS9rYjaUqwpXVJEb701YqgoUJHLRXJJuVyR0lBWoygToSEJUgJoM2lIRpiEAGOGyXzvH2ef8ZkwJ+ecZJ7z8/Na66x5zj7POWfPs2C+2d/v3vtRRGBmZlapSfXugJmZNRcHDjMzq4oDh5mZVcWBw8zMquLAYWZmVXHgMDOzqjhwmJlZVRw4zMysKg4cZmZWlcn17kAejjzyyJgzZ069u2Fm1lTWrl37i4iYXu68lgwcc+bMobe3t97dMDNrKpJ+Xsl5TlWZmVlVHDjMzKwqDhxmZlYVBw4zM6uKA4eZmVWlJWdVmZm1m5GR4JkXhhga3kPn5A6OOLSTSZOUy3c5cJiZNbmRkeCxbbu44MZe+gYGmdXdxXXn93DcjKm5BI/cUlWSDpb0Y0n/JmmDpL9M7d+S9JSkh9PjhNQuSV+RtEnSOkknZT5rsaTH02NxXn02M2smIyNB/67dbN05OBo0APoGCs+feWEol+/Ns8axG3hrRLwBOAFYIOnU9NqlEXFCejyc2t4JzE2PJcDXACS9ErgCOAU4GbhCUneO/TYza3jFUcai5WvoGxgcDRpFfQODDA3vyeW7cwscUfB8enpQesQ+3rIQuDG97wFgmqSjgXcA90TEsxExANwDLMir32ZmjWy8UcaOwZeY1d015rxZ3V10Tu7IpQ+5zqqS1CHpYWA7hT/+P0ovXZnSUV+WNCW1zQQ2Z97el9pKte/9XUsk9Urq7e/vn/DfxcysHoqBYsvAizz7wu5xRxnX3vcEV73n9aPBo1jjOOLQzlz6lGtxPCL2ACdImgaslHQ88BngaaATWAFcBnx+Ar5rRfo8enp69jWyMTNrCnsXvb/54Tdx+R3rx4wy+gYGeWjzDr5492MsW3g8xx51GF0H5TurqibrOCJiB7AaWBARW1M6ajfwTQp1C4AtwOzM22altlLtZmYtqVTR+5DOjpKjjP7nd/OqVxzMrGldTJ86JbegAfnOqpqeRhpI6gLeDvw01S2QJODdwPr0llXA+Wl21anAzojYCtwNnCWpOxXFz0ptZmYtoxgstu0c5NGnnxu36J2tZWRHGfd/6gxWLp2f2/TbveU54jgaWC1pHfAghRrHncC3JT0CPAIcCfxVOv8u4ElgE3AdsBQgIp4FlqXPeBD4fGozM2sJ2RlSD/ft5MKb1o5b9L72vie4+tz6jDKycqtxRMQ64MRx2t9a4vwALi7x2vXA9RPaQTOzOsuu9i6mpKZ1HfSydNRl311H38Ag/c/vZsbhB3P70tN4aXgk9xXipXjluJlZHWQL39e89w2jwaKeRe9KeZNDM7MaKrcOo55F70p5xGFmlrNiSmpkZIRfvDDEhTetHTPKyKakHtq8gxt+8BQ3/+EpdExS3dJR++LAYWaWo2xK6vJz5rHszo11X4dxoJyqMjPLwXgpqfEK342ckirFIw4zswnWzIXvSnjEYWY2QVqh8F0JjzjMzA5AqxW+K+HAYWa2n1qx8F0Jp6rMzKrUyoXvSnjEYWZWgXIpqVYfZWR5xGFmVkYlmxC2+igjyyMOM7MSqtmEsFUK35Vw4DAzy3BKqjynqszMEqekKuMRh5m1PaekquPAYWZtrdW3B8mDU1Vm1pbaZXuQPHjEYWZtIZuO6ursYNtzu182ynBKqjK5BQ5JBwP3A1PS99wWEVdIeg1wC3AEsBb4UEQMSZoC3Ai8EXgGeF9E/Cx91meAjwJ7gI9FxN159dvMWk82HdU3MMg3P/wmLr9jfVtsD5KHPFNVu4G3RsQbgBOABZJOBa4CvhwRvwEMUAgIpJ8Dqf3L6TwkzQPOA34LWAAsl9SRY7/NrEWMl44COKSzo222B8lDboEjCp5PTw9KjwDeCtyW2m8A3p2OF6bnpNfPlKTUfktE7I6Ip4BNwMl59dvMmlsxWGzbOcijTz/HouVr6BsYHA0UwJhaRnaUcf+nzmDl0vkcN2OqA8Y+5Focl9Qh6WFgO3AP8ASwIyKG0yl9wMx0PBPYDJBe30khnTXaPs57st+1RFKvpN7+/v48fh0za3CVrMOAwijj6nM9ythfuRbHI2IPcIKkacBK4HU5ftcKYAVAT09P5PU9ZtZ4qlmH0TcwSP/zu5lx+MHcvvQ0XhoeceG7SjWZVRUROyStBt4MTJM0OY0qZgFb0mlbgNlAn6TJwCsoFMmL7UXZ95hZm/LWIPWTW6pK0vQ00kBSF/B24FFgNXBuOm0xcEc6XpWek17/54iI1H6epClpRtZc4Md59dvMGp+3BqmvPEccRwM3pBlQk4BbI+JOSRuBWyT9FfAQ8I10/jeAmyRtAp6lMJOKiNgg6VZgIzAMXJxSYGbWZrw1SGPILXBExDrgxHHan2ScWVER8UvgvSU+60rgyonuo5k1PqekGo+3HDGzhuWUVGPyliNm1nCckmpsDhxm1hCckmoeTlWZWd05JdVcPOIws7pxSqo5OXCYWU05JdX8nKoys5pxSqo1eMRhZrlzSqq1OHCYWS6ckmpdTlWZ2YRzSqq1ecRhZhPGKan24MBhZgfEKan241SVme03p6Tak0ccZrbfnnlhyCmpNuTAYWb7bWh4j1NSbcipKjOryshI0L9rN1sGXkSSU1JtyCMOMxtXdoZUV2cHwyNBjMRoAbxvYJCz5h3FtR98Ixf9w1qnpNqIA4dZmysXIKYfNoVPLTiOS29bx+XnzGPZnRtH01Pf37gdgFsvfDMR4WDRJnJLVUmaLWm1pI2SNkj6eGr/nKQtkh5Oj7Mz7/mMpE2SHpP0jkz7gtS2SdKn8+qzWSvLppiefWE323f9km07B3n06edYtHwNl9z8EI89vYvfW/6DMTOkLjr9WC69bd3LCuBF39+4nYhgZvchTkm1iTxHHMPAn0bETyRNBdZKuie99uWI+GL2ZEnzgPOA3wJeDfyTpNeml78KvB3oAx6UtCoiNubYd7OWMN4ai1IjiMvPmTdugMgeZwvgRbO6u+ic3FGX38/qI7cRR0RsjYifpONdwKPAzH28ZSFwS0TsjoingE3AyemxKSKejIgh4JZ0rpmNoziyyI4mKhlBjBcg9j7euwA+q7uL687v4YhDO+vwm1q91GRWlaQ5wInAj1LTJZLWSbpeUndqmwlszrytL7WVajezpFywOJAAce19T3D1uYXjbAF8zWVnsHLpfI6bMdXpqTaTe3Fc0mHAd4FPRMRzkr4GLAMi/bwG+IMJ+J4lwBKAY4455kA/zqzhjZeGyqaeSqWYssfZhXrFAHHpbS9ftNfV2cHtS0/jpeERF8At38Ah6SAKQePbEXE7QERsy7x+HXBneroFmJ15+6zUxj7aR0XECmAFQE9PT0zQr2DWUPY3WDhA2ETKLXBIEvAN4NGI+FKm/eiI2JqeLgLWp+NVwM2SvkShOD4X+DEgYK6k11AIGOcBH8ir32aNqrgv1AU39lYdLBwgbCLlOeKYD3wIeETSw6nts8D7JZ1AIVX1M+BCgIjYIOlWYCOFGVkXR8QeAEmXAHcDHcD1EbEhx36bNZRyW5VXGiwcIGyi5BY4IuJfKYwW9nbXPt5zJXDlOO137et9Zq0qO8ootVW5g4XVmleOmzWw7O6zDhbWKBw4zBpQMT314tCwtyq3huPAYdZg9i6Ce6tyazTeVt2swWTTU96q3BqRRxxmDSZ7c6TiKOPyc+bxm6+aSlfnZI8yrO484jBrMJ2TO0ZHGFAIHsvu3EhX52SPMqwhOHCYNZgjDu3kuvN7vJGgNSynqswazKRJ4rgZU1m5dD5Dw3s8Y8oajgOHWQOaNElMnzql3t0wG5cDh1mDyG4t4lGGNTIHDrMGkF270TcwOFrX8L0urBG5OG7WALJrNwD6Bga54MZennlhqM49M3s5Bw6zBpBdu1HUNzDI0PCeOvXIrDQHDrMGsPfaDShMw+2c3FGnHpmV5sBh1gC8dsOaiYvjZg3AazesmThwmDUIr92wZuHAYVYnXrdhzcqBw6wOvG7DmlluxXFJsyWtlrRR0gZJH0/tr5R0j6TH08/u1C5JX5G0SdI6SSdlPmtxOv9xSYvz6rNZrXjdhjWzPGdVDQN/GhHzgFOBiyXNAz4N3BsRc4F703OAdwJz02MJ8DUoBBrgCuAU4GTgimKwMWtWXrdhzSy3wBERWyPiJ+l4F/AoMBNYCNyQTrsBeHc6XgjcGAUPANMkHQ28A7gnIp6NiAHgHmBBXv02qwWv27BmVpN1HJLmACcCPwJmRMTW9NLTwIx0PBPYnHlbX2or1W7WtLxuw5pZ7sVxSYcB3wU+ERHPSb8q/EVESIoJ+p4lFFJcHHPMMRPxkWa58boNa2a5jjgkHUQhaHw7Im5PzdtSCor0c3tq3wLMzrx9Vmor1T5GRKyIiJ6I6Jk+ffrE/iJmE2RkJOjftZstAy/yzAtDHHFoJzO7D/EtYa2p5DmrSsA3gEcj4kuZl1YBxZlRi4E7Mu3np9lVpwI7U0rrbuAsSd2pKH5WajNrKsUpuIuWr2H+VatZtHwNj23bxcjIhAy6zWomzxHHfOBDwFslPZweZwNfAN4u6XHgbek5wF3Ak8Am4DpgKUBEPAssAx5Mj8+nNrOm4im41ipyq3FExL8CpcbeZ45zfgAXl/is64HrJ653ZrXnKbjWKvY54pD0wczx/L1euySvTpm1Ik/BtVZRLlX1J5nj/7nXa38wwX0xa2megmutolyqSiWOx3tuZvvgKbjWKsoFjihxPN5zMyvDW6dbKygXOF4naR2F0cWx6Zj0/Ndz7ZlZi/D26dZqygWO36xJL8xalLdPt1a0z+J4RPw8+wCeB04CjkzPzWwfvHbDWlG56bh3Sjo+HR8NrKcwm+omSZ+oQf/MmprXblgrKjcd9zURsT4df4TC9ub/lcK9MTwd16wMr92wVlQucLyUOT6TwrYgxftrjOTVKbNW4bUb1orKFcc3S/pjCvfAOAn4HoCkLuCgnPtm1vS8dsNaUbnA8VHg8xQ2I3xfROxI7acC38yzY2bNzFNwrZXtM3BExHbgonHaVwOr8+qUWTPzFFxrdfsMHJJW7ev1iHjXxHbHrPmVmoK7cul8rxq3llAuVfVmCvf7/g6F+4X7n0tmZXgKrrW6crOqXgV8Fjge+Fvg7cAvIuJfIuJf8u6cWTPyFFxrdeVWju+JiO9FxGIKBfFNwH2+F4dZaZ6Ca62u7B0AJU0Bfhd4PzAH+AqwMt9umTUvT8G1VleuOH4jhTTVXcBfZlaRm9lePAXX2kW5GscHgbnAx4EfSHouPXZJem5fb5R0vaTtktZn2j4naYukh9Pj7Mxrn5G0SdJjkt6RaV+Q2jZJ+vT+/Zpm+SpOwV20fA3zr1rNouVreGzbLkZGfNsaaz3lahyTImJqehyeeUyNiMPLfPa3gAXjtH85Ik5Ij7sAJM0DzgN+K71nuaQOSR3AV4F3AvOA96dzzRqKd8G1dlK2xrG/IuJ+SXMqPH0hcEtE7AaekrQJODm9tikingSQdEs6d+MEd9fsgHgKrrWTcqmqPFwiaV1KZXWntpkU1osU9aW2Uu1mDcVTcK2d1DpwfA04FjgB2ApcM1EfLGmJpF5Jvf39/RP1sWYV8RRcaye5parGExHbiseSrgPuTE+3ALMzp85Kbeyjfe/PXgGsAOjp6XFF0mrKU3CtndQ0cEg6OiK2pqeLKNxREGAVcLOkLwGvpjCT68cUtjiZK+k1FALGecAHatlns33xFFxrR7kFDknfAU4HjpTUB1wBnC7pBCCAnwEXAkTEBkm3Uih6DwMXR8Se9DmXAHcDHcD1EbEhrz6bVcO74Fq7UkTrZXV6enqit7e33t2wFte/azeLlq8ZM5tqVneXd8G1piVpbUT0lDuvHrOqzFqCp+Bau3LgMNtPnoJr7cqBw2w/eQqutauazqoyawXZmVQzDp/C7UtP46XhEc+qsrbhwGFWBc+kMnOqyqwq3szQzIHDrCqeSWXmwGFWFc+kMnPgMKuKZ1KZuThuVhHPpDL7FQcOszI8k8psLKeqzMrwTCqzsRw4zMrwTCqzsRw4zMrwTCqzsRw4zMrwTCqzsVwcNyvBM6nMxufAYTYOz6QyK82pKrNxeCaVWWkOHGbj8Ewqs9JyCxySrpe0XdL6TNsrJd0j6fH0szu1S9JXJG2StE7SSZn3LE7nPy5pcV79NcvyTCqz0vIccXwLWLBX26eBeyNiLnBveg7wTmBueiwBvgaFQANcAZwCnAxcUQw2ZnnyTCqz0nIrjkfE/ZLm7NW8EDg9Hd8A3AdcltpvjIgAHpA0TdLR6dx7IuJZAEn3UAhG38mr39bePJPKrLxaz6qaERFb0/HTwIx0PBPYnDmvL7WVajebcJ5JZVaZuhXH0+giJurzJC2R1Cupt7+/f6I+1tqIZ1KZVabWgWNbSkGRfm5P7VuA2ZnzZqW2Uu0vExErIqInInqmT58+4R231ueZVGaVqXXgWAUUZ0YtBu7ItJ+fZledCuxMKa27gbMkdaei+FmpzWzCeSaVWWXynI77HeCHwHGS+iR9FPgC8HZJjwNvS88B7gKeBDYB1wFLAVJRfBnwYHp8vlgoN5soIyNB/67djIyM8PUPvdEzqczKUKHU0Fp6enqit7e33t2wJrB3QfyseUfx5787j45J8kwqazuS1kZET7nzvHLc2treBfHvb9zOB/7+R3RO7mD61CkOGmbjcOCwtuaCuFn1HDisrbkgblY9Bw5rSy6Im+0/34/D2s54BfGb//AUF8TNKuQRh7UdF8TNDowDh7UdF8TNDowDh7UdF8TNDowDh7UNF8TNJoaL49YWXBA3mzgecVhbcEHcbOI4cFhbcEHcbOI4cFhLK9Y1ABfEzSaIA4e1rGJdY9HyNVxy80Ncfe7rXRA3mwAujlvLytY1+gYG+ZvvPcayhcdz7FGH0XWQC+Jm+8sjDmtZe9c1Htq8g49860E6hAviZgfAgcNajusaZvly4LCW4rqGWf5c47CW4rqGWf7qMuKQ9DNJj0h6WFJvanulpHskPZ5+dqd2SfqKpE2S1kk6qR59tubguoZZ/uqZqjojIk7I3Bj908C9ETEXuDc9B3gnMDc9lgBfq3lPraEVaxpbBl5EkusaZjlrpBrHQuCGdHwD8O5M+41R8AAwTdLR9eigNZ5sTWP+Vav53Kr1XPtBb2Bolqd61TgC+L6kAL4eESuAGRGxNb3+NDAjHc8ENmfe25fatmJtb7w9qABuvfDNRIQ3MDTLQb1GHL8dESdRSENdLOkt2RcjIigEl4pJWiKpV1Jvf3//BHbVGtl4e1B9f+N2IoKZ3Ye4rmGWg7oEjojYkn5uB1YCJwPbiimo9HN7On0LMDvz9lmpbe/PXBERPRHRM3369Dy7bw3AazXM6qfmgUPSoZKmFo+Bs4D1wCpgcTptMXBHOl4FnJ9mV50K7MyktKwNea2GWX3Vo8YxA1gpqfj9N0fE9yQ9CNwq6aPAz4HfT+ffBZwNbAJeBD5S+y5bI/FaDbP6qnngiIgngTeM0/4McOY47QFcXIOuWZMotVZjzWVnMH3qlDr2zKw9eOW4NbSRkeCZF4YYGt5DV2cHwyOFOROzurvGBA/XNcxqx4HDGk4xWIyMjPCLF4a48Ka1TD9sCp9acByX3raO6YdN4epzX8+lt62jb2DQdQ2zGnPgsLoZbzQRIzEaLC4/Zx7L7txI38Agl58zbzRQuK5hVl+NtHLcWlR2S5BnX9jN9l2/ZNvOQR59+rnRmVGPPb2L31v+Ax7u28mFN62lb2CQaV0HjaajssfgPajM6smBw3JRDBaVBIiLTj92dDSRDRA7Bl8anWabPS5yXcOsPhw4MrL/Mu7ftZuRkaoWr7eN8UYQpUYTlQSIUsHi2vue4Kr3FNZoXHvfE16vYdYgXONIiovKiusDin+Yjpsx1WkQKitYF4+ztYlyAaJvYHDMcTFYXPbddTy0eQc3/OApbv7DU+iYJLo6O7h96Wm8NDziPajM6sgjjmTvzfL6Bga54MZennlhqM49q4+9RxXFldqlRhDVpptKjSaywWLNZWdw5aLXM6v7EGZ2H8IrD53CUVMP9h5UZnXmEUcy3mZ5fQODDA3vqVOP6mfv0dc3P/wmLr9j/T5HEKVGE9kRRDFAXHqbRxNmzcyBI+mc3DFmUdmJs6fxsTPnsicK//Juhz9k2emx2dHXIZ0dZVNMTjeZtQ+nqpIjDu3kuvN7mNXdxYmzp/GpBcdx+R3recvf3Mei5Wt4bNuuli6WZzcOLK6VKKokxeR0k1n7UGErqNbS09MTvb29Vb8v+y/u96144GVbWqxcOr/l9kIa73f++ofeOFrcBkYDabGGcda8o/jz3503OoIYHgleGh4Zc+zRhFnzkbQ2czvvkpyqypg0SUyfOoUtAy+OCRonzp7GRacfy4tDw/TvomX+IGZrGde89w2jv3M21dQ3MEj/87uZcfjBTjGZGeDAMa5svePE2dP45DuOG/0j2grTdMerZWRrFA9t3sEX7/aWHmY2Ptc4xpGtd1x0+rGjQQOaf5puqVpGtnYB0P/8bl71ioOZNa3LtQgzG8MjjnFMmiSOmzGVlUvn8+LQcEukrTzKMLOJ4hFHCcV6xyGdk0f/FV5MWy27cyO/c3VjzrYqt6GgRxlmdqA84iijmLa64Mbel6Wtph82had3/pJDp3TQddDkmv4LvdyW5KW2APEow8wOlANHGaXSVnkWzccLCtnprpUEiOz9K7KrusebMVUcZThgmFklmiZwSFoA/C3QAfx9RHyhVt9dTFv17/rVLUsnYvRRzaih2gBRagsQjzLM7EA1RY1DUgfwVeCdwDzg/ZLm1bof2dlW2T/MxdFHcaX5n61cR9/Aiy/bdrx/126Gh0f26z4V1W5JXmq1N7iWYWYHpikCB3AysCkinoyIIeAWYGGtO5FNW83q7hr9Q5wdfZw4exqLT3sNH/j7H40JCPOvWs2frVzHT8vsMlvNJoJ7H1e64+zKpfObeh2KmdVXs6SqZgKbM8/7gFPq0ZFi2mpkJEaL5tk/6tkgkk0jAbznjbO56B/2fVvUajcR9I6zZlZrzRI4ypK0BFgCcMwxx+T+fdnRx9DwntE/6vu6T3a1NzLKBgUHCDNrFM0SOLYAszPPZ6W2URGxAlgBhU0Oa9Gp8UYfpQICHPh9KhwgzKwRNMXuuJImA/8OnEkhYDwIfCAiNox3/v7ujnsgyt1atbir7MfOfO1ouqrULrMOCmZWD5XujtsUgQNA0tnA/6AwHff6iLiy1Ln1CBxZpdZhdE7uoLvrIAYGX2JoeI8DhJk1lJbbVj0i7gLuqnc/KlFMYZXSavf0MLP20izTcc3MrEE4cJiZWVUcOMzMrCoOHGZmVhUHDjMzq0rTTMethqR+4Of17scEOBL4Rb070SB8Lcby9fgVX4uxDuR6/FpETC93UksGjlYhqbeSOdXtwNdiLF+PX/G1GKsW18OpKjMzq4oDh5mZVcWBo7GtqHcHGoivxVi+Hr/iazFW7tfDNQ4zM6uKRxxmZlYVB446k7RA0mOSNkn69Div/4mkjZLWSbpX0q/Vo5+1Uu56ZM57j6SQ1LKzaSq5FpJ+P/33sUHSzbXuYy1V8P/KMZJWS3oo/f9ydj36WQuSrpe0XdL6Eq9L0lfStVon6aQJ7UBE+FGnB4Ut4p8Afh3oBP4NmLfXOWcAh6TjPwL+V737Xc/rkc6bCtwPPAD01LvfdfxvYy7wENCdnh9V737X+XqsAP4oHc8Dflbvfud4Pd4CnASsL/H62cD/AwScCvxoIr/fI476OhnYFBFPRsQQcAuwMHtCRKyOiBfT0wco3P2wVZW9Hsky4Crgl7XsXI1Vci0uAL4aEQMAEbG9xn2spUquRwCHp+NXAP9Zw/7VVETcDzy7j1MWAjdGwQPANElHT9T3O3DU10xgc+Z5X2or5aMU/hXRqspejzTknh0R/7eWHauDSv7beC3wWklrJD0gaUHNeld7lVyPzwEflNRH4d49f1ybrjWkav+2VKVpbuTU7iR9EOgBfqfefakXSZOALwEfrnNXGsVkCumq0ymMRO+X9F8iYkdde1U/7we+FRHXSHozcJOk4yNipN4dazUecdTXFmB25vms1DaGpLcBfwa8KyJ216hv9VDuekwFjgfuk/QzCrnbVS1aIK/kv40+YFVEvBQRTwH/TiGQtKJKrsdHgVsBIuKHwMEU9m1qRxX9bdlfDhz19SAwV9JrJHUC5wGrsidIOhH4OoWg0co5bChzPSJiZ0QcGRFzImIOhZrPuyKifjeYz0/Z/zaA/0NhtIGkIymkrp6sZSdrqJLr8R/AmQCSfpNC4OivaS8bxyrg/DS76lRgZ0RsnagPd6qqjiJiWNIlwN0UZo1cHxEbJH0e6I2IVcDVwGHA/5YE8B8R8a66dTpHFV6PtlDhtbgbOEvSRmAPcGlEPFO/Xuenwuvxp8B1kv47hUL5hyNNMWo1kr5D4R8NR6aazhXAQQARcS2FGs/ZwCbgReAjE/r9LXpdzcwsJ05VmZlZVRw4zMysKg4cZmZWFQcOMzOrigOHmZlVxYHDrEKSnp+gz/mcpE9WcN63JJ07Ed9pNpEcOMzMrCoOHGZVknRYujfKTyQ9Imlhap8j6adppPDvkr4t6W1pE8LHJZ2c+Zg3SPphar8gvV+S/i7dc+KfgKMy3/kXkh6UtF7SCqXVoGb14MBhVr1fAosi4iQK90u5JvOH/DeAa4DXpccHgN8GPgl8NvMZrwfeCrwZ+AtJrwYWAcdRuJfE+cBpmfP/LiLeFBHHA13AOTn9bmZlecsRs+oJ+GtJbwFGKGxXPSO99lREPAIgaQNwb0SEpEeAOZnPuCMiBoFBSasp3G/iLcB3ImIP8J+S/jlz/hmSPgUcArwS2AD8Y26/odk+OHCYVe+/AdOBN0bES2mn3oPTa9k98qM3AAAAxklEQVTdi0cyz0cY+//b3nv9lNz7R9LBwHIKdzvcLOlzme8zqzmnqsyq9wpgewoaZwD7cx/4hZIOlnQEhc3qHqRwO9z3SepId2s7I51bDBK/kHQY4JlWVlcecZhV79vAP6b0Uy/w0/34jHXAagr3i1gWEf8paSWFusdGCluE/xAgInZIug5YDzxNIciY1Y13xzUzs6o4VWVmZlVx4DAzs6o4cJiZWVUcOMzMrCoOHGZmVhUHDjMzq4oDh5mZVcWBw8zMqvL/ARDfOMdDeBzkAAAAAElFTkSuQmCC\n",
"text/plain": [
"<Figure size 432x288 with 1 Axes>"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"# LASSO Regression - Cross-Validation with test MSE\n",
"\n",
"results = pd.DataFrame()\n",
"alpha_range = np.linspace(10**-1, 1, num=100)\n",
"for train_idx, test_idx in model_selection.KFold(n_splits=10).split(data):\n",
" train, test = data.iloc[train_idx], data.iloc[test_idx]\n",
" errors = []\n",
" for alpha in alpha_range:\n",
" reg = linear_model.Lasso(alpha=alpha, max_iter=1000000)\n",
" reg.fit(train.drop('y', axis=1), train['y'])\n",
" error = metrics.mean_squared_error(test['y'], reg.predict(test.drop('y', axis=1)))\n",
" errors.append(error)\n",
" results = pd.concat([results, pd.DataFrame(errors, index=alpha_range).T], axis=0, ignore_index=True)\n",
" \n",
"\n",
"df = pd.DataFrame({'lambda' : alpha_range, 'MSE' : results.mean()})\n",
"sns.scatterplot(x='lambda', y='MSE', data=df)"
]
},
{
"cell_type": "code",
"execution_count": 193,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"array([ 1.78982643e+00, 1.52503944e+00, 4.13617586e+00, 5.68803568e-01,\n",
" 1.11648904e-04, 0.00000000e+00, 0.00000000e+00, -1.94037799e-02,\n",
" -1.34039474e-03, 1.99471493e-03])"
]
},
"execution_count": 193,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"reg = linear_model.Lasso(alpha=0.15, max_iter=1000000)\n",
"reg.fit(data.drop('y', axis=1), data['y']).coef_\n",
"# The cubic term has the largest coefficient\n",
"# The 5th order and higher terms have very small coefficeints"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"(f) Now generate a response vector Y according to the model Y = β0 + β7X7 + ε,\n",
"and perform best subset selection and the lasso. Discuss the results obtained."
]
},
{
"cell_type": "code",
"execution_count": 197,
"metadata": {},
"outputs": [],
"source": [
"coeffs = np.array([1,2])\n",
"eps = np.random.normal(0,1, 100)\n",
"x7 = x ** 7\n",
"design_matrix = pd.DataFrame({ 'c' : np.repeat(1, 100),\n",
" 'x7' : x7})\n",
"y = (design_matrix @ coeffs.reshape(-1, 1)) + eps.reshape(-1, 1)"
]
},
{
"cell_type": "code",
"execution_count": 201,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"1/10 : 10 combinations, 0.03963327407836914 seconds\n",
"2/10 : 45 combinations, 0.18586397171020508 seconds\n",
"3/10 : 120 combinations, 0.7698366641998291 seconds\n",
"4/10 : 210 combinations, 1.2466669082641602 seconds\n",
"5/10 : 252 combinations, 1.5642271041870117 seconds\n",
"6/10 : 210 combinations, 1.5838689804077148 seconds\n",
"7/10 : 120 combinations, 1.0425281524658203 seconds\n",
"8/10 : 45 combinations, 0.4454309940338135 seconds\n",
"9/10 : 10 combinations, 0.11003971099853516 seconds\n",
"10/10 : 1 combinations, 0.012729883193969727 seconds\n"
]
}
],
"source": [
"data['y'] = y\n",
"best = best_subset(10, 'y', data)"
]
},
{
"cell_type": "code",
"execution_count": 202,
"metadata": {},
"outputs": [],
"source": [
"# Select between models with different numbers of predictors using adjusted training metrics\n",
"best['adj_R_sq'] = best['model'].map(lambda m: m.rsquared_adj)\n",
"best['bic'] = best['model'].map(lambda m: m.bic)\n",
"best['aic'] = best['model'].map(lambda m: m.aic)"
]
},
{
"cell_type": "code",
"execution_count": 203,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"<matplotlib.axes._subplots.AxesSubplot at 0x11a1d54a8>"
]
},
"execution_count": 203,
"metadata": {},
"output_type": "execute_result"
},
{
"data": {
"image/png": "iVBORw0KGgoAAAANSUhEUgAAA3kAAANgCAYAAACRKaGGAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzs3X+0rXVdL/r3h70hBBEQ0EsgYmWoWSit4UXxR4ImegrL0ODmz7ju7kgNy1tHs+GvU2PYMa2OlbddIliGKUqhmT9ShCMqtjai8kPyt4LkVvkpjBTwc/9Yk9M+273Za64913rWetbrNcYca85nPnt+34xnzrV57+/zfGd1dwAAABiHPYYOAAAAwOwoeQAAACOi5AEAAIyIkgcAADAiSh4AAMCIKHkAAAAjouQBAACMiJIHAAAwIkoeAADAiGwcOsBiHXzwwX3kkUcOHQMAAGAQW7Zs+VZ3H7Kr/dZMyTvyyCMzPz8/dAwAAIBBVNVXFrOf0zUBAABGZM3M5F159bfz07/95qFjAAAAI7XlNc8cOsJMmMkDAAAYESUPAABgRJQ8AACAEVHyAAAARkTJAwAAGJFBSl5VnVFVW6vqsiHGBwAAGKuhZvLOTHLiQGMDAACM1iAlr7svTHLdEGMDAACM2aq+Jq+qNlXVfFXN337rzUPHAQAAWPVWdcnr7s3dPdfdcxv32W/oOAAAAKveqi55AAAATEfJAwAAGJGhvkLh7CQfS3JUVV1dVacNkQMAAGBsNg4xaHefOsS4AAAAY+d0TQAAgBFR8gAAAEZEyQMAABgRJQ8AAGBEBll4ZSkeePhBmX/NM4eOAQAAsKqZyQMAABgRJQ8AAGBElDwAAIARUfIAAABGZM0svPK9ay/PV1/1k0PHAAAARuqIl31m6AgzYSYPAABgRJQ8AACAEVHyAAAARkTJAwAAGBElDwAAYEQGKXlVdZ+qOr+qrqiqy6vq9CFyAAAAjM1QX6Fwe5IXdfclVbVfki1V9YHuvmKgPAAAAKMwyExed1/b3ZdM7t+c5Mokhw2RBQAAYEwGvyavqo5M8tAkF+/guU1VNV9V89fdcsdKRwMAAFhzBi15VXX3JO9I8sLuvmn757t7c3fPdffcPffdsPIBAQAA1pjBSl5V7ZmFgveW7n7nUDkAAADGZKjVNSvJG5Nc2d2vGyIDAADAGA01k3dckmckOb6qLp3cnjRQFgAAgNEY5CsUuvsjSWqIsQEAAMZs8NU1AQAAmB0lDwAAYESUPAAAgBFR8gAAAEZkkIVXlmKvQ38iR7xsfugYAAAAq5qZPAAAgBFR8gAAAEZEyQMAABgRJQ8AAGBE1szCK5/d+tkc9/rjho4BAACM1EUvuGjoCDNhJg8AAGBElDwAAIARUfIAAABGRMkDAAAYESUPAABgRAYpeVW1d1V9oqo+VVWXV9Urh8gBAAAwNkN9hcJ3kxzf3d+pqj2TfKSq/rm7Pz5QHgAAgFEYpOR1dyf5zuThnpNbD5EFAABgTAa7Jq+qNlTVpUm2JvlAd1+8g302VdV8Vc3f9p3bVj4kAADAGjNYyevuO7r7IUkOT/KwqnrwDvbZ3N1z3T235933XPmQAAAAa8zgq2t29w1Jzk9y4tBZAAAA1rqhVtc8pKoOmNy/W5LHJ/nsEFkAAADGZKjVNQ9NclZVbchC0Xxbd797oCwAAACjMdTqmp9O8tAhxgYAABizwa/JAwAAYHaUPAAAgBFR8gAAAEZEyQMAABiRoVbXnNoD7vWAXPSCi4aOAQAAsKqZyQMAABgRJQ8AAGBElDwAAIARUfIAAABGZM0svHLzVVflgkc/ZugYAADASD3mwguGjjATZvIAAABGRMkDAAAYESUPAABgRJQ8AACAEVHyAAAARmTQkldVG6rqk1X17iFzAAAAjMXQM3mnJ7ly4AwAAACjMVjJq6rDk/yXJH89VAYAAICxGXIm70+S/E6S7+9sh6raVFXzVTV/4223rVwyAACANWqQkldVP5dka3dvuav9untzd89199z+e+65QukAAADWrqFm8o5LclJVfTnJW5McX1V/O1AWAACA0Rik5HX3S7r78O4+MskpST7U3U8fIgsAAMCYDL26JgAAADO0cegA3f3hJB8eOAYAAMAomMkDAAAYESUPAABgRBZd8qpq36raY3L/x6vqpKryvQYAAACryDQzeRcm2buqDkvy/iTPSHLmcoQCAABgaaZZeKW6+9aqOi3JX3T3f6+qS5cr2Pb2O+qoPObCC1ZqOAAAgDVpmpm8qqqHJ/mVJP802bZh9pEAAABYqmlK3ulJXpLk3O6+vKp+JMn5yxMLAACApVjU6ZpVtSHJSd190p3buvuLSX5juYIBAAAwvUXN5HX3HUkeucxZAAAA2E3TLLzyyao6L8nbk9xy58bufufMU+3A1qtvzJ+96F0rMRQAALAOPf+1Pz90hJmYpuTtneTbSY7fZlsnWZGSBwAAwK4tuuR193OWMwgAAAC7b9Gra1bV4VV1blVtndzeUVWHL2c4AAAApjPNVyi8Kcl5SX54cnvXZBsAAACrxDQl75DuflN33z65nZnkkGXKBQAAwBJMs/DKt6vq6UnOnjw+NQsLsSxJVX05yc1J7khye3fPLfW1AAAAWDBNyfvVJK9P8sdZWFXzo0mevZvjP7a7v7WbrwEAAMDENCXv8O4+adsNVXVckq/NNhIAAABLNc01ea9f5LbF6iTvr6otVbVpRztU1aaqmq+q+e/ceuNuDAUAALA+7HImr6oenuQRSQ6pqt/a5ql7JNmwG2M/sruvqap7JflAVX22uy/cdofu3pxkc5Ic8X/cv3djLAAAgHVhMTN5eyW5exYK4X7b3G5KcvJSB+7uayY/tyY5N8nDlvpaAAAALNjlTF53X5Dkgqo6s7u/MotBq2rfJHt0982T+z+b5FWzeG0AAID1bJpr8v66qg6480FVHVhV71viuPdO8pGq+lSSTyT5p+5+7xJfCwAAgIlpVtc8uLtvuPNBd18/uZ5uat39xSRHL+XPAgAAsHPTzOR9v6qOuPNBVd03CytkAgAAsEpMM5P30iycYnlBkkryqCQ7/OoDAAAAhrHoktfd762qY5IcO9n0wu7+1vLEAgAAYCl2ebpmVT1g8vOYJEck+frkdsRkGwAAAKvEYmbyXpTkuUleu4PnOsnxM020E/c6fP88/7U/vxJDAQAArFmL+Z68505+Pnb54wAAALA7dlnyquopd/V8d79zdnEAAADYHYs5XfPOcyTvleQRST40efzYJB9NouQBAACsEos5XfM5SVJV70/yoO6+dvL40CRnLms6AAAApjLN9+Td586CN/GNLKy2uSKu/dIX8gdPP3mlhgMAANaZl/7tOUNHmIlpSt4Hq+p9Sc6ePP7lJP8y+0gAAAAs1TRfhv78qvrFJI+ebNrc3ecuTywAAACWYpqZvCS5JMnN3f0vVbVPVe3X3TcvRzAAAACmt8did6yq5yY5J8lfTjYdluQfliMUAAAAS7PokpfkeUmOS3JTknT357LwtQoAAACsEtOUvO929/fufFBVG5P0UgeuqgOq6pyq+mxVXVlVD1/qawEAALBgmmvyLqiq301yt6p6fJJfT/Ku3Rj7T5O8t7tPrqq9kuyzG68FAABAppvJe3GSbyb5TJJfS/KeJL+3lEGrav8srNL5xiTp7u919w1LeS0AAAD+06Jm8qpqQ5I3d/evJPmrGYx7vywUxjdV1dFJtiQ5vbtv2W7cTUk2Jcn++9xtBsMCAACM26Jm8rr7jiT3nZxWOQsbkxyT5A3d/dAkt2RhpnD7cTd391x3z+279w/NaGgAAIDxmuaavC8muaiqzstCKUuSdPfrljDu1Umu7u6LJ4/PyQ5KHgAAANOZpuR9YXLbI8l+uzNod/97VX2tqo7q7quSnJDkit15TQAAAKYoed39yiSpqnssPOybd3PsFyR5y+QU0C8mec5uvh4AAMC6t+iSV1VzSd6UySxeVd2Y5Fe7e8tSBu7uS5PMLeXPAgAAsGPTnK55RpJf7+7/mSRV9cgslL6fWo5gAAAATG+a78m7486ClyTd/ZEkt88+EgAAAEs1zUzeBVX1l0nOTtJJfjnJh6vqmCTp7kuWIR8AAABTmKbkHT35+fLttj80C6Xv+JkkAgAAYMmqu2fzQlXP6u6zZvJiOzA3N9fz8/PL9fIAAACrWlVt6e5dLl45zTV5u3L6DF8LAACAJZhlyasZvhYAAABLMMuSN5vzPgEAAFgyM3kAAAAjsujVNavqh5L8UpIjt/1z3f2qyd2LZppsO/9x7c258g8+tJxDAAAA69gDXzqOLwyY5isU/jHJjUm2JPnu9k929/NnFQoAAIClmabkHd7dJy5bEgAAAHbbNNfkfbSqfnLZkgAAALDbppnJe2SSZ1fVl7JwumYl6e7+qWVJBgAAwNSmKXlPXLYUAAAAzMSiS153f2VWg1bVUUn+fptNP5LkZd39J7MaAwAAYD2aZiZvZrr7qiQPSZKq2pDkmiTnDpEFAABgTGb5ZehLdUKSL8xyphAAAGC9Wg0l75QkZ+/oiaraVFXzVTV/3S03rHAsAACAtWfQkldVeyU5Kcnbd/R8d2/u7rnunrvnvgesbDgAAIA1aOiZvCcmuaS7vzFwDgAAgFEYuuSdmp2cqgkAAMD0Bit5VbVvkscneedQGQAAAMZmkK9QSJLuviXJQUONDwAAMEZDn64JAADADCl5AAAAI6LkAQAAjIiSBwAAMCKDLbwyrb0P3S8PfOnxQ8cAAABY1czkAQAAjIiSBwAAMCJKHgAAwIgoeQAAACOyZhZe+frXv55XvOIVQ8cAAABGaix9w0weAADAiCh5AAAAI6LkAQAAjIiSBwAAMCJKHgAAwIgMVvKq6jer6vKquqyqzq6qvYfKAgAAMBaDlLyqOizJbySZ6+4HJ9mQ5JQhsgAAAIzJkKdrbkxyt6ramGSfJF8fMAsAAMAoDFLyuvuaJH+U5KtJrk1yY3e/f/v9qmpTVc1X1fytt9660jEBAADWnKFO1zwwyZOT3C/JDyfZt6qevv1+3b25u+e6e26fffZZ6ZgAAABrzlCnaz4uyZe6+5vdfVuSdyZ5xEBZAAAARmOokvfVJMdW1T5VVUlOSHLlQFkAAABGY6hr8i5Ock6SS5J8ZpJj8xBZAAAAxmTjUAN398uTvHyo8QEAAMZoyK9QAAAAYMaUPAAAgBFR8gAAAEZEyQMAABiR6u6hMyzK3Nxcz8/PDx0DAABgEFW1pbvndrWfmTwAAIARUfIAAABGRMkDAAAYESUPAABgRDYOHWCxrr/+yrzt7Q8bOgYAADBST3vqJ4aOMBNm8gAAAEZEyQMAABgRJQ8AAGBElDwAAIARUfIAAABGZLCSV1WnV9VlVXV5Vb1wqBwAAABjMkjJq6oHJ3lukoclOTrJz1XVjw2RBQAAYEyGmsl7YJKLu/vW7r49yQVJnjJQFgAAgNEYquRdluRRVXVQVe2T5ElJ7rP9TlW1qarmq2r+pptuX/GQAAAAa83GIQbt7iur6g+TvD/JLUkuTXLHDvbbnGRzkvzoj+7bKxoSAABgDRps4ZXufmN3/3R3PzrJ9Un+bagsAAAAYzHITF6SVNW9untrVR2Rhevxjh0qCwAAwFgMVvKSvKOqDkpyW5LndfcNA2YBAAAYhcFKXnc/aqixAQAAxmqwa/IAAACYPSUPAABgRJQ8AACAEVHyAAAARmTI1TWncuCBD8zTnvqJoWMAAACsambyAAAARkTJAwAAGBElDwAAYESUPAAAgBFZMwuvXHH9TTn6nPcNHQMAABipT538hKEjzISZPAAAgBFR8gAAAEZEyQMAABgRJQ8AAGBElDwAAIARWdaSV1VnVNXWqrpsm233rKoPVNXnJj8PXM4MAAAA68lyz+SdmeTE7ba9OMkHu/v+ST44eQwAAMAMLGvJ6+4Lk1y33eYnJzlrcv+sJL+wnBkAAADWkyGuybt3d187uf/vSe69sx2ralNVzVfV/O033bgy6QAAANawQRde6e5O0nfx/ObunuvuuY332H8FkwEAAKxNQ5S8b1TVoUky+bl1gAwAAACjNETJOy/Jsyb3n5XkHwfIAAAAMErL/RUKZyf5WJKjqurqqjotyauTPL6qPpfkcZPHAAAAzMDG5Xzx7j51J0+dsJzjAgAArFeDLrwCAADAbCl5AAAAI6LkAQAAjIiSBwAAMCLLuvDKLD3owHtk/uQnDB0DAABgVTOTBwAAMCLV3UNnWJSqujnJVUPnYF06OMm3hg7BuuX9x1C89xiK9x5DWu3vv/t29yG72mnNnK6Z5Krunhs6BOtPVc177zEU7z+G4r3HULz3GNJY3n9O1wQAABgRJQ8AAGBE1lLJ2zx0ANYt7z2G5P3HULz3GIr3HkMaxftvzSy8AgAAwK6tpZk8AAAAdkHJAwAAGJFVVfKq6sSquqqqPl9VL97B8z9UVX8/ef7iqjpy5VMyVot4//1WVV1RVZ+uqg9W1X2HyMn47Oq9t81+v1RVXVVrfmlnVo/FvP+q6mmT33+XV9XfrXRGxmkRf+8eUVXnV9UnJ3/3PmmInIxPVZ1RVVur6rKdPF9V9T8m781PV9UxK51xd62akldVG5L8eZInJnlQklOr6kHb7XZakuu7+8eS/HGSP1zZlIzVIt9/n0wy190/leScJP99ZVMyRot876Wq9ktyepKLVzYhY7aY919V3T/JS5Ic190/keSFKx6U0Vnk777fS/K27n5oklOS/MXKpmTEzkxy4l08/8Qk95/cNiV5wwpkmqlVU/KSPCzJ57v7i939vSRvTfLk7fZ5cpKzJvfPSXJCVdUKZmS8dvn+6+7zu/vWycOPJzl8hTMyTov53Zck/y0L/7D1HysZjtFbzPvvuUn+vLuvT5Lu3rrCGRmnxbz3Osk9Jvf3T/L1FczHiHX3hUmuu4tdnpzkzb3g40kOqKpDVybdbKymkndYkq9t8/jqybYd7tPdtye5MclBK5KOsVvM+29bpyX552VNxHqxy/fe5DSR+3T3P61kMNaFxfzu+/EkP15VF1XVx6vqrv71GxZrMe+9VyR5elVdneQ9SV6wMtFg6v8vXHU2Dh0A1pqqenqSuSSPGToL41dVeyR5XZJnDxyF9WtjFk5Z+pksnMFwYVX9ZHffMGgq1oNTk5zZ3a+tqocn+ZuqenB3f3/oYLDaraaZvGuS3Gebx4dPtu1wn6ramIWp+2+vSDrGbjHvv1TV45K8NMlJ3f3dFcrGuO3qvbdfkgcn+XBVfTnJsUnOs/gKM7KY331XJzmvu2/r7i8l+bcslD7YHYt5752W5G1J0t0fS7J3koNXJB3r3aL+v3A1W00l71+T3L+q7ldVe2XhAtvzttvnvCTPmtw/OcmH2re5Mxu7fP9V1UOT/GUWCp5rUpiVu3zvdfeN3X1wdx/Z3Udm4XrQk7p7fpi4jMxi/u79hyzM4qWqDs7C6ZtfXMmQjNJi3ntfTXJCklTVA7NQ8r65oilZr85L8szJKpvHJrmxu68dOtQ0Vs3pmt19e1U9P8n7kmxIckZ3X15Vr0oy393nJXljFqbqP5+FiyVPGS4xY7LI999rktw9ydsn6/18tbtPGiw0o7DI9x4si0W+/96X5Ger6ookdyT57e52Fg27ZZHvvRcl+auq+s0sLMLybP+4zyxU1dlZ+MergyfXfL48yZ5J0t3/XxauAX1Sks8nuTXJc4ZJunTlswIAADAeq+l0TQAAAHaTkgcAADAiSh4AAMCIKHkAAAAjouQBAACMiJIHwLpUVc+uqj+b3P9/quqZd7HvmVX1paq6tKo+VVUnrFxSAJjOqvmePAAYyuR7kXblt7v7nKp6bJLNSe6/zLEAYEnM5AEwSlX1D1W1paour6pNk23Pqap/q6pPJDlum31fUVX/7yJf+mNJDtvF2K+uqiuq6tNV9UeTbferqo9V1Weq6ver6jtL/E8DgLtkJg+AsfrV7r6uqu6W5F+r6p+SvDLJTye5Mcn5ST65hNc9Mck/7OzJqjooyS8meUB3d1UdMHnqT5O8obvfXFXPW8K4ALAoZvIAGKvfqKpPJfl4kvskeUaSD3f3N7v7e0n+fsrXe01V/VuSv0vyh3ex341J/iPJG6vqKUlunWw/LsnZk/t/M+XYALBoSh4Ao1NVP5PkcUke3t1HZ2HG7rO7+bK/3d0/nuS/JjljZzt19+1JHpbknCQ/l+S92z69mxkAYJeUPADGaP8k13f3rVX1gCTHJrlbksdU1UFVtWeSpy7xtf8syR5V9YQdPVlVd0+yf3e/J8lvJjl68tRFSU6Z3P+VJY4NALuk5AEwRu9NsrGqrkzy6iycsnltkldkYeGUi5Jcud2fWdQsW3d3kt9P8js72WW/JO+uqk8n+UiS35psPz3J86rqM9nFwi0AsDtq4e8qAFi/qur1SS7p7jet4Jjf6e67r9R4AKwfZvIAWNeq6r8l+T+TnDd0FgCYBTN5ADBRVX+ebb4/b+JPdzbDV1XnJrnfdpv/a3e/bznyAcBiKHkAAAAj4nRNAACAEVHyAAAARkTJAwAAGBElDwAAYESUPAAAgBFR8gAAAEZEyQMAABgRJQ8AAGBElDwAAIARUfIAAABGRMkDAAAYESUPAABgRJQ8AACAEVHyAAAARmTj0AEW6+CDD+4jjzxy6BgAAACD2LJly7e6+5Bd7bdmSt6RRx6Z+fn5oWMAAAAMoqq+spj9nK4JAAAwImtmJu/Kq7+dn/7tNw8dAwAAVpUtr3nm0BFYZczkAQAAjIiSBwAAMCJKHgAAwIgoeQAAACMySMmrqjOqamtVXTbE+AAAAGM11EzemUlOHGhsAACA0Rqk5HX3hUmuG2JsAACAMXNNHgAAwIis6pJXVZuqar6q5m+/9eah4wAAAKx6q7rkdffm7p7r7rmN++w3dBwAAIBVb1WXPAAAAKYz1FconJ3kY0mOqqqrq+q0IXIAAACMzcYhBu3uU4cYFwAAYOycrgkAADAiSh4AAMCIKHkAAAAjouQBAACMiJIHAAAwIoOsrrkUDzz8oMy/5plDxwAAAFjVzOQBAACMiJIHAAAwIkoeAADAiCh5AAAAI7JmFl753rWX56uv+smhYwAAwJpwxMs+M3QEBmImDwAAYESUPAAAgBFR8gAAAEZEyQMAABiRQUpeVd2nqs6vqiuq6vKqOn2IHAAAAGMz1Oqatyd5UXdfUlX7JdlSVR/o7isGygMAADAKg8zkdfe13X3J5P7NSa5MctgQWQAAAMZk8GvyqurIJA9NcvGwSQAAANa+QUteVd09yTuSvLC7b9rB85uqar6q5q+75Y6VDwgAALDGDFbyqmrPLBS8t3T3O3e0T3dv7u657p67574bVjYgAADAGjTU6pqV5I1Jruzu1w2RAQAAYIyGmsk7LskzkhxfVZdObk8aKAsAAMBoDPIVCt39kSQ1xNgAAABjNvjqmgAAAMyOkgcAADAiSh4AAMCIKHkAAAAjouQBAACMyCCray7FXof+RI542fzQMQAAAFY1M3kAAAAjouQBAACMiJIHAAAwIkoeAADAiKyZhVc+u/WzOe71xw0dAwAARuGiF1w0dASWiZk8AACAEVHyAAAARkTJAwAAGBElDwAAYEQGKXlVtXdVfaKqPlVVl1fVK4fIAQAAMDZDra753STHd/d3qmrPJB+pqn/u7o8PlAcAAGAUBil53d1JvjN5uOfk1kNkAQAAGJPBrsmrqg1VdWmSrUk+0N0XD5UFAABgLAYred19R3c/JMnhSR5WVQ/efp+q2lRV81U1f9t3blv5kAAAAGvM4KtrdvcNSc5PcuIOntvc3XPdPbfn3fdc+XAAAABrzFCrax5SVQdM7t8tyeOTfHaILAAAAGMy1OqahyY5q6o2ZKFovq273z1QFgAAgNEYanXNTyd56BBjAwAAjNng1+QBAAAwO0oeAADAiCh5AAAAI6LkAQAAjIiSBwAAMCJDfYXC1B5wrwfkohdcNHQMAACAVc1MHgAAwIgoeQAAACOi5AEAAIyIkgcAADAia2bhlZuvuioXPPoxQ8cAAIB15TEXXjB0BKZkJg8AAGBElDwAAIARUfIAAABGRMkDAAAYkUFLXlVtqKpPVtW7h8wBAAAwFkPP5J2e5MqBMwAAAIzGYCWvqg5P8l+S/PVQGQAAAMZmyJm8P0nyO0m+P2AGAACAURmk5FXVzyXZ2t1bdrHfpqqar6r5G2+7bYXSAQAArF1DzeQdl+SkqvpykrcmOb6q/nb7nbp7c3fPdffc/nvuudIZAQAA1pxBSl53v6S7D+/uI5OckuRD3f30IbIAAACMydCrawIAADBDG4cO0N0fTvLhgWMAAACMgpk8AACAEVHyAAAARmTRJa+q9q2qPSb3f7yqTqoqS14CAACsItPM5F2YZO+qOizJ+5M8I8mZyxEKAACApZmm5FV335rkKUn+orufmuQnlicWAAAASzHN6ppVVQ9P8itJTpts2zD7SDu231FH5TEXXrBSwwEAAKxJ08zknZ7kJUnO7e7Lq+pHkpy/PLEAAABYikXN5FXVhiQndfdJd27r7i8m+Y3lCgYAAMD0FjWT1913JHnkMmcBAABgN01zTd4nq+q8JG9PcsudG7v7nTNPBQAAwJJMU/L2TvLtJMdvs62TrEjJ23r1jfmzF71rJYYCAAAW6fmv/fmhI7CdRZe87n7OcgYBAABg9y16dc2qOryqzq2qrZPbO6rq8OUMBwAAwHSm+QqFNyU5L8kPT27vmmwDAABglZim5B3S3W/q7tsntzOTHLJMuQAAAFiCaRZe+XZVPT3J2ZPHp2ZhIZYlqaovJ7k5yR1Jbu/uuaW+FgAAAAumKXm/muT1Sf44C6tqfjTJs3dz/Md297d28zUAAACYmKbkHd7dJ227oaqOS/K12UYCAABgqaa5Ju/1i9y2WJ3k/VW1pao27cbrAAAAMLHLmbyqeniSRyQ5pKp+a5un7pFkw26M/cjuvqaq7pXkA1X12e6+cLuxNyXZlCQH7meNFwAAgF1ZzEzeXknunoVCuN82t5uSnLzUgbv7msnPrUnOTfKwHeyzubvnunvu7vvsv9ShAAAA1o1dzuR19wVJLqiqM7v7K7MYtKo+qxwkAAAgAElEQVT2TbJHd988uf+zSV41i9cGAABYz6a5Ju+vq+qAOx9U1YFV9b4ljnvvJB+pqk8l+USSf+ru9y7xtQAAAJiYZnXNg7v7hjsfdPf1k+vpptbdX0xy9FL+LAAAADs3zUze96vqiDsfVNV9s7BCJgAAAKvENDN5L83CKZYXJKkkj8pk5UsAAABWh0WXvO5+b1Udk+TYyaYXdve3licWAAAAS7HL0zWr6gGTn8ckOSLJ1ye3IybbAAAAWCUWM5P3oiTPTfLaHTzXSY6faSIAAACWrLrXxtopc3NzPT8/P3QMAACAQVTVlu6e29V+u5zJq6qn3NXz3f3OaYIBAACwfBZzuubPT37eK8kjknxo8vixST6aRMkDAABYJXZZ8rr7OUlSVe9P8qDuvnby+NAkZy5rOgAAAKYyzffk3efOgjfxjSystrkirv3SF/IHTz95pYYDAABm6KV/e87QEdaNaUreB6vqfUnOnjz+5ST/MvtIAAAALNU0X4b+/Kr6xSSPnmza3N3nLk8sAAAAlmKambwkuSTJzd39L1W1T1Xt1903L0cwAAAAprfHYnesqucmOSfJX042HZbkH5YjFAAAAEuz6JKX5HlJjktyU5J09+ey8LUKAAAArBLTlLzvdvf37nxQVRuT9FIHrqoDquqcqvpsVV1ZVQ9f6msBAACwYJpr8i6oqt9NcreqenySX0/yrt0Y+0+TvLe7T66qvZLssxuvBQAAQKabyXtxkm8m+UySX0vyniS/t5RBq2r/LKzS+cYk6e7vdfcNS3ktAAAA/tOiZvKqakOSN3f3ryT5qxmMe78sFMY3VdXRSbYkOb27b5nBawMAAKxbi5rJ6+47ktx3clrlLGxMckySN3T3Q5PckoWZwv9NVW2qqvmqmr/lP747o6EBAADGa5pr8r6Y5KKqOi8LpSxJ0t2vW8K4Vye5ursvnjw+Jzsoed29OcnmJDnsoAOXvMgLAADAejFNyfvC5LZHkv12Z9Du/veq+lpVHdXdVyU5IckVu/OaAAAATFHyuvuVSVJV91h42Dfv5tgvSPKWySmgX0zynN18PQAAgHVv0SWvquaSvCmTWbyqujHJr3b3lqUM3N2XJplbyp8FAABgx6Y5XfOMJL/e3f8zSarqkVkofT+1HMEAAACY3jTfk3fHnQUvSbr7I0lun30kAAAAlmqambwLquovk5ydpJP8cpIPV9UxSdLdlyxDPgAAAKYwTck7evLz5dttf2gWSt/xM0kEAADAkk2zuuZj7+r5qnpWd5+1+5EAAABYquqezXeMV9Ul3X3MTF5sB+bm5np+fn65Xh4AAGBVq6ot3b3LbyiYZuGVXY45w9cCAABgCWZZ8mYzJQgAAMCSmckDAAAYkUUvvFJVP5Tkl5Icue2f6+5XTe5eNNNk2/mPa2/OlX/woeUcAgAAGMgDX2qx/lmZ5isU/jHJjUm2JPnu9k929/NnFQoAAIClmabkHd7dJy5bEgAAAHbbNNfkfbSqfnLZkgAAALDbppnJe2SSZ1fVl7JwumYl6e7+qWVJBgAAwNSmKXlPXLYUAAAAzMSiS153f2VWg1bVUUn+fptNP5LkZd39J7MaAwAAYD2aZiZvZrr7qiQPSZKq2pDkmiTnDpEFAABgTGb5ZehLdUKSL8xyphAAAGC9Wg0l75QkZw8dAgAAYAwGLXlVtVeSk5K8fSfPb6qq+aqav+6WG1Y2HAAAwBo09EzeE5Nc0t3f2NGT3b25u+e6e+6e+x6wwtEAAADWnqFL3qlxqiYAAMDMDFbyqmrfJI9P8s6hMgAAAIzNIF+hkCTdfUuSg4YaHwAAYIyGPl0TAACAGVLyAAAARkTJAwAAGBElDwAAYESUPAAAgBEZbHXNae196H554EuPHzoGAADAqmYmDwAAYESUPAAAgBFR8gAAAEZkzVyT9/Wvfz2veMUrho4BAAAMTC+4a2byAAAARkTJAwAAGBElDwAAYESUPAAAgBFR8gAAAEZksJJXVb9ZVZdX1WVVdXZV7T1UFgAAgLEYpORV1WFJfiPJXHc/OMmGJKcMkQUAAGBMhjxdc2OSu1XVxiT7JPn6gFkAAABGYZCS193XJPmjJF9Ncm2SG7v7/UNkAQAAGJOhTtc8MMmTk9wvyQ8n2beqnr6D/TZV1XxVzd96660rHRMAAGDNGep0zccl+VJ3f7O7b0vyziSP2H6n7t7c3XPdPbfPPvuseEgAAIC1ZqiS99Ukx1bVPlVVSU5IcuVAWQAAAEZjqGvyLk5yTpJLknxmkmPzEFkAAADGZONQA3f3y5O8fKjxAQAAxmjIr1AAAABgxpQ8AACAEVHyAAAARkTJAwAAGBElDwAAYESqu4fOsChzc3M9Pz8/dAwAAIBBVNWW7p7b1X5m8gAAAEZEyQMAABgRJQ8AAGBENg4dYLGuv/7KvO3tDxs6BgAAsEY87amfGDrCIMzkAQAAjIiSBwAAMCJKHgAAwIgoeQAAACOi5AEAAIzIYCWvqk6vqsuq6vKqeuFQOQAAAMZkkJJXVQ9O8twkD0tydJKfq6ofGyILAADAmAw1k/fAJBd3963dfXuSC5I8ZaAsAAAAozFUybssyaOq6qCq2ifJk5LcZ6AsAAAAo7FxiEG7+8qq+sMk709yS5JLk9yx/X5VtSnJpiQ5+OC9VjQjAADAWjTYwivd/cbu/unufnSS65P82w722dzdc909d497DNJHAQAA1pTBmlNV3au7t1bVEVm4Hu/YobIAAACMxZDTY++oqoOS3Jbked19w4BZAAAARmGwktfdjxpqbAAAgLEa7Jo8AAAAZk/JAwAAGBElDwAAYESUPAAAgBFR8gAAAEZkzXzD+IEHPjBPe+onho4BAACwqpnJAwAAGBElDwAAYESUPAAAgBFZM9fkXXH9TTn6nPcNHQMAABipT538hKEjzISZPAAAgBFR8gAAAEZEyQMAABgRJQ8AAGBElDwAAIARWdaSV1VnVNXWqrpsm233rKoPVNXnJj8PXM4MAAAA68lyz+SdmeTE7ba9OMkHu/v+ST44eQwAAMAMLGvJ6+4Lk1y33eYnJzlrcv+sJL+wnBkAAADWkyGuybt3d187uf/vSe49QAYAAIBRGnThle7uJL2z56tqU1XNV9X87TfduILJAAAA1qYhSt43qurQJJn83LqzHbt7c3fPdffcxnvsv2IBAQAA1qohSt55SZ41uf+sJP84QAYAAIBRWu6vUDg7yceSHFVVV1fVaUleneTxVfW5JI+bPAYAAGAGNi7ni3f3qTt56oTlHBcAAGC9GnThFQAAAGZLyQMAABgRJQ8AAGBElDwAAIARUfIAAABGZFlX15ylBx14j8yf/IShYwAAAKxqZvIAAABGpLp76AyLUlU3J7lq6BwM4uAk3xo6BINw7Nc3x3/9cuzXL8d+fXP8d+2+3X3IrnZaM6drJrmqu+eGDsHKq6p5x359cuzXN8d//XLs1y/Hfn1z/GfH6ZoAAAAjouQBAACMyFoqeZuHDsBgHPv1y7Ff3xz/9cuxX78c+/XN8Z+RNbPwCgAAALu2lmbyAAAA2IVVX/Kq6sSquqqqPl9VLx46D8uvqr5cVZ+pqkuran6y7Z5V9YGq+tzk54FD52T3VdUZVbW1qi7bZtsOj3Ut+B+T3wWfrqpjhkvO7trJsX9FVV0z+exfWlVP2ua5l0yO/VVV9YRhUjMLVXWfqjq/qq6oqsur6vTJdp/9deAujr/P/8hV1d5V9Ymq+tTk2L9ysv1+VXXx5Bj/fVXtNdn+Q5PHn588f+SQ+deaVV3yqmpDkj9P8sQkD0pyalU9aNhUrJDHdvdDtllG98VJPtjd90/ywclj1r4zk5y43badHesnJrn/5LYpyRtWKCPL48z84LFPkj+efPYf0t3vSZLJ7/1TkvzE5M/8xeTvB9am25O8qLsflOTYJM+bHGOf/fVhZ8c/8fkfu+8mOb67j07ykCQnVtWxSf4wC8f+x5Jcn+S0yf6nJbl+sv2PJ/uxSKu65CV5WJLPd/cXu/t7Sd6a5MkDZ2IYT05y1uT+WUl+YcAszEh3X5jkuu027+xYPznJm3vBx5McUFWHrkxSZm0nx35nnpzkrd393e7+UpLPZ+HvB9ag7r62uy+Z3L85yZVJDovP/rpwF8d/Z3z+R2LyGf7O5OGek1snOT7JOZPt23/27/ydcE6SE6qqVijumrfaS95hSb62zeOrc9e/CBiHTvL+qtpSVZsm2+7d3ddO7v97knsPE40VsLNj7ffB+vD8ySl5Z2xzWrZjP1KT068emuTi+OyvO9sd/8Tnf/SqakNVXZpka5IPJPlCkhu6+/bJLtse3/917CfP35jkoJVNvHat9pLH+vTI7j4mC6foPK+qHr3tk72wJKxlYdcBx3rdeUOSH83CaTzXJnntsHFYTlV19yTvSPLC7r5p2+d89sdvB8ff538d6O47uvshSQ7PwozsAwaONFqrveRdk+Q+2zw+fLKNEevuayY/tyY5Nwu/BL5x5+k5k59bh0vIMtvZsfb7YOS6+xuT/wH4fpK/yn+ekuXYj0xV7ZmF/8F/S3e/c7LZZ3+d2NHx9/lfX7r7hiTnJ3l4Fk7B3jh5atvj+7+O/eT5/ZN8e4WjrlmrveT9a5L7T1bd2SsLF96eN3AmllFV7VtV+915P8nPJrksC8f9WZPdnpXkH4dJyArY2bE+L8kzJyvtHZvkxm1O7WIEtrvO6hez8NlPFo79KZOV1u6XhQU4PrHS+ZiNyTU1b0xyZXe/bpunfPbXgZ0df5//8auqQ6rqgMn9uyV5fBauyTw/ycmT3bb/7N/5O+HkJB9qX/C9aBt3vctwuvv2qnp+kvcl2ZDkjO6+fOBYLK97Jzl3cl3txiR/193vrap/TfK2qjotyVeSPG3AjMxIVZ2d5GeSHFxVVyd5eZJXZ8fH+j1JnpSFi+5vTfKcFQ/MzOzk2P9MVT0kC6fpfTnJryVJd19eVW9LckUWVuZ7XnffMURuZuK4JM9I8pnJtTlJ8rvx2V8vdnb8T/X5H71Dk5w1WR11jyRv6+53V9UVSd5aVb+f5JNZ+EeATH7+TVV9PgsLdZ0yROi1qhRiAACA8Vjtp2sCAAAwBSUPAABgRJQ8AACAEVHyAAAARkTJAwAAGBElDwC2UVVHVtVlO9j+11X1oCEyAcA0VvX35AHAatHd//fQGQBgMczkAcAP2lhVb6mqK6vqnKrap6o+XFVzSVJVJ1bVJVX1qar64NBhAWBbZvIA4AcdleS07r6oqs5I8ut3PlFVhyT5qySP7u4vVdU9hwoJADtiJg8AftDXuvuiyf2/TfLIbZ47NsmF3f2lJOnu61Y6HADcFSUPAH5Q7+IxAKxaSh4A/KAjqurhk/v/V/L/t3fvwZaV5ZnAn9duCILcBKQIl2BmHLwlIjlloXhFjegkEB01UvESY9mpCijOWDOlcUqNVVY5ldEk40ysdCKCEwejCBEzjmK8wIgRPU1AaVrGu4JoiyAgVBTwnT/O7qRtu+mzT+9z1jmrf7+qU2fvtfbZ6/njYzVPfWt9K5/ebt9nkzyxqh6cJC7XBGC1UfIA4Oddn+SsqtqS5NAk79i2o7u/n2RDkouq6pokfzNMRADYuep2BQoAAMBYmMkDAAAYESUPAABgRJQ8AACAEVHyAAAARkTJAwAAGBElDwAAYESUPAAAgBFR8gAAAEZEyQMAABgRJQ8AAGBElDwAAIARUfIAAABGRMkDAAAYESUPAABgRJQ8AACAEVHyAAAARkTJAwAAGBElDwAAYETWDx1gsQ4//PA+/vjjh44BAAAwiE2bNt3c3Ufs7nNrpuQdf/zxmZ+fHzoGAADAIKrqm4v53JopeVtu+EF+7T++e+gYAADACtv0xy8eOsKa4p48AACAEVHyAAAARkTJAwAAGBElDwAAYESUPAAAgBEZpORV1blVtbWqrh3i+AAAAGM11EzeeUlOG+jYAAAAozVIyevuy5PcMsSxAQAAxsw9eQAAACOyqkteVW2oqvmqmr/nrjuGjgMAALDqreqS190bu3uuu+fW73/g0HEAAABWvVVd8gAAAJjOUI9QuCDJPyQ5oapuqKqXDZEDAABgbNYPcdDuPnOI4wIAAIydyzUBAABGRMkDAAAYESUPAABgRJQ8AACAEVHyAAAARmSQ1TWX4mHHHJb5P37x0DEAAABWNTN5AAAAI6LkAQAAjIiSBwAAMCJr5p68n9y0Od96068MHQMAAFhhx73+i0NHWFPM5AEAAIyIkgcAADAiSh4AAMCIKHkAAAAjouQBAACMyCAlr6qOrapPVtV1VbW5qs4ZIgcAAMDYDPUIhXuSvLq7r6qqA5NsqqqPdfd1A+UBAAAYhUFm8rr7pu6+avL6jiRbkhw9RBYAAIAxGfyevKo6Psmjk1w5bBIAAIC1b9CSV1UPSPKBJK/q7tt3sn9DVc1X1fwtd9678gEBAADWmMFKXlXtk4WC957uvmhnn+nujd09191zDzxg3coGBAAAWIOGWl2zkrwzyZbuftsQGQAAAMZoqJm8U5K8KMmpVXX15OdZA2UBAAAYjUEeodDdn05SQxwbAABgzAZfXRMAAIDZUfIAAABGRMkDAAAYESUPAABgRJQ8AACAERlkdc2l2PeoR+S4188PHQMAAGBVM5MHAAAwIkoeAADAiCh5AAAAI7Jm7sn70tYv5ZS3nzJ0DAAAYCBXvOKKoSOsCWbyAAAARkTJAwAAGBElDwAAYESUPAAAgBFR8gAAAEZkkJJXVftV1eeq6pqq2lxVfzREDgAAgLEZ6hEKP05yanf/qKr2SfLpqvo/3f3ZgfIAAACMwiAlr7s7yY8mb/eZ/PQQWQAAAMZksHvyqmpdVV2dZGuSj3X3lUNlAQAAGIvBSl5339vdJyY5JsljquqRO36mqjZU1XxVzd/9o7tXPiQAAMAaM/jqmt39wySfTHLaTvZt7O657p7b5wH7rHw4AACANWao1TWPqKpDJq/vn+TpSb40RBYAAIAxGWp1zaOSnF9V67JQNN/X3X83UBYAAIDRGGp1zS8kefQQxwYAABizwe/JAwAAYHaUPAAAgBFR8gAAAEZEyQMAABgRJQ8AAGBEhnqEwtQe+qCH5opXXDF0DAAAgFXNTB4AAMCIKHkAAAAjouQBAACMyJq5J++O66/PZU980tAxAACAgT3p8suGjrCqmckDAAAYESUPAABgRJQ8AACAEVHyAAAARkTJAwAAGJFBS15Vrauqf6yqvxsyBwAAwFgMPZN3TpItA2cAAAAYjcFKXlUdk+TfJvmroTIAAACMzZAzeX+a5D8l+emAGQAAAEZlkJJXVb+RZGt3b9rN5zZU1XxVzd92990rlA4AAGDtGmom75Qkp1fVN5K8N8mpVfXXO36ouzd291x3zx28zz4rnREAAGDNGaTkdfdru/uY7j4+yQuSfKK7XzhEFgAAgDEZenVNAAAAZmj90AG6+1NJPjVwDAAAgFEwkwcAADAiSh4AAMCILLrkVdUBVXW/yet/U1WnV5UlLwEAAFaRaWbyLk+yX1UdneTSJC9Kct5yhAIAAGBppil51d13JXlOkj/v7uclecTyxAIAAGAppllds6rqsUl+J8nLJtvWzT7Szh14wgl50uWXrdThAAAA1qRpZvLOSfLaJBd39+aq+uUkn1yeWAAAACzFombyqmpdktO7+/Rt27r7a0leuVzBAAAAmN6iZvK6+94kj1/mLAAAAOyhae7J+8equiTJ+5PcuW1jd18081Q7sfWG2/LfX/2hlTgUAACwBpz91t8cOsKqNE3J2y/JD5Kcut22TrIiJQ8AAIDdW3TJ6+6XLmcQAAAA9tyiV9esqmOq6uKq2jr5+UBVHbOc4QAAAJjONI9QeFeSS5L84uTnQ5NtAAAArBLTlLwjuvtd3X3P5Oe8JEcsUy4AAACWYJqFV35QVS9McsHk/ZlZWIhlSarqG0nuSHJvknu6e26p3wUAAMCCaUre7yV5e5I/ycKqmp9J8rt7ePyndPfNe/gdAAAATExT8o7p7tO331BVpyT59mwjAQAAsFTT3JP39kVuW6xOcmlVbaqqDXvwPQAAAEzsdiavqh6b5HFJjqiq/7DdroOSrNuDYz++u2+sqgcl+VhVfam7L9/h2BuSbEiSQw+0xgsAAMDuLGYmb98kD8hCITxwu5/bkzx3qQfu7hsnv7cmuTjJY3bymY3dPdfdcw/Y/+ClHgoAAGCvsduZvO6+LMllVXVed39zFgetqgOS3K+775i8/vUkb5rFdwMAAOzNprkn76+q6pBtb6rq0Kr66BKPe2SST1fVNUk+l+R/d/dHlvhdAAAATEyzuubh3f3DbW+6+9bJ/XRT6+6vJXnUUv4WAACAXZtmJu+nVXXctjdV9UtZWCETAACAVWKambzXZeESy8uSVJInZLLyJQAAAKvDokted3+kqk5KcvJk06u6++bliQUAAMBS7PZyzap66OT3SUmOS/Kdyc9xk20AAACsEouZyXt1kpcneetO9nWSU2eaaBcedMzBOfutv7kShwIAAFizFvOcvJdPfj9l+eMAAACwJ3Zb8qrqOfe1v7svml0cAAAA9sRiLtfcdo3kg5I8LsknJu+fkuQzSZQ8AACAVWIxl2u+NEmq6tIkD+/umybvj0py3rKmAwAAYCrTPCfv2G0Fb+J7WVhtc0Xc9PWv5s0vfO5KHQ4AAFhjXvfXFw4dYVWYpuR9vKo+muSCyfvfTvL3s48EAADAUk3zMPSzq+rZSZ442bSxuy9enlgAAAAsxTQzeUlyVZI7uvvvq2r/qjqwu+9YjmAAAABM736L/WBVvTzJhUn+YrLp6CR/uxyhAAAAWJpFl7wkZyU5JcntSdLdX87CYxUAAABYJaYpeT/u7p9se1NV65P0Ug9cVYdU1YVV9aWq2lJVj13qdwEAALBgmnvyLquqP0xy/6p6epI/SPKhPTj2nyX5SHc/t6r2TbL/HnwXAAAAmW4m7zVJvp/ki0l+P8mHk/znpRy0qg7Owiqd70yS7v5Jd/9wKd8FAADAv1jUTF5VrUvy7u7+nSR/OYPjPjgLhfFdVfWoJJuSnNPdd87guwEAAPZai5rJ6+57k/zS5LLKWVif5KQk7+juRye5MwszhT+jqjZU1XxVzd/5Tz+e0aEBAADGa5p78r6W5IqquiQLpSxJ0t1vW8Jxb0hyQ3dfOXl/YXZS8rp7Y5KNSXL0YYcueZEXAACAvcU0Je+rk5/7JTlwTw7a3d+tqm9X1QndfX2Spya5bk++EwAAgClKXnf/UZJU1UELb/uOPTz2K5K8Z3IJ6NeSvHQPvw8AAGCvt+iSV1VzSd6VySxeVd2W5Pe6e9NSDtzdVyeZW8rfAgAAsHPTXK55bpI/6O7/myRV9fgslL5fXY5gAAAATG+a5+Tdu63gJUl3fzrJPbOPBAAAwFJNM5N3WVX9RZILknSS307yqao6KUm6+6plyAcAAMAUpil5j5r8fsMO2x+dhdJ36kwSAQAAsGTVPZvHz1XVS7r7/Jl82U7Mzc31/Pz8cn09AADAqlZVm7p7t4tXTnNP3u6cM8PvAgAAYAlmWfJqht8FAADAEsyy5M3muk8AAACWzEweAADAiCx6dc2q+oUk/y7J8dv/XXe/afLyipkm28E/3XRHtrz5E8t5CAAAYAQe9rq9e+H/aR6h8MEktyXZlOTHO+7s7rNnFQoAAIClmabkHdPdpy1bEgAAAPbYNPfkfaaqfmXZkgAAALDHppnJe3yS362qr2fhcs1K0t39q8uSDAAAgKlNU/KeuWwpAAAAmIlFl7zu/uasDlpVJyT5m+02/XKS13f3n87qGAAAAHujaWbyZqa7r09yYpJU1bokNya5eIgsAAAAYzLLh6Ev1VOTfHWWM4UAAAB7q9VQ8l6Q5IKhQwAAAIzBoCWvqvZNcnqS9+9i/4aqmq+q+Vvu/OHKhgMAAFiDhp7Je2aSq7r7ezvb2d0bu3uuu+ceeMAhKxwNAABg7Rm65J0Zl2oCAADMzGAlr6oOSPL0JBcNlQEAAGBsBnmEQpJ0951JDhvq+AAAAGM09OWaAAAAzJCSBwAAMCJKHgAAwIgoeQAAACMy2MIr09rvqAPzsNedOnQMAACAVc1MHgAAwIgoeQAAACOi5AEAAIyIkgcAADAia2bhle985zt54xvfOHQMAABgjdnbeoSZPAAAgBFR8gAAAEZEyQMAABgRJQ8AAGBElDwAAIARGazkVdW/r6rNVXVtVV1QVfsNlQUAAGAsBil5VXV0klcmmevuRyZZl+QFQ2QBAAAYkyEv11yf5P5VtT7J/km+M2AWAACAURik5HX3jUn+a5JvJbkpyW3dfekQWQAAAMZkqMs1D01yRpIHJ/nFJAdU1Qt38rkNVTVfVfN33XXXSscEAABYc4a6XPNpSb7e3d/v7ruTXJTkcTt+qLs3dvdcd8/tv//+Kx4SAABgrRmq5H0ryclVtX9VVZKnJtkyUBYAAIDRGOqevCuTXJjkqiRfnOTYOEQWAACAMVk/1IG7+w1J3jDU8QEAAMZoyEcoAAAAMGNKHgAAwIgoeQAAACOi5AEAAIxIdffQGRZlbm6u5+fnh44BAAAwiKra1N1zu/ucmTwAAIARUfIAAABGRMkDAAAYESUPAABgRNYPHWCxbr11S973/scMHQMAAFjjnv+8zw0dYVmZyQMAABgRJQ8AAGBElDwAAIARUfIAAABGRMkDAAAYkcFKXlWdU1XXVtXmqnrVUDkAAADGZJCSV1WPTPLyJI9J8qgkv1FV/3qILAAAAGMy1Ezew5Jc2d13dfc9SS5L8pyBsgAAAIzGUCXv2iRPqKrDqmr/JM9KcuxAWQAAAEZj/RAH7e4tVfVfklya5M4kVye5d8fPVdWGJBuS5PDD913RjAAAAGvRYAuvdPc7u/vXuvuJSW5N8v928pmN3T3X3XMHHTRIHwUAAFhTBmtOVfWg7t5aVUnMI3IAAAcYSURBVMdl4X68k4fKAgAAMBZDTo99oKoOS3J3krO6+4cDZgEAABiFwUpedz9hqGMDAACM1WD35AEAADB7Sh4AAMCIKHkAAAAjouQBAACMyJp5+Nyhhz4sz3/e54aOAQAAsKqZyQMAABgRJQ8AAGBElDwAAIARUfIAAABGZM0svHLdrbfnURd+dOgYAADASF3z3GcMHWEmzOQBAACMiJIHAAAwIkoeAADAiCh5AAAAI6LkAQAAjMiylryqOreqtlbVtdtte2BVfayqvjz5fehyZgAAANibLPdM3nlJTtth22uSfLy7H5Lk45P3AAAAzMCylrzuvjzJLTtsPiPJ+ZPX5yf5reXMAAAAsDcZ4p68I7v7psnr7yY5coAMAAAAozTowivd3Ul6V/urakNVzVfV/D2337aCyQAAANamIUre96rqqCSZ/N66qw9298bunuvuufUHHbxiAQEAANaqIUreJUleMnn9kiQfHCADAADAKC33IxQuSPIPSU6oqhuq6mVJ3pLk6VX15SRPm7wHAABgBtYv55d395m72PXU5TwuAADA3mrQhVcAAACYLSUPAABgRJQ8AACAEVHyAAAARmRZF16ZpYcfelDmn/uMoWMAAACsambyAAAARkTJAwAAGJHq7qEzLEpV3ZHk+qFzsGYcnuTmoUOwZhgvTMN4YRrGC9MyZrgvv9TdR+zuQ2vmnrwk13f33NAhWBuqat54YbGMF6ZhvDAN44VpGTPMgss1AQAARkTJAwAAGJG1VPI2Dh2ANcV4YRrGC9MwXpiG8cK0jBn22JpZeAUAAIDdW0szeQAAAOzGqi95VXVaVV1fVV+pqtcMnYfVp6q+UVVfrKqrq2p+su2BVfWxqvry5PehQ+dkOFV1blVtraprt9u20zFSC/7b5Jzzhao6abjkDGEX4+WNVXXj5DxzdVU9a7t9r52Ml+ur6hnDpGYoVXVsVX2yqq6rqs1Vdc5ku3MMP+c+xotzDDO1qkteVa1L8j+SPDPJw5OcWVUPHzYVq9RTuvvE7ZYcfk2Sj3f3Q5J8fPKevdd5SU7bYduuxsgzkzxk8rMhyTtWKCOrx3n5+fGSJH8yOc+c2N0fTpLJv0kvSPKIyd/8+eTfLvYe9yR5dXc/PMnJSc6ajAvnGHZmV+MlcY5hhlZ1yUvymCRf6e6vdfdPkrw3yRkDZ2JtOCPJ+ZPX5yf5rQGzMLDuvjzJLTts3tUYOSPJu3vBZ5McUlVHrUxSVoNdjJddOSPJe7v7x9399SRfycK/Xewluvum7r5q8vqOJFuSHB3nGHbiPsbLrjjHsCSrveQdneTb272/Iff9HwJ7p05yaVVtqqoNk21HdvdNk9ffTXLkMNFYxXY1Rpx32JWzJ5fXnbvdJeDGC/+sqo5P8ugkV8Y5ht3YYbwkzjHM0GovebAYj+/uk7JwCcxZVfXE7Xf2whKylpFll4wRFuEdSf5VkhOT3JTkrcPGYbWpqgck+UCSV3X37dvvc45hRzsZL84xzNRqL3k3Jjl2u/fHTLbBP+vuGye/tya5OAuXMXxv2+Uvk99bh0vIKrWrMeK8w8/p7u91973d/dMkf5l/uVzKeCFVtU8W/of9Pd190WSzcww7tbPx4hzDrK32kvf5JA+pqgdX1b5ZuPH0koEzsYpU1QFVdeC210l+Pcm1WRgnL5l87CVJPjhMQlaxXY2RS5K8eLIC3slJbtvukiv2UjvcM/XsLJxnkoXx8oKq+oWqenAWFtP43ErnYzhVVUnemWRLd79tu13OMfycXY0X5xhmbf3QAe5Ld99TVWcn+WiSdUnO7e7NA8didTkyycUL58ysT/K/uvsjVfX5JO+rqpcl+WaS5w+YkYFV1QVJnpzk8Kq6IckbkrwlOx8jH07yrCzc3H5XkpeueGAGtYvx8uSqOjELl9x9I8nvJ0l3b66q9yW5Lgur5p3V3fcOkZvBnJLkRUm+WFVXT7b9YZxj2LldjZcznWOYpVq4TBwAAIAxWO2XawIAADAFJQ8AAGBElDwAAIARUfIAAABGRMkDAAAYESUPABahqj5cVYcMnQMAdscjFAAAAEbETB4A7KCq/raqNlXV5qraMNn2jao6fPL6xVX1haq6pqr+57BpAeBnmckDgB1U1QO7+5aqun+Szyd5UpJNSeaSHJnk4iSP6+6bt312wLgA8DPWDx0AAFahV1bVsyevj03ykO32nZrk/d19c5IoeACsNkoeAGynqp6c5GlJHtvdd1XVp5LsN2goAJiCe/IA4GcdnOTWScF7aJKTd9j/iSTPq6rDkoVLO1c6IADcFyUPAH7WR5Ksr6otSd6S5LPb7+zuzUnenOSyqromydtWPiIA7JqFVwAAAEbETB4AAMCIKHkAAAAjouQBAACMiJIHAAAwIkoeAADAiCh5AAAAI6LkAQAAjIiSBwAAMCL/H3PUs4T7jmkzAAAAAElFTkSuQmCC\n",
"text/plain": [
"<Figure size 1080x1080 with 3 Axes>"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"# Plot the error metrics\n",
"_, (ax1, ax2, ax3) = plt.subplots(nrows=3, figsize=(15,15))\n",
"sns.barplot(y='n_predictors', x='adj_R_sq', data=best, ax=ax1, orient='h')\n",
"sns.barplot(y='n_predictors', x='bic', data=best, ax=ax2, orient='h')\n",
"sns.barplot(y='n_predictors', x='aic', data=best, ax=ax3, orient='h')"
]
},
{
"cell_type": "code",
"execution_count": 210,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"<matplotlib.axes._subplots.AxesSubplot at 0x11a255978>"
]
},
"execution_count": 210,
"metadata": {},
"output_type": "execute_result"
},
{
"data": {
"image/png": "iVBORw0KGgoAAAANSUhEUgAAAY4AAAEKCAYAAAAFJbKyAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAG6pJREFUeJzt3X2UXHWd5/H3p7vTITQPaZI2MulAAkYwcpgj6UUe9jggswiIBCeM4ugQnQyRxadZn0BdB4TZHVlnhpER3Q2CBGVBlgyH4KBulofjGTRIRzAkUSDyYDqGpEkagkmTTrq++0f9OlY6Hapup6tuVfJ5ndMnv/u7v6r77UqlPrn3d+teRQRmZmaVasq7ADMzaywODjMzy8TBYWZmmTg4zMwsEweHmZll4uAwM7NMHBxmZpaJg8PMzDJxcJiZWSYteRdQDZMnT47p06fnXYaZWUNZvnz5SxHRUW7cfhkc06dPp7u7O+8yzMwaiqQXKhnnQ1VmZpaJg8PMzDJxcJiZWSYODjMzy8TBYWZmmeyXZ1WNVqEQbNo6QKFQQBI7Bgs0NUGEiAhaW5qZ1NZKU5PyLtXMLDcOjqRQCJ7a8CrXL32Ky898E/0Dg3znkeeYd9oMFv30OebOnkZn+wQGCwUGC0FTEzSryeFiZgccH6pKNm0d4NLbupk7exp9W3fwubtXMHf2NBb9tBgei5evZcdggede2spX7lvFhi3b+U3v7/nKfat4tncb7/tfP+P06x7ivd98hKc2vEqh4Fvymtn+ycGRDOwcpKevn4kTxnFwa/Ou9tzZ07hi8Yo9AqW0fcXiFfT09QPQ09fPpbd1s2nrQM6/kZlZdfhQVdLa0kxn+wRe7t9Ba3PTrvakttZdIQKM2B4KjSE9ff0M7Bys+e9gZlYL3uNIJrW1ctMlXSxevpb2tnF87aITWbx8LUe0te4KkW0DgyO2O9sn7PZcne0TaG1pzuk3MTOrLkXsf8fiu7q6YjTXqhrprKpxLU30btnO1x94erdJ8+ET6EOHqzrbJ3DTJV0cN+VQT5CbWUORtDwiusqOc3CUN/JpuqK5pB2Bz6oys4ZWaXB4jqMCTU2i49DxeZdhZlYXqjbHIekWSRslrRxh3WckhaTJaVmSbpC0RtIKSSeVjJ0n6Zn0M69a9ZqZWWWqOTl+K3DO8E5J04Czgd+WdJ8LzEw/C4BvpbFHAFcBbwdOBq6S1F7Fms3MrIyqBUdE/ATYPMKq64HPA6WTK3OA26JoGTBR0pHAu4ClEbE5IvqApYwQRmZmVjs1PR1X0hxgXUT8ctiqqcDakuWe1Le3fjMzy0nNJsclHQx8keJhqmo8/wKKh7k46qijqrEJMzOjtnscxwIzgF9Keh7oBH4h6Y3AOmBaydjO1Le3/j1ExMKI6IqIro6OsvdaNzOzUapZcETEkxHxhoiYHhHTKR52OikiXgSWAJeks6tOAV6JiPXAj4GzJbWnSfGzU5+ZmeWkmqfj3gH8DDhOUo+k+a8z/H7gWWANcBNwOUBEbAauBR5LP9ekPjMzy4m/OW5mZkDl3xz3RQ7NzCwTB4eZmWXi4DAzs0wcHGZmlomDw8zMMnFwmJlZJg4OMzPLxMFhZmaZODjMzCwTB4eZmWXi4DAzs0wcHGZmlomDw8zMMnFwmJlZJg4OMzPLxMFhZmaZODjMzCwTB4eZmWVSzXuO3yJpo6SVJX1fk/RrSSsk3SNpYsm6L0haI+kpSe8q6T8n9a2RdGW16jUzs8pUc4/jVuCcYX1LgRMi4kTgaeALAJJmARcDb02P+aakZknNwI3AucAs4ANprJmZ5aRqwRERPwE2D+v7vxGxMy0uAzpTew5wZ0Rsj4jngDXAyelnTUQ8GxEDwJ1prJmZ5STPOY6/An6Y2lOBtSXrelLf3vrNzCwnuQSHpC8BO4Hbx/A5F0jqltTd29s7Vk9rZmbD1Dw4JH0YOB/4YERE6l4HTCsZ1pn69ta/h4hYGBFdEdHV0dEx5nWbmVlRTYND0jnA54ELImJbyaolwMWSxkuaAcwEfg48BsyUNENSK8UJ9CW1rNnMzHbXUq0nlnQHcAYwWVIPcBXFs6jGA0slASyLiMsiYpWku4DVFA9hfSwiBtPzfBz4MdAM3BIRq6pVs5mZlac/HC3af3R1dUV3d3feZZiZNRRJyyOiq9w4f3PczMwycXCYmVkmDg4zM8vEwWFmZpk4OMzMLBMHh5mZZeLgMDOzTBwcZmaWiYPDzMwycXCYmVkmDg4zM8vEwWFmZpk4OMzMLBMHh5mZZeLgMDOzTBwcZmaWiYPDzMwycXCYmVkmDg4zM8ukasEh6RZJGyWtLOk7QtJSSc+kP9tTvyTdIGmNpBWSTip5zLw0/hlJ86pVr5mZVaaaexy3AucM67sSeCAiZgIPpGWAc4GZ6WcB8C0oBg1wFfB24GTgqqGwMTOzfFQtOCLiJ8DmYd1zgEWpvQi4sKT/tihaBkyUdCTwLmBpRGyOiD5gKXuGkZmZ1VCt5zimRMT61H4RmJLaU4G1JeN6Ut/e+vcgaYGkbkndvb29Y1u1mZntktvkeEQEEGP4fAsjoisiujo6Osbqac3MbJhaB8eGdAiK9OfG1L8OmFYyrjP17a3fzMxyUuvgWAIMnRk1D7i3pP+SdHbVKcAr6ZDWj4GzJbWnSfGzU5+ZmeWkpVpPLOkO4AxgsqQeimdHfRW4S9J84AXgfWn4/cB5wBpgG/ARgIjYLOla4LE07pqIGD7hbmZmNaTiVMP+paurK7q7u/Muw8ysoUhaHhFd5cb5m+NmZpaJg8PMzDJxcJiZWSYODjMzy8TBYWZmmTg4zMwsEweHmZll4uAwM7NMHBxmZpaJg8PMzDJxcJiZWSYODjMzy8TBYWZmmTg4zMwsEweHmZll4uAwM7NMHBxmZpaJg8PMzDLJJTgk/RdJqyStlHSHpIMkzZD0qKQ1kr4vqTWNHZ+W16T10/Oo2czMimoeHJKmAp8EuiLiBKAZuBi4Drg+It4E9AHz00PmA32p//o0zszMcpLXoaoWYIKkFuBgYD3wTuDutH4RcGFqz0nLpPVnSVINazUzsxKvGxySPlTSPn3Yuo+PZoMRsQ74B+C3FAPjFWA58HJE7EzDeoCpqT0VWJseuzONnzRCrQskdUvq7u3tHU1pZmZWgXJ7HJ8uaf/LsHV/NZoNSmqnuBcxA/gjoA04ZzTPVSoiFkZEV0R0dXR07OvTmZnZXpQLDu2lPdJypf4UeC4ieiNiB/CvwOnAxHToCqATWJfa64BpAGn94cCmUW7bzMz2UbngiL20R1qu1G+BUyQdnOYqzgJWAw8BF6Ux84B7U3tJWiatfzAiRrttMzPbRy1l1h8vaQXFvYtjU5u0fMxoNhgRj0q6G/gFsBN4HFgI/Btwp6S/S303p4fcDHxX0hpgM8UzsMzMLCflguMt1dhoRFwFXDWs+1ng5BHGvgb8eTXqMDOz7F43OCLihdJlSZOAdwC/jYjl1SzMzMzqU7nTcX8g6YTUPhJYSfFsqu9K+psa1GdmZnWm3OT4jIhYmdofAZZGxHuAtzPK03HNzKyxlQuOHSXts4D7ASLiVaBQraLMzKx+lZscXyvpExS/yX0S8CMASROAcVWuzczM6lC5PY75wFuBDwPvj4iXU/8pwHeqWJeZmdWpcmdVbQQuG6H/IYpf2DMzswPM6waHpCWvtz4iLhjbcszMrN6Vm+M4leKVae8AHmX016cyM7P9RLngeCPwn4APAH9B8bIgd0TEqmoXZmZm9el1J8cjYjAifhQR8yhOiK8BHh7tvTjMzKzxldvjQNJ44N0U9zqmAzcA91S3LDMzq1flJsdvA06g+MW/r5R8i9zMzA5Q5fY4PgRsBT4FfLLkVt8CIiIOq2JtZmZWh8p9j6PcFwTNzOwA42AwM7NMHBxmZpaJg8PMzDLJJTgkTZR0t6RfS/qVpFMlHSFpqaRn0p/taawk3SBpjaQVkk7Ko2YzMyvKa4/j68CPIuJ44I+BXwFXAg9ExEzggbQMcC4wM/0sAL5V+3LNzGxIzYND0uEU71t+M0BEDKTLtc8BFqVhi4ALU3sOcFsULQMmptvYmplZDvLY45gB9ALfkfS4pG9LagOmRMT6NOZFYEpqT6V4ocUhPanPzMxykEdwtFC8m+C3IuJtFL9geGXpgIgIILI8qaQFkroldff29o5ZsWZmtrs8gqMH6ImIR9Py3RSDZMPQIaj058a0fh0wreTxnalvNxGxMCK6IqKro6OjasWbmR3oah4cEfEixXuZH5e6zgJWA0uAealvHnBvai8BLklnV50CvFJySMvMzGqs7NVxq+QTwO2SWoFngY9QDLG7JM0HXgDel8beD5xH8ZLu29JYMzPLSS7BERFPAF0jrDprhLEBfKzqRZmZWUX8zXEzM8vEwWFmZpk4OMzMLBMHh5mZZeLgMDOzTBwcZmaWiYPDzMwycXCYmVkmDg4zM8vEwWFmZpk4OMzMLBMHh5mZZeLgMDOzTBwcZmaWiYPDzMwycXCYmVkmDg4zM8vEwWFmZpnkFhySmiU9LukHaXmGpEclrZH0/XQ/ciSNT8tr0vrpedVsZmb57nF8CvhVyfJ1wPUR8SagD5if+ucDfan/+jTOzMxykktwSOoE3g18Oy0LeCdwdxqyCLgwteekZdL6s9J4MzPLQV57HP8MfB4opOVJwMsRsTMt9wBTU3sqsBYgrX8ljTczsxzUPDgknQ9sjIjlY/y8CyR1S+ru7e0dy6c2M7MSeexxnA5cIOl54E6Kh6i+DkyU1JLGdALrUnsdMA0grT8c2DT8SSNiYUR0RURXR0dHdX8DM7MDWM2DIyK+EBGdETEduBh4MCI+CDwEXJSGzQPuTe0laZm0/sGIiBqWbGZmJerpexxXAJ+WtIbiHMbNqf9mYFLq/zRwZU71mZkZ0FJ+SPVExMPAw6n9LHDyCGNeA/68poWZmdle1dMeh5mZNQAHh5mZZeLgMDOzTBwcZmaWiYPDzMwycXCYmVkmDg4zM8vEwWFmZpk4OMzMLBMHh5mZZeLgMDOzTBwcZmaWiYPDzMwycXCYmVkmDg4zM8vEwWFmZpk4OMzMLBMHh5mZZVLz4JA0TdJDklZLWiXpU6n/CElLJT2T/mxP/ZJ0g6Q1klZIOqnWNZuZ2R/kscexE/hMRMwCTgE+JmkWcCXwQETMBB5IywDnAjPTzwLgW7Uv2czMhtQ8OCJifUT8IrVfBX4FTAXmAIvSsEXAhak9B7gtipYBEyUdWeOyzcwsyXWOQ9J04G3Ao8CUiFifVr0ITEntqcDakof1pD4zM8tBbsEh6RBgMfA3EbGldF1EBBAZn2+BpG5J3b29vWNYqZmZlcolOCSNoxgat0fEv6buDUOHoNKfG1P/OmBaycM7U99uImJhRHRFRFdHR0f1ijczO8DlcVaVgJuBX0XEP5WsWgLMS+15wL0l/Zeks6tOAV4pOaRlZmY11pLDNk8H/hJ4UtITqe+LwFeBuyTNB14A3pfW3Q+cB6wBtgEfqW25ZmZWqubBERH/Dmgvq88aYXwAH6tqUWOgUAg2bR1gYOcgrS3NTGprpalpb7+mmVnjymOPY79TKARPbXiVS2/rpuOQ8XzxvLewfccgBYKDxjUzuW28Q8TM9hsOjjGwaevArtC46oJZ9A8McvFNy3aFyMCOgkPEzPYbDo4xMLBzkJ6+fr58/iz6tu7gy/eu3CNEevr66WyfwE2XdHHclEMdHmbWsHyRwzHQ2tJMZ/sEJk4Yx8GtzfT09XPZGcfSt3UHn7t7BT19/QD09PVz6W3dbNo6kHPFZmaj5z2OMTCprZWbLunixVdeA9gVIsCu0BjS09fPwM7BmtdoZjZWHBxjoKlJHDflUKYcNp7NWwf42kUnsm2gGA6d7RN2C4/O9gm0tjTnVaqZ2T7zoaox0tQkjmgbzzGTD+G4Nx7KrCMPZdoRE/jaRSfS2T4BYNccx6S21pyrNTMbPe9xjLGhAAHoKARHtLXy/QWnMBhw0Lgmn1VlZg3PwVFFu0KkLe9KzMzGjg9VmZlZJg4OMzPLxMFhZmaZODjMzCwTB4eZmWXis6pqpFAIXu4foH9gkMHwBQ/N6sHQ7RAKhQKS2DFYoKkJmtVUth0hmkXmx+1Lu9w2I0REVP3WDg6OGigUguc3bWXDltf43N0r6DhkPJ88ayZvnnIIUm3+om3/U3oPmAmtzewsBFEIBoO6+0Crx+2PaxG9Wwb4+gNPc/mZb6J/YJDvPPJcRe15p81g0U8rGztW7XLbnHfaDK5YvKImF1RV8T5J+5eurq7o7u7Ou4xdel/dzsp1r+y6au5n33Uci376hzfC3NnT6GyfwKS2VgYLUZU9kn35n9WB9oHSCNsc+tD76PeWc9oxk/jQqUfzjQefqcsPtHrd/pXnvoVLbvk5Xz5/Fq3NTXz53pUVt6/9wepRPW5f2uW2ee0PVu9xeaN7Lj+djkPHV/w5IWl5RHSVG+c5jhoY2Dm421Vzr1i8grmzp+0Kj8XL17JjsMBzL23l/QuX8ak7nuCFl7bxu5f7Wf/KNjZueY11fdtG3d7wSj+r12/hS/esYP2W1/hN7+/5yn2r2LBle1Xbz/Zu4+olK2u6zby3X6tt/v61QT76veV0HDKey844lstv/wVzZ0/b9d4aujJzLdp5bHMstr956wA9ff27XdW60vZoH7cv7UrWl6rmBVUdHDXQ2tLMtoHBXVfNHfqLHulNP3Qfj52FAtf8YGw+aFavf5XLvre8Yf5BN/L2a7XNoQ+9y844drcPwHr8QKvX7W/aOkBn+wRe7t+x699npe3RPm5f2pWsL1XNC6o2THBIOkfSU5LWSLoy73qymNTWytGTDt511dyhv+hJba17vKFL7+MxVh80jfYPupG3X6vtDH3oDf8ArMcPtHrd/uLla7lu7oksXr6W9rZxfO2iytujfdy+tMtt87q5tbugakMEh6Rm4EbgXGAW8AFJs/KtqnJNTWL6pLZdV839nx+azeLlazmirXWPN3Q1Pmj8gbL/bXPog2LbwOBuHyr1+IFWr9svnWMc19zEjMltXPWetzLlsIM4tuOQ120f09HG1RecwJEVjB2rdrltHtPRxl0fPZVHrjiTey4/vap3Gm2IyXFJpwJXR8S70vIXACLi70caX2+T48MNTVSLYMOW7bud1fHajgLAmE6slU7IN8KkZSNvv1bbHNrOR06fwSHjW/iXB5/ZdZLF4RPGMa5JJZPzornK7djjTK7qb3Msth+Bz2osUenkeKMEx0XAORHx12n5L4G3R8THS8YsABYAHHXUUbNfeOGFXGrNavjZThHBq9t30vvq9jH7oBl+CvC45qa6/wfdyNuv1TaHPvSGTsXdsbPgD0DbJwdccJSq9z2Ockq/LDhWHzS+H4iZlVNpcDTKFwDXAdNKljtT337J9/Ews3rWEJPjwGPATEkzJLUCFwNLcq7JzOyA1BB7HBGxU9LHgR8DzcAtEbEq57LMzA5IDREcABFxP3B/3nWYmR3oGuVQlZmZ1QkHh5mZZdIQp+NmJakX2JcvckwGXhqjcqqtkWqFxqq3kWqFxqrXtVbPvtR7dER0lBu0XwbHvpLUXcm5zPWgkWqFxqq3kWqFxqrXtVZPLer1oSozM8vEwWFmZpk4OEa2MO8CMmikWqGx6m2kWqGx6nWt1VP1ej3HYWZmmXiPw8zMMjmgg6PcXQUlXSbpSUlPSPr3PG8eVekdECXNlRSScjsLpILX9cOSetPr+oSkv86jzpJ6yr62kt4nabWkVZL+d61rLKmj3Gt7fcnr+rSkl/Oos6SecvUeJekhSY9LWiHpvDzqTLWUq/VoSQ+kOh+W1JlHnamWWyRtlLRyL+sl6Yb0u6yQdNKYFhARB+QPxWte/QY4BmgFfgnMGjbmsJL2BcCP6rXWNO5Q4CfAMqCrXmsFPgx8I+/3QIZ6ZwKPA+1p+Q31Wuuw8Z+geF23en5tFwL/ObVnAc/Xca3/B5iX2u8Evpvja/sO4CRg5V7Wnwf8EBBwCvDoWG7/QN7jOBlYExHPRsQAcCcwp3RARGwpWWwD8poQKltrci1wHfBaLYsbptJa60Ul9V4K3BgRfQARsbHGNQ7J+tp+ALijJpWNrJJ6AzgstQ8HflfD+kpVUuss4MHUfmiE9TUTET8BNr/OkDnAbVG0DJgo6cix2v6BHBxTgbUlyz2pbzeSPibpN8D/AD5Zo9qGK1tr2hWdFhH/VsvCRlDR6wrMTbvQd0uaNsL6Wqmk3jcDb5b0iKRlks6pWXW7q/S1RdLRwAz+8EGXh0rqvRr4kKQeihcx/URtSttDJbX+Eviz1H4vcKikSTWobTQqfq+MxoEcHBWJiBsj4ljgCuC/5l3PSCQ1Af8EfCbvWip0HzA9Ik4ElgKLcq6nnBaKh6vOoPi/+JskTcy1ovIuBu6OiMG8CynjA8CtEdFJ8fDKd9P7uR59FvgTSY8Df0LxZnL1/vpWRb3+BdVC1rsK3glcWNWK9q5crYcCJwAPS3qe4jHNJTlNkJd9XSNiU0RsT4vfBmbXqLaRVPI+6AGWRMSOiHgOeJpikNRalvfsxeR7mAoqq3c+cBdARPwMOIjitZZqrZL37e8i4s8i4m3Al1JfricfvI7q3jU1r8mdvH8o/i/yWYq780OTYW8dNmZmSfs9QHe91jps/MPkNzleyet6ZEn7vcCyOn8fnAMsSu3JFA8BTKrHWtO444HnSd/TqvPX9ofAh1P7LRTnOGped4W1TgaaUvu/Adfk/PpOZ++T4+9m98nxn4/ptvP8xfP+obhr/DTFsym+lPquAS5I7a8Dq4AnKE6G7fXDOu9ah43NLTgqfF3/Pr2uv0yv6/F1/j4QxUOBq4EngYvrtda0fDXw1Txf0wyv7SzgkfReeAI4u45rvQh4Jo35NjA+x1rvANYDOyjuEc8HLgMuS+sF3Jh+lyfH+vPA3xw3M7NMDuQ5DjMzGwUHh5mZZeLgMDOzTBwcZmaWiYPDzMwycXCYVUjS78foea6W9NkKxt0q6aKx2KbZWHJwmJlZJg4Os4wkHZLuy/CLdL+WOal/uqRfpz2FpyXdLulP08URn5F0csnT/LGkn6X+S9PjJekb6Z4Q/w94Q8k2/1bSY5JWSlooSbX9rc3+wMFhlt1rwHsj4iTgTOAfSz7I3wT8I8XLfhwP/AXwHyleIO+LJc9xIsV7OpwK/K2kP6J4+ZXjKH6b+hLgtJLx34iI/xARJwATgPOr9LuZldWSdwFmDUjAf5f0DqBA8XLVU9K65yLiSQBJq4AHIiIkPUnx2kJD7o2IfqBf0kMU7wfxDuCOKF7R9neSSi+JfqakzwMHA0dQvGTLfVX7Dc1eh4PDLLsPAh3A7IjYka5IfFBat71kXKFkucDu/96GX+tnr9f+kXQQ8E2K1xtaK+nqku2Z1ZwPVZlldziwMYXGmcDRo3iOOZIOSjcCOgN4jOJtf98vqTndre3MNHYoJF6SdAjFi+2Z5cZ7HGbZ3Q7clw4/dQO/HsVzrKB4ZeDJwLUR8TtJ91Cc91gN/Bb4GRTv+SDpJmAl8CLFkDHLja+Oa2ZmmfhQlZmZZeLgMDOzTBwcZmaWiYPDzMwycXCYmVkmDg4zM8vEwWFmZpk4OMzMLJP/D+w2ulC113WQAAAAAElFTkSuQmCC\n",
"text/plain": [
"<Figure size 432x288 with 1 Axes>"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"# LASSO Regression - Cross-Validation with test MSE\n",
"\n",
"results = pd.DataFrame()\n",
"alpha_range = np.linspace(0.3, 1, num=200)\n",
"for train_idx, test_idx in model_selection.KFold(n_splits=10).split(data):\n",
" train, test = data.iloc[train_idx], data.iloc[test_idx]\n",
" errors = []\n",
" for alpha in alpha_range:\n",
" reg = linear_model.Lasso(alpha=alpha, max_iter=1000000)\n",
" reg.fit(train.drop('y', axis=1), train['y'])\n",
" error = np.sqrt(metrics.mean_squared_error(test['y'], reg.predict(test.drop('y', axis=1))))\n",
" errors.append(error)\n",
" results = pd.concat([results, pd.DataFrame(errors, index=alpha_range).T], axis=0, ignore_index=True)\n",
" \n",
"\n",
"df = pd.DataFrame({'lambda' : alpha_range, 'RMSE' : results.mean()})\n",
"sns.scatterplot(x='lambda', y='RMSE', data=df)"
]
},
{
"cell_type": "code",
"execution_count": 212,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"array([ 0. , -0. , 0. , -0.62718828, 0.36328617,\n",
" 0.60771921, 1.62992381, -0.07840308, 0.07485172, -0.01063306])"
]
},
"execution_count": 212,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"reg = linear_model.Lasso(alpha=0.5, max_iter=1000000)\n",
"reg.fit(data.drop('y', axis=1), data['y']).coef_\n",
"# The 7th order term has the largest coefficient\n",
"# The first, second and third order terms have zero coeffecients"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"9. In this exercise, we will predict the number of applications received using the other variables in the College data set."
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>Private</th>\n",
" <th>Apps</th>\n",
" <th>Accept</th>\n",
" <th>Enroll</th>\n",
" <th>Top10perc</th>\n",
" <th>Top25perc</th>\n",
" <th>FUndergrad</th>\n",
" <th>PUndergrad</th>\n",
" <th>Outstate</th>\n",
" <th>RoomBoard</th>\n",
" <th>Books</th>\n",
" <th>Personal</th>\n",
" <th>PhD</th>\n",
" <th>Terminal</th>\n",
" <th>SFRatio</th>\n",
" <th>percalumni</th>\n",
" <th>Expend</th>\n",
" <th>GradRate</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>1</td>\n",
" <td>1660</td>\n",
" <td>1232</td>\n",
" <td>721</td>\n",
" <td>23</td>\n",
" <td>52</td>\n",
" <td>2885</td>\n",
" <td>537</td>\n",
" <td>7440</td>\n",
" <td>3300</td>\n",
" <td>450</td>\n",
" <td>2200</td>\n",
" <td>70</td>\n",
" <td>78</td>\n",
" <td>18.1</td>\n",
" <td>12</td>\n",
" <td>7041</td>\n",
" <td>60</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>1</td>\n",
" <td>2186</td>\n",
" <td>1924</td>\n",
" <td>512</td>\n",
" <td>16</td>\n",
" <td>29</td>\n",
" <td>2683</td>\n",
" <td>1227</td>\n",
" <td>12280</td>\n",
" <td>6450</td>\n",
" <td>750</td>\n",
" <td>1500</td>\n",
" <td>29</td>\n",
" <td>30</td>\n",
" <td>12.2</td>\n",
" <td>16</td>\n",
" <td>10527</td>\n",
" <td>56</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>1</td>\n",
" <td>1428</td>\n",
" <td>1097</td>\n",
" <td>336</td>\n",
" <td>22</td>\n",
" <td>50</td>\n",
" <td>1036</td>\n",
" <td>99</td>\n",
" <td>11250</td>\n",
" <td>3750</td>\n",
" <td>400</td>\n",
" <td>1165</td>\n",
" <td>53</td>\n",
" <td>66</td>\n",
" <td>12.9</td>\n",
" <td>30</td>\n",
" <td>8735</td>\n",
" <td>54</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>1</td>\n",
" <td>417</td>\n",
" <td>349</td>\n",
" <td>137</td>\n",
" <td>60</td>\n",
" <td>89</td>\n",
" <td>510</td>\n",
" <td>63</td>\n",
" <td>12960</td>\n",
" <td>5450</td>\n",
" <td>450</td>\n",
" <td>875</td>\n",
" <td>92</td>\n",
" <td>97</td>\n",
" <td>7.7</td>\n",
" <td>37</td>\n",
" <td>19016</td>\n",
" <td>59</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>1</td>\n",
" <td>193</td>\n",
" <td>146</td>\n",
" <td>55</td>\n",
" <td>16</td>\n",
" <td>44</td>\n",
" <td>249</td>\n",
" <td>869</td>\n",
" <td>7560</td>\n",
" <td>4120</td>\n",
" <td>800</td>\n",
" <td>1500</td>\n",
" <td>76</td>\n",
" <td>72</td>\n",
" <td>11.9</td>\n",
" <td>2</td>\n",
" <td>10922</td>\n",
" <td>15</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" Private Apps Accept Enroll Top10perc Top25perc FUndergrad \\\n",
"0 1 1660 1232 721 23 52 2885 \n",
"1 1 2186 1924 512 16 29 2683 \n",
"2 1 1428 1097 336 22 50 1036 \n",
"3 1 417 349 137 60 89 510 \n",
"4 1 193 146 55 16 44 249 \n",
"\n",
" PUndergrad Outstate RoomBoard Books Personal PhD Terminal SFRatio \\\n",
"0 537 7440 3300 450 2200 70 78 18.1 \n",
"1 1227 12280 6450 750 1500 29 30 12.2 \n",
"2 99 11250 3750 400 1165 53 66 12.9 \n",
"3 63 12960 5450 450 875 92 97 7.7 \n",
"4 869 7560 4120 800 1500 76 72 11.9 \n",
"\n",
" percalumni Expend GradRate \n",
"0 12 7041 60 \n",
"1 16 10527 56 \n",
"2 30 8735 54 \n",
"3 37 19016 59 \n",
"4 2 10922 15 "
]
},
"execution_count": 2,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"college_dat = pd.read_csv('college.csv')\n",
"college_dat = college_dat.drop(college_dat.columns[0], axis=1)\n",
"college_dat['Private'] = college_dat['Private'].map({'Yes' : 1, 'No' : 0})\n",
"college_dat = college_dat.apply(pd.to_numeric, errors='coerce')\n",
"college_dat.columns = [c.translate({ord(c):None for c in '.'}) for c in college_dat.columns]\n",
"college_dat.head()"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"(a) Split the data set into a training set and a test set."
]
},
{
"cell_type": "code",
"execution_count": 62,
"metadata": {},
"outputs": [],
"source": [
"train, test = model_selection.train_test_split(college_dat, test_size=0.2)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"(b) Fit a linear model using least squares on the training set, and\n",
"report the test error obtained."
]
},
{
"cell_type": "code",
"execution_count": 63,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"RMSE: 1097.8135996785918\n"
]
}
],
"source": [
"f = 'Apps ~ ' + ' + '.join(train.drop('Apps', axis=1).columns)\n",
"model = smf.ols(formula=f, data=train).fit()\n",
"\n",
"test_error_ols = np.sqrt(metrics.mean_squared_error(test['Apps'], model.predict(test)))\n",
"print('RMSE: {}'.format(test_error_ols))"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"(c) Fit a ridge regression model on the training set, with λ chosen\n",
"by cross-validation. Report the test error obtained."
]
},
{
"cell_type": "code",
"execution_count": 64,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"<matplotlib.axes._subplots.AxesSubplot at 0x1196bff98>"
]
},
"execution_count": 64,
"metadata": {},
"output_type": "execute_result"
},
{
"data": {
"image/png": "iVBORw0KGgoAAAANSUhEUgAAAZgAAAEKCAYAAAAvlUMdAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3XuUpHd93/n356lb9/T0jDQzPc1III0IuiAHJKDhEGRyQBdEHFsiG9Y2jhOhozW757BZfGzFyLs+WS8cb3AU7CRHyUlgzSJ8iGxCZBDx8YIYaUPCkVlmkMRIg4MUXWCk0dx0mUt3XZ56vvvH81RPdU/N9Fy6uruqPq9z5tRTT9Vz667pb/2+v+/v9ygiMDMzW27Jap+AmZkNJwcYMzPrCwcYMzPrCwcYMzPrCwcYMzPrCwcYMzPrCwcYMzPrCwcYMzPrCwcYMzPri/Jqn8Bq2rJlS2zfvn21T8PMbKDs2rXrUERMLfW+kQ4w27dvZ+fOnat9GmZmA0XS82fyPqfIzMysLxxgzMysLxxgzMysL/oWYCR9QdIBSU90rdsk6UFJTxWPFxbrN0r6hqTHJT0p6fZi/fslPdb1ry7pQz2OVZP0p5KelvQ9Sdv7dV1mZnZm+tmC+SLwwUXr7gJ2RMTlwI7iOcDHgT0RcQ3wPuCzkqoR8XBEXBsR1wLXA7PAt3oc6w7glYh4E/CHwO8v98WYmdnZ6VuAiYjvAC8vWn0rcG+xfC/QaY0EMClJwPpiu3TRth8G/iIiZnscrnu/XwVuKPZlZmarZKXLlKcjYl+x/BIwXSzfAzwAvAhMAr8UEdmibX8Z+INT7Pdi4KcAEZFKeg3YDBxaxnM3MxtorVabl2ebtLKgnQXjlRJb1tdIkv58H1+1cTAREZI692u+GXiMPA3214AHJf3niDgCIGkb8Bbgm+d7XEkfAz4GcMkll5zv7szMBkK9nvLTI3McOtrgH331h+x9ZY7XXzjO5//+DFe+brIvQWalq8j2F8GiEzQOFOtvB+6P3NPAs8BVXdv9IvBnEdE6xX5fAN5Q7LcMbAQO93pjRHwuImYiYmZqasmBqGZmA69eTzk812Tvy3PzwQVg7ytz/Nof7+Tw8WZfjrvSAeYB4LZi+Tbg68XyT4AbACRNA1cCz3Rt9xHgvjPc74eBhyIiTvN+M7OR0AkurSxYVy3NB5eOva/M0UzbfTl2P8uU7wMeAa6UtFfSHcBngJskPQXcWDwH+DTwHkm7yavLPhkRh4r9bCdvnfynRfv/lKRbiqd/BGyW9DTwG5yoTjMzG1md4JJmQSIx22zz+gvHF7zn9ReOUy2X+nJ8jfIX/ZmZmfBcZGY2jLpbLuVEfPvJfVx3xdZl6YORtCsiZpZ630hPdmlmNoy6g0siUW+1ecdlW/jujw9w81su4k8+9m6yLKiWE7ZOjvWtisxTxZiZDZHFweWhPfsIYH0t4Yart9FMMyKgUlZfgws4wJiZDY3FfS4P7dk333IZq5SRQEC1lDA9Od7X4AJOkZmZDYVefS7vuGwLu549xPVFy6WciK0TNcbGVuZPvwOMmdkAy7Jgttnktbn2fFrs0ecPLwguEXnQ2TxeXbHgAk6RmZkNrDTN2H90jtfm2gvSYtunNvDcwSPc+DOrF1zAAcbMbCBlWfDS0TppG1pZUEpO9LnsevYQ11yymXa2esEFnCIzMxs4WRYcOFonbQeINZUW6+YWjJnZAOmkxRppRikRpTWWFuvmAGNmNiC602KlRARBmrXXVFqsmwOMmdkA6E6LtSPvc5lrprTawXglWdBy2bJu9YMLOMCYma15vdJi39z9IkmS0EqDWrlEu+jo37KuSq22+sEFHGDMzNa0NM1OmRb77o8PMDleIeu0XMbXTnABBxgzszXrRHAZnLRYNwcYM7M1KE+L5cFlkNJi3RxgzMzWmPmWSxFABikt1m1tnpWZ2YhqtdrsP9YgbQflksgimGumSJpPi3VKkddqy6XDLRgzszWiO7iUEs33uUg6KS02NbG2gws4wJiZrQlpmnGgK7gEQb2Vzve5bCjSYqVETK2rUq2u7eACTpGZma26Tsul3dXn0mm5JBIb11UGJi3WzS0YM7NVtDgt1h1cBjEt1s0BxsxslSxOi5USyCLmg0t3WmzrxGCkxboN1tmamQ2JXmmxI3MptUqJybEy7Yz5tNjUAAYXcAvGzGzFNZvpKdNix+tt0jYL0mKDGFzAAcbMbEW1Wm0OHG8ObVqs2+CeuZnZgGm12ieVIg9bWqybWzBmZitgPi3WoxR5mNJi3RxgzMz6bHFa7Mhca2jTYt36FmAkfUHSAUlPdK3bJOlBSU8VjxcW6zdK+oakxyU9Ken2rm0ukfQtST+StEfS9h7HukTSw5IelfRDST/Xr+syMzsbi9NipQQqZdFotZkcK7Nx3YmJK6fX14YmuEB/WzBfBD64aN1dwI6IuBzYUTwH+DiwJyKuAd4HfFZStXjtS8DdEfFm4F3AgR7H+h3gKxHxNuCXgX+9jNdhZnZOeqXFjsy1qJYTxqtl0jbzr21dX6NSKa32KS+rvgWYiPgO8PKi1bcC9xbL9wIf6rwdmJQkYH2xXSrpaqAcEQ8W+zwWEbO9DgdsKJY3Ai8u24WYmZ2DZjM9ZVrseL2N0HzLZetEdeiCC6x8Fdl0ROwrll8Cpovle4AHyAPDJPBLEZFJugJ4VdL9wGXAt4G7IqK9aL+/C3xL0j8EJoAb+3sZZmantji4dKfFNoxXFlSLDWPLpWPVOvkjIshbHgA3A48BFwHXAvdI2kAeAN8L3Am8E3gj8NEeu/sI8MWIeD3wc8AfS+p5bZI+JmmnpJ0HDx5cxisyM8uDy8Gu4DJqabFuKx1g9kvaBlA8dvpTbgfuj9zTwLPAVcBe4LGIeCYiUuBrwNt77PcO4CsAEfEIMAZs6XUCEfG5iJiJiJmpqallvDQzG3XzLZdsdNNi3VY6wDwA3FYs3wZ8vVj+CXADgKRp4ErgGeD7wAWSOpHgemBPj/12b/9m8gDj5omZrZhepcijUi12Kv0sU74PeAS4UtJeSXcAnwFukvQUeT/JZ4q3fxp4j6Td5NVln4yIQ0Vfy53AjuI1AZ8v9v8pSbcU2/8m8GuSHgfuAz5apODMzPruVKXIEXFSWmx6cmzoWy4dGuW/wzMzM7Fz587VPg0zG2CdtFg7CxLlLZeJsTyAJBJCtLtaLsMQXCTtioiZpd7nkfxmZueoVylyJy1WKyckyvtcKkMUXM7G8CcBzcz64FSlyN1psfmWy+QY5fLofZ93gDEzO0u9S5HzWZF7pcVGMbiAU2RmZmfFpchnzi0YM7Mz1J0WK5fEa7OtkRyhf6YcYMzMzsDitJj7XJbmAGNmtoTuUuROWmxirES1qBRzn0tv/imYmZ2GS5HPnVswZman4FLk8+MAY2bWw+I+F6fFzp5/ImZmi/QqRXZa7Ow5wJiZdTlVn8uoT1x5LhxgzMwKS5Uid6fFXuc+lyW5D8bMDJci94N/QmY28lyK3B9uwZjZSHMpcv84wJjZyHIpcn/5p2VmI8mlyP3nAGNmI8elyCvDAcbMRopLkVeO+2DMbGS4FHll+adnZiPBpcgrzy0YMxt6LkVeHQ4wZjbUXIq8evyTNLOh5VLk1eUWjJkNpUYj5eBs3nIpl8RrsyeXIjst1l/+iZrZ0OkOLi5FXj1uwZjZUGk2Uw7Nus9lLejbT1bSFyQdkPRE17pNkh6U9FTxeGGxfqOkb0h6XNKTkm7v2uYSSd+S9CNJeyRtP8XxfrF4/UlJ/65f12Vma5f7XNaWfobuLwIfXLTuLmBHRFwO7CieA3wc2BMR1wDvAz4rqVq89iXg7oh4M/Au4MDiA0m6HPht4LqI+Bng15f3UsxsrfP0L2tP3wJMRHwHeHnR6luBe4vle4EPdd4OTEoSsL7YLpV0NVCOiAeLfR6LiNkeh/s14F9FxCvF+04KQmY2vBaXItdbbaqVxH0uq2ylf8rTEbGvWH4JmC6W7wHeDLwI7AY+EREZcAXwqqT7JT0q6W5Jvb52XAFcIem7kv5S0uKWk5kNqV5psSyCcgIbxqsL0mIOLitr1X7SERHkLReAm4HHgIuAa4F7JG0gL0J4L3An8E7gjcBHe+yuDFxOnl77CPB5SRf0Oq6kj0naKWnnwYMHl+16zGzlNRqnTouVkoS0HQvSYg4uK2ulf9r7JW0DKB47qazbgfsj9zTwLHAVsBd4LCKeiYgU+Brw9h773Qs8EBGtiHgW+DF5wDlJRHwuImYiYmZqampZL87MVo5Lkde+lf6JPwDcVizfBny9WP4JcAOApGngSuAZ4PvABZI6keB6YE+P/X6NvPWCpC3kKbNnlv/0zWwt6FWK3EgzquXkpODiUuTV088y5fuAR4ArJe2VdAfwGeAmSU8BNxbPAT4NvEfSbvLqsk9GxKGIaJOnx3YUrwn4fLH/T0m6pdj+m8BhSXuAh4F/FBGH+3VtZrZ6XIo8OJR3hYymmZmZ2Llz52qfhpmdoe5S5F7TvzgttjIk7YqImaXe55++mQ0ElyIPHv8GzGzNcynyYPJvwczWNJciDy7/JsxszXIp8mDzbMpmtiZ5VuTB59+Ima0582kxlyIPNAcYM1tTFqfFPCvy4HKAMbM1Y3FazKXIg82/HTNbE1yKPHxO+xuSdH3X8mWLXvvv+nVSZjZaXIo8nJb6Lf2zruX/sOi131nmczGzEbS4z8VpseGx1G9Kp1ju9dzM7Kz0KkV2Wmx4LDUOJk6x3Ou5mdkZ67Rc2lmvtFiJtB0nxrk4uAykpQLMGyU9QN5a6SxTPL/s1JuZmZ1ad1rMsyIPr6UCzK1dy/9s0WuLn5uZLelUpchZljm4DJnTBpiI+E/dzyVVgL8OvBARB3pvZWbW26nSYpWSGKtVaWdBOytG6Du4DLylypT/jaSfKZY3Ao8DXwIelfSRFTg/MxsS9fqpR+i7FHk4LfUbfG9EPFks3w78OCLeArwD+K2+npmZDY1GI+XwnEuRR81Sv8Vm1/JNwNcAIuKlvp2RmQ2VRiPvc2l5hP7IWaqT/1VJPw+8AFwH3AEgqQyM9/nczGzA1esph+byPpdELkUeNUv9Nv9H4H8G/m/g17taLjcAf97PEzOzwbY4LfbQnn1Oi42YparIfgx8sMf6bwLf7NdJmdlgW5wWq7favOOyLXz3xwe4+S0X0c6CLAvGyglbJ8dIEk8MMoxOG2Ak/cvTvR4R/8vyno6ZDbpTpcXW1xJuuHobzTQjkaiU5eAy5Jbqg/mfgCeArwAv4vnHzOw06vUTabFySXz7yX1cd8VWsiyjlOQ3BgtAgqkJB5dht1SA2Qb898AvASnwp8BXI+LVfp+YmQ2WTnBxWsw6TturFhGHI+LfRMT7ycfBXADskfT3V+TszGwgdAeXTlosi1iQFovAabERc0ZlG5LeDnwC+FXgL4Bd/TwpMxsMWRYcqzfytFgRXLqrxUpJ/ifGabHRtFQn/6eAvw38CPgT4LcjIl2JEzOztS3Lgv1H58gyaGV5ufG3n9zXMy02Xi4xNVlzcBkxS7Vgfoc8LXYN8E+AH0j6oaTdkn54ug0lfUHSAUlPdK3bJOlBSU8VjxcW6zdK+oakxyU9Ken2rm0ukfQtST+StEfS9tMc8+9KCkkzS165mZ2Xg0cbpG3m02KdPpddzx7i+iItJmDTRJnpjW65jKKlOvnP554vXwTuIZ8cs+MuYEdEfEbSXcXzTwIfB/ZExC9ImgL+q6QvR0Sz2P73IuJBSeuBrNfBJE2Sp/G+dx7nbGZLyLJgttmknrYJmE+LXXfF1vk+lyzyoJMksK5aXe1TtlWyVCf/873+AT8FfnaJbb8DvLxo9a3AvcXyvcCHOm8HJiUJWF9sl0q6GihHxIPFPo9FxOwpDvlp4PeB+unOy8zOXSct9tpcm1IiSkVw6aTFxiplpHw8Q7WUMD057pbLCFtquv4Nkn5b0j2SPqDcPwSeAX7xHI43HRH7iuWXgOli+R7gzeRjbXYDn4iIDLiCfD60+yU9KuluSaUe5/l24A0RseT0NZI+JmmnpJ0HDx48h0swG13dabFSItKsd1ps60TNaTFbsg/mj4Eryf/o/w/Aw8CHgQ9FxK2n23ApERHkLReAm4HHgIuAa4F7JG0gT+G9F7gTeCfwRuCj3fuRlAB/APzmGR73cxExExEzU1NT53MJZiOjUy1WT9u0i/TXN3e/SJrBeCXhxp/ZRhTzim0erzI2tlT23UbBUgHmjRHx0Yj4t8BHgKuBmyPisXM83n5J2wCKx85dMW8H7o/c08CzwFXAXuCxiHimqF77GvD2RfucJL/L5v8r6Tng3cAD7ug3Wx5pmp02LVYtl2hnDi52sqUCTKuzEBFtYG9EnE8fxwPAbcXybcDXi+WfkM/QjKRp8lbTM8D3gQuKjn+A64E93TuMiNciYktEbI+I7cBfArdExM7zOE8zIw8uLx2tnzYt5paLncpSAeYaSUeKf0eBt3aWJR053YaS7gMeAa6UtFfSHcBngJskPQXcWDyHvIP+PZJ2AzuAT0bEoSKo3QnsKF4T8Pli/5+SdMu5XbaZLSVvudTn79nitJidLeVdIaNpZmYmdu50Q8dssU7LpTMjMsCOIi3mlotJ2hURS3ZD+A4/ZrbAibRYnhILwmkxOycOMGY2rzstVkpEKRFzzZRWO05Ki21Z5+Bip+cAY2YAtFrtvOWSnWi51FspSZLQSoNaUS1WKoJLrebgYqfnT4iZ0WymHDh+4kZhWQRzzRRJJBIb11XmS5EdXOxMuQVjNuK6g0t3WkzSSS2XqQkHFztzDjBmI6zZTDnYFVwWp8U2jFfIoggu66pUqw4udub8aTEbUY1GysHZ5nzrJHBazJaXWzBmI6gTXDotl84tjp0Ws+XkT43ZiGk0Ug7Ndve5QKUsGq02G8YrtDPmWy5b19eoVE6awNzsjDjAmI2Qej3l0NyJtNiRuRYTYyWq5YREIm1DOxxcbHk4RWY2Iur1lMNzC9NinZZLrQgwWQSVREw7uNgycAvGbAR0gktnRuR6q021kpBlGePV8oKWy/TkGOWyv3va+XOAMRtiWRbMNpu8NtemVUxc2Wm5VEpirFalnQXtrGi5OLjYMvInyWxIZVnM3ygsLYLLQ3v2Ua0kRASlJCFtx/zUMA4uttz8aTIbQlkWHFh0o7Duu1BuGK+SSEQE4+USF20cd3CxZecUmdmQSdOMg8frNNMggETi0ecPL5huv5lmeaXYRM0zIlvf+CuL2RCZnxG5TT7GpUiLbZ/awHMHj/gulLaiHGDMhkSr1Wb/scb8LY5LiRbcKOyaSzbPD6B0cLGV4ABjNgRarTYHiuDSabl8c/eLpBkn3SjMwcVWigOM2YBrNNK85ZKdfIvj7/74ANViXjEHF1tp/qSZDbDuSSsX3yis03LxjMi2WtyCMRtQvSat9IzItpb4E2c2YDqj81+day+4l8uRuZRapcTkWHnBjMhTE75RmK0Ot2DMBsiC0fk97uVyvN7O5xXrark4uNhq8SfPbEB0j87vlCF3Jq30vVxsLXKAMRsArVabQ7ONBaPzTzVppWdEtrXCn0CzNe7EAMqFo/M9aaWtdf4Umq1h3QMoF4/O96SVttb17ZMo6QuSDkh6omvdJkkPSnqqeLywWL9R0jckPS7pSUm3d21ziaRvSfqRpD2Stvc41m8Ur/1Q0g5Jl/brurIsePl4gxdemeUnLx/nwNE6WRb9OpyNsMUDKHuNzm+mGQK2TtSY3jhGkmi1T9tsXj+/6nwR+OCidXcBOyLicmBH8Rzg48CeiLgGeB/wWUnV4rUvAXdHxJuBdwEHehzrUWAmIt4KfBX4p8t4HfOyLHju8HEOHq0z18pI23lp6AuvzjrI2LKq108MoPTofBtUfQswEfEd4OVFq28F7i2W7wU+1Hk7MClJwPpiu1TS1UA5Ih4s9nksImZ7HOvhrvV/Cbx+WS+mcPh4k7lmm0Ya/NP/50c8deAYr842yQKONpr9OKSNoM7tjRcPoGy146R5xbasc3CxtWulk7XTEbGvWH4JmC6W7wHeDLwI7AY+EREZcAXwqqT7JT0q6W5JS9Vd3gH8xalelPQxSTsl7Tx48OBZnXwzzUtB73noKX7jA1fw16bWs2miSqsdvDaXuhVj5yXLgmP1BofnmvM3CcsHULYAmBwrM1YpLRhA6dH5tpatWm9gRAR5ywXgZuAx4CLgWuAeSRvIy6jfC9wJvBN4I/DRU+1T0q8CM8Ddpznu5yJiJiJmpqamzuqcO2mJj7//TSQSrxxv0p3xdivGzlWr1T7p9sYeQGmDbqUDzH5J2wCKx05/yu3A/ZF7GngWuArYCzwWEc9ERAp8DXh7rx1LuhH434BbIqLRj5PfPFGlXBJbN4wx12xTLYvnDs9y+FiD/3bgOAePNt2KsbPWbKbzZcjdtzfuDKCcHCuzcV2FrEiLTa+vObjYQFjpAPMAcFuxfBvw9WL5J8ANAJKmgSuBZ4DvAxdI6jQ1rgf2LN6ppLcB/5Y8uPQqAlgWSfGfu50FzTQPJOtrZbZO1njDpnWUk4Qjdbdi7Mw1GikHjzfny5AT5aPzu8uQ21mePquVE7ZtHPfofBsY/SxTvg94BLhS0l5JdwCfAW6S9BRwY/Ec4NPAeyTtJq8u+2REHIqINnl6bEfxmoDPF/v/lKRbiu3vJi8O+PeSHpP0QL+uq1LkwLdtrAGwYbxEOyARpFlwpO6+GDsz85ViXWXID+3ZRwDrawk3XJ2XIUdApSy2TroM2QaL8q6Q0TQzMxM7d+486+0OHKnTSDOOzLVoR/Dnj7/Ah2cuoZSIajlhw3iJDWO1PpyxDYtOpVir6z4u7SzjWCNj17OHuPktF9HOomi5lJiarDm42JohaVdEzCz1Pg/5PQdb1tfIItgwXuHPH3+Bj7z7UvIKa6i3Ml6ZTUnTbJXP0taiXpVivcqQOwMop9d7AKUNLgeYc5AkYrxIld123WUcmUsXjIvB42Ksh1NVijWKLyMuQ7Zh4wBzjrasrxWpDXqOiznaaLsvxuYtVSlWKycnTbXvSjEbdA4w56i7ouzj738TzTT4999/nnYWJMqnUz/WdCvGXClmo8sB5jxUKiUqpYRNEzX+/PEX+Lszb+DV2RYi74t5dbZNq9Ve7dO0VeRKMRtlDjDnaWvR4f8r797OXLO9oGy5mWbs82zLIynLguNFZ/6pJqwcq5SR8tr7ailhenLcwcWGigPMeSqXEyqlhESinCQ0nCobeZ3O/Ffn2q4Us5HmALMMtrps2Qrz93Bp40oxG3kOMMugXE7m/2h0ypa7WzElieNuxQy1zviWzj1cOnefdKWYjTIHmGUy1VW23Onwb6YZlVJ+v/SjjYxmM13t07Q+WDC+pX2iM//R5w/3rBQbK5dcKWYjwQFmmXSXLf/Ku7cTEVTK4qXX6kQE9VbGS8eariobMt3jWzqVYp3O/O1TG3ju4BHf2thGlgPMMuqULScS49Wyq8qG3OLxLaXkxD1cOp35b7t083xKzHeftFHjALPMOh3+Iq8qk29MNnTSNONovX7S+JZ6q73gHi7dnflb19fcmW8jxwFmmXV3+G9aX51vxZRKolZOyDI4Wm+7qmxANZspB47XOTKXnTS+JYBywnx/Sye4TE+Oub/FRpIDTB90Ovzb7Zhvxcw1UxppRlb0x+w7MudU2YDppMTaXfOJdY9vETBeKc9Ps9/pzC+X/d/MRpM/+X2QJOJ1k2O0i7ExrTROGoApD8AcGJ1R+Z2UWGc+sV7jWxpFZ/7rJt2Zb+YA0yflcjI/pX9nAObi0uUjc22XLq9xrVabQ8frvLqoBNnjW8yW5gDTR50p/btLlyfGSgtTZcca7o9Zg7IsOFpvsP9Yg2YaC1JiSRIe32J2Bhxg+qgzNqZcEonEhvEKR+ZS/suP91MuJZRLoiwx23KqbC1ZPHCyvWjKl1Y7mN5Q5SbPJ2Z2Wg4wfVaplNg6US3uuQ4/eC4f3f3Qnn0kEq0sOFrPqNedKlsLGo2UA52Bk52UWI8pX5ppkLoE2ey0HGBWQLV6YkzE9VdvY9ezh3jHZVv48iPP0kozGmnGobkmjYaDzGrp1ZHfKUEGp8TMzoUDzAqZWl+jnIgs8iDzHx/by0fefSm1SjLf6X9w1lPJrIZmM+3Zkd8ZlT/bbLNxvLwgJeYpX8yW5gCzQpIilVIpgsxt111GM82Ya6YE+R+1tB35vFbu9F8RnRH5B443F3XkizRbOCo/kRakxDzli9nSHGBWUKVSYsu6ahFkoJXG/CDM7iDz0tG6g0yfNRoLR+R3d+RnkZFm+Z0mNzolZnbOHGBWWK1WZvN4dX58jIPMyuq0Wg7NLh6Rf6Ij/1g9ZeN4mXEPnDQ7Lw4wq2BsrEwlSU4KMu0sy0uXi7Ez+x1kllW93tVq6RqR36sjv5nmrZpOq8UDJ83OngPMKtk6eWIQ5obxSpHjhyNzTbLiD1/LLZll0Wq181bL3KJWSzEiP8Ad+WZ90LcAI+kLkg5IeqJr3SZJD0p6qni8sFi/UdI3JD0u6UlJt3dtc4mkb0n6kaQ9krb3OFZN0p9KelrS93q9Z60plxO2TlTng8xYpUSzlTldtoxOlB43TvS1FMG7e0T+w3te4oJ1VUpFR34lSZieHHNHvtl56mcL5ovABxetuwvYERGXAzuK5wAfB/ZExDXA+4DPSqoWr30JuDsi3gy8CzjQ41h3AK9ExJuAPwR+fxmvo2+q1fKCION02fLIsuBIvTFfetzuuttkp9XSbMeCEflzzbw8/HXra1x0oWdANlsOfftfFBHfAV5etPpW4N5i+V7gQ523A5OSBKwvtkslXQ2UI+LBYp/HImK2x+G69/tV4IZiX2vefJBJeqfLSglF2WweZDxO5vSazZT9R+fyiUSL0uPuu012t1rWVcu0ihH5nVaLR+SbLZ+V/po2HRH7iuWXgOli+R7gzcCLwG7gExGRAVcAr0q6X9Kjku6W1KtG9GLgpwARkQKvAZv7eB3Lqlots2XdyemyWqXE0XpKvdWeDzIHjjU84r+H7jEtnWleOqXHJYlHnz88XyG2uNWybdKtFrN+WLX/URGRl+7kbgYeAy4CrgXukbQBKAPvBe4E3glAvip7AAAMU0lEQVS8Efjo+RxX0sck7ZS08+DBg+ezq2VVq5WZWlc9qeO/UhJzrYxHnz88H2QOzTY9d1khy4Jj9cbCMS2xsPS4VILtUxv4qxdfZaMrxMxWzEoHmP2StgEUj53+lNuB+yP3NPAscBWwF3gsIp4pWiZfA97eY78vAG8o9lsGNgKHe51ARHwuImYiYmZqamoZL+381WoL02VClJMSzx08wvapDXz7yX1UywkB7D/eYP9r9ZG9K2aaZrw2Wz9lP0t36fG3du/jogtqvP3SzfPjWrZtGHOFmFmfrXSAeQC4rVi+Dfh6sfwT4AYASdPAlcAzwPeBCyR1IsH1wJ4l9vth4KGihTRwFqfL2hG87dLN7Hr2ENddsZUjc02qZVFORD1ts++1uZG6aVl3i+VoI8tnNW4vnJwyzdonlR4fq+fpsOmJGhdfuM6j8c1WQD/LlO8DHgGulLRX0h3AZ4CbJD0F3Fg8B/g08B5Ju8mryz4ZEYciok2eHttRvCbg88X+PyXplmL7PwI2S3oa+A1OVKcNpO50WUman4W52cpYP1Zm/5EmP3j+MLWiNbPv6PC3Znq2WIp+lk6rpd7KA0urHbTSjAtdemy2qjSgX/SXxczMTOzcuXO1T+OUWq02h2YbZFkxOLAojHvsJ4e56qILyLKMCyeq85VQ5URMTVSHqk8hTTNmW02ONTOyoko7bQcIShISvDbb4oKJMrPNjHICE7UKrTRbMDmlWyxmy0fSroiYWep9LptZwyqVEtOT41wwXqKSLEyZLW7NrKvmf0D3HW3w4itzAz9mptlMeW2unqfC6lnPFkunn6V7/rB1lTLNNJuvxvPklGarZ3i+6g6pJBETYzVKpByaaxLBfJVZlsFzB4/w1ks28cpsk0Ri43iFZprxwmtzVJIkn5JmQMpv0zTjaKNJuQRHG3mLJQKyyENJKVnYYkmzvJ9lrFKiVirTTIMsC2rlElMba+7AN1tlDjADYmyszJTgcL1JkuXzlCF426WbySKolESSJLwy22SsUmKims8EvNYDTZYFRxtNIDjezFtd0YSsCKQIyomIgCAQzLdYLpyoMlHMeFxPMyqlhNdtHFuT12k2ihxgBkitVmZrKaGeNnl1LptvzSTKy5mDYF21RDvg5aJFs2V9heONPNCslT6a7qAy28zyRFeWBxVY2GIBKCX5ut4tloxaucQ2t1jM1hwHmAFTLiesL49RJuXlxsLWTKcIQOQtmolamZ++0mDXs4f4hbddjICDx5ukRxsr3vndarU52mxRSmC2mVFKRDMNYlFQgRMtliNzeSqs2YaIPHguaLEkbrGYrWUOMANqbKzM1vLJrRmJ+RZNMw12PXuID7xlG4004/CxFs8dPMJ1l+fDio42W8wdayx7tVWnhZK3riBJ4Fgjo5yI2WbMn2unGgxOBBWAR58/zDsu27wgFZYgGmlGmuWpsIsdWMzWPAeYAdZpzVSU8nK9SWd+z84fbgVcf/U22sX8mM8dPMI7LttMvagwO95oI2DTRCV/3mqRNVsAZBnFH/SgUkrYun5hH06nRQKQ6ER6KxHMtfK0V7mY2qaT/sq6goriRAoMTqTBJLF9agO7nj3MdZdP0UpjPhWWV4U5FWY2KBxghkCtVma6UuJYs0m9VXSUB0j59Ckdb7t083ywKZdEOYFapUQ9zeaDAeQDFTspqfEkDyrHmk2yRr5tpajyAhZsB4v6UrKitVI0TbqDipQHlHaWT+rZnD9mwsUX1Ni0bjNH623KiZj2OBazgeQAMySSRGwYq7G+GhxvNjnWyGdgVnriD347O9HPoQzKSWk+4HSCAUC1JLLIg1MnfdYJIuVEzDXjpCDS0d2XoshnM00kxMKgMlErM9fK78nSztpsmqjMt5rmnAYzGwoOMEMmScTkWI2Jat4PMl6J+dZGkqlnsIETwQBAmYg4kYbqDiJZEXR6bQcL+1KkvCW1Y88+PvCWbXlwKoLKkbmULesrpEWn/dF6m0opn87FQcVsODjADKkkERvHawCMlfL+ku7UVnewgRPBAJYIPnGiRbJ4O8hTYPVWSq1Smm/5dGY0/oW3XczGsdJ8UHltzkHFbJg5wIyASqXEpqIPo5akHGulCzrjYWFfyumCjwTffjJvkSzeDihmM4Z2I097VUti7IIam4oZjT03mNnocIAZMdVqmU3FQMsNXeXEcKLvpZQwP6oeOKkPprtFoq7tOsbKCXPNbL6Tfuv6GpsmHFDMRo0DzAjrTqMtNlHJ5wXrxI3uIPKGC2tsWneiRdJzdoCJ/p23mQ0GBxjrqVxOuLA8dsrXL1i3gidjZgPJPatmZtYXDjBmZtYXDjBmZtYXDjBmZtYXDjBmZtYXiu5h2CNG0kHg+XPcfAtwaBlPZxD4mkeDr3k0nM81XxoRU0u9aaQDzPmQtDMiZlb7PFaSr3k0+JpHw0pcs1NkZmbWFw4wZmbWFw4w5+5zq30Cq8DXPBp8zaOh79fsPhgzM+sLt2DMzKwvHGDOgaQPSvqvkp6WdNdqn08/SPqCpAOSnuhat0nSg5KeKh4vXM1zXE6S3iDpYUl7JD0p6RPF+mG+5jFJ/5+kx4tr/j+K9ZdJ+l7x+f5TSdXVPtflJqkk6VFJ/7F4PtTXLOk5SbslPSZpZ7Gu759tB5izJKkE/CvgbwFXAx+RdPXqnlVffBH44KJ1dwE7IuJyYEfxfFikwG9GxNXAu4GPF7/XYb7mBnB9RFwDXAt8UNK7gd8H/jAi3gS8AtyxiufYL58AftT1fBSu+f0RcW1XaXLfP9sOMGfvXcDTEfFMRDSBPwFuXeVzWnYR8R3g5UWrbwXuLZbvBT60oifVRxGxLyJ+UCwfJf/jczHDfc0REceKp5XiXwDXA18t1g/VNQNIej3wt4H/q3guhvyaT6Hvn20HmLN3MfDTrud7i3WjYDoi9hXLLwHTq3ky/SJpO/A24HsM+TUXqaLHgAPAg8B/A16NiLR4yzB+vv858FtA57atmxn+aw7gW5J2SfpYsa7vn23fcMzOSUSEpKErQZS0HvgPwK9HxJH8y21uGK85ItrAtZIuAP4MuGqVT6mvJP08cCAidkl632qfzwr62Yh4QdJW4EFJf9X9Yr8+227BnL0XgDd0PX99sW4U7Je0DaB4PLDK57OsJFXIg8uXI+L+YvVQX3NHRLwKPAz8DeACSZ0vn8P2+b4OuEXSc+Tp7euBf8FwXzMR8ULxeID8i8S7WIHPtgPM2fs+cHlRdVIFfhl4YJXPaaU8ANxWLN8GfH0Vz2VZFXn4PwJ+FBF/0PXSMF/zVNFyQdI4cBN539PDwIeLtw3VNUfEb0fE6yNiO/n/3Yci4u8xxNcsaULSZGcZ+ADwBCvw2fZAy3Mg6efI87gl4AsR8XurfErLTtJ9wPvIZ1zdD/zvwNeArwCXkM9C/YsRsbgQYCBJ+lngPwO7OZGb/1/J+2GG9ZrfSt65WyL/svmViPiUpDeSf7vfBDwK/GpENFbvTPujSJHdGRE/P8zXXFzbnxVPy8C/i4jfk7SZPn+2HWDMzKwvnCIzM7O+cIAxM7O+cIAxM7O+cIAxM7O+cIAxM7O+cIAxW2aSji39rjPaz+9KuvMM3vdFSR9e6n1mK80BxszM+sIBxqxPJK2XtEPSD4p7cdxarN8u6a+KlsePJX1Z0o2Svlvcm+NdXbu5RtIjxfpfK7aXpHuU35Po28DWrmP+Y0nfl/SEpM+pezI1sxXmAGPWP3Xg70TE24H3A5/t+oP/JuCz5JNLXgX8CvCzwJ3kMwh0vJV8vqy/AfxjSRcBfwe4kvx+RP8AeE/X+++JiHdGxF8HxoGf79O1mS3Jsymb9Y+A/1PS3ySffuZiTkyJ/mxE7AaQ9CT5jZ9C0m5ge9c+vh4Rc8CcpIfJJyn8m8B9xUzIL0p6qOv975f0W8A68mlPngS+0bcrNDsNBxiz/vl7wBTwjohoFTP4jhWvdc9zlXU9z1j4/3LxXE6nnNtJ0hjwr4GZiPippN/tOp7ZinOKzKx/NpLfe6Ql6f3Apeewj1sljRUTE76PfDbv7wC/VNwsbBt5+g1OBJNDxX1tXFlmq8otGLP++TLwjSLttRP4qyXe38sPyaeS3wJ8OiJelPRn5P0ye4CfAI9Afk8XSZ8nn4r9JfJgZLZqPJuymZn1hVNkZmbWFw4wZmbWFw4wZmbWFw4wZmbWFw4wZmbWFw4wZmbWFw4wZmbWFw4wZmbWF/8/ZF88GWGhR1gAAAAASUVORK5CYII=\n",
"text/plain": [
"<Figure size 432x288 with 1 Axes>"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"results = pd.DataFrame()\n",
"alpha_range = np.linspace(10**-3, 50, num=1000)\n",
"for train_idx, test_idx in model_selection.KFold(n_splits=8).split(college_dat):\n",
" train, test = college_dat.iloc[train_idx], college_dat.iloc[test_idx]\n",
" errors = []\n",
" for alpha in alpha_range:\n",
" reg = linear_model.Ridge(alpha=alpha)\n",
" reg.fit(train.drop('Apps', axis=1), train['Apps'])\n",
" error = np.sqrt(metrics.mean_squared_error(test['Apps'], reg.predict(test.drop('Apps', axis=1))))\n",
" errors.append(error)\n",
" results = pd.concat([results, pd.DataFrame(errors, index=alpha_range).T], axis=0, ignore_index=True)\n",
"\n",
"df = pd.DataFrame({'lambda' : alpha_range, 'RMSE' : results.mean()})\n",
"sns.scatterplot(x='lambda', y='RMSE', data=df)"
]
},
{
"cell_type": "code",
"execution_count": 65,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"RMSE: 1085.9207169016076\n"
]
}
],
"source": [
"test_error_ridge = df.sort_values('RMSE', ascending=True)['RMSE'].iloc[0]\n",
"print('RMSE: {}'.format(test_error_ridge))"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"(d) Fit a lasso model on the training set, with λ chosen by cross-validation. Report the test error obtained, along with the num-\n",
"ber of non-zero coefficient estimates."
]
},
{
"cell_type": "code",
"execution_count": 66,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"<matplotlib.axes._subplots.AxesSubplot at 0x1197b8358>"
]
},
"execution_count": 66,
"metadata": {},
"output_type": "execute_result"
},
{
"data": {
"image/png": "iVBORw0KGgoAAAANSUhEUgAAAYsAAAERCAYAAACKHYuuAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3Xt0XOV57/Hvz/JNFr6BFTDYYIc6BMOBUOsQEtIsCE0gKcWh5GJTEuhKuayU0OaEFHKhJKbpoivJaZIV0thwciMpJAeX4JA0LiWXnpUAtVTAwXZNjGmwbIyF78iyZGme88fsEVtjSSPZszUjze+zlpZn9uyZeQcJPXqe533frYjAzMxsMOMqPQAzM6t+DhZmZlaSg4WZmZXkYGFmZiU5WJiZWUkOFmZmVlLNBAtJ75W0TlJOUtMg510iaaOkTZJuTR3/f5KeSr62Sfphifc7N3X+05IuL+fnMTMbSRqL6ywkXQBcExHXpI6dDuSA5cDNEdHcz/PqgGeBtwOtwBpgaUSsLzpvJfBQRHxnkDFMAboiolvSbOBp4MSI6D7Kj2dmNuJqJrOIiA0RsbHEaecCmyJic0R0AfcDi9MnSJoGvA34YXK/QdI3JP2HpCclLU7e70AqMEwGxl5UNrOaUTPBYohOArak7rcmx9LeDTwaEfuS+58CfhYR5wIXAp+X1AAg6Y2S1gG/AW5wVmFmo9X4Sg+gnCQ9AUwCjgGOlfRU8tAtEbG6TG+zFLgndf8dwGWSbk7uTwZOBjZExBPAGUkJ7NuS/iUiDpZpHGZmI2ZMBYuIeCP037MYoq3A3NT9OckxktedRb5UlW5WC7hisBJXRGyQ9ApwJnBYr8TMrNq5DNXXGmCBpPmSJgJLgFWpx98DPFyUHawGPiJJAJLOSf6dL2l8cvsU4PXAf2f/EczMyq9mgoWkyyW1Am8CfixpdXL8REk/AUh6CjeSDwAbgB9ExLrUyywB7it66TuACcDapD9xR3L8LcDTSSnsQeDDEfFyNp/OzCxbY3LqrJmZlVfNZBZmZnbkxkyDe9asWTFv3rxKD8PMbFRpaWl5OSIaS503ZoLFvHnzaG72RCMzs+GQ9LuhnOcylJmZleRgYWZmJTlYmJlZSZkGi4G2+049frKknycb8K2V9K7k+DxJHaktvr+e5TjNzGxwmTW4k+2+7yK13bekVUXbfX+a/MK3f5S0EPgJMC957LmIeENW4zMzs6HLMrMoud03+W27pyW3pwPbMhyPmZkdoSyDxVC2+/4McFWyDcdPgI+kHpuflKd+KekPMhynmdmolMsFbfs72br7AG37O8nlstuRo9LrLJYC34qIL0p6E3CvpDOBF4GTI2KnpEXADyWdkbqGBACSrgOuAzj55JNHeuxmZhWTywUbX9rPtd9ppnV3B3Nm1nP3B5s47fipjBunsr9flpnFoNt9Jz4E/AAgIh4jfy2IWRHRGRE7k+MtwHPA64rfICJWRERTRDQ1NpZcgGhmNmbsbO/qDRQArbs7uPY7zexs78rk/bIMFqW2+wZ4AbgIeq+RPRlok9SYNMiR9FpgAbA5w7GamY0auVzQcai7N1AUtO7uoKu7J5P3zKwMFRHdkgrbfdcB34iIdZKWAc0RsQr4GHC3pI+Sb3ZfExEh6a3AMkmHgBz5S5LuymqsZmajRaH8tH3vQebMrO8TMObMrGfi+LpM3nfMbFHe1NQU3hvKzMa6tv2dXP61X9F4zCRuvvg0blm59qh6FpJaIqKp1HmVbnCbmdkwdHX30Lq7g9bdHXxh9UZuu3QhM+onMGdmPbOn12fS3AZv92FmNmrkckFPLpgzsx6AJ7fs4fp7W/jY/32aiePrMgsU4GBhZjZq7Gzv4m9/vJ6/v+Ks3oAxZ2Y9yz+wiOMaJmb63i5DmZmNAoUZUP+6fgdt+7t6y097Og4xq2FiplkFOFiYmVW94hlQhfIT5DOLBz98fuZjcBnKzKzKFRbgfeXR3x5Wgrr7g02Zl6DAmYWZWdWr1AyoNGcWZmZVrJIzoNIcLMzMqlglZ0CluQxlZlalKj0DKs3BwsysClXDDKg0l6HMzKrQy+2dFZ8BlebMwsysyuRywYHOys+ASnNmYWZWZXa2d/H8y+0VnwGV5mBhZlZFCk3t/spPy68a2RlQaS5DmZlViXRTu+2Vzj7lpwNdPcyeMbkiWQU4szAzqxrF23q0vdLZW346YfpkZtRXJqsAZxZmZlWjGrb1GIgzCzOzKlAt23oMxMHCzKwKVMu2HgNxGcrMrMKqaVuPgWSaWUi6RNJGSZsk3drP4ydL+rmkJyWtlfSu1GOfSJ63UdLFWY7TzKxSCjOgntvR3mdbj/eveJw7Hl7PuHHVUQDKbBSS6oC7gHcCC4GlkhYWnfZp4AcRcQ6wBPha8tyFyf0zgEuAryWvZ2Y2plTDhY2GIssy1LnApojYDCDpfmAxsD51TgDTktvTgW3J7cXA/RHRCTwvaVPyeo9lOF4zsxFXzTOg0rLMb04CtqTutybH0j4DXCWpFfgJ8JFhPBdJ10lqltTc1tZWrnGbmY2Iap8BlVbpYthS4FsRMQd4F3CvpCGPKSJWRERTRDQ1NjZmNkgzs3LL5YLt+w5W9QyotCzLUFuBuan7c5JjaR8i35MgIh6TNBmYNcTnmpmNSoWmdntndc+ASssys1gDLJA0X9JE8g3rVUXnvABcBCDpdGAy0Jact0TSJEnzgQXAf2Q4VjOzEVNoau9s76rqGVBpmY0oIrqBG4HVwAbys57WSVom6bLktI8B10p6GrgPuCby1gE/IN8M/ynwFxHRk9VYzcxGUqGp/fVfPFfVM6DSFBGVHkNZNDU1RXNzc6WHYWY2qFwuaN19gCvveYLW3R2cM3cGN1xwKsc1TOTEGfWcMG1kd5aV1BIRTaXOq75cx8xsDCve1uPJLXu44+H11E+sG/FAMRze7sPMbISMhm09BuJgYWY2AtIXNko3tSHfq3jww+dXeISDcxnKzGwEvNzeOSq29RiIMwszs4zlcsGBztGxrcdAnFmYmWVsZ3sXz7/cPiq29RiIg4WZWYYKTe3+yk/Lr6q+bT0G4jKUmVlG0k3ttlc6+5SfDnT1MHtG9U6VLebMwswsI8XXqmh7pbO3/HTC9MnMqB8dWQU4szAzy8xouVbFUDizMDPLwGi6VsVQOFiYmZXZaLtWxVC4DGVmVkaj8VoVQ+HMwsysjEbjtSqGYnSO2sysChXWVIy2a1UMhctQZmZl0N9GgYUZUJW6VkU5ObMwMyuD/jYKLFyromHS+FEdKMCZhZnZURsLGwWW4szCzOwojYWNAktxsDAzOwpjZaPAUjItQ0m6BPgyUAfcExF3Fj3+D8CFyd0pwGsiYkbyWA/wm+SxFyLisizHamY2XGNpo8BSMgsWkuqAu4C3A63AGkmrImJ94ZyI+Gjq/I8A56ReoiMi3pDV+MzMjlahqd14zCT+/oqzuGXlWq6/t6V3muxo2iiwlCwzi3OBTRGxGUDS/cBiYP0A5y8Fbs9wPGZmZVMLTe20LHsWJwFbUvdbk2OHkXQKMB/4WerwZEnNkh6X9O7shmlmNny10NROq5YG9xLggYjoSR07JSKagCuBL0k6tfhJkq5LAkpzW1vbSI3VzGpcrTS107IsQ20F5qbuz0mO9WcJ8BfpAxGxNfl3s6RfkO9nPFd0zgpgBUBTU1OUZdRmZoOopaZ2WpaZxRpggaT5kiaSDwirik+S9HpgJvBY6thMSZOS27OA8xm412FmNmKKV2qP5qvfDUdmmUVEdEu6EVhNfursNyJinaRlQHNEFALHEuD+iEhnBqcDyyXlyAe0O9OzqMzMKqHWmtpp6vs7evRqamqK5ubmSg/DzMawtv2dPLN1L7c99Aytuzt6j8+ZWc+DHz6fxqmTKji6IyOpJekPD6paGtxmZlWtFpvaad5I0MyshFptaqc5szAzK6FWm9ppzizMzAZRy03tNGcWZmaDqLWV2gNxsDAzG0CtN7XTXIYyM+uHm9p9ObMwM+uHm9p9ObMwMyvipvbhnFmYmaXkcsH2fQfd1C7iYGFmlij0Kbbt6XBTu4jLUGZmiZ3tXVz7nWZuu3Shm9pFnFmYmSW6uvN9iq//4jk3tYs4szAzI1+C6skFc2bW8+SWPb1ZxXENEzlxRj0nTKvdrAKcWZiZ9Ta1//bH63v7FE9u2cMdD6+nfmJdzQcKcGZhZjWu0NRu7+zmX9fvoG1/V2+fYk/HIWY1TKz5QAHOLMysxhUW3+1s7+rNKK6/t4X3r3icOx5ez7hx/jUJDhZmVsPSi+8KTe30VNm7P9hUs1Nli7kMZWY1K72jrJvag3NmYWY1qb8dZXub2hPc1C7mzMLMao53lB2+TDMLSZdI2ihpk6Rb+3n8HyQ9lXw9K2lP6rGrJf02+bo6y3GaWW3xjrLDl1lmIakOuAt4O9AKrJG0KiLWF86JiI+mzv8IcE5y+1jgdqAJCKAlee7urMZrZrXBO8oemSwzi3OBTRGxOSK6gPuBxYOcvxS4L7l9MfBIROxKAsQjwCUZjtXMaoB3lD1yWQaLk4AtqfutybHDSDoFmA/8bDjPlXSdpGZJzW1tbWUZtJmNTd5R9ugMGiwkvS11e37RY39SxnEsAR6IiJ7hPCkiVkREU0Q0NTY2lnE4ZjaWFDKKwuK7dFP7+9edxx2Lz3RTu4RSmcUXUrdXFj326RLP3QrMTd2fkxzrzxJeLUEN97lmZgNKZxTeUfbIlWpwa4Db/d0vtgZYkGQkW8kHhCsPewPp9cBM4LHU4dXA30mamdx/B/CJEu9nZnaYwsyn2y5d6MV3R6FUZhED3O7vft8HI7qBG8n/4t8A/CAi1klaJumy1KlLgPsjIlLP3QXcQT7grAGWJcfMzIZsoO08CovvGiaNd6AYIqV+Rx/+YH7dw7+TzyL+ILlNcv8tETFzoOeOtKampmhubq70MMysirTt7+SZrXu57aFnaN3dwTlzZ3DDBac6o0iR1BIRTaXOK1WGSk91/ULRY8X3zcyqRvF2HresXNubUSy/apEDxTANGiwi4pfp+5ImAGcCWyNiR5YDMzM7Ut7Oo/xKTZ39uqQzktvTgaeB7wBPSlo6AuMzMxs2b+dRfqXKUH8QETckt/8MeDYi3i3pBOBf6Dvd1cys4rydRzZKzYbqSt1+O/BDgIjYntmIzMyOkLfzyE6pYLFH0qWSzgHOB34KIGk8UJ/14MzMhsrbeWSrVBnqeuArwAnAX6UyiouAH2c5MDOz4UgvvnNTu/wGzSwi4tmIuCQi3hAR30odXx0RH8t8dGZmQ9Df4js3tctr0MxC0lcGezwibirvcMzMhqe4T+HtPLJRqmdxA/AWYBvQDLQUfZmZVcxAfQpfS7v8SvUsZgPvBd4PdAPfJ7+V+J5Bn2VmNgLcpxg5pXoWOyPi6xFxIfl1FjOA9ZI+MCKjMzMbgPsUI2tI1+CW9PvkL3v6dvKL8VyCMrOKcZ9i5JXa7mOZpBbgfwG/BJoi4kMRsX5ERmdmVsR9isoolVl8GngeODv5+jtJkN+iPCLirGyHZ2bWl/sUlVEqWMwv8biZ2Yjpr09xy8q1XH9vC3Nm1nP3B5vcp8hIqS3Kf9ffcUnjyPcw+n3czKzc3KeorFI9i2mSPiHpq5LeobyPAJuB943MEM2s1rlPUXmlylD3AruBx4A/Bz5Jvl/x7oh4KuOxmZkB7lNUg1LB4rUR8T8AJN0DvAicHBEHMx+ZmRnuU1SLUsHiUOFGRPRIanWgMLOR4j5F9Si1N9TZkvYlX/uBswq3Je0r9eKSLpG0UdImSbcOcM77JK2XtE7SP6WO90h6KvlaNbyPZWajnfsU1aXUbKi6I31hSXXAXeRXfbcCayStSi/ok7QA+ARwfkTslvSa1Et0RMQbjvT9zWx0c5+iupTKLI7GucCmiNgcEV3A/cDionOuBe6KiN0AEbEjw/GY2SjhfZ+qz5D2hjpCJwFbUvdbgTcWnfM6AEm/AuqAz0TET5PHJktqJr/b7Z0R8cPiN5B0HXAdwMknn1ze0ZtZRbhPUZ2yzCyGYjywALiA/CK/uyXNSB47JSKagCuBL0k6tfjJEbEiIpoioqmxsXGkxmxmGenuzrFh+z73KapQlpnFVmBu6v6c5FhaK/BERBwCnpf0LPngsSYitgJExGZJvwDOAZ7LcLxmVkG5XLBtbwfX39viPkUVyjKzWAMskDRf0kRgCVA8q+mH5LMKJM0iX5baLGmmpEmp4+cD3unWbIwqlJ527O90n6JKZZZZRES3pBuB1eT7Ed+IiHWSlgHNEbEqeewdktYDPcDHI2KnpDcDyyXlyAe0O70tutnYVJgi297Zzc72LvcpqpQiotJjKIumpqZobm6u9DDMbBgKGcX7lj/GbZcuZGXLFq5+83xuWbmW1t0dzJlZz/IPLOL0E6Y5UGREUkvSHx5Ulj0LM7MBpTOKQunp5otP49u/fr43o3jN1EmcOL3egaIKVHo2lJnVqMKiu+LS0xWL5vaWnubMnML48f41VQ38XTCzEdfforv0FNmGSePdo6gyLkOZ2YjyorvRyZmFmY0YL7obvZxZmNmI8KK70c2ZhZllzovuRj9nFmaWKS+6GxucWZhZptJTZFe2bDm8TzHRfYrRwJmFmWUilwv2dHR50d0Y4WBhZmVXKD1t33sQoE/p6YYLTnXpaRRyGcrMyq5Qepoysc5TZMcIZxZmVlbp1dl7Og55iuwY4czCzMqmeHW2p8iOHc4szKwsurtzbNyxn46unt7S0y0r1/KF1Ru5Y/GZzJvVQMOkOmY1THJWMQo5szCzo5Zenb2zvatP6enWd74egOn143nNVJefRisHCzM7Kl6dXRtchjKzI5YuPXl19tjmzMLMjkhx6cmrs8c2ZxZmNmz9lZ68OntsyzSzkHSJpI2SNkm6dYBz3idpvaR1kv4pdfxqSb9Nvq7OcpxmNnTpa1L4kqi1I7PvpKQ64C7gncBCYKmkhUXnLAA+AZwfEWcAf5UcPxa4HXgjcC5wu6SZWY3VzIbGpafalWUZ6lxgU0RsBpB0P7AYWJ8651rgrojYDRARO5LjFwOPRMSu5LmPAJcA92U4XjMbhEtPtS3LHPEkYEvqfmtyLO11wOsk/UrS45IuGcZzkXSdpGZJzW1tbUc0yFwuaNvfydbdB2jb30kuF0f0OmZjmUtPVunv6nhgAXABsBS4W9KMoT45IlZERFNENDU2Ng77zQs7Y37qwbU8s20fv9vZTuvuA3R354b9WmZjlUtPBtmWobYCc1P35yTH0lqBJyLiEPC8pGfJB4+t5ANI+rm/KPcAd7Z38Q+PbOTqN8/nlpVrad3dwZyZ9Sz/wCJOP2Gaf/Ct5rn0ZAVZZhZrgAWS5kuaCCwBVhWd80OSoCBpFvmy1GZgNfAOSTOTxvY7kmNl1dXdwxWL5vYGinPmzuC2SxfS0dXD9n0HXZKympXLBbvaO9nwoktPlpfZdzgiuoEbyf+S3wD8ICLWSVom6bLktNXATknrgZ8DH4+InUlj+w7yAWcNsKzQ7C6niePrOK5hYm+guPni01jZsoWd7V1s29PhkpTVpEJ59ukte7n+uy49WZ4ixsZfz01NTdHc3Dys5+RyQevuA1x5zxPcdulCVrZscUnKat6O/Qf5k6/9mi++92zev+Lx3j+kvv3r53szikLpyRnF6CepJSKaSp1X09/pcePEidPzAeG4hokuSVlNK5SeCtfM3tNxyKUn61Xz3+3x48dx+gnTOHFGvUtSVrMKU2Of3rKX/375QJ8LF6VLTw2Txrv0VKO8NxT5DOOEaZPp7skxZ2Y9N1xwKt/+9fO9JanGYyZx00UL6ImgYdJ4X7zFxpT01Ngvvvds7vyX//KFi+wwDhaJdEmqo+vVWVKNx0zi5otP69PHuPuDTZx2/FT/T2OjXvHU2IGumT29fjzHNkyq9HCtgmq+DJXWX0nqhgtO7Q0ayz+wiC++92y27z3Ino6uSg/X7Kj0tyrbFy6ygTizKFJckppRP6FPdlEoSe3tOER3LpyW26iTywV7OrrYf7Cb6+9t6Z0J6NKTDcbBoh/pktSOfZ3cdNGCPiWp9BTCg109nkJoo0ZhDcX2vQeZOnl8yVXZ/rm2Av8kDKBQkjp77nTmzZrSW5IqNL7veHg9n/vxBp5ra2fLngPs2O/ptVb9Xm7v5NrvNDNlYp1XZduw+KdhEOPGiWMbJtEwaXxvSaq48X3bQ89w4Rd+yZ987ddsfGm/A4ZVpf7WUHhVtg2Hy1BDMKthEnd/sIntew/2Nr5vu3RhnwV8N1xwKu2d3Wzfd9D/s1lV6e7OsXHHfnbs6wTobWR7Q0AbDmcWQzBunDjt+KmcPXc6jVMn9WYZXsBn1ayQTWxN1lBMmVjHVx79be9spy+s3sjSc0/hdcdP5eTjprj0ZINyZjFEhZLUtEkTehvfXsBn1SqdTRQa2V5DYUfDf0YMU7rxvfyqvntKFfoY9/3H73j2pVd4YecBZxk24tIrstONbK+hsKPhzOIIFLKMGfUT2b7vIEBvH8NZhlVSfyuyvYbCysHB4ij0t4DP24RYpRRKTx1dPX2yCa+hsHLwT8tRSi/gO9DV421CbMQVN7LTFytyI9vKxZlFGRT6GLOn57dQ8DYhNlL6a2Q7m7As+CenTAp9jJNSWUbxNiFufFu59Dct1iuyLUv+6SkzbxNiWSoEiQ0v5i9U1NZPI9srsi0LLkNloJBldOdiwMa3NyO04UqXnG576Bm++N6zh9TIdqCwcvBvpwwVtgkpbnw7y7Dh6K/kVJxNuJFtWcs0s5B0CfBloA64JyLuLHr8GuDzwNbk0Fcj4p7ksR7gN8nxFyLisizHmoXCNiHHT5vUp/HtLMOGaqCV2J4WayMts58oSXXAXcA7gYXAUkkL+zn1+xHxhuTrntTxjtTxURcoCvprfDvLsFIGa2CnV2IXsonTnE1YxrLMLM4FNkXEZgBJ9wOLgfUZvmfVGmh6rbMMKzZQNuGV2FZJWf4WOgnYkrrfmhwrdoWktZIekDQ3dXyypGZJj0t6d39vIOm65Jzmtra2Mg49G8PNMl7a18mL+w7ywq52Zxo1oFQ2cfWb5/eWnD71R6dzamMDc2fU85qpnu1k2av0n6w/AuZFxFnAI8C3U4+dEhFNwJXAlySdWvzkiFgREU0R0dTY2DgyIy6D9PTa9Jbn6Szj9ssW0p3LsfTux/nL+55i3dZ9vLDL5amxJpcL2vZ38vL+g4NOh3UD2yoty5+0rUA6U5jDq41sACJiZ0R0JnfvARalHtua/LsZ+AVwToZjHXGlsozd7Yf4+AN9r8j30e87aIwl3d05Nmzfx6ceXMu+g91c/11nE1a9sgwWa4AFkuZLmggsAValT5A0O3X3MmBDcnympEnJ7VnA+YzRXsdAWUZhemR6j6nCKvCNL+33KvBRrLjcdMWiuexq73I2YVUtswZ3RHRLuhFYTX7q7DciYp2kZUBzRKwCbpJ0GdAN7AKuSZ5+OrBcUo58QLszIsZksID+L6wE9Lkin7c/Hxv6a17PqJ/gXWKt6ilibJQympqaorm5udLDOGq5XLCno4td7V207e/k4KFc72rdPR2HuOPh9YfNnJozs57p9RMAmDS+juMaJjpwVJHC9/RQd46D3Tn+9J4n+nw/b7t0IStbthz2h4BnOtlIkNSS9IcH5e0+qkz6wkrHNkzkUHeOr1+1iLb9nb09jXSWUfj3y//2bO+U246ubv8lWgX6C/z9TYVNfy+dTVi1crCoUoWgAdA4dTJ7ZvS/PqO/8tQn33U6kgiCyRP8l2klpMtNwKB7OV2xaC4zpkzgb/74DARMdHZoVch/towCg82c8pTb6lCYAvvS3g527DvYZ63EUPZyakwyiZNmTqFxqoO7VR9nFqNIf6vA93Qc6lOe2t1+iNseeqbfvkZXMnPKfY3yyOWCne1diOClfZ18+dFnufrN8+nq7ltumlg3zs1rG/X8EzrKFGcZK1u2cGzDxH6n3BbKUytbtrDnwCGW/Wgd67bt43c72z3t9ij1t0aikOEVr5WY2TCBz7/HeznZ6Oaf0lGqkGV87vKzmDZ5PMuvypenDnT1HNbXuGLRXG8lUgaF9REv7e3od41EYZpzcbnps6vWM37cOO679jy+vPQNnHHSNE4+dooX1tmo4jLUKDZunGicmm+CH9swidkz8jNvPv+es0r2NTq6elh69+O9DfGuQzlybogfJj3tdX9n92GzmtJrJEptHT572mRnETZqOViMEf1NuT3YnSvZ1ygEjiVJ4LjpogW87vhj8rOpImpyZs5AAQIOn9VUPAU2vSvsTRctYP6sBqZ4rYSNAQ4WY0x6ym13d47lH1jEl//tWW595+m95Slg0IZ4Yc5/oTHek8vRkwt6YuxlHoUmdS6X6w2QAwUIYNA1EoUpsPdfd56nwNqY45x4DBtKX6N4D6p0f2NlyxYO9eR4/uV23r8iPxX3dy8fYNuejlHd6yje6fVTD67lxX0Hea7tFda/uJ8tuzr4+ANrD5v2Wvjvlt7gL71G4swTp/GaqZM9BdbGJAeLMa7Q15g1dTKnz85vWDj32PrevkY6cBT3N65YNLfP7reFNRxL7u4bOF7ceyC/tmD3gaoKIoetfdh9gB37O3oDRHoWU+FzDhYgBprV5DUSVgtchqoh/fU19nd29waO4v7GYCWrQq9j2cPr+PCFv0dHV09vUCk0zOvqoE7jONSTY9w4iChvH6S4hFR4nzqNIyJoe6WLrzz6bO/4vvmr57n1nadz/XdbuO3ShX1mMRU+Z/G6iNsvW8jn33MWH39gLZ9dtZ5Pvut07r/2PE8GsJrjzKIGFYLG8dPree2sYzjthKksnD2Vr1/Vd91G+i/r4pJV4S/xgbKPZQ+v46V9nTzX9gqf/dE6Nrcd4DOrnuGZbfvYsf8g2/Z2sD31F386OxnK7XSGUCghffZHr77n+hf3c0NR1lA8zTXdpB4ogyhMe70/mfZ6yqwpnDijnpOPbfDUV6spDhY1Lh04Fs7u299Y2bKl9xfnQL2OoQSR4j5IYYHg9n5+yQ/19isHew4rIRWXk4rHVxwgCk3q9Od0gDDrn8tQ1qt43cbnLj/rsFlChcAB9P7SLZSklV+4AAAHY0lEQVRt0uWc4tvpjQ8L/xbKWkdyu78SUnE5qXh8g81imlA3jhNmTeb7151HT8DkCeNcYjJLcbCwfqUDR0FjLg7rdXzzV8/z4Qt/b9AgUtwHGeyX/FBvpzOE9Puk+w2FoFAY3zd/dfg0V+/0ajY0LkPZkPXX67j9j8/g+GmTObXxGBbOnto70ypd2inugxT3CY7kdn8lpOJyUmHGUuMxkzi18Rhu/+MzeG1jA5+57ExPczUbJl8pz8qqsPq5o6uHceNEXTJLacL4cbSldmYt/MVfmKU03NvFCwePa5hITy76vKfLSWalDfVKeQ4WNmLSU117AupEasrrq7/kh3o7gprdksSsXHxZVas6/fVBzGx0yLRnIekSSRslbZJ0az+PXyOpTdJTydefpx67WtJvk6+rsxynmZkNLrPMQlIdcBfwdqAVWCNpVUSsLzr1+xFxY9FzjwVuB5qAAFqS5+7OarxmZjawLDOLc4FNEbE5IrqA+4HFQ3zuxcAjEbErCRCPAJdkNE4zMyshy2BxErAldb81OVbsCklrJT0gae5wnivpOknNkprb2trKNW4zMytS6XUWPwLmRcRZ5LOHbw/nyRGxIiKaIqKpsbExkwGamVm2wWIrMDd1f05yrFdE7IyIzuTuPcCioT7XzMxGTmbrLCSNB54FLiL/i34NcGVErEudMzsiXkxuXw7cEhHnJQ3uFuD3k1P/E1gUEbsGeb824HfDHOYs4OVhPme0q8XPDLX5uWvxM0Ntfu6j+cynRETJ0kxms6EiolvSjcBqoA74RkSsk7QMaI6IVcBNki4DuoFdwDXJc3dJuoN8gAFYNligSJ4z7DqUpOahLEYZS2rxM0Ntfu5a/MxQm597JD7zmFnBfST8Q1U7avFz1+Jnhtr83CPxmSvd4DYzs1Gg1oPFikoPoAJq8TNDbX7uWvzMUJufO/PPXNNlKDMzG5pazyzMzGwIHCzMzKykmgwWpXbDHYskzZX0c0nrJa2T9JeVHtNIkVQn6UlJD1d6LCNF0oxkC53/krRB0psqPaasSfpo8rP9jKT7JE2u9JiyIOkbknZIeiZ17FhJjyS7dD8iaWa537fmgkVqN9x3AguBpZIWVnZUI6Ib+FhELATOA/6iRj43wF8CGyo9iBH2ZeCnEfF64GzG+OeXdBJwE9AUEWeSX9u1pLKjysy3OHxj1VuBRyNiAfBocr+sai5YcHS74Y5aEfFiRPxncns/+V8e/W3sOKZImgP8EfntZGqCpOnAW4H/AxARXRGxp7KjGhHjgfpk94gpwLYKjycTEfHv5Bcxpy3m1b31vg28u9zvW4vBYqi74Y5ZkuYB5wBPVHYkI+JLwF8DuUoPZATNB9qAbyblt3skNVR6UFmKiK3AF4AXgBeBvRHxr5Ud1Yg6vrB1ErAdOL7cb1CLwaKmSToGWAn8VUTsq/R4siTpUmBHRLRUeiwjbDz5fdX+MSLOAdrJoCxRTZIa/WLygfJEoEHSVZUdVWVEfj1E2ddE1GKwqNkdbSVNIB8ovhcR/1zp8YyA84HLJP03+XLj2yR9t7JDGhGtQGtEFDLHB3h1U86x6g+B5yOiLSIOAf8MvLnCYxpJL0maDfkNWoEd5X6DWgwWa4AFkuZLmki+CbaqwmPKnCSRr2FviIj/XenxjISI+EREzImIeeS/zz+LiDH/12ZEbAe2SDotOXQRUHw547HmBeA8SVOSn/WLGONN/SKrgKuT21cDD5X7DTLbdbZaDbQbboWHNRLOBz4A/EbSU8mxT0bETyo4JsvOR4DvJX8QbQb+rMLjyVREPCHpAfKXM+gGnmSMbvsh6T7gAmCWpFbgduBO4AeSPkT+Ug3vK/v7ersPMzMrpRbLUGZmNkwOFmZmVpKDhZmZleRgYWZmJTlYmJlZSQ4WZoOQ9EqZXuczkm4ewnnfkvSecrynWTk5WJiZWUkOFmZDIOkYSY9K+k9Jv5G0ODk+L7lmxLckPSvpe5L+UNKvkmsLnJt6mbMlPZYcvzZ5viR9Nbm+yr8Br0m9599IWpNcn2FFsjLZrCIcLMyG5iBweUT8PnAh8MXUL+/fA74IvD75uhJ4C3Az8MnUa5wFvA14E/A3kk4ELgdOI39tlQ/Sdz+jr0bE/0yuz1APXJrRZzMrqea2+zA7QgL+TtJbyW93fhKvbgP9fET8BkDSOvIXoQlJvwHmpV7joYjoADok/Zz8tVXeCtwXET3ANkk/S51/oaS/Jn9thmOBdcCPMvuEZoNwsDAbmj8FGoFFEXEo2cm2cNnOztR5udT9HH3/HyveW2fAvXaSS4J+jfyV37ZI+kzq/cxGnMtQZkMznfy1MQ5JuhA45QheY7GkyZKOI78R3Brg34H3J9cJn02+xAWvBoaXk2uQeIaUVZQzC7Oh+R7wo6S01Az81xG8xlrg58As4I6I2CbpQfJ9jPXkt9l+DCAi9ki6G3iG/JXP1hz9RzA7ct511szMSnIZyszMSnKwMDOzkhwszMysJAcLMzMrycHCzMxKcrAwM7OSHCzMzKyk/w+ZqMQiZdp3XQAAAABJRU5ErkJggg==\n",
"text/plain": [
"<Figure size 432x288 with 1 Axes>"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"# LASSO Regression - Cross-Validation with test MSE\n",
"\n",
"results = pd.DataFrame()\n",
"alpha_range = np.linspace(0.1, 10, num=200)\n",
"for train_idx, test_idx in model_selection.KFold(n_splits=12).split(college_dat):\n",
" train, test = college_dat.iloc[train_idx], college_dat.iloc[test_idx]\n",
" errors = []\n",
" for alpha in alpha_range:\n",
" reg = linear_model.Lasso(alpha=alpha, max_iter=1000000)\n",
" reg.fit(train.drop('Apps', axis=1), train['Apps'])\n",
" error = np.sqrt(metrics.mean_squared_error(test['Apps'], reg.predict(test.drop('Apps', axis=1))))\n",
" errors.append(error)\n",
" results = pd.concat([results, pd.DataFrame(errors, index=alpha_range).T], axis=0, ignore_index=True)\n",
" \n",
"\n",
"df = pd.DataFrame({'lambda' : alpha_range, 'RMSE' : results.mean()})\n",
"sns.scatterplot(x='lambda', y='RMSE', data=df)"
]
},
{
"cell_type": "code",
"execution_count": 67,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"RMSE: 1070.514298450505\n"
]
}
],
"source": [
"test_error_lasso = df.sort_values('RMSE', ascending=True)['RMSE'].iloc[0]\n",
"print('RMSE: {}'.format(test_error_lasso))"
]
},
{
"cell_type": "code",
"execution_count": 68,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"array([-4.53468031e+02, 1.58612352e+00, -8.80863226e-01, 4.97995026e+01,\n",
" -1.41633311e+01, 5.86570247e-02, 4.48014731e-02, -8.78149635e-02,\n",
" 1.49626985e-01, 1.84128994e-02, 3.09317619e-02, -8.49867765e+00,\n",
" -3.16424017e+00, 1.56502651e+01, 2.40455396e-02, 7.81736691e-02,\n",
" 8.60426321e+00])"
]
},
"execution_count": 68,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"reg = linear_model.Lasso(alpha=3, max_iter=1000000)\n",
"reg.fit(college_dat.drop('Apps', axis=1), college_dat['Apps']).coef_\n",
"# there are no zero coefficients"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"(e) Fit a PCR model on the training set, with M chosen by cross-validation. Report the test error obtained, along with the value of M selected by cross-validation."
]
},
{
"cell_type": "code",
"execution_count": 69,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"<matplotlib.axes._subplots.AxesSubplot at 0x11987e780>"
]
},
"execution_count": 69,
"metadata": {},
"output_type": "execute_result"
},
{
"data": {
"image/png": "iVBORw0KGgoAAAANSUhEUgAAAY4AAAEKCAYAAAAFJbKyAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAGzpJREFUeJzt3XvYZXVd9/H3R0DFQwIx4gTYEI4ZWg40IT1qF0FyygI1FS7T0TCywLR6SkivPEWXlofnsRQulBE8BBKHGH1QmJCyehIYdDiLTBxiJg6T4CkfKfD7/LF+d+0Z7tO69973fQ/zfl3XvvZav7XWd3/3zN73d6+1fuu3UlVIkjRbj1noBCRJ2xYLhySpFwuHJKkXC4ckqRcLhySpFwuHJKkXC4ckqRcLhySpFwuHJKmXHRc6gXHYfffda9myZQudhiRtU6655pp/q6olM633qCwcy5YtY926dQudhiRtU5LcOZv1PFQlSerFwiFJ6sXCIUnqxcIhSerFwiFJ6sXCIUnqxcIhSerFwiFJ6uVReQHghM2nfWqo7Zf85q+OKBNJevRwj0OS1IuFQ5LUi4VDktSLhUOS1IuFQ5LUi4VDktSLhUOS1IuFQ5LUi4VDktSLhUOS1IuFQ5LUi4VDktTL2ApHkscnuSrJtUluTPLO1n5WktuTrG+PFa09ST6UZEOS65IcMBBrVZJb22PVuHKWJM1snKPjPggcUlXfTbIT8A9JPt+W/X5Vnb/V+kcCy9vjecBpwPOS7Aa8HVgJFHBNkjVV9cAYc5ckTWFsexzV+W6b3ak9appNjgY+0bb7MrBLkqXA4cDaqrq/FYu1wBHjyluSNL2xnuNIskOS9cB9dH/8r2yLTm2Hoz6Y5HGtbU/groHNN7a2qdq3fq0TkqxLsm7z5s0jfy+SpM5YC0dVPVxVK4C9gAOTPAc4BXgW8DPAbsBbRvRaZ1TVyqpauWTJklGElCRNYl56VVXVN4ErgCOq6u52OOpB4OPAgW21TcDeA5vt1dqmapckLYBx9qpakmSXNr0z8CLga+28BUkCHAPc0DZZA7ym9a46CPhWVd0NXAoclmTXJLsCh7U2SdICGGevqqXA2Ul2oCtQ51XV55J8MckSIMB64A1t/UuAo4ANwPeA1wFU1f1J3g1c3dZ7V1XdP8a8JUnTGFvhqKrrgP0naT9kivULOHGKZauB1SNNUJI0J145LknqxcIhSerFwiFJ6sXCIUnqxcIhSerFwiFJ6sXCIUnqxcIhSerFwiFJ6sXCIUnqxcIhSerFwiFJ6sXCIUnqxcIhSerFwiFJ6sXCIUnqxcIhSerFwiFJ6mVshSPJ45NcleTaJDcmeWdr3yfJlUk2JPlMkse29se1+Q1t+bKBWKe09luSHD6unCVJMxvnHseDwCFV9VxgBXBEkoOA9wIfrKpnAA8Ax7f1jwceaO0fbOuRZD/gWODZwBHAR5LsMMa8JUnTGFvhqM532+xO7VHAIcD5rf1s4Jg2fXSbpy0/NEla+7lV9WBV3Q5sAA4cV96SpOmN9RxHkh2SrAfuA9YC/wx8s6oeaqtsBPZs03sCdwG05d8CfniwfZJtJEnzbKyFo6oerqoVwF50ewnPGtdrJTkhybok6zZv3jyul5Gk7d689Kqqqm8CVwA/C+ySZMe2aC9gU5veBOwN0JY/BfjGYPsk2wy+xhlVtbKqVi5ZsmQs70OSNN5eVUuS7NKmdwZeBNxMV0B+pa22Cri4Ta9p87TlX6yqau3Htl5X+wDLgavGlbckaXo7zrzKnC0Fzm49oB4DnFdVn0tyE3Bukj8Gvgqc2dY/E/hkkg3A/XQ9qaiqG5OcB9wEPAScWFUPjzFvSdI0xlY4quo6YP9J2m9jkl5RVfV94OVTxDoVOHXUOUqS+vPKcUlSLxYOSVIvFg5JUi8WDklSLxYOSVIvFg5JUi8WDklSLxYOSVIvFg5JUi8WDklSLxYOSVIvFg5JUi8WDklSLxYOSVIvFg5JUi8WDklSLxYOSVIvFg5JUi8WDklSL2MrHEn2TnJFkpuS3JjkTa39HUk2JVnfHkcNbHNKkg1Jbkly+ED7Ea1tQ5KTx5WzJGlmO44x9kPA71XVV5I8Gbgmydq27INV9b7BlZPsBxwLPBv4EeBvkjyzLf4w8CJgI3B1kjVVddMYc5ckTWFshaOq7gbubtPfSXIzsOc0mxwNnFtVDwK3J9kAHNiWbaiq2wCSnNvWtXBI0gKYl3McSZYB+wNXtqaTklyXZHWSXVvbnsBdA5ttbG1TtW/9GickWZdk3ebNm0f8DiRJE8ZeOJI8CbgAeHNVfRs4DdgXWEG3R/L+UbxOVZ1RVSurauWSJUtGEVKSNIlxnuMgyU50RePTVXUhQFXdO7D8o8Dn2uwmYO+BzfdqbUzTLkmaZ+PsVRXgTODmqvrAQPvSgdVeAtzQptcAxyZ5XJJ9gOXAVcDVwPIk+yR5LN0J9DXjyluSNL1x7nE8H3g1cH2S9a3tD4HjkqwACrgD+A2AqroxyXl0J70fAk6sqocBkpwEXArsAKyuqhvHmLckaRrj7FX1D0AmWXTJNNucCpw6Sfsl020nSZo/XjkuSerFwiFJ6sXCIUnqxcIhSerFwiFJ6sXCIUnqxcIhSerFwiFJ6sXCIUnqxcIhSepl2sKR5JCB6X22WvbScSUlSVq8ZtrjGLy96wVbLXvbiHORJG0DZiocmWJ6snlJ0nZgpsJRU0xPNi9J2g7MNKz6jyVZQ7d3MTFNm99n6s0kSY9WMxWOowem37fVsq3nJUnbgWkLR1X93eB8u4f4c4BNVXXfOBOTJC1OM3XHPT3Js9v0U4BrgU8AX01y3DzkJ0laZGY6Of7Cgft7vw74elX9JPDTwB9Mt2GSvZNckeSmJDcmeVNr3y3J2iS3tuddW3uSfCjJhiTXJTlgINaqtv6tSVbN+d1KkoY2U+H4j4HpFwF/DVBV98wi9kPA71XVfsBBwIlJ9gNOBi6vquXA5W0e4EhgeXucAJwGXaEB3g48DzgQePtEsZEkzb+ZCsc3k7w4yf7A84EvACTZEdh5ug2r6u6q+kqb/g5wM7An3Qn3s9tqZwPHtOmjgU9U58vALkmWAocDa6vq/qp6AFgLHNHzfUqSRmSmXlW/AXwIeBrw5oE9jUOB/zPbF0myDNgfuBLYo6rubovuAfZo03sCdw1strG1TdW+9WucQLenwtOf/vTZpiZJ6mmmXlVfZ5Jf91V1KXDpbF4gyZPohit5c1V9O/nvC86rqpKM5ELCqjoDOANg5cqVXpwoSWMybeFI8qHpllfVb8+w/U50RePTVXVha743ydKqursdipro1rsJ2Htg871a2ybg4K3a/3a615Ukjc9M5zjeALwA+FdgHXDNVo8ppdu1OBO4uao+MLBoDTDRM2oVcPFA+2ta76qDgG+1Q1qXAocl2bWdFD+MWe7tSJJGb6ZzHEuBlwOvpOsl9Rng/Kr65ixiPx94NXB9kvWt7Q+B9wDnJTkeuBN4RVt2CXAUsAH4Hl33X6rq/iTvBq5u672rqu6fxetLksZgpnMc3wBOB05PshdwLHBTkrdU1Sdn2PYfmHoE3UMnWb+AE6eItRpYPd3rSZLmx0x7HAC0i/GOo7uW4/PMcJhKkvToNdPJ8XcBv0h3Dca5wClV9dB8JCZJWpxm2uN4G3A78Nz2+JPWnTZ0R5d+arzpSZIWm5kKh/fckCRtYaaT43dO1p7kMXTnPCZdLkl69JppWPUfSnJKkr9Icli7xuKNwG38dzdaSdJ2ZKZDVZ8EHgD+CXg93XUYAY6pqvXTbShJenSa8Z7j7f4bJPkYcDfw9Kr6/tgzkyQtSjMNOfKfExNV9TCw0aIhSdu3mfY4npvk2206wM5tfqI77g+NNTtJ0qIzU6+qHeYrEUnStmFWQ46oc/dH3jrU9kt/69QRZSJJC2emcxySJG3BPY4F9NXTf2mo7fd/w2dHlIkkzZ57HJKkXiwckqRePFT1KHLJmUfNedujjr9khJlIejRzj0OS1MvYCkeS1UnuS3LDQNs7kmxKsr49jhpYdkqSDUluSXL4QPsRrW1DkpPHla8kaXbGucdxFnDEJO0frKoV7XEJQJL96O5n/uy2zUeS7JBkB+DDwJHAfsBxbV1J0gIZ2zmOqvpSkmWzXP1o4NyqehC4PckG4MC2bENV3QaQ5Ny27k0jTleSNEsLcY7jpCTXtUNZu7a2PYG7BtbZ2Nqman+EJCckWZdk3ebNm8eRtySJ+S8cpwH7Aivohmh//6gCV9UZVbWyqlYuWbJkVGElSVuZ1+64VXXvxHSSjwKfa7ObgL0HVt2rtTFNuyRpAczrHkeSpQOzLwEmelytAY5N8rgk+wDLgauAq4HlSfZJ8li6E+hr5jNnSdKWxrbHkeQc4GBg9yQbgbcDBydZARRwB/AbAFV1Y5Lz6E56PwSc2G4cRZKTgEuBHYDVVXXjuHKWJM1snL2qjpuk+cxp1j8VeMS4463Lrpc1S9Ii4ZXjkqReLBySpF4sHJKkXiwckqReLBySpF4sHJKkXiwckqReLBySpF4sHJKkXrznuCa1+uzDhtr+11ZdNqJMJC027nFIknpxj0Pz4o8/c/jMK03jba+8dIv5Iy+ebCi02fn80edsMX/URe+dcyyAS17yli3mX3zBlEOyzcrnXnb8UNtL4+YehySpF/c4pEXul86/YM7bfvZXXjbCTKSOexySpF7c45C2Iy+54Iqhtr/oZT+/xfwrL/j6UPE+87JnbjF/xoX3DRXvhJc+9b+mv/jpzUPFOuRVS4ba/tHMwiFJs3DrX9w71PbLT9pji/l7PjDczUyf9rvP3mL+vj+/fM6xnvrGQ3ut76EqSVIvFg5JUi9jKxxJVie5L8kNA227JVmb5Nb2vGtrT5IPJdmQ5LokBwxss6qtf2uSVePKV5I0O+Pc4zgLOGKrtpOBy6tqOXB5mwc4EljeHicAp0FXaIC3A88DDgTePlFsJEkLY2yFo6q+BNy/VfPRwNlt+mzgmIH2T1Tny8AuSZYChwNrq+r+qnoAWMsji5EkaR7N9zmOParq7jZ9DzDRzWBP4K6B9Ta2tqnaHyHJCUnWJVm3efNw3fAkSVNbsJPjVVVAjTDeGVW1sqpWLlli/2tJGpf5Lhz3tkNQtOeJq302AXsPrLdXa5uqXZK0QOa7cKwBJnpGrQIuHmh/TetddRDwrXZI61LgsCS7tpPih7U2SdICGduV40nOAQ4Gdk+yka531HuA85IcD9wJvKKtfglwFLAB+B7wOoCquj/Ju4Gr23rvqqqtT7hLkubR2ApHVU11w4RHXNveznecOEWc1cDqEaYmSRqCV45LknqxcEiSerFwSJJ6sXBIknqxcEiSerFwSJJ6sXBIknqxcEiSerFwSJJ6sXBIknqxcEiSerFwSJJ6sXBIknqxcEiSerFwSJJ6sXBIknqxcEiSerFwSJJ6WZDCkeSOJNcnWZ9kXWvbLcnaJLe2511be5J8KMmGJNclOWAhcpYkdRZyj+Pnq2pFVa1s8ycDl1fVcuDyNg9wJLC8PU4ATpv3TCVJ/2UxHao6Gji7TZ8NHDPQ/onqfBnYJcnShUhQkrRwhaOAy5Jck+SE1rZHVd3dpu8B9mjTewJ3DWy7sbVtIckJSdYlWbd58+Zx5S1J270dF+h1X1BVm5I8FVib5GuDC6uqklSfgFV1BnAGwMqVK3ttK0mavQXZ46iqTe35PuAi4EDg3olDUO35vrb6JmDvgc33am2SpAUw74UjyROTPHliGjgMuAFYA6xqq60CLm7Ta4DXtN5VBwHfGjikJUmaZwtxqGoP4KIkE6//l1X1hSRXA+clOR64E3hFW/8S4ChgA/A94HXzn7IkacK8F46qug147iTt3wAOnaS9gBPnITVJ0iwspu64kqRtgIVDktSLhUOS1IuFQ5LUi4VDktSLhUOS1IuFQ5LUi4VDktSLhUOS1IuFQ5LUi4VDktSLhUOS1IuFQ5LUi4VDktSLhUOS1IuFQ5LUi4VDktSLhUOS1Ms2UziSHJHkliQbkpy80PlI0vZqmygcSXYAPgwcCewHHJdkv4XNSpK2T9tE4QAOBDZU1W1V9R/AucDRC5yTJG2XUlULncOMkvwKcERVvb7Nvxp4XlWdNLDOCcAJbfbHgVtmEXp34N9GmOpijreYcxt1vMWc22KPt5hzG3W8xZzbqOPNNtaPVtWSmVbacfh8FoeqOgM4o882SdZV1cpR5bCY4y3m3EYdbzHnttjjLebcRh1vMec26nijzm1bOVS1Cdh7YH6v1iZJmmfbSuG4GlieZJ8kjwWOBdYscE6StF3aJg5VVdVDSU4CLgV2AFZX1Y0jCN3r0NY2Hm8x5zbqeIs5t8UebzHnNup4izm3UccbaW7bxMlxSdLisa0cqpIkLRIWDklSL9tl4UiyOsl9SW4YUby9k1yR5KYkNyZ50xCxHp/kqiTXtljvHFGOOyT5apLPjSDWHUmuT7I+ybohY+2S5PwkX0tyc5KfHSLWj7ecJh7fTvLmIfP7nfb/cEOSc5I8fohYb2pxbpxLXpN9bpPslmRtklvb865Dxnt5y+8HSXp135wi3p+1/9vrklyUZJchYr27xVmf5LIkPzJMbgPLfi9JJdl9mHhJ3pFk08Dn76hhckvyxvZvd2OSPx0yt88M5HVHkvWzjTepqtruHsDPAQcAN4wo3lLggDb9ZODrwH5zjBXgSW16J+BK4KAR5Pi7wF8CnxtBrDuA3Uf0b3c28Po2/VhglxHF3QG4h+6CprnG2BO4Hdi5zZ8HvHaOsZ4D3AA8ga5Tyt8Az+gZ4xGfW+BPgZPb9MnAe4eM9xN0F9D+LbByBPkdBuzYpt872/ymiPVDA9O/DZw+TG6tfW+6Tjd39vlMT5HfO4D/OYfPxmSxfr59Rh7X5p867HsdWP5+4I/m8jmeeGyXexxV9SXg/hHGu7uqvtKmvwPcTPdHZy6xqqq+22Z3ao+hejAk2Qv4ReBjw8QZtSRPofuQnwlQVf9RVd8cUfhDgX+uqjuHjLMjsHOSHen+6P/rHOP8BHBlVX2vqh4C/g54aZ8AU3xuj6YrvrTnY4aJV1U3V9VsRl2YbbzL2vsF+DLdNVhzjfXtgdkn0uN7Mc13/oPAH/SJNUO83qaI9ZvAe6rqwbbOfaPILUmAVwDnzC3bznZZOMYpyTJgf7o9hbnG2KHtSt4HrK2qOcdq/hfdl+MHQ8aZUMBlSa5JN9TLXO0DbAY+3g6jfSzJE0eTIscy5JejqjYB7wP+Bbgb+FZVXTbHcDcAL0zyw0meABzFlhe1ztUeVXV3m74H2GMEMcfl14DPDxMgyalJ7gJeBfzRkLGOBjZV1bXDxNnKSe1w2uo+hw0n8Uy6z8uVSf4uyc+MKL8XAvdW1a3DBLFwjFCSJwEXAG/e6tdRL1X1cFWtoPt1dmCS5wyR04uB+6rqmrnGmMQLquoAutGKT0zyc3OMsyPdLvVpVbU/8O90h1uGku4i0V8G/mrIOLvS/aLfB/gR4IlJfnUusarqZrpDNZcBXwDWAw8Pk98kr1EMuXc6LkneCjwEfHqYOFX11qrau8U5aab1p8nnCcAfMmTx2cppwL7ACrofGu8fItaOwG7AQcDvA+e1vYVhHceQP6jAwjEySXaiKxqfrqoLRxGzHba5AjhiiDDPB345yR10owofkuRTQ+a1qT3fB1xEN3rxXGwENg7sUZ1PV0iGdSTwlaq6d8g4vwDcXlWbq+o/gQuB/zHXYFV1ZlX9dFX9HPAA3bmwYd2bZClAe571IY35kuS1wIuBV7XiNgqfBl42xPb70v0guLZ9N/YCvpLkaXMNWFX3th99PwA+yty/F9B9Ny5sh66vojtaMOuT95Nph1tfCnxmmDhg4RiJ9kvgTODmqvrAkLGWTPQ8SbIz8CLga3ONV1WnVNVeVbWM7vDNF6tqTr+aW05PTPLkiWm6k59z6p1WVfcAdyX58dZ0KHDTXHMbMJJfVXSHqA5K8oT2f3wo3fmrOUny1Pb8dLov8F+OIMc1wKo2vQq4eAQxRybJEXSHSX+5qr43ZKzlA7NHM9z34vqqempVLWvfjY10HVzuGSK/pQOzL2GO34vmr+lOkJPkmXQdR4YdKfcXgK9V1cYh42y3varOoduV/E+6D8zxQ8Z7Ad0hguvoDkGsB46aY6yfAr7aYt3AkL0ftop9MEP2qgJ+DLi2PW4E3jpkvBXAuvZ+/xrYdch4TwS+ATxlRP9m76T7A3UD8ElaL5c5xvp7usJ4LXDoHLZ/xOcW+GHgcuBWul44uw0Z7yVt+kHgXuDSIeNtAO4a+F7MqifUFLEuaP8P1wGfBfYcJretlt9Bv15Vk+X3SeD6lt8aYOkQsR4LfKq9368Ahwz7XoGzgDeM4nvhkCOSpF48VCVJ6sXCIUnqxcIhSerFwiFJ6sXCIUnqxcIhPYolOTjJnC9alCZj4ZAe3Q5miKvdpclYOPSol2RZunt9fLTd2+CydlX+ZOs+I8nfpLsfyleS7JvOn6W7l8b1SV7Z1j24DUB3cZLbkrwnyavS3U/l+iT7tvXOSnJ6knVJvt7GD5u498rH27pfTTJxpfBrk1yY5Avp7rPxpwP5HZbkn1puf9XGR5u4R8o7W/v1SZ7VBtx8A/A77T4ML0x3v40b2vv70jj/3fXoteNCJyDNk+XAcVX160nOoxvnaLIxuz5NN5z1Relu2vQYuuFBVgDPpRsv6OqBP7rPpRsy/X7gNuBjVXVgupt5vRGYuGHTMrqxi/YFrkjyDOBEurEJfzLJs+hGHH5mW38F3SjLDwK3JPlz4P8BbwN+oar+Pclb6O6z8q62zb9V1QFJfovuvhCvT3I68N2qeh9AkuuBw6tqU2Z5UyVpa+5xaHtxe1VN3PXsGro/5FtoY3DtWVUXAVTV96sbX+kFwDnVDWB3L929NCaGub66uvuxPAj8M93ot9ANPTH4GudV1Q+qG876NuBZLe6n2mt9je5mQhOF4/Kq+lZVfZ9umJIfpRspdT/gH9MNu7+qtU+YGFxz0vfX/CNwVpJfp7vZldSbexzaXjw4MP0wMOmhqiHj/mBg/gds+f3aemyfmcb62TrfHenuDrm2qo6bYZuJ9R+hqt6Q5Hl0N/a6JslPV9U3ZshF2oJ7HFJT3d0bNyY5BiDJ49p9G/4eeGW6G2wtobtr4VU9w788yWPaeY8fA25pcV/VXuuZwNNb+1S+DDy/HeaaGKn4mdOsD/AdutsZ07bZt6qurKo/oruJ1ihuJqXtjIVD2tKrgd9Och3wf4Gn0d1z5Dq6UW2/CPxB9R9++1/ois3n6UYo/T7wEeAx7bzDZ+juZ/7gVAGqajPwWuCclt8/0R3yms5ngZdMnBwH/qydPL+hvb9R3v1O2wlHx5XGLMlZdMPZn7/QuUij4B6HJKkX9zi0XUryYbrb6g7631X18YXIR9qWWDgkSb14qEqS1IuFQ5LUi4VDktSLhUOS1IuFQ5LUy/8HHLVGx3m/w8sAAAAASUVORK5CYII=\n",
"text/plain": [
"<Figure size 432x288 with 1 Axes>"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"# PCR Regression - Cross-Validation with test MSE\n",
"\n",
"results = pd.DataFrame()\n",
"predictors = college_dat.drop('Apps', axis=1).columns\n",
"c_range = range(1, len(predictors) + 1)\n",
"for train_idx, test_idx in model_selection.KFold(n_splits=10).split(college_dat):\n",
" train, test = college_dat.iloc[train_idx], college_dat.iloc[test_idx]\n",
" errors = []\n",
" for c in c_range:\n",
" pca = decomposition.PCA(n_components=c)\n",
" X = pca.fit_transform(train.drop('Apps', axis=1))\n",
" reg = linear_model.LinearRegression()\n",
" reg.fit(X, train['Apps'])\n",
" \n",
" test_X = pca.transform(test.drop('Apps', axis=1))\n",
" error = np.sqrt(metrics.mean_squared_error(test['Apps'], reg.predict(test_X)))\n",
" errors.append(error)\n",
" results = pd.concat([results, pd.DataFrame(errors, index=c_range).T], axis=0, ignore_index=True)\n",
"\n",
"df = pd.DataFrame({'n_components' : c_range, 'RMSE' : results.mean()})\n",
"sns.barplot(x='n_components', y='RMSE', data=df)"
]
},
{
"cell_type": "code",
"execution_count": 70,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"RMSE: 1098.9615610719898\n"
]
}
],
"source": [
"test_error_pcr = df.sort_values('RMSE', ascending=True).iloc[0]['RMSE']\n",
"print('RMSE: {}'.format(test_error_pcr))"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"(f) Fit a PLS model on the training set, with M chosen by cross-validation. Report the test error obtained, along with the value of M selected by cross-validation."
]
},
{
"cell_type": "code",
"execution_count": 71,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"<matplotlib.axes._subplots.AxesSubplot at 0x1199aa898>"
]
},
"execution_count": 71,
"metadata": {},
"output_type": "execute_result"
},
{
"data": {
"image/png": "iVBORw0KGgoAAAANSUhEUgAAAY8AAAEKCAYAAADq59mMAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAGk9JREFUeJzt3XuUnXV97/H3R6JWUQQkYiTYIAYtUg2Qg56juFCsArUCWhWWVbxGKnip7bGiLrW2dHlBPccbLJSIF+SiiKAHBESrPT2iBowQbhIuStKQRFGxXqLA9/zxPKM7yUxmnr33zJ6Q92utveZ5fvvZ3/lOMjOfeS7796SqkCSpi/uMugFJ0tbH8JAkdWZ4SJI6MzwkSZ0ZHpKkzgwPSVJnhockqTPDQ5LUmeEhSepszqgbmC677LJLLViwYNRtSNJW44orrvhJVc2dyrb32vBYsGABy5YtG3UbkrTVSPKjqW7rYStJUmeGhySpM8NDktSZ4SFJ6szwkCR1ZnhIkjqbtvBIsjTJuiQresbOTrK8fdyaZHk7viDJb3qeO6XnNfsnuTrJyiQfSpLp6lmSNDXT+T6P04GPAJ8eG6iqF44tJ3k/8Iue7W+qqkXj1DkZeBXwHeBC4BDgomnoV5I0RdMWHlX1rSQLxnuu3Xt4AfD0LdVIMg/Yoaoub9c/DRxBh/BYf/Jnp7rpZub+7d/0/VpJujcb1TmPA4G1VXVjz9geSb6f5JtJDmzHdgNW9Wyzqh0bV5IlSZYlWbZ+/frhdy1JAkYXHkcDZ/asrwEeWVX7Am8EPpdkh65Fq+rUqlpcVYvnzp3S9CySpD7M+NxWSeYAzwX2Hxurqg3Ahnb5iiQ3AXsBq4H5PS+f345JkkZoFHsezwCur6o/HI5KMjfJdu3yo4CFwM1VtQa4M8mT2vMkLwHOH0HPkqQe03mp7pnAt4HHJFmV5BXtU0ex8SErgKcCV7WX7n4BOLaq7mifew3wCWAlcBNeaSVJIzedV1sdPcH4S8cZOxc4d4LtlwH7DLU5SdJAfIe5JKkzw0OS1JnhIUnqzPCQJHVmeEiSOjM8JEmdGR6SpM4MD0lSZ4aHJKkzw0OS1JnhIUnqzPCQJHVmeEiSOjM8JEmdGR6SpM4MD0lSZ4aHJKkzw0OS1JnhIUnqzPCQJHU2beGRZGmSdUlW9Iy9M8nqJMvbx2E9z52QZGWSG5I8q2f8kHZsZZI3T1e/kqSpm849j9OBQ8YZ/2BVLWofFwIk2Rs4Cnhc+5qPJdkuyXbAR4FDgb2Bo9ttJUkjNGe6ClfVt5IsmOLmhwNnVdUG4JYkK4ED2udWVtXNAEnOare9dsjtSpI6GMU5j+OTXNUe1tqpHdsNuK1nm1Xt2ETjkqQRmunwOBnYE1gErAHeP8ziSZYkWZZk2fr164dZWpLUY0bDo6rWVtXdVXUP8HH+eGhqNbB7z6bz27GJxieqf2pVLa6qxXPnzh1u85KkP5jR8Egyr2f1SGDsSqwLgKOS3D/JHsBC4LvA94CFSfZIcj+ak+oXzGTPkqTNTdsJ8yRnAgcBuyRZBbwDOCjJIqCAW4FXA1TVNUnOoTkRfhdwXFXd3dY5HrgY2A5YWlXXTFfPkqSpmc6rrY4eZ/i0LWx/InDiOOMXAhcOsbW+rfnYWwd6/bzXbPblSdJWyXeYS5I6MzwkSZ0ZHpKkzgwPSVJnhockqTPDQ5LUmeEhSerM8JAkdWZ4SJI6MzwkSZ0ZHpKkzgwPSVJnhockqTPDQ5LUmeEhSeps2u7nocl9/5S/Guj1+x775SF1IknduOchSerM8JAkdWZ4SJI6MzwkSZ15wvxe5MLTDuv7tYe94sIhdiLp3m7a9jySLE2yLsmKnrH3Jbk+yVVJzkuyYzu+IMlvkixvH6f0vGb/JFcnWZnkQ0kyXT1LkqZmOg9bnQ4cssnYpcA+VfV44IfACT3P3VRVi9rHsT3jJwOvAha2j01rSpJm2LSFR1V9C7hjk7FLququdvVyYP6WaiSZB+xQVZdXVQGfBo6Yjn4lSVM3yhPmLwcu6lnfI8n3k3wzyYHt2G7Aqp5tVrVj40qyJMmyJMvWr18//I4lScCIwiPJW4G7gDPaoTXAI6tqX+CNwOeS7NC1blWdWlWLq2rx3Llzh9ewJGkjM361VZKXAs8GDm4PRVFVG4AN7fIVSW4C9gJWs/GhrfntmCRphGZ0zyPJIcCbgOdU1a97xucm2a5dfhTNifGbq2oNcGeSJ7VXWb0EOH8me5YkbW7a9jySnAkcBOySZBXwDpqrq+4PXNpecXt5e2XVU4F3Jfk9cA9wbFWNnWx/Dc2VWw+gOUfSe55EkjQC0xYeVXX0OMOnTbDtucC5Ezy3DNhniK1Jkgbk9CSSpM4MD0lSZ4aHJKkzw0OS1JnhIUnqzPCQJHXm/Tw0rqWfeuZAr3/5MZcMqRNJs5F7HpKkzgwPSVJnHrbSjPiXs5810Ovf9sKLh9SJpGEwPLRVOvT88Wa/mZqLDj9ziJ1I2ybDQ9u8w857z0Cvv/DIfxxSJ9LWw/CQhuzZ5447/+eUfeV5r9ho/a++MO6coVP25b9+3h+Wjzz3GwPVOu95T9to/YXn/nCgemc/b6+N1k/94rqB6i157sM2Wv/6Gf3fUfTpL9r4hnI3fmRt37UAFh6/60brt3/gmoHqPfyNj9tofd2HL+u71sNee3Dn13jCXJLUmeEhSerM8JAkdWZ4SJI6MzwkSZ0ZHpKkzgwPSVJn0xoeSZYmWZdkRc/YzkkuTXJj+3GndjxJPpRkZZKrkuzX85pj2u1vTHLMdPYsSZrcFsMjydN7lvfY5LnnTqH+6cAhm4y9GbisqhYCl7XrAIcCC9vHEuDk9vPsDLwDeCJwAPCOscCRJI3GZHseJ/Usb/o217dNVryqvgXcscnw4cCn2uVPAUf0jH+6GpcDOyaZBzwLuLSq7qiqnwGXsnkgSZJm0GThkQmWx1ufql2rak27fDsw9p793YDberZb1Y5NNC5JGpHJwqMmWB5vvbOqqmHUGZNkSZJlSZatX9//nDaSpC2bbGLERyW5gGYvY2yZdn2PiV+2RWuTzKuqNe1hqbGZ0FYDu/dsN78dWw0ctMn4v41XuKpOBU4FWLx48dBCSZK0scnC4/Ce5ZM2eW7T9am6ADgGeHf78fye8eOTnEVzcvwXbcBcDPxrz0nyZwIn9Pm5JUlDsMXwqKpv9q4nuS+wD7C6qiadOznJmTR7DbskWUVz1dS7gXOSvAL4EfCCdvMLgcOAlcCvgZe1PdyR5J+B77XbvauqNj0JL0maQVsMjySnAB+uqmuSPAT4NnA3sHOSf6iqLd6Sraomut3bZpPHt+c/jpugzlJg6ZY+lyRp5kx2wvzAqhq7Y8nLgB9W1Z8D+wNvmtbOJEmz1mTh8bue5b8AvgRQVbdPW0eSpFlvsvD4eZJnJ9kXeDLwVYAkc4AHTHdzkqTZabKrrV4NfAh4OPCGnj2Og4H/M52NSZJmr8mutvoh40wFUlUXAxdPV1OSpNltsqutPrSl56vqdcNtR5K0NZjssNWxwArgHOA/6X8+K0nSvchk4TEPeD7wQuAu4GzgC1X18+luTJI0e23xaquq+mlVnVJVT6N5n8eOwLVJXjwj3UmSZqXJ9jwAaO/qdzTNez0uAq6YzqYkSbPbZCfM3wX8JXAdcBZwQlXdNRONSZJmr8n2PN4G3AI8oX38axJoTpxXVT1+etuTJM1Gk4VHv/fskCTdi032JsEfjTee5D4050DGfV6SdO+2xautkuyQ5IQkH0nyzDReC9zMH+/DIUnaxkx22OozwM9o7uPxSuAtNOc7jqiq5dPcmyRplpr0Hubt/TtI8glgDfDIqvrttHcmSZq1JpuS/fdjC1V1N7DK4JAkTbbn8YQkd7bLAR7Qro9dqrvDtHYnSZqVJrvaaruZakSStPWY7LCVJEmbmfHwSPKYJMt7HncmeUOSdyZZ3TN+WM9rTkiyMskNSZ410z1LkjY2pYkRh6mqbgAWASTZDlgNnEcza+8Hq+qk3u2T7A0cBTwOeATwtSR7tSfwJUkjMOrDVgcDN030TvbW4cBZVbWhqm4BVgIHzEh3kqRxjTo8jgLO7Fk/PslVSZYm2akd2w24rWebVe3YZpIsSbIsybL169dPT8eSpNGFR5L7Ac8BPt8OnQzsSXNIaw3w/q41q+rUqlpcVYvnzp07tF4lSRsb5Z7HocCVVbUWoKrWVtXdVXUP8HH+eGhqNbB7z+vmt2OSpBEZZXgcTc8hqyTzep47EljRLl8AHJXk/kn2ABYC352xLiVJm5nxq60AkmxPc0vbV/cMvzfJIqCAW8eeq6prkpwDXAvcBRznlVaSNFojCY+q+hXw0E3GXryF7U8ETpzuviRJUzPqq60kSVshw0OS1JnhIUnqzPCQJHVmeEiSOjM8JEmdGR6SpM4MD0lSZ4aHJKkzw0OS1JnhIUnqzPCQJHVmeEiSOjM8JEmdGR6SpM4MD0lSZ4aHJKkzw0OS1JnhIUnqzPCQJHU2svBIcmuSq5MsT7KsHds5yaVJbmw/7tSOJ8mHkqxMclWS/UbVtyRp9HseT6uqRVW1uF1/M3BZVS0ELmvXAQ4FFraPJcDJM96pJOkPRh0emzoc+FS7/CngiJ7xT1fjcmDHJPNG0aAkabThUcAlSa5IsqQd27Wq1rTLtwO7tsu7Abf1vHZVO7aRJEuSLEuybP369dPVtyRt8+aM8HM/papWJ3kYcGmS63ufrKpKUl0KVtWpwKkAixcv7vRaSdLUjWzPo6pWtx/XAecBBwBrxw5HtR/XtZuvBnbvefn8dkySNAIjCY8k2yd58Ngy8ExgBXABcEy72THA+e3yBcBL2quungT8oufwliRpho3qsNWuwHlJxnr4XFV9Ncn3gHOSvAL4EfCCdvsLgcOAlcCvgZfNfMuSpDEjCY+quhl4wjjjPwUOHme8gONmoDVJ0hTMtkt1JUlbAcNDktSZ4SFJ6szwkCR1ZnhIkjozPCRJnRkekqTODA9JUmeGhySpM8NDktSZ4SFJ6szwkCR1ZnhIkjozPCRJnRkekqTODA9JUmeGhySpM8NDktSZ4SFJ6szwkCR1ZnhIkjqb8fBIsnuSbyS5Nsk1SV7fjr8zyeoky9vHYT2vOSHJyiQ3JHnWTPcsSdrYnBF8zruAv6+qK5M8GLgiyaXtcx+sqpN6N06yN3AU8DjgEcDXkuxVVXfPaNeSpD+Y8T2PqlpTVVe2y78ErgN228JLDgfOqqoNVXULsBI4YPo7lSRNZKTnPJIsAPYFvtMOHZ/kqiRLk+zUju0G3NbzslVMEDZJliRZlmTZ+vXrp6lrSdLIwiPJg4BzgTdU1Z3AycCewCJgDfD+rjWr6tSqWlxVi+fOnTvUfiVJfzSS8EhyX5rgOKOqvghQVWur6u6qugf4OH88NLUa2L3n5fPbMUnSiIziaqsApwHXVdUHesbn9Wx2JLCiXb4AOCrJ/ZPsASwEvjtT/UqSNjeKq62eDLwYuDrJ8nbsLcDRSRYBBdwKvBqgqq5Jcg5wLc2VWsd5pZUkjdaMh0dV/V8g4zx14RZecyJw4rQ1JUnqxHeYS5I6MzwkSZ0ZHpKkzgwPSVJnhockqTPDQ5LUmeEhSerM8JAkdWZ4SJI6MzwkSZ0ZHpKkzgwPSVJnhockqTPDQ5LUmeEhSerM8JAkdWZ4SJI6MzwkSZ0ZHpKkzgwPSVJnW014JDkkyQ1JViZ586j7kaRt2VYRHkm2Az4KHArsDRydZO/RdiVJ266tIjyAA4CVVXVzVf0OOAs4fMQ9SdI2K1U16h4mleSvgUOq6pXt+ouBJ1bV8ZtstwRY0q4+BrhhktK7AD8ZYqvbUr3Z3Nuw683m3oZdbzb3Nux6s7m3UdX706qaO5VicwbvZ/aoqlOBU6e6fZJlVbV4WJ9/W6o3m3sbdr3Z3Nuw683m3oZdbzb3tjXU21oOW60Gdu9Zn9+OSZJGYGsJj+8BC5PskeR+wFHABSPuSZK2WVvFYauquivJ8cDFwHbA0qq6Zgilp3yIy3rTWmu215vNvQ273mzubdj1ZnNvs77eVnHCXJI0u2wth60kSbOI4SFJ6mybDI8kS5OsS7JiSPV2T/KNJNcmuSbJ6weo9SdJvpvkB22tfxpSj9sl+X6Srwyh1q1Jrk6yPMmyAWvtmOQLSa5Pcl2S/z5Arce0PY097kzyhgH7+7v2/2FFkjOT/MkAtV7f1rmmn77G+75NsnOSS5Pc2H7cacB6z2/7uydJp8s6J6j3vvb/9qok5yXZccB6/9zWWp7kkiSP6LdWz3N/n6SS7DJgb+9Msrrn+++wQeq1469t//2uSfLeAXo7u6evW5Msn2pvE6qqbe4BPBXYD1gxpHrzgP3a5QcDPwT27rNWgAe1y/cFvgM8aQg9vhH4HPCVIdS6FdhlSP92nwJe2S7fD9hxSHW3A26nedNTvzV2A24BHtCunwO8tM9a+wArgAfSXKjyNeDRHWts9n0LvBd4c7v8ZuA9A9b7M5o32P4bsHgI/T0TmNMuv2cI/e3Qs/w64JR+a7Xju9NciPOjLt/TE/T2TuAf+vz+GK/e09rvk/u36w8b5Gvtef79wNv76bP3sU3ueVTVt4A7hlhvTVVd2S7/EriO5hdPP7Wqqv6rXb1v+xjoqoYk84G/BD4xSJ1hS/IQmm/00wCq6ndV9fMhlT8YuKmqfjRgnTnAA5LMofnF/5991vkz4DtV9euqugv4JvDcLgUm+L49nCaAaT8eMUi9qrquqiabmaFLvUvarxfgcpr3aA1S786e1e2Z4s/GFn7mPwi8aap1plCvLxPU+1vg3VW1od1m3aC9JQnwAuDM/rttbJPhMZ2SLAD2pdlj6LfGdu1u5Trg0qrqu1brf9H8gNwzYJ0xBVyS5Io0U8L0aw9gPfDJ9pDaJ5JsP5wWOYoBf0CqajVwEvBjYA3wi6q6pM9yK4ADkzw0yQOBw9j4ja/92rWq1rTLtwO7DqHmdHk5cNGgRZKcmOQ24EXA2weocziwuqp+MGhPPY5vD6st7XIIcQJ70XzPfCfJN5P8tyH0dyCwtqpuHLSQ4TFESR4EnAu8YZO/kDqpqrurahHNX2kHJNlngJ6eDayrqiv6rTGOp1TVfjSzHB+X5Kl91plDs3t9clXtC/yK5tDLQNK8kfQ5wOcHrLMTzV/2ewCPALZP8jf91Kqq62gO21wCfBVYDtw9SH/jfI5iwL3U6ZLkrcBdwBmD1qqqt1bV7m2t4yfbfoJ+Hgi8hQHCZxwnA3sCi2j+2Hj/gPXmADsDTwL+J3BOu+cwiKMZwl4HGB5Dk+S+NMFxRlV9cRg120M43wAOGaDMk4HnJLmVZjbipyf57IB9rW4/rgPOo5n1uB+rgFU9e1ZfoAmTQR0KXFlVawes8wzglqpaX1W/B74I/I9+i1XVaVW1f1U9FfgZzbmxQa1NMg+g/TilQxszKclLgWcDL2oDbljOAJ7X52v3pPmj4Aftz8Z84MokD++3mapa2/7hdw/wcfr/uRizCvhieyj7uzRHDqZ8Un9T7aHX5wJnD9gXYHgMRfvXwGnAdVX1gQFrzR27IiXJA4C/AK7vt15VnVBV86tqAc2hnK9XVV9/Pbc9bZ/kwWPLNCdE+7pqrapuB25L8ph26GDg2n576zGsv65+DDwpyQPb/+ODac5n9SXJw9qPj6T5If7cEHq8ADimXT4GOH8INYcmySE0h0yfU1W/HkK9hT2rh9Pnz0ZVXV1VD6uqBe3Pxiqai15uH6C3eT2rR9Lnz0WPL9GcNCfJXjQXlAwyy+4zgOuratWAfTUGPeO+NT5ofrGsAX5P803zigHrPYXmcMFVNIcjlgOH9Vnr8cD321orGMJVET21D2LAq62ARwE/aB/XAG8dsN4iYFn79X4J2GnAetsDPwUeMqR/s3+i+QW1AvgM7ZUvfdb6d5pw/AFwcB+v3+z7FngocBlwI82VOTsPWO/IdnkDsBa4eMB6K4Hben4upnR11Bbqndv+X1wFfBnYrd9amzx/K92uthqvt88AV7e9XQDMG7De/YDPtl/vlcDTB/lagdOBY4fxc1FVTk8iSerOw1aSpM4MD0lSZ4aHJKkzw0OS1JnhIUnqzPCQ7sWSHJSk7zc2ShMxPKR7t4MY4F3x0kQMD93rJVmQ5l4hH2/vi3BJ++798bZ9dJKvpbmfypVJ9kzjfWnuxXF1khe22x7UTlh3fpKbk7w7yYvS3I/l6iR7ttudnuSUJMuS/LCdb2zs3i2fbLf9fpKxdxO/NMkXk3w1zX063tvT3zOTfLvt7fPtfGpj91j5p3b86iSPbSfpPBb4u/Y+DgemuV/Hivbr+9Z0/rvr3m3OqBuQZshC4OiqelWSc2jmRBpvjq8zaKbBPi/NjZ/uQzOVyCLgCTRzC32v5xfvE2imW78DuBn4RFUdkOaGYK8Fxm76tIBmrqM9gW8keTRwHM18hn+e5LE0MxXv1W6/iGZ25g3ADUk+DPwGeBvwjKr6VZJ/pLlPy7va1/ykqvZL8hqa+0q8MskpwH9V1UkASa4GnlVVq9PhxkzSptzz0Lbilqoau3vaFTS/zDfSztm1W1WdB1BVv61mPqanAGdWM+ndWpp7cYxNj/29au7nsgG4iWbWXGimqej9HOdU1T3VTIV9M/DYtu5n2891Pc0NicbC47Kq+kVV/ZZmSpM/pZlddW/gP9JM2X9MOz5mbELOcb++1n8Apyd5Fc0Ns6S+uOehbcWGnuW7gXEPWw1Y956e9XvY+Odr03mAJpsXaNN+59DcZfLSqjp6kteMbb+Zqjo2yRNpbg52RZL9q+qnk/QibcY9D6lVzV0gVyU5AiDJ/dv7Pvw78MI0N+maS3P3w+92LP/8JPdpz4M8Crihrfui9nPtBTyyHZ/I5cCT20NeYzMc77WF7QF+SXNrZNrX7FlV36mqt9PciGsYN6TSNsjwkDb2YuB1Sa4C/h/wcJp7llxFMxvu14E3Vfepu39MEzgX0cxs+lvgY8B92vMQZ9PcH33DRAWqaj3wUuDMtr9v0xz+2pIvA0eOnTAH3teeUF/Rfn3DvIuetiHOqitNsySn00yF/4VR9yINi3sekqTO3PPQNinJR2lu0dvrf1fVJ0fRj7S1MTwkSZ152EqS1JnhIUnqzPCQJHVmeEiSOjM8JEmd/X9m+fWrwq+77wAAAABJRU5ErkJggg==\n",
"text/plain": [
"<Figure size 432x288 with 1 Axes>"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"# PLS Regression - Cross-Validation with test MSE\n",
"\n",
"results = pd.DataFrame()\n",
"predictors = college_dat.drop('Apps', axis=1).columns\n",
"c_range = range(1, len(predictors) + 1)\n",
"for train_idx, test_idx in model_selection.KFold(n_splits=10).split(college_dat):\n",
" train, test = college_dat.iloc[train_idx], college_dat.iloc[test_idx]\n",
" errors = []\n",
" for c in c_range:\n",
" reg = cross_decomposition.PLSRegression(n_components=c)\n",
" reg.fit(train.drop('Apps', axis=1), train['Apps'])\n",
" error = np.sqrt(metrics.mean_squared_error(test['Apps'], reg.predict(test.drop('Apps', axis=1))))\n",
" errors.append(error)\n",
" results = pd.concat([results, pd.DataFrame(errors, index=c_range).T], axis=0, ignore_index=True)\n",
"\n",
"df = pd.DataFrame({'n_components' : c_range, 'RMSE' : results.mean()})\n",
"sns.barplot(x='n_components', y='RMSE', data=df)"
]
},
{
"cell_type": "code",
"execution_count": 72,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"RMSE: 1098.9586217927474\n"
]
}
],
"source": [
"test_error_pls = df.sort_values('RMSE', ascending=True).iloc[0]['RMSE']\n",
"print('RMSE: {}'.format(test_error_pls))"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"(g) Comment on the results obtained. How accurately can we predict the number of college applications received? Is there much difference among the test errors resulting from these five approaches?"
]
},
{
"cell_type": "code",
"execution_count": 73,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"<matplotlib.axes._subplots.AxesSubplot at 0x119abc518>"
]
},
"execution_count": 73,
"metadata": {},
"output_type": "execute_result"
},
{
"data": {
"image/png": "iVBORw0KGgoAAAANSUhEUgAAAY4AAAEKCAYAAAAFJbKyAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAE0ZJREFUeJzt3X+0XWV95/H3p4mIyo/w45aFAQwzMu04/sTIj8VoncYqWJ0w1lqtLSnDrEy70FoRLbYusZ3apcu2tlqHVRQktNTRhVqwZalMUCk6UBKkJICWLCgmWQixID+ktiDf+eM8McdAmvsk9559b/J+rXXW3fvZz9nne54c7of97H32TVUhSdJ0/djQBUiS5heDQ5LUxeCQJHUxOCRJXQwOSVIXg0OS1MXgkCR1MTgkSV0MDklSl4VDFzAbDj300FqyZMnQZUjSvLJ27drvVNXUzvrtkcGxZMkS1qxZM3QZkjSvJLlzOv2cqpIkdTE4JEldDA5JUheDQ5LUxeCQJHUxOCRJXQwOSVIXg0OS1MXgkCR12SO/Of5veeHbLx66hFmx9gOnDV2C9hBfeclPDV3CjPupq7+yS8/707d9boYrGd6b/vDVu70PjzgkSV32uiMObfOt333O0CXMiqPevW7oEqQ9mkcckqQuBockqYtTVRJw0odPGrqEWfHVN3916BK0B/KIQ5LUxeCQJHUxOCRJXQwOSVIXg0OS1MXgkCR1MTgkSV0MDklSF4NDktRl1oIjyYVJ7kmyfqzt4CRXJrmt/TyotSfJh5JsSHJTkmPHnrOi9b8tyYrZqleSND2zecRxEXDydm3nAKur6hhgdVsHOAU4pj1WAufBKGiAc4HjgeOAc7eGjSRpGLMWHFV1NXDvds3LgVVteRVw6lj7xTVyLbAoyeHAK4Arq+reqroPuJLHh5EkaYImfY7jsKq6qy1/GzisLS8GNo7129TadtT+OElWJlmTZM2WLVtmtmpJ0g8NdnK8qgqoGdzf+VW1tKqWTk1NzdRuJUnbmXRw3N2moGg/72ntm4Ejx/od0dp21C5JGsikg+NyYOuVUSuAy8baT2tXV50A3N+mtL4AvDzJQe2k+MtbmyRpILP2h5ySfAJ4KXBokk2Mro56H/CpJGcAdwKva92vAF4JbAAeBk4HqKp7k/wv4PrW73eravsT7pKkCZq14KiqN+xg07In6FvAmTvYz4XAhTNYmiRpN/jNcUlSF4NDktTF4JAkdTE4JEldDA5JUheDQ5LUxeCQJHUxOCRJXQwOSVIXg0OS1MXgkCR1MTgkSV0MDklSF4NDktTF4JAkdTE4JEldDA5JUheDQ5LUxeCQJHUxOCRJXQwOSVIXg0OS1MXgkCR1MTgkSV0MDklSF4NDktTF4JAkdTE4JEldBgmOJG9NcnOS9Uk+kWTfJEcnuS7JhiSfTLJP6/vktr6hbV8yRM2SpJGJB0eSxcCvA0ur6tnAAuD1wPuBD1bVM4H7gDPaU84A7mvtH2z9JEkDGWqqaiHwlCQLgacCdwE/DVzatq8CTm3Ly9s6bfuyJJlgrZKkMRMPjqraDPwB8C1GgXE/sBb4blU92rptAha35cXAxvbcR1v/QyZZsyRpmyGmqg5idBRxNPB04GnAyTOw35VJ1iRZs2XLlt3dnSRpB4aYqnoZcEdVbamqR4DPACcBi9rUFcARwOa2vBk4EqBtPxD4p+13WlXnV9XSqlo6NTU12+9BkvZaQwTHt4ATkjy1natYBtwCfAl4beuzArisLV/e1mnbr6qqmmC9kqQxQ5zjuI7RSe4bgHWthvOB3wTOSrKB0TmMC9pTLgAOae1nAedMumZJ0jYLd95l5lXVucC52zXfDhz3BH2/D/z8JOqSJO2c3xyXJHUxOCRJXQwOSVIXg0OS1MXgkCR1MTgkSV0MDklSF4NDktTF4JAkdTE4JEldDA5JUheDQ5LUxeCQJHUxOCRJXQwOSVIXg0OS1MXgkCR1MTgkSV0MDklSF4NDktTF4JAkdTE4JEldDA5JUheDQ5LUxeCQJHUxOCRJXQwOSVKXaQVHRn4pybvb+lFJjpvd0iRJc9F0jzj+N3Ai8Ia2/iDwkVmpSJI0p003OI6vqjOB7wNU1X3APrv6okkWJbk0yTeS3JrkxCQHJ7kyyW3t50Gtb5J8KMmGJDclOXZXX1eStPumGxyPJFkAFECSKeCx3XjdPwE+X1U/CTwPuBU4B1hdVccAq9s6wCnAMe2xEjhvN15XkrSbphscHwI+C/x4kvcC1wC/vysvmORA4CXABQBV9a9V9V1gObCqdVsFnNqWlwMX18i1wKIkh+/Ka0uSdt/C6XSqqkuSrAWWAQFOrapbd/E1jwa2AB9P8jxgLfAW4LCquqv1+TZwWFteDGwce/6m1nYXkqSJm+5VVf8euKOqPgKsB34myaJdfM2FwLHAeVX1AuB7bJuWAqCqijYtNl1JViZZk2TNli1bdrE0SdLOTHeq6tPAD5I8E/gz4EjgL3fxNTcBm6rqurZ+KaMguXvrFFT7eU/bvrm93lZHtLYfUVXnV9XSqlo6NTW1i6VJknZmusHxWFU9CrwG+NOqejuwS+cZqurbwMYkP9GalgG3AJcDK1rbCuCytnw5cFq7uuoE4P6xKS1J0oRN6xwHo6uq3gCcBry6tT1pN173zcAlSfYBbgdOZxRin0pyBnAn8LrW9wrglcAG4OHWV5I0kOkGx+nArwLvrao7khwN/PmuvmhV3QgsfYJNy56gbwFn7uprSZJm1nSvqroF+PWx9TuA989WUZKkuWu6V1W9KsnXk9yb5IEkDyZ5YLaLkyTNPdOdqvpjRifG17WpI0nSXmq6V1VtBNYbGpKk6R5xvAO4IslXgH/Z2lhVfzQrVUmS5qzpBsd7gYeAfdmNu+JKkua/6QbH06vq2bNaiSRpXpjuOY4rkrx8ViuRJM0LOw2OJAHOBj6f5J+9HFeS9m47naqqqkpyi1NVkiSY/lTV2iQvmtVKJEnzwnRPjh8PvDHJnYz+fkYYHYw8d9YqkyTNSdMNjlfMahWSpHljujc5vHO2C5EkzQ/TPcchSRJgcEiSOhkckqQuBockqYvBIUnqYnBIkroYHJKkLgaHJKmLwSFJ6mJwSJK6GBySpC4GhySpi8EhSepicEiSuhgckqQugwVHkgVJvp7kr9v60UmuS7IhySeT7NPan9zWN7TtS4aqWZI07BHHW4Bbx9bfD3ywqp4J3Aec0drPAO5r7R9s/SRJAxkkOJIcAfws8LG2HuCngUtbl1XAqW15eVunbV/W+kuSBjDUEccfA+8AHmvrhwDfrapH2/omYHFbXgxsBGjb72/9f0SSlUnWJFmzZcuW2axdkvZqEw+OJK8C7qmqtTO536o6v6qWVtXSqampmdy1JGnMwgFe8yTgvyZ5JbAvcADwJ8CiJAvbUcURwObWfzNwJLApyULgQOCfJl+2JAkGOOKoqndW1RFVtQR4PXBVVb0R+BLw2tZtBXBZW768rdO2X1VVNcGSJUlj5tL3OH4TOCvJBkbnMC5o7RcAh7T2s4BzBqpPksQwU1U/VFVfBr7clm8HjnuCPt8Hfn6ihUmSdmguHXFIkuYBg0OS1MXgkCR1MTgkSV0MDklSF4NDktTF4JAkdTE4JEldDA5JUheDQ5LUxeCQJHUxOCRJXQwOSVIXg0OS1MXgkCR1MTgkSV0MDklSF4NDktTF4JAkdTE4JEldDA5JUheDQ5LUxeCQJHUxOCRJXQwOSVIXg0OS1MXgkCR1MTgkSV0mHhxJjkzypSS3JLk5yVta+8FJrkxyW/t5UGtPkg8l2ZDkpiTHTrpmSdI2QxxxPAq8raqeBZwAnJnkWcA5wOqqOgZY3dYBTgGOaY+VwHmTL1mStNXEg6Oq7qqqG9ryg8CtwGJgObCqdVsFnNqWlwMX18i1wKIkh0+4bElSM+g5jiRLgBcA1wGHVdVdbdO3gcPa8mJg49jTNrU2SdIABguOJPsBnwZ+o6oeGN9WVQVU5/5WJlmTZM2WLVtmsFJJ0rhBgiPJkxiFxiVV9ZnWfPfWKaj2857Wvhk4cuzpR7S2H1FV51fV0qpaOjU1NXvFS9JeboirqgJcANxaVX80tulyYEVbXgFcNtZ+Wru66gTg/rEpLUnShC0c4DVPAn4ZWJfkxtb2W8D7gE8lOQO4E3hd23YF8EpgA/AwcPpky5UkjZt4cFTVNUB2sHnZE/Qv4MxZLUqSNG1+c1yS1MXgkCR1MTgkSV0MDklSF4NDktTF4JAkdTE4JEldDA5JUheDQ5LUxeCQJHUxOCRJXQwOSVIXg0OS1MXgkCR1MTgkSV0MDklSF4NDktTF4JAkdTE4JEldDA5JUheDQ5LUxeCQJHUxOCRJXQwOSVIXg0OS1MXgkCR1MTgkSV0MDklSF4NDktRl3gRHkpOTfDPJhiTnDF2PJO2t5kVwJFkAfAQ4BXgW8IYkzxq2KknaO82L4ACOAzZU1e1V9a/A/wGWD1yTJO2V5ktwLAY2jq1vam2SpAlLVQ1dw04leS1wclX9j7b+y8DxVfWmsT4rgZVt9SeAb0680Mc7FPjO0EXMEY7FNo7FNo7FNnNhLJ5RVVM767RwEpXMgM3AkWPrR7S2H6qq84HzJ1nUziRZU1VLh65jLnAstnEstnEstplPYzFfpqquB45JcnSSfYDXA5cPXJMk7ZXmxRFHVT2a5E3AF4AFwIVVdfPAZUnSXmleBAdAVV0BXDF0HZ3m1NTZwByLbRyLbRyLbebNWMyLk+OSpLljvpzjkCTNEQbHLEny0NA1TFKSK5IseoL29yQ5e4iaJmVv+7fWzEvy5STz4ooqmEfnODR3JQnwqqp6bOhatOdIsrCqHh26Dj2eRxwzIMlZSda3x29st+3wJFcnubFtf/FQdc6kJEvaTScvBtYDP0hyaNv220n+Ick1jL6MufU5L0pyUxuLDyRZ39oXtPXr2/b/Ocib2k1J9kuyOskNSdYlWd7an5bkb5L8ffsM/EJrf1+SW9p7/oPWtiTJVa1tdZKjhnxPM6G9p28kuSTJrUkuTfLU9nn4WhuXv0uyf5JfSXJ5kquA1UPXPtN2NBZj2xckuah9TtYleeuQ9e5QVfnYjQfwQmAd8DRgP+Bm4AXAQ23724DfbssLgP2HrnmG3vcS4DHghLb+j4y++bp1PJ4KHABsAM5ufdYDJ7bl9wHr2/JK4F1t+cnAGuDood9jx1hs/bdeCBzQlg9t7z3AzwEfHet/IHAIo7sbbL1AZVH7+TlgRVv+78BfDf3+ZuizUsBJbf1C4B3A7cCLWtsBbfx+hdEthQ4euu4JjsXZwJeBpe2/nyvH+i8auuYnenjEsfv+M/DZqvpeVT0EfAYYP6q4Hjg9yXuA51TVgwPUOFvurKprt2t7MaPxeLiqHqB9UbOd/9i/qv5f6/eXY895OXBakhuB6xj9Uj1mdkufFQF+P8lNwP9ldD+1wxgF6c8keX+SF1fV/cD9wPeBC5K8Bni47eNEto3NnzP6fO0JNlbVV9vyXwCvAO6qqusBquqB2jYtdWVV3TtEkROy/ViM/xvfDvy7JB9OcjLwwMSrmwaDY5ZV1dXASxjdIuWiJKcNXNJM+t4M7SfAm6vq+e1xdFV9cYb2PUlvBKaAF1bV84G7gX2r6h+AYxkFyO8leXf7JXkccCnwKuDzA9U8Kdtf9/9v/UKcqc/VXLX9WPxwvaruA57H6AjkV4GPTa6s6TM4dt/fAqe2OdunAf+ttQGQ5BnA3VX1UUYfgmOHKXNirmY0Hk9Jsj/waoCq+i7wYJLjW7/Xjz3nC8CvJXkSQJL/0MZyvjkQuKeqHknyX4BnACR5OvBwVf0F8AHg2CT7AQfW6Iutb2X0ywLga2wbmzcy9lma545KcmJb/kXgWuDwJC8CaOc39paLdbYfi2u2bmjnCX+sqj4NvIs5+vtib/mHmjVVdUOSi4C/a00fq6qvjy40AuClwNuTPAI8BOxJRxyP08bjk8DfA/cwmqrb6gzgo0keA77CaLoGRoG6BLihXaG1BTh1YkXPnEuAzyVZx+g8zTda+3OAD7T3/Qjwa8D+wGVJ9mV0xHVW6/tm4ONJ3s5oHE6fYP2z6ZvAmUkuBG4BPgxcBXw4yVOAfwZeNmB9k7T9WJxH+x8sRtObH0+y9X/q3zlAfTvlN8c1MUn2a+eByOjP/x5eVW8ZuCzNsiRLgL+uqmcPXMrg9pSx8IhDk/SzSd7J6HN3J6MraCTNMx5xSJK6eHJcktTF4JAkdTE4JEldDA5pYEn+cet9vnanjzQpBockqYvBIe2CsbucXtTuBHxJkpcl+WqS25Icl+TgJH/V7nR7bZLntucekuSLSW5O8jFGXwDcut9faneKvTHJnyVZMNiblHbA4JB23TOBPwR+sj1+kdEN684Gfgv4HeDrVfXctn5xe965wDVV9Z+AzwJHAST5j8AvMLpz6vOBHzC67Yg0p/gFQGnX3VFV6wCS3AysrqpqtxxZwuheVT8HUFVXtSONAxjd9PI1rf1vktzX9reM0W21r2+3rHkKo9u2SHOKwSHtun8ZW35sbP0xRv9tPdK5vwCrqmpO3p9I2sqpKmn2/C1tqinJS4HvtL9RcjWjaS2SnAIc1PqvBl6b5MfbtoPb3ZWlOcUjDmn2vAe4sP1hp4eBFa39d4BPtOmtrwHfAqiqW5K8C/hiuzvqI8CZjO7rJc0Z3qtKktTFqSpJUheDQ5LUxeCQJHUxOCRJXQwOSVIXg0OS1MXgkCR1MTgkSV3+PySRmeFVvmEoAAAAAElFTkSuQmCC\n",
"text/plain": [
"<Figure size 432x288 with 1 Axes>"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"summary = pd.DataFrame({'rmse' : [test_error_ols, test_error_ridge, test_error_lasso, test_error_pcr, test_error_pls], \\\n",
" 'model': ['ols', 'ridge', 'lasso', 'pcr', 'pls']})\n",
"\n",
"sns.barplot(x='model', y= 'rmse', data=summary)"
]
},
{
"cell_type": "code",
"execution_count": 75,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>rmse</th>\n",
" <th>model</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>1070.514298</td>\n",
" <td>lasso</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>1085.920717</td>\n",
" <td>ridge</td>\n",
" </tr>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>1097.813600</td>\n",
" <td>ols</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>1098.958622</td>\n",
" <td>pls</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>1098.961561</td>\n",
" <td>pcr</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" rmse model\n",
"2 1070.514298 lasso\n",
"1 1085.920717 ridge\n",
"0 1097.813600 ols\n",
"4 1098.958622 pls\n",
"3 1098.961561 pcr"
]
},
"execution_count": 75,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"summary.sort_values('rmse', ascending=True)"
]
},
{
"cell_type": "code",
"execution_count": 80,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"3001.6383526383524"
]
},
"execution_count": 80,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"college_dat['Apps'].mean()"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"10. We have seen that as the number of features used in a model increases, the training error will necessarily decrease, but the test error may not. We will now explore this in a simulated data set.\n"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"(a) Generate a data set with p = 20 features, n = 1,000 observations, and an associated quantitative response vector generated according to the model\n",
"Y =Xβ+ε,\n",
"where β has some elements that are exactly equal to zero."
]
},
{
"cell_type": "code",
"execution_count": 302,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>x0</th>\n",
" <th>x1</th>\n",
" <th>x2</th>\n",
" <th>x3</th>\n",
" <th>x4</th>\n",
" <th>x5</th>\n",
" <th>x6</th>\n",
" <th>x7</th>\n",
" <th>x8</th>\n",
" <th>x9</th>\n",
" <th>...</th>\n",
" <th>x11</th>\n",
" <th>x12</th>\n",
" <th>x13</th>\n",
" <th>x14</th>\n",
" <th>x15</th>\n",
" <th>x16</th>\n",
" <th>x17</th>\n",
" <th>x18</th>\n",
" <th>x19</th>\n",
" <th>y</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>3.783240</td>\n",
" <td>2.822764</td>\n",
" <td>2.104245</td>\n",
" <td>0.085151</td>\n",
" <td>8.070863</td>\n",
" <td>9.613367</td>\n",
" <td>3.890487</td>\n",
" <td>7.576291</td>\n",
" <td>5.795171</td>\n",
" <td>3.736745</td>\n",
" <td>...</td>\n",
" <td>9.745869</td>\n",
" <td>1.079770</td>\n",
" <td>3.823860</td>\n",
" <td>5.726278</td>\n",
" <td>0.746411</td>\n",
" <td>3.317932</td>\n",
" <td>3.490703</td>\n",
" <td>2.305381</td>\n",
" <td>0.182134</td>\n",
" <td>14.017323</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>3.872013</td>\n",
" <td>1.235886</td>\n",
" <td>5.053738</td>\n",
" <td>0.075068</td>\n",
" <td>3.522952</td>\n",
" <td>8.167472</td>\n",
" <td>5.532001</td>\n",
" <td>5.415793</td>\n",
" <td>8.896883</td>\n",
" <td>1.439292</td>\n",
" <td>...</td>\n",
" <td>8.012286</td>\n",
" <td>1.586732</td>\n",
" <td>4.754267</td>\n",
" <td>7.819763</td>\n",
" <td>3.792867</td>\n",
" <td>0.562013</td>\n",
" <td>9.079879</td>\n",
" <td>0.633988</td>\n",
" <td>0.269867</td>\n",
" <td>19.181412</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>2.868493</td>\n",
" <td>4.097188</td>\n",
" <td>5.897901</td>\n",
" <td>2.579025</td>\n",
" <td>0.174440</td>\n",
" <td>2.931412</td>\n",
" <td>9.650473</td>\n",
" <td>6.266575</td>\n",
" <td>1.599330</td>\n",
" <td>4.434026</td>\n",
" <td>...</td>\n",
" <td>0.434932</td>\n",
" <td>7.079660</td>\n",
" <td>4.791268</td>\n",
" <td>6.003995</td>\n",
" <td>5.510795</td>\n",
" <td>8.790729</td>\n",
" <td>7.049488</td>\n",
" <td>2.820926</td>\n",
" <td>9.893762</td>\n",
" <td>10.489754</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>4.747117</td>\n",
" <td>9.260321</td>\n",
" <td>7.629206</td>\n",
" <td>9.956572</td>\n",
" <td>4.509392</td>\n",
" <td>8.280505</td>\n",
" <td>9.809595</td>\n",
" <td>7.245737</td>\n",
" <td>2.299474</td>\n",
" <td>6.779702</td>\n",
" <td>...</td>\n",
" <td>7.811970</td>\n",
" <td>1.374622</td>\n",
" <td>0.747643</td>\n",
" <td>6.002939</td>\n",
" <td>0.019969</td>\n",
" <td>3.797922</td>\n",
" <td>4.139976</td>\n",
" <td>1.060459</td>\n",
" <td>2.778484</td>\n",
" <td>9.574987</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>2.116980</td>\n",
" <td>4.778625</td>\n",
" <td>0.165573</td>\n",
" <td>5.720200</td>\n",
" <td>3.273290</td>\n",
" <td>9.592775</td>\n",
" <td>8.044132</td>\n",
" <td>8.663585</td>\n",
" <td>9.080542</td>\n",
" <td>1.654905</td>\n",
" <td>...</td>\n",
" <td>9.807599</td>\n",
" <td>6.937180</td>\n",
" <td>5.899757</td>\n",
" <td>9.969896</td>\n",
" <td>2.606421</td>\n",
" <td>5.331222</td>\n",
" <td>3.010509</td>\n",
" <td>1.172996</td>\n",
" <td>8.545720</td>\n",
" <td>26.689338</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"<p>5 rows × 21 columns</p>\n",
"</div>"
],
"text/plain": [
" x0 x1 x2 x3 x4 x5 x6 \\\n",
"0 3.783240 2.822764 2.104245 0.085151 8.070863 9.613367 3.890487 \n",
"1 3.872013 1.235886 5.053738 0.075068 3.522952 8.167472 5.532001 \n",
"2 2.868493 4.097188 5.897901 2.579025 0.174440 2.931412 9.650473 \n",
"3 4.747117 9.260321 7.629206 9.956572 4.509392 8.280505 9.809595 \n",
"4 2.116980 4.778625 0.165573 5.720200 3.273290 9.592775 8.044132 \n",
"\n",
" x7 x8 x9 ... x11 x12 x13 \\\n",
"0 7.576291 5.795171 3.736745 ... 9.745869 1.079770 3.823860 \n",
"1 5.415793 8.896883 1.439292 ... 8.012286 1.586732 4.754267 \n",
"2 6.266575 1.599330 4.434026 ... 0.434932 7.079660 4.791268 \n",
"3 7.245737 2.299474 6.779702 ... 7.811970 1.374622 0.747643 \n",
"4 8.663585 9.080542 1.654905 ... 9.807599 6.937180 5.899757 \n",
"\n",
" x14 x15 x16 x17 x18 x19 y \n",
"0 5.726278 0.746411 3.317932 3.490703 2.305381 0.182134 14.017323 \n",
"1 7.819763 3.792867 0.562013 9.079879 0.633988 0.269867 19.181412 \n",
"2 6.003995 5.510795 8.790729 7.049488 2.820926 9.893762 10.489754 \n",
"3 6.002939 0.019969 3.797922 4.139976 1.060459 2.778484 9.574987 \n",
"4 9.969896 2.606421 5.331222 3.010509 1.172996 8.545720 26.689338 \n",
"\n",
"[5 rows x 21 columns]"
]
},
"execution_count": 302,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"n = 100\n",
"p = 20\n",
"x = np.random.uniform(0,10,(n, p))\n",
"eps = np.random.normal(0, 1, n)\n",
"beta = np.random.normal(0, 1, p)\n",
"beta[0] = 0\n",
"beta[1] = 0\n",
"beta[2] = 0\n",
"beta[3] = 0\n",
"beta[4] = 0\n",
"beta[5] = 0\n",
"beta[6] = 0\n",
"beta[7] = 0\n",
"beta[8] = beta[9] ** 2\n",
"beta[10] = beta[11] ** 2\n",
"\n",
"y = (x @ beta.reshape(-1, 1)) + eps.reshape(-1, 1)\n",
"data = pd.DataFrame(x, columns= ['x' + str(i) for i in range(0, x.shape[1])])\n",
"data['y'] = y\n",
"data.head()"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"(b) Split your dataset into a training set containing 100 observations and a test set containing 900 observations."
]
},
{
"cell_type": "code",
"execution_count": 303,
"metadata": {},
"outputs": [],
"source": [
"train, test = model_selection.train_test_split(data, test_size=0.9)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"(c) Perform best subset selection on the training set, and plot the training set MSE associated with the best model of each size."
]
},
{
"cell_type": "code",
"execution_count": 304,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"0.0006680865036813836"
]
},
"execution_count": 304,
"metadata": {},
"output_type": "execute_result"
},
{
"data": {
"image/png": "iVBORw0KGgoAAAANSUhEUgAAAYoAAAD8CAYAAABpcuN4AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3Xl8W/Wd7//XR5u3xEtM9thZSAJlKUtMEmibprRQ6LQEWkrCmt5hGnrbPNrOvTNTOp0yKTO/3w/amc6lF25bClxIWAJNobgUSimB0iWbE8IStoSQxA7Z4yWxZWv7/P44R7YsS7K8y/bn+Xj4YenonKPvkeTz9tF3E1XFGGOMSccz1AUwxhiT2ywojDHGZGRBYYwxJiMLCmOMMRlZUBhjjMnIgsIYY0xGFhTGGGMysqAwxhiTkQWFMcaYjHxDXYD+cMopp+iMGTOGuhjGGDOsbN269aiqju9uvRERFDNmzKCmpmaoi2GMMcOKiOzNZj376skYY0xGFhTGGGMysqAwxhiTUVZBISKXici7IrJLRG5N8XieiDzuPr5JRGa4yy8Rka0i8ob7++KEbea5y3eJyE9ERNzl40TkBRHZ6f4u659DNcYY0xvdBoWIeIF7gMuBM4BrReSMpNVuBupVdTbwX8Cd7vKjwBdU9WxgObAmYZufAl8F5rg/l7nLbwVeVNU5wIvufWOMMUMkmyuK+cAuVd2tqiFgLbAkaZ0lwEPu7XXAp0VEVPVVVf3QXb4DKHCvPiYDxaq6UZ2Zk1YDV6bY10MJy40xxgyBbIJiKlCbcL/OXZZyHVWNAI1AedI6XwK2qWqbu35dmn1OVNUD7u2DwMQsymiMMWaADEo/ChE5E+frqEt7sp2qqoiknKtVRFYAKwAqKyv7XEZjjDGpZXNFsR+oSLg/zV2Wch0R8QElwDH3/jTgKeAmVX0/Yf1pafZ5yP1qCvf34VSFUtV7VbVKVavGj++2Y6Exxowo4WiME63hQXmubIJiCzBHRGaKSABYBlQnrVONU1kNcDWw3r0aKAV+C9yqqn+Jr+x+tdQkIgvd1k43AU+n2NfyhOXGGDPqqSr1zSHq6oMEQ9FBec5ug8Ktc1gJPA+8DTyhqjtE5HYRucJd7X6gXER2Af+DjpZKK4HZwG0ist39meA+9nXgPmAX8D7wnLv8DuASEdkJfMa9b4wxo97Jtgi1x4PUt4Rw2gENDhnMJxsoVVVVamM9GWNGqrZIlGMnQ7SGO19BjMnzMaE4v9f7FZGtqlrV3XojYlBAY4wZiaIx5XhzaNDqItKxoDDGmByjqjQFI9S3hIjlwLc+FhTGGJNDWkIRjp0MEY7Ghroo7SwojDEmB4QiMY43h2gJRYa6KF1YUBhjzBCKxZT6lhBNrZFBbcnUExYUxhgzRJpaw9Q3h4jGcjMg4iwojDFmkLWGoxxrDtEWHpwOc31lQWGMMYMkEo1xvCXEydbcq4fIxILCGGMGWK41d+0pmwrVGGMGUEsoQl19kGPNbf0aEtGY8vsdB3l1X32/7TMdu6IwxpgBMFDNXaMx5eV3j7Bm4172HW/hwlnlPLZiYb8+RzILCmOM6UdRt7nriX5u7uoExGHWbNzHvuMt7cvLivy0hqPk+7399lzJLCiMMaafNAbDNLT0b3PXaEx56d3DrNmwl9r6YPvyRXNP4WuLTuWi2af023OlY0FhjDF9FAxFOdbcRijSf8NuRGPK+ncO8/DGzgHxybnjuXFhJbPGj2FM3uCcwi0ojDGml8JRpx6iua3/6iHiAbFm417q3IAQYNHc8dx04XRmnlLUb8+VLQsKY4zpoVhMaQiGaQyG+60eIhpTXnSvIBID4pNzx3PjEAVEXFZBISKXAXcBXuA+Vb0j6fE8YDUwD2eu7KWqukdEyoF1wAXAg6q60l1/LPCnhF1MAx5W1W+LyFeAH9Exh/bdqnpfL4/PGGP6VVNrmIbmMJFY/3zNFI0pL759iDUb97G/IbcCIq7boBARL3APcAlQB2wRkWpVfSthtZuBelWdLSLLgDuBpUAr8H3gLPcHAFU9AZyb8BxbgScT9vd4PFSMMSYX9Hc9RDSm/OHtQzycFBCLT3MCYkb50AdEXDZXFPOBXaq6G0BE1gJLgMSgWAKscm+vA+4WEVHVZuDPIjI73c5FZC4wgc5XGMYYkxP6uz9EPCDWbNzLhw2tgBMQnzp9AjcsrMypgIjLJiimArUJ9+uABenWUdWIiDQC5cDRLPa/DOcKIvGLvi+JyCLgPeDvVbU29abGGDMw+rs/RDSmvPDWIR7e1DkgFp/mVFJP72FA5Pu9FBf4+1yubORCZfYy4MaE+78BHlPVNhG5BXgIuDh5IxFZAawAqKysHIxyGmNGif4c/jtdQHzq9AncuLCyxwExJs9HcYF/QDvYJcsmKPYDFQn3p9FR0Zy8Tp2I+IASnErtjETkHMCnqlvjy1Q1cbv7gB+m2lZV7wXuBaiqqhp+o2wZY3JOf9ZDRKIxXnj7MI8kBIRH4OLTJ3DDgulUlhdmvS+PCGPyfZQU+PF7B3+IvmyCYgswR0Rm4gTCMuC6pHWqgeXABuBqYL1md612LfBY4gIRmayqB9y7VwBvZ7EfY4zptVAkRn1L//SHiERj7hXEPg40JgXEwulUjss+ILweoTjfT3GBH69H+ly23uo2KNw6h5XA8zjNYx9Q1R0icjtQo6rVwP3AGhHZBRzHCRMARGQPUAwERORK4NKEFlPXAJ9LespvisgVQMTd11f6cHzGGJNWf05DGonG+P1bh3ikHwLC5/FQUuinON+HyNAFRJzk6hytPVFVVaU1NTVDXQxjzDByojVMfT/0h4gHxMMb93GwqSMgPv2RiVy/oLJHAeH3OgExNm9wAkJEtqpqVXfr5UJltjHGDJr+moY0Eo3x/I5DPLq56xXEjQunU9GDgMj3eykp8FM0SGM39VRulsoYM+q8/M5hfv7KbmrrW6goK+SWRbNYfPqEftt/NKYcbw5xojXcp/2E4wGxqfMVxGfcK4ieBMSYPB/b9zXwf/+6Z8COuz9YUBhjhtzL7xzmtuod+L1CaYGfwydaua16B7dDn0+a/TUNaTwgHtm0l0NNbYATEJec4QTEtLLsAsIjwli3BdOfdx7l3599e0COuz9ZUBhjhtzPX9mN3ysUBpxTUmHAR0sows9f2d2nE2ZLKMKxkyHC0d7XQzgBcZCHN+7j8ImOgPjMRyZyw8LsA8Lv9VBc4NQ/eNwWTAN13P3NgsIYM+Rq61soTeplXOD3UlffkmaLzPpj2I1wNMbv3jzII5s6B8QlZ0zkhgXTmVpWkNV+8tz6h1RzR/T3cQ8UCwpjzJCrKCvk8InW9v+sAYLhaNb/rYPz9dXP/vg+e4+3MLE4n2VVFcyfNa7HZUkXEJeeMYnrF1YytTS7gCgMOF8vFQTS96Duj+MeDBYUxpghd8uiWdxWvYOWUIQCv5dgOEo4qtyyaFZW27/09iH+5ekdeD0wJs/LsZNt3LV+J99iTtZhEYrE+N2OgzyaFBCfPXMS1y3IPiDG5PkoKfST5+t+iI2+HvdgsaAwxgy5xadP4Hac7+zr6luY1oPWPydaw/xk/S48AvnuyTl+0l27pbbboAhFYjz35kEe29w1IK5fUMmULAJCEiqoezLERl+OezBZUBhjcsLi0yf06ASZOC7Th41BivM7n87y/R4ONgXTbN0REI9u2seRkx0BcZl7BZFNQPTHEBs9Pe6hYEFhjBlWUlVUTy4u4FhzGwUJI6q2hmNMKu56sncC4gCPbqptDwivR7jUbeaaTUD4PB5KCvyMze9owTSSWVAYY4aFSDRGfUs4ZYe5ZRdUcNf6nQTDUfL9HlrDMSIxZdkFHQNfhyIxnn3jAI9u3sfRkyHACYjPnukExOSS7gPC7/VQWui0YIoPsTHQHQVzgQWFMSanxWJKYzBMYzCctsPc/Fnj+BZzWLulloNNQSYVF7DsAqfVU6aAuGHBdCaV5HdbhnRDbAxkR8FcYkFhjMlZTa1hGrIcuG/+rHGdKq5DkRhPvbqfRzfv41hCQFx+1iSum1+ZVUAUBnyUFqafJGi4dJjrKwsKY0zO6UuP6lAkxm/dK4jeBISIUJTnpbQgQMCXuQXTcOkw11cWFMaYnNEWiXK8OUQw1PORXUORGM+8foDHtnQEhM8jXJZlQCSOweTLsonrcOkw11cWFMaYIReJxjjeEuJka8+H3GgLR/ntGwd4bHMtx5qTAmJBJZOKMweE1yO8UdvIw5v2UtcQ7FGF9HDpMNdXFhTGmCETiykNbkV1TydRawtHeeaNA6xNCojLz3auICZ2ExDxQfq27TnOD3//bq8qpIdLh7m+yiooROQy4C6cqVDvU9U7kh7PA1YD84BjwFJV3SMi5cA64ALgQVVdmbDNy8BkIN4j5lJVPZxuX70+QmNMTurtDHNt4Si/ef0Aa7fUcjwhID539mSunV+RVUAkNnG9908f9KlCejh0mOurboNCRLzAPcAlQB2wRUSqE+a9BrgZqFfV2SKyDLgTWAq0At8HznJ/kl2vqslzmKbblzFmBOjtDHOtbkA8nhAQfq/wubOcgJjQTUDk+b2UpmjiOloqpPsimyuK+cAuVd0NICJrgSVAYlAsAVa5t9cBd4uIqGoz8GcRmd2DMqXb1/Cf3NuYUSwcjVHfHOJkW8/qIeIBsXbzPupbnM52PQmIgoDTgindKK6jpUK6L7IJiqlAbcL9OmBBunVUNSIijUA5cLSbff9fEYkCvwL+3Q2DrPYlIiuAFQCVlZVZHIYxZqCl6qW8aO74XtVDtIaj/Oa1D1m7pbZzQJw9mevmVzJ+bF7G7YvynBZM6fpAxI2WCum+GMrK7OtVdb+IjMUJihtx6iayoqr3AvcCVFVV2dWGMUMsVS/lf/n1m3zz4tlUzcx+Xoh0AfE3Z0/m2m4Coid9IOJGS4V0X2QTFPuBioT709xlqdapExEfUIJTEZ2Wqu53f58QkUdxvuJa3Zt9GWOGXmIv5VhM8Xk8hCTGo5trswqKoBsQjycFxOc/OoVlF1RkDAiPCMUFforzfVn3gUg0Giqk+yKboNgCzBGRmTgn8WXAdUnrVAPLgQ3A1cD6THUKbgCUqupREfEDnwf+0Jt9GWNyQ219C8X5PsLRGLGY8yfb3VDfkD4gsrmC8HqEkgI/xfn+UTGK61DpNijceoKVwPM4zWMfUNUdInI7UKOq1cD9wBoR2QUcxwkTAERkD1AMBETkSuBSYC/wvBsSXpyQ+IW7Sdp9GWNyUyymTCrO51BTa1ZDfYMTEE9v/5AnttTSEOx8BXHt/ApOGZM+IHweDyWFzhVEfBRXM3BkJPyzXlVVpTU1ya1sjTGDoak1TH1ziA27jnHX+p34PNJpqO9vXdx5OtJ0AfGFc5yvmDIFRKphvk3vichWVa3qbj3rmW2M6ZXEGeYg81Df0BEQj2+ppdENiIDPw+c/OrnbgMj3eykt9HdqwmoGj73qxpgeCUedGeaaU/SHSB7qG5xAeXr7fh6vqesSENdeUEF5hoDobphvMzgsKIwxnaSbsS0WU+pbQjS1RrLqDxEMRfn19v08kRQQV5wzmaVV6QMi3sS1pMBPns8CIhdYUBhj2qXqC/H9p9/k1rbTOWtaCdFY9gHx+JZamtzRYOMBseyCSsYVBVJu15thvs3gsKAwxrRLnrEtz+clHI3xiz99wI+XnpNx25ZQhF+/+iFP1HQERJ7Pwxe6CYh4E9ex+X681sQ1J1lQGGPaxQfIi6kSiSqqSp4vc1+IdAFxxTlTWHpBRdqAiA/zbU1cc58FhTGm3dTSAg42BjvVDaTrC9HbgAj4PJQUWBPX4cSCwhhDNKY0tIS46ryp3PXiTmIa7dQXYtkFHaP4NLdF+PX2/fyypq5HAVEQcCqorYnr8GPvmDGjmKrSFIzQEAwRjSnzZ47jWxen7gvR3BbhyVf3s25rHSfcgMj3ebji3ClcU5U+IMbk+SgptBZMw5kFhTGj1Mm2CPXNIcLRzjPMJfeFaG6LsGbj3i4BseTcKVxzQQVlhV0DQhJaMPmtBdOwZ0FhzCiT7QxzJ9siPJXiCiJTQMRHcS0psBZMI4kFhTGjRKYe1YlOtkV4att+frm1rn02uny/hyvPnco1VdMoTREQPo/HbeLqs1FcRyALCmNGmOSe1Ss+MZNzK8to6GaGuZNtEZ7cVse6rfuzDgi/1xnFday1YBrRLCiMGUGSe1YfbAryvV+/yTeTRnBNdLI1wpOv9iwg8vze9iauZuSzd9mYESTeszrf7yUSU/weDxGPsnZLbZegONka4Vfb6li3rY7mNqe+osDv5crzpnDNvApKCv1d9l8QcKYZLQhYC6bRxILCmBFk3/FmxuT5CEc6WjIlzzJ3sjXCum11/CopIK46bwpfThMQY/J8FBfYKK6jVVZBISKXAXfhzEZ3n6rekfR4Hs581/Nw5rdeqqp7RKQcWAdcADyoqivd9QuBXwKnAlHgN6p6q/vYV4Af0TEv992qel9fDtKYkS4WUxqCYcaPzefYybaUs8ylC4gvnj+Vq+dNo6Sgc0CIiNMHosBPwGdNXEezboNCRLzAPcAlQB2wRUSqVfWthNVuBupVdbaILAPuBJYCrcD3gbPcn0T/oaoviUgAeFFELlfV59zHHo+HijEms/gMc9GYsqyqgrvW7yQY7uhZHYrGKB8T4NpfbKQ5lF1AFNsoriZBNlcU84FdqrobQETWAkuAxKBYAqxyb68D7hYRUdVm4M8iMjtxh6raArzk3g6JyDZgWl8OxJiRJN2cEImSZ5iDzrPMfdjYgiA0BMO8+M5hAAoDXq46L3VAWB8Ik042QTEVqE24XwcsSLeOqkZEpBEoB452t3MRKQW+gPPVVtyXRGQR8B7w96pam3JjY0agVHNC3Fa9g9uBxadPIBRx+kO0hFL3hzh98ljOnlbMzsMn2q8gCgPuFcT50yhOCoj4MN/F+X7rA2FSGtLKbBHxAY8BP4lfsQC/AR5T1TYRuQV4CLg4xbYrgBUAlZWVg1RiYwZe8pwQhQEfLaEIP/vj+5w1rYQTaWaYawqGWbetjie37aclISC+dP5UvpQiIKyTnMlWNkGxH6hIuD+Njorm5HXq3JN/CU6ldnfuBXaq6v+KL1DVxO3uA36YakNVvdfdnqqqqu6n3TJmmIjPCRGnqvi9wp5jzTS5U4omagyGWbe1jqde7QiIIvcKIlVAWCc501PZBMUWYI6IzMQJhGXAdUnrVAPLgQ3A1cB67WZSXRH5d5xA+buk5ZNV9YB79wrg7SzKaMyIUVFWyOETrRT4vcQUIrEYwVC0y5wQ6QLiS+dP40vzpjI2v3NA5Pm9lBb4KbJOcqaHuv3EuHUOK4HncZrHPqCqO0TkdqBGVauB+4E1IrILOI4TJgCIyB6gGAiIyJXApUAT8D3gHWCb+19NvBnsN0XkCiDi7usr/XSsxgwLKz4xk+8/vYNwNEaer+ucED0NiHy/l7JC6yRnek+6+cd/WKiqqtKampqhLoYxfXaiNUxDS5i/7DzaZU6I0yaN5Ymttfz61Q8JuiO/FuW5AXF+14AoDPgoLbROciY9EdmqqlXdrWfXoMbkgJZQhGMnO+aGSJwTorElzBNba1n1zA5aw87jRXlerj5/Gl86fxpj8jv/GRflOQFhEwWZ/mJBYcwQaotEOd4cIhjqOjdEQ0uIJ2rq+PX2/e0BMSbPx9XzpvLF87oGxJg8H6WFAetFbfqdBYUxA6C7DnORaIzjLSFOtnbtC9GTgLBhNsxgsKAwpp9l6jC3aO54GoPhlHNDtAfEq/tpjXQExJfnTeOq86d2GtLbkzDVqA2zYQaaBYUx/SxVh7nmtjD3vLSLmeOLiMY6B0R9S4gnttTy9PYP2wNibL6Pq+dN46rzOgeE1yMU5/sptmE2zCCyoDCmnyV3mIvGFJ/HQ219S6eQSBcQX3YDoigpIGyYDTNULCiM6WfxDnP5fi+RqKKqBMMdHebqW0I8vqWW6hRXEF9MERClBQEbZsMMKQsKY/rZzR+bwW2/2UEoEmsf6jsSUz7/0cn87I/v8/T2D2lzA6I438eXq6Zx5bkWECZ3WVAY00/iLZlmTRjDNz81p73DXHlRHuOKAvzo9+92Cohrqiq48rwp7XUZYAFhcpMFhTF9FI0pDS0hmhJGdZ0/axyzJ45xvmJ67UPaDmQOCJ/HGaivON8G6jO5x4LCmF6KxZTGYJjGYJhYQlPX480h1m7ZR/VrB9onFUoXEDaSqxkOLCiM6SHVjoBIbMWULiCWXlDBledO7TQon9/robTQzxgLCDMMWFAYkyVVpSkYoTEYJhLrmH702Mk21m6p5TevdwRESYGfpVXTWJIUEDbUtxmO7NNqTDdUlabWCI0tXQPisS21PJMUENe4rZgSA6Iw4PSitqG+zXBkQWFMBs1tEY43d4zqCnD0ZBtrN9fyzBtJVxAXVLDknCmdwmBMno8SG8nVDHMWFMak0Bp2RnVtDXeM6nrkhPMV0zOvf0g46tRNlLoBccW5Uyhw532ID9RXWujHb+MwmREgq6AQkcuAu3BmuLtPVe9IejwPWA3Mw5kre6mq7hGRcmAdcAHwoKquTNhmHvAgUAA8C3xLVVVExgGPAzOAPcA1qlrfh2M0Jmut4Sj1LZ2H/U4VEGWFTkB84ZyOgPCIUFzgp8TGYTIjTLdBISJe4B7gEqAO2CIi1ar6VsJqNwP1qjpbRJYBdwJLgVbg+8BZ7k+inwJfBTbhBMVlwHPArcCLqnqHiNzq3v9O7w/RmO6FozHqm0OcbOsY9vvIiTYe27yP375xIGNA2DhMZqTL5opiPrBLVXcDiMhaYAmQGBRLgFXu7XXA3SIiqtoM/FlEZifuUEQmA8WqutG9vxq4EicolgCL3VUfAl7GgsIMkFSd5bINCL/XQ3GBdZIzI182QTEVqE24XwcsSLeOqkZEpBEoB45m2Gdd0j6nurcnquoB9/ZBYGIWZTSmR6JuZ7mmhM5yR0608ejmfTybEBA+j5Dv91BRWsiMcUUU+L3k+b2UFPg7Df9tzEiW0590t85CUz0mIiuAFQCVlZWDWi4zfMViSlNrmIaWjoA43NTKY5trefbNjoBwOsI5HeYKA17qgyF+sn4n/1p0Bp89a/JQHoIxgy6boNgPVCTcn+YuS7VOnYj4gBKcSu1M+5yWZp+HRGSyqh5wv6I6nGoHqnovcC9AVVVVyjAxJi7eWa4hGGrvTX24qZVHN9fy3Judv2K6dn4lf37vKPXBEAV+Lx6PUFzgpTUc5cG/7rWgMKNONkGxBZgjIjNxTubLgOuS1qkGlgMbgKuB9Zo8z2MCNwSaRGQhTmX2TcD/TtrXHe7vp7M/HGM6S9VZLh4Qz75xgIgbGuOKAk4dxEcnk+/38qttdZS604x63PqHAr+XuvqWITsWY4ZKt0Hh1jmsBJ7HaR77gKruEJHbgRpVrQbuB9aIyC7gOE6YACAie4BiICAiVwKXui2mvk5H89jn3B9wAuIJEbkZ2Atc0x8HakYXVeVEmxMQ8c5yh5paeXTzPp5742B7QJQXBVg2v4LPnz2ZPL+3vQ/EjPIijp5sI+DrqKQOhqNMKysckuMxZihJhn/8h42qqiqtqakZ6mKYHHHCrYOIB8TBplYe27SP597sHBDXzq/gbxICYmy+r/0q4uV3DnNb9Q78XqHA7yUYjhKOKrdfcSaLT58wlIdnTL8Rka2qWtXdejldmW1MtlJdQRxsauXRTfv4XZeAqOTzH51MwOdJ20lu8ekTuB34+Su7qatvYVpZIbcsmmUhYUYlCwozrGUdEGMCXDe/kr85O3NAJFp8+gQLBmOwoDDDVKpK6oONrTyyaR+/23GwvWVTqoAocQPCelEbkx0LCjOspJoTIlVAnOIGxOcsIIzpMwsKMyyk6gdxoDHII5v28fyOQ50C4tqEK4gtHxznl1vrONAYpHJckdUzGNMLFhQmp8XrIBqaO64gDjQGeWTjPp5/q3NAXL+gksvP6riCeLOukbtf2kXA56GsMMDhE63cVr2D28HCwpgesKAwOSu5mWuqgBg/Jo/rFlRy+VmTunzF9N0n3yDg81AYcD7mhQEfLaEIP39ltwWFMT1gQWFyTnJAfNgQ/4rpIG4+uAFR0ekKIrkOora+hdICf6d9W+9qY3rOgsLkhFTNXPc3OFcQv3+rIyAmjHWuIC47s+sVRHIldUVZIYdPtLZfUYD1rjamNywozJCKN3NtCnYfENcvqOSzWQRE3C2LZnFb9Q5aQpFOvatvWTRrsA7PmBHBgsIMifhw343BcHt9w/76IA9v2ssLbx3KeAWR7XSj1rvamP5hQWEGVaoJg9IFxPULKikvCrBu634e27yPirJCvv7JU7n4jOznsrLe1cb0nQWFGRSRaMwJiIQpR1MFxMTiPK5fMJ3PnjmRV/c2cNf6nQR8Hk4pClDfEmLVM2/h8Yid/I0ZRBYUZkBFojEagmFOJAREXX0LD2/cxx/eTh0Qfq8HgF9uraPA76Eoz2m5VBjwWPNWY4aABYUZEOFojIaWMCfbOgKi9ngLD2/ax4sJATGpOJ/rF1RyaUJAFOX5KCsMcOhEqzVvNSYHWFCYfhWKxGgIhjjZGmlftu94Cw9v3Mv6dw53CogbFlZyyRmdA6K00E+ezwtY81ZjcoUFhekXbZEoje4VRFyqgJhc4l5BnDERnxsQ+X4v44oC5Pu9nfZpzVuNyQ1ZBYWIXAbchTMV6n2qekfS43nAamAecAxYqqp73Me+C9wMRIFvqurzInIa8HjCLmYBt6nq/xKRVcBXgSPuY/+sqs/27vDMQGsNR2loCdMS6j4gbljgXEHEA8Lv9TCuKEBRXuqPoTVvNSY3dBsUIuIF7gEuAeqALSJS7c57HXczUK+qs0VkGXAnsFREzsCZP/tMYArwBxGZq6rvAucm7H8/8FTC/v5LVf+j74dnBkowFKW+JURrONq+bN+xFtZs3MtL73YfEKWFfsbm+1PtuhNr3mrM0MvmimI+sEtVdwOIyFpgCZAYFEuAVe7tdcDdIiLu8rWq2gZ8ICK73P1tSNj208D7qrq3LwdiBkdzW4SGYJi2FAGx/p3DxGdgn1ySz40Lp/OZj0xg294G/mndGxxlaniiAAAaCUlEQVRsClIxrpBvLJ5tJ39jhpFsgmIqUJtwvw5YkG4dVY2ISCNQ7i7fmLTt1KRtlwGPJS1bKSI3ATXA/1TV+uRCicgKYAVAZWVlFodh+uJkW4SGlhChSKx9WaqAmFKazw0LpnPJGRPxeoTNu4/zE7cvRHlRgOPNIRvq25hhZkgrs0UkAFwBfDdh8U+BfwPU/f2fwN8mb6uq9wL3AlRVVWny46bvVNUNiI5xmAD2HGtmzYa9vPzukbQBAeARYd22OvKtL4Qxw1o2QbEfqEi4P81dlmqdOhHxASU4ldrdbXs5sE1VD8UXJN4WkV8Az2RRRtOPUo3kCqkDYmppATcurOTTH+kICBFhbL7TF+Jgk/WFMGa4yyYotgBzRGQmzkl+GXBd0jrVwHKcuoergfWqqiJSDTwqIj/GqcyeA2xO2O5akr52EpHJqnrAvXsV8GbPDsn0Vnwk18aWjtnkIPuAgI7OcgGfU3FtfSGMGf66DQq3zmEl8DxO89gHVHWHiNwO1KhqNXA/sMatrD6OEya46z2BU/EdAb6hqlEAESnCaUl1S9JT/lBEzsX56mlPisdNP0s1kivAB0edgPjjex0BMa2sgBsWdA2IPL+XcusLYcyIJPHhFYazqqoqrampGepiDDupRnIFJyBWb9jLK0kBcePC6Vx8+oROAeH3eigrCjAmTV8IgJffOWx9IYzJQSKyVVWrulvPemaPQmF3JNfEgfqgIyD++N6R9mXpAsLn8VBa5Gdsng+nJXR61hfCmOHNgmIUiY/D1NwW7RQQu4+cZPXGvbzy3tH2ZekCwusRSgsCFBd0HxDGmJHBgmIUaA1HaQyGaU4Yhwng/SMnWbNhL6/s7AiIirICbrpwOotP6xwQ2Uw9aowZmSwoRrBgKEpDMEQwFO20PFVAVI4r5MaF01l82vhOASEiFOf7KC0MdDv1qDFmZLKgGIFSDbMBmQKisssVhIgwJs9HWaG/fYwmY8zoZEExQqTrRQ3w/mGnDuJPCQExfVwhN6S4ggCcgCgKtM8TAR0tl2rrW6iwlkvGjCoWFMNcLNbRizqxkxzArsMnWb1hL3/elRAQ5YXctHA6i+Z2DYjCgI+yoo6Jg+Jefucwt1XvwO8VSgv8HD7RauM1GTOKWFAMU9GY0hQM09TauZMcOAHx0IY9/GXXsfZlmQLCGbAvj4JA54CI+/kru/F7pb13dWHAZ+M1GTOKWFAMM+n6QADsPHSC1Rv28pf3OwfE8gudgPAkNWf1eTyUFXU/L0RtfYuN12TMKGZBMUykmmo07j03IP6aEBAzygu56cIZLJp7SpeA6GlfCBuvyZjRzYIix6WaajQuVUDMPKWIGxdOTxkQvW3qauM1GTO6WVDkqJaQ04KpNamJKzgB8dBf97Jhd+eAuOnC6XxiTteAAHhtXwNra2rZ3xDscaslm7vamNHNgiLHpJpJLu7JrXWs3riXptaOq4tZpxRxY4aA8Hs9vLW/kf96cWefWi3ZeE3GjF4WFDkg3URBce8ePMFdL+7knYMn2pf5vUJRno+/+/hMFp5a3mUbr0coLQxQnO/jn9a9bq2WjDG9ZkExhGIx5URrhMZg1z4QAO8cbGL1hr1s3H28fVnA56G8MMCYPC+tkRhP1NR1CopUYzJZqyVjTF9YUAyBTH0gAN4+4ATEpg86AsLnEcaPceZ9iLdUyvd7ONgUbF9nbL6fcUVdK6qt1ZIxpi+yCgoRuQy4C2eGu/tU9Y6kx/OA1cA8nLmyl6rqHvex7wI3A1Hgm6r6vLt8D3DCXR6JT54hIuOAx4EZODPcXaOq9X04xn7Vl6EsItEYz75xgAf+socDjUEmFxew7IIK5s8aB6QOiFPHF3HThTN4cmsdx1tCnZqztoZjTCouoDDgY1xRx/SjyazVkjGmL7oNChHxAvfgTFtaB2wRkWpVfSthtZuBelWdLSLLgDuBpSJyBs60qGfizJn9BxGZG58OFfiUqh6ls1uBF1X1DhG51b3/nT4cY7/p7VAW4WiMhpYwL759iLte3InP4zRTPdbcxl3rd3LV8anU7Ktnc1JALL9wBhfNLscjQp7Xw13rdxIMR8n3e2gNx4jGlP++eBaTSvIzlttaLRlj+iKbK4r5wC5V3Q0gImuBJTjzYMctAVa5t9cBd4vzr+8SYK2qtgEfuHNqzwc2ZHi+JcBi9/ZDwMvkSFD0dCiL+ERBJ91WSms31+LzCAUJ80rXN4f4P398v/3+7PFjWH7RdC46tbzT1cP8WeP4FnNYu6WWQ01Bpo0r5BuLZ1urJWPMgMsmKKYCtQn364AF6dZR1YiINALl7vKNSdtOdW8r8HsRUeDnqnqvu3yiqh5wbx8EJmZ5LAMu20rhtojTSS55oqADTUGK830Ew1GONYdoSZgnYvaEMSy/sGtAJFpwajmXnDmRssKATR5kjBk0Q1mZ/XFV3S8iE4AXROQdVX0lcQVVVTdIuhCRFcAKgMrKyoEvLd1XCqebKCiuOM9P7fEWWhP6SPi9wrTSQn5+w/kZh9MoynPqIfw2N4QxZpBlc9bZD1Qk3J/mLku5joj4gBKcSu2026pq/Pdh4Cmcr6QADonIZHdfk4HDqQqlqveqapWqVo0fPz6Lw+i7WxbNIhxVWkLOgHwtoQjhqLL8wunsbwhyoDGYMiR2fNjId371OjuPnGwPiTyfh1OKApQXBbhl0ay0IRHweZhcUsDE4nwLCWPMkMjmimILMEdEZuKc5JcB1yWtUw0sx6l7uBpY714NVAOPisiPcSqz5wCbRaQI8KjqCff2pcDtSfu6w/39dF8OsD8lVgrXHm9mcmkB18yrYO6ksV1mkwN4c38jqzfspWZvR6OtqaUFBLweWkJhJpcUdmr1lMjrEcqKAhR3M7KrMcYMtG6Dwq1zWAk8j9M89gFV3SEitwM1qloN3A+scSurj+OECe56T+BUfEeAb6hqVEQmAk+5/0X7gEdV9XfuU94BPCEiNwN7gWv68Xj77JOnjee86WUpJwqKe3N/Iw9t2MvWhICYO3EMyy+cwcJZ4zJ+xSRuh7nShA5zxhgzlCR5ToPhqKqqSmtqagb0OWIxpak1TGMwdSc5SB0Qp00cy/KLprNgZuaAAKfDnM1RbYwZLCKyNd6HLRPrmd2NSDRGU2uEpmCYWJpQfaOukdUb9rB1X0P7stMmjWX5hdkFRLopSI0xJhdYUKQRijgzyZ1s6zqTXNwbdY08tGEP2xIC4vRJY7kpy4DI83spLwqQ77eAMMbkLguKJK3hKI3Brn0gEr1e18DqDXt7HRB+r4dxRQGK8uzlN8bkPjtTuTJNFBT3mhsQryYFxPKLpjN/RvcBkTj0dzZTkBpjTC4Y9UHREopwvDn1REFxr9U18NBf97K9tiMgzpg8luUXzaBqelm3J31ryWSMGc5GfVA0BSNpQ+K12gYe2tD7gAAYk+ejzHpUG2OGsVEfFKlsr21g9YY9bK9tbF/W04DI93sZZxXVxpgRwIIiwfbaBh766x5eq+sIiDOnFLP8wunMyzIgrKLaGDPSjPqzmary6r56Htqwl9f7EBBej1BaEKC4wCqqjTEjy6gNClVlw/vH+OHz73aqgzhzSjFfuWgG51eWZnXCF3EmISot7DoFqTHGjASjNihOtEX46uoamt3RXs+aUsxNPbiCAKuoNsaMDqM2KIrz/XzlYzP4886j3LhwOudleQUB1qPaGDO6jNqgAPj2Z+Zyw4LpBDN0skvk93ooKwowxiqqjTGjyKg+4/m9nqyuIjwilBb6KSnwW0W1MWbUGdVB0R0RYWy+jzKrqDbGjGIWFGkUBpw5qgM+q6g2xoxuWZ0FReQyEXlXRHaJyK0pHs8TkcfdxzeJyIyEx77rLn9XRD7rLqsQkZdE5C0R2SEi30pYf5WI7BeR7e7P5/p+mNnzez1MKslnUkm+hYQxxpDFFYWIeIF7gEuAOmCLiFSr6lsJq90M1KvqbBFZBtwJLBWRM3CmRT0TZ87sP4jIXJxpUf+nqm4TkbHAVhF5IWGf/6Wq/9FfB5kNjwhlhdZhzhhjkmXzL/N8YJeq7lbVELAWWJK0zhLgIff2OuDT4pxtlwBrVbVNVT8AdgHzVfWAqm4DUNUTwNvA1L4fTu+MzfdTMa6QkkKrrDbGmGTZBMVUoDbhfh1dT+rt66hqBGgEyrPZ1v2a6jxgU8LilSLyuog8ICJlWZSx18aPzWP82DyrrDbGmDSG9Et4ERkD/Ar4tqo2uYt/CpwKnAscAP4zzbYrRKRGRGqOHDnS6zJYQBhjTGbZBMV+oCLh/jR3Wcp1RMQHlADHMm0rIn6ckHhEVZ+Mr6Cqh1Q1qqox4Bc4X311oar3qmqVqlaNHz8+i8MwxhjTG9kExRZgjojMFJEATuV0ddI61cBy9/bVwHpVVXf5MrdV1ExgDrDZrb+4H3hbVX+cuCMRmZxw9yrgzZ4elDHGmP7TbasnVY2IyErgecALPKCqO0TkdqBGVatxTvprRGQXcBwnTHDXewJ4C6el0zdUNSoiHwduBN4Qke3uU/2zqj4L/FBEzgUU2APc0o/Ha4wxpofE+cd/eKuqqtKampqhLoYxxgwrIrJVVau6W896lBljjMnIgsIYY0xGFhTGGGMysqAwxhiTkQWFMcaYjCwojDHGZGRBYYwxJiMLCmOMMRlZUBhjjMnIgsIYY0xGNmd2T733Avz1LmjYC6XT4aJvwdxLhrpUZqTpy+esr5/RoXpuK3fOnlNsrKeeeO8FeO4fwBMAfwGEgxALweX/kfsfSPsjHD7l7svnrK+f0YF8blX3J4Yz5ifOfRR2/gGev9XZ1pcPkVZn28/8G5y62FlHE7aJ7yO+bPcfYf0PQPzgz4dwK8TCsPifYcZFbgET9xHreO49f4E//cjZ1pcHkTZn24//PVQuSFg3vg86nn/fJth4t7ttwN02Agu+BtPmpSirtu8GjcH+bVBzP3h84A1ANOQ89/lfgckf7dgu+Xjj+0r1O3ldtPPxJuxDFt6S1VhPIyMo5s3Tmo1/TvGiJb/BKT4k3f5OWPdXX4Xmo84HMS4UhKJyuPL/dP+m7dvknDzE1/GB1DAsXAnTqlL/AcX3UbcVttzrbhuASMjZdt7NMOXc1M8XP+4Pt8Ora5xtvX7nw6gR+Oh1MOms9Mcb/33oLXhzHYjX3T7s/DGccRVMOC3Dh1ThyHvw7m+dbT0+Z1uNwtzLoPzUzK/Zsd2wez3gAa8PohFn25mfhLLpqZ8v8XdDLez7q/vcXqfMGoNp86F4SvrjVYUTB+DDV0E87rZRZ91JZ0PR+KT3KGkfzUfgyDvOtuJxH49B+WwoKEv9/iY+/+F3nJOFeDo+D7GY8xqMm+UuSvOeNdQ6x5m4rapzDGMmpNg2aR/Beue2dGwO6uzPX5T+tdaY87nq2MgMA/KDplEUFFO8WrNizFAXwxhjkgiIpPiNE76RNvd20kybqpBf7K7vSbOPNL+Tl8WfK8W6snJzVkFhdRQ5z31jNdZxP5m/gM4fKDp/SNqaOvaTSNX5L7P9A+Zxbyc8LwKNtR2Pddre/S8504f08FvOf7PJ28aiMOW8zB/02k3OVUh7mXD+YY1FYOaiFB9+Ot/f+YJzBZT83NEwfOTzSa9Z0nG/+Svw5nV9zSIhOPfapDInve5bH3S+Pkl+3kgrLLglxfEmve7b1kDopFt2d1k0BHlj4YK/S7Ot+1wbfwptJ5yvMeL7i4Sck87HvuWuT/p9/PGH0NrkXLXGyx9pg4JSuPh7HcfbaR/u8b/wrxBsAH9e0rZlcNn/m+L1jr8+As/+I7Qcd6/W49u2QmE5XHFXmtc7YdlTX4OWY+7fgiscdK7+vvxAxz67vGYCT9wIJ4+Av7DjLQsFYcx4uP6Jzusm7+PhL8KJwxAo7HjeUAuMnQj/7bd068HPw4lDqbf/yjPdb99XK1OcT1IYGUFxyly4eXXqP/jEE1yqD0nihy++Trp1P/gTvPz/4HwPWtDxPerFq5zvUbvsI+kD/cg1cPKws218nfiH4qbqNCc9V18+UH39MA7Vc2fa9von+lbuL96bedvjH6Tf9nM/yrzth9vTb3vxv3Rf7knnpPmu/0fd1xOUzui6rccLn/3/squjyCvpur0Al/xb99srzrbi69jW64NP/6sT7Jlc/H1nWySh3B741Pdg8jndl3vxd53tY9HO5f7kd6C0MvO2n/hHZ9toqGNbovCJf4D8kszbfuzvnW3DwY5tNQwf+3b3ZQan7uq5f4AQnd/ri76V3faDxLtq1aqhLkOf3Xv/g6tW/OMPnO+d238mw9jJMHaS8wc6diKMmej8Bz1mvPOfRtEp7k85FI7r/FNQ5vwXVVDqfFjyS2Dy2VA+F468DS1HobQCPr0KzvwCBIqcE0Og0HnD/QXOf0e+fKc+wpcHRRNh5++cPyiv36lw0zBc/K8wfk5HsIh0DgmAwvGw8znnu2qPr+MDdfFtznf9mfRl26F87tFa7vJTYdxsOPwmnDwEJVOdbbM50fdl26F8bit377bvox/84AcHVq1a1c1/TWRXRyEilwF34UyFep+q3pH0eB6wGpgHHAOWquoe97HvAjcDUeCbqvp8pn26c2uvBcqBrcCNqhrKVL5hNcNde2uYfc5/Or1qSTPI21q5B7/cxgyCbGe46zYoRMQLvAdcAtQBW4BrVfWthHW+DnxUVb8mIsuAq1R1qYicATwGzAemAH8A5rqbpdynO8f2k6q6VkR+Brymqj/NVMZhFRTGGJMj+nMq1PnALlXd7f5nvxZYkrTOEuAh9/Y64NMiIu7ytarapqofALvc/aXcp7vNxe4+cPd5ZRZlNMYYM0CyCYqpQG3C/Tp3Wcp1VDUCNOJ8dZRu23TLy4EGdx/pngsAEVkhIjUiUnPkyJEsDsMYY0xvDNuxnlT1XlWtUtWq8ePHD3VxjDFmxMomKPYDFQn3p7nLUq4jIj6gBKdSO9226ZYfA0rdfaR7LmOMMYMom6DYAswRkZkiEgCWAdVJ61QDy93bVwPr1aklrwaWiUie25ppDrA53T7dbV5y94G7z6d7f3jGGGP6qtsOd6oaEZGVwPM4TVkfUNUdInI7UKOq1cD9wBoR2QUcxznx4673BPAWEAG+oapRgFT7dJ/yO8BaEfl34FV338YYY4bIyBjryZrHGmNMj/Vn81hjjDGj2Ii4ohCRI8DeXmx6CnC0n4vTH3K1XJC7ZbNy9Vyuli1XywW5W7belmu6qnbbbHREBEVviUhNNpddgy1XywW5WzYrV8/latlytVyQu2Ub6HLZV0/GGGMysqAwxhiT0WgPim6H1x0iuVouyN2yWbl6LlfLlqvlgtwt24CWa1TXURhjjOneaL+iMMYY041RERQicpmIvCsiu0Tk1hSP54nI4+7jm0RkxiCUqUJEXhKRt0Rkh4h0mftQRBaLSKOIbHd/bhvociU89x4RecN93i69GcXxE/c1e11Ezh+EMp2W8FpsF5EmEfl20jqD9pqJyAMiclhE3kxYNk5EXhCRne7vsjTbLnfX2Skiy1Ot08/l+pGIvOO+V0+JSGmabTO+7wNQrlUisj/h/fpcmm0z/g0PUNkeTyjXHhHZnmbbgXzNUp4nBv1zpqoj+gdniJD3gVlAAHgNOCNpna8DP3NvLwMeH4RyTQbOd2+PxZnIKblci4Fnhuh12wOckuHxzwHP4cxMvBDYNATv60GcduBD8poBi4DzgTcTlv0QuNW9fStwZ4rtxgG73d9l7u2yAS7XpYDPvX1nqnJl874PQLlWAf+QxXud8W94IMqW9Ph/ArcNwWuW8jwx2J+z0XBF0ZeJlwaMqh5Q1W3u7RPA26SZeyNHLQFWq2Mjzqi/kwfx+T8NvK+qvelo2S9U9RWcsc0SJX6W0k289VngBVU9rqr1wAvAZQNZLlX9vXbM87IRZ2TmQZXm9cpGNn/DA1Y291xwDc5snYMqw3liUD9noyEo+jLx0qBwv+o6D9iU4uELReQ1EXlORM4crDIBCvxeRLaKyIoUj2fzug6kZaT/wx2q1wxgoqoecG8fBCamWGeoX7u/xbkaTKW7930grHS/EnsgzVcoQ/16fQI4pKo70zw+KK9Z0nliUD9noyEocpqIjAF+BXxbVZuSHt6G89XKOcD/Bn49iEX7uKqeD1wOfENEFg3ic2ckztD0VwC/TPHwUL5mnahz/Z9TzQpF5Hs4Izk/kmaVwX7ffwqcCpwLHMD5iifXXEvmq4kBf80ynScG43M2GoKiLxMvDSgR8eO8+Y+o6pPJj6tqk6qedG8/C/hF5JSBLpf7fPvd34eBp3Au/xNl87oOlMuBbap6KPmBoXzNXIfiX8G5vw+nWGdIXjsR+QrweeB69+TSRRbve79S1UOqGlXVGPCLNM83ZJ8193zwReDxdOsM9GuW5jwxqJ+z0RAUfZl4acC433veD7ytqj9Os86keF2JiMzHeb8GI8CKRGRs/DZOReibSatVAzeJYyHQmHApPNDS/oc3VK9ZgsTPUrqJt54HLhWRMverlkvdZQNGRC4D/gm4QlVb0qyTzfve3+VKrNe6Ks3zZfM3PFA+A7yjqnWpHhzo1yzDeWJwP2cDUVOfaz84LXTew2k58T132e04fzQA+ThfY+zCmYFv1iCU6eM4l4uvA9vdn88BXwO+5q6zEtiB08pjI3DRIL1es9znfM19/vhrllg2Ae5xX9M3gKpBKlsRzom/JGHZkLxmOGF1AAjjfP97M07d1ovATuAPwDh33SrgvoRt/9b9vO0C/tsglGsXzvfV8c9avJXfFODZTO/7AJdrjfv5eR3n5Dc5uVzu/S5/wwNdNnf5g/HPVsK6g/mapTtPDOrnzHpmG2OMyWg0fPVkjDGmDywojDHGZGRBYYwxJiMLCmOMMRlZUBhjjMnIgsIYY0xGFhTGGGMysqAwxhiT0f8PN6k5Kh0gb0sAAAAASUVORK5CYII=\n",
"text/plain": [
"<Figure size 432x288 with 1 Axes>"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"# Best subsets with smf is SLOW.\n",
"\n",
"# Investigating performance of best subsets with sklearn\n",
"\n",
"i = 1\n",
"\n",
"smf_times = []\n",
"while i < data.shape[1] - 1 :\n",
" cols = ['x' + str(j) for j in range(1, i + 1)]\n",
" f = 'y ~ ' + ' + '.join(cols)\n",
" tick = time.time()\n",
" smf.ols(formula=f, data=data).fit()\n",
" tock = time.time()\n",
" smf_times.append(tock - tick)\n",
" i += 1\n",
" \n",
"sns.regplot(x=[x for x in range(1, data.shape[1] - 1)], y=smf_times)\n",
"\n",
"i = 1\n",
"sk_times = []\n",
"while i < data.shape[1] - 1 :\n",
" cols = ['x' + str(j) for j in range(1, i + 1)]\n",
" x = data[cols]\n",
" tick = time.time()\n",
" model = linear_model.LinearRegression().fit(x, data['y'])\n",
" model.score(x, data['y'])\n",
" tock = time.time()\n",
" sk_times.append(tock - tick)\n",
" i += 1\n",
" \n",
"sns.regplot(x=[x for x in range(1, data.shape[1] - 1)], y=sk_times)\n",
"\n",
"# SKlearn kills statsmodels.\n",
"np.array(sk_times).mean()"
]
},
{
"cell_type": "code",
"execution_count": 420,
"metadata": {},
"outputs": [],
"source": [
"def n_choose_k(n, k):\n",
" return int(np.math.factorial(n) / (np.math.factorial(n - k) * np.math.factorial(k)))\n",
"\n",
"def n_choose_upto_k_sum(n, up_to_k):\n",
" return np.array([n_choose_k(n, k) for k in range(1, up_to_k + 1)]).sum()\n",
" \n",
"def best_subset(n_predictors, target_column, data):\n",
" i = 1\n",
" predictors = data.drop(target_column, axis=1).columns\n",
" search_size = n_choose_upto_k_sum(len(predictors), n_predictors)\n",
" single_fit_time = 0.0012\n",
" print('searching over {} models'.format(search_size))\n",
" print('estimated time: {} seconds'.format(single_fit_time * search_size))\n",
" top_models = []\n",
" top_combs = []\n",
" loop_tick = time.time() \n",
" while i <= n_predictors:\n",
" print('-----------')\n",
" print('n choose: {}'.format(i))\n",
" cmbs = list(combinations(predictors, i))\n",
" print('{} combinations'.format(len(cmbs)))\n",
" tick = time.time()\n",
" top_model = None\n",
" top_score = None\n",
" top_comb = None\n",
" for s in cmbs:\n",
" x = data[list(s)]\n",
" model = linear_model.LinearRegression().fit(x, data[target_column])\n",
" score = model.score(x, data[target_column])\n",
" if top_score == None or score < top_score:\n",
" top_model = model\n",
" top_score = score\n",
" top_comb = s\n",
" tock = time.time()\n",
" print('done: {} seconds ({} secs/model)'.format(tock - tick, (tock - tick) / len(cmbs)))\n",
" top_models.append(top_model)\n",
" top_combs.append(top_comb)\n",
" tock = time.time()\n",
" i += 1\n",
" \n",
" print('--- FINISHED ---')\n",
" loop_tock = time.time()\n",
" print('total time: {} seconds'.format((loop_tock-loop_tick)))\n",
" print('secs/model: {}'.format((loop_tock-loop_tick)/ search_size))\n",
" return pd.DataFrame({'model' : top_models, 'n_predictors' : range(1, n_predictors + 1)}), top_combs"
]
},
{
"cell_type": "code",
"execution_count": 306,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"searching over 6195 models\n",
"estimated time: 7.433999999999999 seconds\n",
"-----------\n",
"n choose: 1\n",
"20 combinations\n",
"done: 0.025238990783691406 seconds (0.0012619495391845703 secs/model)\n",
"-----------\n",
"n choose: 2\n",
"190 combinations\n",
"done: 0.22151517868041992 seconds (0.0011658693614758943 secs/model)\n",
"-----------\n",
"n choose: 3\n",
"1140 combinations\n",
"done: 1.356029987335205 seconds (0.0011894999888905308 secs/model)\n",
"-----------\n",
"n choose: 4\n",
"4845 combinations\n",
"done: 5.599212884902954 seconds (0.001155668294097617 secs/model)\n",
"--- FINISHED ---\n",
"total time: 7.204585075378418 seconds\n",
"secs/model: 0.0011629677280675411\n"
]
}
],
"source": [
"df, combs = best_subset(4, 'y', train)"
]
},
{
"cell_type": "code",
"execution_count": 307,
"metadata": {},
"outputs": [],
"source": [
"rmses = []\n",
"for idx, m in enumerate(df['model']):\n",
" comb = combs[idx]\n",
" x = test[list(comb)]\n",
" preds = m.predict(x)\n",
" rmses.append(np.sqrt(metrics.mean_squared_error(test['y'], preds)))\n",
" \n",
"df = pd.DataFrame({'test_rmse' : rmses, 'n_predictors' : np.arange(1, len(rmses) + 1)})"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"(d) Plot the test set MSE associated with the best model of each size."
]
},
{
"cell_type": "code",
"execution_count": 308,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"<matplotlib.axes._subplots.AxesSubplot at 0x132608320>"
]
},
"execution_count": 308,
"metadata": {},
"output_type": "execute_result"
},
{
"data": {
"image/png": "iVBORw0KGgoAAAANSUhEUgAAAXwAAAEKCAYAAAARnO4WAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAEQ1JREFUeJzt3XuwXWV9xvHvQ8KdKLYEBwkQWxXrpZWYsWoQChYGqsXW2iojWrUDtt6gOlq1jo5OO9ZSqTPO1DFydVQcLmIRUfCC4SIiSUDCtVUKiqJBvBCw5frrH3sdPIZczj7snbVP3u9nZs/Ze+3LesiE57x591rvSlUhSdr6bdN3AEnSlmHhS1IjLHxJaoSFL0mNsPAlqREWviQ1wsKXpEZY+JLUCAtfkhoxv+8A0+222261ePHivmNI0pyyatWqn1bVws29bqIKf/HixaxcubLvGJI0pyS5dSavc0pHkhph4UtSIyx8SWrERM3h33DbnTz77Z/sO4YkjcSq41/dd4Tf4Ahfkhph4UtSIyx8SWqEhS9JjbDwJakRFr4kNcLCl6RGWPiS1AgLX5IaYeFLUiMsfElqhIUvSY2w8CWpERa+JDXCwpekRlj4ktQIC1+SGmHhS1Ijxlr4SU5OsjbJtePcjyRp88Y9wj8VOGzM+5AkzcBYC7+qLgZ+Ns59SJJmpvc5/CTHJFmZZOUDv1rXdxxJ2mr1XvhVtbyqllbV0vk7Leg7jiRttXovfEnSlmHhS1Ijxn1Y5unA5cC+SW5L8jfj3J8kaePmj/PDq+rIcX6+JGnmnNKRpEZY+JLUCAtfkhph4UtSIyx8SWqEhS9JjbDwJakRFr4kNcLCl6RGWPiS1AgLX5IaYeFLUiMsfElqhIUvSY2w8CWpERa+JDVirBdAGdbvLfptVh7/6r5jSNJWyRG+JDXCwpekRlj4ktQIC1+SGmHhS1IjLHxJaoSFL0mNsPAlqREWviQ1wsKXpEZM1NIK991+Hd//wDP7jiFJI7X3e9f0HQFwhC9JzbDwJakRFr4kNcLCl6RGWPiS1AgLX5IaYeFLUiMsfElqhIUvSY2w8CWpERa+JDXCwpekRlj4ktQIC1+SGmHhS1IjLHxJaoSFL0mNGGvhJ9kryUVJrk9yXZJjx7k/SdLGjfsShw8Ab6uq1UkWAKuSfKWqrh/zfiVJ65nxCD/Jzkm26e4/JckRSbbd1Huq6vaqWt3dXwfcAOz5aAJLkmZnmCmdi4EdkuwJXAi8Cjh1pm9OshjYD7hive3HJFmZZOXP7nlwiDiSpGEMU/ipql8BLwX+o6r+Enj6jN6Y7AKcDRxXVXdNf66qllfV0qpa+ls7zxsijiRpGEMVfpLnAa8Evtht22xDd9M+ZwOfrqrPDR9RkjQKwxT+scC7gHOq6rokvwNctKk3JAlwEnBDVZ0w+5iSpEdrRkfpJJkHHFFVR0xtq6qbgbds5q3LGMz1r0lydbft3VV1/mzCSpJmb0aFX1UPJtl/2A+vqkuBDJ1KkjRywxyHf1WSc4EzgXumNjovL0lzwzCFvwNwJ3DwtG0FWPiSNAfMuPCr6rXjDCJJGq9hzrRdlOScJGu729lJFo0znCRpdIY5LPMU4FzgCd3tC902SdIcMEzhL6yqU6rqge52KrBwTLkkSSM2TOHfmeSoJPO621EMvsSVJM0BwxT+64C/An4M3A68DHjNGDJJksZgmMMyF00/0xYgyTLgB6ONJEkah2FG+B+d4TZJ0gTa7Ai/WyHz+cDCJG+d9tRjmMFqmZKkyTCTKZ3tgF261y6Ytv0uBvP4kqQ5YLOFX1UrgBVJTq2qW7dAJknSGAwzh39ikl2nHiR5XJILxpBJkjQGwxT+blX1i6kHVfVzYPfRR5IkjcMwhf9Qkr2nHiTZh8FqmZKkOWCY4/D/Ebg0yQoGFzV5AXDMKMNst8fT2fu9K0f5kZKkzjDLI385yRLgud2m46rqp+OJJUkatc1O6SR5avdzCbA38KPutne3TZI0B8xkhP824Gjgwxt4rvjNK2BJkibUTI7DP7r7edD440iSxmUmSyu8dFPPexFzSZobZjKl86fdz90ZrKnz9e7xQcA38SLmkjQnzGRK57UASS4EnlZVt3eP9wBOHWs6SdLIDHPi1V5TZd/5CYOjdiRJc8AwJ159rVs75/Tu8cuBr44+kiRpHIY58epNSf4cOKDbtLyqzhlPLEnSqA0zwgdYDayrqq8m2SnJgqpaN6owN669kWUfXTaqj5OkkbrszZf1HeFRmfEcfpKjgbOAj3eb9gQ+P45QkqTRG+ZL2zcCyxhc6Yqq+m9cHlmS5oxhCv/eqrpv6kGS+bg8siTNGcMU/ook7wZ2THIIcCbwhfHEkiSN2jCF/07gDmAN8HrgfOA94wglSRq9GR2lk2Qe8MmqeiXwifFGkiSNw4xG+FX1ILBPku3GnEeSNCbDHId/M3BZknOBe6Y2VtUJI08lSRq5YQr/e91tG2DBeOJIksZlmKUV3g+Q5DGDh6M7w1aSNH7DnGm7NMka4BpgTZLvJHn2+KJJkkZpmCmdk4E3VNUlAEn2B04Bfn8cwSRJozXMcfgPTpU9QFVdCjww+kiSpHEYZoS/IsnHGayHXwzWw/9GkiUAVbV6DPkkSSMyTOH/Qffzfett34/BL4CDR5JIkjQWwxylc9Cmnk/y11V12qOPJEkah2Hm8Dfn2BF+liRpxEZZ+BnhZ0mSRmyUhf+ItfGT7JDk290x+9clef8I9ydJGsKw17TdlA2N8O8FDq6qu5NsC1ya5EtV9a0R7leSNAMzLvwk2wN/ASye/r6q+kB39xFX962qAu7uHm7b3bxKliT1YJgpnf8EXsLgZKt7pt0AqKo3behNSeYluRpYC3ylqq5Y7/ljkqxMsvL+u+8fNr8kaYaGmdJZVFWHDbuDbi39ZyXZFTgnyTOq6tppzy8HlgPssvcujv4laUyGGeF/M8kzZ7ujqvoFcBEw9C8NSdKjN0zh7w+sSnJTkmuSrElyzabekGRhN7InyY7AIcCNs48rSZqtYaZ0Dp/F5+8BnNZdE3cb4IyqOm8WnyNJepSGWVrh1mE/vKquYbDWjiSpZ6M88UqSNMEsfElqhIUvSY2w8CWpERa+JDXCwpekRlj4ktQIC1+SGmHhS1IjLHxJaoSFL0mNsPAlqREWviQ1wsKXpEZY+JLUCAtfkhoxzBWvxu6puz+Vy958Wd8xJGmr5Ahfkhph4UtSIyx8SWqEhS9JjbDwJakRFr4kNcLCl6RGWPiS1AgLX5IaMVFn2q676SZWHHBg3zEkaYs68OIVW2Q/jvAlqREWviQ1wsKXpEZY+JLUCAtfkhph4UtSIyx8SWqEhS9JjbDwJakRFr4kNcLCl6RGWPiS1AgLX5IaYeFLUiMsfElqhIUvSY2w8CWpERa+JDViixR+knlJrkpy3pbYnyTpkbbUCP9Y4IYttC9J0gaMvfCTLAJeBJw47n1JkjZuS4zwPwK8A3hoQ08mOSbJyiQrf3n//VsgjiS1aayFn+TFwNqqWrWx11TV8qpaWlVLH7vttuOMI0lNG/cIfxlwRJJbgM8CByf51Jj3KUnagLEWflW9q6oWVdVi4BXA16vqqHHuU5K0YR6HL0mNmL+ldlRV3wC+saX2J0n6TY7wJakRFr4kNcLCl6RGWPiS1AgLX5IaYeFLUiMsfElqhIUvSY2w8CWpERa+JDXCwpekRlj4ktQIC1+SGmHhS1IjLHxJaoSFL0mN2GIXQJmJBfvuy4EXr+g7hiRtlRzhS1IjLHxJaoSFL0mNsPAlqRGpqr4zPCzJOuCmvnNsxG7AT/sOsRFmG96k5gKzzcak5oItk22fqlq4uRdN1FE6wE1VtbTvEBuSZKXZhjep2SY1F5htNiY1F0xWNqd0JKkRFr4kNWLSCn953wE2wWyzM6nZJjUXmG02JjUXTFC2ifrSVpI0PpM2wpckjYmFL0mNmJjCT3JYkpuSfDfJO/vOMyXJyUnWJrm27yzTJdkryUVJrk9yXZJj+840JckOSb6d5Dtdtvf3nWl9SeYluSrJeX1nmS7JLUnWJLk6ycq+80xJsmuSs5LcmOSGJM/rOxNAkn27P6up211Jjus7F0CSv+/+/l+b5PQkO/SeaRLm8JPMA/4LOAS4DbgSOLKqru81GJDkAOBu4JNV9Yy+80xJsgewR1WtTrIAWAX82YT8mQXYuaruTrItcClwbFV9q+doD0vyVmAp8JiqenHfeaYkuQVYWlUTdRJRktOAS6rqxCTbATtV1S/6zjVd1yM/BP6wqm7tOcueDP7eP62q/jfJGcD5VXVqn7kmZYT/HOC7VXVzVd0HfBZ4Sc+ZAKiqi4Gf9Z1jfVV1e1Wt7u6vA24A9uw31UAN3N093La79T+y6CRZBLwIOLHvLHNBkscCBwAnAVTVfZNW9p0XAt/ru+ynmQ/smGQ+sBPwo57zTEzh7wn8YNrj25iQ8poLkiwG9gOu6DfJr3VTJlcDa4GvVNXEZAM+ArwDeKjvIBtQwIVJViU5pu8wnScCdwCndNNgJybZue9QG/AK4PS+QwBU1Q+BfwO+D9wO/LKqLuw31eQUvmYpyS7A2cBxVXVX33mmVNWDVfUsYBHwnCQTMR2W5MXA2qpa1XeWjdi/qpYAhwNv7KYU+zYfWAJ8rKr2A+4BJuZ7NoBumukI4My+swAkeRyDWYonAk8Adk5yVL+pJqfwfwjsNe3xom6bNqGbHz8b+HRVfa7vPBvS/dP/IuCwvrN0lgFHdHPlnwUOTvKpfiP9WjcypKrWAucwmO7s223AbdP+lXYWg18Ak+RwYHVV/aTvIJ0/Bv6nqu6oqvuBzwHP7znTxBT+lcCTkzyx+039CuDcnjNNtO6L0ZOAG6rqhL7zTJdkYZJdu/s7Mvgy/sZ+Uw1U1buqalFVLWbw9+zrVdX7yAsgyc7dF/B0UyaHAr0fHVZVPwZ+kGTfbtMLgd4PDljPkUzIdE7n+8Bzk+zU/b/6Qgbfs/VqIlbLrKoHkrwJuACYB5xcVdf1HAuAJKcDfwTsluQ24H1VdVK/qYDBSPVVwJpurhzg3VV1fo+ZpuwBnNYdNbENcEZVTdThjxPq8cA5g35gPvCZqvpyv5Ee9mbg092A7GbgtT3neVj3y/EQ4PV9Z5lSVVckOQtYDTwAXMUELLEwEYdlSpLGb1KmdCRJY2bhS1IjLHxJaoSFL0mNsPAlqREWvrYK3WqOb5jle49LstOoM0mTxsLX1mJXYFaFDxzHYHGrGenOL5DmHAtfW4t/AX63WxP9+CRvT3Jlkmum1uPvzmT9YrdO/7VJXp7kLQzWOrkoyUUb+/Akdyf5cJLvAM/r1q3/4NS69UmWJLkgyfeS/G33nj2SXNy95tokL+i2H5rk8iSrk5zZrYckjZ0nXmmr0K0Yel5VPSPJocDLGJx5GQbLdPwrsBA4rKqO7t7z2Kr65UzWoE9SwMur6ozu8S3Ah6rqY0n+ncGp88uAHYBrq+rxSd4G7FBV/9z9q2AnYHsG66ocXlX3JPkHYPuq+sCI/0ikR5iIpRWkETu0u13VPd4FeDJwCfDhJB9i8MvhkiE+80EGC9VNN7Xe0xpgl+66BOuS3NutJXQlcHK3yN3nq+rqJAcCTwMu65ZQ2A64fOj/QmkWLHxtjQJ8sKo+/ognkiXAnwD/lORrQ4ys/6+qHlxv273dz4em3Z96PL+qLu6WN34RcGqSE4CfM7g+wJFD/PdII+EcvrYW64AF3f0LgNdNzY0n2TPJ7kmeAPyqqj4FHM+vl/id/t6RSbIP8JOq+gSDq2stAb4FLEvypO41Oyd5yqj3LW2II3xtFarqziSXZXCx+S8BnwEu76ZN7gaOAp4EHJ/kIeB+4O+6ty8HvpzkR1V10Ahj/RHw9iT3dxleXVV3JHkNcHqS7bvXvYfBNZ2lsfJLW0lqhFM6ktQIp3SkaZJcweDQyeleVVVr+sgjjZJTOpLUCKd0JKkRFr4kNcLCl6RGWPiS1AgLX5Ia8f83fEnwMENPSwAAAABJRU5ErkJggg==\n",
"text/plain": [
"<Figure size 432x288 with 1 Axes>"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"sns.barplot(y='n_predictors', x='test_rmse', data=df, orient='h')"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"(e) For which model size does the test set MSE take on its minimum value? Comment on your results. If it takes on its minimum value for a model containing only an intercept or a model containing all of the features, then play around with the way that you are generating the data in (a) until you come up with a scenario in which the test set MSE is minimized for an intermediate model size."
]
},
{
"cell_type": "code",
"execution_count": 309,
"metadata": {},
"outputs": [],
"source": [
"# Running best subsets for p=20 is going to take ~20 hours!!\n",
"# I'm going to do forward stepwise instead.\n",
"\n",
"# Select between models with different numbers of predictors using adjusted training metrics\n",
"best = fwd_selection('y', train)\n",
"rmses = []\n",
"for m in best['model']:\n",
" preds = m.predict(test)\n",
" rmses.append(np.sqrt(metrics.mean_squared_error(test['y'], preds)))\n",
"\n",
"df = pd.DataFrame({'model' : best['model'] ,'test_rmse' : rmses, 'n_predictors' : np.arange(1, len(rmses) + 1)})\n"
]
},
{
"cell_type": "code",
"execution_count": 397,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"<matplotlib.axes._subplots.AxesSubplot at 0x117a37438>"
]
},
"execution_count": 397,
"metadata": {},
"output_type": "execute_result"
},
{
"data": {
"image/png": "iVBORw0KGgoAAAANSUhEUgAAAYMAAAEKCAYAAADw2zkCAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAGhBJREFUeJzt3XuwnFWd7vHvY8ItVwIJCAQJw83jMIq4D4UiCKgMIAOjokiJA8ghDoKCh9KDlxp0jlOj4LVmPBwDQpgjkxG5KIrcBtGoA8zsQIBw0QAChlu4BJJACQSe80e/wU6y9+7unf322jv9fKq6dvfb633Xr9Xyl/dda/2WbBMREb3tNaUDiIiI8pIMIiIiySAiIpIMIiKCJIOIiCDJICIiSDKIiAiSDCIigiSDiIgAxpcOoF3Tp0/3rFmzSocRETGmLFiw4EnbM1q1GzPJYObEKVx1wmmlw4iIDsw46ZjSIfQ8SQ+2067YYyJJ50taKmlRqRgiIqKh5JjBXODggv1HRESlWDKwPR94ulT/ERHxJ6N6NpGk2ZL6JfU/tXJ56XAiIjZYozoZ2J5ju89235aTppQOJyJigzWqk0FERHRHkkFERBSdWjoPuBHYTdISSSeUiiUiotcVW3Rm++hO2o+fsUUWsERE1CSPiSIiYuyUo3jpiUd57Jwvlw4jYoPw2pO+UDqEGGWK3BlI2l7SDZLuknSnpFNLxBEREQ2l7gxWAafbvkXSZGCBpOts31UonoiInlbkzsD2o7Zvqd6vAO4GtisRS0REjIIBZEmzgDcDNw/wXVM5iue6HVpERM8omgwkTQIuBU6zvU7xoTXLUUzsfoARET2i5KKzjWgkgotsX1YqjoiIKDebSMD3gLttf6NEDBER8Sel7gz2AT4CHChpYfU6tFAsERE9r8jUUtu/BtTJORvN2CYLZSIialJ8NlFERJQ3ZspR/HHpvdzznSNKhxERw/T6k39cOoQYQqkB5E0l/aek26pyFF8qEUdERDSUujN4ATjQ9spqiumvJV1l+6ZC8URE9LRSA8gGVlYfN6peLhFLRESUXXQ2TtJCYClwne0hy1EsW/li94OMiOgRxZKB7Zdt7wHMBPaStPsAbV4tRzFt0sbdDzIiokcUn1pq+xngBuDg0rFERPSqUrOJZkjavHq/GfBu4J4SsURERLnZRNsAF0oaRyMhXWz7p0OdsOlWO2eeckRETUrNJrqdxh4GERExChQfM4iIiPLGTDmKFU8u5hfnvqd0GBFjyv4nXlk6hBgjSu90Nk7SrZKGHC+IiIh6lX5MdCpwd+EYIiJ6XskVyDOB9wDnlYohIiIaSt4ZfAv4DPDKYA2ay1E8uyLlKCIi6lJq0dlhwFLbC4Zq11yOYurklKOIiKhLyT2QD5f0APBvNPZC/n6hWCIiel6RZGD7s7Zn2p4FfAj4ue1jSsQSERFjaJ3B5Om7ZM50RERNiicD278AflE4jIiInlZ6nUFERIwCxe8M2rXsycVcckG2PIgYriOPv7p0CDGKFUsG1UyiFcDLwCrbfaViiYjodaXvDA6w/WThGCIiel7GDCIiomgyMHCtpAWSZg/UoLkcxfKVKUcREVGXko+J3m77YUlbAddJusf2/OYGtucAcwB2mjXVJYKMiOgFxe4MbD9c/V0KXA7sVSqWiIheV6pQ3URJk1e/Bw4CFpWIJSIiyj0m2hq4XNLqGP7V9pCToKdN3yXzpCMialIkGdi+H3hTib4jImJdpdcZtO2Jpxbz3f/3l6XDiIgafewj15QOoWeV3PZyc0mXSLpH0t2S3loqloiIXlfyzuDbwNW2j5S0MTChYCwRET2tSDKQNBXYDzgOwPaLQFaVRUQUUuox0Y7AE8AFkm6VdF41xTQiIgoolQzGA3sC59h+M/AccMbajZrLUaxckRuHiIi6lEoGS4Altm+uPl9CIzmswfYc2322+yZN3rirAUZE9JIiycD2Y8AfJO1WHXoncFeJWCIiouxsok8AF1Uzie4Hji8YS0RET5M9NoqB9vX1ub+/v3QYERFjiqQF7ewkmc1tIiJi7JSjeGTZYr54ccpRRIx1X/xgSk6MRqVKWO8maWHTa7mk00rEEhER5aqW/hbYA0DSOOBhGhvcREREAaNhzOCdwH22HywdSERErxoNyeBDwLzSQURE9LKiyaBaY3A48MNBvn+1HMXzy1OOIiKiLqXvDA4BbrH9+EBfNpejmDAl5SgiIupSOhkcTR4RRUQUV3Kns4nAu4HLSsUQERENxRad2X4O2LLd9ttO2yWLVSIialL6MVFERIwCY6YcxeJn7uOQH7+/dBgR0WVXHXFp6RB6Qskxg09JulPSIknzJG1aKpaIiF5XqjbRdsAngT7buwPjaCw+i4iIAtpOBpImSnpN9X5XSYdL2mg9+h4PbCZpPDABeGQ9rhUREeuhkzuD+cCm1b/qrwU+AswdTqe2Hwa+BjwEPAo8a/vatds1r0B+cfkLw+kqIiLa0EkykO3ngfcB/8f2B4A/H06nkqYBRwA7AtsCEyUds3a75hXIG0/ZZDhdRUREGzpKBpLeCnwYuLI6Nm6Y/b4L+L3tJ2y/RGPh2duGea2IiFhPnSSDU4HPApfbvlPSnwE3DLPfh4C9JU2QJBplrO8e5rUiImI9tbXOoNqA5nDbh68+Zvt+GjOCOmb7ZkmXALcAq4BbgTnDuVZERKw/2W6voXST7b1rjmdQfX197u/vL9V9RMSYJGmB7b5W7TpZgXyrpCto7D3w3OqDtlNoLiJijOskGWwKPAUc2HTMdKnq6OJnHuXQy7/cja4iYhT52Xu/UDqEntB2MrB9/Eh2LOlU4ERAwLm2vzWS14+IiPZ1sgJ5pqTLJS2tXpdKmjmcTiXtTiMR7AW8CThM0s7DuVZERKy/TqaWXgBcQWOR2LbAT6pjw/HfgJttP297FfBLGovZIiKigE6SwQzbF9heVb3mAjOG2e8iYF9JW0qaABwKbL92ozXLUTy3zkUiImJkdJIMnpJ0jKRx1esYGgPKHbN9N/BVGjWOrgYWAi8P0K6pHMXE4XQVERFt6CQZfBT4IPAYjeJyRwLHDbdj29+z/Rbb+wHLgN8N91oREbF+OplaOrN5BTKApH2APwynY0lb2V4q6XU0xguKLWiLiOh1ndwZ/FObx9p1qaS7aAxEn2z7mfW4VkRErIeWdwZVpdK3ATMk/c+mr6Yw/Kql2N63k/a7bL5NFp9ERNSkncdEGwOTqraTm44vpzFuEBERY1wnhep2sP1gzfEMaurOO/jtZ51RqvuIiCKufN9J63V+u4XqOhkzOE/S5k0dTJN0TYsgzq9WKy9qOraFpOskLa7+TusghoiIqEEnyWB68yCv7WXAVi3OmQscvNaxM4Drbe8CXF99joiIgjpJBq9U00CBxmMjGlVLB2V7PvD0WoePAC6s3l8I/HUHMURERA06WWfweeDXkn5Jo9LovsDsYfS5te1Hq/ePAVsP1lDS7NV9bDp9i2F0FRER7eikhPXVkvbkT4vDTrP95Pp0btuSBr27sD2HajvMqTvv0N5Id0REdKzlYyJJr6/+7gm8Dniker2uOtapxyVtU11zG2DpMK4REREjqJ07g9Np7D3w9QG+M2vufNaOK4Bjga9Uf3/c4fkRETHCWiYD2ydWfw/o9OKS5gH7A9MlLQHOpJEELpZ0AvAgjeJ3ERFRUMtFZ5KG3HTGdlf2QO7r63N/f383uoqI2GC0u+isncdEf1X93YpGjaKfV58PAP4D6EoyiIiI+rTzmOh4AEnXAm9YPS20GvydW2t0Te5d9jSHXXJRt7qLiKjFT4/8cOkQBtTJorPtm9YHADxOY3bRoAYpR/EBSXdKekVSy1uXiIioXyfJ4HpJ10g6TtJxwJXAv7c4Zy7rlqNYRGMzm/kd9B0RETXqZNHZKZLeC+xXHZpj+/IW58yXNGutY3cDSOos0oiIqE0n5SgAbgFW2P53SRMkTba9oo7AYM1yFJtN37KubiIiel7bj4kknQhcAny3OrQd8KM6glrN9hzbfbb7Np4ypc6uIiJ6WidjBicD+9DY4Qzbi2ldwjoiIsaATpLBC7ZfXP1B0nhalLCOiIixoZMxg19K+hywmaR3Ax8HfjLUCYOUo3ga+CdgBnClpIW2/7JV5ztP22LUzs+NiBjrOkkGZwAnAHcAHwN+Bpw31Am2jx7kqyFnIUVERHe1lQwkjQP+xfaHgXPrDSkiIrqtrWRg+2VJO0jauHncoJvuXfYsh18y5FOpiIgRd8WRf9W60Qagk8dE9wO/kXQF8Nzqg7a/MdgJks4HDgOW2t69OnY2jeJ3LwL3AcfbfmYYsUdExAjpZDbRfcBPq3MmN72GMpd1y1FcB+xu+43A74DPdhBDRETUoJNyFF8CkDSl8bH1yuNBylFc2/TxJuDIdmOIiIh6dLICuU/SHcDtwB2SbpP0lvXs/6PAVUP0OVtSv6T+F5c/u55dRUTEYDp5THQ+8HHbs2zPorEi+YLhdizp88AqYNBNCtYsRzF1uF1FREQLnQwgv2z7V6s/2P61pFXD6bQqgX0Y8E632nczIiJq1+kK5O8C82iUoTgK+IWkPQFs39LORSQdDHwGeIft5zuMNyIiaqB2/2Eu6YYhvrbtAwc459VyFDR2RjuTxuyhTYCnqmY32f7bVv339fW5v7+/rVgjIqJB0gLbLXeV7GQ20QEtOjzW9oVrnTNQOYrvtdtnRER0RycDyK2cOoLXioiILup0p7Oh1LqP5X3LVvLeS39dZxcREUO6/P1vLx1CbUbyzmCdwQdJ50taKmlR07H/Lel2SQslXStp2xGMISIihmEkk8FAdwZzWbccxdm232h7DxrlLf5uBGOIiIhhaPsxkaRNgPcDs5rPs/331dvfrH3OIOUoljd9nEh2S4uIKK6TMYMfA88CC4AX1v7S9intXkjSPwB/U11v0FlKkmYDswE2m751B6FGREQnOkkGM22v/chnWGx/Hvi8pM8Cp9BYfzBQuznAHIBpO70+dxARETXpZMzgPyT9xQj3fxGNR08REVFQJ3cGbweOk/R7Go+JRGPl8Rs76VDSLrYXVx+PAO7p5PyIiBh5nSSDQzq9eHM5CklLaDwOOlTSbsArwINAy1IUADtNm7RBz/GNiCipk3IUD3Z68ZSjiIgYG0ZyBXKt7n/mBY667N7SYUREDOgH79u5dAjrZSQXna1joBXITd+dLsmSptcZQ0REtFZrMmDgFchI2h44CHio5v4jIqINtSYD2/OBpwf46ps0NrjJ2oGIiFGg7juDdUg6AnjY9m3d7jsiIgbW1QFkSROAz9F4RNRO+1fLUUyYnuKmERF16fadwU7AjsBtkh4AZgK3SHrtQI1tz7HdZ7tvk6lbdDHMiIje0tU7A9t3AFut/lwlhD7bT3YzjoiIWFPdU0vnATcCu0laIumEOvuLiIjhqfXOYJAVyM3fz2r3Wn+2+SZjflFHRMRo1fXZRBERMfqMmXIUS595ie9c/njpMCJiA3Pye7NxFhQoRyHpi5IelrSweh1aZwwREdFakXIUwDdt71G9flZzDBER0UKpchQRETGKlBpAPkXS7dVjpGmFYoiIiEqJZHAOjZXIewCPAl8frKGk2ZL6JfWvXJ4bjIiIunQ9Gdh+3PbLtl8BzgX2GqLtq+UoJk1JOYqIiLqUqFq6TdPH9wLrbHwTERHdVes6g6ocxf7AdElLgDOB/SXtQWMvgweAj9UZQ0REtCZ7bOwv09fX5/7+/tJhRESMKZIW2O5r1S7lKCIiYuyUo3h22Squ+kEqXUfE+jvkqOmlQxh1ul6Oojr+CUn3SLpT0ll1xhAREa11vRyFpAOAI4A32f5z4Gs1xxARES2UKEdxEvAV2y9UbZbWGUNERLRWYgB5V2BfSTdL+qWk/z5Yw+YVyMuXP9XFECMiekuJZDAe2ALYG/g0cLEkDdSweQXylClbdjPGiIieUiIZLAEuc8N/Aq8AGdqPiCioRDL4EXAAgKRdgY2BzBmNiCioRDmK84Hzq+mmLwLHeqwsg46I2EDVmgxsHz3IV8d0eq2p08ZnoUhERE1SjiIiIsZOOYrnn1zFredlSUJEjJw3/4+tSocwanS9HIWkH0haWL0ekLSwzhgiIqK1uu8M5gL/DPzL6gO2j1r9XtLXgWdrjiEiIlqoewB5vqRZA31XLTT7IHBgnTFERERrJQeQ9wUet714sAbN5SiWrUg5ioiIupRMBkcD84Zq0FyOYtrklKOIiKhLkdlEksYD7wPeUqL/iIhYU6k7g3cB99heUqj/iIhoUvfU0nnAjcBukpZIOqH66kO0eEQUERHdU6Qche3jOr3WhOnjs0AkIqImKUcRERFjpxzFS4+9xKNnPVw6jIjYgGzzme1KhzBqlChHsYekm6pyFP2S9qozhoiIaK3ux0RzgYPXOnYW8CXbewB/V32OiIiCak0GtucDT699GJhSvZ8KPFJnDBER0VqJMYPTgGskfY1GMnrbYA0lzQZmA2y3eZ7tRUTUpcRsopOAT9neHvgU8L3BGjaXo9hyYspRRETUpUQyOBa4rHr/QyADyBERhZVIBo8A76jeHwgMWrU0IiK6o9Yxg6ocxf7AdElLgDOBE4FvV8Xq/kg1JhAREeUUKUfBMKqVbvTajbJAJCKiJilHERERY6gcxePP8/i3FpQOIyK6aOvTsuVJt5QoR/EmSTdKukPSTyRNGeoaERFRvxLlKM4DzrD9F8DlwKdrjiEiIlooUY5iV2B+9f464P11xhAREa2VGEC+Eziiev8BYPvBGkqaXVU27X/6uWVdCS4ioheVSAYfBT4uaQEwGXhxsIbN5Si2mDitawFGRPSars8msn0PcBCApF2B93Q7hoiIWFPX7wwkbVX9fQ3wBeD/djuGiIhYU4lyFJMknVw1uQy4oJ1rbbT1hMw5joioSalyFN+us9+IiOhMylFERMTYKUexaulylv7ztaXDiIgu2uqUg0qH0DPqLkexvaQbJN0l6U5Jp1bHt5B0naTF1d/MG42IKKjux0SrgNNtvwHYGzhZ0huAM4Drbe8CXF99joiIQuouR/Go7Vuq9yuAu4HtaKxAvrBqdiHw13XGERERQ+vaALKkWcCbgZuBrW0/Wn31GLD1IOe8Wo7iqZXPdiXOiIhe1JVkIGkScClwmu3lzd/ZNuCBzmsuR7HlpKldiDQiojfVngwkbUQjEVxk+7Lq8OOStqm+3wZYWnccERExuLpnEwn4HnC37W80fXUFcGz1/ljgx3XGERERQ1PjKU1NF5feDvwKuAN4pTr8ORrjBhcDrwMeBD5oe+19D9bQ19fn/v7+2mKNiNgQSVpgu69luzqTwUiStAL4bek4CpkOPFk6iAJ69XdDfnsv/va6fvcOtme0ajRmViADv20nu22IJPX34m/v1d8N+e29+NtL/+7UJoqIiCSDiIgYW8lgTukACurV396rvxvy23tR0d89ZgaQIyKiPmPpziAiImqSZBAREaM/GUg6WNJvJd0rqWdKXQ+2F0QvkTRO0q2Sflo6lm6StLmkSyTdI+luSW8tHVM3SPpU9b/1RZLmSdq0dEx1kXS+pKWSFjUdK7rPy6hOBpLGAd8BDgHeABxd7YfQCwbbC6KXnEqj7Hmv+TZwte3XA2+iB/4zkLQd8Emgz/buwDjgQ2WjqtVc4OC1jhXd52VUJwNgL+Be2/fbfhH4Nxp7IWzwhtgLoidImgm8BzivdCzdJGkqsB+Nml7YftH2M2Wj6prxwGaSxgMTgEcKx1Mb2/OBtUvwFN3nZbQng+2APzR9XkIP/R/iamvtBdErvgV8hj/VtOoVOwJPABdUj8jOkzSxdFB1s/0w8DXgIeBR4FnbvbbpeVv7vNRltCeDnjfUXhAbKkmHAUttLygdSwHjgT2Bc2y/GXiOHtgWtno+fgSNZLgtMFHSMWWjKmeofV7qMtqTwcPA9k2fZ1bHesIge0H0gn2AwyU9QOPR4IGSvl82pK5ZAiyxvfou8BIayWFD9y7g97afsP0ScBnwtsIxdVvRfV5GezL4L2AXSTtK2pjGgNIVhWPqiiH2gtjg2f6s7Zm2Z9H47/zntnviX4m2HwP+IGm36tA7gbsKhtQtDwF7S5pQ/W//nfTAwPlaiu7zMqqrltpeJekU4BoaswvOt31n4bC6ZR/gI8AdkhZWxz5n+2cFY4ru+ARwUfUPoPuB4wvHUzvbN0u6BLiFxky6W9mAy1JImgfsD0yXtAQ4E/gKcLGkE6j2eelqTClHERERo/0xUUREdEGSQUREJBlERESSQUREkGQQEREkGUQPqKqAfnyY554macJIxxQx2iQZRC/YHBhWMgBOo1E0rS1Vpd2IMSfJIHrBV4CdJC2UdLakT0v6L0m3S/oSgKSJkq6UdFtVT/8oSZ+kUSfnBkk3DHZxSSslfV3SbcBbJT0g6R+r/vol7SnpGkn3Sfrb6pxtJM2v2iyStG91/CBJN0q6RdIPq9pUEbXLorPY4FVVX39qe3dJBwFHAh8DRKMEwFnADOBg2ydW50y1/WxVH6nP9pNDXN/AUbYvrj4/AHzV9jmSvkmjtMI+wKbAIttbSzod2NT2P1R3ExOATWjU5DnE9nOS/hewie2/H+H/SCLWMarLUUTU4KDqdWv1eRKwC/Ar4OuSvkojcfyqg2u+TKOgYLPVNbTuACZVe1KskPSCpM1p1N06vypG+CPbCyW9g8YmTr9plOdhY+DGjn9hxDAkGUSvEfCPtr+7zhfSnsChwJclXd/Bv8j/aPvltY69UP19pen96s/jbc+XtB+NDXzmSvoGsAy4zvbRHfyeiBGRMYPoBSuAydX7a4CPrn4WL2k7SVtJ2hZ43vb3gbP5U9no5nNHjKQdgMdtn0tjN7c9gZuAfSTtXLWZKGnXke47YiC5M4gNnu2nJP2m2nz8KuBfgRurRzErgWOAnYGzJb0CvAScVJ0+B7ha0iO2DxjBsPYHPi3ppSqGv7H9hKTjgHmSNqnafQH43Qj2GzGgDCBHREQeE0VERB4TRbRN0s00pn82+4jtO0rEEzGS8pgoIiLymCgiIpIMIiKCJIOIiCDJICIiSDKIiAjg/wOOqPpBpvGU2QAAAABJRU5ErkJggg==\n",
"text/plain": [
"<Figure size 432x288 with 1 Axes>"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"sns.barplot(y='n_predictors', x='test_rmse', data=df, orient='h')"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"(f) How does the model at which the test set MSE is minimized compare to the true model used to generate the data? Comment on the coefficient values.\n"
]
},
{
"cell_type": "code",
"execution_count": 311,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"/usr/local/lib/python3.6/site-packages/scipy/stats/stats.py:1394: UserWarning: kurtosistest only valid for n>=20 ... continuing anyway, n=10\n",
" \"anyway, n=%i\" % int(n))\n"
]
},
{
"data": {
"text/html": [
"<table class=\"simpletable\">\n",
"<caption>OLS Regression Results</caption>\n",
"<tr>\n",
" <th>Dep. Variable:</th> <td>y</td> <th> R-squared: </th> <td> 0.889</td>\n",
"</tr>\n",
"<tr>\n",
" <th>Model:</th> <td>OLS</td> <th> Adj. R-squared: </th> <td> 0.800</td>\n",
"</tr>\n",
"<tr>\n",
" <th>Method:</th> <td>Least Squares</td> <th> F-statistic: </th> <td> 9.993</td>\n",
"</tr>\n",
"<tr>\n",
" <th>Date:</th> <td>Mon, 01 Oct 2018</td> <th> Prob (F-statistic):</th> <td>0.0133</td> \n",
"</tr>\n",
"<tr>\n",
" <th>Time:</th> <td>19:14:07</td> <th> Log-Likelihood: </th> <td> -19.122</td>\n",
"</tr>\n",
"<tr>\n",
" <th>No. Observations:</th> <td> 10</td> <th> AIC: </th> <td> 48.24</td>\n",
"</tr>\n",
"<tr>\n",
" <th>Df Residuals:</th> <td> 5</td> <th> BIC: </th> <td> 49.76</td>\n",
"</tr>\n",
"<tr>\n",
" <th>Df Model:</th> <td> 4</td> <th> </th> <td> </td> \n",
"</tr>\n",
"<tr>\n",
" <th>Covariance Type:</th> <td>nonrobust</td> <th> </th> <td> </td> \n",
"</tr>\n",
"</table>\n",
"<table class=\"simpletable\">\n",
"<tr>\n",
" <td></td> <th>coef</th> <th>std err</th> <th>t</th> <th>P>|t|</th> <th>[0.025</th> <th>0.975]</th> \n",
"</tr>\n",
"<tr>\n",
" <th>Intercept</th> <td> -7.5641</td> <td> 3.561</td> <td> -2.124</td> <td> 0.087</td> <td> -16.717</td> <td> 1.589</td>\n",
"</tr>\n",
"<tr>\n",
" <th>x10</th> <td> 0.6322</td> <td> 0.298</td> <td> 2.121</td> <td> 0.087</td> <td> -0.134</td> <td> 1.398</td>\n",
"</tr>\n",
"<tr>\n",
" <th>x11</th> <td> 1.3608</td> <td> 0.387</td> <td> 3.513</td> <td> 0.017</td> <td> 0.365</td> <td> 2.357</td>\n",
"</tr>\n",
"<tr>\n",
" <th>x13</th> <td> 0.6353</td> <td> 0.307</td> <td> 2.072</td> <td> 0.093</td> <td> -0.153</td> <td> 1.423</td>\n",
"</tr>\n",
"<tr>\n",
" <th>x16</th> <td> 0.6766</td> <td> 0.384</td> <td> 1.761</td> <td> 0.139</td> <td> -0.311</td> <td> 1.664</td>\n",
"</tr>\n",
"</table>\n",
"<table class=\"simpletable\">\n",
"<tr>\n",
" <th>Omnibus:</th> <td> 1.865</td> <th> Durbin-Watson: </th> <td> 1.442</td>\n",
"</tr>\n",
"<tr>\n",
" <th>Prob(Omnibus):</th> <td> 0.394</td> <th> Jarque-Bera (JB): </th> <td> 1.269</td>\n",
"</tr>\n",
"<tr>\n",
" <th>Skew:</th> <td>-0.768</td> <th> Prob(JB): </th> <td> 0.530</td>\n",
"</tr>\n",
"<tr>\n",
" <th>Kurtosis:</th> <td> 2.173</td> <th> Cond. No. </th> <td> 58.1</td>\n",
"</tr>\n",
"</table><br/><br/>Warnings:<br/>[1] Standard Errors assume that the covariance matrix of the errors is correctly specified."
],
"text/plain": [
"<class 'statsmodels.iolib.summary.Summary'>\n",
"\"\"\"\n",
" OLS Regression Results \n",
"==============================================================================\n",
"Dep. Variable: y R-squared: 0.889\n",
"Model: OLS Adj. R-squared: 0.800\n",
"Method: Least Squares F-statistic: 9.993\n",
"Date: Mon, 01 Oct 2018 Prob (F-statistic): 0.0133\n",
"Time: 19:14:07 Log-Likelihood: -19.122\n",
"No. Observations: 10 AIC: 48.24\n",
"Df Residuals: 5 BIC: 49.76\n",
"Df Model: 4 \n",
"Covariance Type: nonrobust \n",
"==============================================================================\n",
" coef std err t P>|t| [0.025 0.975]\n",
"------------------------------------------------------------------------------\n",
"Intercept -7.5641 3.561 -2.124 0.087 -16.717 1.589\n",
"x10 0.6322 0.298 2.121 0.087 -0.134 1.398\n",
"x11 1.3608 0.387 3.513 0.017 0.365 2.357\n",
"x13 0.6353 0.307 2.072 0.093 -0.153 1.423\n",
"x16 0.6766 0.384 1.761 0.139 -0.311 1.664\n",
"==============================================================================\n",
"Omnibus: 1.865 Durbin-Watson: 1.442\n",
"Prob(Omnibus): 0.394 Jarque-Bera (JB): 1.269\n",
"Skew: -0.768 Prob(JB): 0.530\n",
"Kurtosis: 2.173 Cond. No. 58.1\n",
"==============================================================================\n",
"\n",
"Warnings:\n",
"[1] Standard Errors assume that the covariance matrix of the errors is correctly specified.\n",
"\"\"\""
]
},
"execution_count": 311,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"best = df.sort_values('test_rmse', ascending=True).iloc[0]['model']\n",
"best.summary()"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"(g) Create a plot displaying: `sqrt(sum ((bj - bjr) ^ 2))` for a range of values\n",
"of r, where βˆjr is the jth coefficient estimate for the best model containing r coefficients. Comment on what you observe. How does this compare to the test MSE plot from (d)?"
]
},
{
"cell_type": "code",
"execution_count": 401,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"<matplotlib.axes._subplots.AxesSubplot at 0x1215fd2e8>"
]
},
"execution_count": 401,
"metadata": {},
"output_type": "execute_result"
},
{
"data": {
"image/png": "iVBORw0KGgoAAAANSUhEUgAAAmIAAAJQCAYAAAAt5AhAAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzs3XuUZGV97//3RwZELsNtABHQMYp4CImAfVgiahCiBy+BoERlqQHDkRyjBjyu+PN2ouYkaxnjLSdmmSDiYEKIyCXiFYgxEBMlpwdRQDCgER1EhvsAngAD398ftcc0w/R0V9NVT1X1+7VWra7a9dTen1S3zDfP3vv5pqqQJEnS8D2mdQBJkqSlykJMkiSpEQsxSZKkRizEJEmSGrEQkyRJasRCTJIkqRELMUmSpEYsxCRJkhqxEJMkSWpkWesA87VixYpauXJl6xiSJElzWr169a1Vtetc48amENtr2+V8+cRTWseQNItd3/Ca1hEkaWQkuWE+4zw1KUmS1EizQizJ6UnWJrmqVQZJkqSWWs6IrQKObHh8SZKkppoVYlV1KXB7q+NLkiS1NtLXiCU5Kcl0kunb7lnXOo4kSdKiGulCrKpOraqpqpraZbvlreNIkiQtqpEuxCRJkiaZhZgkSVIjLZevOAv4BrBvkjVJTmyVRZIkqYVmK+tX1XH9jF+2686u3C1JkiaKpyYlSZIaGZtekw/cchM//fgfto4hacge/4Z3t44gSQPjjJgkSVIjTQqxJHsn+VqS7ya5OsnJLXJIkiS11OrU5HrgrVV1eZLtgdVJLq6q7zbKI0mSNHRNZsSq6qaqurx7fjdwDbBniyySJEmtNL9GLMlK4EDgsk28N6PX5L3DjiZJkjRQTQuxJNsB5wKnVNUjuno/vNfktsMPKEmSNEAtV9bfkl4RdmZVndcqhyRJUiut7poM8Engmqr6cIsMkiRJrbWaETsUeC1weJIruseLG2WRJElqosnyFVX1dSD9fGbLXfdwhW1JkjRRmt81KUmStFRZiEmSJDUyNk2//2Pt9Vz750e3jjFvT3/j51pHkCRJI84ZMUmSpEZaLV+xdZJ/TfLtrun3+1rkkCRJaqnVqcn7gMOr6p5uYdevJ/lyVX2zUR5JkqSha7V8RQH3dC+37B7VIoskSVIrLVscbZHkCmAtcHFVbbbp9x333D/8kJIkSQPUrBCrqger6gBgL+DgJPtvYszPm37vtN1Www8pSZI0QM3vmqyqO4GvAUe2ziJJkjRMre6a3DXJjt3zxwEvAK5tkUWSJKmVVndN7gGckWQLesXg2VX1hUZZJEmSmmh11+R3gAP7+czWuz3V1eolSdJEaX6NmCRJ0lI1Nr0m7771Ov7xEy9pHWMkHPb6L7aOIEmSFoEzYpIkSY00LcS6RV2/lcQL9SVJ0pLTekbsZOCaxhkkSZKaaNniaC/gJcBprTJIkiS11HJG7KPA24CHZhsws9fkXXfba1KSJE2WVivrvxRYW1WrNzduZq/JHba316QkSZosrWbEDgWOSvJD4G+Bw5P8daMskiRJTTQpxKrqHVW1V1WtBF4F/ENVvaZFFkmSpFZa3zUpSZK0ZKWqWmeYl6mpqZqenm4dQ5IkaU5JVlfV1FzjnBGTJElqZGx6Td5x63Wc86kjW8eQRt6xr/tK6wiSpHlyRkySJKmRZjNi3dIVdwMPAuvncx5VkiRpkrQ+Nfn8qrq1cQZJkqQmPDUpSZLUSMtCrICLkqxOctKmBszsNbnuHntNSpKkydLy1ORzqurGJLsBFye5tqounTmgqk4FTgV4ysodxmPBM0mSpHlqNiNWVTd2P9cC5wMHt8oiSZLUQpNCLMm2Sbbf8Bx4IXBViyySJEmttDo1uTtwfpINGf6mqlyFUpIkLSlNCrGq+gHwjH4+s9OKfVwxXJIkTRSXr5AkSWrEQkySJKmR1ivrz9stt13HX/7Vf2sdQ5JG2m+/9sLWEST1wRkxSZKkRpoVYkl2THJOkmuTXJPkkFZZJEmSWmh5avJPga9U1bFJtgK2aZhFkiRp6JoUYkl2AJ4HnABQVfcDNpOUJElLSqtTk08GbgE+leRbSU7rVth/mJlNv++52zpNkiRNllaF2DLgIODjVXUgcC/w9o0HVdWpVTVVVVPbbb/VsDNKkiQNVKtCbA2wpqou616fQ68wkyRJWjKaFGJV9VPgx0n27TYdAXy3RRZJkqRWWt41+WbgzO6OyR8Ar2uYRZIkaehSVa0zzMvU1FRNT0+3jiFJkjSnJKuramquca6sL0mS1MjY9Jr8yR3X8d6z7TUpScPy3lfYt1IaNGfEJEmSGmlSiCXZN8kVMx7rkpzSIoskSVIrTU5NVtX3gAMAkmwB3Aic3yKLJElSK6NwavII4PtVdUPrIJIkScM0CoXYq4CzNvXGzF6TP1tnr0lJkjRZmhZi3WKuRwGf3dT7M3tNbrPcXpOSJGmytJ4RexFweVXd3DiHJEnS0LUuxI5jltOSkiRJk65ZIZZkW+AFwHmtMkiSJLXUbGX9qroX2GW+45+w0z6u8ixJkiZK61OTkiRJS5aFmCRJUiNj0/T7uju/z4s+9/LWMSRJeoQvH31u6wgaUy0v1n9LkquTXJXkrCRbt8oiSZLUQqum33sCvwtMVdX+wBb0VtiXJElaMlpeI7YMeFySZcA2wE8aZpEkSRq6JoVYVd0IfBD4EXATcFdVXbTxuJm9Ju9fd9+wY0qSJA1Uq1OTOwFHA08GngBsm+Q1G4+b2Wtyq+WPHXZMSZKkgWp1avJXgX+vqluq6gF6q+s/u1EWSZKkJuZdiCXZNsljuudPS3JUki0XeNwfAc9Ksk2SAEcA1yxwX5IkSWOpnxmxS4GtuzseLwJeC6xayEGr6jLgHOBy4Moux6kL2ZckSdK4SlXNb2ByeVUdlOTNwOOq6gNJrqiqAwYbsWdqaqqmp6eHcShJkqRHJcnqqpqaa1w/M2JJcgjwauCL3bYtFhJOkiRJ/RViJwPvAM6vqquT/ALwtcHEkiRJmnzz6jWZZAvgqKo6asO2qvoBvdXxh+K6O2/ixef/4bAOJ0nSI3zpmHe3jqAJM68Zsap6EHjOgLNIkiQtKfOaEet8K8kFwGeBezdsrKrzFnLgJCcDrwcCfKKqPrqQ/UiSJI2rfgqxrYHbgMNnbCt6i7H2Jcn+9Iqwg4H7ga8k+UJVXd/vviRJksbVvAuxqnrdIh73vwCXVdXPAJJcArwM+MAiHkOSJGmk9bOy/l5Jzk+ytnucm2SvBR73KuC5SXZJsg3wYmDvTRxzRtPvex+xE0mSpHHWz/IVnwIuoNek+wnA57ttfauqa4A/prdC/1eAK4AHNzFuRtPvbRdyKEmSpJHVTyG2a1V9qqrWd49VwK4LPXBVfbKqnllVzwPuAP5tofuSJEkaR/0UYrcleU2SLbrHa+hdvL8gSXbrfj6R3vVhf7PQfUmSJI2jfu6a/C3gz4CP0Ltb8l+AEx7Fsc9NsgvwAPDGqrrzUexLkiRp7PRTiO01c2V9gCSHAj9eyIGr6rn9jN9nxz1c0ViSJE2Ufk5N/tk8t0mSJGke5pwRS3II8Gxg1yT/c8Zby4EtBhVsY9fdeQsvOe/jwzqcJGmEffFlb2gdQVoU8zk1uRWwXTd2+xnb1wHHDiKUJEnSUjBnIVZVlwCXJFlVVTf0s/MkpwMvBdZW1f7dtp2BzwArgR8Cr6iqO/rMLUmSNPb6uUbstCQ7bniRZKckF87xmVXAkRttezvw1araB/hq91qSJGnJ6acQWzFziYluFmu3zX2gqi4Fbt9o89HAGd3zM4Bf7yODJEnSxOinEHuoW3wVgCRPoreeWL92r6qbuuc/BXafbeDDek3edc8CDiVJkjS6+llH7F3A15NcAgR4LnDSozl4VVWSWYu5qjoVOBVgh6c+aSFFnyRJ0siadyFWVV9JchDwrG7TKVV16wKOeXOSParqpiR7AGsXsA9JkqSxN+epySRP734eBDwR+En3eGK3rV8XAMd3z48HPreAfUiSJI29+cyIvRV4PfChTbxXwOGzfTDJWcBhwIoka4D3AO8Hzk5yInAD8Io+M0uSJE2EVI3HpVdTU1M1PT3dOoYkSdKckqyuqqm5xs2nxdHLNvd+VZ3XTzBJkiT1zOfU5K91P3ej13PyH7rXzwf+BbAQkyRJWoD5tDh6HUCSi4D9NqwB1t3xuGqg6Wa4/o7beek5Zw7rcJKkEfaFY1/dOoK0KPpZ0HXvGQuxAtxM7y7KWSU5PcnaJFfN2PYbSa5O8lCSOc+dSpIkTap+CrGvJrkwyQlJTgC+CPz9HJ9ZxSN7TV4FvAy4tI9jS5IkTZx+FnR9U5JjgOd1m06tqvPn+MylSVZutO0agCT9JZUkSZow/bQ4ArgcuLuq/j7JNkm2r6q7BxFMkiRp0s371GSS1wPnAH/ZbdoT+LtBhJpxzP9s+r1u3SAPJUmSNHT9XCP2RuBQYB1AVV1Hb0mLgamqU6tqqqqmtlq+fJCHkiRJGrp+CrH7qur+DS+SLKPX4kiSJEkL0E8hdkmSdwKPS/IC4LPA5zf3ga7X5DeAfZOsSXJikmO6vpOHAF9McuFCw0uSJI2zefeaTPIY4ETghUCAC4HTakjNKu01KUmSxsWi9ZrsdrYF8OmqejXwiUcbTpIkSfM8NVlVDwJPSrLVgPNIkiQtGf2sI/YD4J+TXADcu2FjVX140VNtwvV33MVR52z2kjRJkh6VC479tdYRtMT0U4h9v3s8Bth+MHEkSZKWjn5aHL0PIMny3su5V9RPcjrwUmBtVe3fbfsT4NeA++kVdq+rqjsXkF2SJGms9bOy/lSSK4HvAFcm+XaSZ87xsVU8sun3xcD+VfXLwL8B7+gjryRJ0sToZx2x04HfqaqVVbWS3kr7n9rcB6rqUuD2jbZdVFXru5ffBPbqI4MkSdLE6KcQe7Cq/mnDi6r6OrB+M+Pn47eAL8/25sN7Td71KA8lSZI0Wvq5WP+SJH8JnEWvtdErgX9MchBAVV3ez4GTvIteIXfmbGOq6lTgVIAdn7KP7ZQkSdJE6acQe0b38z0bbT+QXmF2+Hx3lOQEehfxHzGslfklSZJGTT93TT5/c+8nOb6qzphrP0mOBN4G/EpV/Wy+x5ckSZo0/VwjNpeTN96wqabfwMforUN2cZIrkvzFImaQJEkaG/2cmpxLNt5QVcdtYtwnF7Lzp+60gyseS5KkibKYM2Je6yVJktSHgc6ILabv33EPx5z79UEeQpKkeTv/5c9pHUETYN6FWJLHAi8HVs78XFX9Qff0nxc1mSRJ0oTr59Tk54Cj6a39de+MBwBV9aaNP5Dk9CRrk1w1Y9v/TvKd7kL9i5I8YeHxJUmSxlc/pyb3qqqN+0bOZRW9uyQ/PWPbn1TV/wJI8rvA7wP/o8/9SpIkjb1+ZsT+Jckv9bPzWXpNrpvxclu8yF+SJC1R/cyIPQc4Icm/A/fRuzi/quqX+z1okj8CfhO4C5h1odgkJwEnATxuxe79HkaSJGmk9VOIvWixDlpV7wLeleQdwJt4ZNukDeN+3mtyp6c83ZkzSZI0UfppcXTDAI5/JvAlZinEJEmSJtliLug6L0n2mfHyaODaYWeQJEkaBYu5oOsjdL0mDwNWJFlDb+brxUn2BR4CbsA7JiVJ0hKVqvG49Gpqaqqmp6dbx5AkSZpTktVVNTXXuKGfmpQkSVKPhZgkSVIjA71GbDH94M77eOV517eOIUkSAJ952VNbR9AEGOiM2KZ6Tc54761JKsmKQWaQJEkaVYM+NbkKeER/yiR7Ay8EfjTg40uSJI2sgRZim+o12fkI8DbsMylJkpawFgu6Hg3cWFXfHvaxJUmSRslQL9ZPsg3wTnqnJecz/udNv7dZ8YQBJpMkSRq+Yc+IPQV4MvDtJD8E9gIuT/L4TQ2uqlOraqqqph67w85DjClJkjR4Q50Rq6orgd02vO6KsamqunWYOSRJkkbBoJevOAv4BrBvkjVJThzk8SRJksbJQGfEquq4Od5fOcjjS5IkjbKxWVn/F3Z8rKsYS5KkiWKvSUmSpEbGZkZs7Z0P8Ofn39w6hiRpiXjjMbu3jqAlwBkxSZKkRobe9DvJe5PcmOSK7vHiQWaQJEkaVU2afgMfqaoDuseXBpxBkiRpJLVq+i1JkrTktbpG7E1JvtOdutxptkFJTkoynWT6nnXWc5IkabK0KMQ+Tq/n5AHATcCHZhs4s9fkdsvtNSlJkibL0Auxqrq5qh6sqoeATwAHDzuDJEnSKBh6IZZkjxkvjwGumm2sJEnSJBvogq5d0+/DgBVJ1gDvAQ5LcgBQwA+B3x5kBkmSpFGVqmqdYV6mpqZqenq6dQxJkqQ5JVldVVNzjXNlfUmSpEbGptfkXXes58ufubV1DEmSxsqLXrmidQRthjNikiRJjQy912S3/c1Jrk1ydZIPDDKDJEnSqBp6r8kkzweOBp5RVb8IfHDAGSRJkkZSi16TbwDeX1X3dWPWDjKDJEnSqGpxjdjTgOcmuSzJJUn+62wDZ/aaXLfutiFGlCRJGrwWhdgyYGfgWcDvAWcnyaYGzuw1uXz5LsPMKEmSNHAtCrE1wHnV86/AQ4D31kqSpCWnRSH2d8DzAZI8DdgKcIEwSZK05LToNXk6cHq3pMX9wPE1Ln2WJEmSFtFAC7GqOm6Wt17T77522GmZqwNLkqSJ4sr6kiRJjViISZIkNTI2Tb9/dut6vnWaa79KkgbnwP++W+sIWmKcEZMkSWpk6E2/k3wmyRXd44dJrhhkBkmSpFE16FOTq4CPAZ/esKGqXrnheZIPAXcNOIMkSdJIGvTyFZcmWbmp97q2Rq8ADh9kBkmSpFHV8hqx5wI3V9V1sw2Y2fT7jrtt+i1JkiZLy0LsOOCszQ2Y2fR7p+1t+i1JkiZLk+UrkiwDXgY8s8XxJUmSRkGrGbFfBa6tqjWNji9JktTcoJevOAv4BrBvkjVJTuzeehVznJaUJEmadE2aflfVCf3ua5sVy1zxWJIkTRRX1pckSWpkbHpNPvDTB7jpAze2jiFJUnN7vG3P1hG0SJwRkyRJaqRFr8kDknyz6zU5neTgQWaQJEkaVYOeEVsFHLnRtg8A76uqA4Df715LkiQtOQMtxKrqUuD2jTcDy7vnOwA/GWQGSZKkUdXiYv1TgAuTfJBeIfjs2QYmOQk4CWDPHb0wUZIkTZYWF+u/AXhLVe0NvAX45GwDZ/aa3GVbe01KkqTJ0qIQOx44r3v+WcCL9SVJ0pLUohD7CfAr3fPDgesaZJAkSWpuoNeIdb0mDwNWJFkDvAd4PfCnSZYB/0F3DZgkSdJS06TXJPDMfve15eO3dCVhSZI0UVxZX5IkqZHx6TV588+4+aOrW8eQJEkLtPspfZ8Qm3jOiEmSJDXSotfkM5J8I8mVST6fZPnm9iFJkjSpWvSaPA14e1X9EnA+8HsDziBJkjSSWvSafBpwaff8YuDlg8wgSZI0qlpcI3Y1cHT3/DeAvWcbmOSkJNNJpm+/946hhJMkSRqWFoXYbwG/k2Q1sD1w/2wDZ/aa3HnbnYYWUJIkaRiGvnxFVV0LvBAgydOAlww7gyRJ0igY+oxYkt26n48B3g38xbAzSJIkjYJBL19xFvANYN8ka5KcCByX5N+Aa+k1AP/UIDNIkiSNqlRV6wzzMjU1VdPT061jSJIkzSnJ6qqammucK+tLkiQ1YiEmSZLUyNg0/V6/dh1rP3ZR6xiSJGlM7famF7aO8AjOiEmSJDUy6Lsm907ytSTfTXJ1kpO77TsnuTjJdd1PV2uVJElLzqBnxNYDb62q/YBnAW9Msh/wduCrVbUP8NXutSRJ0pIy6KbfN1XV5d3zu4FrgD3p9Zo8oxt2BvDrg8whSZI0ioZ2jViSlcCBwGXA7lV1U/fWT4HdZ/nMz5t+33bPXUPJKUmSNCxDKcSSbAecC5xSVetmvle9FWU3uarszKbfu2y3wxCSSpIkDc/AC7EkW9Irws6sqvO6zTcn2aN7fw9g7aBzSJIkjZpB3zUZ4JPANVX14RlvXQAc3z0/HvjcIHNIkiSNokEv6Hoo8FrgyiRXdNveCbwfOLtrAn4D8IoB55AkSRo5Ay3EqurrQGZ5+4h+9rVst+UjuSKuJEnSQqV3rfzoS3I38L3WOSbMCuDW1iEmjN/p4vM7XXx+p4vP73Txjft3+qSq2nWuQWPTaxL4XlVNtQ4xSZJM+50uLr/Txed3uvj8Thef3+niWyrfqb0mJUmSGrEQkyRJamScCrFTWweYQH6ni8/vdPH5nS4+v9PF53e6+JbEdzo2F+tLkiRNmnGaEZMkSZooFmKSJEmNjFwhluTIJN9Lcn2St2/i/ccm+Uz3/mVJVg4/5XiZx3d6QpJbklzRPf57i5zjIsnpSdYmuWqW95Pk/3Tf93eSHDTsjONmHt/pYUnumvE3+vvDzjhukuyd5GtJvpvk6iQnb2KMf6t9mOd36t9qH5JsneRfk3y7+07ft4kxE/3v/kgVYkm2AP4ceBGwH3Bckv02GnYicEdVPRX4CPDHw005Xub5nQJ8pqoO6B6nDTXk+FkFHLmZ918E7NM9TgI+PoRM424Vm/9OAf5pxt/oHwwh07hbD7y1qvYDngW8cRP/2/dvtT/z+U7Bv9V+3AccXlXPAA4AjkzyrI3GTPS/+yNViAEHA9dX1Q+q6n7gb4GjNxpzNHBG9/wc4Iiuubg2bT7fqfpQVZcCt29myNHAp6vnm8COSfYYTrrxNI/vVH2qqpuq6vLu+d3ANcCeGw3zb7UP8/xO1Yfub++e7uWW3WPjuwgn+t/9USvE9gR+POP1Gh75R/7zMVW1HrgL2GUo6cbTfL5TgJd3pybOSbL3cKJNrPl+5+rPId3piy8n+cXWYcZJdyrnQOCyjd7yb3WBNvOdgn+rfUmyRZIrgLXAxVU169/pJP67P2qFmNr4PLCyqn4ZuJj//P88pFFxOb2+bc8A/gz4u8Z5xkaS7YBzgVOqal3rPJNgju/Uv9U+VdWDVXUAsBdwcJL9W2caplErxG4EZs7G7NVt2+SYJMuAHYDbhpJuPM35nVbVbVV1X/fyNOCZQ8o2qebzd6w+VNW6DacvqupLwJZJVjSONfKSbEmvYDizqs7bxBD/Vvs013fq3+rCVdWdwNd45PWiE/3v/qgVYv8X2CfJk5NsBbwKuGCjMRcAx3fPjwX+oVyVdnPm/E43uibkKHrXPWjhLgB+s7sj7VnAXVV1U+tQ4yzJ4zdcE5LkYHr/7ZqY/xAPQvd9fRK4pqo+PMsw/1b7MJ/v1L/V/iTZNcmO3fPHAS8Art1o2ET/u7+sdYCZqmp9kjcBFwJbAKdX1dVJ/gCYrqoL6P2P4K+SXE/v4t5XtUs8+ub5nf5ukqPo3RF0O3BCs8BjIMlZwGHAiiRrgPfQu8CUqvoL4EvAi4HrgZ8Br2uTdHzM4zs9FnhDkvXA/wNeNUn/IR6QQ4HXAld2198AvBN4Ivi3ukDz+U79W+3PHsAZ3R3+jwHOrqovLKV/921xJEmS1MionZqUJElaMizEJEmSGrEQkyRJasRCTJIkqRELMUmSpEYsxCQtSUnumXuUJA2WhZgkSVIjFmKSJkKS9yd544zX703y7iRfTXJ5kiuTHL2Jzx2W5AszXn8syQnd82cmuSTJ6iQXbtSFQpIeNQsxSZPiM8ArZrx+Bb0G9sdU1UHA84EPbWg/M5eup+CfAcdW1TOB04E/WtzIkpa6kWpxJEkLVVXfSrJbkicAuwJ3AD8FPpLkecBDwJ7A7t32uewL7A9c3NVuWwD2YZS0qCzEJE2Sz9Lr9fd4ejNkr6ZXlD2zqh5I8kNg640+s56Hnx3Y8H6Aq6vqkIEmlrSkeWpS0iT5DL2GwMfSK8p2ANZ2RdjzgSdt4jM3APsleWySHYEjuu3fA3ZNcgj0TlUm+cWB/18gaUlxRkzSxKiqq5NsD9xYVTclORP4fJIrgWng2k185sdJzgauAv4d+Fa3/f4kxwL/J8kO9P57+VHg6iH9nyNpCUhVtc4gSZK0JHlqUpIkqRELMUmSpEYsxCRJkhqxEJMkSWrEQkySJKkRCzFJkqRGLMQkSZIasRCTJElqxEJMkiSpEQsxSZKkRizEJEmSGrEQkyRJasRCTJIkqRELMUmSpEYsxCRJkhqxEJMkSWrEQkySJKkRCzFJkqRGlrUOMF8rVqyolStXto4hSZI0p9WrV99aVbvONW5sCrGVK1cyPT3dOoYkSdKcktwwn3FjU4itv+V2bvn4X7eOIUkacbu+4TWtI0jz1uwasSSnJ1mb5KpWGSRJklpqebH+KuDIhseXJElqqlkhVlWXAre3Or4kSVJrI718RZKTkkwnmb7tnnWt40iSJC2qkS7EqurUqpqqqqldtlveOo4kSdKiGulCTJIkaZJZiEmSJDXScvmKs4BvAPsmWZPkxFZZJEmSWmi2oGtVHdfP+GW77uwifZIkaaJ4alKSJKkRCzFJkqRGxqbX5AO33MRPP/6HrWNI0pL0+De8u3UEaSI5IyZJktRIk0Isyd5Jvpbku0muTnJyixySJEkttTo1uR54a1VdnmR7YHWSi6vqu43ySJIkDV2TGbGquqmqLu+e3w1cA+zZIoskSVIrza8RS7ISOBC4bBPvzWj6fe+wo0mSJA1U00IsyXbAucApVbVu4/cf3vR72+EHlCRJGqCWLY62pFeEnVlV57XKIUmS1EqruyYDfBK4pqo+3CKDJElSa61mxA4FXgscnuSK7vHiRlkkSZKaaLJ8RVV9HUg/n9ly1z1c2VmSJE2U5ndNSpIkLVVj02vyP9Zez7V/fnTrGJKkMfX0N36udQTpEZwRkyRJaqTVXZNbJ/nXJN/uek2+r0UOSZKkllqdmrwPOLyq7unWE/t6ki9X1Tcb5ZEkSRq6VndNFnBP93LL7lEtskiSJLXScmX9LZJcAawFLq6qzfaavOOe+4cfUpIkaYCaFWJV9WBVHQDsBRycZP9NjPl5r8mdtttq+CElSZIGqPldk1V1J/A14MjWWSRJkoap1V2TuybZsXv+OOAFwLUtskiSJLXS6q7JPYAzkmxBrxg8u6q+0CiLJElSE63umvwOcGA/n9l6t6e6KrIkSZooza/yNuBxAAAZ40lEQVQRkyRJWqosxCRJkhoZm6bfd996Hf/4iZe0jiFJE+ew13+xdQRpyWo6I9Yt6vqtJF6oL0mSlpzWpyZPBq5pnEGSJKmJli2O9gJeApzWKoMkSVJLLWfEPgq8DXioYQZJkqRmWq2s/1JgbVWtnmPcz5t+33W3Tb8lSdJkaTUjdihwVJIfAn8LHJ7krzceNLPp9w7b2/RbkiRNliaFWFW9o6r2qqqVwKuAf6iq17TIIkmS1ErruyYlSZKWrFRV6wzzMjU1VdPT061jSJIkzSnJ6qqammucM2KSJEmNWIhJkiQ1Mja9Ju+49TrO+dSRrWNI0pJw7Ou+0jqCtCQ4IyZJktRIsxmxbg2xu4EHgfXzuaBNkiRpkrQ+Nfn8qrq1cQZJkqQmPDUpSZLUSMtCrICLkqxOctKmBszsNbnuHntNSpKkydLy1ORzqurGJLsBFye5tqounTmgqk4FTgV4ysodxmPlWUmSpHlqNiNWVTd2P9cC5wMHt8oiSZLUQpNCLMm2Sbbf8Bx4IXBViyySJEmttDo1uTtwfpINGf6mqlw9UJIkLSlNCrGq+gHwjH4+s9OKfVzpWZIkTRSXr5AkSWqk9YKu83bLbdfxl3/131rHkCQtQb/92gtbR9CEckZMkiSpkWaFWJIdk5yT5Nok1yQ5pFUWSZKkFlqemvxT4CtVdWySrYBtGmaRJEkauiaFWJIdgOcBJwBU1f2APYwkSdKS0urU5JOBW4BPJflWktO6hV0fZmavyXvutk6TJEmTpVUhtgw4CPh4VR0I3Au8feNBVXVqVU1V1dR222817IySJEkD1aoQWwOsqarLutfn0CvMJEmSlowmhVhV/RT4cZJ9u01HAN9tkUWSJKmVlndNvhk4s7tj8gfA6xpmkSRJGrpUVesM8zI1NVXT09OtY0iSJM0pyeqqmpprnCvrS5IkNWIhJkmS1MjYNP3+yR3X8d6zbfotSXp03vsKG3hrdDgjJkmS1EiTQizJvkmumPFYl+SUFlkkSZJaaXJqsqq+BxwAkGQL4Ebg/BZZJEmSWhmFU5NHAN+vqhtaB5EkSRqmUSjEXgWctak3Zjb9/tk6m35LkqTJ0rQQ61bVPwr47Kben9n0e5vlNv2WJEmTpfWM2IuAy6vq5sY5JEmShq51IXYcs5yWlCRJmnTNCrEk2wIvAM5rlUGSJKmlZivrV9W9wC7zHf+EnfZxNWRJkjRRWp+alCRJWrLGptfkdXd+nxd97uWtY0iS9AhfPvrc1hE0ppwRkyRJaqTlxfpvSXJ1kquSnJVk61ZZJEmSWmjV9HtP4HeBqaraH9iC3gr7kiRJS0bLU5PLgMclWQZsA/ykYRZJkqSha1KIVdWNwAeBHwE3AXdV1UUbj5vZa/L+dfcNO6YkSdJAtTo1uRNwNPBk4AnAtkles/G4mb0mt1r+2GHHlCRJGqhWpyZ/Ffj3qrqlqh6gt7r+sxtlkSRJamLehViSbZM8pnv+tCRHJdlygcf9EfCsJNskCXAEcM0C9yVJkjSW+pkRuxTYurvj8SLgtcCqhRy0qi4DzgEuB67scpy6kH1JkiSNq1TV/AYml1fVQUneDDyuqj6Q5IqqOmCwEXumpqZqenp6GIeSJEl6VJKsrqqpucb1MyOWJIcArwa+2G3bYiHhJEmS1F+vyZOBdwDnV9XVSX4B+NpgYj3SdXfexIvP/8NhHU6SpHn70jHvbh1BY2pehViSLYCjquqoDduq6gf0VseXJEnSAszr1GRVPQg8ZzEPnOTkrs/k1UlOWcx9S5IkjYN+Tk1+K8kFwGeBezdsrKrz+j1okv2B1wMHA/cDX0nyhaq6vt99SZIkjat+CrGtgduAw2dsK3qLsfbrvwCXVdXPAJJcArwM+MAC9iVJkjSW5l2IVdXrFvG4VwF/lGQX4P8BLwYesTZFkpOAkwC23nWHRTy8JElSe/2srL9XkvOTrO0e5ybZayEHraprgD+mtzDsV4ArgAc3MW5Gr8ltF3IoSZKkkdXPOmKfAi6g16T7CcDnu20LUlWfrKpnVtXzgDuAf1voviRJksZRP4XYrlX1qapa3z1WAbsu9MBJdut+PpHe9WF/s9B9SZIkjaN+Lta/LclrgLO618fRu3h/oc7trhF7AHhjVd35KPYlSZI0dvrpNfkk4M+AQ+jdLfkvwJur6seDi/ef7DUpSZLGxXx7TfYzI7bXzJX1u4McCgylEJMkSZo0/Vwj9mfz3CZJkqR5mHNGLMkhwLOBXZP8zxlvLQe2GFSwjV135y285LyPD+twkiRpwnzxZW9oHeER5nNqcitgu27s9jO2rwOOHUQoSZKkpWDOQqyqLgEuSbKqqm7oZ+dJTgdeCqytqv27bTsDnwFWAj8EXlFVd/SZW5Ikaez1c43YaUl23PAiyU5JLpzjM6uAIzfa9nbgq1W1D/DV7rUkSdKS008htmLmWl/dLNZum/tAVV0K3L7R5qOBM7rnZwC/3kcGSZKkidFPIfZQtwo+8PN1xea3CNnD7V5VN3XPfwrsPtvAJCclmU4yff9d9yzgUJIkSaOrn3XE3gV8PcklQIDnAic9moNXVSWZtZirqlOBUwF2eOqTFlL0SZIkjax5F2JV9ZUkBwHP6jadUlW3LuCYNyfZo6puSrIHsHYB+5AkSRp7c56aTPL07udBwBOBn3SPJ3bb+nUBcHz3/HjgcwvYhyRJ0tibz4zYW4HXAx/axHsFHD7bB5OcBRwGrEiyBngP8H7g7CQnAjcAr+gzsyRJ0kSYd9Pv1mz6LUmSxsWiNf1O8rLNvV9V5/UTTJIkST3zOTX5a93P3ej1nPyH7vXzgX8BhlKIXX/H7bz0nDOHcShJkjRCvnDsq1tHGJj5tDh6HUCSi4D9NqwB1t3xuGqg6SRJkiZYPwu67j1jIVaAm+ndRTmrJKcnWZvkqhnbfiPJ1UkeSjLnuVNJkqRJ1U8h9tUkFyY5IckJwBeBv5/jM6t4ZK/Jq4CXAZf2cWxJkqSJ08+Crm9KcgzwvG7TqVV1/hyfuTTJyo22XQOQpL+kkiRJE6afFkcAlwN3V9XfJ9kmyfZVdfcggkGv1yRdG6XHrdhlUIeRJElqYt6nJpO8HjgH+Mtu057A3w0i1AZVdWpVTVXV1FbLlw/yUJIkSUPXzzVibwQOBdYBVNV19Ja0kCRJ0gL0U4jdV1X3b3iRZBm9FkeSJElagH4KsUuSvBN4XJIXAJ8FPr+5D3S9Jr8B7JtkTZITkxzT9Z08BPhikgsXGl6SJGmczbvXZJLHACcCLwQCXAicVkNqVmmvSUmSNC4Wrddkt7MtgE9X1auBTzzacJIkSZpnIVZVDyZ5UpKtZl4nNkzX33EXR52z2TOhkiRpEV1w7K/NPUiPSj/riP0A+OckFwD3bthYVR9e9FSSJElLQD8X638f+EL3me1nPGY1S6/JP0lybZLvJDk/yY4LCS5JkjTu+mlx9D6AJMt7L+e1ov4q4GPAp2dsuxh4R1WtT/LHwDuA/2/eiSVJkiZEPyvrTyW5EvgOcGWSbyd55uY+U1WXArdvtO2iqlrfvfwmsFefmSVJkiZCP6cmTwd+p6pWVtVKeivtf+pRHv+3gC/P9maSk5JMJ5m+f91dj/JQkiRJo6WfQuzBqvqnDS+q6uvA+s2M36wk7+o+f+ZsYx7ea3KHhR5KkiRpJPVz1+QlSf4SOItea6NXAv+Y5CCAqrp8vjtKcgLwUuCIYS0IK0mSNGr6KcSe0f18z0bbD6RXmB0+n50kORJ4G/ArVfWzPo4vSZI0Ufq5a/L5m3s/yfFVdcZG284CDgNWdP0l30PvLsnHAhcnAfhmVf2PPnNLkiSNvXn3mpxzR8nlVXXQouxsE+w1KUmSxsV8e032c7H+nMdcxH1JkiRNvMUsxLzoXpIkqQ/9XKw/l4HOiH3/jns45tyvD/IQkiRpM85/+XNaR5g48y7EkjwWeDmwcubnquoPuqf/vKjJJEmSJlw/pyY/BxxNbxHWe2c8AKiqN238gVmafv/vruH3FUkuSvKEhceXJEkaX/2cmtyrqo7sc/+reGTT7z+pqv8FkOR3gd8HXL5CkiQtOf3MiP1Lkl/qZ+ezNP1eN+PltniRvyRJWqL6mRF7DnBCkn8H7qN3cX5V1S/3e9AkfwT8JnAXMOtCsUlOAk4CeNyK3fs9jCRJ0kjrpxB70WIdtKreBbwryTuAN/HItkkbxp0KnAqw01Oe7syZJEmaKP20OLphAMc/E/gSsxRikiRJk2wxF3SdlyT7zHh5NHDtsDNIkiSNgsVc0PURZmn6/eIk+wIPATfgHZOSJGmJWrSm34Nm029JkjQuWjT9liRJUh8GempyMf3gzvt45XnXt44hSZLm6TMve2rrCCPPGTFJkqRGBlqIbarX5Iz33pqkkqwYZAZJkqRRNegZsVXAI/pTJtkbeCHwowEfX5IkaWQNtBDbVK/JzkeAt2GfSUmStIS1WND1aODGqvr2PMaelGQ6yfR9d22qnpMkSRpfQ71rMsk2wDvpnZac08xekzs/9ZecPZMkSRNl2DNiTwGeDHw7yQ+BvYDLkzx+yDkkSZKaG+qMWFVdCey24XVXjE1V1a3DzCFJkjQKBr18xVnAN4B9k6xJcuIgjydJkjROBjojVlXHzfH+yvnu6xd2fKwr9EqSpIniyvqSJEmNWIhJkiQ1MjZNv9fe+QB/fv7NrWNIkjR23njM7q0jaBZD7zWZ5L1JbkxyRfd48SAzSJIkjaomvSaBj1TVAd3jSwPOIEmSNJJa9ZqUJEla8lpdrP+mJN/pTl3uNNugmb0m71lnPSdJkiZLi0Ls4/RaHR0A3AR8aLaBVXVqVU1V1dR2y3ceVj5JkqShGHohVlU3V9WDVfUQ8Ang4GFnkCRJGgVDL8SS7DHj5THAVbONlSRJmmQDXUes6zV5GLAiyRrgPcBhSQ4ACvgh8NuDzCBJkjSqUlWtM8zL1NRUTU9Pt44hSZI0pySrq2pqrnG2OJIkSWrEQkySJKmRsek1edcd6/nyZ25tHUOSpJH3oleuaB1B8+SMmCRJUiNDb/rdbX9zkmuTXJ3kA4PMIEmSNKqG3vQ7yfOBo4FnVNUvAh8ccAZJkqSR1KLp9xuA91fVfd2YtYPMIEmSNKpaXCP2NOC5SS5LckmS/zrbwJlNv9etu22IESVJkgavRSG2DNgZeBbwe8DZSbKpgTObfi9fvsswM0qSJA1ci0JsDXBe9fwr8BDgfbaSJGnJaVGI/R3wfIAkTwO2AlwgTJIkLTktmn6fDpzeLWlxP3B8jUvDS0mSpEU00EKsqo6b5a3X9LuvHXZa5krBkiRporiyviRJUiNj02vyZ7eu51unueSYJEnzdeB/3611BM3BGTFJkqRGht5rMslnklzRPX6Y5IpBZpAkSRpVgz41uQr4GPDpDRuq6pUbnif5EHDXgDNIkiSNpEHfNXlpkpWbeq9bTf8VwOGDzCBJkjSqWl4j9lzg5qq6brYBM3tN3nG3vSYlSdJkaVmIHQectbkBM3tN7rS9vSYlSdJkabJ8RZJlwMuAZ7Y4viRJ0ihoNSP2q8C1VbWm0fElSZKaG/TyFWcB3wD2TbImyYndW69ijtOSkiRJk65Jr8mqOqHffW2zYpkrBEuSpIniyvqSJEmNWIhJkiQ1MjZNvx/46QPc9IEbW8eQJGls7PG2PVtH0Bxa9Jo8IMk3u16T00kOHmQGSZKkUTXoU5OrgCM32vYB4H1VdQDw+91rSZKkJWeghVhVXQrcvvFmYHn3fAfgJ4PMIEmSNKpaXCN2CnBhkg/SKwSf3SCDJElScy3umnwD8Jaq2ht4C/DJ2QbObPp92702/ZYkSZOlRSF2PHBe9/yzwKwX689s+r3Ltjb9liRJk6VFIfYT4Fe654cD1zXIIEmS1NxArxHrek0eBqxIsgZ4D/B64E+TLAP+AzhpkBkkSZJGVZNek8Az+93Xlo/f0oXpJEnSRLHFkSRJUiMWYpIkSY2MT6/Jm3/GzR9d3TqGJEkPs/spfV9tI/2cM2KSJEmNtGj6/Ywk30hyZZLPJ1m+uX1IkiRNqhZNv08D3l5VvwScD/zegDNIkiSNpBZNv58GXNo9vxh4+SAzSJIkjaoW14hdDRzdPf8NYO/ZBs7sNXn7vXcMJZwkSdKwtCjEfgv4nSSrge2B+2cbOLPX5M7b7jS0gJIkScMw9OUrqupa4IUASZ4GvGTYGSRJkkbB0GfEkuzW/XwM8G7gL4adQZIkaRQMevmKs4BvAPsmWZPkROC4JP8GXAv8BPjUIDNIkiSNqlRV6wzzMjU1VdPT061jSJIkzSnJ6qqammucK+tLkiQ1Mja9JtevXcfaj13UOoYkST+325te2DqCxpwzYpIkSY0M+mL9vZN8Lcl3k1yd5ORu+85JLk5yXffTRcIkSdKSM+gZsfXAW6tqP+BZwBuT7Ae8HfhqVe0DfLV7LUmStKQMutfkTVV1eff8buAaYE96LY7O6IadAfz6IHNIkiSNoqFdI5ZkJXAgcBmwe1Xd1L31U2D3WT7z816Tt91z11BySpIkDctQCrEk2wHnAqdU1bqZ71VvIbNNLmY2s9fkLtvtMISkkiRJwzPwQizJlvSKsDOr6rxu881J9uje3wNYO+gckiRJo2bQd00G+CRwTVV9eMZbFwDHd8+PBz43yBySJEmjaNALuh4KvBa4MskV3bZ3Au8Hzu56T94AvGLAOSRJkkbOQAuxqvo6kFnePqKffS3bbbkrGEuSpIkyNk2/k9wNfK91Ds1qBXBr6xDaLH9Ho83fz+jzdzTaRu3386Sq2nWuQWPTaxL43ny6mKuNJNP+fkabv6PR5u9n9Pk7Gm3j+vux16QkSVIjFmKSJEmNjFMhdmrrANosfz+jz9/RaPP3M/r8HY22sfz9jM3F+pIkSZNmnGbEJEmSJoqFmCRJUiMjX4glOTLJ95Jcn+TtrfPo4ZLsneRrSb6b5OokJ7fOpEdKskWSbyX5QusseqQkOyY5J8m1Sa5JckjrTPpPSd7S/fftqiRnJdm6daalLsnpSdYmuWrGtp2TXJzkuu7nTi0zztdIF2JJtgD+HHgRsB9wXJL92qbSRtYDb62q/YBnAW/0dzSSTgauaR1Cs/pT4CtV9XTgGfi7GhlJ9gR+F5iqqv2BLYBXtU0lYBVw5Ebb3g58tar2Ab7avR55I12IAQcD11fVD6rqfuBvgaMbZ9IMVXVTVV3ePb+b3j8ge7ZNpZmS7AW8BDitdRY9UpIdgOcBnwSoqvur6s62qbSRZcDjkiwDtgF+0jjPkldVlwK3b7T5aOCM7vkZwK8PNdQCjXohtifw4xmv1+A/8iMryUrgQOCytkm0kY8CbwMeah1Em/Rk4BbgU93p49OSbNs6lHqq6kbgg8CPgJuAu6rqorapNIvdq+qm7vlPgd1bhpmvUS/ENCaSbAecC5xSVeta51FPkpcCa6tqdessmtUy4CDg41V1IHAvY3JKZSnorjM6ml7B/ARg2ySvaZtKc6ne2lxjsT7XqBdiNwJ7z3i9V7dNIyTJlvSKsDOr6rzWefQwhwJHJfkhvVP7hyf567aRtJE1wJqq2jCTfA69wkyj4VeBf6+qW6rqAeA84NmNM2nTbk6yB0D3c23jPPMy6oXY/wX2SfLkJFvRu0DygsaZNEOS0Lu25Zqq+nDrPHq4qnpHVe1VVSvp/e/nH6rK/29+hFTVT4EfJ9m323QE8N2GkfRwPwKelWSb7r93R+DNFKPqAuD47vnxwOcaZpm3Za0DbE5VrU/yJuBCeneqnF5VVzeOpYc7FHgtcGWSK7pt76yqLzXMJI2bNwNndv8P5w+A1zXOo05VXZbkHOByeneJf4sxbaUzSZKcBRwGrEiyBngP8H7g7CQnAjcAr2iXcP5scSRJktTIqJ+alCRJmlgWYpIkSY1YiEmSJDViISZJktSIhZgkSVIjFmKSRlaSHZP8zgI/e0qSbRY7kyQtJgsxSaNsR2BBhRhwCr0GzfOSZIsFHkeSFsxCTNIoez/wlCRXJPmTJL+X5P8m+U7+//buHcSuKgzD8PuCOMFEtFFRCy2ihVgNNhKM2gxeWiEIKiKI2tiI2NhElJCEaCleEAtR0MZC0SASSAijIGrIVIKQShAVkYnBEJLP4qyBwUtg4MAex++Bw9lrn3Xbp/pZa7F/3Qugblc/Vk+oK+oe9WlmeQGPqEf+rXP1tHpIPQHcrp5S943xvlIX1cPq9+qTo8216tFRZ0W9Y9xfUpfVr9UPRv7VqqqL6gtdq2rTUm8EPkpyq7oEPAA8AcgsnckB4CrgniSPjzZXJPlt5Ne8LcnPF+k/wJ4k74/yKWB/klfVV5ils9kFbANWklyjPgNsS/LSWEW7DFhgloPw3iS/q88BC0lemPNfUlVbzKZOcVRVtc7S+HwzyjuAm4BjwCF1P7Og7dgG+jzPLGH9emv5bE8CO5KsAqvqWfVKZjlw3xrJ7j9M8q16J3ALcHyWjpBLgeUNP2FV/e80EKuq/wqBfUle+9sPugjcB7yofr6Blag/kpz/y72z4/vCuuu18iVJjqq7gfuBt9WXgV+Bz5I8uIHnqarqGbGq2tRWgcvH9WHgsbWzV+r16tXqdcCZJO8AB4HFf2g7N+oNwI9J3gDeHON9AexSd44629Wb5z12VW09XRGrqk0ryS/qcXUF+AR4F1ge23+ngYeAncBB9QJwDnhqNH8d+FT9Icndc5zWXcCz6rkxh0eS/KQ+CrynLox6zwPfzXHcqtqCeli/qqqqaiLdmqyqqqqaSLcmq2rLU79k9oqJ9R5OcnKK+VRVrenWZFVVVdVEujVZVVVVNZEGYlVVVVUTaSBWVVVVNZEGYlVVVVUTaSBWVVVVNZE/ATbapexdDRbZAAAAAElFTkSuQmCC\n",
"text/plain": [
"<Figure size 720x720 with 2 Axes>"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"beta_lookup = pd.Series(beta, index=['x' + str(i) for i in range(0, len(beta))])\n",
"vs = []\n",
"\n",
"i = 0\n",
"while i < df.shape[0]:\n",
" row = sorted_df.iloc[i]\n",
" coeffs = row['model'].params.drop('Intercept')\n",
" betas = beta_lookup[coeffs.index]\n",
" v = beta_lookup[list(coeffs.index)] - coeffs\n",
" vs.append(np.sqrt((v ** 2).sum()))\n",
" i += 1\n",
"\n",
"xy = pd.DataFrame({'value': vs, 'n_predictors' : np.arange(1, i + 1)})\n",
"\n",
"_, (ax1, ax2) = plt.subplots(nrows=2, figsize=(10,10))\n",
"sns.barplot(y='n_predictors', x='value', data=xy, orient='h', ax=ax1)\n",
"sns.barplot(y='n_predictors', x='test_rmse', data=df, orient='h', ax=ax2)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"11. We will now try to predict per capita crime rate in the Boston data set."
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"(a) Try out some of the regression methods explored in this chapter, such as best subset selection, the lasso, ridge regression, and PCR. Present and discuss results for the approaches that you consider.\n",
"\n",
"(b) Propose a model (or set of models) that seem to perform well on this data set, and justify your answer. Make sure that you are evaluating model performance using validation set error, cross- validation, or some other reasonable alternative, as opposed to using training error.\n",
"\n",
"(c) Does your chosen model involve all of the features in the data set? Why or why not?"
]
},
{
"cell_type": "code",
"execution_count": 414,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>CRIM</th>\n",
" <th>ZN</th>\n",
" <th>INDUS</th>\n",
" <th>CHAS</th>\n",
" <th>NOX</th>\n",
" <th>RM</th>\n",
" <th>AGE</th>\n",
" <th>DIS</th>\n",
" <th>RAD</th>\n",
" <th>TAX</th>\n",
" <th>PTRATIO</th>\n",
" <th>B</th>\n",
" <th>LSTAT</th>\n",
" <th>MEDV</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>0.00632</td>\n",
" <td>18.0</td>\n",
" <td>2.31</td>\n",
" <td>0.0</td>\n",
" <td>0.538</td>\n",
" <td>6.575</td>\n",
" <td>65.2</td>\n",
" <td>4.0900</td>\n",
" <td>1.0</td>\n",
" <td>296.0</td>\n",
" <td>15.3</td>\n",
" <td>396.90</td>\n",
" <td>4.98</td>\n",
" <td>24.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>0.02731</td>\n",
" <td>0.0</td>\n",
" <td>7.07</td>\n",
" <td>0.0</td>\n",
" <td>0.469</td>\n",
" <td>6.421</td>\n",
" <td>78.9</td>\n",
" <td>4.9671</td>\n",
" <td>2.0</td>\n",
" <td>242.0</td>\n",
" <td>17.8</td>\n",
" <td>396.90</td>\n",
" <td>9.14</td>\n",
" <td>21.6</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>0.02729</td>\n",
" <td>0.0</td>\n",
" <td>7.07</td>\n",
" <td>0.0</td>\n",
" <td>0.469</td>\n",
" <td>7.185</td>\n",
" <td>61.1</td>\n",
" <td>4.9671</td>\n",
" <td>2.0</td>\n",
" <td>242.0</td>\n",
" <td>17.8</td>\n",
" <td>392.83</td>\n",
" <td>4.03</td>\n",
" <td>34.7</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>0.03237</td>\n",
" <td>0.0</td>\n",
" <td>2.18</td>\n",
" <td>0.0</td>\n",
" <td>0.458</td>\n",
" <td>6.998</td>\n",
" <td>45.8</td>\n",
" <td>6.0622</td>\n",
" <td>3.0</td>\n",
" <td>222.0</td>\n",
" <td>18.7</td>\n",
" <td>394.63</td>\n",
" <td>2.94</td>\n",
" <td>33.4</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>0.06905</td>\n",
" <td>0.0</td>\n",
" <td>2.18</td>\n",
" <td>0.0</td>\n",
" <td>0.458</td>\n",
" <td>7.147</td>\n",
" <td>54.2</td>\n",
" <td>6.0622</td>\n",
" <td>3.0</td>\n",
" <td>222.0</td>\n",
" <td>18.7</td>\n",
" <td>396.90</td>\n",
" <td>5.33</td>\n",
" <td>36.2</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" CRIM ZN INDUS CHAS NOX RM AGE DIS RAD TAX \\\n",
"0 0.00632 18.0 2.31 0.0 0.538 6.575 65.2 4.0900 1.0 296.0 \n",
"1 0.02731 0.0 7.07 0.0 0.469 6.421 78.9 4.9671 2.0 242.0 \n",
"2 0.02729 0.0 7.07 0.0 0.469 7.185 61.1 4.9671 2.0 242.0 \n",
"3 0.03237 0.0 2.18 0.0 0.458 6.998 45.8 6.0622 3.0 222.0 \n",
"4 0.06905 0.0 2.18 0.0 0.458 7.147 54.2 6.0622 3.0 222.0 \n",
"\n",
" PTRATIO B LSTAT MEDV \n",
"0 15.3 396.90 4.98 24.0 \n",
"1 17.8 396.90 9.14 21.6 \n",
"2 17.8 392.83 4.03 34.7 \n",
"3 18.7 394.63 2.94 33.4 \n",
"4 18.7 396.90 5.33 36.2 "
]
},
"execution_count": 414,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"boston_data = datasets.load_boston()\n",
"df_boston = pd.DataFrame(boston_data.data,columns=boston_data.feature_names)\n",
"df_boston['MEDV'] = pd.Series(boston_data.target)\n",
"df_boston.head()"
]
},
{
"cell_type": "code",
"execution_count": 427,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"searching over 8191 models\n",
"estimated time: 9.829199999999998 seconds\n",
"-----------\n",
"n choose: 1\n",
"13 combinations\n",
"done: 0.017893075942993164 seconds (0.0013763904571533203 secs/model)\n",
"-----------\n",
"n choose: 2\n",
"78 combinations\n",
"done: 0.08938980102539062 seconds (0.0011460230900691105 secs/model)\n",
"-----------\n",
"n choose: 3\n",
"286 combinations\n",
"done: 0.3350977897644043 seconds (0.0011716705935818333 secs/model)\n",
"-----------\n",
"n choose: 4\n",
"715 combinations\n",
"done: 0.874377965927124 seconds (0.0012229062460519216 secs/model)\n",
"-----------\n",
"n choose: 5\n",
"1287 combinations\n",
"done: 1.4666271209716797 seconds (0.0011395704125654077 secs/model)\n",
"-----------\n",
"n choose: 6\n",
"1716 combinations\n",
"done: 1.9497790336608887 seconds (0.001136234868100751 secs/model)\n",
"-----------\n",
"n choose: 7\n",
"1716 combinations\n",
"done: 1.9710617065429688 seconds (0.0011486373581252732 secs/model)\n",
"-----------\n",
"n choose: 8\n",
"1287 combinations\n",
"done: 1.492173194885254 seconds (0.0011594197318455742 secs/model)\n",
"-----------\n",
"n choose: 9\n",
"715 combinations\n",
"done: 0.8320097923278809 seconds (0.0011636500591998334 secs/model)\n",
"-----------\n",
"n choose: 10\n",
"286 combinations\n",
"done: 0.3377799987792969 seconds (0.0011810489467807583 secs/model)\n",
"-----------\n",
"n choose: 11\n",
"78 combinations\n",
"done: 0.09338688850402832 seconds (0.0011972678013336964 secs/model)\n",
"-----------\n",
"n choose: 12\n",
"13 combinations\n",
"done: 0.018061161041259766 seconds (0.0013893200800969051 secs/model)\n",
"-----------\n",
"n choose: 13\n",
"1 combinations\n",
"done: 0.0015816688537597656 seconds (0.0015816688537597656 secs/model)\n",
"--- FINISHED ---\n",
"total time: 9.48312497138977 seconds\n",
"secs/model: 0.0011577493555597327\n",
"searching over 8191 models\n",
"estimated time: 9.829199999999998 seconds\n",
"-----------\n",
"n choose: 1\n",
"13 combinations\n",
"done: 0.01477503776550293 seconds (0.0011365413665771484 secs/model)\n",
"-----------\n",
"n choose: 2\n",
"78 combinations\n",
"done: 0.09007978439331055 seconds (0.0011548690306834686 secs/model)\n",
"-----------\n",
"n choose: 3\n",
"286 combinations\n",
"done: 0.3248109817504883 seconds (0.0011357027333933156 secs/model)\n",
"-----------\n",
"n choose: 4\n",
"715 combinations\n",
"done: 0.7975690364837646 seconds (0.0011154811699073631 secs/model)\n",
"-----------\n",
"n choose: 5\n",
"1287 combinations\n",
"done: 1.4798541069030762 seconds (0.0011498477909114811 secs/model)\n",
"-----------\n",
"n choose: 6\n",
"1716 combinations\n",
"done: 2.13156795501709 seconds (0.0012421724679586772 secs/model)\n",
"-----------\n",
"n choose: 7\n",
"1716 combinations\n",
"done: 2.1127989292144775 seconds (0.001231234807234544 secs/model)\n",
"-----------\n",
"n choose: 8\n",
"1287 combinations\n",
"done: 1.602759838104248 seconds (0.0012453456395526404 secs/model)\n",
"-----------\n",
"n choose: 9\n",
"715 combinations\n",
"done: 0.9456429481506348 seconds (0.0013225775498610277 secs/model)\n",
"-----------\n",
"n choose: 10\n",
"286 combinations\n",
"done: 0.37534165382385254 seconds (0.0013123834049785055 secs/model)\n",
"-----------\n",
"n choose: 11\n",
"78 combinations\n",
"done: 0.11243391036987305 seconds (0.0014414603893573468 secs/model)\n",
"-----------\n",
"n choose: 12\n",
"13 combinations\n",
"done: 0.018174171447753906 seconds (0.001398013188288762 secs/model)\n",
"-----------\n",
"n choose: 13\n",
"1 combinations\n",
"done: 0.0018270015716552734 seconds (0.0018270015716552734 secs/model)\n",
"--- FINISHED ---\n",
"total time: 10.012573003768921 seconds\n",
"secs/model: 0.0012223871326784179\n",
"searching over 8191 models\n",
"estimated time: 9.829199999999998 seconds\n",
"-----------\n",
"n choose: 1\n",
"13 combinations\n",
"done: 0.015893936157226562 seconds (0.0012226104736328125 secs/model)\n",
"-----------\n",
"n choose: 2\n",
"78 combinations\n",
"done: 0.10020971298217773 seconds (0.0012847399100279196 secs/model)\n",
"-----------\n",
"n choose: 3\n",
"286 combinations\n",
"done: 0.37825489044189453 seconds (0.0013225695469996311 secs/model)\n",
"-----------\n",
"n choose: 4\n",
"715 combinations\n",
"done: 0.8888750076293945 seconds (0.0012431818288523 secs/model)\n",
"-----------\n",
"n choose: 5\n",
"1287 combinations\n",
"done: 1.602207899093628 seconds (0.0012449167825125313 secs/model)\n",
"-----------\n",
"n choose: 6\n",
"1716 combinations\n",
"done: 2.1144068241119385 seconds (0.001232171808923041 secs/model)\n",
"-----------\n",
"n choose: 7\n",
"1716 combinations\n",
"done: 2.12587308883667 seconds (0.0012388537813733508 secs/model)\n",
"-----------\n",
"n choose: 8\n",
"1287 combinations\n",
"done: 1.5516738891601562 seconds (0.001205651817529259 secs/model)\n",
"-----------\n",
"n choose: 9\n",
"715 combinations\n",
"done: 0.883814811706543 seconds (0.0012361046317574027 secs/model)\n",
"-----------\n",
"n choose: 10\n",
"286 combinations\n",
"done: 0.3632950782775879 seconds (0.0012702625114600974 secs/model)\n",
"-----------\n",
"n choose: 11\n",
"78 combinations\n",
"done: 0.09531021118164062 seconds (0.001221925784380008 secs/model)\n",
"-----------\n",
"n choose: 12\n",
"13 combinations\n",
"done: 0.017831802368164062 seconds (0.0013716771052433895 secs/model)\n",
"-----------\n",
"n choose: 13\n",
"1 combinations\n",
"done: 0.0014450550079345703 seconds (0.0014450550079345703 secs/model)\n",
"--- FINISHED ---\n",
"total time: 10.142664909362793 seconds\n",
"secs/model: 0.0012382694310051022\n",
"searching over 8191 models\n",
"estimated time: 9.829199999999998 seconds\n",
"-----------\n",
"n choose: 1\n",
"13 combinations\n",
"done: 0.017348051071166992 seconds (0.0013344654670128455 secs/model)\n",
"-----------\n",
"n choose: 2\n",
"78 combinations\n",
"done: 0.10265707969665527 seconds (0.0013161164063673753 secs/model)\n",
"-----------\n",
"n choose: 3\n",
"286 combinations\n",
"done: 0.3676578998565674 seconds (0.0012855171323656203 secs/model)\n",
"-----------\n",
"n choose: 4\n",
"715 combinations\n",
"done: 0.8679542541503906 seconds (0.00121392203377677 secs/model)\n",
"-----------\n",
"n choose: 5\n",
"1287 combinations\n",
"done: 1.564488172531128 seconds (0.001215608525665212 secs/model)\n",
"-----------\n",
"n choose: 6\n",
"1716 combinations\n",
"done: 2.1087839603424072 seconds (0.0012288950817846197 secs/model)\n",
"-----------\n",
"n choose: 7\n",
"1716 combinations\n",
"done: 2.1522388458251953 seconds (0.0012542184416230741 secs/model)\n",
"-----------\n",
"n choose: 8\n",
"1287 combinations\n",
"done: 1.5691421031951904 seconds (0.0012192246334072963 secs/model)\n",
"-----------\n",
"n choose: 9\n",
"715 combinations\n",
"done: 0.8493759632110596 seconds (0.001187938410085398 secs/model)\n",
"-----------\n",
"n choose: 10\n",
"286 combinations\n",
"done: 0.3537921905517578 seconds (0.0012370356312998525 secs/model)\n",
"-----------\n",
"n choose: 11\n",
"78 combinations\n",
"done: 0.09138798713684082 seconds (0.0011716408607287284 secs/model)\n",
"-----------\n",
"n choose: 12\n",
"13 combinations\n",
"done: 0.01631307601928711 seconds (0.0012548520014836239 secs/model)\n",
"-----------\n",
"n choose: 13\n",
"1 combinations\n",
"done: 0.002048015594482422 seconds (0.002048015594482422 secs/model)\n",
"--- FINISHED ---\n",
"total time: 10.066420793533325 seconds\n",
"secs/model: 0.0012289611516949487\n",
"searching over 8191 models\n",
"estimated time: 9.829199999999998 seconds\n",
"-----------\n",
"n choose: 1\n",
"13 combinations\n",
"done: 0.015763282775878906 seconds (0.0012125602135291467 secs/model)\n",
"-----------\n",
"n choose: 2\n",
"78 combinations\n",
"done: 0.08764505386352539 seconds (0.001123654536711864 secs/model)\n",
"-----------\n",
"n choose: 3\n",
"286 combinations\n",
"done: 0.3282649517059326 seconds (0.001147779551419345 secs/model)\n",
"-----------\n",
"n choose: 4\n",
"715 combinations\n",
"done: 0.8312163352966309 seconds (0.001162540329086197 secs/model)\n",
"-----------\n",
"n choose: 5\n",
"1287 combinations\n",
"done: 1.627681016921997 seconds (0.0012647094148578067 secs/model)\n",
"-----------\n",
"n choose: 6\n",
"1716 combinations\n",
"done: 2.1096181869506836 seconds (0.0012293812278267387 secs/model)\n",
"-----------\n",
"n choose: 7\n",
"1716 combinations\n",
"done: 2.1005899906158447 seconds (0.0012241200411514247 secs/model)\n",
"-----------\n",
"n choose: 8\n",
"1287 combinations\n",
"done: 1.6426849365234375 seconds (0.001276367472046183 secs/model)\n",
"-----------\n",
"n choose: 9\n",
"715 combinations\n",
"done: 0.8941559791564941 seconds (0.0012505678030160757 secs/model)\n",
"-----------\n",
"n choose: 10\n",
"286 combinations\n",
"done: 0.3765528202056885 seconds (0.0013166182524674422 secs/model)\n",
"-----------\n",
"n choose: 11\n",
"78 combinations\n",
"done: 0.0985708236694336 seconds (0.0012637285085824819 secs/model)\n",
"-----------\n",
"n choose: 12\n",
"13 combinations\n",
"done: 0.01629781723022461 seconds (0.0012536782484788161 secs/model)\n",
"-----------\n",
"n choose: 13\n",
"1 combinations\n",
"done: 0.0013277530670166016 seconds (0.0013277530670166016 secs/model)\n",
"--- FINISHED ---\n",
"total time: 10.13389277458191 seconds\n",
"secs/model: 0.0012371984830401549\n"
]
},
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>0</th>\n",
" <th>1</th>\n",
" <th>2</th>\n",
" <th>3</th>\n",
" <th>4</th>\n",
" <th>5</th>\n",
" <th>6</th>\n",
" <th>7</th>\n",
" <th>8</th>\n",
" <th>9</th>\n",
" <th>10</th>\n",
" <th>11</th>\n",
" <th>12</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>4.411323</td>\n",
" <td>4.755900</td>\n",
" <td>4.784093</td>\n",
" <td>5.129547</td>\n",
" <td>2.046046</td>\n",
" <td>2.147097</td>\n",
" <td>1.916061</td>\n",
" <td>2.925837</td>\n",
" <td>3.284684</td>\n",
" <td>3.023006</td>\n",
" <td>2.982842</td>\n",
" <td>2.266282</td>\n",
" <td>2.028975</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>3.782029</td>\n",
" <td>4.368218</td>\n",
" <td>4.670622</td>\n",
" <td>4.736891</td>\n",
" <td>4.789007</td>\n",
" <td>4.404211</td>\n",
" <td>5.677229</td>\n",
" <td>5.227295</td>\n",
" <td>7.070136</td>\n",
" <td>6.761502</td>\n",
" <td>6.654373</td>\n",
" <td>3.910064</td>\n",
" <td>3.350659</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>4.128744</td>\n",
" <td>3.383647</td>\n",
" <td>3.193658</td>\n",
" <td>3.447774</td>\n",
" <td>2.880217</td>\n",
" <td>2.803286</td>\n",
" <td>2.962172</td>\n",
" <td>2.837483</td>\n",
" <td>2.051755</td>\n",
" <td>1.982576</td>\n",
" <td>2.524979</td>\n",
" <td>2.174489</td>\n",
" <td>2.267941</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>12.421952</td>\n",
" <td>12.135320</td>\n",
" <td>11.946773</td>\n",
" <td>11.663072</td>\n",
" <td>11.055813</td>\n",
" <td>10.782303</td>\n",
" <td>10.723177</td>\n",
" <td>10.263207</td>\n",
" <td>10.244673</td>\n",
" <td>10.011402</td>\n",
" <td>11.265268</td>\n",
" <td>10.567198</td>\n",
" <td>10.058744</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>14.559303</td>\n",
" <td>13.630074</td>\n",
" <td>13.646259</td>\n",
" <td>13.835853</td>\n",
" <td>13.060408</td>\n",
" <td>12.657820</td>\n",
" <td>12.651616</td>\n",
" <td>14.276134</td>\n",
" <td>13.376914</td>\n",
" <td>12.917513</td>\n",
" <td>12.566299</td>\n",
" <td>11.688774</td>\n",
" <td>11.411557</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" 0 1 2 3 4 5 \\\n",
"0 4.411323 4.755900 4.784093 5.129547 2.046046 2.147097 \n",
"1 3.782029 4.368218 4.670622 4.736891 4.789007 4.404211 \n",
"2 4.128744 3.383647 3.193658 3.447774 2.880217 2.803286 \n",
"3 12.421952 12.135320 11.946773 11.663072 11.055813 10.782303 \n",
"4 14.559303 13.630074 13.646259 13.835853 13.060408 12.657820 \n",
"\n",
" 6 7 8 9 10 11 12 \n",
"0 1.916061 2.925837 3.284684 3.023006 2.982842 2.266282 2.028975 \n",
"1 5.677229 5.227295 7.070136 6.761502 6.654373 3.910064 3.350659 \n",
"2 2.962172 2.837483 2.051755 1.982576 2.524979 2.174489 2.267941 \n",
"3 10.723177 10.263207 10.244673 10.011402 11.265268 10.567198 10.058744 \n",
"4 12.651616 14.276134 13.376914 12.917513 12.566299 11.688774 11.411557 "
]
},
"execution_count": 427,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"# Select between models with different numbers of predictors using cross validation\n",
"results = pd.DataFrame()\n",
"for train_idx, test_idx in model_selection.KFold(n_splits=5).split(df_boston):\n",
" train = df_boston.iloc[train_idx]\n",
" test = df_boston.iloc[test_idx]\n",
" df, combs = best_subset(13, 'CRIM', train)\n",
" rmses = []\n",
" for idx, m in enumerate(df['model']):\n",
" comb = combs[idx]\n",
" x = test[list(comb)]\n",
" preds = m.predict(x)\n",
" rmses.append(np.sqrt(metrics.mean_squared_error(test['CRIM'], preds)))\n",
" rmse = pd.DataFrame({'test_rmse' : rmses})\n",
" results = results.append(rmse.T, ignore_index=True)\n",
" \n",
"results\n"
]
},
{
"cell_type": "code",
"execution_count": 433,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"<matplotlib.axes._subplots.AxesSubplot at 0x11b4110f0>"
]
},
"execution_count": 433,
"metadata": {},
"output_type": "execute_result"
},
{
"data": {
"image/png": "iVBORw0KGgoAAAANSUhEUgAAAXwAAAELCAYAAADawD2zAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAFMdJREFUeJzt3XuUZWWd3vHvYzcotCAgJUHAaTSEDGFlkFQICkMQxEEhMAYnwBomMoptzCigmXFgzGhcySTxEtYwowunQZQgMktuE68IcgkjjowFtNjQGC8gglzKmOE6guAvf5zdUkJX9enu/VZ31/5+1jqr9jm1z/t7T12e2vXufd43VYUkaeF7zsbugCRpfhj4kjQQBr4kDYSBL0kDYeBL0kAY+JI0EAa+JA2EgS9JA2HgS9JALN7YHZhpxx13rKVLl27sbkjSZuXGG2/8cVVNrG2/TSrwly5dytTU1MbuhiRtVpL8YJz9HNKRpIFoGvhJ3pnk1iQrk1yY5Hkt60mSZtcs8JPsApwMTFbV3sAi4LhW9SRJc2s9pLMY2CrJYmBr4EeN60mSZtEs8KvqHuDDwF3AvcCDVXVFq3qSpLm1HNLZHjga2B14MbAkyQlr2G9ZkqkkU9PT0626I0mD13JI59XAHVU1XVU/Ay4FXvnMnapqeVVNVtXkxMRaLyOVJK2nloF/F7B/kq2TBDgUWNWwniRpDi3H8G8ALgZuAr7V1Vreqp4kaW5N32lbVe8D3rc+z50+61O99mXibc86fSBJg+I7bSVpIDapuXS0YT553mt6be/EN3oVrbSQeIQvSQNh4EvSQAx6SOfuj7yp9zZ3ffu5z3rsmnOO6L3Oq076Qu9tSlrYPMKXpIEw8CVpIAx8SRoIA1+SBsLAl6SBMPAlaSAMfEkaCANfkgbCwJekgTDwJWkgDHxJGohBz6UjLTTHXPK3vbZ3yTH79dreurj2U9O9t3nwCcNeN9sjfEkaiGaBn2TPJCtm3B5KcmqrepKkuTUb0qmqbwP7ACRZBNwDXNaqnrQp+1cX9/uj/7k3vL7X9jQM8zWkcyjwvar6wTzVkyQ9w3wF/nHAhWv6RJJlSaaSTE1P93+SRpI00jzwk2wJHAVctKbPV9XyqpqsqsmJiWGfQZeklubjsszXAjdV1f3zUEtSY2dcdl/vbb7r9f+g9zb1bPMxpHM8swznSJLmT9PAT7IEOAy4tGUdSdLaNR3SqapHgRe2rCFJGo9TK2jQjrjk7N7b/MIxb+m9TakPBr7W2XsuOrzX9v7kty7vtT1Ja+ZcOpI0EAa+JA2EgS9JA2HgS9JAeNJWm6TX/dUf9d7mF3/zv/beprQ58QhfkgbCwJekgTDwJWkgDHxJGggDX5IGwsCXpIEw8CVpIAx8SRoIA1+SBsLAl6SBaL3E4XZJLk5ye5JVSV7Rsp4kaXat59I5E7i8qt6QZEtg68b1JGmd3PvBe3ttb+d379xre31qFvhJXgAcBJwIUFVPAE+0qidJmlvLIZ3dgWngE0luTnJOkiUN60mS5tAy8BcD+wJnVdXLgUeB0565U5JlSaaSTE1PTzfsjiQNW8vAvxu4u6pu6O5fzOgPwC+pquVVNVlVkxMTEw27I0nD1izwq+o+4IdJ9uweOhS4rVU9SdLcWl+l8w7ggu4Kne8Dv9u4niRpFk0Dv6pWAJMta0jS5uD+P/tqr+3tdPKB6/wc32krSQNh4EvSQBj4kjQQBr4kDYSBL0kDYeBL0kAY+JI0EAa+JA2EgS9JA2HgS9JAGPiSNBAGviQNhIEvSQNh4EvSQBj4kjQQBr4kDYSBL0kD0XTFqyR3Ag8DTwFPVpWrX0nSRtJ6TVuAV1XVj+ehjiRpDg7pSNJAtA78Aq5IcmOSZY1rSZLm0HpI58CquifJi4Ark9xeVdfN3KH7Q7AM4CUveUnj7kjScK31CD/J1kn+OMnZ3f09khw5TuNVdU/38QHgMmC/NeyzvKomq2pyYmJi3XovSRrbOEM6nwAeB17R3b8H+C9re1KSJUm2Wb0NvAZYuZ79lCRtoHGGdF5WVccmOR6gqh5LkjGetxNwWbfrYuDTVXX5+ndVkrQhxgn8J5JsxegELElexuiIf05V9X3g1zase5KkvowT+O8DLgd2S3IBcABwYstOSZL6t9bAr6ork9wE7A8EOMU3UknS5mecq3QOAH5aVV8AtgP+KMmvNO+ZJKlX41ylcxbwWJJfA94FfA/4n017JUnq3TiB/2RVFXA08NGq+iiwTdtuSZL6Ns5J24eTnA6cAByU5DnAFm27JUnq2zhH+McyugzzzVV1H7Ar8KGmvZIk9W6cq3TuA85Ism2SHYBHgM8375kkqVdrDfwkbwXeD/yU7s1X3ceXNuyXJKln44zh/z6wt9feS9LmbZwx/O8Bj7XuiCSprXGO8E8HvpbkBmbMoVNVJzfrlSSpd+ME/l8AVwPfAn7etjuSpFbGCfwtqupdzXsiSWpqnDH8LyVZlmTnJDusvjXvmSSpV+Mc4R/ffTx9xmNelilJm5k5A7+bRuGEqrp+nvojSWpkziGdqvo58JF56oskqaFxxvCvSnLMmOvYPkuSRUluTuJ0DJK0EY0T+G8FLgIeT/JQkoeTPLQONU4BVq1X7yRJvVlr4FfVNlX1nKrasqq27e5vu/rzSf7JbM9NsitwBHBOP92VJK2vcY7w1+b8OT73p8C78Q1bkrTR9RH4axzbT3Ik8EBV3Tjnk0fX+E8lmZqenu6hO5KkNekj8GuWxw8AjkpyJ/CXwCFJPvWsJ1ctr6rJqpqcmJjooTuSpDXpI/DXqKpOr6pdq2opcBxwdVWd0KqeJGlufQT+Ez20IUlqbK2Bn+SquR6rqv3X1kZVXVtVR6579yRJfZl1aoUkzwO2BnZMsj1Pn5zdFthlHvomSerRXHPpvBU4FXgxcCNPB/5DON2CJG12Zg38qjoTODPJO6rqz+exT5KkBsY5aXtfkm0AkvzHJJcm2bdxvyRJPRsn8P+4qh5OciDwauDjwFltuyVJ6ts4gf9U9/EIYHlVfQHYsl2XJEktjBP49yT5C+BY4ItJnjvm8yRJm5BxgvvfAF8GfqOq/g7YAfiDpr2SJPVunOmRHwMeAA7sHnoS+E7LTkmS+jfOO23fB/whTy9ivgXwrEnQJEmbtnGGdF4PHAU8ClBVPwK2adkpSVL/xgn8J6qq6KZBTrKkbZckSS2ME/if6a7S2S7JW4CvAGe37ZYkqW9zzaWz2gRwMaM5dPYE3svoDViSpM3IOIF/WFX9IXDl6geS/A9GJ3IlSZuJuaZHfhvw74GXJrllxqe2Aa5v3TFJUr/mOsL/NPAl4L8Bp814/OGq+knTXkmSejfX9MgPAg8Cx69Pw90CKtcBz+3qXFxV71uftiRJG26cMfz19ThwSFU9kmQL4KtJvlRVX29YU5I0i2aB3127/0h3d4vuVq3qSZLm1nTWyySLkqxgNBfPlVV1Q8t6kqTZNQ38qnqqqvYBdgX2S7L3M/dJsizJVJKp6enplt2RpEGbl3ntu2mVrwEOX8PnllfVZFVNTkxMzEd3JGmQmgV+kokk23XbWwGHAbe3qidJmlvLq3R2Bs5LsojRH5bPVNXnG9aTJM2h5VU6twAvb9W+JGnduDatJA2EgS9JA2HgS9JAGPiSNBAGviQNhIEvSQNh4EvSQBj4kjQQBr4kDYSBL0kDYeBL0kAY+JI0EAa+JA2EgS9JA2HgS9JAGPiSNBAGviQNRMs1bXdLck2S25LcmuSUVrUkSWvXck3bJ4H/UFU3JdkGuDHJlVV1W8OakqRZNDvCr6p7q+qmbvthYBWwS6t6kqS5zcsYfpKljBY0v2E+6kmSnq154Cd5PnAJcGpVPbSGzy9LMpVkanp6unV3JGmwmgZ+ki0Yhf0FVXXpmvapquVVNVlVkxMTEy27I0mD1vIqnQAfB1ZV1Rmt6kiSxtPyCP8A4HeAQ5Ks6G6va1hPkjSHZpdlVtVXgbRqX5K0bnynrSQNhIEvSQNh4EvSQBj4kjQQBr4kDYSBL0kDYeBL0kAY+JI0EAa+JA2EgS9JA2HgS9JAGPiSNBAGviQNhIEvSQNh4EvSQBj4kjQQBr4kDYSBL0kD0TTwk5yb5IEkK1vWkSStXesj/E8ChzeuIUkaQ9PAr6rrgJ+0rCFJGo9j+JI0EBs98JMsSzKVZGp6enpjd0eSFqyNHvhVtbyqJqtqcmJiYmN3R5IWrI0e+JKk+dH6sswLgb8B9kxyd5I3t6wnSZrd4paNV9XxLduXJI3PIR1JGggDX5IGwsCXpIEw8CVpIAx8SRoIA1+SBsLAl6SBMPAlaSAMfEkaCANfkgbCwJekgTDwJWkgDHxJGggDX5IGwsCXpIEw8CVpIAx8SRqI1kscHp7k20m+m+S0lrUkSXNrFvhJFgEfBV4L7AUcn2SvVvUkSXNreYS/H/Ddqvp+VT0B/CVwdMN6kqQ5tAz8XYAfzrh/d/eYJGkjSFW1aTh5A3B4VZ3U3f8d4F9U1dufsd8yYFl3d0/g2+tYakfgxxvY3U2lzkJ6LQutzkJ6LQutzkJ6Letb51eqamJtOy1ev/6M5R5gtxn3d+0e+yVVtRxYvr5FkkxV1eT6Pn9TqrOQXstCq7OQXstCq7OQXkvrOi2HdL4B7JFk9yRbAscBn21YT5I0h2ZH+FX1ZJK3A18GFgHnVtWtrepJkubWckiHqvoi8MWWNdiA4aBNsM5Cei0Lrc5Cei0Lrc5Cei1N6zQ7aStJ2rQ4tYIkDcRmG/hJzk3yQJKVDWvsluSaJLcluTXJKY3qPC/J3yb5Zlfn/S3qdLUWJbk5yecb1rgzybeSrEgy1bDOdkkuTnJ7klVJXtGgxp7d61h9eyjJqX3X6Wq9s/v+r0xyYZLnNahxStf+rX2+jjX9PibZIcmVSb7Tfdy+UZ3f6l7Pz5P0cnXLLHU+1P2s3ZLksiTbNarzn7saK5JckeTFG1rnF6pqs7wBBwH7Aisb1tgZ2Lfb3gb4P8BeDeoEeH63vQVwA7B/o9f0LuDTwOcbft3uBHach5+B84CTuu0tge0a11sE3Mfomue+294FuAPYqrv/GeDEnmvsDawEtmZ0/u4rwD/sqe1n/T4CHwRO67ZPAz7QqM6vMnoPz7XAZMPX8xpgcbf9gYavZ9sZ2ycDH+vrZ2CzPcKvquuAnzSucW9V3dRtPwysosG7hWvkke7uFt2t95MrSXYFjgDO6bvt+ZbkBYx+WT4OUFVPVNXfNS57KPC9qvpBo/YXA1slWcwolH/Uc/u/CtxQVY9V1ZPA/wb+dR8Nz/L7eDSjP8p0H3+zRZ2qWlVV6/qGzfWpc0X3dQP4OqP3FrWo89CMu0voMQs228Cfb0mWAi9ndPTdov1FSVYADwBXVlWLOn8KvBv4eYO2ZyrgiiQ3du+kbmF3YBr4RDdEdU6SJY1qrXYccGGLhqvqHuDDwF3AvcCDVXVFz2VWAr+e5IVJtgZexy+/ObJvO1XVvd32fcBODWvNtzcBX2rVeJI/SfJD4LeB9/bVroE/hiTPBy4BTn3GX9/eVNVTVbUPo6OG/ZLs3Wf7SY4EHqiqG/tsdxYHVtW+jGZK/b0kBzWosZjRv8JnVdXLgUcZDRs00b158Cjgokbtb8/oiHh34MXAkiQn9FmjqlYxGoq4ArgcWAE81WeNOWoXDf5r3RiSvAd4ErigVY2qek9V7dbVePva9h+Xgb8WSbZgFPYXVNWlret1wxLXAIf33PQBwFFJ7mQ0c+khST7Vcw3gF0erVNUDwGWMZk7t293A3TP+E7qY0R+AVl4L3FRV9zdq/9XAHVU1XVU/Ay4FXtl3kar6eFX9s6o6CPh/jM5LtXJ/kp0Buo8PNKw1L5KcCBwJ/Hb3R6y1C4Bj+mrMwJ9DkjAaI15VVWc0rDOx+ox/kq2Aw4Db+6xRVadX1a5VtZTR0MTVVdXrESRAkiVJtlm9zehEV+9XUlXVfcAPk+zZPXQocFvfdWY4nkbDOZ27gP2TbN393B3K6JxRr5K8qPv4Ekbj95/uu8YMnwXe2G2/EfhfDWs1l+RwRkOiR1XVYw3r7DHj7tH0mQV9nf2d7xujX757gZ8xOtp7c4MaBzL6N/QWRv/+rgBe16DOPwVu7uqsBN7b+Gt3MI2u0gFeCnyzu90KvKfh69gHmOq+bn8FbN+ozhLg/wIvaPx9eX/3y70SOB94boMaf83oD+M3gUN7bPdZv4/AC4GrgO8wuiJoh0Z1Xt9tPw7cD3y5UZ3vMpryfXUWbPDVM7PUuaT7GbgF+BywS1/fJ99pK0kD4ZCOJA2EgS9JA2HgS9JAGPiSNBAGviQNhIEvSQNh4EtjSvLJJG/ots9Jstcc+x6cpPd3ykoboukSh9KmLsnienoGxLFV1Ulr2eVg4BHga637Io3LI3xtVpIs7RY7Obtb9OKKbjqKNe17bZIzu4UkVibZr3v8PyU5P8n1wPndTKUfSvKNbuGJt3b7JclHknw7yVeAFz2j7clu+/AkN2W0gM1V3cyq/w54Z1f717t+X921f1U3tcHq/xo+luQG4INJ/mWeXmzl5tXTVEh98Ahfm6M9gOOr6i1JPsNocqnZJoLbuqr26WbsPJfRIiAAezGa1fPvuymcH6yqf57kucD1Sa5gNB32nt2+OzGakuDcmY0nmQDOBg6qqjuS7FBVP0nyMeCRqvpwt9/ngPOq6rwkbwL+jKfnh98VeGVVPdXt93tVdX03S+tPN/SLJa3mEb42R3dU1Ypu+0Zg6Rz7Xgi/WGhi2xnL0n22qv6+234N8G+79QhuYDQHzB6MFli5sEZTV/8IuHoN7e8PXFdVd3R1ZluU5xU8PVHZ+YzmaVrtoqpaPU3x9cAZSU5mtIKXQzzqjYGvzdHjM7afYu7/VJ85WdTq+4/OeCzAO6pqn+62e/W/+MhcftGXqvrvwEnAVoz+0/jH89gPLXAGvha6YwGSHMho2ObBNezzZeBt3doHJPlH3dTO1wHHdmP8OwOvWsNzvw4clGT37rk7dI8/zGgd5NW+xmhaahitYvTXa+pskpdV1beq6gPANwADX71xDF8L3U+T3MxoneA3zbLPOYyGhW7q5qKfZjS+fhlwCKOx+7uAv3nmE6tqujsHcGmS5zBa5OMwRtPaXpzkaOAd3e0TSf6ga/93Z+nLqUlexWgZyltpuIyehsfpkbVgJbkW+P2qmtrYfZE2BQ7pSNJAOKSjzV6SjzJas3emM6vq4I3QHWmT5ZCOJA2EQzqSNBAGviQNhIEvSQNh4EvSQBj4kjQQ/x+YMeIg/y5h6AAAAABJRU5ErkJggg==\n",
"text/plain": [
"<Figure size 432x288 with 1 Axes>"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"xy = pd.DataFrame({'test_rmse' : results.mean(), 'n_predictors' : results.columns + 1})\n",
"sns.barplot(x='n_predictors', y='test_rmse', data = xy)"
]
},
{
"cell_type": "code",
"execution_count": 435,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"5.823575152341391"
]
},
"execution_count": 435,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"# rmse for ols with all predictors (best performing subset)\n",
"xy.sort_values('test_rmse', ascending=True)['test_rmse'].iloc[0]"
]
},
{
"cell_type": "code",
"execution_count": 455,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"<matplotlib.axes._subplots.AxesSubplot at 0x119978a58>"
]
},
"execution_count": 455,
"metadata": {},
"output_type": "execute_result"
},
{
"data": {
"image/png": "iVBORw0KGgoAAAANSUhEUgAAAYsAAAEKCAYAAADjDHn2AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3X943GWZ7/H3naRJ0zRt0jaUQlJaEWorC2KzHqXneIp4xIVukS0q7BbYvTz8OOwKq1YR1grS9RxZYUXPLksLR0VwQaS61OouuvzYvS5+rCbyw7ZY3FKlaWkT2qRJk0kmydznj5nv8M00ySRt5vfndV29OvnON5lnoMmd57nv537M3RERERlPWa4HICIi+U/BQkRE0lKwEBGRtBQsREQkLQULERFJS8FCRETSUrAQEZG0FCxERCQtBQsREUmrItcDmCrz5s3zRYsW5XoYIiIFpbW19U13b0h3X9EEi0WLFtHS0pLrYYiIFBQz+91E7tMylIiIpKVgISIiaSlYiIhIWgoWIiKSloKFiIikpWAhIiJpFU3prIjkv1jMOdgbJTo0TGVFOXNrKikrs1wPSyZAwUJEMioWc7oiUQaHYnQcifKNJ15lzfIm5tZUEokOcdLsaioqtMiR7xQsRCQjgiBxqDdKR88A/YMxHvr577jynMXcuPllGmZWcf15pzHsTk1VBfNqqjTLyGMK5yIy5YaGYryyv5uX9hxmz6EIn330ZWZUlrNmeVMyUKw7fwkP/fx3vHrgCK8f7KOts4+hoViuhy5jULAQkSkRizmHegc4cDjC3sMRrnmglRmV5cyoLKetM0JXZJC5NZW0dUa4duWp3P/sbq48ZzEbtu7gyz9+hV0dvezp6qO9p59YzHP9diSFlqFE5LiMttxUO70iGSAqy8torK/mnqd3cefHzqKxvpq66mlHzTLCS1OL59Uwo6pcS1N5RDMLETlmYy03HeyNJgNEfc00vnrJmXQcGeCep3dx95+8m77o8IhZRurS1M4DPVqayjMKFiJyTGIxZ98Yy02bW/dw+5p4gPjSlh1UlJXx0FXv5S/Oezsn10/nzKZZNNRWJWcZWprKf1qGEpFJi8Wc/d39tPcMjLrctO78Jdz/7G7Wr1rG3JpKTqitYsGs6SNKZGdXxdh4+XLauwfGXZpq64zQWF/NvVc0s2R+rZalckQzCxGZlGDpaV9XZMzlpjse38ll7zmFJfNrWTh3Bo31M47aS1FRUcbSE2dxVtNsNq5dPubS1MbLl3PnR89i/+F+uiLRHL1rUbAQkQkJqp2CSqeDvdExl5u+ftm7eOfJs1g4ZwYn1E4fczZQVmbMqali6YJ40AgvTQWziw1bd/CVf/41AIcjg1qSyhEtQ4lIWkNDMXa299DePZCsdJroctNEBEFjVtW05NLU9eedNmJJ6v5ndyd3fvdHh7XzO8v0X1pExpQ6mwhXOr2wp4s7Ht+Z/AF+Ul31qMtNkxFemlo0b4YS33lEwUJERhWLOTsP9PDSnsN0hBLZwdJTEDA2bN1BdWU5J84ae7lpMoJZRk1VxZiJb+38zj4tQ4nIUYJqp6u+08KdHz1rRCJ7tKWnk2ZXT3mV0ryaKu69opn9h/uTie/1q5YlZxlB8Lj5gqWYGY4zfZo28mWKZhYiMkK42mm0fRNBpdPp41Q6TYWyMmPJ/NqjEt/hWcYtq5cxFItx2b3Pc8NDL7J9bzevH9LyVCZkPFiYWbmZvWBmW0d5bqGZPZV4/mUzuyBxfZGZRczsxcSfezI9TpFSN1q1UzCbuPKcxcnZxF9duJRTG2poqqset9JpKgRLUifPrmbj5UeX13b2DvLZR7X7OxuysQx1A/AKMGuU574APOLu/2Bmy4CfAIsSz+1y93dlYXwiJW+saqfb15zJjZtf5o7Hd+a0Z1OQ+F4wO0pP/1BylgGMuTyl9udTK6MzCzNrBC4E7hvjFuetIDIb2JfJ8YjISBOpdgpmE0tOrE27byKTRptl9EWHlQTPkkwvQ90FfA4Y6//SrcBaM2sjPqv4ZOi5xYnlqX8zs/+W2WGKlJ5wE8BsVjsdr3B5bdOcar56yZlHLU+p1HbqZWwZysxWAe3u3mpmK8e47TLg2+5+p5m9D3jAzM4A3gAWuvtBM1sO/JOZvdPdu1Ne42rgaoCFCxdm6q2IFJWgpXhP/xDXPNCas2qn4xHMMuqqK5lTU8ngUIz+odiYswxt6Dt+mfyvtQJYbWa/BR4GPmBmD6bc8wngEQB3fw6YDsxz9wF3P5i43grsAk5PfQF33+Tuze7e3NDQkLl3IlIk0u2dyGa101QIgsb82dVjJsE1y5gaGfsX4O43uXujuy8CLgWedPe1Kbe9DpwHYGZLiQeLDjNrMLPyxPW3AacBr2VqrCKl4s3eAa76TstRZ07kqtppKoWXp8YqtV13/hLWP7aNT3/vJZXZTlLWf10ws9vMbHXiw88AV5nZS8BDwJ+6uwPvB142sxeBR4Fr3f1QtscqUiyCRHbvwFDBzybGk67UdrRDljp6Bmjv6WdvZx+vH+pV8BhDVnZwu/vTwNOJx18MXd9BfLkq9f7NwOZsjE2k2IXLYoG0uYlCDBKpxiq1TS2zvf/Z3Vx37tvZ/WZvcr/GzRcsJToYI6Yd4SMU/r8KERnVaGWx33jiN0U3mxjLaLOM1KWpNcubRmzsC3aEX5rYEf67N/vY1xXRjAP1hhIpSqNtsuuKDCaDxPpVy6irnkZfdJjZ1RXMqanK9ZAzJjzL2Lh2Oe09A8mlqdSNfZ29g6x/bFsycESiw1x67/MjZhzl5eBuuDuVFeXMraksiZlH8fwaISLA0WdjhxPZwazimgda+cz3X+LE2dOpq67M9ZAzbqxDlroigyNmHME54qmtRIIZx21bt/NaRx+3btnGtn3dtPf0s+9whLbOPjp6Bop65qFgIVJERjsbOzWRveGiM3hq3Up+cN05JXemderS1ObWPcnjYFN3hI8WONYsb0rmOza37qGrb5DbfrSd7SUQOLQMJVIkgj0UvQNDE9pkV0z5ickKlqa+fPGZxGKxeItzd3oGhpKBAziqB1U435GaKL/ynMV8/V9fZc3yJhrrqxkYGqaizCgzY3A4RlkZlFtZ8nGhLWWV7r8WkSJzsDfKVd9pOeps7GJPZB+rsjKjoTa+oe+EWdOZP7uat82byZITa1m2oHZEK5FgxtEVGRyR7wgnykebcezv7mdXxxG+9KPtHOgeSD5OXcoKSnffONxHe/foj/d1Rdibw1mL/sWIFIno0HCyW2wxbLLLhfCO8NECx+bWPcypqUwbOFIrrVIfhwPL4HCM3W/2HhVQJhpcshVEFCxEikAs5gzHPGNnY5ei0QLHLX/4TmZOr2Dj2uXjBo7g7yDvkfp4IgFlIsHltY4+PrbxOVbc/hQX3/0MOw/0ZCxg6F+PSBE42Bvlr3+8I6+7xRayZGK8Pt6ifemCeL5j1hiBI7XSKvVxOLCMFVAmElxu3PwybZ0RIJ5TCZYhM0EJbpECF4s5kcEhfrqjnY6eaHIPRVdkkHkFkDgtREG+A2BOTRVfvvhMDGfj2uV8/YlXuX3Nmcnd4V+95Ey+9czIx5//g6XJwFFZXpb28Wj7QoIZTFhbZ4To0HBm3nNGvqqIZEVQAbWrvTc5o7jmgVY+vul5NmzdQVmZvsUzLQgc80Izjrc31HDr6jNYMGs6pzbM5JY/fCfzQ4/DS1lB6e54j8OzlvAMpbG+esRYGuurqawoz8j7tHjfvsLX3NzsLS0tuR6GSFZ19Axw8d3PJJvjBcsSjfXV3HtFc8ntoygksZhzsDeaLN2Nl9Qa5aM8nlZRRkf3AF9/4lWuO/ftRKLDfOuZt46RPZ7/52bW6u7Nae9TsBApXHs7+1hx+1MAnN1Ux7UrT6WuehqN9dUsyLMDi+T4jBVc3Dmu/RoTDRbKWYgUqHAFVFtnJLkE1VhfzQ+vW6FAUWTCeZKcvH7OXllEjktqBRTE16w3Xr6cuTXF3+9JskszC5ECpAooyTYFC5ECE1RA7T/cP6ICCkguQYlMNS1DiRSYoAdUcJBReAnq3iuatQQlGaGZhUgBCZaf2jojtHVGRhxkpAooySTNLEQKROoGPCC5BPWZ779EZUW5AoVkjIKFSIHQ8pPkkpahRApE0IJcy0+SC5pZiBSA8AY80PKTZJ+ChUgB0AY8ybWML0OZWTnQAux191Upzy0E7gfqgHLg8+7+k8RzNwGfAIaB69398UyPVSQfaQOe5INs5CxuAF4BZo3y3BeAR9z9H8xsGfATYFHi8aXAO4GTgH81s9PdPTON2kXylDbgSb7I6DKUmTUCFwL3jXGL81YQmQ3sSzy+CHjY3QfcfTfwn8B7MjlWkXz0Zu+AKqAkL2R6ZnEX8DmgdoznbwV+amafBGqADyaunww8H7qvLXFtBDO7GrgaYOHChVMzYpE8EYs5fQOqgJL8kLGZhZmtAtrdvXWc2y4Dvu3ujcAFwANmNuExufsmd2929+aGhobjHLFIfjnYG2X3m9qAJ/khk8tQK4DVZvZb4GHgA2b2YMo9nwAeAXD354DpwDxgL9AUuq8xcU2kJARJ7dGWnzauVQWUZF/GlqHc/SbgJgAzWwmsc/e1Kbe9DpwHfNvMlhIPFh3AFuAfzexviSe4TwN+nqmxiuSTcFK748jAiOWnvugwC+qma1YhWZf1fRZmdpuZrU58+BngKjN7CXgI+FOP2058xrED+Bfgz1UJJaUita1Hx5GB5PLTibOnU1etWYVkn87gFskzOldbsmmiZ3BrB7dIHlFbD8lXChYieSIWc/Z396uth+QldZ0VyQNBUrt3QG09JD9pZiGSB4Kk9sHe6Ii2Hh/f9Dwbtu6grEzfqpJb+hcokgeCsyrueXqX2npIXtIylEiOhZPaL+zpSu6rmFtTyUl11Zw4S/sqJPc0sxDJsdSzKl7Y08WGrTuorixXoJC8oZmFSI5Fh4aV1Ja8p2AhkkOxmGNmOqtC8p6WoURyJCiXvXXLNiW1Je9pZiGSI0G5bFtnJLkEpaS25CsFC5EcCcplgRFLUM/ceK4CheQdLUOJ5EBqD6hAY301lRXlORqVyNgULERyILVcFtQDSvKblqFEckDlslJoFCxEskzlslKItAwlkkUql5VCpZmFSBapXFYKlYKFSBapXFYKlZahRLJoWkWZymWlIClYiGRJLOYc6R/iq5coVyGFR8tQIllysDfKFd/8OQ0zq5Llsn3RYebPqtISlOQ9BQuRLIjFnMjgEG2dEdo6I8lcBcTzFdTkcHAiE5DxYGFm5UALsNfdV6U89zXg3MSHM4AT3L0u8dww8KvEc6+7++pMj1UkE4Jy2f2H+2msr04muEH5Cikc2ZhZ3AC8AsxKfcLdPxU8NrNPAmeHno64+7syPzyRzArKZRtmVnH7mjO5cfPLtHVGlK+QgpLRYGFmjcCFwJeBT6e5/TLglkyORyQXgnLZts5I8nztuuppNNZXs2B2tfIVUhAyXQ11F/A5IDbeTWZ2CrAYeDJ0ebqZtZjZ82b2kQyOUSRjUrvLBnsrPvP9l6isKFegkIKRsWBhZquAdndvTXszXAo86u7DoWunuHsz8MfAXWZ26iivcXUioLR0dHRMzcBFppC6y0qxyOQy1ApgtZldAEwHZpnZg+6+dpR7LwX+PHzB3fcm/n7NzJ4mns/YlXLPJmATQHNzs0/5OxA5TuouK8UiY8HC3W8CbgIws5XAutEChZm9A6gHngtdqwf63H3AzOYRDzx/k6mximSCustKMcn6Dm4zu83MwmWwlwIPu3t4ZrAUaDGzl4CngK+4+45MjCcWczp6Btjb2UdHzwCxmCYocvzUXVaKjY38GZ3ypNkH3P3JxOPF7r479NwfufsPsjDGCWlubvaWlpZJfU7wDR10AQ2+kZfMr9USgRyXjp4BLr77Gdo6I5zdVMe1K09Vd1nJS2bWmsgPjyvdzOKO0OPNKc99YdKjyjPhdtEAbZ0RrvpOCwd7ozkemRS60brLXnLPc7i7AoUUpHTBwsZ4PNrHBSf8DR1o64wQHRoe4zNE0kstlw1ot7YUsnTBwsd4PNrHBaeyolzf0DLlVC4rxShdNdTbzGwL8VlE8JjEx4szOrIsmFtTyb1XNB+Vs9A3tByroGGgymWl2KRLcP/38T7Z3f9tykd0jI4lwQ3xb+6DvVGiQ8NUVpQzV9/QcozCDQPXP7btqIaBP7xuBQ21VTkcocjRJprgHndmkRoMzGwacAbxDrLtxzfE/FBWZvoGlimhhoFSzMYNFmZ2D/B/3X27mc0mvnFuGJhjZuvc/aFsDFKkEKhhoBSzdAnu/+bu2xOP/wx41d1/D1hOvEGgiDBytzaoYaAUn3TBIrzh4H8A/wTg7vszNiKRAqPd2lIK0lVDdSW6x+4l3p/pEwBmVgFUj/eJIqUivLkzqIDSbm0pNumCxTXAN4ATgb8MzSjOA36cyYGJFIrRdmtD/GxtBQopFumqoV4FPjzK9ceBxzM1KJFCEd6trbO1pZilq4b6xnjPu/v1UzsckcIS3q0dLpXVbm0pNumWoa4FtgGPAPsogn5QIlNFu7WllKQLFguAjwIfB4aA7xE//rQr0wPLJu3ilskK79bW4UZSCsYtnXX3g+5+j7ufS3yfRR2ww8wuz8rosiD4pr/47mdYcftTXHz3M+w80KNDkGRcQQXUN574jcplpSRM6FhVM3s3cBnxvRb/DLRmclDZNNaZFurjI+PRbm0pNekS3LcBFwKvAA8DN7n7UDYGli0600ImK7UCKliCCpafFCikGKXbwf0F4ktPZwH/B/ilmb1sZr8ys5czPros0JkWMlk6r0JKUbplqII/syIdnWkhk6EKKClV6Tbl/W6062ZWRjyHMerzhaSszFgyv5YfXrdC1VAyLlVASSkbdxnKzGaZ2U1m9ndm9iGL+yTwGvCx7Awx84IzLU6un0FDbZUChYxKFVBSytItQz0AdBI/x+J/AjcT35j3EXd/McNjE8krqoCSUpb2DO7E+RWY2X3AG8BCd+/P+MhE8ogqoKTUpauGGgweuPsw0DbZQGFm5Wb2gpltHeW5r5nZi4k/r5pZV+i5K83sN4k/V07mNUWmmiqgpNSlm1mcZWbdiccGVCc+NsDdfdYEXuMG4vs0jrrX3T8VPE7kQs5OPJ4D3AI0Aw60mtkWd++cwOuJTClVQImkr4Y6rs0GZtZIfFPfl4FPp7n9MuIBAuB84GfufijxdX5GvFW6zvyWrFIFlEhcumWo43UX8bO6Y+PdZGanEN/T8WTi0snAntAtbYlrqZ93tZm1mFlLR0fH1IxYJEQVUCJxE+oNdSwSx7G2u3urma1Mc/ulxLvZTqrHhrtvAjYBNDc3q/OfTDlVQInEZXJmsQJYbWa/Jd5X6gNm9uAY917KyCWmvUBT6OPGxDWRrAlXQMFbR6Z+5vsvUVlRrkAhJSVjwcLdb3L3RndfRDwYPOnua1PvM7N3APXE93IEHgc+ZGb1ZlYPfIgsHOMaizkdPQPs7eyjo2dAbcpLnCqgRN6SsWWosSQ62ba4+5bEpUuBh909+ZPZ3Q+Z2QbgF4lLtwXJ7kwJEpmpPaKWzK/Vb5AlSBVQIiNZ6Gd0QWtubvaWlpZj/vyOngEuvvuZEe3Kg2oXnWtRWsIVUOsf26Z/E1LUzKzV3ZvT3ZfpaqiCoXMtJKAKKJGjZX0ZKl8F51qk/hapcy1KjyqgRI6mmUVCcK6FfossbaqAEhmdZhYJOtdCYGQF1I2bX04WO6gCSkqdgkVIcK6FlCZVQImMTcFCBPWAEklHOQsRVAElko5mFiKoAkokHc0spOSpAkokPQULKXnqASWSnpahpKSpAkpkYhQsUsRizsHeqPZalABVQIlMnIJFiDrPlpY3ewe46jstNMysOmoTniqgREZSsAgJyieD/lBtnRGu+k6LuowWoVjM6RtQBZTIRCnBHaLOs6XjYG+U3W/2qgJKZIIULEKCzrNh6jxbfIKk9mgb8DauVQWUyGi0DBUSdJ5NzVnoh0fxCCe1O44MjFh+6osOs6BuumYVIqPQSXkpVA1V3IITERtmVrHu/CVHJbVVzCClZqIn5WlmkUKdZ4ub2nqIHBvlLKRkqK2HyLFTsJCSobYeIsdOy1BSEtTWQ+T4KFhI0VNbD5Hjl/FlKDMrN7MXzGzrGM9/zMx2mNl2M/vH0PVhM3sx8WdLpseZKhZzOnoG2NvZR0fPALFYcVSNlaKgrYcONhI5dtmYWdwAvALMSn3CzE4DbgJWuHunmZ0Qejri7u/KwviOoh5RxUNtPUSmRkZnFmbWCFwI3DfGLVcBf+/unQDu3p7J8UzUWD2iDvZGczwymSy19RCZGplehroL+BwQG+P504HTzewZM3vezD4cem66mbUkrn8kw+McQT2iioPaeohMnYwtQ5nZKqDd3VvNbOU4r38asBJoBP7dzH7P3buAU9x9r5m9DXjSzH7l7rtSXuNq4GqAhQsXTtnYgx5R4YChHlGFRW09RKZWJmcWK4DVZvZb4GHgA2b2YMo9bcAWdx90993Aq8SDB+6+N/H3a8DTwNmpL+Dum9y92d2bGxoapmzgQY8oJUILUyzm7O/uH5HU7jgykFx+OnH2dOqq9f9SZDKy0hsqMbNY5+6rUq5/GLjM3a80s3nAC8C7iC9b9bn7QOL6c8BF7r5jrNeYqt5QgVjM6YpEiUSHGXZn+rRy5tVU6bfRPBfMKHoHhrjknucAOLupjmtXnqqktsgoJtobKus7uM3sNjNbnfjwceCgme0AngI+6+4HgaVAi5m9lLj+lfECRaYc6B7g45ue5/1/8zR/dPez7DzQoxLaPBcUJxzsjSqpLTKF1HV2DEF30tS8hU7Ny297O/tYcftTnN1Up66yIhOgrrPHSRVRhSfcKPCFPV3JpPbcmkpOqqvmxFlKaoscKzUSHINOzSssQVI73CjwhT1dbNi6g+rKcgUKkeOkmcUYwqfmNcys4vrzTmPxvBocJxbzCf/g0WFKmRdOaqtRoEhmKFiMoazMWDK/li1/sYI3uvq55sHWSa99Dw3F2NnewzUPTP5zZeKCpPb6VcvUKFAkQ7QMNY6yMmM4RjJQwMRbf8Rizr7DkWSgmMznyuQE+aV7nt6lRoEiGaKZRRrHmug+2BulvWdASfIMU1JbJDs0s0jjWBPd0aHhEbX+k/lcmRgltUWyRzOLNMKJ7nDeYbyljVjMMTM2t+7h9jVncuPml5NJ8kXzZkw6SS5HU1JbJLsULNIIJ7rDrT/GEvwQ+9rPdnLlOYu5/9ndfPWSM5lZVcH/+u4vleieIsGBRkpqi2SHlqEmaKKtP4LKnJ/uaOeOx3eyZnkTJ86engwUoET38QofaKSktkh2KFhMwGQOQwonxIPfdtu7leieSuEDjcJJ7UevfR+PXPM+zdhEMkDBYgImWhEVrswJ64sOK9E9RUY70CiZ1J6mpLZIpihnMQHhw5CCdtdzayoxsxGJ6oO90WRlTriBXdOc+NLI134WX5aaW1PJCbVV1FdPy/E7Kyw60EgkdxQsJiCoiAqS1qN1MgWIDI5emVNbVcHcmipu+ODp2s19HIKkdsPMqmRAvuaB1uR/Sx1oJJI5alE+QUFN/8c2PkfDzKrkYTp90WHefcps2jr7ae8eYP1j20Ztaw6o5flxiMWc1w/1sfKOpwEdaCQyVfL28KNCVVZmuDsNM6tYd/4SNmzdwVf++dfMrKqgq2+Iax5oHbGODiMrc9Ty/NgFgTpIaoMONBLJNi1DTUJlRTnXn3dacpPduvOXcGRgCOuJ/+Bv64yMWEcP/8YbznsElOROL7z5LgjG4WXAjWuXq0xWJAs0s5iEuTWVLJ5XQ1tnhGtXnsqNm19mRmX5hI7wDPIejfXVnN1Ux7f+9Pd58BP/JbmbW0YX5CkO9kZHJLW/d/V72XDRGUpqi2SJgsUklJUZM6riM4S66mm0dUboigwm23qEl582Xj7yN97wTvC//sgZrH9sGyvv0Nne4xlt813HkYFkMD5x9nQltUWyRAnuSQqXb65/bFtyOer+Z3ePKIs9aXY1FRVHx2Kd7T0xQZ5i5/6eZNFAuGxZHWVFpoYS3BkSzBDOaprNxrXLk0sjl73nFJbMr2Xh3Bk01s8YNVDAWxv8zm6qY+Ply/ne1e9l/aplxGKxLL+T/BUE5H1dEW2+E8kTSnAfg7IyY05NFXXVlfzwuhWTOjK1sqKcDy074aj9GhsvX05DrX4Awsgmgdp8J5IfNLM4DmVlRkNtFSfXz6ChtmpCP8Dm1lTyhQuXJQMFxCuprnmgVY0FUZ5CJF9pZpFlZWVGeZklA0V4c1l0aLikz7lI3U+hk+9E8kfGZxZmVm5mL5jZ1jGe/5iZ7TCz7Wb2j6HrV5rZbxJ/rsz0OLMp2HNxdlNdcoPfxzc9z8c3PV+ylVHKU4jkt2zMLG4AXgFmpT5hZqcBNwEr3L3TzE5IXJ8D3AI0Aw60mtkWd+/MwngzLthzsf9wf3KDX7Amv/9wP/NnVTGnprQqo5SnEMlvGZ1ZmFkjcCFw3xi3XAX8fRAE3L09cf184Gfufijx3M+AD2dyrNkUVFSdekLNUe1DAA5HBmnv6S+ZGYbyFCL5L9Mzi7uAzwG1Yzx/OoCZPQOUA7e6+78AJwN7Qve1Ja4VjbIyo3paxVHtQ8L7Nfqjw2Pu1ygWylOIFIaM/RQys1VAu7u3jnNbBXAasBK4DLjXzOom8RpXm1mLmbV0dHQc13hzIbV9yP3P7ubKcxazYesOvvzjVzjQPcAb3f28fqi3KGcaQ0MxXtnfrTyFSAHI5MxiBbDazC4ApgOzzOxBd18buqcN+A93HwR2m9mrxIPHXuIBJNAIPJ36Au6+CdgE8R3cmXgTmZTaPmTN8qbkLOOW1cuIRIe57N7naZhZxfXnncbieTXMqCpnXs3EynTzWSzm7DscLxlWnkIk/2VsZuHuN7l7o7svAi4FnkwJFAD/RCIomNk84stSrwGPAx8ys3ozqwc+lLhWdObVVHHvFc30RYeZW1OZnGV09g7y2UffWp5a/9g2PvW9F9m+t5vXD/UV9EwjWHpq7xlQnkKkQGR9n4WZ3Qa0uPsW3goKO4Bh4LPufjBx3wbgF4lPu81hwDaUAAAJy0lEQVTdD2V7rNkQJLvnz6qip38oOcuA+Ga99auWjchpjHZKX6H89h2LOV2RKG909RMZHE5261WeQiT/ZSVz6u5Pu/uqxOMvJgIFHvdpd1/m7r/n7g+HPueb7v72xJ9vZWOcuRK0Dzl5drztR190mL7o8IjutkFL9IaZVWy8fDl3fvQs9h/upytSGLu+g30UL+05zDUPxnerh7v1JvMUlcpTiOQj7eDOIxUVZSw9cRYLZkc51Bvlq5ecOSJohGcXQR7jcGSQoZjnfR4j2Edx50fPSi49BdVfwYwi6Nabz+9DpFQpWOSZcJPCOTWVDA7FuGftcjp6BgqyzDZYeuodGEqe/xFeelLLcZHCkF8/WSQpCBrzZ1ezbMEszmqazaJ5M8Yss93V0cuervxKfAelsS/tOcxv3+yjsb46mcwOLz3VVFUoUIjkOQWLAhAEjpqqilHLbNedv4SHfv47Xj1whNcP9tHW2cfQUO7Ox4jFnEO9A+xNlMbOqCxP7qMISmQ3XHQGT61byQ+uO6egkvQipUrLUAUkKLPdf7g/WWa7ftWy5CwjnMsYdqemqiLruYyhoRg723to7x6gdnpFculptH0Us6srSq4Hlkih0syigIRP6WuorcqrWcZos4mgNFb7KEQKn2YWBSZYkppVNY2Nly+nvXtg3FnGzRcsxcxwnOnTpnb3dyzmHOyNYjgHugdo7xk5mwhKY2/c/HJy6WnRvBpqimQXukgpUbAoUOEy2/BmvvFahtx8wVKigzHKy8HdcPcJHwcLb1U2RaLDTKswOrqjfP2JV/n8HyzlmgdbufOjZ42YTYxVGptvFVsikp6+awvYaJv5xmoZcsvqZQzFYty2dTuvdfRx65ZtbNvXTXtPP+09/ezt7OONw33s64pw4HCE9u63rrV393PgcITX3jzCzv09fOlH2znSP8w1D7ayZnkTh3qjR80mghzFZe85hdPn17Jw7gwa62coUIgUKH3nFoFglpGay5hRWX5U4FizvCm5VLW5dQ+DwzF2v9nLl370VhB5o7ufXR1H+NKPtnOge4BdHUfY8UYPew5Fkl8jCBB11dNGzCauPGdxcjbxVxcu5dSGGprqqjmhVqWxIoVMwaJIpGsZEgSO8FLVmuVNI4LIaNeCxzMqy0d8jSBAaDYhUhqUsygyY7UMAZI/3IOlqnDDwqAHVeq14HFXZJDK8rKjAkQwS1FuQqS46bu5CAWzjLfNm8mSE2tZtqCWpjnVfPWSM9ncuoc5NZXJH/rB7CNow5F6LXh8z9O7qK+ZlvwaQYBYs7yJuhnT+OIfvpMzTprFKXNrNJsQKULmnh+tIY5Xc3Ozt7S05HoYeWtkJVMZHd0DfP2JV7nu3LcTiQ7zrWfemiGErwWPg0T5zRcsZcHs6cdcUSUi+cXMWt29Oe19ChalKdgjEYvFMDMGh2OUlRnuUG6MuFaeeDzsMH1amfZIiBSRiQYL5SxKVFmZ0VCrVhsiMjFaWBYRkbQULEREJC0FCxERSUvBQkRE0lKwEBGRtBQsREQkraLZZ2FmHcDvJvlp84A3MzCcfFaK7xlK832X4nuG0nzfx/OeT3H3hnQ3FU2wOBZm1jKRzSjFpBTfM5Tm+y7F9wyl+b6z8Z61DCUiImkpWIiISFqlHiw25XoAOVCK7xlK832X4nuG0nzfGX/PJZ2zEBGRiSn1mYWIiExASQYLM/uwme00s/80s8/nejzZYGZNZvaUme0ws+1mdkOux5QtZlZuZi+Y2dZcjyVbzKzOzB41s1+b2Stm9r5cjynTzOxTiX/b28zsITObnusxZYKZfdPM2s1sW+jaHDP7mZn9JvF3/VS/bskFCzMrB/4e+ANgGXCZmS3L7aiyYgj4jLsvA94L/HmJvG+AG4BXcj2ILPs68C/u/g7gLIr8/ZvZycD1QLO7nwGUA5fmdlQZ823gwynXPg884e6nAU8kPp5SJRcsgPcA/+nur7l7FHgYuCjHY8o4d3/D3X+ZeNxD/IfHybkdVeaZWSNwIXBfrseSLWY2G3g/8P8A3D3q7l25HVVWVADVZlYBzAD25Xg8GeHu/w4cSrl8EXB/4vH9wEem+nVLMVicDOwJfdxGCfzQDDOzRcDZwH/kdiRZcRfwOSCW64Fk0WKgA/hWYvntPjOryfWgMsnd9wJ3AK8DbwCH3f2nuR1VVs139zcSj/cD86f6BUoxWJQ0M5sJbAb+0t27cz2eTDKzVUC7u7fmeixZVgG8G/gHdz8b6CUDyxL5JLFGfxHxQHkSUGNma3M7qtzweInrlJe5lmKw2As0hT5uTFwremY2jXig+K67/yDX48mCFcBqM/st8eXGD5jZg7kdUla0AW3uHswcHyUePIrZB4Hd7t7h7oPAD4BzcjymbDpgZgsAEn+3T/ULlGKw+AVwmpktNrNK4kmwLTkeU8aZmRFfw37F3f821+PJBne/yd0b3X0R8f/PT7p70f+26e77gT1mtiRx6TxgRw6HlA2vA+81sxmJf+vnUeRJ/RRbgCsTj68EHpvqF6iY6i+Y79x9yMz+AniceMXEN919e46HlQ0rgMuBX5nZi4lrN7v7T3I4JsmcTwLfTfxC9BrwZzkeT0a5+3+Y2aPAL4lX/r1Ake7kNrOHgJXAPDNrA24BvgI8YmafIN59+2NT/rrawS0iIumU4jKUiIhMkoKFiIikpWAhIiJpKViIiEhaChYiIpKWgoXIOMzsyBR9nVvNbN0E7vu2mV0yFa8pMpUULEREJC0FC5EJMLOZZvaEmf3SzH5lZhclri9KnBnxbTN71cy+a2YfNLNnEmcLvCf0Zc4ys+cS169KfL6Z2d8lzlf5V+CE0Gt+0cx+kTifYVNiZ7JITihYiExMP3Cxu78bOBe4M/TD++3AncA7En/+GPivwDrg5tDXOBP4APA+4ItmdhJwMbCE+NkqVzCyn9HfufvvJ85nqAZWZei9iaRVcu0+RI6RAf/bzN5PvN35ybzVBnq3u/8KwMy2Ez+Exs3sV8Ci0Nd4zN0jQMTMniJ+tsr7gYfcfRjYZ2ZPhu4/18w+R/xshjnAduBHGXuHIuNQsBCZmD8BGoDl7j6Y6GQbHNs5ELovFvo4xsjvsdTeOmP22kkcCXo38ZPf9pjZraHXE8k6LUOJTMxs4mdjDJrZucApx/A1LjKz6WY2l3gjuF8A/w58PHFO+ALiS1zwVmB4M3EGiSqkJKc0sxCZmO8CP0osLbUAvz6Gr/Ey8BQwD9jg7vvM7IfE8xg7iLfZfg7A3bvM7F5gG/GTz35x/G9B5Nip66yIiKSlZSgREUlLwUJERNJSsBARkbQULEREJC0FCxERSUvBQkRE0lKwEBGRtBQsREQkrf8Po7PcHjoLpzQAAAAASUVORK5CYII=\n",
"text/plain": [
"<Figure size 432x288 with 1 Axes>"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"# LASSO Regression - Cross-Validation with test MSE\n",
"\n",
"results = pd.DataFrame()\n",
"alpha_range = np.linspace(0.1, 10, num=200)\n",
"for train_idx, test_idx in model_selection.KFold(n_splits=12).split(df_boston):\n",
" train, test = df_boston.iloc[train_idx], df_boston.iloc[test_idx]\n",
" errors = []\n",
" for alpha in alpha_range:\n",
" reg = linear_model.Lasso(alpha=alpha, max_iter=1000000)\n",
" reg.fit(train.drop('CRIM', axis=1), train['CRIM'])\n",
" error = np.sqrt(metrics.mean_squared_error(test['CRIM'], reg.predict(test.drop('CRIM', axis=1))))\n",
" errors.append(error)\n",
" results = pd.concat([results, pd.DataFrame(errors, index=alpha_range).T], axis=0, ignore_index=True)\n",
" \n",
"\n",
"df = pd.DataFrame({'lambda' : alpha_range, 'RMSE' : results.mean()})\n",
"sns.scatterplot(x='lambda', y='RMSE', data=df)"
]
},
{
"cell_type": "code",
"execution_count": 456,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"4.575267023519401"
]
},
"execution_count": 456,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"lmbda, RMSE = df.sort_values('RMSE', ascending=True).iloc[0]\n",
"# best rmse for lasoo\n",
"RMSE"
]
},
{
"cell_type": "code",
"execution_count": 457,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"<matplotlib.axes._subplots.AxesSubplot at 0x11ccc2160>"
]
},
"execution_count": 457,
"metadata": {},
"output_type": "execute_result"
},
{
"data": {
"image/png": "iVBORw0KGgoAAAANSUhEUgAAAYwAAAEKCAYAAAAB0GKPAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3XmYHHd95/H3p6+Z8UiydYyEY1k+gjmccBjPghc7gEhibEIwR54sBLDDEhQnsIGHhxAICfDAkpPsJjwEHMOSYO5NgoJDwNjhiJ8FnFgC39jG+Ig1FpIsS9YxR1/f/aOqRzWtOWpG3T0azef1PPNMdf2qun5T01Xf/p2liMDMzGwuhcXOgJmZLQ0OGGZmlosDhpmZ5eKAYWZmuThgmJlZLg4YZmaWiwOGmZnl4oBhZma5OGCYmVkupcXOQCetW7cuzjzzzMXOhpnZkrF9+/ZHI2Ioz7YnVMA488wz2bZt22Jnw8xsyZD0UN5tXSVlZma5OGCYmVkuDhhmZpZLV9swJD0IHAQaQD0ihtvSXwP8HqB0u9+KiFvz7GtmZr3Vi0bvzRHx6AxpDwDPj4h9ki4Frgaek3NfMzProUXtJRUR3828vAnYuFh5MTOz2XW7DSOA6yVtl7Rljm3fAHxtvvtK2iJpm6Rte/bsmXcGm81gz8EJRvaNsufgBM2mn0BoZjadbpcwLoqIEUnrgRsk3R0RN7ZvJGkzScC4aL77RsTVJFVZDA8Pz+tu32wG9+w6yBuv2caOfWNsXD3Axy8f5skbVlIoaD5vZWZ2wutqCSMiRtLfu4GtwLPbt5H0dOATwGURsXc++x6rvYerk8ECYMe+Md54zTb2Hq52+lBmZkte1wKGpEFJK1vLwMXAHW3bbAK+BLwuIu6dz76dUK03JoNFy459Y1TrjU4fysxsyetmldQGYKuk1nE+FxHXSboSICKuAt4DrAU+mm7X6j477b6dzmClVGTj6oEpQWPj6gEqpWKnD2VmtuQp4sRp5B0eHo75zCXlNgwzW+4kbc87zu2EmnxwvgoF8eQNK9n62xdSrTeolIqsHaw4WJiZTWNZBwxIgsbQyr7FzoaZ2XHPc0mZmVkuDhhmZpaLA4aZmeXigGFmZrk4YJiZWS4OGGZmlosDhpmZ5eKAYWZmuThgmJlZLg4YZmaWiwOGmZnl4oBhZma5OGCYmVkuDhhmZpaLA4aZmeXigGFmZrk4YJiZWS5dDRiSHpR0u6RbJB31sG1Jr5F0W7rNdyU9I5N2iaR7JN0n6Z3dzKeZmc2tF49o3RwRj86Q9gDw/IjYJ+lS4GrgOZKKwF8DvwjsAG6WdG1E3NWD/JqZ2TQWtUoqIr4bEfvSlzcBG9PlZwP3RcT9EVEFvgBcthh5NDOzRLcDRgDXS9ouacsc274B+Fq6fBrwcCZtR7rOzMwWSberpC6KiBFJ64EbJN0dETe2byRpM0nAuGi+B0gD0RaATZs2HWt+zcxsBl0tYUTESPp7N7CVpKppCklPBz4BXBYRe9PVI8Dpmc02puumO8bVETEcEcNDQ0OdzL6ZmWV0LWBIGpS0srUMXAzc0bbNJuBLwOsi4t5M0s3AOZLOklQBXgVc2628mpnZ3LpZJbUB2CqpdZzPRcR1kq4EiIirgPcAa4GPptvV09JCXdKbga8DReCTEXFnF/NqZmZzUEQsdh46Znh4OLZtO2q4h5mZzUDS9ogYzrOtR3qbmVkuDhhmZpaLA4aZmeXigGFmZrk4YJiZWS4OGGZmlosDhpmZ5eKAYWZmuThgmJlZLg4YZmaWiwOGmZnl4oBhZma5OGCYmVkuDhhmZpaLA4aZmeXigGFmZrk4YJiZWS4OGGZmlosDhpmZ5dLVgCHpQUm3S7pF0lEP25b0FEnfkzQh6e3z2dfMzHqr1INjbI6IR2dIewz4HeBlC9jXzMx6aFGrpCJid0TcDNQWMx9mZja3bgeMAK6XtF3Slh7ua2ZmHdbtKqmLImJE0nrgBkl3R8SNndw3DSZbADZt2tS5nJuZ2RRdLWFExEj6ezewFXh2p/eNiKsjYjgihoeGho4902ZmNq2uBQxJg5JWtpaBi4E7ur2vmZl1RzerpDYAWyW1jvO5iLhO0pUAEXGVpCcA24BVQFPSW4FzgXXT7dvFvJqZ2Ry6FjAi4n7gGdOsvyqz/BNg4zS7H5huXzMzWzwe6W1mZrk4YJiZWS4OGGZmlosDhpmZ5eKAYWZmuThgmJlZLg4YZmaWiwOGmZnl4oBhZma5OGCYmVkuDhhmZpaLA4aZmeXigGFmZrk4YJiZWS4OGGZmlosDhpmZ5eKAYWZmuThgmJlZLg4YZmaWS1cDhqQHJd0u6RZJ26ZJf4qk70makPT2trRLJN0j6T5J7+xmPs3MbG6lHhxjc0Q8OkPaY8DvAC/LrpRUBP4a+EVgB3CzpGsj4q6u5tTMzGa0qFVSEbE7Im4Gam1Jzwbui4j7I6IKfAG4rOcZNDOzSd0OGAFcL2m7pC3z2O804OHM6x3pOjMzWyTdrpK6KCJGJK0HbpB0d0Tc2MkDpIFoC8CmTZvmvX+zGew9XKVab1ApFVk7WKFQUCezaGZ2QuhqCSMiRtLfu4GtJFVNeYwAp2deb0zXTXeMqyNiOCKGh4aG5pW/ZjO4Z9dBXv7R73Dhn36Ll3/0O9yz6yDNZszrfczMloOuBQxJg5JWtpaBi4E7cu5+M3COpLMkVYBXAdd2Oo97D1d54zXb2LFvDIAd+8Z44zXb2Hu42ulDmZkted2sktoAbJXUOs7nIuI6SVcCRMRVkp4AbANWAU1JbwXOjYgDkt4MfB0oAp+MiDs7ncFqvTEZLFp27BujWm90+lBmZkverAFD0gsj4pvp8lkR8UAm7RUR8aWZ9o2I+4FnTLP+qszyT0iqm6bb/6vAV+f8C45BpVRk4+qBKUFj4+oBKqViNw9rZrYkzVUl9aHM8j+2pf1Bh/PSc2sHK3z88mE2rh4AkmDx8cuHWTtYWeScmZkdf+aqktIMy9O9XnIKBfHkDSvZ+tsXupeUmdkc5goYMcPydK+XpEJBDK3sW+xsmJkd9+YKGGdLupakNNFaJn19VldzZmZmx5W5AkZ2Oo4PtaW1vzYzsxPYrAEjIv4t+1pSGfhZYCQdjGdmZsvErL2kJF0l6WfS5ZOBW4FrgB9IenUP8mdmZseJubrV/lxmwNzrgXsj4mnA+cA7upqzHmk2gz0HJxjZN8qegxOeFsTMbAZztWFk58j4ReDvIRlwl47gXtJac0m1pgdpjcN48oaV7lprZtZmrhLGfkkvkXQecCFwHYCkEjDQ7cx1m+eSMjPLb64Sxm8CHwaeALw1ncoD4OeBf+lmxnrBc0mZmeU3Vy+pe4FLpln/dZKJAZc0zyVlZpbfXJMPfni29Ij4nc5mp7dac0n97xvu4ZXnn87awQrrV/axeqC82FkzMzvuzFUldSXJMyz+L/AIJ8D8UVmFgjhnaAVv+YUn8Zuf3u6GbzOzWczV6H0qcDXwIuB1QBn4ckR8KiI+1e3M9cK+sdpksAA3fJuZzWSuNoy9wFXAVZI2kjz57i5JvxcRn+5FBrutWm8wtKKPP3zJuZwyUGb/WI2rvv1jN3ybmbXJ9cQ9Sc8CXk0yFuNrwPZuZqqXBipF3nHJk/ndf7htskrqz3/l6QxU3PBtZpY119Qg75e0HXgb8G/AcES8ISLu6knueqDejMlgAUmV1O/+w23UPeLbzGyKuUoYfwA8QPKo1WcAf5SO8BYQEfH07mav+2r15rRVUrV6c7GzZmZ2XJkrYBzTMy8kPQgcBBpAPSKG29IF/BXwYmAU+PWI+H6a1gBuTzf9z4h46bHkZSaukjIzy2euRu+HplsvqUDSpjFtepvNEfHoDGmXAuekP88BPpb+BhiLiGfmeP9j0qqSypYyRqsN4sR4oKCZWcfM1YaxStK7JH1E0sVK/A/gfuBXO3D8y4BrInETcIqkUzvwvrm1qqTe+9JzqRSPnI59h2ueudbMLGOuKqlPA/uA7wG/Afw+SfvFyyLilhzvH8D1kgL4m4i4ui39NODhzOsd6bqdQL+kbUAd+JOI+Kccx5u3SqnI77/4qYxVG/zhl++YUi21ZrDCmkE/79vMDHI80zt9/gWSPkFyI98UEeM53/+iiBiRtB64QdLdEXFjzn3PSPc9G/impNsj4sftG0naAmwB2LRpU863PmLtYIWJWoP3f+XOKQ3ff/udB3jvL/8MDM77Lc3MTkhzjfSutRYiogHsmEewICJG0t+7ga3As9s2GQFOz7zemK7L7ns/8G3gvBmOcXVEDEfE8NDQUN6sTSoURLEIVzz3LP5x+8PsH6uxdrDCOy99KuXSXKfHzGz5mOuO+AxJB9Kfg8DTW8uSDsy2o6RBSStby8DFJPNSZV0LXJ62jVwAPB4ROyWtltSX7ruO5FkcXRv7ESE+9d0H+O3NT6RSLNBoBg/tHeXAmNsxzMxa5uoldSx9SzcAW9NxGyXgcxFxnaQr0/e+CvgqSZfa+0i61b4+3fepwN9IapIEtT/p5mDBiOD1F5511MyKhyfq7B+ruh3DzIycU4MsRFqV9Ixp1l+VWQ7gTdNs813gad3KW7tKqcjpawbYe+joCQc9gM/MLNG1gLGUrB2ssPtgk9Fpekq5QsrMLOGAQdLw3RrA99yz1/LG551NsSCaAUU/E8PMDHDAmNSM4Llnr+U3nncWO/aNcVKlmIz4XjPA6oEKJfeYMrNlznfBVH+5yJt//onsPVTl8//xEPvHaqzsL1GUeHzcD1MyM3PASK0b7KMg8bffObp77b5Rd681M3PASBUKotGcvXutmdly5oCRMVBOuteOVhtTqqVW9pddwjCzZc+N3hnrVvTxyONj/O13HuBtFz+JUqFIQdAMGK01aDaDgntNmdky5RJGRqEgms3gTZufSEHi4cdG2XNwgocfG2W81uCAG7/NbBlzwGjTXymyflX/Ub2lSoUC9Yarpcxs+XKVVJt1g66WMjObjksYbVq9pd60+YlU68GfXfdDfrT7EPtHq0TAwQlXS5nZ8uQSxjQGykUKg3184Ct3Till1BrB42N1VvZVXMows2XHJYxprFvRRzNcyjAzy3IJYxqFgigXC6xxKcPMbJJLGDNY71KGmdkULmHMoFQquJRhZpbhEsYssqWMgsS+w9Up80y5lGFmy4kDxixapYz1q/oZqzaolMSDe0fZe2iCH+8+zJ6DVc8xZWbLhgPGHNav6KPRDKr1JDCs6CuxfmUfp685iVKh4OlCzGzZ6GrAkPSgpNsl3SJp2zTpkvRhSfdJuk3SszJpV0j6UfpzRTfzOZtSKXkuxqkn9wGwaqBII6AgqDeDA+N1lzLMbFnoRQljc0Q8MyKGp0m7FDgn/dkCfAxA0hrgvcBzgGcD75W0ugd5ndZAuUggSoUCE/Xg729+iEYzKAgKEoeqLmWY2YlvsaukLgOuicRNwCmSTgVeBNwQEY9FxD7gBuCSxcpkayDfqoEy/3LrCK8cPp39ozUEjNea7B9tUKs1Fit7ZmY90e2AEcD1krZL2jJN+mnAw5nXO9J1M60/iqQtkrZJ2rZnz54OZXuqQkEMlIs0msGvXXAmY9XGlKqpar3JzoPjrpoysxNatwPGRRHxLJKqpzdJel6nDxARV0fEcEQMDw0NdfrtJ61b0UepKApKqqbkbrZmtsx0NWBExEj6ezewlaQ9ImsEOD3zemO6bqb1i6ZQEBvSqqk1KyrTlDKSwXwuZZjZiaprAUPSoKSVrWXgYuCOts2uBS5Pe0tdADweETuBrwMXS1qdNnZfnK5bVOVykf5ykUYj3ABuZstON6cG2QBsldQ6zuci4jpJVwJExFXAV4EXA/cBo8Dr07THJH0AuDl9r/dHxGNdzGtuQyv6eHjfKKsGynzmew9MNoCvW1FhvNakGTBQbFAuFxc7q2ZmHaWIE6cKZXh4OLZtO2q4R8ftPjDOaLWBBPtHq/SXCxQzT+arlMTGU07yPFNmdtyTtH2GYQ9HWexutUuSG8DNbDlywFiAmRrAi0XRVyrQbMLB8Qb1enOxs2pm1jEOGAvU3gAuibFqnYl6k2YE47UmOw+MudeUmZ0wHDCOwdCKPhrpCPBaPZioB//v3l2UioXJKqvD7jVlZicIB4xjkB0BvmqgzPcf3Mv5Z63js997gFq9yUS9yeNjDSYm6oudVTOzY+aAcYxaDeCNZvDCc0/lK7fs4NUXnEFfOXmWRr0R7Bmteq4pM1vyHDCOUasBvFQUzQiuuPAsqvXm0e0ZnmvKzJY4B4wOKJeLrB+sUC6IZjBte0bR7RlmtsQ5YHRIpVJi3UmVo9ozvnnXTgoStWZwYKzB+LjbM8xsaXLA6KC+vhLlQmGyPWP7A49y4ZPWc3CsRlGi1ggeHau6EdzMliQHjA5bv/JIe8aLnvZTNJtNyiVxYKxGsaDJRvBq1UHDzJYWB4wOK5UKk+0ZjWYwUCkREUcFjd2HHTTMbGlxwOiCbHuGkIOGmZ0QHDC6pK+vRH+peFTQGOwvEsSUoOExGma2FDhgdNHQyiOD+oRYNVCeHKNRKjKZtvvQhIOGmR33HDC6qFAQT1jZPxkYGs1kjEZfucjB8TqPj9aSLreNYNehCVdPmdlxzQGjy0qlwpSgsWqgTEGarJ7KljT2uE3DzI5jDhg9MBk0ClOrpwAOjtcZrzWSNo00aHichpkdjxwweqRUKrA+M1FhowkFiXJRjNWa/OudO6mUCgTwk0MT7Hrcc0+Z2fGl6wFDUlHSDyR9ZZq0MyR9Q9Jtkr4taWMmrSHplvTn2m7nsxdac05lG8JLheLkiPDxWp1iQRQLYrzeYOfjY35qn5kdN3pRwngL8MMZ0j4EXBMRTwfeD/xxJm0sIp6Z/ry025nslUqllMxum1ZPNSKZRkRAvQkHxqoUC0xWUe06OO4eVGZ2XOhqwEhLDL8EfGKGTc4Fvpkufwu4rJv5OV6Uy8XJ6qmikmlE+stFqrXmZA+qbLvG7kMTbtcws0XX7RLGXwLvAGaqV7kVeEW6/HJgpaS16et+Sdsk3STpZV3OZ8+Vy8X0ORpMTiPS6kFVLooAxmsN+tyuYWbHia4FDEkvAXZHxPZZNns78HxJPwCeD4wArfqXMyJiGPg14C8l/fQMx9mSBpZte/bs6eBf0H3lcpENKwc4eaA4pQdVf7lEqQABPJ6pomq1a7iKyswWQzdLGBcCL5X0IPAF4IWSPpPdICIeiYhXRMR5wLvTdfvT3yPp7/uBbwPnTXeQiLg6IoYjYnhoaKhbf0vXFApiRX8fawcqmR5UQakwtYqqGUEpU0Xl52qYWa91LWBExLsiYmNEnAm8CvhmRLw2u42kdZJaeXgX8Ml0/WpJfa1tSILPXd3K6/Ggv7/E0EmZoBFHqqhOqhR5fKzODXfuZKBSBGDX4Qke2edeVGbWOz0fhyHp/ZJavZ5eANwj6V5gA/DBdP1TgW2SbiVpDP+TiDihAwYkExa2elAVdaSKCsT2Bx5l87lPYP9olUYE/aUCTYKRx8fctmFmPaGIE+dGMzw8HNu2bVvsbByzer3JnsPjNJpQbwQSSAKCgmCs1qTZbLJ6sEKtHtSbSXXV0GCFSqW02Nk3syVE0va0vXhOHul9HCqVCkcaw4vJQL5mHCltVGtNVvSX2HWgyg137mRFf1JNtfPgBCP7Rt0obmZd4a+jx6lCQazs76O/2GDfeBXVoZaWNlYNlGk2YfsDj3Lx007lscM1BJycTp/+yIFxyoVC8rjYkr8TmFlnOGAc58rlIuuK/YxVq+wbS0oO9UaASEeIi1IBCoUC+0arFCTWDJYZqzYZeXyMUkGsX9FHuVxc5L/EzJY6B4wloFAQg/19lAt19k/UAIiARiSTGJYKyVP8ykXRVy6y1yUOM+sC3z2WkEqlxLrBfk4ZKFLMjBBvxJGJDFsljnIpKXE0IliZPhZ25PExt3GY2YI5YCwxrYF+6wf7WTVQmNIFNylxRBo44KRK0mg+sn9isnG8VBC7Dk3w0N7D7D7g7rhmlp+rpJaoUqnAylI/fYU6e8eqSEqqqZpBAEUJgEZjauN4qZA0mkcTJupNHt43SrmYPqvD1VVmNgsHjCWuUimxoVTkULXK4YkGzaZopoFDSuajeuG5p9JogIC+cpFDEw0mag36y0VW9hep1YOfHByn0QwGykXWreijUNBi/2lmdpxxwDgBFApiVX8fKyoxGTiUljBajeMA/eXiZOA4qVKkUBB7D9cmBwFWigUOTySlDg8ENLN2roM4gbQCx4aVA6waKFIpabJxPNvO0RoA2GgwOQgQ4OF9R9o6BvsK7Dlc5aG9hxnZN0q16skOzZY7f308AbWXOCA4NNGkME11VWsQYDVisq2j0QweOVBl+wOP8svnnYaAAxN1Jg5XqTfDbR5my5QDxgmsFTgABooNxho1Dk00p1ZXNZNBgIojbR3N5pHgMVFvUmsEEcFJlSIDhSRI7Do4PjmHlQcGmi0PDhjLRLlcpFwuMlBscLhWo9pIJi1s9a6SoJ52sc0GD4BKMSmZ1JvJPocnGghYM1gG4GC1xtihCQcQsxOcA8YyUy4XOaVcpNlMqqsigsPVJqWCUB2abcEDQE0RkZRKSsVkYGBfuch4+iyObAApyAHE7ETlgLFMZaurBstNDk5UGRgQByemBg84MrYDQOlUJK3SRzaAtEogew/VJts/imkAaY7XmKg33QZitoQ5YBilUoHVpX4A+osNDlZrk8EDmGwsh6nBA6YGkGYziGBK4/lEM6g1Ysq4D4ADE1UKVTg80fTzPMyWCF+dNkW5XGRNWn3UCh7FAhyuHh08oK30EVMHCrYCSHbcR6saq1QQu9p6Yo3W6xyqJd13a/Wglj7f3IMJbTmr1ZLrsKUgplyDY9Vmz6p/HTBsRtng0aq2UvrEv9bnNRtApKkDBVsBJDtNSUt7T6xSQZPtJocnGpNTmLRCxIHxicnjNNNpTVy9Zce7ZjM4OFGl0YwpN/r2m/5MaaUiHEpL+sCU6wRg76Eav/WZ7ezYN8bG1QN87LXn85T1K7oWNPyIVpu39ougFUBaH+ZoHmk8b3XdbY37aBHJY2dbAaG1LcBEvcFJfUc+8NmLJNvFt5SWOKa72FxCsWMx040eZr65t78uFmC0OvXagKNv+rOlta6llux1AvDqj9/Ejn1jk683rh7gi1su4LTVJ+X+W+fziNaulzAkFYFtwEhEvKQt7Qzgk8AQ8Bjw2ojYkaZdAfxBuun/jIhPdTuvlk+hIE4e6Jt8vSpzcVWKolBIvhW1PvzZcR8tR/XEiiMBpTWFSUuraguO7uI73cWWzKl1ZLqTZsD+sYkZL26XWJa+VrXNQr7FT5fW/iWoZT43/tHqkc9t9jOcXZ4zLdraDGPqF69ssGi9zuap03pRJfUW4IfAqmnSPgRcExGfkvRC4I+B10laA7wXGCY5P9slXRsR+3qQX5un9gACyUDBg9UalWISLFrjPlrae2Jlvzkd1bCeuUiyXXxh+outWmtyymDy0R6tHWkzme7inq1BfiE3m17WJx/PWt22640jJ+pYbuCzpbV/QYGF3+hh6rf6ed3c29JaT8aEts9w201/trRSQUd90cq+3rh64KgSRqvk3Q1dDRiSNgK/BHwQeNs0m5ybWf8t4J/S5RcBN0TEY+n73ABcAny+m/m1zsm2f8D0N5BsT6zsRdvesJ47mMTR053MdXHP1CC/kJvN3kM1HtxzgAvPGQKS7sSFWq0jN8WZ0jr1Pp1Mg+m/ICz0m/pcaa0b/IK/xc/yrX4+N/f2tGLmxp39DLff9GdLG6/NXD0L8LHXnn9UG8b6FVO/vHVSt0sYfwm8A1g5Q/qtwCuAvwJeDqyUtBY4DXg4s92OdJ0tUdlxH1mtnlhwpLpptoa+2YJJts2kNd3JXBf3TA3yC7nZPLjnAOeftXbawNONG2YvjrGQtGb6r5v1ptyBm3trub2TRfvyfNJaf8dCbu7tac0IGs0mfeXiwtsw6jBRa0zpANK6TgBOO6WPL265YOn3kpL0EmB3RGyX9IIZNns78BFJvw7cCIwA83p+qKQtwBaATZs2LTi/tjjaSyItA8WpXQlbF0m2iy9Mf7FlpzuZ6+KetkF+gTeb885YO2Pg6cYNsxfHWEhaJ89pnrRC2nliod/i218XC2K8Vj/qRg/zC6ZjtaTKs9FssGawPOVGn11uf51d7i8VqNWDsWqTZjPon64DxyA9080SxoXASyW9GOgHVkn6TES8trVBRDxCUsJA0grglRGxX9II8ILMe20Evj3dQSLiauBqSHpJdeHvsEUwUyCBI118W//s9otNYnK6k7ku7pka5Bdys8lTXdbJtF4cYyFpxYLmvCl34ubeWv7GXTu5+Gmndq4Ng6DehMbE0Td6mPnm3v765P4i9XKRiXqTg+ONE6JDRU+61aYljLdP00tqHfBYRDQlfRBoRMR70kbv7cCz0k2/D5zfatOYibvVWku9fmTcSK/q22frAtmNG2YvjrGQtCDSxugj34K7WSXWPhXNsbbLtCyXDgzHVbfadpLeD2yLiGtJShF/LClIqqTeBBARj0n6AHBzutv75woWZlnZ6U7mMl2D/EK+VZaLTDbiw/Jtw6g1YLSa9DxbkWmwXeg39bnS+k/pY83PnMqh8UZnb+49rOpZKjxwz6yDqtUj05tA93sp9eIYC0nz2Jal47guYZidyCqVEms8gaKdoBzyzcwsFwcMMzPLxQHDzMxyccAwM7NcHDDMzCwXBwwzM8vlhBqHIWkP8NACd18HPNrB7HSK8zU/ztf8OF/zcyLm64yIGMqz4QkVMI6FpG15B6/0kvM1P87X/Dhf87Pc8+UqKTMzy8UBw8zMcnHAOOLqxc7ADJyv+XG+5sf5mp9lnS+3YZiZWS4uYZiZWS7LPmBIukTSPZLuk/TOHh/7dEnfknSXpDslvSVd/z5JI5JuSX9enNnnXWle75H0oi7m7UFJt6fH35auWyPpBkk/Sn+vTtdL0ofTfN0m6Vmzv/uC8/TkzDm5RdIBSW9drPMl6ZOSdku6I7Nu3udI0hXp9j+SdEUX8vTnku5Oj7tV0inp+jMljWXO21WZfc5P///3pfnWdMfrQN7m/b/r9DU7Q76+mMlLhP33AAAGCUlEQVTTg5JuSdf35JzNcm9Y1M8XEbFsf4Ai8GPgbKAC3Aqc28Pjnwo8K11eCdwLnAu8j+QJhe3bn5vmsQ84K817sUt5exBY17buz4B3psvvBP40XX4x8DVAwAXAv/fof/cT4IzFOl/A80ieCnnHQs8RsAa4P/29Ol1e3eE8XQyU0uU/zeTpzOx2be/zH2k+leb70i6dr3n977pxzU6Xr7b0vwDe08tzNsu9YVE/X8u9hPFs4L6IuD8iqsAXgMt6dfCI2BkR30+XDwI/BE6bZZfLgC9ExEREPADcR/I39MplwKfS5U8BL8usvyYSNwGnSDq1y3n5eeDHETHbQM2unq+IuBFofxLkfM/Ri4AbIuKxiNgH3ABc0sk8RcT1EdF6qtNNwMbZ3iPN16qIuCmSu841mb9jwWY4XzOZ6X/X8Wt2tnylpYRfBT4/23t0+pzNcm9Y1M/Xcg8YpwEPZ17vYPYbdtdIOhM4D/j3dNWb06LlJ1vFTnqb3wCul7Rd0pZ03YaI2Jku/wTYsAj5ankVUy/ixT5fLfM9R73O438n+SbacpakH0j6N0k/l8nrjh7maT7/u16fr58DdkXEjzLrenrO2u4Ni/r5Wu4B47ggaQXwj8BbI+IA8DHgp4FnAjtJisS9dlFEPAu4FHiTpOdlE9NvUYvSxU5SBXgp8PfpquPhfB1lMc/RdCS9G6gDn01X7QQ2RcR5wNuAz0la1eNsHZf/u4xXM/WLSU/P2TT3hkmL8fla7gFjBDg983pjuq5nJJVJPhCfjYgvAUTErohoREQT+DhHqlF6lt+IGEl/7wa2pnnY1apqSn/v7nW+UpcC34+IXWkeF/18Zcz3HPUkj5J+HXgJ8Jr0RkNa3bM3Xd5O0jbwpPT42Wqrbn7O5vu/69n/VFIJeAXwxUx+e3bOprs3sMifr+UeMG4GzpF0Vvqt9VXAtb06eFo/+n+AH0bE/8qsz9b/vxxo9d64FniVpD5JZwHnkDS0dTpfg5JWtpZJGk3vSI/f6mVxBfDlTL4uT3tqXAA8nik2d8OUb32Lfb7azPccfR24WNLqtDrm4nRdx0i6BHgH8NKIGM2sH5JUTJfPJjk/96f5OiDpgvQzennm7+ioBfzvennN/gJwd0RMVjX16pzNdG9gsT9fC20tP1F+SHoX3EvyTeHdPT72RSRFytuAW9KfFwOfBm5P118LnJrZ591pXu+hAz1XZsjX2SS9T24F7mydF2At8A3gR8C/AmvS9QL+Os3X7cBwF8/ZILAXODmzblHOF0nQ2gnUSOqG37CQc0TSrnBf+vP6LuTpPpJ67NZn7Kp021em/99bgO8Dv5x5n2GSm/ePgY+QDvLtQt7m/b/r9DU7Xb7S9X8HXNm2bU/OGTPfGxb18+WR3mZmlstyr5IyM7OcHDDMzCwXBwwzM8vFAcPMzHJxwDAzs1wcMMxmIelQh97nfZLenmO7v5P0K504plmnOWCYmVkuDhhmOUhaIekbkr6v5JkHl6Xrz1TyrIm/k3SvpM9K+gVJ30mfP5CdHfcZkr6Xrn9jur8kfUTJ8x3+FVifOeZ7JN0s6Q5JV6ejf80WjQOGWT7jwMsjmZBxM/AXmRv4E0kmzXtK+vNrJCN13w78fuY9ng68EPivwHsk/RTJdBhPJnnWweXAczPbfyQi/ktE/CwwQDIXlNmiKS12BsyWCAF/lM7a2ySZIro1tfQDEXE7gKQ7gW9EREi6neSBOy1fjogxYEzSt0gm2nse8PmIaACPSPpmZvvNkt4BnETyAJw7gX/u2l9oNgcHDLN8XgMMAedHRE3Sg0B/mjaR2a6Zed1k6jXWPg/PjPPySOoHPkoyJ9DDkt6XOZ7ZonCVlFk+JwO702CxmeTRsPN1maR+SWuBF5DMvHoj8N8kFdOZWzen27aCw6PpMxHcc8oWnUsYZvl8FvjntJppG3D3At7jNuBbwDrgAxHxiKStJO0adwH/CXwPICL2S/o4yeynPyEJLmaLyrPVmplZLq6SMjOzXBwwzMwsFwcMMzPLxQHDzMxyccAwM7NcHDDMzCwXBwwzM8vFAcPMzHL5/0MMFpAakW5gAAAAAElFTkSuQmCC\n",
"text/plain": [
"<Figure size 432x288 with 1 Axes>"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"# RIDGE Regression - Cross-Validation with test MSE\n",
"\n",
"results = pd.DataFrame()\n",
"alpha_range = np.linspace(10**-3, 2000, num=1000)\n",
"for train_idx, test_idx in model_selection.KFold(n_splits=8).split(df_boston):\n",
" train, test = df_boston.iloc[train_idx], df_boston.iloc[test_idx]\n",
" errors = []\n",
" for alpha in alpha_range:\n",
" reg = linear_model.Ridge(alpha=alpha)\n",
" reg.fit(train.drop('CRIM', axis=1), train['CRIM'])\n",
" error = np.sqrt(metrics.mean_squared_error(test['CRIM'], reg.predict(test.drop('CRIM', axis=1))))\n",
" errors.append(error)\n",
" results = pd.concat([results, pd.DataFrame(errors, index=alpha_range).T], axis=0, ignore_index=True)\n",
"\n",
"df = pd.DataFrame({'lambda' : alpha_range, 'RMSE' : results.mean()})\n",
"sns.scatterplot(x='lambda', y='RMSE', data=df)"
]
},
{
"cell_type": "code",
"execution_count": 458,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"4.897662699449212"
]
},
"execution_count": 458,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"lmbda, RMSE = df.sort_values('RMSE', ascending=True).iloc[0]\n",
"# best rmse for ridge\n",
"RMSE"
]
},
{
"cell_type": "code",
"execution_count": 460,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"<matplotlib.axes._subplots.AxesSubplot at 0x11c530a58>"
]
},
"execution_count": 460,
"metadata": {},
"output_type": "execute_result"
},
{
"data": {
"image/png": "iVBORw0KGgoAAAANSUhEUgAAAXwAAAEKCAYAAAARnO4WAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAEilJREFUeJzt3X20ZXVdx/H3hxkSxMfihiTS4EOaWYJOZIEuhGSRmEpkyhKXVEZWJvawSLNlWatWPiyXuSpdhIiFYghiPiRCidKDYTMwwMD4FKJCwFwt0mqJAt/+2Hta12HmnrnM/t17Z37v11pnzd7n7PP7/s6dez7nd/fZ+7dTVUiS9n77rHQHJEnLw8CXpE4Y+JLUCQNfkjph4EtSJwx8SeqEgS9JnTDwJakTBr4kdWLtSndgoQMPPLDWrVu30t2QpD3Gxo0bv1JVc7uy7aoK/HXr1rFhw4aV7oYk7TGSfHFXt3WXjiR1wsCXpE4Y+JLUCQNfkjph4EtSJwx8SeqEgS9JnTDwJakTq+rEK61+r37vCZO3+YfPu2TyNiXdmyN8SeqEgS9JnTDwJakTBr4kdcIvbbUqPfP9vz15m3/73D+avE1pT+IIX5I6YeBLUicMfEnqhIEvSZ3wS1tJauz2t/zj5G0e9PKjl/wcR/iS1AlH+HuJc995/ORtnvbiSydvU9LKcYQvSZ1whL8MLj/7xEnbe/pLPjxpe5L64AhfkjrR9Qj/5j/9ucnbPORl50zepto58aK/mLS9D5/8C5O2p7Zuff2tk7d58JkHT97mVLoOfEmr18fPm5+0vWNOnZu0vT1R08BPchPwdeBu4K6qWt+yniRp55ZjhP/0qvrKMtSRJC3CXTqSluRNF982eZu/ftLDJm9T99b6KJ0CLk2yMcnpjWtJkhbReoR/dFXdkuS7gcuSfLqqrli4wfhBcDrAoYce2rg70vL7yQsvnrzND/70SZO3qb1f08CvqlvGf7cmuRg4Erhiu23OAs4CWL9+fQHMv/W8yfsy90unTt6mtJqcfNGnJm/zopOPnLxNrZxmu3SSHJDkgduWgeOBza3qSZIW13KEfxBwcZJtdd5dVZc0rCdJWkSzwK+qG4EntmpfkrQ0zqUjSZ0w8CWpEwa+JHXCwJekThj4ktQJA1+SOmHgS1InDHxJ6oSBL0mdMPAlqRMGviR1wsCXpE4Y+JLUCQNfkjph4EtSJwx8SeqEgS9JnTDwJakTBr4kdcLAl6ROGPiS1AkDX5I6YeBLUicMfEnqhIEvSZ0w8CWpEwa+JHXCwJekTjQP/CRrklyd5EOta0mSdm45RvhnAFuWoY4kaRFNAz/JIcCJwNkt60iSZms9wn8zcCZwT+M6kqQZmgV+kmcBW6tq44ztTk+yIcmG+fn5Vt2RpO61HOEfBTw7yU3Ae4Bjk5y3/UZVdVZVra+q9XNzcw27I0l9axb4VfWqqjqkqtYBLwA+VlWntqonSVqcx+FLUifWLkeRqvo48PHlqCVJ2jFH+JLUCQNfkjph4EtSJwx8SeqEgS9JnTDwJakTBr4kdcLAl6ROGPiS1AkDX5I6YeBLUicMfEnqhIEvSZ0w8CWpEwa+JHXCwJekThj4ktQJA1+SOmHgS1InDHxJ6oSBL0mdMPAlqRMGviR1YtHAT3LsguXDtnvsp1p1SpI0vVkj/DcuWL5ou8d+Z+K+SJIamhX42cnyjtYlSavYrMCvnSzvaF2StIqtnfH4I5N8gGE0v22Zcf2wnT9NkrTazAr85yxYfuN2j22/LklaxRYN/Kr6xML1JPsCTwBuqaqtiz03yX7AFcD9xjoXVtXv7l53JUn31azDMt+W5AfG5QcD1wB/CVyd5JQZbd8JHFtVTwQOB05I8pQJ+ixJug9mfWn71Kq6flz+WeCzVfWDwJOBMxd7Yg3+e1zdd7z5Ra8krZBZgf/NBcvPAN4PUFW37UrjSdYk2QRsBS6rqit3sM3pSTYk2TA/P7+L3ZYkLdWswL8jybOSHAEcBVwCkGQtsP+sxqvq7qo6HDgEODLJE3awzVlVtb6q1s/NzS39FUiSdsmso3R+EXgL8DDgFQtG9scBH97VIlV1R5LLgROAzfelo5Kk3TPrKJ3PMoT09vd/FPjoYs9NMgd8awz7/Rl2Cb1uN/oqSdoNiwZ+krcs9nhVvXyRhw8G3plkDcOuowuq6kNL76IkaQqzdum8lGEXzAXAv7OE+XOq6lrgiPveNUnSlGYF/sHA84DnA3cBf81wAtUdrTsmSZrWokfpVNVXq+ptVfV0huPwHwLckORFy9I7SdJkZo3wAUjyJOAUhi9ePwJsbNkpSdL0Zn1p+/vAicAW4D3Aq6rqruXomCRpWrNG+L8DfAF44nj7oyQwfHlbVfVDbbsnSZrKrMB3zntJ2kvMOvHqizu6P8k+DPv0d/i4JGn1mTU98oOSvCrJnyY5PoNfBW4EfmZ5uihJmsKsXTp/Bfwn8EngJcBvM+y/f25VbWrcN0nShGZe03ac/54kZwO3AodW1Tea90ySNKlZ0yN/a9tCVd0N3GzYS9KeadYI/4lJvjYuB9h/XN92WOaDmvZOkjSZWUfprFmujkiS2pq1S0eStJcw8CWpEwa+JHXCwJekThj4ktQJA1+SOmHgS1InDHxJ6oSBL0mdMPAlqRMGviR1wsCXpE4Y+JLUCQNfkjph4EtSJ5oFfpJHJLk8yQ1Jrk9yRqtakqTZZl3xanfcBfxGVV2V5IHAxiSXVdUNDWtKknai2Qi/qm6tqqvG5a8DW4CHt6onSVrcsuzDT7IOOAK4cgePnZ5kQ5IN8/Pzy9EdSepS88BP8gDgIuAVVfW17R+vqrOqan1VrZ+bm2vdHUnqVtPAT7IvQ9i/q6re17KWJGlxLY/SCfB2YEtVvalVHUnSrmk5wj8KeBFwbJJN4+2ZDetJkhbR7LDMqvpHIK3alyQtjWfaSlInDHxJ6oSBL0mdMPAlqRMGviR1wsCXpE4Y+JLUCQNfkjph4EtSJwx8SeqEgS9JnTDwJakTBr4kdcLAl6ROGPiS1AkDX5I6YeBLUicMfEnqhIEvSZ0w8CWpEwa+JHXCwJekThj4ktQJA1+SOmHgS1InDHxJ6oSBL0mdMPAlqRPNAj/JOUm2JtncqoYkade1HOGfC5zQsH1J0hI0C/yqugL4j1btS5KWZsX34Sc5PcmGJBvm5+dXujuStNda8cCvqrOqan1VrZ+bm1vp7kjSXmvFA1+StDwMfEnqRMvDMs8HPgk8NsnNSX6+VS1J0mxrWzVcVae0aluStHTu0pGkThj4ktQJA1+SOmHgS1InDHxJ6oSBL0mdMPAlqRMGviR1wsCXpE4Y+JLUCQNfkjph4EtSJwx8SeqEgS9JnTDwJakTBr4kdcLAl6ROGPiS1AkDX5I6YeBLUicMfEnqhIEvSZ0w8CWpEwa+JHXCwJekThj4ktQJA1+SOmHgS1InmgZ+khOSfCbJ55O8smUtSdLimgV+kjXAnwE/ATweOCXJ41vVkyQtruUI/0jg81V1Y1V9E3gP8JyG9SRJi0hVtWk4+WnghKp6ybj+IuBHqupl2213OnD6uPpY4DNLKHMg8JUJumudPbOGdVZvDessX43vraq5Xdlw7dL7M62qOgs46748N8mGqlo/cZess4fUsM7qrWGd1Vmj5S6dW4BHLFg/ZLxPkrQCWgb+vwKPSXJYku8AXgB8oGE9SdIimu3Sqaq7krwM+CiwBjinqq6fuMx92hVknWWpsze9lr2tzt70Wva2Ok1rNPvSVpK0unimrSR1wsCXpE7skYGf5JwkW5NsblznEUkuT3JDkuuTnNGozn5JPpXkmrHOa1vUGWutSXJ1kg81rHFTkuuSbEqyoWGdhyS5MMmnk2xJ8qMNajx2fB3bbl9L8ooGdX5t/L/fnOT8JPtNXWOsc8ZY4/opX8eO3pNJvjPJZUk+N/770EZ1nje+nnuS7PYhjTup8Ybx9+zaJBcneUijOn8w1tiU5NIk37O7db5NVe1xN+BpwJOAzY3rHAw8aVx+IPBZ4PEN6gR4wLi8L3Al8JRGr+nXgXcDH2r4c7sJOHAZfg/eCbxkXP4O4CGN660BbmM40WXKdh8OfAHYf1y/ADitQf+fAGwG7s9wwMbfAY+eqO17vSeB1wOvHJdfCbyuUZ3vZzhp8+PA+kY1jgfWjsuva/haHrRg+eXA26b8HdgjR/hVdQXwH8tQ59aqumpc/jqwheHNOXWdqqr/Hlf3HW+Tf5ue5BDgRODsqdtebkkezPCGeTtAVX2zqu5oXPY44N+q6osN2l4L7J9kLUMg/3uDGt8PXFlV/1tVdwGfAH5qioZ38p58DsOHMuO/z21Rp6q2VNVSztC/LzUuHX9mAP/CcF5RizpfW7B6ABPnwB4Z+CshyTrgCIbRd4v21yTZBGwFLquqFnXeDJwJ3NOg7YUKuDTJxnHqjBYOA+aBd4y7qM5OckCjWtu8ADh/6kar6hbgjcCXgFuB/6qqS6euwzC6f2qS70pyf+CZfPvJkVM7qKpuHZdvAw5qWGs5/RzwkVaNJ/nDJF8GXgi8Zsq2DfxdkOQBwEXAK7b7BJ5MVd1dVYczjByOTPKEKdtP8ixga1VtnLLdnTi6qp7EMFPqryR5WoMaaxn+HH5rVR0B/A/DboMmxpMHnw28t0HbD2UYDR8GfA9wQJJTp65TVVsYdkdcClwCbALunrrOTmoXDf5qXW5JXg3cBbyrVY2qenVVPWKs8bJZ2y+FgT9Dkn0Zwv5dVfW+1vXG3RKXAydM3PRRwLOT3MQwc+mxSc6buAbw/yNWqmorcDHDzKlTuxm4ecFfQhcyfAC08hPAVVV1e4O2fxz4QlXNV9W3gPcBP9agDlX19qp6clU9DfhPhu+lWrk9ycEA479bG9ZqLslpwLOAF44fYK29Czh5ygYN/EUkCcM+4i1V9aaGdea2feufZH/gGcCnp6xRVa+qqkOqah3DromPVdXko8gkByR54LZlhi+7Jj+aqqpuA76c5LHjXccBN0xdZ4FTaLA7Z/Ql4ClJ7j/+zh3H8H3R5JJ89/jvoQz779/dos7oA8CLx+UXA3/TsFZTSU5g2B367Kr634Z1HrNg9TlMnAOTHgWwXDeGN96twLcYRno/36jO0Qx/hl7L8OfvJuCZDer8EHD1WGcz8JrGP79jaHSUDvBI4Jrxdj3w6oav43Bgw/hzez/w0EZ1DgC+Cjy44Wt57fjm3gz8FXC/RnX+geGD8RrguAnbvdd7Evgu4O+BzzEcEfSdjeqcNC7fCdwOfLRBjc8DX16QA7t99MxO6lw0/g5cC3wQePiU//9OrSBJnXCXjiR1wsCXpE4Y+JLUCQNfkjph4EtSJwx8aZVJckySJideqW8GvrT6HEOjM23VNwNfq1KSdeMc938xznV+6XgW8o62fXSSvxuvJ3BVkkdl8IZx7vfrkjx/3PaYJJ9I8jdJbkzyx0lemOF6BNcledS43blJ3pZkQ5LPjnMRbbt2wTvGba9O8vTx/tOSvC/JJeP8769f0L/jk3xy7Nt7x7mZtl034LXj/dcledw4Sd9LgV8b50R/6jjf++bx9V3R8ueuvVyrMwe9edudG7COYZKqw8f1C4BTd7LtlcBJ4/J+DNMLnwxcxjCH/UEM0xcczDB6vmNcvh9wC/Da8blnAG8el89lmGBsH+AxDGdC7gf8BnDOuM3jxnb3A04DbgQePK5/kWEmygOBK4ADxuf8FuOZ1AzXDfjVcfmXgbPH5d8DfnPB67uO8YxLGs/5723vvjnC12r2haraNC5vZPgQ+DbjvD0Pr6qLAarqGzXMdXI0cH4Ns5DezjD3+w+PT/vXGq51cCfwbwyzR8IQrAtrXFBV91TV5xjC/HFju+eNtT7NEOzfN27/91X1X1X1DYbpC74XeArweOCfxumvXzzev822Cfl2+PpG/wScm+QXGD7ApPtk7Up3QFrEnQuW7wZ2uEtnN9u9Z8H6PXz7e2L7eUdmzUOyfX/XMlzN7LKqOmXGc7Ztfy9V9dIkP8Jw8ZqNSZ5cVV+d0RfpXhzha49Ww5XIbk7yXIAk9xsv7vEPwPPHC8vMMVwd61NLbP55SfYZ9+s/EvjM2O4Lx1rfBxw63r8z/wIcleTR43MOGJ+3mK8zXFKT8TmPqqorq+o1DBd9aXnREu3FDHztDV4EvDzJtcA/Aw9jmIf/WoZZIT8GnFnDlMpL8SWGD4mPAC8dd9X8ObBPkuuAv2a49uydO2ugquYZ9u+fP/bvkwy7hhbzQeCkbV/aAm8Yv9TdPL6+a5b4OiQAZ8uUdiTJuQxTSF+40n2RpuIIX5I64Qhfe4wkf8ZwqcaF/qSq3rES/ZH2NAa+JHXCXTqS1AkDX5I6YeBLUicMfEnqhIEvSZ34P60tw9vQQ6UzAAAAAElFTkSuQmCC\n",
"text/plain": [
"<Figure size 432x288 with 1 Axes>"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"# PCR Regression - Cross-Validation with test MSE\n",
"\n",
"results = pd.DataFrame()\n",
"predictors = df_boston.drop('CRIM', axis=1).columns\n",
"c_range = range(1, len(predictors) + 1)\n",
"for train_idx, test_idx in model_selection.KFold(n_splits=10).split(df_boston):\n",
" train, test = df_boston.iloc[train_idx], df_boston.iloc[test_idx]\n",
" errors = []\n",
" for c in c_range:\n",
" pca = decomposition.PCA(n_components=c)\n",
" X = pca.fit_transform(train.drop('CRIM', axis=1))\n",
" reg = linear_model.LinearRegression()\n",
" reg.fit(X, train['CRIM'])\n",
" \n",
" test_X = pca.transform(test.drop('CRIM', axis=1))\n",
" error = np.sqrt(metrics.mean_squared_error(test['CRIM'], reg.predict(test_X)))\n",
" errors.append(error)\n",
" results = pd.concat([results, pd.DataFrame(errors, index=c_range).T], axis=0, ignore_index=True)\n",
"\n",
"df = pd.DataFrame({'n_components' : c_range, 'RMSE' : results.mean()})\n",
"sns.barplot(x='n_components', y='RMSE', data=df)"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.6.5"
}
},
"nbformat": 4,
"nbformat_minor": 2
}