![](\"https://www.tilburguniversity.edu/upload/c67b70dd-9a4f-499b-82e0-0aee20dfe12a_jads%20logo.png\",width=500)
\n",
"\n",
"\n",
"\n",
" \n",
"
\n",
"
"
],
"text/plain": [
" Total Percent\n",
"PoolQC 2907 0.997256\n",
"MiscFeature 2810 0.963979\n",
"Alley 2717 0.932075\n",
"Fence 2345 0.804460\n",
"FireplaceQu 1420 0.487136\n",
"LotFrontage 486 0.166724\n",
"GarageCond 159 0.054545\n",
"GarageQual 159 0.054545\n",
"GarageYrBlt 159 0.054545\n",
"GarageFinish 159 0.054545\n",
"GarageType 157 0.053859\n",
"BsmtCond 82 0.028130\n",
"BsmtExposure 82 0.028130\n",
"BsmtQual 81 0.027787\n",
"BsmtFinType2 80 0.027444\n",
"BsmtFinType1 79 0.027101\n",
"MasVnrType 24 0.008233\n",
"MasVnrArea 23 0.007890\n",
"MSZoning 4 0.001372\n",
"BsmtHalfBath 2 0.000686\n",
"Utilities 2 0.000686\n",
"Functional 2 0.000686\n",
"BsmtFullBath 2 0.000686\n",
"BsmtFinSF2 1 0.000343\n",
"BsmtFinSF1 1 0.000343\n",
"Exterior2nd 1 0.000343\n",
"BsmtUnfSF 1 0.000343\n",
"TotalBsmtSF 1 0.000343\n",
"Exterior1st 1 0.000343\n",
"SaleType 1 0.000343\n",
"Electrical 1 0.000343\n",
"KitchenQual 1 0.000343\n",
"GarageArea 1 0.000343\n",
"GarageCars 1 0.000343"
]
},
"execution_count": 42,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"def count_missing_data(df):\n",
" \"\"\" Counts the missing values for each features\n",
" and display the Total and the Percentage\n",
" \"\"\"\n",
" # Calculates the total number of missing values for each feature\n",
" total = df.isnull().sum().sort_values(ascending=False)\n",
" \n",
" # Calculates the percentage of missing values for each features\n",
" percent = (df.isnull().sum()/df.isnull().count()).sort_values(ascending=False)\n",
" \n",
" # Combines the percentage and totals\n",
" missing_data = pd.concat([total, percent], axis=1, keys=['Total', 'Percent'])\n",
" \n",
" # Only returns data that have missing values\n",
" return missing_data[missing_data['Percent']>0]\n",
"\n",
"print(\"Number of features with missing values: {}\".format(len(count_missing_data(all_features))))\n",
"count_missing_data(all_features)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"I will fill the missing values for the categorical features with the median using the function below. "
]
},
{
"cell_type": "code",
"execution_count": 43,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"def fillna_numerica_average(df):\n",
" \"\"\" Fills the numeric features that contain \n",
" NaN with the average in those columns\n",
" \"\"\"\n",
" # Get features that contain missing values\n",
" features_with_nan = count_missing_data(df).index \n",
" \n",
" # Get numeric features that contain missing values\n",
" numeric_missing_features = df[features_with_nan].dtypes[df.dtypes != \"object\"].index \n",
" \n",
" # Fill each missing values for the numeric features with the corresponding median\n",
" for feature in numeric_missing_features:\n",
" df[feature].fillna(df[feature].median(), inplace=True)\n",
" \n",
"fillna_numerica_average(all_features)"
]
},
{
"cell_type": "code",
"execution_count": 44,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Number of features with missing values: 23\n",
"Categorical features\n"
]
},
{
"data": {
"text/html": [
"| \n", " | Total | \n", "Percent | \n", "
|---|---|---|
| PoolQC | \n", "2907 | \n", "0.997256 | \n", "
| MiscFeature | \n", "2810 | \n", "0.963979 | \n", "
| Alley | \n", "2717 | \n", "0.932075 | \n", "
| Fence | \n", "2345 | \n", "0.804460 | \n", "
| FireplaceQu | \n", "1420 | \n", "0.487136 | \n", "
| LotFrontage | \n", "486 | \n", "0.166724 | \n", "
| GarageCond | \n", "159 | \n", "0.054545 | \n", "
| GarageQual | \n", "159 | \n", "0.054545 | \n", "
| GarageYrBlt | \n", "159 | \n", "0.054545 | \n", "
| GarageFinish | \n", "159 | \n", "0.054545 | \n", "
| GarageType | \n", "157 | \n", "0.053859 | \n", "
| BsmtCond | \n", "82 | \n", "0.028130 | \n", "
| BsmtExposure | \n", "82 | \n", "0.028130 | \n", "
| BsmtQual | \n", "81 | \n", "0.027787 | \n", "
| BsmtFinType2 | \n", "80 | \n", "0.027444 | \n", "
| BsmtFinType1 | \n", "79 | \n", "0.027101 | \n", "
| MasVnrType | \n", "24 | \n", "0.008233 | \n", "
| MasVnrArea | \n", "23 | \n", "0.007890 | \n", "
| MSZoning | \n", "4 | \n", "0.001372 | \n", "
| BsmtHalfBath | \n", "2 | \n", "0.000686 | \n", "
| Utilities | \n", "2 | \n", "0.000686 | \n", "
| Functional | \n", "2 | \n", "0.000686 | \n", "
| BsmtFullBath | \n", "2 | \n", "0.000686 | \n", "
| BsmtFinSF2 | \n", "1 | \n", "0.000343 | \n", "
| BsmtFinSF1 | \n", "1 | \n", "0.000343 | \n", "
| Exterior2nd | \n", "1 | \n", "0.000343 | \n", "
| BsmtUnfSF | \n", "1 | \n", "0.000343 | \n", "
| TotalBsmtSF | \n", "1 | \n", "0.000343 | \n", "
| Exterior1st | \n", "1 | \n", "0.000343 | \n", "
| SaleType | \n", "1 | \n", "0.000343 | \n", "
| Electrical | \n", "1 | \n", "0.000343 | \n", "
| KitchenQual | \n", "1 | \n", "0.000343 | \n", "
| GarageArea | \n", "1 | \n", "0.000343 | \n", "
| GarageCars | \n", "1 | \n", "0.000343 | \n", "
\n",
"\n",
" \n",
"
\n",
"
"
],
"text/plain": [
" Total Percent\n",
"PoolQC 2907 0.997256\n",
"MiscFeature 2810 0.963979\n",
"Alley 2717 0.932075\n",
"Fence 2345 0.804460\n",
"FireplaceQu 1420 0.487136\n",
"GarageCond 159 0.054545\n",
"GarageFinish 159 0.054545\n",
"GarageQual 159 0.054545\n",
"GarageType 157 0.053859\n",
"BsmtExposure 82 0.028130\n",
"BsmtCond 82 0.028130\n",
"BsmtQual 81 0.027787\n",
"BsmtFinType2 80 0.027444\n",
"BsmtFinType1 79 0.027101\n",
"MasVnrType 24 0.008233\n",
"MSZoning 4 0.001372\n",
"Functional 2 0.000686\n",
"Utilities 2 0.000686\n",
"SaleType 1 0.000343\n",
"Exterior1st 1 0.000343\n",
"Electrical 1 0.000343\n",
"KitchenQual 1 0.000343\n",
"Exterior2nd 1 0.000343"
]
},
"execution_count": 44,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"# Shows the missing values again, but seeing as we have filled all numerical features, the categorical features are left\n",
"list_missing_data = count_missing_data(all_features)\n",
"print(\"Number of features with missing values: {}\".format(len(list_missing_data)))\n",
"print(\"Categorical features\")\n",
"list_missing_data"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"I've tried filling the missing values for the categorical features with the mode, but that didn't help the score. Thus, I simply dropped all categorical features that have missing values. "
]
},
{
"cell_type": "code",
"execution_count": 45,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"all_features = all_features.fillna(all_features.median())"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"**Things I tried, but didn't work...** \n",
"I've tried some different methods for filling in the missing values. So far, simply taking the median of numeric features and taking the median of categoricals gave me the best score. I will list here what I have tried and my view on why it didn't work. \n",
"\n",
"- Filling the missing values for numerical features, for example GarageYrBlt, with 0. \n",
" - Since the corresponding feature \"GarageCars\" has the value 0 for the missing values in GarageYrBlt, this led me to think that there is no Garage for that house and GarageYrBlt cannot be filled with the median. However, 0 is technically also not correct, since no garage was built in the year 0. Dropping would make sense, but it lowered my score compared to filling it with the median. I may be overfitting the data or there actually may be a garage, but it cannot hold any cars. \n",
"- Filling the missing values for categorical features, for example GarageType, with \"No Garage\"\n",
" - Similarly, some categorical features were missing and using the mode didn't help the score, which assumes there are not other categories than is listed in the features. Since it didn't help, there might be more categories, which is why I figured that the missing values for GarageType meant that there was no garage. So I filled it with \"No Garage\", which also didn't help my score. Overfitting might have happened, but it may also be that there actually is a Garage. "
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### Normality"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"**Why are we log transforming the features?** \n",
"The models that I'm going to use are mostly (linear) regressors which use loss minimizers (or optimizers) like minimizing the RMSE. These models often perform better when the variables have standard deviation and means that are not too far apart. Thus, when doing a log() or sqrt() usually helps to reduce the scale of the variables so that they are closer to each other. However, it should be noted that log transforming the features is not common practice and should only be done if the features are highly skewed. Moreover, linear regression models have an important assumption, namely that of normality in the dependent variables. \n",
"\n",
"There are several ways to approach this problem:\n",
"- Calculating Skewness/kurtosis\n",
"- Using Kolmogorov-Smirnof test\n",
"\n",
"The Kolmogorov-Smirnof test is used to compare the sample distribution to a normal distribution and then returns a p-value that will determine if the two distributions are similar. Kurtosis measure the taildness of the given distribution. Skeweness is a symmetry measure where distributions with 0 skew are of a normal shape. For skewness, we often look for values outside the range of -0.75 to 0.75 and for kurtosis outside the range of -3.0 to 3.0 which then doesn't satisfy the assumption for normality. \n",
"All values that are not normally distributed, we can do a non-linear transformation like a log-transformation: f(x) = log(x+1). We're doing +1 when there might be 0's in the column seeing as log(0) = -∞. \n",
"\n",
"For the purpose of this notebook, I'll be using the Skewedness to calculate the normality. In a later version, I might extend this by adding a kurtosis and Kolmogorov-Smirnof test. \n",
"\n",
"\n",
"**Skewness** \n",
"I use the *skewness* here to identify distribution that are not normal, but I also could have use *kurtosis*. \n",
"*Skewness* refers to how skewed the data is to the left or right of a normal distribution. \n",
"\n",
"| \n", " | Total | \n", "Percent | \n", "
|---|---|---|
| PoolQC | \n", "2907 | \n", "0.997256 | \n", "
| MiscFeature | \n", "2810 | \n", "0.963979 | \n", "
| Alley | \n", "2717 | \n", "0.932075 | \n", "
| Fence | \n", "2345 | \n", "0.804460 | \n", "
| FireplaceQu | \n", "1420 | \n", "0.487136 | \n", "
| GarageCond | \n", "159 | \n", "0.054545 | \n", "
| GarageFinish | \n", "159 | \n", "0.054545 | \n", "
| GarageQual | \n", "159 | \n", "0.054545 | \n", "
| GarageType | \n", "157 | \n", "0.053859 | \n", "
| BsmtExposure | \n", "82 | \n", "0.028130 | \n", "
| BsmtCond | \n", "82 | \n", "0.028130 | \n", "
| BsmtQual | \n", "81 | \n", "0.027787 | \n", "
| BsmtFinType2 | \n", "80 | \n", "0.027444 | \n", "
| BsmtFinType1 | \n", "79 | \n", "0.027101 | \n", "
| MasVnrType | \n", "24 | \n", "0.008233 | \n", "
| MSZoning | \n", "4 | \n", "0.001372 | \n", "
| Functional | \n", "2 | \n", "0.000686 | \n", "
| Utilities | \n", "2 | \n", "0.000686 | \n", "
| SaleType | \n", "1 | \n", "0.000343 | \n", "
| Exterior1st | \n", "1 | \n", "0.000343 | \n", "
| Electrical | \n", "1 | \n", "0.000343 | \n", "
| KitchenQual | \n", "1 | \n", "0.000343 | \n", "
| Exterior2nd | \n", "1 | \n", "0.000343 | \n", "
\n",
"\n", "**Kurtosis** \n", "*Kurtosis* is an indication of how the tails and the peak of a distribution differ from the normal distribution. This is also why it tells you not much about the shape of the distribution and is the reason for choosing *skewness* over *kurtosis*. \n", "
\n", "\n", "\n", "
| 
\n",
"\n", "**Distribution of the target variable** \n", "As an example, let's take the distribution of the target variable below." ] }, { "cell_type": "code", "execution_count": 46, "metadata": {}, "outputs": [ { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAAcIAAAHCCAMAAABLxjl3AAADAFBMVEUAAAABAQECAgIDAwMEBAQF\nBQUGBgYHBwcICAgJCQkKCgoLCwsMDAwNDQ0ODg4PDw8QEBARERESEhITExMUFBQVFRUWFhYXFxcY\nGBgZGRkaGhobGxscHBwdHR0eHh4fHx8gICAhISEiIiIjIyMkJCQlJSUmJiYnJycoKCgpKSkqKior\nKyssLCwtLS0uLi4vLy8wMDAxMTEyMjIzMzM0NDQ1NTU2NjY3Nzc4ODg5OTk6Ojo7Ozs8PDw9PT0+\nPj4/Pz9AQEBBQUFCQkJDQ0NERERFRUVGRkZHR0dISEhJSUlKSkpLS0tMTExNTU1OTk5PT09QUFBR\nUVFSUlJTU1NUVFRVVVVWVlZXV1dYWFhZWVlaWlpbW1tcXFxdXV1eXl5fX19gYGBhYWFiYmJjY2Nk\nZGRlZWVmZmZnZ2doaGhpaWlqampra2tsbGxtbW1ubm5vb29wcHBxcXFycnJzc3N0dHR1dXV2dnZ3\nd3d4eHh5eXl6enp7e3t8fHx9fX1+fn5/f3+AgICBgYGCgoKDg4OEhISFhYWGhoaHh4eIiIiJiYmK\nioqLi4uMjIyNjY2Ojo6Pj4+QkJCRkZGSkpKTk5OUlJSVlZWWlpaXl5eYmJiZmZmampqbm5ucnJyd\nnZ2enp6fn5+goKChoaGioqKjo6OkpKSlpaWmpqanp6eoqKipqamqqqqrq6usrKytra2urq6vr6+w\nsLCxsbGysrKzs7O0tLS1tbW2tra3t7e4uLi5ubm6urq7u7u8vLy9vb2+vr6/v7/AwMDBwcHCwsLD\nw8PExMTFxcXGxsbHx8fIyMjJycnKysrLy8vMzMzNzc3Ozs7Pz8/Q0NDR0dHS0tLT09PU1NTV1dXW\n1tbX19fY2NjZ2dna2trb29vc3Nzd3d3e3t7f39/g4ODh4eHi4uLj4+Pk5OTl5eXm5ubn5+fo6Ojp\n6enq6urr6+vs7Ozt7e3u7u7v7+/w8PDx8fHy8vLz8/P09PT19fX29vb39/f4+Pj5+fn6+vr7+/v8\n/Pz9/f3+/v7////isF19AAAapklEQVR4nO2daWAUVbqGv7CFhH0XRnFBYVwZBjfmjjruo2M3IGAg\nRmWTAURBHUEZNN7RkQCyKAGRQVEREBAZEFkGWYVgkObK4oLKMoAsYvadNH1uVXVXV3UqlXNS3UlV\nJ+/zo3O6+nz1Vp0ntXSH5hADUQ7ZvQEgXKAw6oHCqAcKox7nKCxKvfWihlf1Ti0MXZxA9GPFha+Q\nQptbF/sqUzZFqYrv2m+pT6C///WuVJffqbpxjMLMzn4TdN2ZkOXCCiWerEzZlGDZX33l9F9MNMWw\nGWYKA51rucJEossmTB3RjCgpZLmQwluSk8f2IIrxBBcuS07OqLhMUtgjOfmp7pLDReX0L6PQ/zpH\nIT+0KnCKQl8rapkl/fwhnhp49S8IKfy7vIb7iCZXIlFS+Kxc9jxRZ5/x5TIK/XAU2oNTFOYSdVEG\nMuWxxySVn939mwYdH9jIggpPP35t3GVPHpV7bHN1jO2UsDtYGlDIZhENZXcSZb56wVtqWVFyj6YX\n/mUnC12DQkAhO3cB0R61f3Dd9yqn2NIy65MUnklof+ljx6SuXaiRXF6XugY7B1biTb2jbevbJp2T\nmlL5ib9d1/j6T6pw6JyikLUm6vdpUeDJO/6LVJ31qkJPG2VB6z3S2Srw2jq1UlU4m+hpecyeI1KH\nPONqf+dFIWvwoypkI4g+CvTX1j2wLVHLTqVl1teVYq6UO7TbG6JQ7ezvVPhH/1q6ZioKeyhP1lbd\nyDlG4XR5Rxve+PiSbMbOt6Jmry0fTdQzoNDXg5ov3JNSn67zsiuozowNE+Opi1qpnkjvJ5otj1n9\npl0XB0bzKaK/LJkZT03y9GvwE1T4T6JXAv116w6cG0PX11XaxmGz/kB0Z4jC0NuZF4mu+Nd7vyN6\nQimPGz39ZqJ7qm7kHKPQt+Q6/y9v0/fZLwkJ8xjLr09XBoblC6IUptzy7PfWoXbfMzbviVHnApWB\n2xlpYONPymPWt4AFyorrUUfJ2FjpINCtIVAWVDiHaLi/v37dmkJtfbLC8YyVXEV0wFRhaRw1Oi1d\nGdpSnRy5fKF0ty1tR9WNnGMUSvy8bIJ81qkj3VeWbJ02oAPJR4MyLG8TXdCpU6eWRMuY9CtNV45Z\np7191N5UvK4M+T55oVL2LdHjUrvo+PE8/Rr86I/CVwOKdOvWFGrrkxUe8ZcuMlX4I1Gi/MIool1y\nufwe6SJqVXXD5iSFMofvkX/RP2ohjeU1DYIKXwhaeo2dHB0nN5qnqiUBhY1vle8ZpDFT7uuVsnVE\nEwKd9GvwU961ULduTaG2PlmhfOwvIZpkqnAj0Vj5hUlES+Ry+T774tqgcNdtt72sNNZK95U/1afL\nP85mzYIKpQvlGq1v/ooh7aSB/jrwVL2dUQiMmb9sr/S2XWqXnD1bGLoGmeAdaXvdHam2bk2htj71\nKJwqS+9C8VKzNKaco/BhueJJorRapTCDqMVhuSEdLm/OVy5c32kn0rXyHQdjq5OTD3+dnLxdGrkh\nRAsCpeYKC2LowhLGpFvK9bo1BHqq7wvHK29nlP76dS+Wj7RyFL4oSb+W6Bu5fZKxTRRQOEntVNqQ\nGv/CWF47ismuVQpZL6JmI2dM+ot0RvxpEdHl7y/4raawpDPFpn41owFdfO5rootnbZp/rXZfYq6Q\nDSa6a/nsOGpRqFtDoKf/05nRN5B8SAUU6tYtvb9wfe8zKox5cu6tRPcz1pfo+pkTWigKA539nSYQ\n/fa9D39PNJLVLoXZgRtSil3BfmmuXNwaUwd1WDY2Ul5ruYf5Hgv0G6x+pFKBwtOdlK4xn+rXEED7\njPRJ9TNS/bq/C761D1UoX6Wpg3TfulXpea1yIv1O/9a+8H/8K+mWXcsUstIFd3WKveCmv0lnJ7b/\nvuYXJx3rLd/4BS5ShxI7x3UZc0Lu9+EdHRu0vXmeejRVpJDlj72+0UW9/o+FrMGP+peKhz7R/lKh\nX3dqx9jLjQrrHryvdcdH5W1kn97Q+PIh2crtTKCz+unMzD+1bXnLlHOstikEFoHCqAcKox4ojHqg\nMOqBwqjHVGHB1EeTUgsYW+CSmMzYudlJwzdU55YBQUwVjh/9tWfYNMamTvN4PIcYmztsz7qeyof2\n7Jcxb1plquXK16fbEDrFltAZgh3fuL2gIoU/uyRtnp4lbOx65XlxwleMzZqotM98ZPk3Jt9yZU6x\nDaHZJZZLCyxXZp3j91Hwdlf2zEzhD894GfvRdZYN8n8iddAtDeGOR5Q2FApgv0IZX+pgX6k7JWnQ\n3CKWLv8Z8zv3eWlAcg4v9lkl33JlTpENodnFlksLLFdmlQh2LOUqLHjDnc5OuWYc2DZwOts4RFpy\n1JXL2IPd73nnLHAAp3kK9w8cLN29eHN8jO10FwWOQuXfDuFEKoADTqRp7rfUfxLIzrhOfu+W9ibt\nYf9TKORjv8KihI+Vn+njpL3Y3dtbkuBhbM4kZRkUCmC/wu29duySKC1ImLwvbch8xuaN+mlHn73K\na1AogP0Kl7kUctixl/oPXSRdAr2zk0Zu9L8GhQLYr7AioFAAKDQAhQagUAAoNAKFfKAwkqFQaAQK\n+UBhJEOh0IhxNDeTjmHmlVBowCkK/33VyiDDE8wrodCAYxReuynIGChkUCgWKgoUGoFCPlAoEioK\nFBqBQj5QKBIqChQagUI+UCgSKgoUGoFCPlAoEioKFBqBQj5QKBIqChQagUI+UCgSKgoUGoFCPlAo\nEioKFBqBQj5QKBIqChQagUI+UCgSKgoUGoFCPtWh8PQH+VbJMiz5SK+wj3llZk4EQ0XJsB6abT00\nV7BjDo5CPjX2KIRCAaDQABQagEIBoNAIFPKBQpFQUaDQCBTygUKRUFGg0AgU8oFCkVBRoNAIFPKB\nQpFQUaDQCBTygUKRUFGg0AgU8oFCkVBRoNAIFPKBQpFQUaDQCBTygUKRUFGg0AgU8oFCkVBRoNAI\nFPKBQpFQUaDQCBTygUKRUFGg0AgU8oFCkVBRoNAIFPKBQpFQUaJboTqvvTqfvW5eeygUwAEK1Xnt\n1fnsdfPaQ6EA9itU57VX57OP6Lz2uRPGBXkQCssQ6Xnt1fnsIzqv/a7GA4JcBYVliPS89up89tq8\n9pFQ2EHTlgiFZYigQmVee3U+e21e+7f+OfH9PKtkKY9bzBT2Ma/MyA4z1AoZOU4OzRab116dz16b\n1/6T995ZUGiVbOXxCzOFfc0rM3PDDLVCZp7l0hzLlRmioXli89qr89lHdF57nEgrItLz2qvz2Ud0\nXnsorIhIz2sfnM8+kvPaQ2FFRHxee3U++0jOaw+FFREVn5FCYUVAoVmoFaDQCBTygUKzUCtAoREo\n5AOFZqFWgEIjUMgHCs1CrQCFRqCQDxSahVoBCo1AIR8oNAu1AhQagUI+UGgWagUoNAKFfKDQLNQK\nUGgECvlAoVmoFaDQCBTyqTEKZ5COd6GwHJyu8EX3yiB3T4bCcnC8wr7a8vuhsDygUAAoNAKFfKDQ\nLNQKUGgECvlAoVmoFaDQCBTygUKzUCtAoREo5FMtChd5rZKnPH5ppvAhrecEvcIUrze7MMxQK2QV\nWS7Nt1yZWSzYscS6wtMLiqySrTxuN/2uvdbzBb3C14qKsvLCDLVCpvXQHOuh+YId83Ei5VNzT6RQ\nyKe2KuydGeQ5KOThRIX36v9GCIU8nKjwjtu19nVQyAMKBYBCI1DIBwrNQq0AhUagkA8UmoVaAQqN\nQCEfKDQLtQIUGoFCPlBoFmoFKDQChXyg0CzUClBoBAr5QKFZqBWg0AgU8oFCs1ArQKERKOQDhWah\nVoBCI1DIBwrNQq0AhUagkA8UmoVaAQqNQCEfKDQLtQIUGoFCPlBoFmqFaFd45nH5cYE8B540hOdm\nJw3fEHgBCvk4QWF+iqJw6jSPx3OIsbnD9qzruU95BQoFcIDChW6XonDseuVpccJXjM2aqLShUAAH\nKMw8vkxROGiP8vSgWxrCHY8obSgUwAEKGftcVljqTkkaNLdInRz9PGOHvk1fVGqVXOVxZ2UVppSW\nZheEGWqFrELLpXmWKzOLBDsWiyk85ZpxYNvA6WzjEOnJUVcuYyPcSe9mWiVDedxYWYX/m5n5a0aY\noVZwduhZMYXeHB9jO91FgaPQK7+AE6kAzjmRKpxxnfzeLe1N2sP+p1DIxzkK08dJe7G7t7ckwcPY\nnEnKC1AogHMUFiRM3pc2ZD5j80b9tKPPXuUFKBTAOQrZsZf6D10kXQK9s5NGbvS/AIUCOEKhKVAo\nABRCIR8oFAAKjUAhHyg0C7UCFBqBQj5QaBZqBSg0AoV8oNAs1ApQaAQK+UChWagVoNAIFPKBQrNQ\nK0ChESjkA4VmoVaILoVdZyo/HruPXwqFAlS3wrWjR9MNoyWGNGzIL4VCAapbYbI2t8Dv+aVQKEB1\nK1zSpw916SMz4gd+KRQKYMO18Ka3hDOgUADckUIhn7IKve8OuFOBXwqFAtigcLx6P8MvhUIBbFDY\ngTo/myzDL4VCAWxQ2JqOiGZAoQA2KOxPGaIZUCiADQpXtrln9WYZfikUCmCDQsLtjJHoUlhXhV8K\nhQLgrT0U8glD4ekP8q2SpTxurazCV/PzM3PCDLVChvXQbOuhuYIdc8ooTFDhK8RRKABuZ6CQT1mF\nT8gMatloJL8UCgWw7VrooX/wS6FQANsU5sdczi+FQgFsULhfJn0kNeWXQqEANt7OPMAvhUIBbFNY\n947/8kuhUAAbFJ5VENpkKBTAntsZX8Z5oVIoFMAOhQfub0yN/vyNQCkUCmCDwoMNlYth3I/8UigU\nwAaFven6FXtXdKe+/FIoFMAGhRc2yJIeMxpcxC+FQgFsUNi0pU969LVsxi+FQgFsUNiDXixhJROo\nB78UCgWwQeFHRE2vakq0lF8KhQLYoNA3Ub4ljUsRKIVCAWx5a3/6szmfnRYphUIB7FDo++E4O3FE\npBQKBbBBYW4vWsyW0SNF/FIoFMAGhc8RrWSriV7ml0KhADYovJS2SI8b6Ap+KRQKYIPC+LbKj9bx\n/FIoFMAGhb+jDwrZ+XnUjV8KhQLYoHAmUb2r2xLN4JdCoQA2KDz/XH35H148JfBXXygUwJa39sdX\npi4/IlIKhQI4+2sxUCgAFEIhH9F57dX57MOe1/7UIYn98sOhFVBYARGf116dzz7cee3Pxmj/VR9B\nYQVEel57dT77sOe1PxGrKXkeCisg0vPaq/PZhz2vfTgK735q9+6tabsVPKI7qFFbFapz+Qbms9fm\ntbdBYccm7YPELql0dC1XqM5nr81r//KIp+dnV57vwlB40V+19o1zKx2dYWFz/fyaZbm0GkIzKnUU\nerV57XdsWPVhceU5HCmF8ysdnWNhc/1k5lsuDSO0QLBjgZhCdT77sOe1D+dEGqLww0pH1/ITqTqf\nfdjz2kOhKBGfFF2dzz7cee2hUJSIK1Tnsw93XnsoFMWxn5FCoShQaAQKDUChAFAIhXygUAAo1AOF\nokChESg0AIUCQKGpwq7PbwiyqVQoGgoN2KqwefvOQWK3CUVDoQFbFTb7u9a+bLNQNBQagEIBoBAK\n+UChAFCoBwpFgUIjUGgACgWAQijkA4UCQKEeKBQFCo1AoQEoFAAKoZAPFAoAhXqgUBQoNAKFBqBQ\nACiEQj5QKAAU6oFCUaplXvuF5yrP0apQuEEoOtfC5vrJKrRcmme5MlM0tAgK+dRYhTiRCuDsEykU\nCgCFUMgHCgWAQj1QKAoUGoFCA1AoABRCIR8oFAAK9UChKM5SeGmLIM2qQGF8E239bU+ZbgUUGhBW\n6KO3F6q8WgUKG0wIrn5hk4OmmwGFBiqhcGVwuOdUhcJpuuVQWAmgUAAohEI+UCgAFEKhFaDQCBQa\ngEIBoBAK+UChAFAIhVaAQiNQaAAKBYBCKOQDhQJAIRRaAQqNQKEBKBQACqGQj6jCBS6JyYydm500\nfIN/ERQK4CCFU6d5PJ5DjM0dtmddz33KIigUwEEKx65XfhQnfMXYrIlKGwoFcJDCQXuUHwfdxYzt\neERpQ6EAzlFY6k5JGjS3SJ0c/XylJkUvqkaFjaZq/yBxdehm1N5J0RVOuWYc2DZwOts4RHpy1JXL\n2Msjnp6fLUZWNSqMubq7Sjc6FbIZGYKba+TXLMul1RCaIabQm+NjbKe7KHAUeuVljjyRxnwQbP6H\n8kI2o5afSBXOuE5+75b2Ju1h/1Mo5OMchenjpL3Y3dtbkuBhbM4kZRkUCuAchQUJk/elDZnP2LxR\nP+3os1dZBoUCOEchO/ZS/6GLpEugd3bSyI3+RVAogIMUlgMUCgCFUMgHCgWAQii0AhRCoQBOV7ie\nbr8ryCAoLAenK1xDLySrjG4KheXgfIWfBduLoLA8oFAAKIRCPlAoABRCoRWgEAoFgEIBoBAK+UCh\nAFAIhVaAQigUAAoFgEIo5AOFAkAhFFoBCqFQACgUAAqhkA8UCgCFUGgFKDRXeKAuafQV2QEodJjC\nHReuDDLuDpEdgEKnKeyodU+u3QpPLygUo8AJCgsLswObs0mv8E8iO5CZJ7inRnIsV2aIhubVjqNw\nfv1+/Xr383OHXmHtPgqjSeEbDZ4JchcUqkSVwnityzNQqAKFAkAhFPKBQgGgEAqtAIVQKAAUCgCF\nUMinyhTOHacBhRawX2GH+wcEgUILOEDhO8Fx2giFFoDC8BRuGqZjDxQ6WOFLPQ5p5GpbP67r40Gu\nmAGFDlbYv26TILGDtK0fN0DrczcUOllhv25ae1SStvVQyKAwABRCIR8ohEIzok/h4JvfDnIvFEaj\nwlva3RakKRRGpcK7tfZVUAiFAaAQCvlAYRAoLEPNUdjtDwOHqp94jzhSyUGKNoXZus+M29UYhe17\naB95d5ip7eF/RQYp2hT+MU770JhqjsIndLENtT2ss1tgkKJN4fWTtZ2tUyMVNknW2s1v7xckyewr\nxA5UWOG89rVLYf3+2ldtGqzaHeRb/ZA4UGGF89rXMoVv6mLbtg9SRz/2zlNY8bz2tVjhQl1ssW5I\nyle464buGrPLH+qqUljxvPZQqMQuXKqyZOKCYHvh5GBz6VNdpgR5YLA2gP/Vuiyd+C+t/bq2mqWf\nlFVbSYXavPZQaKJwBXULHmBX1NMOtovitHabG7XS4TqFz7bX+jToqLXrdtLasV+Fp1Cb136EO+nd\nzDLc0Eq7IlAbXfuCYLN1jLa4eT2t3aSB1o6L09qxjbR2vSa6C05zrR2jj9VdlExim+liG8dq7Ya6\n2AaNtXbdprrYFtzYC3RRreqUH9tIFxvf5JIgzfSxzcqPrbelzKiftXQUehk79G36otIyHEvXWKdr\nry3b3uZvr9EW71yvtbdv0NpbNmntz6WyLdv97f+k6aK+rCBKawdC07/ULU/TxX6hj92si/1i8w61\nvV4Xu1Ygds22YPNL3Yik/UcX+7nW3rxFa6/aqrX1e7un7KgXV06hpXntjVj/j5hyivl9Ih5aoz4j\ntTSvvREo5FNl7wutzGtvBAr5VJlCK/PaG4FCPvb/salCoJAPFEYyFAqNQCEfKIxkKBQagUI+UBjJ\nUCg0AoV8oDCSoVBoBAr5QGEkQ6HQCBTyqRaFLyy3yiLLlQuX2BD6ofXQxZYrFywV7PixdYXZ42zA\nNciG0PsftyH03uGiPV85b1WhLYxZbUPo0C02hA4o+29kOESNwlc22RD6QroNoaP3Va5/1CgEZkBh\n1ONQhQVTH01KLee+XP1mxwKXxOTI5656qpyFVRu6elT/N8t5wyQe6lCF40d/7Rk2zbhc/WbH1Gke\nj+dQxGNP9SlPYZWGrkncvn3g/HBCnanwZ5e00Z6ehjfWwW92jF1fFbG+50eWo7BKQ31DpJverZPC\nCXWmwh+e8TL2o+ssy5+elDirSF6Um8F03+wYtKcqYlc/p5xIqzP0sLvI37Ae6kyFMr7UwT7fhFcO\nep5IkZ+umcO0b3aUulOSBs0tinDiqcTjqyWF1RrqeXjDyEekq34YoY5VWPCGO5399JB0Lj34oI8F\ndkz9Zscp14wD2wZOj2yib/xyJius1tBNPcd4vhr+ejihTlW4f+Bg6Vq+yZ2QkPCQ61cW+rvp9eZI\n+7rTHdkjYu3TXkVhtYZud/0sHYquojBCHaowzf2WvNUbhp+RKfUkJvZ9MDFxlf6bHeyM62REM990\n9+zpdvX0VGvo/l6Sol9cP4cR6kyFRQkfKz+/ezCbsT0pvpKzZ5dNP3u2QP1mR/o4aQd39/ZGNPTX\no0ePvj/8aGG1hhb0OSodcr1Kwwh1psLtvXbskij1Pfvs3u2D35UXKacX9ZsdBQmT96UNmR/5YOV2\nplpDU0d9vWvI3HBCnalwmUshh2VPSnxknvInUP+Oqd/sOPZS/6GLIns8KMgKqze0dPajg94rDSfU\nmQpBJYDCqAcKox4ojHqgMOqBwqinJio8N+/m1k1vHJenX9aKVoR2uokkYq9J9Zr2iBZqosKRpHDJ\nr7pl5SuUmGzaI1qogQrPEE04XrQinl7ULSxH4aN5ud/cSG3VBXl5pdW0gRGmBircSiQffyl3/oOx\nwwN+E3vZc4V+hSXjr4y77h2fv9dNJP85ZzFRBqtLn4/urPTwzereqNNY6Qwc0tXh1ECFx+rQH985\noTSLu1BMO6IxfoUPUP1r69Cr/l5+hUuICiSF91IrpcdYorZEvX2hXR1ODVTIptSTrnHt+v3bx76g\n2BPsNbpaUbiF4n5mn1BsptJJOZF+exNdwSSFbWaukXucqE8fsM1EP4R2dTg1USE7Miuxs2TxBVaS\ndeLziZfTxYrCV+iSceOeJdqh9FFvZ96XFb7NlB4rqZ50zl25+HhoV4dTIxXKfPMg1T/n/Ws9iv9t\nQOHQgLT5yuuKwjrXyE/qkvwnAalHqny8yoR2dTg1UOHyq2+Uv/GzXbpTWUgtPi/9MKBwArl0vfzX\nQoW6tJkpPZZTO6nys2UnQrs6nBqo8BuiESdL0l10CZtAnUuK/hxQuIpanWI7LrnslNKrPIVH6tA8\ntoHocGhXh1MDFbJnAqfBVexjoiZNY6iDIsh3CzWWbjOf93cqTyF7kqgNURIL7epwaqLC80tuax97\nWd90qZXcvnXiRqLtiqD80Z3iu4W+L1TQFHqndo3rlCzd0oR0dTg1UWEtAwqjHiiMeqAw6oHCqOf/\nAapkuUiNBYWVAAAAAElFTkSuQmCC\n" }, "metadata": {}, "output_type": "display_data" } ], "source": [ "%%R -w 450 -h 450 \n", "ggplot(train, aes(x=SalePrice)) + \n", " geom_histogram(color = 'black', fill = 'grey', bins = 30) + \n", " theme_bw() +\n", " labs(x = 'Sale Price', title =\"Sales Price Distribution\") +\n", " theme(legend.position=\"none\", axis.text=element_text(size=14), axis.title=element_text(size=16,face=\"bold\"),\n", " plot.title = element_text(lineheight=1, face=\"bold\", size=20, hjust = 0.5))" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Clearly, the distribution is skewed to the right, which means that we have to transform the current distribution to one that has a normal distribution. Below we will simply do that by computing the log(1+x) of all data points within the target variable. " ] }, { "cell_type": "code", "execution_count": 47, "metadata": { "collapsed": true }, "outputs": [], "source": [ "# #log transform the target:\n", "train[\"SalePrice\"] = np.log1p(train[\"SalePrice\"])" ] }, { "cell_type": "code", "execution_count": 48, "metadata": {}, "outputs": [ { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAAcIAAAHCCAMAAABLxjl3AAAC/VBMVEUAAAABAQECAgIDAwMEBAQF\nBQUGBgYHBwcICAgJCQkKCgoLCwsMDAwNDQ0ODg4PDw8QEBARERESEhITExMUFBQVFRUWFhYXFxcY\nGBgZGRkaGhobGxscHBwdHR0eHh4fHx8gICAhISEiIiIjIyMkJCQlJSUmJiYnJycoKCgpKSkqKior\nKyssLCwtLS0uLi4vLy8wMDAxMTEyMjIzMzM0NDQ1NTU2NjY3Nzc4ODg5OTk6Ojo7Ozs8PDw9PT0+\nPj4/Pz9AQEBBQUFCQkJDQ0NERERFRUVGRkZHR0dISEhJSUlKSkpLS0tMTExNTU1OTk5PT09QUFBR\nUVFSUlJTU1NUVFRVVVVWVlZXV1dYWFhZWVlaWlpbW1tcXFxdXV1eXl5fX19gYGBhYWFiYmJjY2Nk\nZGRlZWVmZmZnZ2doaGhpaWlqampra2tsbGxtbW1ubm5vb29wcHBxcXFycnJzc3N0dHR1dXV2dnZ3\nd3d4eHh5eXl6enp7e3t8fHx9fX1+fn5/f3+AgICBgYGCgoKDg4OEhISFhYWGhoaHh4eIiIiJiYmK\nioqLi4uMjIyNjY2Ojo6Pj4+QkJCRkZGSkpKTk5OUlJSVlZWWlpaXl5eYmJiZmZmampqbm5ucnJyd\nnZ2enp6fn5+goKChoaGioqKjo6OkpKSlpaWmpqanp6eoqKipqamqqqqrq6usrKytra2urq6vr6+w\nsLCxsbGysrKzs7O0tLS1tbW2tra3t7e4uLi5ubm7u7u8vLy9vb2+vr6/v7/AwMDBwcHCwsLDw8PE\nxMTFxcXGxsbHx8fIyMjJycnKysrLy8vMzMzNzc3Ozs7Pz8/Q0NDR0dHS0tLT09PU1NTV1dXW1tbX\n19fY2NjZ2dna2trb29vc3Nzd3d3e3t7f39/g4ODh4eHi4uLj4+Pk5OTl5eXm5ubn5+fo6Ojp6enq\n6urr6+vs7Ozt7e3u7u7v7+/w8PDx8fHy8vLz8/P09PT19fX29vb39/f4+Pj5+fn6+vr7+/v8/Pz9\n/f3+/v7///93UAGYAAAaZklEQVR4nO3dd3wUdf7H8U8IEAgdQpOmBEHkBBEb/FTup/5UThPkkIti\nAAUpogg2ODw0/iwQQIqUCAiHdGly3FmQ4lngDiGcoBygFA+QIiQkJARS2O/jpny/u9/dzGb3u0uy\nM8n79Ud2Mjvfme/Ok92dJRRiyOFRpCeAwg2Ejg+Ejg+Eji9swouz7mpR7fpes/K8VycR/VTywDfJ\nqOFdK1wqwyYbo2I79VnlCmJ78/5OFB14o5J7j2hooG2M7id3NYIawPu0Q5Wq+1UGuAuXMLMtn2/H\n017rgybUGqEybLJ72FCXxfYriCYXm4Y/Qr6xHQgv1NEG7FMY4Clcwr5ErcdNeVqbQLLX+qAI70xJ\nGd2VKCrdvXJ1SkpGycM0wq4pKc910R7xcovtfQjN+wMQBj6oAuGSlJSUx4iu1W7eDmqA2fdE3Q7l\nKwzwFCahqwHVP6fd/hhLVYvkO4Ii/JO+hx5EkxSOqBG+qA/7I1FbV/G7fQjNAhAGU9CEen8jejDo\njc12qOzfuzAJzxO1M05k6oABGuUn/9esasuHNjM34anBN1RvPeJnfYuvElrGxCftdA/lhGw20VPs\nHqLMt5q8J4ZdTOlau/mD/2DeezDihKygCdEusb173+bLWKHP/jTC00lNrxlwVNu0nfn6Fk2d3Bvz\nnRTNurtRXPeJBdqiNvz4Sx1r3vyR+7BehNKWLHfUdU36n2un7c6Tm1BMQzoxXrv2mfdG30kYg7tQ\n9Nm+ja8ZeOp03xb1enxfzCDcF9I4oj5/u8i/WWC+B1T6XBCmNzRWxO3SXq34fRvESEGYRvS8Pt2X\nicQpz+hgbrzcaw9mgpA9TfQh396z7ycaEdWPL/TZXyeKaq9v0Hi3F6HY2Nwo7w5zL50yjbPX1fjm\nM3FYmVDeMusmfen2Fn4JjWlIJ0betTTvZkS14r/xnYQxuAtFGRccN8XrX5ucu9KE0/TdVrt18Mos\nxi43oDrj144k6skJXV2p7rJdqVWoYxG7lipN3zghltr5ELp+R5SmT7dK7U4r+Nl8TjsFK2fGUq0c\neQ8+hG8Tvcm3l/bNXxu999dJm+OQ2d2I7vEi9L6ceVV7A3v/gxuJnjGGVx857Xai+6wI5S215fbv\njoomv4T6NOQTI+9amjd/IfWZhPEYtLf9puOHaYeIeelPDfVft1eY0LWyo/nrpvYi9mtS0nztlaUK\nteen5WuiVGZc8nxfVIkaa9fM8595tsBDqF/OaCc29oQ+3UcuiLN5qTK11MRGa79SpT34Es4hGmZu\nL+/bQ+jZn074CmP51xP94JewsDrVOKW9MzSiStn68GXa1bY2DwtCecuCmlT7jHGvP0J9GvKJkXYt\nz9sk9J2E8Rg0wm8YSzTOxFSiYhdJV+Cj/S+rx+kvDZW068r8L6c+dhXpv6qM0zJXe97Hx8fXJ1rN\ntF931H7UBs/HR8+HineMB7aHibP5b6LB2vLFY8dy5D34EGrPwrc4kbRvD6FnfzrhEXPocr+EPxH1\n1e94luhbfbj+GakFNRCzlQjlLX8k6q8tFtTwS2hMQzox8q6leZuEvpMwBmuE2i977Tm8hbF1+kuP\nT1fod2cO36f/Qv+wnjan31R1E451K41nJ0ZW1xfqzhJDOGHNuz4yH6txXW8M20A0jm8k78HM6r1Q\n2reH0LM/nVB/7q8kmuiXcDPRaP2OiUQr9eH6W04rS0J5yw38Db29X0JjGtKJkXctzdsk9J2EMVgj\nZAbh16VC+G337q8bC59p15UHq1CbNVmsjptQe6P81LNt7rpBjbUJf8e/FZcz4rEab9PGsN3m6co/\ncybPew967ivSptIVqWffHkLP/sSzcIqO3o5itcXCKItn4eP6iBFE20omlLfcpV9Oa28ndf0S6vuR\nT4z3rt3zdj8Li02itAkziOod1he0p8uMhcbL9T7PC+ln5gE/Tkk5/F1KivaCXjiIaAkf6p/wQhQ1\n1z7lapdjn0t74FuKz4WvGB9njO3lfa/Qn2kWhK9q6DcQ7dWXTzC2hTjhRLFRYTWq+StjOY0pKqtk\nQnlL7UNVXLbx/C6JUD4x0q7lefP3QotJlDYhe5iozvDpEx/UXhEPLidqs2jJdR7C/LYUM2vH9KrU\nquA7olaztyy8wXNd4p+QDSS6d21adaqXJ+3BQ9g1JWXkLWRcm5mE0r616/SE/a7ihFEj5t1F9DvG\nHiG6eea4esY55xubG40juu6DpdpnhOG+TxVO2CXF6BOvLR/VLvbnjI0pmVA+MTKhNG9+RWoxiVIn\nzOIXpBSzjv1a13hzq0lXidOyuYZxX/1dzDWAbzdQ/JZKCYSnjE9AFPU3eQ8eQt4I8Xuk8r736beF\nxQn1NyO6Srv++9LY8gbjhZRvzD8X/o+5k85Zfgh5z3hteVB/HaSm9UsklE+MtGt53pzQYhKlTsgK\nl9wbH9Pktpe0Vyf2fY+6rZKP9tIv/Pib1KG+bau3G3Vc327p3S2rNrp9vng2lUTIckffXKPFw/9i\nXnswEz+p+MNHnp9UyPue1TKmTXHC6AM94lr2P2Gc31tqthmUZVzO8I3F787M/G2j+ndOLmCBCOUt\n2S99W8X1PNa2REL5xMi7luYtfoOt+CRKn7BiV3TkiP6rq6im5zcByjwQhlVRLYraxIr+SPR6xOYA\nwvCaob20tmxAdOOFiE0BhGH2dULrmMa3Tc+N3AxA6PhA6PhA6PhA6PhA6PhA6PhCIfx11Azrpk/3\nc0cQTZ8c+tgZU8IY+8600Me+OzWMsZNCH2s+4Hf/90KohKeL/fENXn6BnzuCqKDYH+tRKJxPZdkX\nA2/jr8vhjD0b+ljzARd1MW5ACEI5EKqMBaEcCBUDoQiEciBUCYRyIJQDocpYEMqB0LrT+p+KZ0sS\ntCZppzgtedhGfgcIeTYnzE01CKdMTU9PP8TYvCG7NvQ0/4IACEX2JlyWmGAQjv7c+PZS0g7GZk8w\nlkEosjdh5rHVBuGT5h/CPZB4ibGt/YxlEIrsTcjYJp2wMDE1+cl5F9l2/S9N7Uu8zNhHHyxYkmdd\nTo6fO4IoJyP0sXlZYYzNzA597IXzYYw9G/pY8wHnBEd4MmH6D189MY1tHqR983PCecbee3vCohzr\nsrP93BFE2Rmhj805F8bYjKzQx54PZ+zZ0MeaDzgrOMKibBdj/0i8yJ+Fxl+XduYLacYhkfe/kFPu\nX0iNTiec2J+Yz9i2x81vHUnYtXots9hmXuvLPeH2MRrdzl5F+UnpjM2ZaNzhTMIuk7eYLWrotb7c\nE15ImrRn26CFjM1/9uDW3ruNO0AocgQhO/rao08t194Ci9KSh2827wChyO6EfgOhCIRyIFQJhHIg\nlAOhylgQyoFQMRCKQCgHQpVAKAdCORCqjAWhHAgVA6EIhHIgVAmEciCUA6HKWBDKgVAxEIpAKAdC\nlUAoB0I5EKqMBaEcCBUDoQiEciBUCYRyIJQDocpYEMqBUDEQikAoB0KVQCgHQjnnEk4ZIpqiclwQ\nykWWsM3Dg80ebqNyXBDKRZgwjd+ZBsKQA6FKIJQDoRwIVcaCUA6EioFQBEI5EKoEQjkQyoFQZSwI\n5UCoGAhFIJQDoUoglAOhHAhVxoJQDoSKgVAEQjkQqgRCORDKgVBlLAjlQKgYCEXOJVzhsu5Svp87\ngij/XOhjXblBbCMReq3PztO+SIQqxy3KU9naZ+zZ0MeaD7gwdMJTjvwPfzq7CeO81hv/4U+8mzBe\n5bh2/w9//D8L8ULKc+4LKQh5IJQDoUoglAOhHAhVxoJQDoSKgVAEQjkQqgRCORDKgVBlLAjlQKgY\nCEUglCslwqNzRa1A6MlJhFObdOdFg9CTkwinPMBttlTz/LApU1TIQOidMwinRpFoLAOhd84gTK0r\nVg0YyUDoHQhVxoJQDoSKgVAEQjkQqgRCEIKQB0I5EKoEQhCCkAdCORCqBEIQgpAHQjkQqgRCEIKQ\nB0I5EKoEQhCCkAdCORCqBEIQgpAHQjkQqgRCEIKQB0I5EKoEQt7jj+3cufPLbdqXlm7CljtFFwIf\nF4RykSC8sUZTXpQgHFtZrKrxRuDjglAuEoSdeoulyoJwdDOxqvdrgY8LQjkQqgRCEIKQB0I5EKoE\nQhCCkAdCORCqFCTh6cH614K05GEbpVsGQk82J8xNNQjnDdm1oecezy0DoSd7Ey5LTNAJLyXtYGz2\nBPetHghF9ibMPLZaJzyQeImxrf3ct3ogFNmbkLFNOuH2vtqXfYmXxS1ju7dvWlZgXV6enzuCKC8z\n9LEF5/3eMyl0wnGBj5ufG/qcL50Nfaz5gC8GR7h5kPbl54Tz4paxl5OHLDxnXWamnzuCKPNs6GPP\nZfi9563QCUeHdeDAhf2Azyo9C4vErX4HXkhFjngh3Z+Yz9i2x923eiAUOYIwPymdsTkT3bd6IBQ5\ngpDNf/bg1t67PbcMhJ6cQViUljx8s3TLQOjJ7oR+A6EIhHIgVAmEIAQhD4RyIFQJhCAEIQ+EciBU\nCYQgBCEPhHIgVAmEIAQhD4RyIFQJhCAEIQ+EciBUCYQgBCEPhHIgVAmEIAQhD4RyIFQJhCCsEITt\nSVTluL+9g1DOdoRtHl3Pq7Pf395BKGc/wgFiqS4IgwqEKoEQhCDkgVAOhCqBsHwSdppp3AzoEXg8\nCEU2Ivxs5Ei6ZaTWoGrVAo8HochGhCnu34mgmwKPB6HIRoQre/emdr31nv4x8HgQimxEqHXbe0GP\nB6HIXoQKgVBkL8KiPz92j1Hg8SAU2YvwFXE9E3g8CEX2IryK2r6Yohd4vP0J8xLu5bWtQIRxdCTY\n8fYnPBmdwmtXgQgfpYxgxzuAsLI4/d0qEOH6hvd9/IVe4PEgFNmLkMrT5UzFJIwWBR4PQpG9CBUC\noQiEciBUyZIwSRR4PAhF9iLE5YzjCZ/Re7J+jeGBx4NQZC9Cs3QK4r8CB6HIjoS5UW0Cjz+1ONe6\n7PN+7gii8xmhj8095/P9oStL+C9/x83JDn3OOWdDH2s+4Gwfwu/1tg+n2oEJ8SwU2etZKK5mHgo8\n3raE47rwOlZgwui7/xN4vG0J7+4/2eyFCkl4xig/mPH2JUzhJ31+hSTUcmVcDmo8CEU2I/zhdzWp\nxgN7gxhf0QjrbDzEy/Q5rr0ID1Qz3gyr/xR4fEUjrBRby6x6N5/j2ouwF928bve6LvRI4PEVjTBq\nEV+Y3MXnuPYibF5VP4cZVVsEHg9Ckb0Ia9d3aV9d9esEHg9Ckb0Iu9Kr+Sx/HHUNPB6EInsRfkhU\n+/raRKsCjwehyF6Ergn6JWn11CDGg1BkL0LGTn0y55NTwYwHochmhK4fj7HjR4IZD0KRvQjPP0wr\n2GrqF8SkQCiyF+HLROvZx0SvBx4PQpG9CK+hv2tfN9K1gceDUGQvwthGxk1cbODxIBTZi/BGWpzH\nLs+nzoHHV1jCSe13ikw7exHOJKrcoRHR9MDjKyzhsCpNebEzjAPai/Dyy1X0P3jxXBA/9a2whIPb\niVUPTDEOaC9Cxo6tn7X2SDDjQWhXwqADIQitAqHyAwYhCC0CoUIg9AqEioEQhFaBUPkBgxCEFoFQ\nIRB6BULFQAhCq0Co/IBBCEKLQKgQCL0CoWIgBKFVIFR+wCAEoUUgVAiEXoFQMRCC0CoQKj9gEILQ\nIhAqBEKvQKgYCEFoFQiVHzAIQWgRCBUqO8IlCVqTtFOcljxso7kKhM4inDI1PT39EGPzhuza0HOP\nsQqEziIc/blxcylpB2OzJxjLINxyV/9VeiuXrfo01DmXHeGTu4ybA4mXGNvaz1gG4ZZmzcU/xh91\nOsQ5lxlhYWJq8pPzLrLtfbVv9iUaf40bhFuaPe0eeTLEOZcZ4cmE6T989cQ0tnmQ9s3PCecZ+32X\n+xacsWd3RoJwb8Qe7qngCIuyXYz9I/EifxYWab8Asg+vcFl3Kd/PHUGUfy70sa5c8yYiz8ITIc65\n6Gy4D7hQ4XPh6YQT+xPzGdv2uPktXkid9EK6fYxGt7NXUX5SOmNzJhrrQOgowgtJk/ZsG7SQsfnP\nHtzae7exDoSOImRHX3v0qeXaW2BRWvLwzeYqEDqL0CIQgtAqECo/YBCC0CIQKgRCr0CoGAglwujJ\nc3nfqs0ZhF5FkJA68h9ZtL5Hbc4g9CqShGv4QsrdanMGoVcgVAyEILQKhMoPGIQgtAiECoHQKxAq\nBkIQWgVC5QcMQhBaBEKFQOgVCBUDIQitAqHyAwYhCC0CoUIg9AqEioEQhFaBUPkBg/AKE77cJpU3\n6Zdg5gxCr+xA2CfuIV6TBcHMGYRe2YLwJrHqjvnBzBmEXoFQMRCC0CoQKj9gEILQIhCCMNRAqBgI\nQWgVCJUfMAhBaBEIQRhqIFQskoSbxY8DUmeKVT+npr5hrooHYZBFkrBXJ/7jgB6Ux1fNbyJ+RFAF\nhEEWUcKX+Hn6zEN4hzh3tUAYZCAEoVUgVAmEIAQhCC1SJ3xxjFkPECpnC8L19IfHzK4BoXI2IfyM\nL90PQuVACEKrQKhSuISnFudal33ezx1BdD4jqM0SHUDYIu4aXscs/48k52zoJyv3nP4lG8/C0iJs\n0G+O2Uz3Bx+L8EJqZ8Lin10tAiEIQQhCq0AIwsCB0AiEIAQhCC0CoUogBCEIQWgRCFUCIQjLnPDr\nBvV4VUGo5zzC1dcv49UDoZ4DCTsXOz3OIvz30CG8YYf5KhA6i/D9awbzrlrNV4HQYYR3ijs7gxCE\neiAEIQhBaBEIVQIhCEEIQotAqBIIQQhCEFpUHgk/JncgdCbhelrIf8ByJwidSuieIwhB6B0IQQhC\nEFoEQpVAGEHCuu7PGY1DP1kgjCRhrTHrzRZVDf1kgTCihGKOa0AIQhCC0CIQgtBfIPQJhCAEIQgt\n+qlXH9FXxe4EoU+2JPy00Qu8Du8UuxOEPtmTsK14kHc8Mpf34hy+MLQcEq6uLB7m3L3KZ8vmhM3j\nu/OoK19oXg4J50eJhxk/UPlsRZ7QlSnKFas8hM2GFT895ZDw/Spi1bAnlQkiTzjO/dv1UUf5KhCq\nFHnCUf3F9Ovu46tAqFIpEp578F7RyhL2AEJrwlfdZ29oyQSlSLg/NoXXfWQJewChNWGXZH72nmlY\nMkFIhAVpycM2movFCf9uXhzPTnvDfXoGBEdY4//5hfWIikvYr5v4dNFKzHGRm/Cw+6PHgkthEs4b\nsmtDzz1+CG/pyK+PO6oSRt3KR7auuITdmoiPF9HFCV9rLu6s/m14hJeSdjA2e4Ifwpsn8UNPVCZ0\nn56KS9i1+BwXxYlPXaPdc2y2PTzCA4na03hrPxDyOZYy4dQo98euK0a4va/2ZV/iZRCacyxlQs8c\nO14xws2DtC8/J5xn7PWnn1+Y5dNN13Yxu7YyX+hSv/lveS278YVuLcSq5k3FZtSBLzSLFatimrvv\n7MgXGtcSq6q04gudqDNfalBX3Bndmi9cHyVW1W0glioVn2PNRmIp6jq+cHWMWBVrtznGfOE54RnG\nl5CehUXaq+nGvy695NM/zb+/s2TJ0rfEP5Y2e4ZYGr+YLyweL1bNmC2W3lrKF/78hlg1ba5YelMs\nLJgklt55v9id77+zbJG5NGlBsTvnTBNLEz7gCz5zXKhP7+0lwczxgwli1VQ+x8XBz3GZzxyXvhrc\nHGeKpVW5nhOerX+5oEa4PzGfsW2PG8uR/McsrcsNvInfsi+GPvZyOGPL+k9z5yelMzZnorEMQpGj\nCNn8Zw9u7b3bWAShyFmERWnJwzebiyAUOYtQCoQiEMqBUCUQyoFQDoQqY0EoB0LFQCgCoRwIVQKh\nHAjlQKgyFoRyIFQMhCLnEo5da93KVX7uCKJVS0Ifu3Z5GGOXfRj62DXhjF0c+ljzAa8JnTBrTCk0\n9P7S2GsQJT4RmeM+f1fYu3jzcqiEpdI/kyN04Jf+Epnjnul6hXZkG8J/PR+hA4/fGJnjZvS+Qjuy\nDSEKNRA6PlsQnh7sfVvGB74wpX/yrAtlf9yTKUmDVrrC3pkdCHNTB3vdlvWBXxn5XfqQqWV+XNdz\nKXu/7BP+O7ENCJclJgyWb8v6wL8kHGIsvWd+WR/3SEIWY2lvhr07GxBmHls9WL4t6wP/+EIRYz8l\nnCnr4x5I1b4sfD3s3dmAkLFNg71vy/zAzDVrYPhvSurHLTj81Kdh7wyEehfeTdxe8palc9z+Ca+W\nj8uZiBN+/8TAPZE4Ljv53fPvhr0zEDK2LfG9MH7iEPJxT/9H+7KnT1G4OwMhu5i0powPax5303Dt\nRXT7I5fD3RkI2TcPb/1Wq7Csj/trrz//9O1QvJBegQOvTjDKLuvjsh1jkwYuCP8l3BaEKJxA6PhA\n6PhA6PhA6PhA6PjKI2HB/Nvjat86Jkde14DWeW90m/4vK8X8ZlaR3y2cUnkkHG7+y1dXy3/W1ppQ\na5LfLZxSOSQ8TTTu2MV1sfSqtNKCsH/O+b23UiOxIienLH9/5gpWDgm/JNKff6n3vMHY4ceaxbR+\nOc8kzH+lffWOC/hPd24j/Z8kW0GUwaJp08i2xhau2V1qxI/WXoG9NrV55ZDwaCW6Y8FxY/FSO4pq\nTDTKJHyIqtxQid4ytzIJVxJd0AjvpwbGFqOJGhH1cnlvavPKISGbXFn/n3L7/MXFvqaY42w8dTAI\n/07Vf2EfUUymsZHxQvrv2+haphE2nPmpvsXxKrSYfUH0o/emNq88ErIjs/u21RTHsvxzxzdNaEOt\nDMI36eoxY14k2mpsIy5nFumEc5mxxXqqrL3mrl9xzHtTm1cuCfX2/p6qFBQNrUyx13HCpzjaQuN+\ng7DSb/Rvokn/Z620LWbpz1c9701tXjkkXNvhVv3HqN9oVyrLqN6mwqWccBwlSFuZ74VG0fQFM7ZY\nS421kZ+sPu69qc0rh4R7iZ4+kb89ga5m46ht/sUHOOFfqcFJtvXq1ieNrawIj1Si+Wwj0WHvTW1e\nOSRkL/CXwb+yNUS1akfRVQaQ606qqV1m/tHcyIqQjSBqSJTMvDe1eeWR8PLK7k1jWj+yXVtKaRrX\ndzPRNwZQ7sj42M7enwuNPIRFUzpVj0/RLmm8NrV55ZGwggVCxwdCxwdCxwdCx/dfuege+nMSREkA\nAAAASUVORK5CYII=\n" }, "metadata": {}, "output_type": "display_data" } ], "source": [ "%%R -w 450 -h 450 -i train\n", "ggplot(train, aes(x=SalePrice)) + \n", " geom_histogram(color = 'black', fill = 'grey', binwidth=0.08, bins = 30) + \n", " theme_bw() +\n", " labs(x = 'Sale Price', title =\"Sales Price Distribution Log Transform\") +\n", " theme(legend.position=\"none\", axis.text=element_text(size=14), axis.title=element_text(size=16,face=\"bold\"),\n", " plot.title = element_text(lineheight=1, face=\"bold\", size=20, hjust = 0.5))" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "This already looks a lot better! Next, it would be wise to do the same for the independent variables that have a high skewedness." ] }, { "cell_type": "code", "execution_count": 49, "metadata": { "collapsed": true }, "outputs": [], "source": [ "# # Selecting only the numeric features\n", "numeric_feature_names = all_features.dtypes[all_features.dtypes != \"object\"].index\n", "\n", "# # Calculates the skewedness of the features and then gets the features with a skewedness above a certain threshold\n", "skewed_features = train[numeric_feature_names].apply(lambda x: skew(x.dropna()))\n", "skewed_features = skewed_features[skewed_features > 0.65].index\n", "\n", "all_features[skewed_features] = np.log1p(all_features[skewed_features])" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Homoscedasticity (homogeneity of variance) \n", "Homoscedasticity is an assumption that the standard deviations of the error terms are constant and do not depend on the x-value. Basically, this means that each probability distribution for y has the same standard deviation regardless of the x-value. Thus, if all dependent variables exhibit the same levels of variance across predictor variables, then we have homoscedasticity and the assumption is not violated. We want the error term to be the same across all values of the independent variables. \n", "\n", "What may be easier to remember is that homoscedasticity means \"same variance\". It refers to situations in which the error term (i.e., noise) is the same across all values of the independent variables. In contrast, heteroscedasticity (which violates homoscedasticity) refers to situations in which the size of the error term is different across values of the independent variables. \n", "\n", "When we started plotting outliers, we saw that the graph displayed a cone. The shape of a cone violates the assumption of homoscedasticity since the error terms were not constant and seemed to increase with an increase of x. What we would like to see is the same variance at every x. Luckily, we've already normalized the data (which often also removes the violation of the assumption of homoscedasticity). Let's check that below. " ] }, { "cell_type": "code", "execution_count": 50, "metadata": {}, "outputs": [ { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAAcIAAAHCCAMAAABLxjl3AAADAFBMVEUAAAABAQECAgIDAwMEBAQF\nBQUGBgYHBwcICAgJCQkKCgoLCwsMDAwNDQ0ODg4PDw8QEBARERESEhITExMUFBQVFRUWFhYXFxcY\nGBgZGRkaGhobGxscHBwdHR0eHh4fHx8gICAhISEiIiIjIyMkJCQlJSUmJiYnJycoKCgpKSkqKior\nKyssLCwtLS0uLi4vLy8wMDAxMTEyMjIzMzM0NDQ1NTU2NjY3Nzc4ODg5OTk6Ojo7Ozs8PDw9PT0+\nPj4/Pz9AQEBBQUFCQkJDQ0NERERFRUVGRkZHR0dISEhJSUlKSkpLS0tMTExNTU1OTk5PT09QUFBR\nUVFSUlJTU1NUVFRVVVVWVlZXV1dYWFhZWVlaWlpbW1tcXFxdXV1eXl5fX19gYGBhYWFiYmJjY2Nk\nZGRlZWVmZmZnZ2doaGhpaWlqampra2tsbGxtbW1ubm5vb29wcHBxcXFycnJzc3N0dHR1dXV2dnZ3\nd3d4eHh5eXl6enp7e3t8fHx9fX1+fn5/f3+AgICBgYGCgoKDg4OEhISFhYWGhoaHh4eIiIiJiYmK\nioqLi4uMjIyNjY2Ojo6Pj4+QkJCRkZGSkpKTk5OUlJSVlZWWlpaXl5eYmJiZmZmampqbm5ucnJyd\nnZ2enp6fn5+goKChoaGioqKjo6OkpKSlpaWmpqanp6eoqKipqamqqqqrq6usrKytra2urq6vr6+w\nsLCxsbGysrKzs7O0tLS1tbW2tra3t7e4uLi5ubm6urq7u7u8vLy9vb2+vr6/v7/AwMDBwcHCwsLD\nw8PExMTFxcXGxsbHx8fIyMjJycnKysrLy8vMzMzNzc3Ozs7Pz8/Q0NDR0dHS0tLT09PU1NTV1dXW\n1tbX19fY2NjZ2dna2trb29vc3Nzd3d3e3t7f39/g4ODh4eHi4uLj4+Pk5OTl5eXm5ubn5+fo6Ojp\n6enq6urr6+vs7Ozt7e3u7u7v7+/w8PDx8fHy8vLz8/P09PT19fX29vb39/f4+Pj5+fn6+vr7+/v8\n/Pz9/f3+/v7////isF19AAAgAElEQVR4nO2dB3zcRPbHH3DAUe6OGjg4OgSOfsAB96emxzVxeu+N\n9AbpJJBeSUgISSCVFNJJSO+9B9J7d4odO4m7vV575z/vzUgr7Wq1JfZ67ej3+dgrjTQa7Xx3pClv\n3gCzVMgFBX0Dlm5WFsJCLwthoZeFsNAr9BHWBviFNp4BOJkP138MHvNyRnXXhN0CClQWQhOEswCG\nMo8I5dECl4XQO8K5vXtf04WLAAuhryp4hAEdDZ4KHcKcMSWLPfLZ4Gwe8C7ckVjrsecaxcXXeurB\nsIN4juYo2xT19N0vVN+Dm5m9//f3f0Vsx824pm/c83ybc7h544uXHq14XiBcWubJu56OXKuLVw5Q\nduVB6rwIBcijdQHG8KBVAC2Cmy+qChvCjI8p5+Ct64jwtuK4/c4L+P/xG0x3dK7YvH0FY9deE9sz\nGdv7KG098gdjF1/CrWfuQ4QT5ckrtfEaFAN46AUFoeYiFCCPrgQoy2+uPcDmAsqgQoFQ1UnWC+Cl\nn6a8DdAKEcI/B7S4A+Duzj04mV+Z7uhLcPvI1QPvhZcZawsQMXv0vfC3VMf/4IEZfwy6E97MYQ0A\n3h3d7nbgCHMfhn8MmN8OoIIunq4647yIrjqT80+4M5mx4vB0bgFlUCFDaL8H7otjLKUY3J6MCLcw\nFg0wiLERAP2Z9mjO7fDYMcZ+btU6O+sv8HQOY18BLN9MJ7NaAAev3AZPpTM2FBFerV79Z8bS7oR/\nM008HULNRfQ10k4As9lJgC4FlUGFAmF0D9QDPON4VtXCwNYAuxAhf+nxsrOOsYUAfZnu6Icc+b/b\nr8hg7AhAUx6aGRubOp4/cF944YWHAOauBujGQ5OA3oW2jSNqPgFY9JzxdAg1F9Ej3IeJfg9woIDy\np3AgFO9C/sY7uRbgK9wejD99jpARws0Soe7o5Xb3YMF9YAxbAdBTXqybWqAH/AwwFoP+jgh/fZAH\nvX4XInTG0yHUXESP0PE6PJBdHl4PYpboVcgQ8nJWG7fbAGxzR6g7yh+MCxs/xmHs2w/QnO/aEhIy\nvgNYJq/LmfTgH6lYCk/dCS/OS2L/QITOeDqEmou4NO35D2bJX2FAMPNEp0KG0P5XuP8qz/fH4LYk\nd4Tao/t69+YvSntjHjv9NviXjbEvAVYux9MYW9K795lYgGf50/J7RDiZXpFH8UGqiYeQBjOJUHMR\nJ0I8ymJvg38DnC2wDCpkCFlPgFemTH8HoCVzR6g9yl9Rz/ywbvIbvOLCGgGUnj/2Hngww1Yc7h6z\ne+Rd8Ew2qwTwwbgv70CEMwFenPrLK4RQE4+3L6KOOSQx50VkgDzKWCleZD8quAwqbAgzPhKvsv8k\nGSHUHHXUly+9Rg4WRw1HuO13xtbeR5sP8XbhsX/h1n13YY30Ady8/354wqGNd1TbtNdcRATIo4xN\n4Z8/FFwGFTaELGf058Ue+mSo6J1xRag9ap9e8um7in34M26nffXefU9V/BOvcrpW8Xtebn8RN+Ma\nv/hQ6f3UO3Mw7IFn6lyIwYa7Nt6Yp+9+Ue2dcV5EBoijvBFzD9xxNej5oir0EYa+HM9AeAEmbyG8\nWaXG9qVetwKThfBmxSu58Gx2Ad6AhfBmtfrOZ2OOFeQNWAgLvSyEhV4WwkIvC2Ghl4Ww0MtCWOjl\nJ8Kr7b/3rhE+nGMQa3hA0b4bGVC0Yd8FFC2we/x+yKj8SG1UifQAEMb/6v2c3Az/rimVkRpQNFtg\nrerkzICipQUUi13LyY/Uct6lEyyEfslCaCYLob+pWQgDkYXQTBZCf1OzEAYiC6GZLIT+pmYhDEQW\nQjNZCP1NzUIYiCyEZrIQ+puahTAQWQjNZCH0NzULYSAqSIRud2whDEQFh/BwKQi7pD9sIQxEBYew\nMojZcRpZCH3R2RG/6K5fcAjDOcIY/WELoQ86zfOtpjag4BAO5bcyS3/YQuiDvsP5ZymagAKszizq\nvd7lsIXQB/2CCLXZbzUqzBSKCLNruExashCaKRQRMnZDf3kLoZlCE6GLvCP8s3rMKrdAC6GZQgxh\nOr49T7uGWgjNFGII9yNCt0yyEJopxBCmIkK3GaMWQjOFGEK2pUL4PLdAC6GZQg2h0Mk9OteWFkIz\n5T/CaWXLqeXKR4RdACKzNPsWQjPlO8JT+Ha7Ind8Q5iIUbSPUwuhmfId4TLksUXu+IYwDlz6pC2E\nZsp3hFe1Xd0+PkibAJTSfh1zhFcPezhsITSTH+/CfU2aH1W2fa3ObFmmu74pwvEAZZMNj1gIzRQ6\no/a5WMpHBZKahTAQ5QNCOyIcGEhqFsJAlB8jFZ05wguBpGYhDET5Mti0aWZiQKlZCAORNV5oJguh\nv6lZCAORhdBMoYHwRl2oGe85moXQTKGBsC2vHjb0HM1CaKbQQBjJEZr4SrcQmik0EA4DzdpM7rIQ\nmik0EOaMrf2dyRXzBOGisIhlns70JzULYSDKC4QXsEctLg9SsxAGorxAuBIRbvQztT9nX3I7bCEM\nRHmBMB4RXvcvte95lD9dD1sIA5EeYeJ+m2/R9O/C7dVr7PEzNaTe0vWwhRB1oFLEj/5E0yGcDVDC\n/fFmpJuukeIK7B1dD1sIuRz4617vRzQdQozc3adoN41wIU/qvOthCyHXNaQw3I9obgjdioahbr5d\neO3PLLfDPiGMx0Wk2ZXe1RvPdoiAooWQleYUdhgdXz32pFGwDiF2Axz2KbWCa9qnDUKEjra9D2+s\nuppCihrC0/UqzjU63IvT2W0Qrs/UPb9eMTjHQAWGcEZ0FCI8G5XE2Ni+FFTUEHoSPiPbGIQbZ+rS\nMl6exgWG8HrsXER4fBD/N7kPBRUNhGd3ZntBWJYj7GUQbpipNKdwu9nlCrCPdE1T8Zl9pgn26sVd\nPDQzx6uy07yfY6C0lICiZWb5HWUAQKkL6aanLAUoE2cQbniPxxDhFzaDI0kNoPI5/nnN6KB3eckR\nmz8I60X1wupM3RIxExO9K8GHcwpQVzDHh3i5ycv7DU8wDIz/FK9Yx+BIBx5eyVM07/ISLd4fhFf2\ndRDGqkXhQZqFGT44LzvYYj/GSxrcSCUMZ54fpLv7zXP4nZoq3x+k8dimPFCV7qIoIGRf4XR4HxBu\nmnrZNchDpv6IqBQYac6pheNky9EDwrX88FcmN5BnCNe05De3swrdWJFAyHYvSXVWZxZ0W2R8VneD\nKdceMjUzRp2MllEZYKkS7vi5zjBskntA2EwWUk/KM4RXYyad3NW86DxISSpCHAMYa3QGdb71cAn0\nmKmJym2MNsLiASG2PsNMbjLvaqS7u1VvNFF85aKHMIJn44cL0g1OQRYDXMK8Dzb1cvX5hfKA8EYk\nwDaTa1lDvmZSEdbBLIeSBrPDZvKGxTWXMLdMTRnRQ9/Hhl5Lmrqe5bFdaP6dLYRmUhGeCyOGMw3O\nuXHULefdMhUt3k7pQo4MnOkWzSPCQeHVjnu+SQuhmTS9MzSKvsC3aK6ZmoBxR3uN5gnhPB67nO+p\nuehWR7itetVNcpu/vmK8XSXph9H4UHXN1GxE6KzRbqhQYYNRbE8Ie2B0u8dELYRmisXMU4bczx82\na2CjskoBlEgzyNTfANqokeM92aZ5Qricnx/lOVULoVYnjojP8/2GnE2euSR3Deb2Yp9T2Y6nrzbM\nVI2PoHXyLDd5fBdOiW5mYrpRpBGmjf8+QWy5IpzXeKhB+i0A6mBezwKh6ufw/1mf0zuJp+/zlql+\nl0IvKsoI7WgOlESbLgixgtDc7fwjiu+YchIhHF0WGf6bH3f3LUA35jVT10VXWGsUbiF00wGkIAi4\nIGxs2Ge1FEMxc8srCM/JGmnuuuVensQrKjXAen8Gtf6NM9Xbm9RC6K5LSGErbbogHMwPRLueTg6c\nwtDmE/2lf14JoK9sVDiq81q9aY/3EV1vi1GmrgZo4OVn4DdCEaEoI8RVGzqILReEmY0g2s0daAXs\nRBPnnfmdHz0Vx5Kvr0DraBqtNZ2jMhG0jXd9pia1Cfs6mzriJprfrp8IkyNErahII2SZytmea6Qc\n809iC4ftKuHGzrLQVJx/ngc1iKO1RAzrkKr24Bmq0bY+U3GkYSA1DfuY362fCAfwK5ZxS81NhRyh\nKmOE2YsW/UEtv/nly89m2xTbFgwaRyd8T2/ExvbWAFXN83diuWhnV7Q+U/ESFVktMJjvoJefCDvK\nF/qtjNDOn0TvYT7Qv/MsaTtVX2n8SIyxjhG1mhjHsX1eKyMa6TO1KY1lZI7pttdLND8R7pIjxbcy\nwjmglXPRARyW2Exblz4Rx876l5o+U2+0jOjh08QYf6szh4YucLil5qaijDCH6CxuKxE6fSulD2u9\nXmwlTxPHrvqXmjVR20x5iPAswvmEOdDcPqbBPt2xhIW0n7yJCPb1MzWZqTMABvsTzRBhttcn+C2M\nkCqJuxm70Kqma2cJ9pRh7idnfgMwws8yqGQqOW729v7TygjhVwATfErNo4oyQrYjutwk49O7ycpe\nAFNEdy5OkZlKnUOGczE8yADhFv0j3lC3MkLP6h0wwi+xC1tkqq2Uh+5sTzJASDWuI+bRLIRGuswz\nbgrTIdwI0EwZdj0VDpVdioZtDc2qtmGWfycz9XLPzm5dQGYyQIguv8t6qeVYCA2VuZPsezUIkc1k\nuV2Vb7eX25c23OD/08oAfMGk894xxpl6cJvnkXeS0bvwQr/vkrzcq4XQTE6EVPfpLXc0ix4v4JsH\naEK9eGgOAyiTpGaqY9H3Z5TtdgBh5t/ZalSY6SYRbhh9qDo4p4P2ks9ZLrRsa6T0ElT8Y3znJm0u\nOTO1FagzfGkVhDmmqVkIzeQB4Y7GHYz9XUtJhCN47v8+rP1WJThnUhvFlg0RNogf/WNZYvhP/Fci\nXclUcmv/pdimhUVmm96khdBMxghxBKKka779UTVaHVeSCPHBWVsG7ahccY3mbCx+6/lf2JYmalfd\nxrQZkXVO4FHaleNdX/CKiZE5uFMWQjMZI5yKOXzCJQFtmESIlrytRUgSHtaW3HPLE+di2LFVKsIT\nOBepJB78mfb/EGfuWZPXQ75CtzRCeoO5lIyDGKZYbUuEWwFKxYqQHXiYTEKvbFHybp3oRp1TrTPV\nagazQfiBhldtCOEa5psshGZyRWg/nS5afw8vdQbmzu69VUyJV5pzSnUmO1YaER6lOYLn+NZ8/imt\n5B0teRtDZn+W3S6O0UTC2rjxkcs3jU3wcJMWQjO5ILxcCmA5zmgBmOYMxW61ZWxHxSjV7t61d2Yx\nP+O/0UQdTaRUf2ctAMKdp44CeLwKVUTJnSHU1KLJre9snLjIQmgmF4Q4g7eczOCTGy7KUNyrz9iZ\nJWq/tivCiqDaVSPCZjKYapti8mfckvMZmkfxvv64ozUmJutiY8NeDwhzN2zKNTwgdWsi/ALz8bBa\n/8DbzW0Pxfjme3PQDlhxQeiCMBsbh5XENlaF9stwfCKLDm003hiLe+PloYu4M0NzjYXaB7Vexgjt\n/HdTxYzhLYjwSvdWOJI78DUVYXkma6eo10VZZOz42LU6hNsBcGhxl9w9udj5TmsOUDz8G35yTbzC\nB6Dp3q6qKXT8iiyF71cwHgM0RrgVNL8pI916CHORw4YJm+ygQXj5O2z7wUP8D8tiQ56fONrbQ4sQ\nH51vtDZenmJXI36ws6y/wK+TnJ3gaW8BlBHO7XDKxbcsecpcD40LY4Q7NO0SQ916CGmwHi068fNv\nlOW/XAUNT/530FZN7HSf7XyEiZDGxgnIZQ8ov+F6bP8RN+SBzRggLHHo8W1yk8YIc2sA1M1vpyVu\nCmmEmZiROGlwPS9UuD36DJpICP0PYuKvbExmU1SiTlei0sjGOAEcJ2wmzEmhPbpGLy2HJWjg9wBt\nYo3XZKqnp+qMY69ZGbwVEbIlAP3xs4uExA8pPSvNlV/7cGepVOPlTCMrG2V3UZsJmoumtoCG11gM\nP+Gj9aK1clAe6QjQSWzFl9baybnLalSYyaB3JkciwiVDHLxh16PfLOd77gw/UGcCHo7QXqUlVD4t\nt38VLz+90DFGJdldQxfLbQ0wW30x5p6THl8vT15p8Gi0EJrJACFZ/EI5+ZBKwY5OTeX//MhfaZqM\nx2JTy+ihOgmo9y2tm2IOjrPgQL5O7ZNLAXyPW/gDae1+SQuhmYz6SH/C7O2mjKRjqz1SdzwCzN5d\nPbQPVVU7f8C2YhrLltjogSyH3VvRTwI75wbK57eLLIRm0iE8WjWSCsPFzqC21KlFV0kXB5sZr7m2\nIS4ckHCSq0O4J8sYZ6bipLUKcls8t3HQkez83Z+kASFc/UWfFNMTiiRCrJW8j2OCs5RiwYV9NXqj\nT2z/r3e5zgDedDAf9ENpysXeTsOUHKaJpyXxa6eFq6P+WgWCENss1U3PKIoIM0RxGJsrpvUK102x\nF23HXb3LH5l9xKWPlKLKzM+d0W62h+Za2omop0rudw3lbcZXvxa9NrlnjGyaAkFIDk1M4xU1hEen\nHVbnYQ9hOXUUk3n+nmplEM21mzsZ48nuzz7g0R3QGdAUb40MXsjH5l9UtwNBOMlzU1WqiCHEwSLo\nc66qaMbzgItiVaR9uL/PPRohTNaUTuxgEW0Emjaq1H9iB47R3pfoKfAyp5drwzc4R1CdlxgIwpyW\nUGaX+RmFH+Hx1q2PKQgrUt7yIvd3/PxdnnGo4oe4u8U9LkfoaC5bAkIb58sb+BqjNBHb6IaknGYw\nQUymMfRqoZWcHFdH2bdqpMYiY5ez+zLZ8Ul7WHXQSrFoKkN75eOmLZEYDkw6wDLahTW7zhGucr4v\nUZcmrxMb/TBc+gOmapG0t3F8CZGb0Eqxhddbqyxuo5GybyE01gaRTyXRZnfYSBXfs6D2e9GQReU5\n6KyrLgVgK39eT/6vKUdIeJTWw2WstwqrwviSzkcl9cbIheqocy1nandDL2t6tRS3ok6ZsBAa67xk\n9qb46NCpyeZIsfnOVZawL+vapgR0dfGbaOnje+7CR3wjOor/C0/OTBYDw2v3DFvBEoSxKOwmoll7\nnIvAtHOO8dKMmvaAJhxra7ZyWwRLq8vloOSWXU4CFkIPkhN1/y5R8pDMkTTGVB1fRu/wjQXo62IX\nFTfe5qOeUaiK/iTaJg8DqFha/gC691DLcDvXRLLU6RI4nSwaz2mM8EuZT/DU576F0LP6ON9/ZXlt\n8QcxbFT3WxlGJa66jYMcwZROlH3ZvSK6pdFiFYo7r8c/cF7moufEltfvdRHLeW90fA+xJrd1owbE\naFseFkLPWi3yHV+H41UK/1G3ojHHq/J2AtVaMKR51maAYh1yqM6Iw3wPKOf2OknuMeRIUuJRQ6OW\ntD9KQ6Vkmt5pNpmpEz9eQ7NvIfQsmf1pGdvjdiksXu2sIlyKzUXFcyz7ju+c+fN9PEDlCK4eGblS\neYYuVi62EE/lx8unZozrr7p9Su8e2S+bMhVpjCofQQ2VP7YYs8HRxXDNfighjJue5VWZKd7PMVBK\nkv9xvsE8f7LazqzE3nWnSxaNMmjtlunnzy44lZV1YsE55+n7Fsa3FifRNIk2GJYxvk5D3MnIyqJP\nKIWBuDEKBzS2yahovd0nKytZlzy/Snl9iBSaFXfR7Cdm+P/dslxTc1N6YKVwpt2rbGnezzFQWrLf\nUdIFj352O/7sJ3Zs/Tb/uETODL50O3fjMf7/lOQ8HbtxNmP4itLQvTbAwGq19yR9+zI9IeWFaQjp\nPRkd5zlF2O0p2ktSb9sCoxvLntb4h0zN/jWb398NlWJ+OKsIPEgzJZCSZM5XjWZ19hJN/v41/q+i\n1nziRinyUEINg+KfTmBxg9vSA5aajiszyLViMq1QgEv9KdY0IDqal3eYiI6eurk82tCQxsusNKlQ\nepCGFkIyTVJVkSw7wym0NIUoi0jmzO9Lj1ybcHiBX1zp5iZz7TFiZGMrYxen7mTst7q9k9ZNeVtc\ntZuY79u6GbRNd81UbLJU9eU+LYSetWf6ZyrCzuWfxlfZkpiGyzZLe255VlV5RiY7BGRKmrLxmDJS\ngQ2Lo8J+UVrrY8fbhzXrKpdlrJ7zUvpMlb8Lp+ZUaGY4MdVCqFz4m7Cmbva6Nlq7qmTF8orZGleJ\nXDIQVIYbboBaoNjZT3hDEQ3rpzK24btdjF3t0RzXY54XFrZcXlFXtHG8H/tF5QC9PlOnaH4mKOwE\nL2905xZCKcz5mryRp1sD2Xb1CaBp7/M0+X6B5gGWlmOz4pU5kIacDimnlKHLzTdIZqJ6mQaVIewY\nL7NVofwhcUyfqTlN9aP/2O8DRl/DQijVAHMop6Z+lV1bdu6CUWeZrFxIxdEkXLULrA/uiXbaXuWU\nt4QR1BjNpdIaQbWLPGfakSFx1CEDA5odkfClps2f1QIqnXbutnQplaoshFLYHq+Nq3OAsxw6tqnz\nSrbWbPiVwPMuwOe6Eoa1m7K0lfO8ijmGPjTTUrCVX4u2FgG86jLNWwhj/OLcHQJaYzdyYrvTKJqF\nEPVjVOu4geEtEmierew0ubDsSmWqnrDYIRMw1S+cBTFVm2ufOF+MSg8AbKMBfS0QZdlWp2W/0Dm1\nOOYqr1SphrpiR6NR+nXUpCyETHjwofp7SkmAKuIR+ZuEcZhhpzUaZy9wItSN6mFAXbn9tsJYzNw+\ni6P0U1jGuL7HsU/laaiR4ZxfQ+LVm+oKAjQOWO+8LkJzrnR3ju+VNLCisVsIUW2U3/visPdHyAyJ\nkDAOOugojuDNfIFToFpFD9fYb9cWj8aEznWX7WvSaL9Y8QCifsf/iVgCT06ohd0zQ8QsN3Lu5MgW\nqzutlBe62L2Oc2Ivb05Mq9VPk887G7Y+w1y1GKDJVQshY5NBVOxPYIZKe08chniSV1IdZEMNYl7n\n6X3RAuxAXtsY12tPb2SS9R3iLo3Hk/mL8v8OsuQ10dH1nWWWzMVGiuLamrEr3Vqi5elOXuhaH8cw\nZWEZTabyl/Lr67x/LXr2/mQh5MWhT3hNHIHDOSvKNGqcGrFr9QYHWWejwUzympNiotgjuH9UtMm5\nvvz2PLUycKlQ8skdoQ75azWbvCPAOrusc9JaW7C0KoevGAlrMpUOmq6PQMK5v/DtLY/wz68nKn0g\nxzWlkMVtuM4bFbt7/VcgwB7O+na1bK3L1eCZqpRCsSt8G2hGFrne5/mcPf5LtHCqQ2800Vnwc+7y\nBarRqTNTyfueD5ZQZAq104nwfL8R3hwgqio6CLEtV1/ZWRxZSWNdmTKnZVWl1MGr+K8Vdnd+xes8\nkMY0fLCl9yc71IqWgIBGshXZZ+JTzlOkKz2aDFoTt8RaFrrReU2m0iLA/Xz4XmO+PuSszqBVYxlf\ny2TRQYgmZ8am6ZcNHoeH34cPrl0dPCie/E9qtSpebnyezpLRvRpM7KYcK15Dcc9GRsXULZBVE6Dx\naV2Cmky18WZkZR/7dZ0IyYuUF2fARqkZqRAhJF8h/LH5Frx6Un9kkDvBF/BfVzp6zuXQocVyY9/x\ny8mZKwHqZVeQIUOzWPLWMVPxzq/LIKyquBlX6DPViyNZp5wIaYK+6zrdnlR0EGJpWMPY//iXf1UJ\n21Wl4mpalkkUK/r/hgqrenOIOiSqPk71Z3L8YluM8HiRLdsV1IxfgZ9hmNVnpFWjkYmal0z1JE27\n8Cv+dvU1WtFBSC0wxh7j2fqoDKB63rnsppJO7PB75Nan9B8n6n4+p5EeIZPlGWhixETx6qMWZXPG\nVj5Khw7yVsmckzZcQs+wv/rmEapzTH1QUUJIwvb2O1caRHTnz6/dmMcLFBPuCLaWPj/oNCAaPAlN\ntckmo+5kEdCHnoNkUSEmNgGaIGJp/A3roiWN7iEPEPqhIocw+XP4vys1eOaOkvMpztCSofB83+tM\nmK+9P+gTA3YfdcQGZFVsJ9Bpvy1RD/EHKLpSG0wWF0D+MtCCOEoWTXdZCM3kBWHuDzUHZYlmXT2+\nu7lC1GI5Gh9mG1JzjBu45/jf3bTVlOGyEsCrlnYcWYDL29WTXuP5MK7+hBzRsb0yRSxTDhWVUusm\nC6GZvCAcBTTbBTtcaJ2cS726nhZTKJpoxutVldiqbvZVDN2oxMEnuY4K6jH18seb1qYVDnGEHtbg\n9BlDCwoLoZm8IMSSEc6r/G0jh2NVMQMb750Iw1fYv/aRZPIPUf5ev0blteWEElAlVXnRZYl+8Xph\nhP5h/ve4WzJi9KPGpb3Gq8lYCM3kBWFXwPGiLDlRTL78CFwX9Kr2juMMjjI8K2kdEI/cvvxu4nNX\nysCwR7XltAGWyGmnZ+Pc/PF1his9eCPcBvO1shCayQvC1MZQ88okgNqiPe3WL0MPSXhf7g0nW5oy\nsz/59FUxdUKvv0LZ1aecu3hCFyUhcjzU3cNdWAjN5IMRIq2bJO9gkguVR/W7/IzkE3KEHmszZZZu\n0hz9B9qRZh8rI3dxYpNzCgQ2Jz15CbUQmskTwktOX8nU99WAssPRzK1o6bR9B39jfi62/73zi14J\nogUoFSNMgZ+Tu2i2VFdNJXvS1x5XV7YQmskDwjaqUbZ9aAXqQqNZXxtxq+6H8rGo6Ellg9dVnq7z\np5w/+DrLWTgxgV2WVsPlijfLFgjF/hujfqkN1S/7dJMhjvD61snsiqfTpYKOkF5xYogXx+b/gbs4\nL4x6rA/R0hNwTWP1pNOP4qMrecaLYzRxdF9Gzo5D5zfFIcJ66pk3jNI2UEgjdPS9i79Bnu9qPv84\n2AhFrwlnlnWDRSr5jQZH2EkakUuTWiqsBA+KrIyPyHpp1K82gYaLWzHZtEAjuLgqypm/GKRtpJBG\nSD4F2LNe3OIEGyFVSD62sfEA9dWZn3ejm8qkn2dkSafn1KvWg6beQimn6wtFXReRg/XZzFYR7ZgW\nKkUTlSC7wsf6eJMhjfD12xd0Brb1tv+Yxgo2Qlw9Gc5tJEuXX+uA7AJ9UBzcEU4+EchtOrx1mQzX\ngOxM4dNOL8O/JEJ+uDhvQh6Lpd+AYrgv+8/SRUHUG25vX+7pK8T3qjLB+Zw6vci3N6iK8PyXbQ3t\nTJ2yr17vHHtZFbwAACAASURBVMnwE+Ff32Rd+M4b95rGCvq7cE2VsM8lChzfrd39TuJAUoqZaAzW\ny6Uu0z5/GA1W9FG3DqYJByVH6RJJIrCBLk3+ai3pwb4FbX9VT4fYDWfgWMpAEiH1xBo74pfKjgKo\nrDL0E+Erd17iCK/c/5JprODXSKkcUquPWujZOCp/N3Vn0mxqrKOuvIArULy014EV1JVOe+Do8p8r\no8BKCxC1fPXOo8tiRQeaWMNimy5FaroYeKRE4SG1+YEv1RrG57lIIqQJHQvNTqSKtup500+EXeGe\nZyHqYXUlW2MFHeFp2SU9dOQ5mkLvuEENh6l4DIvbjt8H8W8sXoDbz9WPmMguq6MW/1In69ZuqC2S\n2JfGGxW7V9ukeb4+SepSnWFwL0z4QRii7GD9qo7xeS6SCGms+aDZidQ5f0DZ8xOhrKh9Yh4t2Aiv\nKNk+KoOej1NsshLZd3DMsCvd6q8UbvRysRhCW5aWo7j1EipJ/6NOOOekoUqnIsKWuJGOZ/C6zL6p\np5yJdve8Nu+x6tBanROKNWGPnQA6SYT13X8vLsKlKxqqL1u/24Xr+zbpudi8TRF0hLOUbG8gzH0V\nj0+khjjFkxBuFAHPA7QUUyY0toUwiiUO+iZKy5AXseQLYiNn8+xY0VuncW94YqenGqQ+UxO2JPv2\n3SRCHBWL9HLqnwec2/4idJyIZRfPekkgqAjj92dvBp0aSdtOVe9ULvNZJ1scuOpt7U4mdpLWob6Z\nRmQwNSj9zG6aiK18G/x9+PRWozwbGkYL4DkOu7uW9SCJ8FqUq4tpH1LzLFeEKRVhFq9j1TUeJ1MU\nTIS8qlHyqm6UoXwamfBG/wt0GrTGDeHr2h33IX2pl5Q7jQKN50kzYZ7hPJrSbG+HNzTDG16ktgv9\nyxo/EX6J6zAs4dVv01jBQxhLU977sp1YGX0FcOGsf+HSgWP+1zDjdL13tCTCr7jBET05Xcu6BD/I\n/2p+rO6O4lWAEY1niUnyhlNCXYV5Rv10f4or+Nku9FN+InyOZuSthhBpVNhFTaRHCk1+/qGUyLGD\nuTgDu4mc1a6oe47se5syhJYJgReTyDYffj9d+yX8fFg5E38OjZQ5bVAhXTiY+YVlpx2yebovnTDP\n0A48XHYCnfXtuwUH4b3F6OOREGnaHxVZJFHxujw6UAPZL31jma5sNZMGvehQJo025DLmG6Q79A+V\nM9HudBdNxoVSSdTGxs2qlcUiEz6I8mxmhWaxwkCjlo/fLTgI34ZpGYz/yEOkg41sfQeKudTwFui1\nvIJ+X/ERs5NdjdSGp0pf2nCX+BjEDvx6kf88Zp1ZOVN2v+C8IyziZuueaaRmqqM9QM/VXurvqoKD\ncDTAX14rRvMkTRS8d+Ei3pxn0gyNKpPrY96uI9EcdXnHdZSfg47KxQnFwOFQpVAq6u+8vOL96XRF\nqI+TCn0cNw3pbu7cL7H38Y625ubi+YZwfHgdfQdwBn94zpPmgzib4u+DHwC4nXZHMv2DFKp0ffGT\n4uCm6fwyF7+mwcQRPyH1Q87La9epwFYlrpV1fVylcu3MW3ohjZDXAReNmX/Wy0XzCyEyeeKANoSG\na/Fx9aQLmHvjRAezoZ7R7nyHNyMsvquwZTVba7u2dEuN7Og5za50eRsacasKXYQLF9oXKjKNlV8I\naW0I0K4yRfa9R4wwlUlPf84oHKV/ay6XE5YAItBd2raLDSNpFRhHottqMYwpA4mm9xm6CPmbX/3e\nprHyCyG5nYDt6v7hfxKsGWCsO72GU6ui1OYJI8QueYNqgbWgueTip/RRd4Q0aVTjg8RIoYvw4YdT\nH1ZkGivf3oXkKFRdizzJUyn7i6cDrjqmef5+PGwTOTyknp6OjCHJRu4I0SjulZbmVjShi9Bn5V+N\ndHbZssvUHfVd96ILmQrvg29K2yE3ao+ZiiszreFl+ioOdnzLGHbPVDTqSEzw2msd0ghfeniZ55NV\nBalR8ZNConQnPZntZ2qXN6h7kl7S7vwnUrq+7ygvaYt1sGMYcIxME2eZ9wV7UkgjbAM/+HDR/EXY\nFyJ2MHszMcxHSw68PR30GsLr/iVw4zY3hBe6feAWxsve3hZvNpTGDjRqThevZVCdcZFjt3YsMHcH\n1WdDGuGacsXGr13PZRorXxGSd2WHUgKXUz83Tt+9S8dELIft1MNQOZwK4P/c+eH4MBpHfSi6UU6q\nB7olZx43XbKH5VQFqKf2vtiiAL5gIY5Q/Xbak+KpfpY+vF6dMcJdT74iJB8jN7rL+zhNZWbB+f4j\nZa3yCXlAX9Ye539DzmiIvgKq7ZqzjSFwkbdJ9DP04qUb1QxWPteKbBrVlir9us6FOMI7FGkvMYgQ\ndm+3b2+zERSSrwjRaU95yrtHH4VpLAv7Lk9NHydtskFZ346a77ohwxdYE83e3TGJXR8VbFWEYq1J\nuuL2ClBi7PnFkolHkSGA+iil08+EOEIDzYiOQoSXok7z10gFGo3JJ4TbVmThu3BFrQ6XGVtH40t/\nMHaxa8ddGjSPa6nhCCLcq+w59AbdZK/WgP+9Ib2aKEtS8iseFQ0YOnDU5JbsvPpaTe1tzCwvLBVD\nHWHqMf2S0tdj5yLCEx35DZyMokZb/iBszKueac4aKbmP/JE2Pfak6VSJTCc6ddQFrlWLoHrhNcKI\nG8NalFLdmnpQzrpNmuPZq8lSMbQRLn6Xf7P39e4Z1yh9FY4xjej75AtCmu4524kwB/eFW5ih4EHa\nussn8WL93Thb8nJnaLq6TIFSlnJlgcSPrtfnLwsgX0Ma4WJRS79D555RQZg+Khp9Tld6t+zEhLwX\nNdcmK3uXvw5v9s4DD731fkkeZFDLFHrTufn69IsJ5OhpaULCHyLouRfDViRIR93wsXJl8mX6xrqE\naQCfHffj/g7XLz9Cu39wwto8+NY3rzgXhB9AlV2XN34CHxogPNigEdXN0pLPzHJ4VU6G93P06gwQ\nmZmeKnb6OOGs/84TQdQ7ZEAzvDZv6t0gG7YkHnuWHKvoleJc3VdNRxgypjiST9iTNTeZs/uo6e2h\n/e8CsZnG/3C21TD/vuA1u3/nO5ypmcjugvBvT2Bhv37HP9wRbov+UWkI51N15tiuXLV3JgJc9LFr\ngE706Hyz3966DXeOHZnImLRLiuhLH7ioaBs1mWuElR7RmqY9+tJzWzxUI5p8L1cHxjxrh/v+fb/g\nPEg/fwS/1OXbympPIoSZ1eepAXmGMGNwPRyRTekaOZxeVVurVd28vkqdg+yQa/9ZI3W9HkPJ6dcL\nWDbWQ1NEC463+WQz4/+iemnqaNS2J5MLDUJyBmWyeCgZ3s8Um5hnOAxW1uR0AwUH4W/3ffLb7lmv\nF1t1kEsJJIRbKm7dxUWeJvIMIa4sN42x5iDWlnc3I9RAolUin/Fw9LGu9PFWLvUEjE/ASS1PlmuT\npnjpWi/em2I6IZsfHi1c52sQ0jjhWe3NJU+eozVoO1Mjor+snGKeXSqBvxi/FOTeGZQSSAjnRpGo\nFz/PEGIqtcVHFcZcLSn0up/+q2uExPQrpjn4XOZYKrdPSJu3MWe79ybrzlXtq2lOg6X69DUIM8oK\nn2DOY5xRlLEBCuWZbf91w4OeVYAI3ZRnCNFyrC9jWPHvz65t001b8aK3PtXujVSqLXWkf1mnn9d0\n/pJT1rKgsq6RtpvbtmKzroVIcYyb/Wks7pLhAXMFB6GuuuoxVp4hPFMRGvPz4utBh6x1PMeGh5Vf\nOOsFV1p9zFDyhkSTERsvKrsvodEbl0x9Q8VKW1hSDjn+QrkswGQ2UoH3A8ak0njlubmvpodOFckh\nX8fkZuvZ6Qioek3MaHgCIucnOt36whOPP8FbGtuU3aeahrsz/D+AVtlK6w+ez6BJhNKNKL1ceQWV\nJfVrPaZx6S92u6RvhtDR0OO8BHLfsMOH76dXkUSItcyf8GX12UqnK4sHnXg2sqQ2La78qO5nJMjH\np+KkQuImG5nyYpuqPcPF9anTdIPn9M3HCy8mejhA4xfeF6dwVZFEiFlRXJSsbxa6FzCOBVsLLdXd\n2rk5x2kGdstabqe+YhOf5KS5lxhXosV5r/Jc6F9/rlH63od8DZXEb7mM/1GLJEKcoPJmL5H17MzC\nNkYUdZrPmOjGltYzmkbGp+wo9Yf2ED8JMSy2IroC9vfiw9W1BYDvsmSjm5wRXe+0QbBGaVlzZqab\nn2KkYCHkVbkLW7PMY+UZQjQc3ZhIi+E+nl4NSulGklRpVnMZqnd+OFQ0B2k6b9me1xl2xZ1N+4Ge\nyTm5hycANKKqKe7rrXtX4iF+vKMbLZyPWsb8tkO6mzuz+b3oefdl85mreYYQK4pNhUOuUu7OJl2E\n3WQPl9MZr03L0g4tfcjYkaXY60LGGsnC2Ro5rcCq0ncsc+EKNRPxkJgr3MClaklun5N2LzMpZiGN\nsC9A+t/gdo1HQCPl6btQ0WPy80V4zZBgU4OwE2JISpFsa9O8s55TZCA+UY9FQwtbRhmAypJXFh6R\n71iXGWlop1MKfxmelwIJaYTFYd5S+CD2zn+ZxsozhFqTpgeUjV4ZatgjmuMfgauKLeKXOFbOGSDN\nQrGLvIxcjQLEEoSb63xxmvp+lFk36GN/j5hy6lq1HBdRlbrpxnm87ZBG+NfnWBsYyV66yzRWniH8\nuwbIfcpGDSfCezTHlRa/03HhmqT2kYNyaYEs2c9d/AV49NHncdS6Sr3SymmbxSJaD7VSeKIyRnf7\ngyX0Ks7foQYvfnLQ9JPH2w5phP+85+hTcDr1L0+ZxsojhAn6eX88N+n/Lw638oZ6WX7uQLtgKp61\n0PvdaAf1rTU+4R5DtiFHsPUP0QaP1IEna3NOjU9OXTRfQ9C5fCt/oZT17J84pBHW5t/0ffYKdTp7\nVp4gzK3nnunt+KtwiHPJckONV9+gNeifnLZU2f1BKxGG25SA6M1xNPfm4/HSrlDXLsQ1D9Vu8NM7\nTFbTCmmE5/8Ndyxn9z1g7s4oTxD+rmQsvgS1HWdNOoORnpWfh6iQ4voh63C+7zCm+Jtp+R+DWFxR\nqr/RJ3FxX7HZmO5Bh3AKhvvU85nGTk/zzeGTTkFqF+YcusTYj2fNY+UJQq0HJ4hWYQItSK/qvg+E\nC6cXy6DB4Wf/F8PLCbUr6r77OHzRKKJLNvvpHTEOVTuhbWSDNro5FahSwgDgbrkbKz9PsCMDpiZK\nhDv7LnAI/xk+2RqkoXmp51elJxUpx84bv57jSNDms250Qh0IxPbiO2K872kQI4aVmVyOhwam5vO9\nPvzzA5xpNojvJFWms1tq6kHKc7dPmw74cVSW8SN4gZoCIXal9iAPbTV1t3m4z3jDbrQ0LP5h3vPA\nRUXJsTPahXZkV3+c6VzN2tNwvNsUbWz7DYeI7dSl05jfPQ0Cn64KLXm9ZAPRD9u0RBOBNRCfe1hq\nSd7ucDiWYCmv6SCzGjEPjQY3GDvYe7yudorDx9WM7j8N26gR3vPARUXJsfMTIssYu6GuK/H+3zwy\n1EupNcoVKkq0Rgu2x8iNnmK4If3VVI7AdsVEdqNd5EfC32TazIVYSXFsGLrBwWhejShkXY2JUD+N\nUQamofWND0uhu6goOXamdxeLGz1tABjIOeHsF50fPJrcW0y5RrZSSanPfwadBEJhuIHOmvnT913e\nfGgMZWjFXISx3+UmbLxCO1sgvMZfl7uYm2hKnJHpRRqzHfJx/V6tgtS0D4pjZ6yPPBuvQ2OgRTl2\neigo9ogdcRLMc3beGFxelb/+jkhfJuE4eteVEJ7D/U+oNZDJH5E5WLIjGZtDsyva6u+hG8DkXKqR\npo/qsd8QiL0BwGK5nah9tYR0oyI4jp2n8Bxt0csYHNQWvWr8OSB9M2+TJqT92/N/XSYr573YXlyh\nJ7mUEc/XWU9DOE2TT8BF64Tfe+Xx2lV3CzRlKYUQYg3Ig7tz5dV4nj+S9zmDQxphcBw7YwcCtDcm\n+KlDtin2KVXJ5GNyY2u7iM5be2rO3bv0q4jeWRjy6CadN8I+vNGR+wu1UUoxsdZkSb27QnLYdhIR\nUhfRYPMvg9WXGOduSCMMjmNnNKOtYNAjRrKVEJ/PZMuA4el9lGMuoxXChW9vsTPeeX1q7ItIdyw6\nS55YNmXrbwHH88vSg5R6CmaafxksqM4VuUIbYVAcO6f/+gl8uJhtrxX5KbjLIV1WAlPWwi6TtVjj\nBljTdj+LV7Mre84EaEU7mgaMzckpJ1o1VybqJm5V3Fmc6D74uuidWcp/Gl4yF5e90LilC12ECzUy\njXVzCG8ArfHZetRDYt1yyUnZ2CeN6591DtBz0p/WfU/uOA3zJ9HlElX2fMexZhZ/F9IUFhoNxBrQ\nG87xazQsFFZsCfWgRZrSweY9a7cN0g5JhS5CbVkwjXVzCKcYFD0hcva0T86in+HqgRtE5fTPMY1X\n0sBUP3G52fLYt7iDcyjiecmc/0tK2nPq/EN1JRDsx6tOW1jCuwZq/hS6CB/WyDTWzSH05I/rg3Fk\nDXqeDGgixkkfeho13sMOzb9SAdvzuCsXyaJ66+F55LCcFpf4UaZjW7pOrpqtGtqjUc1/VuEW9qtH\neUGY3RyiDuHn7HFXdQdCF6HPujmEGeHUJ0oLDnR7z0lIesXbik7RX8YTk+XDtYp8KZLzO5qWJFz6\nUq8os/Gi1UU07cVQ7VRnUg7hu7SFsi8M83sxYQc3xAtCfApgJaYKuCzQUygQmi9CdJPVmZw9JxrA\n+7vq4USKGm6FMQNNyOiFlT2/Pg7rd5gyav36x5RHJbXylFcoWVTn7j3FJEI2nP80dFgOous8p5Xa\nGapx843Mb6KH53hB2FqcS/XbydoDIY0wo91rz3A9kc+eEHHBgX9jzriuLfFgzFF+zwcz2YkWDbZF\nuhz8ZE6umOmgarpyQY7wxBeDeBbFHdPVpm/8MO7SQe0UM2xEOqcFmiPEzvIGjFGjRFe/C2mE6mir\naayAEV4dPZFs+1SfatRZrTVyosdp85xMt8JJaq6fegWxbOkw6jWxZeP8+ldd08OVfErpRh8S+MN5\nlbrnCeGvwv3zwoYDMHtGA9TR5X5II3wGBrz4yvRyt631HIUFjhCbE2HYb7xeYTAF/7kbHU4eaIxQ\nOEx9Re68foCa9ThkYBO93ljrWPndwXPfzxO5RWVWb2ToiNUUSmOE/ejqO50BqS5L1YU0wrseZwPu\ntWfeX9s0VqAIqfJIQwdrWsslBx4E+G99cFUvrSHGq7R4pBCrxP8pqxksFe0gtKGwZTfAzTQyXEKJ\ntQgP4KbJ6iGGCOVEqlmeo4U0wsdvu7ga1truf9Q0VqAIqW85jjZTvlfKEq+USDv8p2oqITtVbzGw\n7Cqz3VCmzHRkKW3LzCbPMfAMTnRB1Ljeju3K2Ad4q585G7cCTnecYBY72dPkJkOEstFjsgRMSCOs\nDnfH/uXB4vCkaayA34WtlWljOAv6U9kZKnszyyWyRMWsN/FUSblkXYTjQjVokSmXpIhkE4FmhXI1\nRNP7LTweTmXFRZzK0JRcZd166eEwLZ2ha+/uxjdpiBCta57veMTku4U0wtPF4UYzyJ81mxxb1tqV\nqUS01u4BuVgrNf/+gfbvKYIbdLjO4h20KtJeHL2F/nLOUxUqdnIoowN9gVNLj7JLS+hccmKxibf8\ncWe4mi45VTS+SeN34cEOveJMv1tII2RZm20503vOy4d1Khy1eUNZqR3S3M0rmYvp0Uq2T/dMT9/8\nh/r4pBnVbcgTDO7VkHbAB6iL7FOlSGZuOY892sqcNsHDfoXmWUxQEx7ggvDQbvXbFbkONqlsz/NB\npAJCiOvJ44ID4mgdXtLjS8jWoYGoiF3Daba4igsNEg65YGcMO68/FrZSQzC+Ynb/uNYVwgTO1zkx\n6Tov5XOc98GvEK2MPBU9hIm9vmWOrn+BZ383ixMgQhoBUppkGYnn4h1Oj9vapXtSaOihM8sYVI9G\n8dL7VJ1AnoCwxyZdPe2zUSm9NbEiT2l/eNdPaRv4OYc1kwhpZXXFaLvIIUx9FkoJFzq3ufoV0Cuw\nd2FzgBhp5T6OF5twXYte1UB2Ggf5zuKi5LL35aAwEQ67weY4z1s9XN3EMrrS5FbmiRXuhMh1sGIR\nU+QQfgX3DWJVoNKWSGPzSVUB1kgP7JZlIx08aMSWo6m5tM7g+6Pwv5jmqNixKYO9JarzUicHE/tn\nbIpl1zddynZLzSk8b4y6x1uh5RVyRQ7he/AbczwGh9l2eNE0VmAIF5TBVaspvsrMZcGJdk0gcpQ2\noOwSfvrTunPCFv8IL1e5Rs/lLofktX8trRpeXN3o8i6nYf1v1F3H1tUqb18RXp6ps1EMXYR/gyx2\nEYrzhjLcbRorIITXMCelX7fKCg/0NuJig6i6nRF4U4WlrlPHRdM7ATvBFVhk3C+s0LDvzmXeLrp4\nM3bj5CNCNNUfqNkPXYQPwUX2M66JeTE/mvbkX3Je1pi2q3mld+DT0j3lwBFiuEnMo3hAy0qYG2IT\n+/s3Sx1YMgJXP6v/Isgy+VPm1LGqfyqaJioGE7AToLLLrcwfeYYZSo/wj7r19xmeRp3/mv3QRfhf\n6JX4EX/Y5TaHz01jBYSQBh8uY4lYIQJojtFQOdNopbq0couDwpVTMxrXfV9UgBbH1D2SWf+dPk43\npf/9QDO0S/OCRXMcG46VNKlOHWfSRtIhpJqwoWu8XoUGIc0y+Mt5NLY2dLSjKrB34Zl2zYVt6Bcy\nAJ+U5WhE/t2ZwksFisfDfrZFYnxe9FFPxU1nvymIKdzTndc+1rqFNLjHaU/O6Q652JuT5PEmdQhX\n43XX02balOna248rqWtZhjDCbF6Pv304Y69Bp3xbRRSzVLG7VZeV+Ge6qOyj4Vk0P3C2fhQaxqyo\nUIMebLmqFzxRKf24wizylad5VNicNdLY3zW+72h+zHyPN6lDiP2swm9eRmn+29LWcbMP6Epn6CLk\nP7eNsfz/fLMuXtRNIPyjHNRWjqCFxcAI5Rl4Y+nR7LZQ2c00/noFKE1ToXBoo0vX52gwltwqN3Ge\nZMu2G06KvOIsWUbSvwvnhYWJ9ykNM5q4yQtlhD4qrzxeHB2zXrhgnib2T5bULkAppdRH75pZDior\nz0Ss0t6v+W62tQBl4l2jcg3iL1fPhs0eaqQ02d9kBRILoU5kNyj73erJB6nQzvLQjj/QlQW04W1N\nrCmgH0CyoRvEL42un2rW4+upUdEW4CuTaBZC1PSouuOjax1il7t8CvDctKv8AZ7dB83USjJ2rl40\nut1CbOVTqMdPuHPGeAdqVKAO3LmtftK6pCCnFo01AesqVtVYTniSx3ZhvGd/usxCSNosS1a26uCn\noUPMF32PMSxRa6Rrrua8ptjlFxv2sVVkEpXBq9rWB9B7qSpqqniuiSoqch1sPuvmEQ6R4LY4mwnr\n5QCwaEB+y+SqBvI+Wz1MPiZoOtTk1N82uVzPlr2i9WiN4zvyiLLN603ewghn2r3KlmZ2dJXkpllx\neYFYEj3GbkdTmi/t9ms4xamBjEDrzt2w02RA7ENrplwpvk3kyGx7RmYDHpioXp96UK96vcmkdO9f\nxEApAcWyX7PlR2pZASGMm57lVZkppof7vfx277AK24Rb9Cdf5428xKT25d8pXedYFnkpbMzPSe4Q\nVrbzGnE+Tdk+nZW1uULYtB9wO15eCPvnJmal0TDgJOf1F4RHrfJ+kzdSvZ9joOSAYmUlZuRHaukF\n9CDFfumyOdgbS1o0d5qc7Y4L/mALXywpjM/b2bSFDbwoGZn84Sl+s3G7CbNdBfVUP3QLP0hvGuEc\n2fKSo37UVZb42yHmQITnqsNX/Nve+LohLtcq+6xjh/ykXHE0RlHaCmgCPJa/C/ljOMrFp+H6Rj30\nM5LcZCE0kznCjUpBEsNOuIW2NW9Cqc0bp0nzMTki9YVbZOSuumm6WCe8Rw52sJ3d59IriEMj4a5x\n9bIQmslLu7CjnAeY/evoceOQYEpbQQznVJToY5NLUD4D5Q1Mcq+MnWtnjsuaHLEZjNoP896wsBCa\nyVvvTJb++3wDOqGFNm385in+mVJoZKrICCE1LMy9BlgIzaRDeKN5eHtTpGf1BOEZHrbnZWe7kF2I\ndYmBg1QV6WZOMWOErBeUN58naSE0lQ4hGoj31B1OGDpE9kon14Oql8gIGCe/lBYT7Iuv+Pp31sGJ\nkFdCW+mvr/gR6QdQxW6M0LsshGbSIRQz21n2iFqjxdfIQMt70Y7Asd469nIAJfegOWnN62j7i86F\nvt8NStc1LZJEY4fp+6TXkYWi5YHz3WBhfiLMPeS2MOWtiBCH9wYIAwaaH8+2Ys4LPyAVqDilTxx3\nQ7wQszfwf58D+iY81rTOBufp+FBEQyQ5Jr97xPc9puXQkj6/5CPCzAi6dZ1uQYSTyrzwQd9sURiF\n00havlxYEI6RtRemrI+LfaU4NFGDjQA5zS+7PEB5pITDUZHyqmhi0ZEm61/PR4TkDtElF289hDRR\nM4UJSwu5LPIQWnuSscVfL/upzndy4i3aqMHXb2GxLAeP/W8VIheuz7LmzqV2ew1NW4/sNlju7zOS\nPVRnvMsHhOhuwdUo6tZDSPMB0YXW1dpQX8kNe/KxbJaFj1ank+s+ao10MXtcPk6bqkeTT+aQlahi\n9dQbnI5gzRFedq3Oqpf0jjBRdw9Ctx7CcwjFdUEPnA4t3MBEqWEpMYqD5y40qlS+JJRTc382QJlr\n7OwM1bgmOQZKKy1DU4SdAJoZtw99qc6kzFnvGvnWQzjiw2eLf9aVMzxQNVx1OYhvvGcJl3YRZLuc\n7BKZLhr4GlevAG71Cue4vRlCmhXnZpJDshoVZnIilE7S+jCaI6/MmcJWxRPiCM5KuryJvywz6kJZ\n4fFpIDYKn514VnNBDO7mKTVbduK0FR56YvY6a7+ushCayYlQTiOMFCNLvC13tFHt7dTS+1q++YTP\n/KM0HxdNLP47mBeqXNxrXul7pfO6B5hYldlwvLCx8TF7hMc1Py2EZnIinCs4lRWlcI+wnifnsNLD\nCwhzQ9+xTgAADTRJREFUmRaqbXAFjKaskP6dcp0Nkz2vRG4bpdR6jQ7O/dVD756F0Eyad+GosCdF\n5fFQ9QgekzrTyDBGLs7UQSBsSl7u8P2I/ibV6UwVfLkj20SjKpNXWQjNpOudaQSy+b48PHwx9Ylt\nLaW4AHpmhEOs/rOdzILRN9B+6fiMHJR29ZAAY3/GlB8tN20pUdrJ9r7KQmgmHcKrdaEpVi5pbmjs\nirCwaSznWPJSWc6wGnIyjNdh6OmaeQQLEyHsdwmdCXkypyCn2rK3zZbtOOHVy4O7LIRm0iHMHlF3\nAlZLNikVUNIBifDbuj87aBgfe7zLyYOdAUoli6VAe9YzdN9Ac06lexlrpCK/EeJSy+ie/DrmenzS\nYfldhA8gfPsBuWTbUwVKzbwh4+xbk6V4tPOwug62P6TJtoUwvxGij+wSuLGnRnVcMOsxUbfE7rLX\nGit+vCAil10vrTpsJqkrISwySOJs4xhlsXoLYX4jxDUIVWehSOT52jQN7ODINXLhEa63+a0MFW0M\nLsdPNfulM3bue+lCz/jrp31TczJ+WgjzG2FpbePA9dl4SVm24KDs6KbGPLYShPHaPvSpAMnMSFgB\nwlJrIcxvhNh8WKHsyFn1mrllWaIReEE4TxfTwmqp5ZEWl6lonAqehH2sFsJ8t2DbP/m4up3dhSbA\nzNAev/y59L54drycn4QVGeHAiNoX05mh0IJmCLMQBt8IcX8UdHLplI4XA4kZythEZnOodoG2yNZ+\nkgzO6RJW87wz1tkqtHqohTD/EV5eo59zuQ0g0tA/SIbBSoL4TFWi45SYGPdTLIT5jRCX99S5H0Df\nvT2ZgVSEWZOGKaUtd9FUFXcz5ytSKwthfiNEZ1s1tEcRhKs5A0lFiG+58+7H0XtXHfdgC2EwEFbX\nHu3qqcdFQUi9Zz8YnDAuqpPBE9hCmN8IsUNba/pweMvsfsYuTxWENKne3AeVVhbCfK/OXFyhzPRb\nPuo4lsEyHlakVh+k0wAa+P6VLYTBa1Sgu3paK2aKcTRnjdTuz7LjFsLgIUR66A1RvxSZU0aNCh9k\nIQweQpz58nUHGgs0lIXQ39SCjnAVQLlr7I9VnsxcLIT+phb8DrZsz2ZoKAuhv6kVvBs9F1kI/U0t\nHxFe7VVlgvlMdyPpEKaOGWDQQWMkC2F+IGzoudppIi1CB/akGnqKdZOFMD8QgnFvptt5+oqNc7BJ\n+lCY6h7FQBbC/ECINr2DvZ42DaCl9nErEZ4rB2VOpCDCVcbxXGQhzA+EJ6pBK68W8jQHbakmQCJs\nQEMbuBiTiX9ljSyEBVYjpclOP2sCJEI0zYggTwr1jCO6yEJYcI0KdJCvdQoiEeJKMaPpYGXjeC6y\nEBYcwrQf+p9Vttd8/ZtDqc6s6r1UOFFf5tNlLIQh0bTHBX4G65v2e3/2tmSGlIUwJBBiFTbc6p3x\nN7VQQojLhla2EPqbWoEjTB7eWzESvhwGpY9YCP1NrcARhglDfKFMq5vb/9R8QhjfVP+ZhwhpSZCf\nNAEWQn9T8wVh2qCmus+8REjeLxY2gnqKtbaF0N/UfEA4IzqqqfaT5emD9FeADuiYRlnIzkLob2o+\nILweO7ep9pOZIzxTCZphxnhDuKo8jBK3QIbCintDU4SnY6C5cberhdD8Xbimqf7TFCEa0qOrNC8I\n6Qkqp1kMAqdrElOEMR4HPyyEfiGsWyJmYqJHIZpquJHg+RwudNoMo8V2fN+Iry+bnq25dg1fTvRV\nCeY36TFagKnlS7T4ABCePrLTZNktnHo0xe5t2S17FjpJOGG3ry1fboQ2PC3ZJBI6+PrF8EhGpmlq\nnnTLLLvl14M0vX9tWtbV27vwfNtG2xnLxoKlXYnA9EGa3q+2h9m+1oM0DxEq8q1GSguejdcEWDVS\nf1MraIS5pVwG5i2E/qZW0AjZ+dZ1N2j3LYT+plbgfaSuQoTb6rc87Wc0C2FIITzDn60l/URiISwA\nhFt7TrYbBHOEtLjHkbSRvX0zXiNZCIOPcAPH1NIgnCMkr8FpUfyf749TC2HwEbY1dj5C78IfIWwl\n9cSM8fmOLITBRzgYnN5jtVJqpNTm97j+pJsshMFHmF7VePkPtVEx38VS31wWwoKokRpVZnTtQuMT\njGUhDBLC/WWhioe1JBRZTXt/UwsuQhzbbVM17CuTr2Ih9De14CLEKgr6eB7nOZqF0N/Ugouwo/AH\nbNgglLIQ+ptacBHaf+64upa5bzULob+pBb9Ger1zhUnGR0gWQn9TC8Vu7kBkIbQQ+iULoZkshP6m\nZiEMRBZCM1kI/U3NQhiILIRmshD6m5qFMBBZCM1kIfQ3NQthILIQmslC6G9qFsJAZCE0k4XQ39Qs\nhIHIQmgmC6G/qVkIA5GF0EwWQn9TsxAGIguhmSyE/qZmIQxEFkIzWQj9Tc1CGIgshGayEPqbmoUw\nEFkIzWQh9Dc1C2EgshCayTvCP9v1incLtBAWIoTolaaM2wxuC+HNIowfNuqGS1B+IZyEs9vOuYZa\nCG8SYTLP1VI2fVh+IUSHNeCW8xbCm0S4HLN1lz4s396F3wD87hZoIbxJhOSvyWXhZKtG6qZQRsh6\nAAxzCQoAYeahNAuh36nlVY3UPQv9R3iEF+VNFkJ/UwulRkU9jjDKQuhvaqGEsJq1fmEgqYUSwjUc\n4TQLob+phRJCdmraAas643dqIYUQZSH0N7UAEc5yeFVOuvdzDJSeGlC0LFtA0ZIzAoqWFlAsxzV7\nfqRmDwhh3C8ZXpWe7P0cAyXfCChaampA0a4HdpNJAcXKSEzLj9RS8+dBuimm8mbrQeqqkHqQekEY\nhx1uF42P7Zl92SyqhdDf1PIH4SpEuMLw0DB+5KBJVAuhv6nlD8JLBt3eUnikvUlUC6G/qeVTo2JF\nVPRq43chaFYMNZKF0N/Ugt4unAtQ4pJJNAuhv6kFv2l/bb/N+ICQhdDf1KzemUBkITSThdDf1CyE\ngchCaCYLob+pWQgDkYXQTBZCf1OzEAYiC6GZLIT+pmYhDEQWQjNZCP1NzUIYiCyEZrIQ+puahTAQ\nFQWE3eZ71bxfvZ9joF9nBhRt9pyAos0I7CYDu8f50+bmR2rzAkKY1CXfVLty/l3bXdENgplaibb5\nctm+uQEgzEdN7RvM1DotCmZqpd3ml+ehQgfh/NHBTK3/6mCmVteDUV+eKHQQWgpQFsJCr4JFGN8U\n/2ePrdNitcFn3ip9eL06Y9KDldqV3tUbz3YEJbUCRZg2iBBOaPbHigoH3D/zVt3b7dvbbESQUnO0\n7X14Y9XVQUmtIBHOiI5ChFnVdzP2w0C3z7zVpajTjO2tYAtOamejkhgb2zco360gEV6PnYsIj0dn\nMba1rttn3upExxzGTkYlBCe144P4v8l9gvLdCvZduAYR7qzF/x2NznX9zPvkHGMaOYKWWvaZJsuC\n8t1CAOHaxvzfuagU1888Ty19VPTO4KVWL6qXIyiphQBC+cvMcf3M68QONmh0IHipsSv7OowKSmoh\ngPBYtI2xbbXdPvNY26J/zAxaavE4u+tA1ZxgpBYCCG3V9zI2brDbZ94qs/o8+gxOamta8jbhziq5\nwUgtBBCyn1uf2lp5v/tnnmpLxa27uOzBSe1qzKSTu5qPCsp3CwWEOWPrtFxr8JmnmhtFSg5Oamx3\nt+qNJmYG5btZfaSFXhbCQi8LYaGXhbDQy0JY6GUhLPQqSgjLQQW59TAsdAbrdtzVCODnfLyp/Nct\njzDrAYCy+XlX+a6iiTA11e4M1u24aTEA3JGQn7eV3yqaCHnBawSlGMt9CH6gUngHLKr5yJM9cxlz\nDH35iV4DoLoSqSaU/itMwK07YE274szW/d/3vDnRwdiZmk/e/fyXgU0uCKqKLMIVcFcq2wu3XZEI\nH0UHYpMZ6w1wJzytIky7DyZFQmncvAPKwcMsEu5843box7JehtseM/caFyIqsgizH4JFbAh8xiTC\nMolni0NNdvVe6GNfdZeKcDbcfnUC/2OI8NHRyzbAPZfYArj7+ma4+yIbAK8VzFfxR0UWIWsCLVhZ\nGKMgXMJYJ/5sXQIPZjNWR0VYET5mVwDGMUQ4nrG+8GyXLp0AttpuXFwz8EV4pmC+ij8qughXwTNZ\n99x2WUG4Hp+hpdgYeIsf768gTLoL+qamvgMlGSJcyzh4ock5zf8C975iIQyu9Ajtj8CP8ClzQTgH\nHslBUBLhFEns9jgmzukJUeLIDHhwjX26hTC4KgcRqSi7aAo2h2KAU210CE/dBoNyNtyjICwnEcJY\niXAxPHyFbX32+Ss9obgts7yFMLhSeKwWCNcC4HNUj5DVxxrpvRLh1TtgOH6+B59LhI5P4H5eI+3K\n5gH87e+3wRMF93V8VRFGaC8Gn2CwHqGt09PF2nwrEf4Id2JdlI3Fxgedw9LavXDvf3i7MLf3Px+p\nxX8FWwrq2/isooTQd/WAPDeQKzjdagj3ASxjOe/xtnuR0a2GMOdjKNbmY3g4tqBvJO90qyFkKR1f\nuvvpmscK+jbyULccwqInC2Ghl4Ww0MtCWOj1/8SiDMkSFGzQAAAAAElFTkSuQmCC\n" }, "metadata": {}, "output_type": "display_data" } ], "source": [ "%%R -w 450 -h 450 \n", "ggplot(train, aes(y=SalePrice, x=GrLivArea)) + \n", " geom_point() + \n", " theme_bw() + \n", " labs( y = \"Sales price\", x = 'Living Area', title =\"Homoscedasticity\") +\n", " theme(legend.position=\"none\", axis.text=element_text(size=14), axis.title=element_text(size=16,face=\"bold\"),\n", " plot.title = element_text(lineheight=1, face=\"bold\", size=20, hjust = 0.5))" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "We can clearly see that the shape of a cone does not exist anymore and the graph follow more the pattern of equal variances. Thus, we don't have to any data manipulation to make sure that the assumption of homoscedasticity is met. " ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "**Transforming Categorical Features** \n", "If I would simply label the categorical features, then that would imply some meaning in the distances between data points, that is not the case. Thus, I choose to make dummy features of all categorical features so that they will displayed as having some category or not. " ] }, { "cell_type": "code", "execution_count": 51, "metadata": { "collapsed": true }, "outputs": [], "source": [ "all_features = pd.get_dummies(all_features)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "**Scaling Features** \n", "Scaling all the numeric features so that they will have a better distribution" ] }, { "cell_type": "code", "execution_count": 52, "metadata": { "collapsed": true }, "outputs": [], "source": [ "from sklearn.preprocessing import StandardScaler\n", "\n", "scaler = StandardScaler()\n", "scaler.fit(all_features[numeric_feature_names])\n", "scaled = scaler.transform(all_features[numeric_feature_names])" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Variance Threshold\n", "Here, I'm selecting features for which most values are the same. If most values are the same for a certain feature, then it only adds noise and will likely make our model overfit the data. Thus, we drop the features that have a low variance (which significantly improved my score). " ] }, { "cell_type": "code", "execution_count": 53, "metadata": { "collapsed": true }, "outputs": [], "source": [ "from sklearn.feature_selection import VarianceThreshold" ] }, { "cell_type": "code", "execution_count": 54, "metadata": { "collapsed": true }, "outputs": [], "source": [ "variance = VarianceThreshold(0.01)\n", "all_features = variance.fit_transform(all_features)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Modeling" ] }, { "cell_type": "code", "execution_count": 55, "metadata": { "collapsed": true }, "outputs": [], "source": [ "import xgboost as xgb\n", "from sklearn.linear_model import Lasso, Ridge, ElasticNet\n", "from sklearn.kernel_ridge import KernelRidge\n", "from sklearn.ensemble import GradientBoostingRegressor\n", "from sklearn.model_selection import train_test_split\n", "from sklearn.metrics import mean_squared_error\n", "from sklearn.model_selection import GridSearchCV" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "**RMSE** \n", "The root mean squared error (RMSE) is a quadratic scoring real that measures the average magniute of the error. It takes the square root of the average of squared differences between prediction and actual observation. This is mostly used to measure the accuracy of continuous variables. The formula for RMSE is as follows: $RMSE = {\\sqrt {\\frac{1} {n}{\\sum\\limits_{j = 1}^n {(y_{j} - \\hat{y}_{i} } })^{2} } }$. One benefit of RMSE is that is penalizes large errors more, which can be helpfull in certain cases. If you are off by 10, then that is more than twice as bad as being off by 5. " ] }, { "cell_type": "code", "execution_count": 56, "metadata": { "collapsed": true }, "outputs": [], "source": [ "from sklearn.model_selection import cross_val_score\n", "\n", "def rmse_cross_validation(model):\n", " rmse= np.sqrt(-cross_val_score(model, X_train, y_train, scoring=\"neg_mean_squared_error\", cv = 5))\n", " return(rmse)\n", "\n", "def rmse(y_true, y_pred):\n", " return np.sqrt(mean_squared_error(y_true, y_pred))" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Because we concatenated the data in an earlier stage, we can simply use the shape of the train data to mark where we have added the test data so that we can split all_features into X_train and X_test. That way, we have the same data that was in the training dataset and in the testing dataset." ] }, { "cell_type": "code", "execution_count": 57, "metadata": { "collapsed": true }, "outputs": [], "source": [ "y_train = train['SalePrice']\n", "\n", "X_train = all_features[:train.shape[0]]\n", "X_test = all_features[train.shape[0]:]" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "**Ridge Regression** \n", "Ridge regression is a method that redues the size of coefficients in a linear regression using the L2 norm (see below) and thereby reduces the complexity in the model. Furthermore, it helps in overfitting and dealing with outliers. \n", "\n", "The L2 norm means that the weight (which you find in a linear regression) are penalized if they were to grow to large. This way you won't put too much emphasis on a few features, but you make sure the weights are similar. Therefore, it also helps with generalization and makes it more robust. \n", "\n", "**Lasso** \n", "Lasso (Least Absolute Shrinkage and Selection Operator) also uses regulization to prevent overfitting. In contrast, however, it can set the weights of coefficients to zero, whereas ridge regression can only bring those close to zero but never zero. Therefore, lasso often is a model that is less complex since it can fully remove features. In our case, it helps with the highly correlated predictors. \n", "\n", "**ElasticNet** \n", "Elastic net solves the limitations of lasso and ridge, while also including each as special cases. It uses both the L1 and the L2 norm. In layman-terms, it combines the penalties of lasso and ridge and it often prefered above both models. \n", "\n", "**Gradient Booster** \n", "The gradient booster basically takes many weak predictive models and boosts it into a stronger one, which represents an ensemble of weak models. Often, the weak model can be any model that works a little better than a random guess. \n", "\n", "**XGBoost** \n", "XGBoost is very similar to a gradient booster in that it follows gradient boosting. XGBoost uses a more regularized model formalization to control over-fitting, which gives it better performance." ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### GridSearch\n", "I use a grid search to find the best parameters for each model. Each model has its own parameter grid, which can be used to specify the parameters. I suggest you don't execute this part, seeing as it will take forever to find all the best parameters. " ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": true }, "outputs": [], "source": [ "def grid_search(param_grid, estimator):\n", " grid = GridSearchCV(estimator,param_grid,refit=True,verbose=3)\n", " grid.fit(X_train, y_train)\n", " print(grid.best_params_)\n", " print(grid.best_estimator_)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": true }, "outputs": [], "source": [ "param_grid_gradient = {'n_estimators': [500, 3000], \n", " 'learning_rate': np.arange(0.01, 0.1, 0.01), \n", " 'max_depth': [1, 3, 5, 7], \n", " 'min_samples_leaf': [10, 15, 20],\n", " 'min_samples_split': [10, 15, 20]}\n", "\n", "param_grid_xgb = {'colsample_bytree' :[0.2],\n", " 'gamma': [0.0],\n", " 'learning_rate': [0.01, 0.05, 0.1],\n", " 'min_child_weight': [1.5],\n", " 'n_estimators': np.arange(5000, 30000, 5000), \n", " 'reg_alpha': [0.9],\n", " 'reg_lambda': [0.6],\n", " 'subsample': [0.2, 0.3],\n", " 'max_depth': [1, 2, 3, 4]}\n", "\n", "param_grid_lasso = {'alpha':np.arange(0.0001, 0.001, 0.0001), \n", " 'max_iter': np.arange(10000, 50000, 10000)}\n", "\n", "param_grid_net = {'alpha': np.arange(0.0001, 0.001, 0.0001)}\n", "\n", "param_grid_ridge = {'alpha': np.arange(0.1, 0.9, 0.1), \n", " 'kernel': ['polynomial'], \n", " 'degree': [2], \n", " 'coef0':[1.85]} \n", "\n", "# grid_search(param_grid_gradient, GradientBoostingRegressor())\n", "# grid_search(param_grid_xgb, XGBRegressor())\n", "# grid_search(param_grid_lasso, Lasso())\n", "# grid_search(param_grid_net, ElasticNet())\n", "# grid_search(param_grid_ridge, KernelRidge())" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Training Models\n", "Here the models are trained and test on the X_train and X_test data to measure its performance. The parameters used are the ones I found using a grid search for each model. \n", "\n", "Based on the prediction scores, I devised a ensemble of multiple predictors using certain weights. I found dat XGB and ElasticNet gave me the best scores, so I increased the weight of those predictors so that those would dominate the output. All the other predictors are then used to slightly adjust the dominating predictors to avoid overfitting. " ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": true }, "outputs": [], "source": [ "# Gradient Boosting\n", "gradient = GradientBoostingRegressor(n_estimators = 3000, min_samples_leaf = 15, learning_rate = 0.05, max_features = 'sqrt',\n", " max_depth = 3, min_samples_split = 10, loss = 'huber')\n", "gradient.fit(X_train, y_train)\n", "y_pred_gradient = gradient.predict(X_train)\n", "\n", "# XGBoost\n", "xgb = xgb.XGBRegressor(n_estimators = 30000, colsample_bytree = 0.2, gamma = 0.0, reg_lambda = 0.6, min_child_weight = 1.5,\n", " reg_alpha = 0.9, learning_rate = 0.01, max_depth = 4, subsample = 0.2, seed = 42, silent = 1)\n", "xgb.fit(X_train, y_train)\n", "y_pred_xgb = xgb.predict(X_train)\n", "\n", "# Lasso\n", "lasso = Lasso(max_iter = 50000, alpha = 0.00098)\n", "lasso.fit(X_train, y_train)\n", "y_pred_lasso = lasso.predict(X_train)\n", "\n", "# ElasticNet\n", "net = ElasticNet(alpha=0.001)\n", "net.fit(X_train, y_train)\n", "y_pred_net = net.predict(X_train)\n", "\n", "# Ridge\n", "ridge = KernelRidge(alpha = 0.3, coef0 = 1.85, kernel = 'polynomial', degree = 2)\n", "ridge.fit(X_train, y_train)\n", "y_pred_ridge = ridge.predict(X_train)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": true }, "outputs": [], "source": [ "# Averaging scores using weights\n", "y_pred = (0.70*y_pred_xgb + 0.05*y_pred_lasso + 0.15*y_pred_net + 0.05*y_pred_ridge + 0.05*y_pred_gradient)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": true }, "outputs": [], "source": [ "# print('XGBoost CV RMSE: {}'.format(rmse_cross_validation(xgb)))\n", "# print('Lasso CV RMSE: {}'.format(rmse_cross_validation(lasso)))\n", "# print('ElasticNet CV RMSE: {}'.format(rmse_cross_validation(net)))\n", "# print('Ridge CV RMSE: {}'.format(rmse_cross_validation(ridge)))\n", "# print('Gradient CV RMSE: {}'.format(rmse_cross_validation(gradient)))\n", "# print('Averaging Ensemble RMSE: {}'.format(rmse(y_train, y_pred)))" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Submission\n", "Below is mostly the same code as above. The only change is that I predict the X_test instead of y so that I can submit my predictions. A file is created that is used for submission with the correct structure. It should be noted that since I've not set a seed, the results are somewhat randomized and the submission may slightly be different from the leaderboard score. " ] }, { "cell_type": "code", "execution_count": 20, "metadata": { "collapsed": true }, "outputs": [], "source": [ "# Lasso\n", "lasso = Lasso(max_iter = 50000, alpha = 0.00098)\n", "lasso.fit(X_train, y_train)\n", "y_pred_lasso = lasso.predict(X_test)\n", "\n", "# ElasticNet\n", "net = ElasticNet(alpha = 0.001)\n", "net.fit(X_train, y_train)\n", "y_pred_net = net.predict(X_test)\n", "\n", "# Ridge\n", "ridge = KernelRidge(alpha = 0.3, coef0 = 1.8, kernel = 'polynomial', degree = 2)\n", "ridge.fit(X_train, y_train)\n", "y_pred_ridge = ridge.predict(X_test)\n", "\n", "# Gradient boosting\n", "gradient = GradientBoostingRegressor(n_estimators = 3000, min_samples_leaf = 15, learning_rate = 0.05, max_features = 'sqrt',\n", " max_depth = 3, min_samples_split = 10, loss = 'huber', random_state = 42)\n", "gradient.fit(X_train, y_train)\n", "y_pred_gradient = gradient.predict(X_test)\n", "\n", "# XGBoost\n", "xgb = xgb.XGBRegressor(n_estimators = 35000, colsample_bytree = 0.2, reg_lambda = 0.6, min_child_weight = 1.5,\n", " reg_alpha = 0.9, learning_rate = 0.01, max_depth = 4, subsample = 0.2, seed = 42)\n", "xgb.fit(X_train, y_train)\n", "y_pred_xgb = xgb.predict(X_test)" ] }, { "cell_type": "code", "execution_count": 21, "metadata": { "collapsed": true }, "outputs": [], "source": [ "# Averaging Scores\n", "y_pred = (3*y_pred_xgb + y_pred_lasso + 3.6*y_pred_net + 1.5*y_pred_ridge + 2*y_pred_gradient) / 11.1\n", "\n", "# y_pred = (3*y_pred_xgb + y_pred_lasso + 3.6*y_pred_net + y_pred_ridge + 2*y_pred_gradient) / 10.6" ] }, { "cell_type": "code", "execution_count": 22, "metadata": { "collapsed": true }, "outputs": [], "source": [ "solution = pd.DataFrame({\"id\":test.Id, \"SalePrice\": np.expm1(y_pred)})\n", "# solution.to_csv(\"submission.csv\", index = False)" ] } ], "metadata": { "kernelspec": { "display_name": "Python [default]", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.5.4" }, "latex_envs": { "LaTeX_envs_menu_present": true, "bibliofile": "biblio.bib", "cite_by": "apalike", "current_citInitial": 1, "eqLabelWithNumbers": true, "eqNumInitial": 1, "labels_anchors": false, "latex_user_defs": false, "report_style_numbering": false, "user_envs_cfg": false } }, "nbformat": 4, "nbformat_minor": 2 }