{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Regularized linear models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Generating the data set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.random.seed(100)\n",
    "\n",
    "p = 20\n",
    "X = 3 * np.random.rand(p, 1)\n",
    "y = 1 + 0.5 * X + np.random.randn(p, 1) / 1.5\n",
    "X_new = np.linspace(0, 3, 100).reshape(100, 1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1. Ridge regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.preprocessing import PolynomialFeatures\n",
    "from sklearn.pipeline import Pipeline\n",
    "\n",
    "from sklearn.linear_model import LinearRegression\n",
    "from sklearn.linear_model import Ridge\n",
    "\n",
    "ridge_reg = Ridge(alpha=1, solver=\"cholesky\", random_state=100)\n",
    "ridge_reg.fit(X, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_model(model_class, polynomial, alphas, **model_kargs):\n",
    "    for alpha, style in zip(alphas, (\"b-\", \"g--\", \"r:\")):\n",
    "        model = model_class(alpha, **model_kargs) if alpha > 0 else LinearRegression()\n",
    "        if polynomial:\n",
    "            model = Pipeline([\n",
    "                    (\"poly_features\", PolynomialFeatures(degree=10, include_bias=False)),\n",
    "                    (\"std_scaler\", StandardScaler()),\n",
    "                    (\"regul_reg\", model),\n",
    "                ])\n",
    "        model.fit(X, y)\n",
    "        y_new_regul = model.predict(X_new)\n",
    "        lw = 2 if alpha > 0 else 1\n",
    "        plt.plot(X_new, y_new_regul, style, linewidth=lw, label=r\"$\\alpha = {}$\".format(alpha))\n",
    "    plt.plot(X, y, \"b.\", linewidth=3)\n",
    "    plt.legend(loc=\"upper left\", fontsize=15)\n",
    "    plt.xlabel(\"$x_1$\", fontsize=18)\n",
    "    plt.axis([0, 3, 0, 4])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure(figsize=(15,4))\n",
    "plt.subplot(121)\n",
    "plot_model(Ridge, polynomial=False, alphas=(0, 10, 100), random_state=100)\n",
    "plt.ylabel(\"$y$\", rotation=0, fontsize=18)\n",
    "plt.savefig('ridge_regression_model1.pdf')\n",
    "plt.subplot(122)\n",
    "plot_model(Ridge, polynomial=True, alphas=(0, 10**-5, 1), random_state=100)\n",
    "plt.savefig('figures/ridge_regression_polynomial.pdf')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Lasso"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.linear_model import Lasso\n",
    "\n",
    "plt.figure(figsize=(15,4))\n",
    "plt.subplot(121)\n",
    "plot_model(Lasso, polynomial=False, alphas=(0, 0.1, 1), random_state=42)\n",
    "plt.ylabel(\"$y$\", rotation=0, fontsize=18)\n",
    "plt.subplot(122)\n",
    "plot_model(Lasso, polynomial=True, alphas=(0, 10**-7, 1), random_state=42)\n",
    "plt.savefig('figures/lasso_regression_plot_sim.pdf')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Elastic Net"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.linear_model import ElasticNet\n",
    "elastic_net = ElasticNet(alpha=0.1, l1_ratio=0.5, random_state=100)\n",
    "elastic_net.fit(X, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure(figsize=(15,4))\n",
    "plt.subplot(121)\n",
    "plot_model(ElasticNet, polynomial=False, alphas=(0, 0.1, 1), random_state=42)\n",
    "plt.ylabel(\"$y$\", rotation=0, fontsize=18)\n",
    "plt.subplot(122)\n",
    "plot_model(ElasticNet, polynomial=True, alphas=(0, 10**-7, 1), random_state=42)\n",
    "plt.savefig('figures/elastic_net_plot_sim.pdf')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}