{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Hyperparameter search"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Import H2O and other libraries that will be used in this tutorial \n",
    "import h2o\n",
    "import matplotlib as plt\n",
    "\n",
    "#Import the Estimators\n",
    "from h2o.estimators.glm import H2OGeneralizedLinearEstimator\n",
    "from h2o.estimators import H2ORandomForestEstimator\n",
    "from h2o.estimators.gbm import H2OGradientBoostingEstimator\n",
    "\n",
    "#Import h2o grid search \n",
    "import h2o.grid \n",
    "from h2o.grid.grid_search import H2OGridSearch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Checking whether there is an H2O instance running at http://localhost:54321 . connected.\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div style=\"overflow:auto\"><table style=\"width:50%\"><tr><td>H2O cluster uptime:</td>\n",
       "<td>21 hours 35 mins</td></tr>\n",
       "<tr><td>H2O cluster timezone:</td>\n",
       "<td>Etc/UTC</td></tr>\n",
       "<tr><td>H2O data parsing timezone:</td>\n",
       "<td>UTC</td></tr>\n",
       "<tr><td>H2O cluster version:</td>\n",
       "<td>3.28.0.2</td></tr>\n",
       "<tr><td>H2O cluster version age:</td>\n",
       "<td>1 month and 14 days </td></tr>\n",
       "<tr><td>H2O cluster name:</td>\n",
       "<td>H2O_from_python_unknownUser_b8im2o</td></tr>\n",
       "<tr><td>H2O cluster total nodes:</td>\n",
       "<td>1</td></tr>\n",
       "<tr><td>H2O cluster free memory:</td>\n",
       "<td>2.931 Gb</td></tr>\n",
       "<tr><td>H2O cluster total cores:</td>\n",
       "<td>4</td></tr>\n",
       "<tr><td>H2O cluster allowed cores:</td>\n",
       "<td>4</td></tr>\n",
       "<tr><td>H2O cluster status:</td>\n",
       "<td>locked, healthy</td></tr>\n",
       "<tr><td>H2O connection url:</td>\n",
       "<td>http://localhost:54321</td></tr>\n",
       "<tr><td>H2O connection proxy:</td>\n",
       "<td>{'http': None, 'https': None}</td></tr>\n",
       "<tr><td>H2O internal security:</td>\n",
       "<td>False</td></tr>\n",
       "<tr><td>H2O API Extensions:</td>\n",
       "<td>Amazon S3, XGBoost, Algos, AutoML, Core V3, TargetEncoder, Core V4</td></tr>\n",
       "<tr><td>Python version:</td>\n",
       "<td>3.6.10 final</td></tr></table></div>"
      ],
      "text/plain": [
       "--------------------------  ------------------------------------------------------------------\n",
       "H2O cluster uptime:         21 hours 35 mins\n",
       "H2O cluster timezone:       Etc/UTC\n",
       "H2O data parsing timezone:  UTC\n",
       "H2O cluster version:        3.28.0.2\n",
       "H2O cluster version age:    1 month and 14 days\n",
       "H2O cluster name:           H2O_from_python_unknownUser_b8im2o\n",
       "H2O cluster total nodes:    1\n",
       "H2O cluster free memory:    2.931 Gb\n",
       "H2O cluster total cores:    4\n",
       "H2O cluster allowed cores:  4\n",
       "H2O cluster status:         locked, healthy\n",
       "H2O connection url:         http://localhost:54321\n",
       "H2O connection proxy:       {'http': None, 'https': None}\n",
       "H2O internal security:      False\n",
       "H2O API Extensions:         Amazon S3, XGBoost, Algos, AutoML, Core V3, TargetEncoder, Core V4\n",
       "Python version:             3.6.10 final\n",
       "--------------------------  ------------------------------------------------------------------"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import h2o\n",
    "h2o.init(max_mem_size=16)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Parse progress: |█████████████████████████████████████████████████████████| 100%\n"
     ]
    }
   ],
   "source": [
    "loan_level = h2o.import_file(\"https://s3.amazonaws.com/data.h2o.ai/DAI-Tutorials/loan_level_500k.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train:350268 valid:74971 test:74898\n"
     ]
    }
   ],
   "source": [
    "train, valid, test = loan_level.split_frame([0.7, 0.15], seed=42)\n",
    "print(\"train:%d valid:%d test:%d\" % (train.nrows, valid.nrows, test.nrows))\n",
    "y = \"DELINQUENT\"\n",
    "ignore = [\"DELINQUENT\", \"PREPAID\", \"PREPAYMENT_PENALTY_MORTGAGE_FLAG\", \"PRODUCT_TYPE\"] \n",
    "x = list(set(train.names) - set(ignore))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Grid Search/ Cartesian Search by default or not specified"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "glm Grid Build progress: |████████████████████████████████████████████████| 100%\n",
      "CPU times: user 755 ms, sys: 55.9 ms, total: 811 ms\n",
      "Wall time: 35.5 s\n"
     ]
    }
   ],
   "source": [
    "\n",
    "glm_grid = h2o.grid.H2OGridSearch (\n",
    "    H2OGeneralizedLinearEstimator( \n",
    "        family = \"binomial\",\n",
    "        lambda_search = True),\n",
    "    \n",
    "    hyper_params = {\n",
    "        \"alpha\": [x*0.01 for x in range(0, 4)],\n",
    "        \"lambda\": [x*1e-6 for x in range(0, 4)],\n",
    "        },\n",
    "    \n",
    "    grid_id = \"glm_grid_2\",\n",
    "    \n",
    ")\n",
    "%time glm_grid.train(x=x, y=y, training_frame=train, validation_frame = valid)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Random Search"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "glm Grid Build progress: |████████████████████████████████████████████████| 100%\n",
      "CPU times: user 4.73 s, sys: 504 ms, total: 5.23 s\n",
      "Wall time: 3min 26s\n"
     ]
    }
   ],
   "source": [
    "\n",
    "glm_grid = h2o.grid.H2OGridSearch (\n",
    "    H2OGeneralizedLinearEstimator( \n",
    "        family = \"binomial\",\n",
    "        lambda_search = True),\n",
    "    \n",
    "    hyper_params = {\n",
    "        \"alpha\": [x*0.01 for x in range(0, 100)],\n",
    "        \"lambda\": [x*1e-6 for x in range(0, 1000)],\n",
    "        },\n",
    "    \n",
    "    grid_id = \"glm_grid\",\n",
    "    \n",
    "    search_criteria = {\n",
    "        \"strategy\":\"RandomDiscrete\", \n",
    "        \"max_models\":100,\n",
    "        \"max_runtime_secs\":300,\n",
    "        \"seed\":42\n",
    "        }\n",
    ")\n",
    "%time glm_grid.train(x=x, y=y, training_frame=train, validation_frame = valid)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>key</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>GLM_model_python_1583377547966_1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>GLM_model_python_1583377547966_11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>GLM_model_python_1583377547966_14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>GLM_model_python_1583377547966_3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>GLM_model_python_1583377547966_5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>GLM_model_python_1583377547966_8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>glm_grid</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>glm_grid_2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>glm_grid_2_model_1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>glm_grid_2_model_10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>glm_grid_2_model_11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>glm_grid_2_model_12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>glm_grid_2_model_13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>glm_grid_2_model_14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>glm_grid_2_model_15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>glm_grid_2_model_16</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>glm_grid_2_model_2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>glm_grid_2_model_3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>glm_grid_2_model_4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>glm_grid_2_model_5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>glm_grid_2_model_6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>glm_grid_2_model_7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>glm_grid_2_model_8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>glm_grid_2_model_9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>glm_grid_model_1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>glm_grid_model_10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>glm_grid_model_100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>glm_grid_model_11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>glm_grid_model_12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>glm_grid_model_13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>358</th>\n",
       "      <td>modelmetrics_glm_grid_model_96@-6013562878500035200_on_py_16_sid_9...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>359</th>\n",
       "      <td>modelmetrics_glm_grid_model_97@-6125460407153251328_on_py_15_sid_9...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>360</th>\n",
       "      <td>modelmetrics_glm_grid_model_97@-6125460407153251328_on_py_16_sid_9...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>361</th>\n",
       "      <td>modelmetrics_glm_grid_model_98@-2192349973378340480_on_py_15_sid_9...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>362</th>\n",
       "      <td>modelmetrics_glm_grid_model_98@-2192349973378340480_on_py_16_sid_9...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>363</th>\n",
       "      <td>modelmetrics_glm_grid_model_99@-3110742583558476800_on_py_15_sid_9...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>364</th>\n",
       "      <td>modelmetrics_glm_grid_model_99@-3110742583558476800_on_py_16_sid_9...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>365</th>\n",
       "      <td>modelmetrics_glm_grid_model_9@9000166489408089856_on_py_15_sid_966...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>366</th>\n",
       "      <td>modelmetrics_glm_grid_model_9@9000166489408089856_on_py_16_sid_966...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>367</th>\n",
       "      <td>prostate.hex</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>368</th>\n",
       "      <td>prostate1.hex</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>369</th>\n",
       "      <td>prostate2.hex</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>370</th>\n",
       "      <td>prostate3.hex</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>371</th>\n",
       "      <td>prostate4.hex</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>372</th>\n",
       "      <td>prostate5.hex</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>373</th>\n",
       "      <td>prostate6.hex</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>374</th>\n",
       "      <td>prostate7.hex</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>375</th>\n",
       "      <td>py_15_sid_9664</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>376</th>\n",
       "      <td>py_16_sid_9664</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>377</th>\n",
       "      <td>py_17_sid_9664</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>378</th>\n",
       "      <td>py_1_sid_9fcd</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>379</th>\n",
       "      <td>py_27_sid_9317</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>380</th>\n",
       "      <td>py_28_sid_9317</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>381</th>\n",
       "      <td>py_29_sid_9317</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>382</th>\n",
       "      <td>py_2_sid_9fcd</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>383</th>\n",
       "      <td>py_30_sid_9317</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>384</th>\n",
       "      <td>py_31_sid_9317</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>385</th>\n",
       "      <td>py_32_sid_9317</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>386</th>\n",
       "      <td>py_3_sid_9fcd</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>387</th>\n",
       "      <td>py_4_sid_9fcd</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>388 rows × 1 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                       key\n",
       "0                                         GLM_model_python_1583377547966_1\n",
       "1                                        GLM_model_python_1583377547966_11\n",
       "2                                        GLM_model_python_1583377547966_14\n",
       "3                                         GLM_model_python_1583377547966_3\n",
       "4                                         GLM_model_python_1583377547966_5\n",
       "5                                         GLM_model_python_1583377547966_8\n",
       "6                                                                 glm_grid\n",
       "7                                                               glm_grid_2\n",
       "8                                                       glm_grid_2_model_1\n",
       "9                                                      glm_grid_2_model_10\n",
       "10                                                     glm_grid_2_model_11\n",
       "11                                                     glm_grid_2_model_12\n",
       "12                                                     glm_grid_2_model_13\n",
       "13                                                     glm_grid_2_model_14\n",
       "14                                                     glm_grid_2_model_15\n",
       "15                                                     glm_grid_2_model_16\n",
       "16                                                      glm_grid_2_model_2\n",
       "17                                                      glm_grid_2_model_3\n",
       "18                                                      glm_grid_2_model_4\n",
       "19                                                      glm_grid_2_model_5\n",
       "20                                                      glm_grid_2_model_6\n",
       "21                                                      glm_grid_2_model_7\n",
       "22                                                      glm_grid_2_model_8\n",
       "23                                                      glm_grid_2_model_9\n",
       "24                                                        glm_grid_model_1\n",
       "25                                                       glm_grid_model_10\n",
       "26                                                      glm_grid_model_100\n",
       "27                                                       glm_grid_model_11\n",
       "28                                                       glm_grid_model_12\n",
       "29                                                       glm_grid_model_13\n",
       "..                                                                     ...\n",
       "358  modelmetrics_glm_grid_model_96@-6013562878500035200_on_py_16_sid_9...\n",
       "359  modelmetrics_glm_grid_model_97@-6125460407153251328_on_py_15_sid_9...\n",
       "360  modelmetrics_glm_grid_model_97@-6125460407153251328_on_py_16_sid_9...\n",
       "361  modelmetrics_glm_grid_model_98@-2192349973378340480_on_py_15_sid_9...\n",
       "362  modelmetrics_glm_grid_model_98@-2192349973378340480_on_py_16_sid_9...\n",
       "363  modelmetrics_glm_grid_model_99@-3110742583558476800_on_py_15_sid_9...\n",
       "364  modelmetrics_glm_grid_model_99@-3110742583558476800_on_py_16_sid_9...\n",
       "365  modelmetrics_glm_grid_model_9@9000166489408089856_on_py_15_sid_966...\n",
       "366  modelmetrics_glm_grid_model_9@9000166489408089856_on_py_16_sid_966...\n",
       "367                                                           prostate.hex\n",
       "368                                                          prostate1.hex\n",
       "369                                                          prostate2.hex\n",
       "370                                                          prostate3.hex\n",
       "371                                                          prostate4.hex\n",
       "372                                                          prostate5.hex\n",
       "373                                                          prostate6.hex\n",
       "374                                                          prostate7.hex\n",
       "375                                                         py_15_sid_9664\n",
       "376                                                         py_16_sid_9664\n",
       "377                                                         py_17_sid_9664\n",
       "378                                                          py_1_sid_9fcd\n",
       "379                                                         py_27_sid_9317\n",
       "380                                                         py_28_sid_9317\n",
       "381                                                         py_29_sid_9317\n",
       "382                                                          py_2_sid_9fcd\n",
       "383                                                         py_30_sid_9317\n",
       "384                                                         py_31_sid_9317\n",
       "385                                                         py_32_sid_9317\n",
       "386                                                          py_3_sid_9fcd\n",
       "387                                                          py_4_sid_9fcd\n",
       "\n",
       "[388 rows x 1 columns]"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "h2o.ls()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Help on class H2OGridSearch in module h2o.grid.grid_search:\n",
      "\n",
      "class H2OGridSearch(H2OGridSearch)\n",
      " |  Grid Search of a Hyper-Parameter Space for a Model\n",
      " |  \n",
      " |  :param model: The type of model to be explored initialized with optional parameters that will be\n",
      " |      unchanged across explored models.\n",
      " |  :param hyper_params: A dictionary of string parameters (keys) and a list of values to be explored by grid\n",
      " |      search (values).\n",
      " |  :param str grid_id: The unique id assigned to the resulting grid object. If none is given, an id will\n",
      " |      automatically be generated.\n",
      " |  :param search_criteria:  The optional dictionary of directives which control the search of the hyperparameter space.\n",
      " |      The dictionary can include values for: ``strategy``, ``max_models``, ``max_runtime_secs``, ``stopping_metric``, \n",
      " |      ``stopping_tolerance``, ``stopping_rounds`` and ``seed``. The default strategy, \"Cartesian\", covers the entire space of \n",
      " |      hyperparameter combinations. If you want to use cartesian grid search, you can leave the search_criteria \n",
      " |      argument unspecified. Specify the \"RandomDiscrete\" strategy to get random search of all the combinations of \n",
      " |      your hyperparameters with three ways of specifying when to stop the search: max number of models, max time, and \n",
      " |      metric-based early stopping (e.g., stop if MSE hasn’t improved by 0.0001 over the 5 best models). \n",
      " |      Examples below::\n",
      " |  \n",
      " |          >>> criteria = {\"strategy\": \"RandomDiscrete\", \"max_runtime_secs\": 600,\n",
      " |          ...             \"max_models\": 100, \"stopping_metric\": \"AUTO\",\n",
      " |          ...             \"stopping_tolerance\": 0.00001, \"stopping_rounds\": 5,\n",
      " |          ...             \"seed\": 123456}\n",
      " |          >>> criteria = {\"strategy\": \"RandomDiscrete\", \"max_models\": 42,\n",
      " |          ...             \"max_runtime_secs\": 28800, \"seed\": 1234}\n",
      " |          >>> criteria = {\"strategy\": \"RandomDiscrete\", \"stopping_metric\": \"AUTO\",\n",
      " |          ...             \"stopping_tolerance\": 0.001, \"stopping_rounds\": 10}\n",
      " |          >>> criteria = {\"strategy\": \"RandomDiscrete\", \"stopping_rounds\": 5,\n",
      " |          ...             \"stopping_metric\": \"misclassification\",\n",
      " |          ...             \"stopping_tolerance\": 0.00001}\n",
      " |  :param parallelism: Level of parallelism during grid model building. 1 = sequential building (default). \n",
      " |       Use the value of 0 for adaptive parallelism - decided by H2O. Any number > 1 sets the exact number of models\n",
      " |       built in parallel.\n",
      " |  :returns: a new H2OGridSearch instance\n",
      " |  \n",
      " |  Examples\n",
      " |  --------\n",
      " |      >>> from h2o.grid.grid_search import H2OGridSearch\n",
      " |      >>> from h2o.estimators.glm import H2OGeneralizedLinearEstimator\n",
      " |      >>> hyper_parameters = {'alpha': [0.01,0.5], 'lambda': [1e-5,1e-6]}\n",
      " |      >>> gs = H2OGridSearch(H2OGeneralizedLinearEstimator(family='binomial'), hyper_parameters)\n",
      " |      >>> training_data = h2o.import_file(\"smalldata/logreg/benign.csv\")\n",
      " |      >>> gs.train(x=range(3) + range(4,11),y=3, training_frame=training_data)\n",
      " |      >>> gs.show()\n",
      " |  \n",
      " |  Method resolution order:\n",
      " |      H2OGridSearch\n",
      " |      H2OGridSearch\n",
      " |      builtins.object\n",
      " |  \n",
      " |  Methods defined here:\n",
      " |  \n",
      " |  __getattr__(self, name)\n",
      " |  \n",
      " |  __getitem__(self, item)\n",
      " |  \n",
      " |  __init__(self, *args, **kwargs)\n",
      " |  \n",
      " |  __iter__(self)\n",
      " |  \n",
      " |  __len__(self)\n",
      " |  \n",
      " |  __repr__(self)\n",
      " |      Return repr(self).\n",
      " |  \n",
      " |  aic(self, train=False, valid=False, xval=False)\n",
      " |      Get the AIC(s).\n",
      " |      \n",
      " |      If all are False (default), then return the training metric value.\n",
      " |      If more than one options is set to True, then return a dictionary of metrics where the keys are \"train\",\n",
      " |      \"valid\", and \"xval\".\n",
      " |      \n",
      " |      :param bool train: If train is True, then return the AIC value for the training data.\n",
      " |      :param bool valid: If valid is True, then return the AIC value for the validation data.\n",
      " |      :param bool xval:  If xval is True, then return the AIC value for the validation data.\n",
      " |      \n",
      " |      :returns: The AIC.\n",
      " |  \n",
      " |  auc(self, train=False, valid=False, xval=False)\n",
      " |      Get the AUC(s).\n",
      " |      \n",
      " |      If all are False (default), then return the training metric value.\n",
      " |      If more than one options is set to True, then return a dictionary of metrics where the keys are \"train\",\n",
      " |      \"valid\", and \"xval\".\n",
      " |      \n",
      " |      :param bool train: If train is True, then return the AUC value for the training data.\n",
      " |      :param bool valid: If valid is True, then return the AUC value for the validation data.\n",
      " |      :param bool xval:  If xval is True, then return the AUC value for the validation data.\n",
      " |      \n",
      " |      :returns: The AUC.\n",
      " |  \n",
      " |  aucpr(self, train=False, valid=False, xval=False)\n",
      " |      Get the aucPR (Area Under PRECISION RECALL Curve).\n",
      " |      \n",
      " |      If all are False (default), then return the training metric value.\n",
      " |      If more than one options is set to True, then return a dictionary of metrics where the keys are \"train\",\n",
      " |      \"valid\", and \"xval\".\n",
      " |      \n",
      " |      :param bool train: If train is True, then return the aucpr value for the training data.\n",
      " |      :param bool valid: If valid is True, then return the aucpr value for the validation data.\n",
      " |      :param bool xval:  If xval is True, then return the aucpr value for the validation data.\n",
      " |      \n",
      " |      :returns: The AUCPR for the models in this grid.\n",
      " |  \n",
      " |  biases(self, vector_id=0)\n",
      " |      Return the frame for the respective bias vector.\n",
      " |      \n",
      " |      :param: vector_id: an integer, ranging from 0 to number of layers, that specifies the bias vector to return.\n",
      " |      :returns: an H2OFrame which represents the bias vector identified by vector_id\n",
      " |  \n",
      " |  build_model(self, algo_params)\n",
      " |      (internal)\n",
      " |  \n",
      " |  catoffsets(self)\n",
      " |      Categorical offsets for one-hot encoding\n",
      " |  \n",
      " |  coef(self)\n",
      " |      Return the coefficients that can be applied to the non-standardized data.\n",
      " |      \n",
      " |      Note: standardize = True by default. If set to False, then coef() returns the coefficients that are fit directly.\n",
      " |  \n",
      " |  coef_norm(self)\n",
      " |      Return coefficients fitted on the standardized data (requires standardize = True, which is on by default). These coefficients can be used to evaluate variable importance.\n",
      " |  \n",
      " |  deepfeatures(self, test_data, layer)\n",
      " |      Obtain a hidden layer's details on a dataset.\n",
      " |      \n",
      " |      :param test_data: Data to create a feature space on.\n",
      " |      :param int layer: Index of the hidden layer.\n",
      " |      :returns: A dictionary of hidden layer details for each model.\n",
      " |  \n",
      " |  get_grid(self, sort_by=None, decreasing=None)\n",
      " |      Retrieve an H2OGridSearch instance.\n",
      " |      \n",
      " |      Optionally specify a metric by which to sort models and a sort order.\n",
      " |      Note that if neither cross-validation nor a validation frame is used in the grid search, then the\n",
      " |      training metrics will display in the \"get grid\" output. If a validation frame is passed to the grid, and\n",
      " |      ``nfolds = 0``, then the validation metrics will display. However, if ``nfolds`` > 1, then cross-validation\n",
      " |      metrics will display even if a validation frame is provided.\n",
      " |      \n",
      " |      :param str sort_by: A metric by which to sort the models in the grid space. Choices are: ``\"logloss\"``,\n",
      " |          ``\"residual_deviance\"``, ``\"mse\"``, ``\"auc\"``, ``\"r2\"``, ``\"accuracy\"``, ``\"precision\"``, ``\"recall\"``,\n",
      " |          ``\"f1\"``, etc.\n",
      " |      :param bool decreasing: Sort the models in decreasing order of metric if true, otherwise sort in increasing\n",
      " |          order (default).\n",
      " |      \n",
      " |      :returns: A new H2OGridSearch instance optionally sorted on the specified metric.\n",
      " |  \n",
      " |  get_hyperparams(self, id, display=True)\n",
      " |      Get the hyperparameters of a model explored by grid search.\n",
      " |      \n",
      " |      :param str id: The model id of the model with hyperparameters of interest.\n",
      " |      :param bool display: Flag to indicate whether to display the hyperparameter names.\n",
      " |      \n",
      " |      :returns: A list of the hyperparameters for the specified model.\n",
      " |  \n",
      " |  get_hyperparams_dict(self, id, display=True)\n",
      " |      Derived and returned the model parameters used to train the particular grid search model.\n",
      " |      \n",
      " |      :param str id: The model id of the model with hyperparameters of interest.\n",
      " |      :param bool display: Flag to indicate whether to display the hyperparameter names.\n",
      " |      \n",
      " |      :returns: A dict of model pararmeters derived from the hyper-parameters used to train this particular model.\n",
      " |  \n",
      " |  get_xval_models(self, key=None)\n",
      " |      Return a Model object.\n",
      " |      \n",
      " |      :param str key: If None, return all cross-validated models; otherwise return the model\n",
      " |          specified by the key.\n",
      " |      :returns: A model or a list of models.\n",
      " |  \n",
      " |  gini(self, train=False, valid=False, xval=False)\n",
      " |      Get the Gini Coefficient(s).\n",
      " |      \n",
      " |      If all are False (default), then return the training metric value.\n",
      " |      If more than one options is set to True, then return a dictionary of metrics where the keys are \"train\",\n",
      " |      \"valid\", and \"xval\".\n",
      " |      \n",
      " |      :param bool train: If train is True, then return the Gini Coefficient value for the training data.\n",
      " |      :param bool valid: If valid is True, then return the Gini Coefficient value for the validation data.\n",
      " |      :param bool xval:  If xval is True, then return the Gini Coefficient value for the cross validation data.\n",
      " |      \n",
      " |      :returns: The Gini Coefficient for the models in this grid.\n",
      " |  \n",
      " |  is_cross_validated(self)\n",
      " |      Return True if the model was cross-validated.\n",
      " |  \n",
      " |  join(self)\n",
      " |      Wait until grid finishes computing.\n",
      " |  \n",
      " |  logloss(self, train=False, valid=False, xval=False)\n",
      " |      Get the Log Loss(s).\n",
      " |      \n",
      " |      If all are False (default), then return the training metric value.\n",
      " |      If more than one options is set to True, then return a dictionary of metrics where the keys are \"train\",\n",
      " |      \"valid\", and \"xval\".\n",
      " |      \n",
      " |      :param bool train: If train is True, then return the Log Loss value for the training data.\n",
      " |      :param bool valid: If valid is True, then return the Log Loss value for the validation data.\n",
      " |      :param bool xval:  If xval is True, then return the Log Loss value for the cross validation data.\n",
      " |      \n",
      " |      :returns: The Log Loss for this binomial model.\n",
      " |  \n",
      " |  mae(self, train=False, valid=False, xval=False)\n",
      " |  \n",
      " |  mean_residual_deviance(self, train=False, valid=False, xval=False)\n",
      " |      Get the Mean Residual Deviances(s).\n",
      " |      \n",
      " |      If all are False (default), then return the training metric value.\n",
      " |      If more than one options is set to True, then return a dictionary of metrics where the keys are \"train\",\n",
      " |      \"valid\", and \"xval\".\n",
      " |      \n",
      " |      :param bool train: If train is True, then return the Mean Residual Deviance value for the training data.\n",
      " |      :param bool valid: If valid is True, then return the Mean Residual Deviance value for the validation data.\n",
      " |      :param bool xval:  If xval is True, then return the Mean Residual Deviance value for the cross validation data.\n",
      " |      :returns: The Mean Residual Deviance for this regression model.\n",
      " |  \n",
      " |  model_performance(self, test_data=None, train=False, valid=False, xval=False)\n",
      " |      Generate model metrics for this model on test_data.\n",
      " |      \n",
      " |      :param test_data: Data set for which model metrics shall be computed against. All three of train, valid\n",
      " |          and xval arguments are ignored if test_data is not None.\n",
      " |      :param train: Report the training metrics for the model.\n",
      " |      :param valid: Report the validation metrics for the model.\n",
      " |      :param xval: Report the validation metrics for the model.\n",
      " |      :return: An object of class H2OModelMetrics.\n",
      " |  \n",
      " |  mse(self, train=False, valid=False, xval=False)\n",
      " |      Get the MSE(s).\n",
      " |      \n",
      " |      If all are False (default), then return the training metric value.\n",
      " |      If more than one options is set to True, then return a dictionary of metrics where the keys are \"train\",\n",
      " |      \"valid\", and \"xval\".\n",
      " |      \n",
      " |      :param bool train: If train is True, then return the MSE value for the training data.\n",
      " |      :param bool valid: If valid is True, then return the MSE value for the validation data.\n",
      " |      :param bool xval:  If xval is True, then return the MSE value for the cross validation data.\n",
      " |      :returns: The MSE for this regression model.\n",
      " |  \n",
      " |  normmul(self)\n",
      " |      Normalization/Standardization multipliers for numeric predictors.\n",
      " |  \n",
      " |  normsub(self)\n",
      " |      Normalization/Standardization offsets for numeric predictors.\n",
      " |  \n",
      " |  null_degrees_of_freedom(self, train=False, valid=False, xval=False)\n",
      " |      Retreive the null degress of freedom if this model has the attribute, or None otherwise.\n",
      " |      \n",
      " |      :param bool train: Get the null dof for the training set. If both train and valid are False, then train is\n",
      " |          selected by default.\n",
      " |      :param bool valid: Get the null dof for the validation set. If both train and valid are True, then train is\n",
      " |          selected by default.\n",
      " |      :param bool xval: Get the null dof for the cross-validated models.\n",
      " |      \n",
      " |      :returns: the null dof, or None if it is not present.\n",
      " |  \n",
      " |  null_deviance(self, train=False, valid=False, xval=False)\n",
      " |      Retreive the null deviance if this model has the attribute, or None otherwise.\n",
      " |      \n",
      " |      :param bool train: Get the null deviance for the training set. If both train and valid are False, then\n",
      " |          train is selected by default.\n",
      " |      :param bool valid: Get the null deviance for the validation set. If both train and valid are True, then\n",
      " |          train is selected by default.\n",
      " |      :param bool xval: Get the null deviance for the cross-validated models.\n",
      " |      \n",
      " |      :returns: the null deviance, or None if it is not present.\n",
      " |  \n",
      " |  pprint_coef(self)\n",
      " |      Pretty print the coefficents table (includes normalized coefficients).\n",
      " |  \n",
      " |  pr_auc(self)\n",
      " |      H2OGridSearch.pr_auc is deprecated, please use ``H2OGridSearch.aucpr`` instead.\n",
      " |  \n",
      " |  predict(self, test_data)\n",
      " |      Predict on a dataset.\n",
      " |      \n",
      " |      :param H2OFrame test_data: Data to be predicted on.\n",
      " |      :returns: H2OFrame filled with predictions.\n",
      " |  \n",
      " |  r2(self, train=False, valid=False, xval=False)\n",
      " |      Return the R^2 for this regression model.\n",
      " |      \n",
      " |      The R^2 value is defined to be ``1 - MSE/var``, where ``var`` is computed as ``sigma^2``.\n",
      " |      \n",
      " |      If all are False (default), then return the training metric value.\n",
      " |      If more than one options is set to True, then return a dictionary of metrics where the keys are \"train\",\n",
      " |      \"valid\", and \"xval\".\n",
      " |      \n",
      " |      :param bool train: If train is True, then return the R^2 value for the training data.\n",
      " |      :param bool valid: If valid is True, then return the R^2 value for the validation data.\n",
      " |      :param bool xval:  If xval is True, then return the R^2 value for the cross validation data.\n",
      " |      \n",
      " |      :returns: The R^2 for this regression model.\n",
      " |  \n",
      " |  residual_degrees_of_freedom(self, train=False, valid=False, xval=False)\n",
      " |      Retreive the residual degress of freedom if this model has the attribute, or None otherwise.\n",
      " |      \n",
      " |      :param bool train: Get the residual dof for the training set. If both train and valid are False, then\n",
      " |          train is selected by default.\n",
      " |      :param bool valid: Get the residual dof for the validation set. If both train and valid are True, then\n",
      " |          train is selected by default.\n",
      " |      :param bool xval: Get the residual dof for the cross-validated models.\n",
      " |      \n",
      " |      :returns: the residual degrees of freedom, or None if they are not present.\n",
      " |  \n",
      " |  residual_deviance(self, train=False, valid=False, xval=False)\n",
      " |      Retreive the residual deviance if this model has the attribute, or None otherwise.\n",
      " |      \n",
      " |      :param bool train: Get the residual deviance for the training set. If both train and valid are False,\n",
      " |          then train is selected by default.\n",
      " |      :param bool valid: Get the residual deviance for the validation set. If both train and valid are True,\n",
      " |          then train is selected by default.\n",
      " |      :param bool xval: Get the residual deviance for the cross-validated models.\n",
      " |      \n",
      " |      :returns: the residual deviance, or None if it is not present.\n",
      " |  \n",
      " |  respmul(self)\n",
      " |      Normalization/Standardization multipliers for numeric response.\n",
      " |  \n",
      " |  respsub(self)\n",
      " |      Normalization/Standardization offsets for numeric response.\n",
      " |  \n",
      " |  rmse(self, train=False, valid=False, xval=False)\n",
      " |  \n",
      " |  rmsle(self, train=False, valid=False, xval=False)\n",
      " |  \n",
      " |  scoring_history(self)\n",
      " |      Retrieve model scoring history.\n",
      " |      \n",
      " |      :returns: Score history (H2OTwoDimTable)\n",
      " |  \n",
      " |  show(self)\n",
      " |      Print models sorted by metric.\n",
      " |  \n",
      " |  sort_by(self, metric, increasing=True)\n",
      " |      grid.sort_by() is deprecated; use grid.get_grid() instead\n",
      " |      \n",
      " |      Deprecated since 2016-12-12, use grid.get_grid() instead.\n",
      " |  \n",
      " |  sorted_metric_table(self)\n",
      " |      Retrieve summary table of an H2O Grid Search.\n",
      " |      \n",
      " |      :returns: The summary table as an H2OTwoDimTable or a Pandas DataFrame.\n",
      " |  \n",
      " |  start(self, x, y=None, training_frame=None, offset_column=None, fold_column=None, weights_column=None, validation_frame=None, **params)\n",
      " |      Asynchronous model build by specifying the predictor columns, response column, and any\n",
      " |      additional frame-specific values.\n",
      " |      \n",
      " |      To block for results, call :meth:`join`.\n",
      " |      \n",
      " |      :param x: A list of column names or indices indicating the predictor columns.\n",
      " |      :param y: An index or a column name indicating the response column.\n",
      " |      :param training_frame: The H2OFrame having the columns indicated by x and y (as well as any\n",
      " |          additional columns specified by fold, offset, and weights).\n",
      " |      :param offset_column: The name or index of the column in training_frame that holds the offsets.\n",
      " |      :param fold_column: The name or index of the column in training_frame that holds the per-row fold\n",
      " |          assignments.\n",
      " |      :param weights_column: The name or index of the column in training_frame that holds the per-row weights.\n",
      " |      :param validation_frame: H2OFrame with validation data to be scored on while training.\n",
      " |  \n",
      " |  summary(self, header=True)\n",
      " |      Print a detailed summary of the explored models.\n",
      " |  \n",
      " |  train(self, x=None, y=None, training_frame=None, offset_column=None, fold_column=None, weights_column=None, validation_frame=None, **params)\n",
      " |      Train the model synchronously (i.e. do not return until the model finishes training).\n",
      " |      \n",
      " |      To train asynchronously call :meth:`start`.\n",
      " |      \n",
      " |      :param x: A list of column names or indices indicating the predictor columns.\n",
      " |      :param y: An index or a column name indicating the response column.\n",
      " |      :param training_frame: The H2OFrame having the columns indicated by x and y (as well as any\n",
      " |          additional columns specified by fold, offset, and weights).\n",
      " |      :param offset_column: The name or index of the column in training_frame that holds the offsets.\n",
      " |      :param fold_column: The name or index of the column in training_frame that holds the per-row fold\n",
      " |          assignments.\n",
      " |      :param weights_column: The name or index of the column in training_frame that holds the per-row weights.\n",
      " |      :param validation_frame: H2OFrame with validation data to be scored on while training.\n",
      " |  \n",
      " |  varimp(self, use_pandas=False)\n",
      " |      Pretty print the variable importances, or return them in a list/pandas DataFrame.\n",
      " |      \n",
      " |      :param bool use_pandas: If True, then the variable importances will be returned as a pandas data frame.\n",
      " |      \n",
      " |      :returns: A dictionary of lists or Pandas DataFrame instances.\n",
      " |  \n",
      " |  weights(self, matrix_id=0)\n",
      " |      Return the frame for the respective weight matrix.\n",
      " |      \n",
      " |      :param: matrix_id: an integer, ranging from 0 to number of layers, that specifies the weight matrix to return.\n",
      " |      :returns: an H2OFrame which represents the weight matrix identified by matrix_id\n",
      " |  \n",
      " |  xval_keys(self)\n",
      " |      Model keys for the cross-validated model.\n",
      " |  \n",
      " |  xvals(self)\n",
      " |      Return the list of cross-validated models.\n",
      " |  \n",
      " |  ----------------------------------------------------------------------\n",
      " |  Data descriptors defined here:\n",
      " |  \n",
      " |  __dict__\n",
      " |      dictionary for instance variables (if defined)\n",
      " |  \n",
      " |  __weakref__\n",
      " |      list of weak references to the object (if defined)\n",
      " |  \n",
      " |  failed_params\n",
      " |  \n",
      " |  failed_raw_params\n",
      " |  \n",
      " |  failure_details\n",
      " |  \n",
      " |  failure_stack_traces\n",
      " |  \n",
      " |  grid_id\n",
      " |      A key that identifies this grid search object in H2O.\n",
      " |  \n",
      " |  hyper_names\n",
      " |  \n",
      " |  model_ids\n",
      "\n"
     ]
    }
   ],
   "source": [
    "help(h2o.grid.H2OGridSearch)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                       alpha                   lambda           model_ids  \\\n",
      "0                     [0.87]  [4.9999999999999996E-6]   glm_grid_model_61   \n",
      "1                      [0.4]                 [1.8E-5]   glm_grid_model_46   \n",
      "2                     [0.07]                 [3.7E-5]   glm_grid_model_48   \n",
      "3                     [0.07]                 [5.6E-5]   glm_grid_model_72   \n",
      "4                     [0.48]                 [2.9E-5]   glm_grid_model_96   \n",
      "5                     [0.88]                 [3.1E-5]   glm_grid_model_37   \n",
      "6                     [0.18]   [8.099999999999999E-5]   glm_grid_model_86   \n",
      "7                     [0.15]  [1.3099999999999999E-4]   glm_grid_model_30   \n",
      "8                      [0.1]                [1.59E-4]   glm_grid_model_38   \n",
      "9                     [0.06]                [2.12E-4]   glm_grid_model_78   \n",
      "10                    [0.53]   [9.499999999999999E-5]   glm_grid_model_28   \n",
      "11     [0.41000000000000003]                [1.37E-4]   glm_grid_model_67   \n",
      "12                    [0.65]                [1.07E-4]   glm_grid_model_34   \n",
      "13                    [0.93]                 [7.7E-5]  glm_grid_model_100   \n",
      "14                    [0.52]                [1.37E-4]   glm_grid_model_42   \n",
      "15                    [0.31]  [2.2999999999999998E-4]   glm_grid_model_22   \n",
      "16                    [0.56]                [1.64E-4]   glm_grid_model_82   \n",
      "17     [0.47000000000000003]  [1.8899999999999999E-4]   glm_grid_model_51   \n",
      "18                    [0.05]                [4.17E-4]   glm_grid_model_16   \n",
      "19                    [0.18]                [3.47E-4]   glm_grid_model_17   \n",
      "20                    [0.92]                [1.45E-4]   glm_grid_model_93   \n",
      "21                    [0.11]                [4.88E-4]   glm_grid_model_73   \n",
      "22                    [0.16]                [4.93E-4]   glm_grid_model_14   \n",
      "23                    [0.75]  [1.8099999999999998E-4]   glm_grid_model_11   \n",
      "24                    [0.09]                [6.68E-4]    glm_grid_model_1   \n",
      "25                    [0.78]                [1.88E-4]    glm_grid_model_2   \n",
      "26                    [0.07]                [7.32E-4]   glm_grid_model_79   \n",
      "27                    [0.17]   [5.459999999999999E-4]   glm_grid_model_98   \n",
      "28                    [0.65]                [2.26E-4]   glm_grid_model_83   \n",
      "29                    [0.02]                [8.87E-4]   glm_grid_model_92   \n",
      ".. ..                    ...                      ...                 ...   \n",
      "70                    [0.55]   [6.129999999999999E-4]   glm_grid_model_59   \n",
      "71                    [0.86]                [4.86E-4]   glm_grid_model_21   \n",
      "72                    [0.63]                [5.75E-4]   glm_grid_model_49   \n",
      "73                     [0.5]   [6.929999999999999E-4]    glm_grid_model_6   \n",
      "74                    [0.73]                [5.53E-4]   glm_grid_model_13   \n",
      "75                    [0.65]                [5.88E-4]   glm_grid_model_89   \n",
      "76                    [0.87]   [5.099999999999999E-4]   glm_grid_model_19   \n",
      "77                    [0.32]                 [9.5E-4]   glm_grid_model_95   \n",
      "78                    [0.88]                [5.18E-4]   glm_grid_model_63   \n",
      "79                    [0.42]   [8.129999999999999E-4]   glm_grid_model_74   \n",
      "80                    [0.43]                [8.42E-4]   glm_grid_model_40   \n",
      "81                    [0.66]                [6.59E-4]   glm_grid_model_18   \n",
      "82                    [0.59]                [7.31E-4]    glm_grid_model_9   \n",
      "83     [0.41000000000000003]                [8.91E-4]   glm_grid_model_24   \n",
      "84                    [0.54]                 [7.8E-4]    glm_grid_model_5   \n",
      "85     [0.41000000000000003]   [9.559999999999999E-4]   glm_grid_model_76   \n",
      "86      [0.9400000000000001]                [5.09E-4]   glm_grid_model_62   \n",
      "87     [0.41000000000000003]                [9.93E-4]   glm_grid_model_50   \n",
      "88                     [0.6]                [8.16E-4]   glm_grid_model_32   \n",
      "89                    [0.55]   [8.759999999999999E-4]   glm_grid_model_97   \n",
      "90                    [0.52]                [9.45E-4]   glm_grid_model_91   \n",
      "91                    [0.84]                [6.74E-4]   glm_grid_model_57   \n",
      "92      [0.5700000000000001]                [9.53E-4]   glm_grid_model_69   \n",
      "93                    [0.65]   [9.379999999999999E-4]   glm_grid_model_56   \n",
      "94      [0.9500000000000001]                [7.41E-4]   glm_grid_model_85   \n",
      "95                    [0.86]                [8.06E-4]   glm_grid_model_39   \n",
      "96                    [0.92]                 [8.6E-4]   glm_grid_model_99   \n",
      "97                    [0.74]                [9.87E-4]   glm_grid_model_15   \n",
      "98                    [0.88]   [9.209999999999999E-4]   glm_grid_model_23   \n",
      "99                    [0.88]                [9.48E-4]    glm_grid_model_7   \n",
      "\n",
      "                   auc  \n",
      "0   0.8460502420206815  \n",
      "1   0.8459231218911992  \n",
      "2   0.8458460074966513  \n",
      "3        0.84578969349  \n",
      "4    0.845718421725669  \n",
      "5   0.8454975463710519  \n",
      "6   0.8454049383408971  \n",
      "7    0.845125597663872  \n",
      "8    0.844990163622469  \n",
      "9   0.8447922633836474  \n",
      "10  0.8446763576704011  \n",
      "11  0.8443068880489722  \n",
      "12  0.8442450331686066  \n",
      "13  0.8441362707601581  \n",
      "14  0.8440820898598193  \n",
      "15  0.8436148511314989  \n",
      "16  0.8436089096500138  \n",
      "17  0.8435708987360638  \n",
      "18   0.843518778104261  \n",
      "19  0.8432349813161218  \n",
      "20  0.8426307867571391  \n",
      "21  0.8426180947745795  \n",
      "22  0.8421399824067635  \n",
      "23  0.8416523933239516  \n",
      "24  0.8416254797655253  \n",
      "25  0.8415763740974008  \n",
      "26   0.841510364862424  \n",
      "27  0.8414721822594293  \n",
      "28  0.8414650545627264  \n",
      "29  0.8413966312757275  \n",
      "..                 ...  \n",
      "70  0.8367248032825947  \n",
      "71   0.836457462629252  \n",
      "72   0.836414769286811  \n",
      "73  0.8362693772808174  \n",
      "74  0.8362422374050139  \n",
      "75  0.8362208751256985  \n",
      "76  0.8361702996952978  \n",
      "77   0.835997372408428  \n",
      "78   0.835987266166935  \n",
      "79  0.8359603526085087  \n",
      "80  0.8355779204642072  \n",
      "81  0.8355752566826656  \n",
      "82  0.8353492462648034  \n",
      "83  0.8353201996000644  \n",
      "84  0.8352437797657427  \n",
      "85  0.8348899104352362  \n",
      "86   0.834719503855704  \n",
      "87  0.8346606457295727  \n",
      "88  0.8339922654813755  \n",
      "89  0.8338713885898811  \n",
      "90  0.8336982271815869  \n",
      "91  0.8335408845775446  \n",
      "92  0.8330923235362049  \n",
      "93  0.8328250505179406  \n",
      "94  0.8325835828833598  \n",
      "95  0.8324377694588019  \n",
      "96  0.8318016979671923  \n",
      "97  0.8317961336813356  \n",
      "98  0.8315479207483787  \n",
      "99  0.8313888092269064  \n",
      "\n",
      "[100 rows x 5 columns]\n"
     ]
    },
    {
     "data": {
      "text/plain": []
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "glm_grid.get_grid(sort_by='auc',decreasing=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model Details\n",
      "=============\n",
      "H2OGeneralizedLinearEstimator :  Generalized Linear Modeling\n",
      "Model Key:  glm_grid_model_61\n",
      "\n",
      "\n",
      "GLM Model: summary\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>family</th>\n",
       "      <th>link</th>\n",
       "      <th>regularization</th>\n",
       "      <th>lambda_search</th>\n",
       "      <th>number_of_predictors_total</th>\n",
       "      <th>number_of_active_predictors</th>\n",
       "      <th>number_of_iterations</th>\n",
       "      <th>training_frame</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td></td>\n",
       "      <td>binomial</td>\n",
       "      <td>logit</td>\n",
       "      <td>Elastic Net (alpha = 0.87, lambda = 5.0E-6 )</td>\n",
       "      <td>nlambda = 100, lambda.max = 0.03808, lambda.min = 5.0E-6, lambda.1...</td>\n",
       "      <td>161</td>\n",
       "      <td>143</td>\n",
       "      <td>7</td>\n",
       "      <td>py_15_sid_9664</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       family   link                                regularization  \\\n",
       "0    binomial  logit  Elastic Net (alpha = 0.87, lambda = 5.0E-6 )   \n",
       "\n",
       "                                                           lambda_search  \\\n",
       "0  nlambda = 100, lambda.max = 0.03808, lambda.min = 5.0E-6, lambda.1...   \n",
       "\n",
       "   number_of_predictors_total number_of_active_predictors  \\\n",
       "0                         161                         143   \n",
       "\n",
       "   number_of_iterations  training_frame  \n",
       "0                     7  py_15_sid_9664  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "ModelMetricsBinomialGLM: glm\n",
      "** Reported on train data. **\n",
      "\n",
      "MSE: 0.031344275605674536\n",
      "RMSE: 0.17704314616972477\n",
      "LogLoss: 0.12279979897845819\n",
      "Null degrees of freedom: 350267\n",
      "Residual degrees of freedom: 350124\n",
      "Null deviance: 108932.13150368733\n",
      "Residual deviance: 86025.67997717319\n",
      "AIC: 86313.67997717319\n",
      "AUC: 0.8519842670925402\n",
      "AUCPR: 0.21046685420921254\n",
      "Gini: 0.7039685341850803\n",
      "\n",
      "Confusion Matrix (Act/Pred) for max f1 @ threshold = 0.13993235618594693: \n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>FALSE</th>\n",
       "      <th>TRUE</th>\n",
       "      <th>Error</th>\n",
       "      <th>Rate</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>FALSE</td>\n",
       "      <td>323805.0</td>\n",
       "      <td>13802.0</td>\n",
       "      <td>0.0409</td>\n",
       "      <td>(13802.0/337607.0)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>TRUE</td>\n",
       "      <td>8129.0</td>\n",
       "      <td>4532.0</td>\n",
       "      <td>0.6421</td>\n",
       "      <td>(8129.0/12661.0)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Total</td>\n",
       "      <td>331934.0</td>\n",
       "      <td>18334.0</td>\n",
       "      <td>0.0626</td>\n",
       "      <td>(21931.0/350268.0)</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             FALSE     TRUE   Error                 Rate\n",
       "0  FALSE  323805.0  13802.0  0.0409   (13802.0/337607.0)\n",
       "1   TRUE    8129.0   4532.0  0.6421     (8129.0/12661.0)\n",
       "2  Total  331934.0  18334.0  0.0626   (21931.0/350268.0)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Maximum Metrics: Maximum metrics at their respective thresholds\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>metric</th>\n",
       "      <th>threshold</th>\n",
       "      <th>value</th>\n",
       "      <th>idx</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>max f1</td>\n",
       "      <td>0.139932</td>\n",
       "      <td>0.292434</td>\n",
       "      <td>200.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>max f2</td>\n",
       "      <td>0.068698</td>\n",
       "      <td>0.389555</td>\n",
       "      <td>264.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>max f0point5</td>\n",
       "      <td>0.212665</td>\n",
       "      <td>0.290398</td>\n",
       "      <td>157.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>max accuracy</td>\n",
       "      <td>0.981772</td>\n",
       "      <td>0.963851</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>max precision</td>\n",
       "      <td>0.562303</td>\n",
       "      <td>0.421203</td>\n",
       "      <td>45.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>max recall</td>\n",
       "      <td>0.000789</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>398.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>max specificity</td>\n",
       "      <td>0.981772</td>\n",
       "      <td>0.999997</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>max absolute_mcc</td>\n",
       "      <td>0.099058</td>\n",
       "      <td>0.269921</td>\n",
       "      <td>234.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>max min_per_class_accuracy</td>\n",
       "      <td>0.038519</td>\n",
       "      <td>0.774240</td>\n",
       "      <td>305.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>max mean_per_class_accuracy</td>\n",
       "      <td>0.036145</td>\n",
       "      <td>0.775428</td>\n",
       "      <td>309.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>max tns</td>\n",
       "      <td>0.981772</td>\n",
       "      <td>337606.000000</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>max fns</td>\n",
       "      <td>0.981772</td>\n",
       "      <td>12661.000000</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>max fps</td>\n",
       "      <td>0.000498</td>\n",
       "      <td>337607.000000</td>\n",
       "      <td>399.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>max tps</td>\n",
       "      <td>0.000789</td>\n",
       "      <td>12661.000000</td>\n",
       "      <td>398.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>max tnr</td>\n",
       "      <td>0.981772</td>\n",
       "      <td>0.999997</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>max fnr</td>\n",
       "      <td>0.981772</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>max fpr</td>\n",
       "      <td>0.000498</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>399.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>max tpr</td>\n",
       "      <td>0.000789</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>398.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                         metric  threshold          value    idx\n",
       "0                        max f1   0.139932       0.292434  200.0\n",
       "1                        max f2   0.068698       0.389555  264.0\n",
       "2                  max f0point5   0.212665       0.290398  157.0\n",
       "3                  max accuracy   0.981772       0.963851    0.0\n",
       "4                 max precision   0.562303       0.421203   45.0\n",
       "5                    max recall   0.000789       1.000000  398.0\n",
       "6               max specificity   0.981772       0.999997    0.0\n",
       "7              max absolute_mcc   0.099058       0.269921  234.0\n",
       "8    max min_per_class_accuracy   0.038519       0.774240  305.0\n",
       "9   max mean_per_class_accuracy   0.036145       0.775428  309.0\n",
       "10                      max tns   0.981772  337606.000000    0.0\n",
       "11                      max fns   0.981772   12661.000000    0.0\n",
       "12                      max fps   0.000498  337607.000000  399.0\n",
       "13                      max tps   0.000789   12661.000000  398.0\n",
       "14                      max tnr   0.981772       0.999997    0.0\n",
       "15                      max fnr   0.981772       1.000000    0.0\n",
       "16                      max fpr   0.000498       1.000000  399.0\n",
       "17                      max tpr   0.000789       1.000000  398.0"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Gains/Lift Table: Avg response rate:  3.61 %, avg score:  3.61 %\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>group</th>\n",
       "      <th>cumulative_data_fraction</th>\n",
       "      <th>lower_threshold</th>\n",
       "      <th>lift</th>\n",
       "      <th>cumulative_lift</th>\n",
       "      <th>response_rate</th>\n",
       "      <th>score</th>\n",
       "      <th>cumulative_response_rate</th>\n",
       "      <th>cumulative_score</th>\n",
       "      <th>capture_rate</th>\n",
       "      <th>cumulative_capture_rate</th>\n",
       "      <th>gain</th>\n",
       "      <th>cumulative_gain</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td></td>\n",
       "      <td>1</td>\n",
       "      <td>0.010001</td>\n",
       "      <td>0.332250</td>\n",
       "      <td>10.724871</td>\n",
       "      <td>10.724871</td>\n",
       "      <td>0.387668</td>\n",
       "      <td>0.466160</td>\n",
       "      <td>0.387668</td>\n",
       "      <td>0.466160</td>\n",
       "      <td>0.107259</td>\n",
       "      <td>0.107259</td>\n",
       "      <td>972.487123</td>\n",
       "      <td>972.487123</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td></td>\n",
       "      <td>2</td>\n",
       "      <td>0.020002</td>\n",
       "      <td>0.241718</td>\n",
       "      <td>7.629032</td>\n",
       "      <td>9.176952</td>\n",
       "      <td>0.275764</td>\n",
       "      <td>0.280435</td>\n",
       "      <td>0.331716</td>\n",
       "      <td>0.373297</td>\n",
       "      <td>0.076297</td>\n",
       "      <td>0.183556</td>\n",
       "      <td>662.903211</td>\n",
       "      <td>817.695167</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td></td>\n",
       "      <td>3</td>\n",
       "      <td>0.030003</td>\n",
       "      <td>0.195284</td>\n",
       "      <td>6.452297</td>\n",
       "      <td>8.268734</td>\n",
       "      <td>0.233229</td>\n",
       "      <td>0.216827</td>\n",
       "      <td>0.298887</td>\n",
       "      <td>0.321140</td>\n",
       "      <td>0.064529</td>\n",
       "      <td>0.248085</td>\n",
       "      <td>545.229734</td>\n",
       "      <td>726.873356</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td></td>\n",
       "      <td>4</td>\n",
       "      <td>0.040001</td>\n",
       "      <td>0.165076</td>\n",
       "      <td>5.166472</td>\n",
       "      <td>7.493334</td>\n",
       "      <td>0.186750</td>\n",
       "      <td>0.179199</td>\n",
       "      <td>0.270859</td>\n",
       "      <td>0.285663</td>\n",
       "      <td>0.051655</td>\n",
       "      <td>0.299739</td>\n",
       "      <td>416.647177</td>\n",
       "      <td>649.333418</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td></td>\n",
       "      <td>5</td>\n",
       "      <td>0.050002</td>\n",
       "      <td>0.143363</td>\n",
       "      <td>4.770119</td>\n",
       "      <td>6.948660</td>\n",
       "      <td>0.172424</td>\n",
       "      <td>0.153562</td>\n",
       "      <td>0.251170</td>\n",
       "      <td>0.259241</td>\n",
       "      <td>0.047706</td>\n",
       "      <td>0.347445</td>\n",
       "      <td>377.011946</td>\n",
       "      <td>594.866013</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td></td>\n",
       "      <td>6</td>\n",
       "      <td>0.100001</td>\n",
       "      <td>0.087455</td>\n",
       "      <td>3.342624</td>\n",
       "      <td>5.145694</td>\n",
       "      <td>0.120825</td>\n",
       "      <td>0.110895</td>\n",
       "      <td>0.185999</td>\n",
       "      <td>0.185070</td>\n",
       "      <td>0.167127</td>\n",
       "      <td>0.514572</td>\n",
       "      <td>234.262433</td>\n",
       "      <td>414.569371</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td></td>\n",
       "      <td>7</td>\n",
       "      <td>0.150002</td>\n",
       "      <td>0.062108</td>\n",
       "      <td>2.397833</td>\n",
       "      <td>4.229723</td>\n",
       "      <td>0.086674</td>\n",
       "      <td>0.073579</td>\n",
       "      <td>0.152890</td>\n",
       "      <td>0.147906</td>\n",
       "      <td>0.119896</td>\n",
       "      <td>0.634468</td>\n",
       "      <td>139.783271</td>\n",
       "      <td>322.972261</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td></td>\n",
       "      <td>8</td>\n",
       "      <td>0.200001</td>\n",
       "      <td>0.047315</td>\n",
       "      <td>1.600226</td>\n",
       "      <td>3.572367</td>\n",
       "      <td>0.057843</td>\n",
       "      <td>0.054179</td>\n",
       "      <td>0.129129</td>\n",
       "      <td>0.124475</td>\n",
       "      <td>0.080009</td>\n",
       "      <td>0.714478</td>\n",
       "      <td>60.022611</td>\n",
       "      <td>257.236725</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td></td>\n",
       "      <td>9</td>\n",
       "      <td>0.300002</td>\n",
       "      <td>0.030452</td>\n",
       "      <td>1.121548</td>\n",
       "      <td>2.755427</td>\n",
       "      <td>0.040540</td>\n",
       "      <td>0.037965</td>\n",
       "      <td>0.099599</td>\n",
       "      <td>0.095638</td>\n",
       "      <td>0.112155</td>\n",
       "      <td>0.826633</td>\n",
       "      <td>12.154798</td>\n",
       "      <td>175.542749</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td></td>\n",
       "      <td>10</td>\n",
       "      <td>0.399999</td>\n",
       "      <td>0.020842</td>\n",
       "      <td>0.645303</td>\n",
       "      <td>2.227908</td>\n",
       "      <td>0.023326</td>\n",
       "      <td>0.025266</td>\n",
       "      <td>0.080531</td>\n",
       "      <td>0.078045</td>\n",
       "      <td>0.064529</td>\n",
       "      <td>0.891162</td>\n",
       "      <td>-35.469658</td>\n",
       "      <td>122.790777</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td></td>\n",
       "      <td>11</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.014542</td>\n",
       "      <td>0.399650</td>\n",
       "      <td>1.862254</td>\n",
       "      <td>0.014446</td>\n",
       "      <td>0.017478</td>\n",
       "      <td>0.067314</td>\n",
       "      <td>0.065932</td>\n",
       "      <td>0.039965</td>\n",
       "      <td>0.931127</td>\n",
       "      <td>-60.034981</td>\n",
       "      <td>86.225417</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td></td>\n",
       "      <td>12</td>\n",
       "      <td>0.600001</td>\n",
       "      <td>0.010215</td>\n",
       "      <td>0.280387</td>\n",
       "      <td>1.598608</td>\n",
       "      <td>0.010135</td>\n",
       "      <td>0.012267</td>\n",
       "      <td>0.057784</td>\n",
       "      <td>0.056988</td>\n",
       "      <td>0.028039</td>\n",
       "      <td>0.959166</td>\n",
       "      <td>-71.961301</td>\n",
       "      <td>59.860838</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td></td>\n",
       "      <td>13</td>\n",
       "      <td>0.699998</td>\n",
       "      <td>0.007058</td>\n",
       "      <td>0.198251</td>\n",
       "      <td>1.398561</td>\n",
       "      <td>0.007166</td>\n",
       "      <td>0.008555</td>\n",
       "      <td>0.050553</td>\n",
       "      <td>0.050069</td>\n",
       "      <td>0.019825</td>\n",
       "      <td>0.978991</td>\n",
       "      <td>-80.174889</td>\n",
       "      <td>39.856142</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td></td>\n",
       "      <td>14</td>\n",
       "      <td>0.799999</td>\n",
       "      <td>0.004607</td>\n",
       "      <td>0.116104</td>\n",
       "      <td>1.238253</td>\n",
       "      <td>0.004197</td>\n",
       "      <td>0.005790</td>\n",
       "      <td>0.044759</td>\n",
       "      <td>0.044534</td>\n",
       "      <td>0.011610</td>\n",
       "      <td>0.990601</td>\n",
       "      <td>-88.389609</td>\n",
       "      <td>23.825309</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td></td>\n",
       "      <td>15</td>\n",
       "      <td>0.899999</td>\n",
       "      <td>0.002610</td>\n",
       "      <td>0.057657</td>\n",
       "      <td>1.107075</td>\n",
       "      <td>0.002084</td>\n",
       "      <td>0.003583</td>\n",
       "      <td>0.040017</td>\n",
       "      <td>0.039984</td>\n",
       "      <td>0.005766</td>\n",
       "      <td>0.996367</td>\n",
       "      <td>-94.234296</td>\n",
       "      <td>10.707492</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td></td>\n",
       "      <td>16</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000095</td>\n",
       "      <td>0.036332</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.001313</td>\n",
       "      <td>0.001619</td>\n",
       "      <td>0.036147</td>\n",
       "      <td>0.036147</td>\n",
       "      <td>0.003633</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>-96.366816</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      group  cumulative_data_fraction  lower_threshold       lift  \\\n",
       "0         1                  0.010001         0.332250  10.724871   \n",
       "1         2                  0.020002         0.241718   7.629032   \n",
       "2         3                  0.030003         0.195284   6.452297   \n",
       "3         4                  0.040001         0.165076   5.166472   \n",
       "4         5                  0.050002         0.143363   4.770119   \n",
       "5         6                  0.100001         0.087455   3.342624   \n",
       "6         7                  0.150002         0.062108   2.397833   \n",
       "7         8                  0.200001         0.047315   1.600226   \n",
       "8         9                  0.300002         0.030452   1.121548   \n",
       "9        10                  0.399999         0.020842   0.645303   \n",
       "10       11                  0.500000         0.014542   0.399650   \n",
       "11       12                  0.600001         0.010215   0.280387   \n",
       "12       13                  0.699998         0.007058   0.198251   \n",
       "13       14                  0.799999         0.004607   0.116104   \n",
       "14       15                  0.899999         0.002610   0.057657   \n",
       "15       16                  1.000000         0.000095   0.036332   \n",
       "\n",
       "    cumulative_lift  response_rate     score  cumulative_response_rate  \\\n",
       "0         10.724871       0.387668  0.466160                  0.387668   \n",
       "1          9.176952       0.275764  0.280435                  0.331716   \n",
       "2          8.268734       0.233229  0.216827                  0.298887   \n",
       "3          7.493334       0.186750  0.179199                  0.270859   \n",
       "4          6.948660       0.172424  0.153562                  0.251170   \n",
       "5          5.145694       0.120825  0.110895                  0.185999   \n",
       "6          4.229723       0.086674  0.073579                  0.152890   \n",
       "7          3.572367       0.057843  0.054179                  0.129129   \n",
       "8          2.755427       0.040540  0.037965                  0.099599   \n",
       "9          2.227908       0.023326  0.025266                  0.080531   \n",
       "10         1.862254       0.014446  0.017478                  0.067314   \n",
       "11         1.598608       0.010135  0.012267                  0.057784   \n",
       "12         1.398561       0.007166  0.008555                  0.050553   \n",
       "13         1.238253       0.004197  0.005790                  0.044759   \n",
       "14         1.107075       0.002084  0.003583                  0.040017   \n",
       "15         1.000000       0.001313  0.001619                  0.036147   \n",
       "\n",
       "    cumulative_score  capture_rate  cumulative_capture_rate        gain  \\\n",
       "0           0.466160      0.107259                 0.107259  972.487123   \n",
       "1           0.373297      0.076297                 0.183556  662.903211   \n",
       "2           0.321140      0.064529                 0.248085  545.229734   \n",
       "3           0.285663      0.051655                 0.299739  416.647177   \n",
       "4           0.259241      0.047706                 0.347445  377.011946   \n",
       "5           0.185070      0.167127                 0.514572  234.262433   \n",
       "6           0.147906      0.119896                 0.634468  139.783271   \n",
       "7           0.124475      0.080009                 0.714478   60.022611   \n",
       "8           0.095638      0.112155                 0.826633   12.154798   \n",
       "9           0.078045      0.064529                 0.891162  -35.469658   \n",
       "10          0.065932      0.039965                 0.931127  -60.034981   \n",
       "11          0.056988      0.028039                 0.959166  -71.961301   \n",
       "12          0.050069      0.019825                 0.978991  -80.174889   \n",
       "13          0.044534      0.011610                 0.990601  -88.389609   \n",
       "14          0.039984      0.005766                 0.996367  -94.234296   \n",
       "15          0.036147      0.003633                 1.000000  -96.366816   \n",
       "\n",
       "    cumulative_gain  \n",
       "0        972.487123  \n",
       "1        817.695167  \n",
       "2        726.873356  \n",
       "3        649.333418  \n",
       "4        594.866013  \n",
       "5        414.569371  \n",
       "6        322.972261  \n",
       "7        257.236725  \n",
       "8        175.542749  \n",
       "9        122.790777  \n",
       "10        86.225417  \n",
       "11        59.860838  \n",
       "12        39.856142  \n",
       "13        23.825309  \n",
       "14        10.707492  \n",
       "15         0.000000  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "ModelMetricsBinomialGLM: glm\n",
      "** Reported on validation data. **\n",
      "\n",
      "MSE: 0.031018805729749764\n",
      "RMSE: 0.17612156520355413\n",
      "LogLoss: 0.12242815235268398\n",
      "Null degrees of freedom: 74970\n",
      "Residual degrees of freedom: 74827\n",
      "Null deviance: 22974.597464481732\n",
      "Residual deviance: 18357.12202006614\n",
      "AIC: 18645.12202006614\n",
      "AUC: 0.8460502420206815\n",
      "AUCPR: 0.2009137545141779\n",
      "Gini: 0.6921004840413629\n",
      "\n",
      "Confusion Matrix (Act/Pred) for max f1 @ threshold = 0.13091994899463488: \n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>FALSE</th>\n",
       "      <th>TRUE</th>\n",
       "      <th>Error</th>\n",
       "      <th>Rate</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>FALSE</td>\n",
       "      <td>69013.0</td>\n",
       "      <td>3300.0</td>\n",
       "      <td>0.0456</td>\n",
       "      <td>(3300.0/72313.0)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>TRUE</td>\n",
       "      <td>1672.0</td>\n",
       "      <td>986.0</td>\n",
       "      <td>0.629</td>\n",
       "      <td>(1672.0/2658.0)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Total</td>\n",
       "      <td>70685.0</td>\n",
       "      <td>4286.0</td>\n",
       "      <td>0.0663</td>\n",
       "      <td>(4972.0/74971.0)</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            FALSE    TRUE   Error               Rate\n",
       "0  FALSE  69013.0  3300.0  0.0456   (3300.0/72313.0)\n",
       "1   TRUE   1672.0   986.0   0.629    (1672.0/2658.0)\n",
       "2  Total  70685.0  4286.0  0.0663   (4972.0/74971.0)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Maximum Metrics: Maximum metrics at their respective thresholds\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>metric</th>\n",
       "      <th>threshold</th>\n",
       "      <th>value</th>\n",
       "      <th>idx</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>max f1</td>\n",
       "      <td>0.130920</td>\n",
       "      <td>0.283986</td>\n",
       "      <td>201.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>max f2</td>\n",
       "      <td>0.071763</td>\n",
       "      <td>0.375903</td>\n",
       "      <td>256.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>max f0point5</td>\n",
       "      <td>0.242376</td>\n",
       "      <td>0.295791</td>\n",
       "      <td>133.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>max accuracy</td>\n",
       "      <td>0.974097</td>\n",
       "      <td>0.964533</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>max precision</td>\n",
       "      <td>0.376628</td>\n",
       "      <td>0.396364</td>\n",
       "      <td>83.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>max recall</td>\n",
       "      <td>0.000735</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>398.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>max specificity</td>\n",
       "      <td>0.974097</td>\n",
       "      <td>0.999986</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>max absolute_mcc</td>\n",
       "      <td>0.119399</td>\n",
       "      <td>0.260856</td>\n",
       "      <td>210.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>max min_per_class_accuracy</td>\n",
       "      <td>0.037620</td>\n",
       "      <td>0.764108</td>\n",
       "      <td>304.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>max mean_per_class_accuracy</td>\n",
       "      <td>0.034897</td>\n",
       "      <td>0.767316</td>\n",
       "      <td>309.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>max tns</td>\n",
       "      <td>0.974097</td>\n",
       "      <td>72312.000000</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>max fns</td>\n",
       "      <td>0.974097</td>\n",
       "      <td>2658.000000</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>max fps</td>\n",
       "      <td>0.000475</td>\n",
       "      <td>72313.000000</td>\n",
       "      <td>399.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>max tps</td>\n",
       "      <td>0.000735</td>\n",
       "      <td>2658.000000</td>\n",
       "      <td>398.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>max tnr</td>\n",
       "      <td>0.974097</td>\n",
       "      <td>0.999986</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>max fnr</td>\n",
       "      <td>0.974097</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>max fpr</td>\n",
       "      <td>0.000475</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>399.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>max tpr</td>\n",
       "      <td>0.000735</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>398.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                         metric  threshold         value    idx\n",
       "0                        max f1   0.130920      0.283986  201.0\n",
       "1                        max f2   0.071763      0.375903  256.0\n",
       "2                  max f0point5   0.242376      0.295791  133.0\n",
       "3                  max accuracy   0.974097      0.964533    0.0\n",
       "4                 max precision   0.376628      0.396364   83.0\n",
       "5                    max recall   0.000735      1.000000  398.0\n",
       "6               max specificity   0.974097      0.999986    0.0\n",
       "7              max absolute_mcc   0.119399      0.260856  210.0\n",
       "8    max min_per_class_accuracy   0.037620      0.764108  304.0\n",
       "9   max mean_per_class_accuracy   0.034897      0.767316  309.0\n",
       "10                      max tns   0.974097  72312.000000    0.0\n",
       "11                      max fns   0.974097   2658.000000    0.0\n",
       "12                      max fps   0.000475  72313.000000  399.0\n",
       "13                      max tps   0.000735   2658.000000  398.0\n",
       "14                      max tnr   0.974097      0.999986    0.0\n",
       "15                      max fnr   0.974097      1.000000    0.0\n",
       "16                      max fpr   0.000475      1.000000  399.0\n",
       "17                      max tpr   0.000735      1.000000  398.0"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Gains/Lift Table: Avg response rate:  3.55 %, avg score:  3.61 %\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>group</th>\n",
       "      <th>cumulative_data_fraction</th>\n",
       "      <th>lower_threshold</th>\n",
       "      <th>lift</th>\n",
       "      <th>cumulative_lift</th>\n",
       "      <th>response_rate</th>\n",
       "      <th>score</th>\n",
       "      <th>cumulative_response_rate</th>\n",
       "      <th>cumulative_score</th>\n",
       "      <th>capture_rate</th>\n",
       "      <th>cumulative_capture_rate</th>\n",
       "      <th>gain</th>\n",
       "      <th>cumulative_gain</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td></td>\n",
       "      <td>1</td>\n",
       "      <td>0.010004</td>\n",
       "      <td>0.334125</td>\n",
       "      <td>10.793417</td>\n",
       "      <td>10.793417</td>\n",
       "      <td>0.382667</td>\n",
       "      <td>0.473002</td>\n",
       "      <td>0.382667</td>\n",
       "      <td>0.473002</td>\n",
       "      <td>0.107976</td>\n",
       "      <td>0.107976</td>\n",
       "      <td>979.341711</td>\n",
       "      <td>979.341711</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td></td>\n",
       "      <td>2</td>\n",
       "      <td>0.020008</td>\n",
       "      <td>0.243096</td>\n",
       "      <td>8.348915</td>\n",
       "      <td>9.571166</td>\n",
       "      <td>0.296000</td>\n",
       "      <td>0.282541</td>\n",
       "      <td>0.339333</td>\n",
       "      <td>0.377771</td>\n",
       "      <td>0.083521</td>\n",
       "      <td>0.191497</td>\n",
       "      <td>734.891497</td>\n",
       "      <td>857.116604</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td></td>\n",
       "      <td>3</td>\n",
       "      <td>0.030012</td>\n",
       "      <td>0.193143</td>\n",
       "      <td>5.829197</td>\n",
       "      <td>8.323843</td>\n",
       "      <td>0.206667</td>\n",
       "      <td>0.216035</td>\n",
       "      <td>0.295111</td>\n",
       "      <td>0.323859</td>\n",
       "      <td>0.058315</td>\n",
       "      <td>0.249812</td>\n",
       "      <td>482.919739</td>\n",
       "      <td>732.384316</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td></td>\n",
       "      <td>4</td>\n",
       "      <td>0.040002</td>\n",
       "      <td>0.162847</td>\n",
       "      <td>4.895532</td>\n",
       "      <td>7.467623</td>\n",
       "      <td>0.173565</td>\n",
       "      <td>0.176602</td>\n",
       "      <td>0.264755</td>\n",
       "      <td>0.287082</td>\n",
       "      <td>0.048909</td>\n",
       "      <td>0.298721</td>\n",
       "      <td>389.553164</td>\n",
       "      <td>646.762264</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td></td>\n",
       "      <td>5</td>\n",
       "      <td>0.050006</td>\n",
       "      <td>0.141925</td>\n",
       "      <td>3.948811</td>\n",
       "      <td>6.763673</td>\n",
       "      <td>0.140000</td>\n",
       "      <td>0.152013</td>\n",
       "      <td>0.239797</td>\n",
       "      <td>0.260061</td>\n",
       "      <td>0.039503</td>\n",
       "      <td>0.338224</td>\n",
       "      <td>294.881114</td>\n",
       "      <td>576.367262</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td></td>\n",
       "      <td>6</td>\n",
       "      <td>0.100012</td>\n",
       "      <td>0.087175</td>\n",
       "      <td>3.310363</td>\n",
       "      <td>5.037018</td>\n",
       "      <td>0.117365</td>\n",
       "      <td>0.110232</td>\n",
       "      <td>0.178581</td>\n",
       "      <td>0.185146</td>\n",
       "      <td>0.165538</td>\n",
       "      <td>0.503762</td>\n",
       "      <td>231.036257</td>\n",
       "      <td>403.701760</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td></td>\n",
       "      <td>7</td>\n",
       "      <td>0.150005</td>\n",
       "      <td>0.061800</td>\n",
       "      <td>2.227565</td>\n",
       "      <td>4.100700</td>\n",
       "      <td>0.078975</td>\n",
       "      <td>0.073379</td>\n",
       "      <td>0.145385</td>\n",
       "      <td>0.147897</td>\n",
       "      <td>0.111362</td>\n",
       "      <td>0.615124</td>\n",
       "      <td>122.756536</td>\n",
       "      <td>310.070006</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td></td>\n",
       "      <td>8</td>\n",
       "      <td>0.200011</td>\n",
       "      <td>0.047553</td>\n",
       "      <td>1.707846</td>\n",
       "      <td>3.502447</td>\n",
       "      <td>0.060549</td>\n",
       "      <td>0.054259</td>\n",
       "      <td>0.124175</td>\n",
       "      <td>0.124486</td>\n",
       "      <td>0.085403</td>\n",
       "      <td>0.700527</td>\n",
       "      <td>70.784615</td>\n",
       "      <td>250.244669</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td></td>\n",
       "      <td>9</td>\n",
       "      <td>0.300009</td>\n",
       "      <td>0.030349</td>\n",
       "      <td>1.106110</td>\n",
       "      <td>2.703703</td>\n",
       "      <td>0.039216</td>\n",
       "      <td>0.038070</td>\n",
       "      <td>0.095856</td>\n",
       "      <td>0.095682</td>\n",
       "      <td>0.110609</td>\n",
       "      <td>0.811136</td>\n",
       "      <td>10.610956</td>\n",
       "      <td>170.370316</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td></td>\n",
       "      <td>10</td>\n",
       "      <td>0.400008</td>\n",
       "      <td>0.020669</td>\n",
       "      <td>0.714833</td>\n",
       "      <td>2.206502</td>\n",
       "      <td>0.025343</td>\n",
       "      <td>0.025116</td>\n",
       "      <td>0.078229</td>\n",
       "      <td>0.078041</td>\n",
       "      <td>0.071482</td>\n",
       "      <td>0.882619</td>\n",
       "      <td>-28.516729</td>\n",
       "      <td>120.650213</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td></td>\n",
       "      <td>11</td>\n",
       "      <td>0.500007</td>\n",
       "      <td>0.014493</td>\n",
       "      <td>0.470285</td>\n",
       "      <td>1.859268</td>\n",
       "      <td>0.016673</td>\n",
       "      <td>0.017395</td>\n",
       "      <td>0.065918</td>\n",
       "      <td>0.065912</td>\n",
       "      <td>0.047028</td>\n",
       "      <td>0.929646</td>\n",
       "      <td>-52.971532</td>\n",
       "      <td>85.926790</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td></td>\n",
       "      <td>12</td>\n",
       "      <td>0.600005</td>\n",
       "      <td>0.010143</td>\n",
       "      <td>0.297220</td>\n",
       "      <td>1.598932</td>\n",
       "      <td>0.010538</td>\n",
       "      <td>0.012191</td>\n",
       "      <td>0.056688</td>\n",
       "      <td>0.056959</td>\n",
       "      <td>0.029722</td>\n",
       "      <td>0.959368</td>\n",
       "      <td>-70.278008</td>\n",
       "      <td>59.893236</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td></td>\n",
       "      <td>13</td>\n",
       "      <td>0.700004</td>\n",
       "      <td>0.007006</td>\n",
       "      <td>0.180589</td>\n",
       "      <td>1.396316</td>\n",
       "      <td>0.006403</td>\n",
       "      <td>0.008512</td>\n",
       "      <td>0.049505</td>\n",
       "      <td>0.050038</td>\n",
       "      <td>0.018059</td>\n",
       "      <td>0.977427</td>\n",
       "      <td>-81.941068</td>\n",
       "      <td>39.631578</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td></td>\n",
       "      <td>14</td>\n",
       "      <td>0.800003</td>\n",
       "      <td>0.004578</td>\n",
       "      <td>0.131680</td>\n",
       "      <td>1.238239</td>\n",
       "      <td>0.004669</td>\n",
       "      <td>0.005761</td>\n",
       "      <td>0.043900</td>\n",
       "      <td>0.044504</td>\n",
       "      <td>0.013168</td>\n",
       "      <td>0.990594</td>\n",
       "      <td>-86.832029</td>\n",
       "      <td>23.823891</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td></td>\n",
       "      <td>15</td>\n",
       "      <td>0.900001</td>\n",
       "      <td>0.002583</td>\n",
       "      <td>0.045147</td>\n",
       "      <td>1.105675</td>\n",
       "      <td>0.001601</td>\n",
       "      <td>0.003547</td>\n",
       "      <td>0.039200</td>\n",
       "      <td>0.039953</td>\n",
       "      <td>0.004515</td>\n",
       "      <td>0.995109</td>\n",
       "      <td>-95.485267</td>\n",
       "      <td>10.567514</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td></td>\n",
       "      <td>16</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000078</td>\n",
       "      <td>0.048910</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.001734</td>\n",
       "      <td>0.001600</td>\n",
       "      <td>0.035454</td>\n",
       "      <td>0.036118</td>\n",
       "      <td>0.004891</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>-95.109039</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      group  cumulative_data_fraction  lower_threshold       lift  \\\n",
       "0         1                  0.010004         0.334125  10.793417   \n",
       "1         2                  0.020008         0.243096   8.348915   \n",
       "2         3                  0.030012         0.193143   5.829197   \n",
       "3         4                  0.040002         0.162847   4.895532   \n",
       "4         5                  0.050006         0.141925   3.948811   \n",
       "5         6                  0.100012         0.087175   3.310363   \n",
       "6         7                  0.150005         0.061800   2.227565   \n",
       "7         8                  0.200011         0.047553   1.707846   \n",
       "8         9                  0.300009         0.030349   1.106110   \n",
       "9        10                  0.400008         0.020669   0.714833   \n",
       "10       11                  0.500007         0.014493   0.470285   \n",
       "11       12                  0.600005         0.010143   0.297220   \n",
       "12       13                  0.700004         0.007006   0.180589   \n",
       "13       14                  0.800003         0.004578   0.131680   \n",
       "14       15                  0.900001         0.002583   0.045147   \n",
       "15       16                  1.000000         0.000078   0.048910   \n",
       "\n",
       "    cumulative_lift  response_rate     score  cumulative_response_rate  \\\n",
       "0         10.793417       0.382667  0.473002                  0.382667   \n",
       "1          9.571166       0.296000  0.282541                  0.339333   \n",
       "2          8.323843       0.206667  0.216035                  0.295111   \n",
       "3          7.467623       0.173565  0.176602                  0.264755   \n",
       "4          6.763673       0.140000  0.152013                  0.239797   \n",
       "5          5.037018       0.117365  0.110232                  0.178581   \n",
       "6          4.100700       0.078975  0.073379                  0.145385   \n",
       "7          3.502447       0.060549  0.054259                  0.124175   \n",
       "8          2.703703       0.039216  0.038070                  0.095856   \n",
       "9          2.206502       0.025343  0.025116                  0.078229   \n",
       "10         1.859268       0.016673  0.017395                  0.065918   \n",
       "11         1.598932       0.010538  0.012191                  0.056688   \n",
       "12         1.396316       0.006403  0.008512                  0.049505   \n",
       "13         1.238239       0.004669  0.005761                  0.043900   \n",
       "14         1.105675       0.001601  0.003547                  0.039200   \n",
       "15         1.000000       0.001734  0.001600                  0.035454   \n",
       "\n",
       "    cumulative_score  capture_rate  cumulative_capture_rate        gain  \\\n",
       "0           0.473002      0.107976                 0.107976  979.341711   \n",
       "1           0.377771      0.083521                 0.191497  734.891497   \n",
       "2           0.323859      0.058315                 0.249812  482.919739   \n",
       "3           0.287082      0.048909                 0.298721  389.553164   \n",
       "4           0.260061      0.039503                 0.338224  294.881114   \n",
       "5           0.185146      0.165538                 0.503762  231.036257   \n",
       "6           0.147897      0.111362                 0.615124  122.756536   \n",
       "7           0.124486      0.085403                 0.700527   70.784615   \n",
       "8           0.095682      0.110609                 0.811136   10.610956   \n",
       "9           0.078041      0.071482                 0.882619  -28.516729   \n",
       "10          0.065912      0.047028                 0.929646  -52.971532   \n",
       "11          0.056959      0.029722                 0.959368  -70.278008   \n",
       "12          0.050038      0.018059                 0.977427  -81.941068   \n",
       "13          0.044504      0.013168                 0.990594  -86.832029   \n",
       "14          0.039953      0.004515                 0.995109  -95.485267   \n",
       "15          0.036118      0.004891                 1.000000  -95.109039   \n",
       "\n",
       "    cumulative_gain  \n",
       "0        979.341711  \n",
       "1        857.116604  \n",
       "2        732.384316  \n",
       "3        646.762264  \n",
       "4        576.367262  \n",
       "5        403.701760  \n",
       "6        310.070006  \n",
       "7        250.244669  \n",
       "8        170.370316  \n",
       "9        120.650213  \n",
       "10        85.926790  \n",
       "11        59.893236  \n",
       "12        39.631578  \n",
       "13        23.823891  \n",
       "14        10.567514  \n",
       "15         0.000000  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "Scoring History: \n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>timestamp</th>\n",
       "      <th>duration</th>\n",
       "      <th>iteration</th>\n",
       "      <th>lambda</th>\n",
       "      <th>predictors</th>\n",
       "      <th>deviance_train</th>\n",
       "      <th>deviance_test</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td></td>\n",
       "      <td>2020-03-06 00:44:12</td>\n",
       "      <td>0.000 sec</td>\n",
       "      <td>7</td>\n",
       "      <td>.5E-5</td>\n",
       "      <td>144</td>\n",
       "      <td>0.2456</td>\n",
       "      <td>0.244856</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "               timestamp    duration  iteration lambda  predictors  \\\n",
       "0    2020-03-06 00:44:12   0.000 sec          7  .5E-5         144   \n",
       "\n",
       "   deviance_train  deviance_test  \n",
       "0          0.2456       0.244856  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": []
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "glm_grid.models[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Grid Summary:\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div style=\"overflow:auto\"><table style=\"width:50%\"><tr><td><b>Model Id</b></td>\n",
       "<td><b>family</b></td>\n",
       "<td><b>link</b></td>\n",
       "<td><b>regularization</b></td>\n",
       "<td><b>lambda_search</b></td>\n",
       "<td><b>number_of_predictors_total</b></td>\n",
       "<td><b>number_of_active_predictors</b></td>\n",
       "<td><b>number_of_iterations</b></td>\n",
       "<td><b>training_frame</b></td></tr>\n",
       "<tr><td>glm_grid_model_61</td>\n",
       "<td>binomial</td>\n",
       "<td>logit</td>\n",
       "<td>Elastic Net (alpha = 0.87, lambda = 5.0E-6 )</td>\n",
       "<td>nlambda = 100, lambda.max = 0.03808, lambda.min = 5.0E-6, lambda.1se = -1.0</td>\n",
       "<td>161</td>\n",
       "<td>143</td>\n",
       "<td>7</td>\n",
       "<td>py_15_sid_9664</td></tr>\n",
       "<tr><td>glm_grid_model_46</td>\n",
       "<td>binomial</td>\n",
       "<td>logit</td>\n",
       "<td>Elastic Net (alpha = 0.4, lambda = 1.8E-5 )</td>\n",
       "<td>nlambda = 100, lambda.max = 0.08282, lambda.min = 1.8E-5, lambda.1se = -1.0</td>\n",
       "<td>161</td>\n",
       "<td>137</td>\n",
       "<td>7</td>\n",
       "<td>py_15_sid_9664</td></tr>\n",
       "<tr><td>glm_grid_model_48</td>\n",
       "<td>binomial</td>\n",
       "<td>logit</td>\n",
       "<td>Elastic Net (alpha = 0.07, lambda = 3.7E-5 )</td>\n",
       "<td>nlambda = 100, lambda.max = 0.4733, lambda.min = 3.7E-5, lambda.1se = -1.0</td>\n",
       "<td>161</td>\n",
       "<td>145</td>\n",
       "<td>5</td>\n",
       "<td>py_15_sid_9664</td></tr>\n",
       "<tr><td>glm_grid_model_96</td>\n",
       "<td>binomial</td>\n",
       "<td>logit</td>\n",
       "<td>Elastic Net (alpha = 0.48, lambda = 2.9E-5 )</td>\n",
       "<td>nlambda = 100, lambda.max = 0.06902, lambda.min = 2.9E-5, lambda.1se = -1.0</td>\n",
       "<td>161</td>\n",
       "<td>110</td>\n",
       "<td>7</td>\n",
       "<td>py_15_sid_9664</td></tr>\n",
       "<tr><td>glm_grid_model_72</td>\n",
       "<td>binomial</td>\n",
       "<td>logit</td>\n",
       "<td>Elastic Net (alpha = 0.07, lambda = 5.6E-5 )</td>\n",
       "<td>nlambda = 100, lambda.max = 0.4733, lambda.min = 5.6E-5, lambda.1se = -1.0</td>\n",
       "<td>161</td>\n",
       "<td>137</td>\n",
       "<td>5</td>\n",
       "<td>py_15_sid_9664</td></tr>\n",
       "<tr><td>glm_grid_model_37</td>\n",
       "<td>binomial</td>\n",
       "<td>logit</td>\n",
       "<td>Elastic Net (alpha = 0.88, lambda = 3.1E-5 )</td>\n",
       "<td>nlambda = 100, lambda.max = 0.03765, lambda.min = 3.1E-5, lambda.1se = -1.0</td>\n",
       "<td>161</td>\n",
       "<td>93</td>\n",
       "<td>7</td>\n",
       "<td>py_15_sid_9664</td></tr>\n",
       "<tr><td>glm_grid_model_86</td>\n",
       "<td>binomial</td>\n",
       "<td>logit</td>\n",
       "<td>Elastic Net (alpha = 0.18, lambda = 8.1E-5 )</td>\n",
       "<td>nlambda = 100, lambda.max = 0.184, lambda.min = 8.1E-5, lambda.1se = -1.0</td>\n",
       "<td>161</td>\n",
       "<td>108</td>\n",
       "<td>7</td>\n",
       "<td>py_15_sid_9664</td></tr>\n",
       "<tr><td>glm_grid_model_30</td>\n",
       "<td>binomial</td>\n",
       "<td>logit</td>\n",
       "<td>Elastic Net (alpha = 0.15, lambda = 1.31E-4 )</td>\n",
       "<td>nlambda = 100, lambda.max = 0.2209, lambda.min = 1.31E-4, lambda.1se = -1.0</td>\n",
       "<td>161</td>\n",
       "<td>105</td>\n",
       "<td>7</td>\n",
       "<td>py_15_sid_9664</td></tr>\n",
       "<tr><td>glm_grid_model_38</td>\n",
       "<td>binomial</td>\n",
       "<td>logit</td>\n",
       "<td>Elastic Net (alpha = 0.1, lambda = 1.59E-4 )</td>\n",
       "<td>nlambda = 100, lambda.max = 0.3313, lambda.min = 1.59E-4, lambda.1se = -1.0</td>\n",
       "<td>161</td>\n",
       "<td>110</td>\n",
       "<td>7</td>\n",
       "<td>py_15_sid_9664</td></tr>\n",
       "<tr><td>glm_grid_model_28</td>\n",
       "<td>binomial</td>\n",
       "<td>logit</td>\n",
       "<td>Elastic Net (alpha = 0.53, lambda = 9.5E-5 )</td>\n",
       "<td>nlambda = 100, lambda.max = 0.06251, lambda.min = 9.5E-5, lambda.1se = -1.0</td>\n",
       "<td>161</td>\n",
       "<td>80</td>\n",
       "<td>7</td>\n",
       "<td>py_15_sid_9664</td></tr>\n",
       "<tr><td>glm_grid_model_78</td>\n",
       "<td>binomial</td>\n",
       "<td>logit</td>\n",
       "<td>Elastic Net (alpha = 0.06, lambda = 2.12E-4 )</td>\n",
       "<td>nlambda = 100, lambda.max = 0.5521, lambda.min = 2.12E-4, lambda.1se = -1.0</td>\n",
       "<td>161</td>\n",
       "<td>119</td>\n",
       "<td>7</td>\n",
       "<td>py_15_sid_9664</td></tr>\n",
       "<tr><td>glm_grid_model_67</td>\n",
       "<td>binomial</td>\n",
       "<td>logit</td>\n",
       "<td>Elastic Net (alpha = 0.41, lambda = 1.37E-4 )</td>\n",
       "<td>nlambda = 100, lambda.max = 0.0808, lambda.min = 1.37E-4, lambda.1se = -1.0</td>\n",
       "<td>161</td>\n",
       "<td>77</td>\n",
       "<td>7</td>\n",
       "<td>py_15_sid_9664</td></tr>\n",
       "<tr><td>glm_grid_model_100</td>\n",
       "<td>binomial</td>\n",
       "<td>logit</td>\n",
       "<td>Elastic Net (alpha = 0.93, lambda = 7.7E-5 )</td>\n",
       "<td>nlambda = 100, lambda.max = 0.03562, lambda.min = 7.7E-5, lambda.1se = -1.0</td>\n",
       "<td>161</td>\n",
       "<td>71</td>\n",
       "<td>7</td>\n",
       "<td>py_15_sid_9664</td></tr>\n",
       "<tr><td>glm_grid_model_34</td>\n",
       "<td>binomial</td>\n",
       "<td>logit</td>\n",
       "<td>Elastic Net (alpha = 0.65, lambda = 1.07E-4 )</td>\n",
       "<td>nlambda = 100, lambda.max = 0.05097, lambda.min = 1.07E-4, lambda.1se = -1.0</td>\n",
       "<td>161</td>\n",
       "<td>69</td>\n",
       "<td>7</td>\n",
       "<td>py_15_sid_9664</td></tr>\n",
       "<tr><td>glm_grid_model_42</td>\n",
       "<td>binomial</td>\n",
       "<td>logit</td>\n",
       "<td>Elastic Net (alpha = 0.52, lambda = 1.37E-4 )</td>\n",
       "<td>nlambda = 100, lambda.max = 0.06371, lambda.min = 1.37E-4, lambda.1se = -1.0</td>\n",
       "<td>161</td>\n",
       "<td>70</td>\n",
       "<td>7</td>\n",
       "<td>py_15_sid_9664</td></tr>\n",
       "<tr><td>glm_grid_model_82</td>\n",
       "<td>binomial</td>\n",
       "<td>logit</td>\n",
       "<td>Elastic Net (alpha = 0.56, lambda = 1.64E-4 )</td>\n",
       "<td>nlambda = 100, lambda.max = 0.05916, lambda.min = 1.64E-4, lambda.1se = -1.0</td>\n",
       "<td>161</td>\n",
       "<td>68</td>\n",
       "<td>7</td>\n",
       "<td>py_15_sid_9664</td></tr>\n",
       "<tr><td>glm_grid_model_51</td>\n",
       "<td>binomial</td>\n",
       "<td>logit</td>\n",
       "<td>Elastic Net (alpha = 0.47, lambda = 1.89E-4 )</td>\n",
       "<td>nlambda = 100, lambda.max = 0.07048, lambda.min = 1.89E-4, lambda.1se = -1.0</td>\n",
       "<td>161</td>\n",
       "<td>69</td>\n",
       "<td>7</td>\n",
       "<td>py_15_sid_9664</td></tr>\n",
       "<tr><td>glm_grid_model_22</td>\n",
       "<td>binomial</td>\n",
       "<td>logit</td>\n",
       "<td>Elastic Net (alpha = 0.31, lambda = 2.3E-4 )</td>\n",
       "<td>nlambda = 100, lambda.max = 0.1069, lambda.min = 2.3E-4, lambda.1se = -1.0</td>\n",
       "<td>161</td>\n",
       "<td>72</td>\n",
       "<td>7</td>\n",
       "<td>py_15_sid_9664</td></tr>\n",
       "<tr><td>glm_grid_model_16</td>\n",
       "<td>binomial</td>\n",
       "<td>logit</td>\n",
       "<td>Elastic Net (alpha = 0.05, lambda = 4.17E-4 )</td>\n",
       "<td>nlambda = 100, lambda.max = 0.6626, lambda.min = 4.17E-4, lambda.1se = -1.0</td>\n",
       "<td>161</td>\n",
       "<td>110</td>\n",
       "<td>6</td>\n",
       "<td>py_15_sid_9664</td></tr>\n",
       "<tr><td>glm_grid_model_17</td>\n",
       "<td>binomial</td>\n",
       "<td>logit</td>\n",
       "<td>Elastic Net (alpha = 0.18, lambda = 3.47E-4 )</td>\n",
       "<td>nlambda = 100, lambda.max = 0.184, lambda.min = 3.47E-4, lambda.1se = -1.0</td>\n",
       "<td>161</td>\n",
       "<td>80</td>\n",
       "<td>7</td>\n",
       "<td>py_15_sid_9664</td></tr>\n",
       "<tr><td>glm_grid_model_93</td>\n",
       "<td>binomial</td>\n",
       "<td>logit</td>\n",
       "<td>Elastic Net (alpha = 0.92, lambda = 1.45E-4 )</td>\n",
       "<td>nlambda = 100, lambda.max = 0.03601, lambda.min = 1.45E-4, lambda.1se = -1.0</td>\n",
       "<td>161</td>\n",
       "<td>58</td>\n",
       "<td>8</td>\n",
       "<td>py_15_sid_9664</td></tr>\n",
       "<tr><td>glm_grid_model_73</td>\n",
       "<td>binomial</td>\n",
       "<td>logit</td>\n",
       "<td>Elastic Net (alpha = 0.11, lambda = 4.88E-4 )</td>\n",
       "<td>nlambda = 100, lambda.max = 0.3012, lambda.min = 4.88E-4, lambda.1se = -1.0</td>\n",
       "<td>161</td>\n",
       "<td>85</td>\n",
       "<td>6</td>\n",
       "<td>py_15_sid_9664</td></tr>\n",
       "<tr><td>glm_grid_model_11</td>\n",
       "<td>binomial</td>\n",
       "<td>logit</td>\n",
       "<td>Elastic Net (alpha = 0.75, lambda = 1.81E-4 )</td>\n",
       "<td>nlambda = 100, lambda.max = 0.04417, lambda.min = 1.81E-4, lambda.1se = -1.0</td>\n",
       "<td>161</td>\n",
       "<td>53</td>\n",
       "<td>7</td>\n",
       "<td>py_15_sid_9664</td></tr>\n",
       "<tr><td>glm_grid_model_2</td>\n",
       "<td>binomial</td>\n",
       "<td>logit</td>\n",
       "<td>Elastic Net (alpha = 0.78, lambda = 1.88E-4 )</td>\n",
       "<td>nlambda = 100, lambda.max = 0.04247, lambda.min = 1.88E-4, lambda.1se = -1.0</td>\n",
       "<td>161</td>\n",
       "<td>53</td>\n",
       "<td>8</td>\n",
       "<td>py_15_sid_9664</td></tr>\n",
       "<tr><td>glm_grid_model_14</td>\n",
       "<td>binomial</td>\n",
       "<td>logit</td>\n",
       "<td>Elastic Net (alpha = 0.16, lambda = 4.93E-4 )</td>\n",
       "<td>nlambda = 100, lambda.max = 0.207, lambda.min = 4.93E-4, lambda.1se = -1.0</td>\n",
       "<td>161</td>\n",
       "<td>77</td>\n",
       "<td>6</td>\n",
       "<td>py_15_sid_9664</td></tr>\n",
       "<tr><td>glm_grid_model_83</td>\n",
       "<td>binomial</td>\n",
       "<td>logit</td>\n",
       "<td>Elastic Net (alpha = 0.65, lambda = 2.26E-4 )</td>\n",
       "<td>nlambda = 100, lambda.max = 0.05097, lambda.min = 2.26E-4, lambda.1se = -1.0</td>\n",
       "<td>161</td>\n",
       "<td>53</td>\n",
       "<td>8</td>\n",
       "<td>py_15_sid_9664</td></tr>\n",
       "<tr><td>glm_grid_model_20</td>\n",
       "<td>binomial</td>\n",
       "<td>logit</td>\n",
       "<td>Elastic Net (alpha = 0.53, lambda = 2.9E-4 )</td>\n",
       "<td>nlambda = 100, lambda.max = 0.06251, lambda.min = 2.9E-4, lambda.1se = -1.0</td>\n",
       "<td>161</td>\n",
       "<td>54</td>\n",
       "<td>8</td>\n",
       "<td>py_15_sid_9664</td></tr>\n",
       "<tr><td>glm_grid_model_1</td>\n",
       "<td>binomial</td>\n",
       "<td>logit</td>\n",
       "<td>Elastic Net (alpha = 0.09, lambda = 6.68E-4 )</td>\n",
       "<td>nlambda = 100, lambda.max = 0.3681, lambda.min = 6.68E-4, lambda.1se = -1.0</td>\n",
       "<td>161</td>\n",
       "<td>84</td>\n",
       "<td>6</td>\n",
       "<td>py_15_sid_9664</td></tr>\n",
       "<tr><td>glm_grid_model_98</td>\n",
       "<td>binomial</td>\n",
       "<td>logit</td>\n",
       "<td>Elastic Net (alpha = 0.17, lambda = 5.46E-4 )</td>\n",
       "<td>nlambda = 100, lambda.max = 0.1949, lambda.min = 5.46E-4, lambda.1se = -1.0</td>\n",
       "<td>161</td>\n",
       "<td>72</td>\n",
       "<td>6</td>\n",
       "<td>py_15_sid_9664</td></tr>\n",
       "<tr><td>glm_grid_model_54</td>\n",
       "<td>binomial</td>\n",
       "<td>logit</td>\n",
       "<td>Elastic Net (alpha = 0.13, lambda = 6.33E-4 )</td>\n",
       "<td>nlambda = 100, lambda.max = 0.2548, lambda.min = 6.33E-4, lambda.1se = -1.0</td>\n",
       "<td>161</td>\n",
       "<td>79</td>\n",
       "<td>7</td>\n",
       "<td>py_15_sid_9664</td></tr>\n",
       "<tr><td>glm_grid_model_79</td>\n",
       "<td>binomial</td>\n",
       "<td>logit</td>\n",
       "<td>Elastic Net (alpha = 0.07, lambda = 7.32E-4 )</td>\n",
       "<td>nlambda = 100, lambda.max = 0.4733, lambda.min = 7.32E-4, lambda.1se = -1.0</td>\n",
       "<td>161</td>\n",
       "<td>88</td>\n",
       "<td>6</td>\n",
       "<td>py_15_sid_9664</td></tr>\n",
       "<tr><td>glm_grid_model_35</td>\n",
       "<td>binomial</td>\n",
       "<td>logit</td>\n",
       "<td>Elastic Net (alpha = 0.99, lambda = 2.06E-4 )</td>\n",
       "<td>nlambda = 100, lambda.max = 0.03346, lambda.min = 2.06E-4, lambda.1se = -1.0</td>\n",
       "<td>161</td>\n",
       "<td>46</td>\n",
       "<td>7</td>\n",
       "<td>py_15_sid_9664</td></tr>\n",
       "<tr><td>glm_grid_model_92</td>\n",
       "<td>binomial</td>\n",
       "<td>logit</td>\n",
       "<td>Elastic Net (alpha = 0.02, lambda = 8.87E-4 )</td>\n",
       "<td>nlambda = 100, lambda.max = 1.6564, lambda.min = 8.87E-4, lambda.1se = -1.0</td>\n",
       "<td>161</td>\n",
       "<td>116</td>\n",
       "<td>5</td>\n",
       "<td>py_15_sid_9664</td></tr>\n",
       "<tr><td>glm_grid_model_60</td>\n",
       "<td>binomial</td>\n",
       "<td>logit</td>\n",
       "<td>Elastic Net (alpha = 0.05, lambda = 8.0E-4 )</td>\n",
       "<td>nlambda = 100, lambda.max = 0.6626, lambda.min = 8.0E-4, lambda.1se = -1.0</td>\n",
       "<td>161</td>\n",
       "<td>90</td>\n",
       "<td>6</td>\n",
       "<td>py_15_sid_9664</td></tr>\n",
       "<tr><td>glm_grid_model_90</td>\n",
       "<td>binomial</td>\n",
       "<td>logit</td>\n",
       "<td>Elastic Net (alpha = 0.02, lambda = 9.92E-4 )</td>\n",
       "<td>nlambda = 100, lambda.max = 1.6564, lambda.min = 9.92E-4, lambda.1se = -1.0</td>\n",
       "<td>161</td>\n",
       "<td>114</td>\n",
       "<td>5</td>\n",
       "<td>py_15_sid_9664</td></tr>\n",
       "<tr><td>glm_grid_model_71</td>\n",
       "<td>binomial</td>\n",
       "<td>logit</td>\n",
       "<td>Elastic Net (alpha = 0.09, lambda = 7.94E-4 )</td>\n",
       "<td>nlambda = 100, lambda.max = 0.3681, lambda.min = 7.94E-4, lambda.1se = -1.0</td>\n",
       "<td>161</td>\n",
       "<td>81</td>\n",
       "<td>6</td>\n",
       "<td>py_15_sid_9664</td></tr>\n",
       "<tr><td>glm_grid_model_26</td>\n",
       "<td>binomial</td>\n",
       "<td>logit</td>\n",
       "<td>Elastic Net (alpha = 0.93, lambda = 2.45E-4 )</td>\n",
       "<td>nlambda = 100, lambda.max = 0.03562, lambda.min = 2.45E-4, lambda.1se = -1.0</td>\n",
       "<td>161</td>\n",
       "<td>44</td>\n",
       "<td>7</td>\n",
       "<td>py_15_sid_9664</td></tr>\n",
       "<tr><td>glm_grid_model_80</td>\n",
       "<td>binomial</td>\n",
       "<td>logit</td>\n",
       "<td>Elastic Net (alpha = 0.23, lambda = 5.77E-4 )</td>\n",
       "<td>nlambda = 100, lambda.max = 0.144, lambda.min = 5.77E-4, lambda.1se = -1.0</td>\n",
       "<td>161</td>\n",
       "<td>64</td>\n",
       "<td>6</td>\n",
       "<td>py_15_sid_9664</td></tr>\n",
       "<tr><td>glm_grid_model_77</td>\n",
       "<td>binomial</td>\n",
       "<td>logit</td>\n",
       "<td>Elastic Net (alpha = 0.6, lambda = 3.29E-4 )</td>\n",
       "<td>nlambda = 100, lambda.max = 0.05521, lambda.min = 3.29E-4, lambda.1se = -1.0</td>\n",
       "<td>161</td>\n",
       "<td>47</td>\n",
       "<td>7</td>\n",
       "<td>py_15_sid_9664</td></tr>\n",
       "<tr><td>glm_grid_model_87</td>\n",
       "<td>binomial</td>\n",
       "<td>logit</td>\n",
       "<td>Elastic Net (alpha = 0.52, lambda = 3.66E-4 )</td>\n",
       "<td>nlambda = 100, lambda.max = 0.06371, lambda.min = 3.66E-4, lambda.1se = -1.0</td>\n",
       "<td>161</td>\n",
       "<td>48</td>\n",
       "<td>6</td>\n",
       "<td>py_15_sid_9664</td></tr>\n",
       "<tr><td>glm_grid_model_45</td>\n",
       "<td>binomial</td>\n",
       "<td>logit</td>\n",
       "<td>Elastic Net (alpha = 0.25, lambda = 5.9E-4 )</td>\n",
       "<td>nlambda = 100, lambda.max = 0.1325, lambda.min = 5.9E-4, lambda.1se = -1.0</td>\n",
       "<td>161</td>\n",
       "<td>59</td>\n",
       "<td>6</td>\n",
       "<td>py_15_sid_9664</td></tr>\n",
       "<tr><td>glm_grid_model_81</td>\n",
       "<td>binomial</td>\n",
       "<td>logit</td>\n",
       "<td>Elastic Net (alpha = 0.8, lambda = 3.02E-4 )</td>\n",
       "<td>nlambda = 100, lambda.max = 0.04141, lambda.min = 3.02E-4, lambda.1se = -1.0</td>\n",
       "<td>161</td>\n",
       "<td>43</td>\n",
       "<td>7</td>\n",
       "<td>py_15_sid_9664</td></tr>\n",
       "<tr><td>glm_grid_model_33</td>\n",
       "<td>binomial</td>\n",
       "<td>logit</td>\n",
       "<td>Elastic Net (alpha = 0.64, lambda = 3.47E-4 )</td>\n",
       "<td>nlambda = 100, lambda.max = 0.05176, lambda.min = 3.47E-4, lambda.1se = -1.0</td>\n",
       "<td>161</td>\n",
       "<td>45</td>\n",
       "<td>6</td>\n",
       "<td>py_15_sid_9664</td></tr>\n",
       "<tr><td>glm_grid_model_43</td>\n",
       "<td>binomial</td>\n",
       "<td>logit</td>\n",
       "<td>Elastic Net (alpha = 0.1, lambda = 8.66E-4 )</td>\n",
       "<td>nlambda = 100, lambda.max = 0.3313, lambda.min = 8.66E-4, lambda.1se = -1.0</td>\n",
       "<td>161</td>\n",
       "<td>75</td>\n",
       "<td>6</td>\n",
       "<td>py_15_sid_9664</td></tr>\n",
       "<tr><td>glm_grid_model_55</td>\n",
       "<td>binomial</td>\n",
       "<td>logit</td>\n",
       "<td>Elastic Net (alpha = 0.66, lambda = 3.52E-4 )</td>\n",
       "<td>nlambda = 100, lambda.max = 0.05019, lambda.min = 3.52E-4, lambda.1se = -1.0</td>\n",
       "<td>161</td>\n",
       "<td>43</td>\n",
       "<td>6</td>\n",
       "<td>py_15_sid_9664</td></tr>\n",
       "<tr><td>glm_grid_model_12</td>\n",
       "<td>binomial</td>\n",
       "<td>logit</td>\n",
       "<td>Elastic Net (alpha = 0.11, lambda = 8.43E-4 )</td>\n",
       "<td>nlambda = 100, lambda.max = 0.3012, lambda.min = 8.43E-4, lambda.1se = -1.0</td>\n",
       "<td>161</td>\n",
       "<td>73</td>\n",
       "<td>6</td>\n",
       "<td>py_15_sid_9664</td></tr>\n",
       "<tr><td>glm_grid_model_65</td>\n",
       "<td>binomial</td>\n",
       "<td>logit</td>\n",
       "<td>Elastic Net (alpha = 0.86, lambda = 3.02E-4 )</td>\n",
       "<td>nlambda = 100, lambda.max = 0.03852, lambda.min = 3.02E-4, lambda.1se = -1.0</td>\n",
       "<td>161</td>\n",
       "<td>40</td>\n",
       "<td>7</td>\n",
       "<td>py_15_sid_9664</td></tr>\n",
       "<tr><td>glm_grid_model_10</td>\n",
       "<td>binomial</td>\n",
       "<td>logit</td>\n",
       "<td>Elastic Net (alpha = 0.83, lambda = 3.23E-4 )</td>\n",
       "<td>nlambda = 100, lambda.max = 0.03991, lambda.min = 3.23E-4, lambda.1se = -1.0</td>\n",
       "<td>161</td>\n",
       "<td>38</td>\n",
       "<td>6</td>\n",
       "<td>py_15_sid_9664</td></tr>\n",
       "<tr><td>glm_grid_model_8</td>\n",
       "<td>binomial</td>\n",
       "<td>logit</td>\n",
       "<td>Elastic Net (alpha = 0.94, lambda = 3.05E-4 )</td>\n",
       "<td>nlambda = 100, lambda.max = 0.03524, lambda.min = 3.05E-4, lambda.1se = -1.0</td>\n",
       "<td>161</td>\n",
       "<td>36</td>\n",
       "<td>8</td>\n",
       "<td>py_15_sid_9664</td></tr>\n",
       "<tr><td>glm_grid_model_3</td>\n",
       "<td>binomial</td>\n",
       "<td>logit</td>\n",
       "<td>Elastic Net (alpha = 0.69, lambda = 3.77E-4 )</td>\n",
       "<td>nlambda = 100, lambda.max = 0.04801, lambda.min = 3.77E-4, lambda.1se = -1.0</td>\n",
       "<td>161</td>\n",
       "<td>39</td>\n",
       "<td>6</td>\n",
       "<td>py_15_sid_9664</td></tr>\n",
       "<tr><td>glm_grid_model_70</td>\n",
       "<td>binomial</td>\n",
       "<td>logit</td>\n",
       "<td>Elastic Net (alpha = 0.71, lambda = 3.86E-4 )</td>\n",
       "<td>nlambda = 100, lambda.max = 0.04666, lambda.min = 3.86E-4, lambda.1se = -1.0</td>\n",
       "<td>161</td>\n",
       "<td>38</td>\n",
       "<td>6</td>\n",
       "<td>py_15_sid_9664</td></tr>\n",
       "<tr><td>glm_grid_model_44</td>\n",
       "<td>binomial</td>\n",
       "<td>logit</td>\n",
       "<td>Elastic Net (alpha = 0.19, lambda = 7.88E-4 )</td>\n",
       "<td>nlambda = 100, lambda.max = 0.1744, lambda.min = 7.88E-4, lambda.1se = -1.0</td>\n",
       "<td>161</td>\n",
       "<td>60</td>\n",
       "<td>6</td>\n",
       "<td>py_15_sid_9664</td></tr>\n",
       "<tr><td>glm_grid_model_41</td>\n",
       "<td>binomial</td>\n",
       "<td>logit</td>\n",
       "<td>Elastic Net (alpha = 0.67, lambda = 3.99E-4 )</td>\n",
       "<td>nlambda = 100, lambda.max = 0.04944, lambda.min = 3.99E-4, lambda.1se = -1.0</td>\n",
       "<td>161</td>\n",
       "<td>38</td>\n",
       "<td>6</td>\n",
       "<td>py_15_sid_9664</td></tr>\n",
       "<tr><td>glm_grid_model_36</td>\n",
       "<td>binomial</td>\n",
       "<td>logit</td>\n",
       "<td>Elastic Net (alpha = 0.94, lambda = 3.4E-4 )</td>\n",
       "<td>nlambda = 100, lambda.max = 0.03524, lambda.min = 3.4E-4, lambda.1se = -1.0</td>\n",
       "<td>161</td>\n",
       "<td>35</td>\n",
       "<td>7</td>\n",
       "<td>py_15_sid_9664</td></tr>\n",
       "<tr><td>glm_grid_model_94</td>\n",
       "<td>binomial</td>\n",
       "<td>logit</td>\n",
       "<td>Elastic Net (alpha = 0.15, lambda = 9.21E-4 )</td>\n",
       "<td>nlambda = 100, lambda.max = 0.2209, lambda.min = 9.21E-4, lambda.1se = -1.0</td>\n",
       "<td>161</td>\n",
       "<td>64</td>\n",
       "<td>6</td>\n",
       "<td>py_15_sid_9664</td></tr>\n",
       "<tr><td>glm_grid_model_29</td>\n",
       "<td>binomial</td>\n",
       "<td>logit</td>\n",
       "<td>Elastic Net (alpha = 0.59, lambda = 4.59E-4 )</td>\n",
       "<td>nlambda = 100, lambda.max = 0.05615, lambda.min = 4.59E-4, lambda.1se = -1.0</td>\n",
       "<td>161</td>\n",
       "<td>39</td>\n",
       "<td>6</td>\n",
       "<td>py_15_sid_9664</td></tr>\n",
       "<tr><td>glm_grid_model_25</td>\n",
       "<td>binomial</td>\n",
       "<td>logit</td>\n",
       "<td>Elastic Net (alpha = 0.3, lambda = 6.75E-4 )</td>\n",
       "<td>nlambda = 100, lambda.max = 0.1104, lambda.min = 6.75E-4, lambda.1se = -1.0</td>\n",
       "<td>161</td>\n",
       "<td>47</td>\n",
       "<td>7</td>\n",
       "<td>py_15_sid_9664</td></tr>\n",
       "<tr><td>glm_grid_model_52</td>\n",
       "<td>binomial</td>\n",
       "<td>logit</td>\n",
       "<td>Elastic Net (alpha = 0.34, lambda = 6.66E-4 )</td>\n",
       "<td>nlambda = 100, lambda.max = 0.09743, lambda.min = 6.66E-4, lambda.1se = -1.0</td>\n",
       "<td>161</td>\n",
       "<td>45</td>\n",
       "<td>7</td>\n",
       "<td>py_15_sid_9664</td></tr>\n",
       "<tr><td>glm_grid_model_47</td>\n",
       "<td>binomial</td>\n",
       "<td>logit</td>\n",
       "<td>Elastic Net (alpha = 0.19, lambda = 8.82E-4 )</td>\n",
       "<td>nlambda = 100, lambda.max = 0.1744, lambda.min = 8.82E-4, lambda.1se = -1.0</td>\n",
       "<td>161</td>\n",
       "<td>56</td>\n",
       "<td>6</td>\n",
       "<td>py_15_sid_9664</td></tr>\n",
       "<tr><td>glm_grid_model_31</td>\n",
       "<td>binomial</td>\n",
       "<td>logit</td>\n",
       "<td>Elastic Net (alpha = 0.58, lambda = 5.09E-4 )</td>\n",
       "<td>nlambda = 100, lambda.max = 0.05712, lambda.min = 5.09E-4, lambda.1se = -1.0</td>\n",
       "<td>161</td>\n",
       "<td>37</td>\n",
       "<td>6</td>\n",
       "<td>py_15_sid_9664</td></tr>\n",
       "<tr><td>glm_grid_model_27</td>\n",
       "<td>binomial</td>\n",
       "<td>logit</td>\n",
       "<td>Elastic Net (alpha = 0.81, lambda = 4.25E-4 )</td>\n",
       "<td>nlambda = 100, lambda.max = 0.0409, lambda.min = 4.25E-4, lambda.1se = -1.0</td>\n",
       "<td>161</td>\n",
       "<td>34</td>\n",
       "<td>6</td>\n",
       "<td>py_15_sid_9664</td></tr>\n",
       "<tr><td>glm_grid_model_75</td>\n",
       "<td>binomial</td>\n",
       "<td>logit</td>\n",
       "<td>Elastic Net (alpha = 0.59, lambda = 5.18E-4 )</td>\n",
       "<td>nlambda = 100, lambda.max = 0.05615, lambda.min = 5.18E-4, lambda.1se = -1.0</td>\n",
       "<td>161</td>\n",
       "<td>36</td>\n",
       "<td>7</td>\n",
       "<td>py_15_sid_9664</td></tr>\n",
       "<tr><td>glm_grid_model_84</td>\n",
       "<td>binomial</td>\n",
       "<td>logit</td>\n",
       "<td>Elastic Net (alpha = 0.96, lambda = 3.8E-4 )</td>\n",
       "<td>nlambda = 100, lambda.max = 0.03451, lambda.min = 3.8E-4, lambda.1se = -1.0</td>\n",
       "<td>161</td>\n",
       "<td>30</td>\n",
       "<td>6</td>\n",
       "<td>py_15_sid_9664</td></tr>\n",
       "<tr><td>glm_grid_model_64</td>\n",
       "<td>binomial</td>\n",
       "<td>logit</td>\n",
       "<td>Elastic Net (alpha = 0.29, lambda = 7.81E-4 )</td>\n",
       "<td>nlambda = 100, lambda.max = 0.1142, lambda.min = 7.81E-4, lambda.1se = -1.0</td>\n",
       "<td>161</td>\n",
       "<td>45</td>\n",
       "<td>7</td>\n",
       "<td>py_15_sid_9664</td></tr>\n",
       "<tr><td>glm_grid_model_68</td>\n",
       "<td>binomial</td>\n",
       "<td>logit</td>\n",
       "<td>Elastic Net (alpha = 0.36, lambda = 6.96E-4 )</td>\n",
       "<td>nlambda = 100, lambda.max = 0.09202, lambda.min = 6.96E-4, lambda.1se = -1.0</td>\n",
       "<td>161</td>\n",
       "<td>42</td>\n",
       "<td>7</td>\n",
       "<td>py_15_sid_9664</td></tr>\n",
       "<tr><td>glm_grid_model_66</td>\n",
       "<td>binomial</td>\n",
       "<td>logit</td>\n",
       "<td>Elastic Net (alpha = 0.48, lambda = 5.93E-4 )</td>\n",
       "<td>nlambda = 100, lambda.max = 0.06902, lambda.min = 5.93E-4, lambda.1se = -1.0</td>\n",
       "<td>161</td>\n",
       "<td>36</td>\n",
       "<td>6</td>\n",
       "<td>py_15_sid_9664</td></tr>\n",
       "<tr><td>glm_grid_model_53</td>\n",
       "<td>binomial</td>\n",
       "<td>logit</td>\n",
       "<td>Elastic Net (alpha = 0.6, lambda = 5.69E-4 )</td>\n",
       "<td>nlambda = 100, lambda.max = 0.05521, lambda.min = 5.69E-4, lambda.1se = -1.0</td>\n",
       "<td>161</td>\n",
       "<td>35</td>\n",
       "<td>6</td>\n",
       "<td>py_15_sid_9664</td></tr>\n",
       "<tr><td>glm_grid_model_58</td>\n",
       "<td>binomial</td>\n",
       "<td>logit</td>\n",
       "<td>Elastic Net (alpha = 0.43, lambda = 6.94E-4 )</td>\n",
       "<td>nlambda = 100, lambda.max = 0.07704, lambda.min = 6.94E-4, lambda.1se = -1.0</td>\n",
       "<td>161</td>\n",
       "<td>36</td>\n",
       "<td>6</td>\n",
       "<td>py_15_sid_9664</td></tr>\n",
       "<tr><td>glm_grid_model_59</td>\n",
       "<td>binomial</td>\n",
       "<td>logit</td>\n",
       "<td>Elastic Net (alpha = 0.55, lambda = 6.13E-4 )</td>\n",
       "<td>nlambda = 100, lambda.max = 0.06023, lambda.min = 6.13E-4, lambda.1se = -1.0</td>\n",
       "<td>161</td>\n",
       "<td>35</td>\n",
       "<td>6</td>\n",
       "<td>py_15_sid_9664</td></tr>\n",
       "<tr><td>glm_grid_model_4</td>\n",
       "<td>binomial</td>\n",
       "<td>logit</td>\n",
       "<td>Elastic Net (alpha = 0.22, lambda = 9.91E-4 )</td>\n",
       "<td>nlambda = 100, lambda.max = 0.1506, lambda.min = 9.91E-4, lambda.1se = -1.0</td>\n",
       "<td>161</td>\n",
       "<td>46</td>\n",
       "<td>6</td>\n",
       "<td>py_15_sid_9664</td></tr>\n",
       "<tr><td>glm_grid_model_21</td>\n",
       "<td>binomial</td>\n",
       "<td>logit</td>\n",
       "<td>Elastic Net (alpha = 0.86, lambda = 4.86E-4 )</td>\n",
       "<td>nlambda = 100, lambda.max = 0.03852, lambda.min = 4.86E-4, lambda.1se = -1.0</td>\n",
       "<td>161</td>\n",
       "<td>30</td>\n",
       "<td>6</td>\n",
       "<td>py_15_sid_9664</td></tr>\n",
       "<tr><td>glm_grid_model_49</td>\n",
       "<td>binomial</td>\n",
       "<td>logit</td>\n",
       "<td>Elastic Net (alpha = 0.63, lambda = 5.75E-4 )</td>\n",
       "<td>nlambda = 100, lambda.max = 0.05258, lambda.min = 5.75E-4, lambda.1se = -1.0</td>\n",
       "<td>161</td>\n",
       "<td>31</td>\n",
       "<td>6</td>\n",
       "<td>py_15_sid_9664</td></tr>\n",
       "<tr><td>glm_grid_model_88</td>\n",
       "<td>binomial</td>\n",
       "<td>logit</td>\n",
       "<td>Elastic Net (alpha = 0.29, lambda = 8.94E-4 )</td>\n",
       "<td>nlambda = 100, lambda.max = 0.1142, lambda.min = 8.94E-4, lambda.1se = -1.0</td>\n",
       "<td>161</td>\n",
       "<td>40</td>\n",
       "<td>7</td>\n",
       "<td>py_15_sid_9664</td></tr>\n",
       "<tr><td>glm_grid_model_13</td>\n",
       "<td>binomial</td>\n",
       "<td>logit</td>\n",
       "<td>Elastic Net (alpha = 0.73, lambda = 5.53E-4 )</td>\n",
       "<td>nlambda = 100, lambda.max = 0.04538, lambda.min = 5.53E-4, lambda.1se = -1.0</td>\n",
       "<td>161</td>\n",
       "<td>31</td>\n",
       "<td>6</td>\n",
       "<td>py_15_sid_9664</td></tr>\n",
       "<tr><td>glm_grid_model_89</td>\n",
       "<td>binomial</td>\n",
       "<td>logit</td>\n",
       "<td>Elastic Net (alpha = 0.65, lambda = 5.88E-4 )</td>\n",
       "<td>nlambda = 100, lambda.max = 0.05097, lambda.min = 5.88E-4, lambda.1se = -1.0</td>\n",
       "<td>161</td>\n",
       "<td>31</td>\n",
       "<td>6</td>\n",
       "<td>py_15_sid_9664</td></tr>\n",
       "<tr><td>glm_grid_model_6</td>\n",
       "<td>binomial</td>\n",
       "<td>logit</td>\n",
       "<td>Elastic Net (alpha = 0.5, lambda = 6.93E-4 )</td>\n",
       "<td>nlambda = 100, lambda.max = 0.06626, lambda.min = 6.93E-4, lambda.1se = -1.0</td>\n",
       "<td>161</td>\n",
       "<td>33</td>\n",
       "<td>6</td>\n",
       "<td>py_15_sid_9664</td></tr>\n",
       "<tr><td>glm_grid_model_19</td>\n",
       "<td>binomial</td>\n",
       "<td>logit</td>\n",
       "<td>Elastic Net (alpha = 0.87, lambda = 5.1E-4 )</td>\n",
       "<td>nlambda = 100, lambda.max = 0.03808, lambda.min = 5.1E-4, lambda.1se = -1.0</td>\n",
       "<td>161</td>\n",
       "<td>30</td>\n",
       "<td>6</td>\n",
       "<td>py_15_sid_9664</td></tr>\n",
       "<tr><td>glm_grid_model_63</td>\n",
       "<td>binomial</td>\n",
       "<td>logit</td>\n",
       "<td>Elastic Net (alpha = 0.88, lambda = 5.18E-4 )</td>\n",
       "<td>nlambda = 100, lambda.max = 0.03765, lambda.min = 5.18E-4, lambda.1se = -1.0</td>\n",
       "<td>161</td>\n",
       "<td>30</td>\n",
       "<td>6</td>\n",
       "<td>py_15_sid_9664</td></tr>\n",
       "<tr><td>glm_grid_model_74</td>\n",
       "<td>binomial</td>\n",
       "<td>logit</td>\n",
       "<td>Elastic Net (alpha = 0.42, lambda = 8.13E-4 )</td>\n",
       "<td>nlambda = 100, lambda.max = 0.07888, lambda.min = 8.13E-4, lambda.1se = -1.0</td>\n",
       "<td>161</td>\n",
       "<td>33</td>\n",
       "<td>6</td>\n",
       "<td>py_15_sid_9664</td></tr>\n",
       "<tr><td>glm_grid_model_95</td>\n",
       "<td>binomial</td>\n",
       "<td>logit</td>\n",
       "<td>Elastic Net (alpha = 0.32, lambda = 9.5E-4 )</td>\n",
       "<td>nlambda = 100, lambda.max = 0.1035, lambda.min = 9.5E-4, lambda.1se = -1.0</td>\n",
       "<td>161</td>\n",
       "<td>35</td>\n",
       "<td>6</td>\n",
       "<td>py_15_sid_9664</td></tr>\n",
       "<tr><td>glm_grid_model_18</td>\n",
       "<td>binomial</td>\n",
       "<td>logit</td>\n",
       "<td>Elastic Net (alpha = 0.66, lambda = 6.59E-4 )</td>\n",
       "<td>nlambda = 100, lambda.max = 0.05019, lambda.min = 6.59E-4, lambda.1se = -1.0</td>\n",
       "<td>161</td>\n",
       "<td>30</td>\n",
       "<td>6</td>\n",
       "<td>py_15_sid_9664</td></tr>\n",
       "<tr><td>glm_grid_model_40</td>\n",
       "<td>binomial</td>\n",
       "<td>logit</td>\n",
       "<td>Elastic Net (alpha = 0.43, lambda = 8.42E-4 )</td>\n",
       "<td>nlambda = 100, lambda.max = 0.07704, lambda.min = 8.42E-4, lambda.1se = -1.0</td>\n",
       "<td>161</td>\n",
       "<td>31</td>\n",
       "<td>6</td>\n",
       "<td>py_15_sid_9664</td></tr>\n",
       "<tr><td>glm_grid_model_9</td>\n",
       "<td>binomial</td>\n",
       "<td>logit</td>\n",
       "<td>Elastic Net (alpha = 0.59, lambda = 7.31E-4 )</td>\n",
       "<td>nlambda = 100, lambda.max = 0.05615, lambda.min = 7.31E-4, lambda.1se = -1.0</td>\n",
       "<td>161</td>\n",
       "<td>30</td>\n",
       "<td>6</td>\n",
       "<td>py_15_sid_9664</td></tr>\n",
       "<tr><td>glm_grid_model_24</td>\n",
       "<td>binomial</td>\n",
       "<td>logit</td>\n",
       "<td>Elastic Net (alpha = 0.41, lambda = 8.91E-4 )</td>\n",
       "<td>nlambda = 100, lambda.max = 0.0808, lambda.min = 8.91E-4, lambda.1se = -1.0</td>\n",
       "<td>161</td>\n",
       "<td>30</td>\n",
       "<td>6</td>\n",
       "<td>py_15_sid_9664</td></tr>\n",
       "<tr><td>glm_grid_model_62</td>\n",
       "<td>binomial</td>\n",
       "<td>logit</td>\n",
       "<td>Elastic Net (alpha = 0.94, lambda = 5.09E-4 )</td>\n",
       "<td>nlambda = 100, lambda.max = 0.03524, lambda.min = 5.09E-4, lambda.1se = -1.0</td>\n",
       "<td>161</td>\n",
       "<td>28</td>\n",
       "<td>6</td>\n",
       "<td>py_15_sid_9664</td></tr>\n",
       "<tr><td>glm_grid_model_5</td>\n",
       "<td>binomial</td>\n",
       "<td>logit</td>\n",
       "<td>Elastic Net (alpha = 0.54, lambda = 7.8E-4 )</td>\n",
       "<td>nlambda = 100, lambda.max = 0.06135, lambda.min = 7.8E-4, lambda.1se = -1.0</td>\n",
       "<td>161</td>\n",
       "<td>30</td>\n",
       "<td>6</td>\n",
       "<td>py_15_sid_9664</td></tr>\n",
       "<tr><td>glm_grid_model_76</td>\n",
       "<td>binomial</td>\n",
       "<td>logit</td>\n",
       "<td>Elastic Net (alpha = 0.41, lambda = 9.56E-4 )</td>\n",
       "<td>nlambda = 100, lambda.max = 0.0808, lambda.min = 9.56E-4, lambda.1se = -1.0</td>\n",
       "<td>161</td>\n",
       "<td>30</td>\n",
       "<td>6</td>\n",
       "<td>py_15_sid_9664</td></tr>\n",
       "<tr><td>glm_grid_model_50</td>\n",
       "<td>binomial</td>\n",
       "<td>logit</td>\n",
       "<td>Elastic Net (alpha = 0.41, lambda = 9.93E-4 )</td>\n",
       "<td>nlambda = 100, lambda.max = 0.0808, lambda.min = 9.93E-4, lambda.1se = -1.0</td>\n",
       "<td>161</td>\n",
       "<td>30</td>\n",
       "<td>6</td>\n",
       "<td>py_15_sid_9664</td></tr>\n",
       "<tr><td>glm_grid_model_32</td>\n",
       "<td>binomial</td>\n",
       "<td>logit</td>\n",
       "<td>Elastic Net (alpha = 0.6, lambda = 8.16E-4 )</td>\n",
       "<td>nlambda = 100, lambda.max = 0.05521, lambda.min = 8.16E-4, lambda.1se = -1.0</td>\n",
       "<td>161</td>\n",
       "<td>28</td>\n",
       "<td>7</td>\n",
       "<td>py_15_sid_9664</td></tr>\n",
       "<tr><td>glm_grid_model_97</td>\n",
       "<td>binomial</td>\n",
       "<td>logit</td>\n",
       "<td>Elastic Net (alpha = 0.55, lambda = 8.76E-4 )</td>\n",
       "<td>nlambda = 100, lambda.max = 0.06023, lambda.min = 8.76E-4, lambda.1se = -1.0</td>\n",
       "<td>161</td>\n",
       "<td>28</td>\n",
       "<td>6</td>\n",
       "<td>py_15_sid_9664</td></tr>\n",
       "<tr><td>glm_grid_model_57</td>\n",
       "<td>binomial</td>\n",
       "<td>logit</td>\n",
       "<td>Elastic Net (alpha = 0.84, lambda = 6.74E-4 )</td>\n",
       "<td>nlambda = 100, lambda.max = 0.03944, lambda.min = 6.74E-4, lambda.1se = -1.0</td>\n",
       "<td>161</td>\n",
       "<td>25</td>\n",
       "<td>6</td>\n",
       "<td>py_15_sid_9664</td></tr>\n",
       "<tr><td>glm_grid_model_91</td>\n",
       "<td>binomial</td>\n",
       "<td>logit</td>\n",
       "<td>Elastic Net (alpha = 0.52, lambda = 9.45E-4 )</td>\n",
       "<td>nlambda = 100, lambda.max = 0.06371, lambda.min = 9.45E-4, lambda.1se = -1.0</td>\n",
       "<td>161</td>\n",
       "<td>28</td>\n",
       "<td>7</td>\n",
       "<td>py_15_sid_9664</td></tr>\n",
       "<tr><td>glm_grid_model_69</td>\n",
       "<td>binomial</td>\n",
       "<td>logit</td>\n",
       "<td>Elastic Net (alpha = 0.57, lambda = 9.53E-4 )</td>\n",
       "<td>nlambda = 100, lambda.max = 0.05812, lambda.min = 9.53E-4, lambda.1se = -1.0</td>\n",
       "<td>161</td>\n",
       "<td>26</td>\n",
       "<td>6</td>\n",
       "<td>py_15_sid_9664</td></tr>\n",
       "<tr><td>glm_grid_model_56</td>\n",
       "<td>binomial</td>\n",
       "<td>logit</td>\n",
       "<td>Elastic Net (alpha = 0.65, lambda = 9.38E-4 )</td>\n",
       "<td>nlambda = 100, lambda.max = 0.05097, lambda.min = 9.38E-4, lambda.1se = -1.0</td>\n",
       "<td>161</td>\n",
       "<td>25</td>\n",
       "<td>6</td>\n",
       "<td>py_15_sid_9664</td></tr>\n",
       "<tr><td>glm_grid_model_85</td>\n",
       "<td>binomial</td>\n",
       "<td>logit</td>\n",
       "<td>Elastic Net (alpha = 0.95, lambda = 7.41E-4 )</td>\n",
       "<td>nlambda = 100, lambda.max = 0.03487, lambda.min = 7.41E-4, lambda.1se = -1.0</td>\n",
       "<td>161</td>\n",
       "<td>21</td>\n",
       "<td>6</td>\n",
       "<td>py_15_sid_9664</td></tr>\n",
       "<tr><td>glm_grid_model_39</td>\n",
       "<td>binomial</td>\n",
       "<td>logit</td>\n",
       "<td>Elastic Net (alpha = 0.86, lambda = 8.06E-4 )</td>\n",
       "<td>nlambda = 100, lambda.max = 0.03852, lambda.min = 8.06E-4, lambda.1se = -1.0</td>\n",
       "<td>161</td>\n",
       "<td>21</td>\n",
       "<td>6</td>\n",
       "<td>py_15_sid_9664</td></tr>\n",
       "<tr><td>glm_grid_model_99</td>\n",
       "<td>binomial</td>\n",
       "<td>logit</td>\n",
       "<td>Elastic Net (alpha = 0.92, lambda = 8.6E-4 )</td>\n",
       "<td>nlambda = 100, lambda.max = 0.03601, lambda.min = 8.6E-4, lambda.1se = -1.0</td>\n",
       "<td>161</td>\n",
       "<td>19</td>\n",
       "<td>7</td>\n",
       "<td>py_15_sid_9664</td></tr>\n",
       "<tr><td>glm_grid_model_15</td>\n",
       "<td>binomial</td>\n",
       "<td>logit</td>\n",
       "<td>Elastic Net (alpha = 0.74, lambda = 9.87E-4 )</td>\n",
       "<td>nlambda = 100, lambda.max = 0.04477, lambda.min = 9.87E-4, lambda.1se = -1.0</td>\n",
       "<td>161</td>\n",
       "<td>21</td>\n",
       "<td>6</td>\n",
       "<td>py_15_sid_9664</td></tr>\n",
       "<tr><td>glm_grid_model_23</td>\n",
       "<td>binomial</td>\n",
       "<td>logit</td>\n",
       "<td>Elastic Net (alpha = 0.88, lambda = 9.21E-4 )</td>\n",
       "<td>nlambda = 100, lambda.max = 0.03765, lambda.min = 9.21E-4, lambda.1se = -1.0</td>\n",
       "<td>161</td>\n",
       "<td>19</td>\n",
       "<td>6</td>\n",
       "<td>py_15_sid_9664</td></tr>\n",
       "<tr><td>glm_grid_model_7</td>\n",
       "<td>binomial</td>\n",
       "<td>logit</td>\n",
       "<td>Elastic Net (alpha = 0.88, lambda = 9.48E-4 )</td>\n",
       "<td>nlambda = 100, lambda.max = 0.03765, lambda.min = 9.48E-4, lambda.1se = -1.0</td>\n",
       "<td>161</td>\n",
       "<td>19</td>\n",
       "<td>6</td>\n",
       "<td>py_15_sid_9664</td></tr></table></div>"
      ],
      "text/plain": [
       "Model Id            family    link    regularization                                 lambda_search                                                                 number_of_predictors_total    number_of_active_predictors    number_of_iterations    training_frame\n",
       "------------------  --------  ------  ---------------------------------------------  ----------------------------------------------------------------------------  ----------------------------  -----------------------------  ----------------------  ----------------\n",
       "glm_grid_model_61   binomial  logit   Elastic Net (alpha = 0.87, lambda = 5.0E-6 )   nlambda = 100, lambda.max = 0.03808, lambda.min = 5.0E-6, lambda.1se = -1.0   161                           143                            7                       py_15_sid_9664\n",
       "glm_grid_model_46   binomial  logit   Elastic Net (alpha = 0.4, lambda = 1.8E-5 )    nlambda = 100, lambda.max = 0.08282, lambda.min = 1.8E-5, lambda.1se = -1.0   161                           137                            7                       py_15_sid_9664\n",
       "glm_grid_model_48   binomial  logit   Elastic Net (alpha = 0.07, lambda = 3.7E-5 )   nlambda = 100, lambda.max = 0.4733, lambda.min = 3.7E-5, lambda.1se = -1.0    161                           145                            5                       py_15_sid_9664\n",
       "glm_grid_model_96   binomial  logit   Elastic Net (alpha = 0.48, lambda = 2.9E-5 )   nlambda = 100, lambda.max = 0.06902, lambda.min = 2.9E-5, lambda.1se = -1.0   161                           110                            7                       py_15_sid_9664\n",
       "glm_grid_model_72   binomial  logit   Elastic Net (alpha = 0.07, lambda = 5.6E-5 )   nlambda = 100, lambda.max = 0.4733, lambda.min = 5.6E-5, lambda.1se = -1.0    161                           137                            5                       py_15_sid_9664\n",
       "glm_grid_model_37   binomial  logit   Elastic Net (alpha = 0.88, lambda = 3.1E-5 )   nlambda = 100, lambda.max = 0.03765, lambda.min = 3.1E-5, lambda.1se = -1.0   161                           93                             7                       py_15_sid_9664\n",
       "glm_grid_model_86   binomial  logit   Elastic Net (alpha = 0.18, lambda = 8.1E-5 )   nlambda = 100, lambda.max = 0.184, lambda.min = 8.1E-5, lambda.1se = -1.0     161                           108                            7                       py_15_sid_9664\n",
       "glm_grid_model_30   binomial  logit   Elastic Net (alpha = 0.15, lambda = 1.31E-4 )  nlambda = 100, lambda.max = 0.2209, lambda.min = 1.31E-4, lambda.1se = -1.0   161                           105                            7                       py_15_sid_9664\n",
       "glm_grid_model_38   binomial  logit   Elastic Net (alpha = 0.1, lambda = 1.59E-4 )   nlambda = 100, lambda.max = 0.3313, lambda.min = 1.59E-4, lambda.1se = -1.0   161                           110                            7                       py_15_sid_9664\n",
       "glm_grid_model_28   binomial  logit   Elastic Net (alpha = 0.53, lambda = 9.5E-5 )   nlambda = 100, lambda.max = 0.06251, lambda.min = 9.5E-5, lambda.1se = -1.0   161                           80                             7                       py_15_sid_9664\n",
       "glm_grid_model_78   binomial  logit   Elastic Net (alpha = 0.06, lambda = 2.12E-4 )  nlambda = 100, lambda.max = 0.5521, lambda.min = 2.12E-4, lambda.1se = -1.0   161                           119                            7                       py_15_sid_9664\n",
       "glm_grid_model_67   binomial  logit   Elastic Net (alpha = 0.41, lambda = 1.37E-4 )  nlambda = 100, lambda.max = 0.0808, lambda.min = 1.37E-4, lambda.1se = -1.0   161                           77                             7                       py_15_sid_9664\n",
       "glm_grid_model_100  binomial  logit   Elastic Net (alpha = 0.93, lambda = 7.7E-5 )   nlambda = 100, lambda.max = 0.03562, lambda.min = 7.7E-5, lambda.1se = -1.0   161                           71                             7                       py_15_sid_9664\n",
       "glm_grid_model_34   binomial  logit   Elastic Net (alpha = 0.65, lambda = 1.07E-4 )  nlambda = 100, lambda.max = 0.05097, lambda.min = 1.07E-4, lambda.1se = -1.0  161                           69                             7                       py_15_sid_9664\n",
       "glm_grid_model_42   binomial  logit   Elastic Net (alpha = 0.52, lambda = 1.37E-4 )  nlambda = 100, lambda.max = 0.06371, lambda.min = 1.37E-4, lambda.1se = -1.0  161                           70                             7                       py_15_sid_9664\n",
       "glm_grid_model_82   binomial  logit   Elastic Net (alpha = 0.56, lambda = 1.64E-4 )  nlambda = 100, lambda.max = 0.05916, lambda.min = 1.64E-4, lambda.1se = -1.0  161                           68                             7                       py_15_sid_9664\n",
       "glm_grid_model_51   binomial  logit   Elastic Net (alpha = 0.47, lambda = 1.89E-4 )  nlambda = 100, lambda.max = 0.07048, lambda.min = 1.89E-4, lambda.1se = -1.0  161                           69                             7                       py_15_sid_9664\n",
       "glm_grid_model_22   binomial  logit   Elastic Net (alpha = 0.31, lambda = 2.3E-4 )   nlambda = 100, lambda.max = 0.1069, lambda.min = 2.3E-4, lambda.1se = -1.0    161                           72                             7                       py_15_sid_9664\n",
       "glm_grid_model_16   binomial  logit   Elastic Net (alpha = 0.05, lambda = 4.17E-4 )  nlambda = 100, lambda.max = 0.6626, lambda.min = 4.17E-4, lambda.1se = -1.0   161                           110                            6                       py_15_sid_9664\n",
       "glm_grid_model_17   binomial  logit   Elastic Net (alpha = 0.18, lambda = 3.47E-4 )  nlambda = 100, lambda.max = 0.184, lambda.min = 3.47E-4, lambda.1se = -1.0    161                           80                             7                       py_15_sid_9664\n",
       "glm_grid_model_93   binomial  logit   Elastic Net (alpha = 0.92, lambda = 1.45E-4 )  nlambda = 100, lambda.max = 0.03601, lambda.min = 1.45E-4, lambda.1se = -1.0  161                           58                             8                       py_15_sid_9664\n",
       "glm_grid_model_73   binomial  logit   Elastic Net (alpha = 0.11, lambda = 4.88E-4 )  nlambda = 100, lambda.max = 0.3012, lambda.min = 4.88E-4, lambda.1se = -1.0   161                           85                             6                       py_15_sid_9664\n",
       "glm_grid_model_11   binomial  logit   Elastic Net (alpha = 0.75, lambda = 1.81E-4 )  nlambda = 100, lambda.max = 0.04417, lambda.min = 1.81E-4, lambda.1se = -1.0  161                           53                             7                       py_15_sid_9664\n",
       "glm_grid_model_2    binomial  logit   Elastic Net (alpha = 0.78, lambda = 1.88E-4 )  nlambda = 100, lambda.max = 0.04247, lambda.min = 1.88E-4, lambda.1se = -1.0  161                           53                             8                       py_15_sid_9664\n",
       "glm_grid_model_14   binomial  logit   Elastic Net (alpha = 0.16, lambda = 4.93E-4 )  nlambda = 100, lambda.max = 0.207, lambda.min = 4.93E-4, lambda.1se = -1.0    161                           77                             6                       py_15_sid_9664\n",
       "glm_grid_model_83   binomial  logit   Elastic Net (alpha = 0.65, lambda = 2.26E-4 )  nlambda = 100, lambda.max = 0.05097, lambda.min = 2.26E-4, lambda.1se = -1.0  161                           53                             8                       py_15_sid_9664\n",
       "glm_grid_model_20   binomial  logit   Elastic Net (alpha = 0.53, lambda = 2.9E-4 )   nlambda = 100, lambda.max = 0.06251, lambda.min = 2.9E-4, lambda.1se = -1.0   161                           54                             8                       py_15_sid_9664\n",
       "glm_grid_model_1    binomial  logit   Elastic Net (alpha = 0.09, lambda = 6.68E-4 )  nlambda = 100, lambda.max = 0.3681, lambda.min = 6.68E-4, lambda.1se = -1.0   161                           84                             6                       py_15_sid_9664\n",
       "glm_grid_model_98   binomial  logit   Elastic Net (alpha = 0.17, lambda = 5.46E-4 )  nlambda = 100, lambda.max = 0.1949, lambda.min = 5.46E-4, lambda.1se = -1.0   161                           72                             6                       py_15_sid_9664\n",
       "glm_grid_model_54   binomial  logit   Elastic Net (alpha = 0.13, lambda = 6.33E-4 )  nlambda = 100, lambda.max = 0.2548, lambda.min = 6.33E-4, lambda.1se = -1.0   161                           79                             7                       py_15_sid_9664\n",
       "glm_grid_model_79   binomial  logit   Elastic Net (alpha = 0.07, lambda = 7.32E-4 )  nlambda = 100, lambda.max = 0.4733, lambda.min = 7.32E-4, lambda.1se = -1.0   161                           88                             6                       py_15_sid_9664\n",
       "glm_grid_model_35   binomial  logit   Elastic Net (alpha = 0.99, lambda = 2.06E-4 )  nlambda = 100, lambda.max = 0.03346, lambda.min = 2.06E-4, lambda.1se = -1.0  161                           46                             7                       py_15_sid_9664\n",
       "glm_grid_model_92   binomial  logit   Elastic Net (alpha = 0.02, lambda = 8.87E-4 )  nlambda = 100, lambda.max = 1.6564, lambda.min = 8.87E-4, lambda.1se = -1.0   161                           116                            5                       py_15_sid_9664\n",
       "glm_grid_model_60   binomial  logit   Elastic Net (alpha = 0.05, lambda = 8.0E-4 )   nlambda = 100, lambda.max = 0.6626, lambda.min = 8.0E-4, lambda.1se = -1.0    161                           90                             6                       py_15_sid_9664\n",
       "glm_grid_model_90   binomial  logit   Elastic Net (alpha = 0.02, lambda = 9.92E-4 )  nlambda = 100, lambda.max = 1.6564, lambda.min = 9.92E-4, lambda.1se = -1.0   161                           114                            5                       py_15_sid_9664\n",
       "glm_grid_model_71   binomial  logit   Elastic Net (alpha = 0.09, lambda = 7.94E-4 )  nlambda = 100, lambda.max = 0.3681, lambda.min = 7.94E-4, lambda.1se = -1.0   161                           81                             6                       py_15_sid_9664\n",
       "glm_grid_model_26   binomial  logit   Elastic Net (alpha = 0.93, lambda = 2.45E-4 )  nlambda = 100, lambda.max = 0.03562, lambda.min = 2.45E-4, lambda.1se = -1.0  161                           44                             7                       py_15_sid_9664\n",
       "glm_grid_model_80   binomial  logit   Elastic Net (alpha = 0.23, lambda = 5.77E-4 )  nlambda = 100, lambda.max = 0.144, lambda.min = 5.77E-4, lambda.1se = -1.0    161                           64                             6                       py_15_sid_9664\n",
       "glm_grid_model_77   binomial  logit   Elastic Net (alpha = 0.6, lambda = 3.29E-4 )   nlambda = 100, lambda.max = 0.05521, lambda.min = 3.29E-4, lambda.1se = -1.0  161                           47                             7                       py_15_sid_9664\n",
       "glm_grid_model_87   binomial  logit   Elastic Net (alpha = 0.52, lambda = 3.66E-4 )  nlambda = 100, lambda.max = 0.06371, lambda.min = 3.66E-4, lambda.1se = -1.0  161                           48                             6                       py_15_sid_9664\n",
       "glm_grid_model_45   binomial  logit   Elastic Net (alpha = 0.25, lambda = 5.9E-4 )   nlambda = 100, lambda.max = 0.1325, lambda.min = 5.9E-4, lambda.1se = -1.0    161                           59                             6                       py_15_sid_9664\n",
       "glm_grid_model_81   binomial  logit   Elastic Net (alpha = 0.8, lambda = 3.02E-4 )   nlambda = 100, lambda.max = 0.04141, lambda.min = 3.02E-4, lambda.1se = -1.0  161                           43                             7                       py_15_sid_9664\n",
       "glm_grid_model_33   binomial  logit   Elastic Net (alpha = 0.64, lambda = 3.47E-4 )  nlambda = 100, lambda.max = 0.05176, lambda.min = 3.47E-4, lambda.1se = -1.0  161                           45                             6                       py_15_sid_9664\n",
       "glm_grid_model_43   binomial  logit   Elastic Net (alpha = 0.1, lambda = 8.66E-4 )   nlambda = 100, lambda.max = 0.3313, lambda.min = 8.66E-4, lambda.1se = -1.0   161                           75                             6                       py_15_sid_9664\n",
       "glm_grid_model_55   binomial  logit   Elastic Net (alpha = 0.66, lambda = 3.52E-4 )  nlambda = 100, lambda.max = 0.05019, lambda.min = 3.52E-4, lambda.1se = -1.0  161                           43                             6                       py_15_sid_9664\n",
       "glm_grid_model_12   binomial  logit   Elastic Net (alpha = 0.11, lambda = 8.43E-4 )  nlambda = 100, lambda.max = 0.3012, lambda.min = 8.43E-4, lambda.1se = -1.0   161                           73                             6                       py_15_sid_9664\n",
       "glm_grid_model_65   binomial  logit   Elastic Net (alpha = 0.86, lambda = 3.02E-4 )  nlambda = 100, lambda.max = 0.03852, lambda.min = 3.02E-4, lambda.1se = -1.0  161                           40                             7                       py_15_sid_9664\n",
       "glm_grid_model_10   binomial  logit   Elastic Net (alpha = 0.83, lambda = 3.23E-4 )  nlambda = 100, lambda.max = 0.03991, lambda.min = 3.23E-4, lambda.1se = -1.0  161                           38                             6                       py_15_sid_9664\n",
       "glm_grid_model_8    binomial  logit   Elastic Net (alpha = 0.94, lambda = 3.05E-4 )  nlambda = 100, lambda.max = 0.03524, lambda.min = 3.05E-4, lambda.1se = -1.0  161                           36                             8                       py_15_sid_9664\n",
       "glm_grid_model_3    binomial  logit   Elastic Net (alpha = 0.69, lambda = 3.77E-4 )  nlambda = 100, lambda.max = 0.04801, lambda.min = 3.77E-4, lambda.1se = -1.0  161                           39                             6                       py_15_sid_9664\n",
       "glm_grid_model_70   binomial  logit   Elastic Net (alpha = 0.71, lambda = 3.86E-4 )  nlambda = 100, lambda.max = 0.04666, lambda.min = 3.86E-4, lambda.1se = -1.0  161                           38                             6                       py_15_sid_9664\n",
       "glm_grid_model_44   binomial  logit   Elastic Net (alpha = 0.19, lambda = 7.88E-4 )  nlambda = 100, lambda.max = 0.1744, lambda.min = 7.88E-4, lambda.1se = -1.0   161                           60                             6                       py_15_sid_9664\n",
       "glm_grid_model_41   binomial  logit   Elastic Net (alpha = 0.67, lambda = 3.99E-4 )  nlambda = 100, lambda.max = 0.04944, lambda.min = 3.99E-4, lambda.1se = -1.0  161                           38                             6                       py_15_sid_9664\n",
       "glm_grid_model_36   binomial  logit   Elastic Net (alpha = 0.94, lambda = 3.4E-4 )   nlambda = 100, lambda.max = 0.03524, lambda.min = 3.4E-4, lambda.1se = -1.0   161                           35                             7                       py_15_sid_9664\n",
       "glm_grid_model_94   binomial  logit   Elastic Net (alpha = 0.15, lambda = 9.21E-4 )  nlambda = 100, lambda.max = 0.2209, lambda.min = 9.21E-4, lambda.1se = -1.0   161                           64                             6                       py_15_sid_9664\n",
       "glm_grid_model_29   binomial  logit   Elastic Net (alpha = 0.59, lambda = 4.59E-4 )  nlambda = 100, lambda.max = 0.05615, lambda.min = 4.59E-4, lambda.1se = -1.0  161                           39                             6                       py_15_sid_9664\n",
       "glm_grid_model_25   binomial  logit   Elastic Net (alpha = 0.3, lambda = 6.75E-4 )   nlambda = 100, lambda.max = 0.1104, lambda.min = 6.75E-4, lambda.1se = -1.0   161                           47                             7                       py_15_sid_9664\n",
       "glm_grid_model_52   binomial  logit   Elastic Net (alpha = 0.34, lambda = 6.66E-4 )  nlambda = 100, lambda.max = 0.09743, lambda.min = 6.66E-4, lambda.1se = -1.0  161                           45                             7                       py_15_sid_9664\n",
       "glm_grid_model_47   binomial  logit   Elastic Net (alpha = 0.19, lambda = 8.82E-4 )  nlambda = 100, lambda.max = 0.1744, lambda.min = 8.82E-4, lambda.1se = -1.0   161                           56                             6                       py_15_sid_9664\n",
       "glm_grid_model_31   binomial  logit   Elastic Net (alpha = 0.58, lambda = 5.09E-4 )  nlambda = 100, lambda.max = 0.05712, lambda.min = 5.09E-4, lambda.1se = -1.0  161                           37                             6                       py_15_sid_9664\n",
       "glm_grid_model_27   binomial  logit   Elastic Net (alpha = 0.81, lambda = 4.25E-4 )  nlambda = 100, lambda.max = 0.0409, lambda.min = 4.25E-4, lambda.1se = -1.0   161                           34                             6                       py_15_sid_9664\n",
       "glm_grid_model_75   binomial  logit   Elastic Net (alpha = 0.59, lambda = 5.18E-4 )  nlambda = 100, lambda.max = 0.05615, lambda.min = 5.18E-4, lambda.1se = -1.0  161                           36                             7                       py_15_sid_9664\n",
       "glm_grid_model_84   binomial  logit   Elastic Net (alpha = 0.96, lambda = 3.8E-4 )   nlambda = 100, lambda.max = 0.03451, lambda.min = 3.8E-4, lambda.1se = -1.0   161                           30                             6                       py_15_sid_9664\n",
       "glm_grid_model_64   binomial  logit   Elastic Net (alpha = 0.29, lambda = 7.81E-4 )  nlambda = 100, lambda.max = 0.1142, lambda.min = 7.81E-4, lambda.1se = -1.0   161                           45                             7                       py_15_sid_9664\n",
       "glm_grid_model_68   binomial  logit   Elastic Net (alpha = 0.36, lambda = 6.96E-4 )  nlambda = 100, lambda.max = 0.09202, lambda.min = 6.96E-4, lambda.1se = -1.0  161                           42                             7                       py_15_sid_9664\n",
       "glm_grid_model_66   binomial  logit   Elastic Net (alpha = 0.48, lambda = 5.93E-4 )  nlambda = 100, lambda.max = 0.06902, lambda.min = 5.93E-4, lambda.1se = -1.0  161                           36                             6                       py_15_sid_9664\n",
       "glm_grid_model_53   binomial  logit   Elastic Net (alpha = 0.6, lambda = 5.69E-4 )   nlambda = 100, lambda.max = 0.05521, lambda.min = 5.69E-4, lambda.1se = -1.0  161                           35                             6                       py_15_sid_9664\n",
       "glm_grid_model_58   binomial  logit   Elastic Net (alpha = 0.43, lambda = 6.94E-4 )  nlambda = 100, lambda.max = 0.07704, lambda.min = 6.94E-4, lambda.1se = -1.0  161                           36                             6                       py_15_sid_9664\n",
       "glm_grid_model_59   binomial  logit   Elastic Net (alpha = 0.55, lambda = 6.13E-4 )  nlambda = 100, lambda.max = 0.06023, lambda.min = 6.13E-4, lambda.1se = -1.0  161                           35                             6                       py_15_sid_9664\n",
       "glm_grid_model_4    binomial  logit   Elastic Net (alpha = 0.22, lambda = 9.91E-4 )  nlambda = 100, lambda.max = 0.1506, lambda.min = 9.91E-4, lambda.1se = -1.0   161                           46                             6                       py_15_sid_9664\n",
       "glm_grid_model_21   binomial  logit   Elastic Net (alpha = 0.86, lambda = 4.86E-4 )  nlambda = 100, lambda.max = 0.03852, lambda.min = 4.86E-4, lambda.1se = -1.0  161                           30                             6                       py_15_sid_9664\n",
       "glm_grid_model_49   binomial  logit   Elastic Net (alpha = 0.63, lambda = 5.75E-4 )  nlambda = 100, lambda.max = 0.05258, lambda.min = 5.75E-4, lambda.1se = -1.0  161                           31                             6                       py_15_sid_9664\n",
       "glm_grid_model_88   binomial  logit   Elastic Net (alpha = 0.29, lambda = 8.94E-4 )  nlambda = 100, lambda.max = 0.1142, lambda.min = 8.94E-4, lambda.1se = -1.0   161                           40                             7                       py_15_sid_9664\n",
       "glm_grid_model_13   binomial  logit   Elastic Net (alpha = 0.73, lambda = 5.53E-4 )  nlambda = 100, lambda.max = 0.04538, lambda.min = 5.53E-4, lambda.1se = -1.0  161                           31                             6                       py_15_sid_9664\n",
       "glm_grid_model_89   binomial  logit   Elastic Net (alpha = 0.65, lambda = 5.88E-4 )  nlambda = 100, lambda.max = 0.05097, lambda.min = 5.88E-4, lambda.1se = -1.0  161                           31                             6                       py_15_sid_9664\n",
       "glm_grid_model_6    binomial  logit   Elastic Net (alpha = 0.5, lambda = 6.93E-4 )   nlambda = 100, lambda.max = 0.06626, lambda.min = 6.93E-4, lambda.1se = -1.0  161                           33                             6                       py_15_sid_9664\n",
       "glm_grid_model_19   binomial  logit   Elastic Net (alpha = 0.87, lambda = 5.1E-4 )   nlambda = 100, lambda.max = 0.03808, lambda.min = 5.1E-4, lambda.1se = -1.0   161                           30                             6                       py_15_sid_9664\n",
       "glm_grid_model_63   binomial  logit   Elastic Net (alpha = 0.88, lambda = 5.18E-4 )  nlambda = 100, lambda.max = 0.03765, lambda.min = 5.18E-4, lambda.1se = -1.0  161                           30                             6                       py_15_sid_9664\n",
       "glm_grid_model_74   binomial  logit   Elastic Net (alpha = 0.42, lambda = 8.13E-4 )  nlambda = 100, lambda.max = 0.07888, lambda.min = 8.13E-4, lambda.1se = -1.0  161                           33                             6                       py_15_sid_9664\n",
       "glm_grid_model_95   binomial  logit   Elastic Net (alpha = 0.32, lambda = 9.5E-4 )   nlambda = 100, lambda.max = 0.1035, lambda.min = 9.5E-4, lambda.1se = -1.0    161                           35                             6                       py_15_sid_9664\n",
       "glm_grid_model_18   binomial  logit   Elastic Net (alpha = 0.66, lambda = 6.59E-4 )  nlambda = 100, lambda.max = 0.05019, lambda.min = 6.59E-4, lambda.1se = -1.0  161                           30                             6                       py_15_sid_9664\n",
       "glm_grid_model_40   binomial  logit   Elastic Net (alpha = 0.43, lambda = 8.42E-4 )  nlambda = 100, lambda.max = 0.07704, lambda.min = 8.42E-4, lambda.1se = -1.0  161                           31                             6                       py_15_sid_9664\n",
       "glm_grid_model_9    binomial  logit   Elastic Net (alpha = 0.59, lambda = 7.31E-4 )  nlambda = 100, lambda.max = 0.05615, lambda.min = 7.31E-4, lambda.1se = -1.0  161                           30                             6                       py_15_sid_9664\n",
       "glm_grid_model_24   binomial  logit   Elastic Net (alpha = 0.41, lambda = 8.91E-4 )  nlambda = 100, lambda.max = 0.0808, lambda.min = 8.91E-4, lambda.1se = -1.0   161                           30                             6                       py_15_sid_9664\n",
       "glm_grid_model_62   binomial  logit   Elastic Net (alpha = 0.94, lambda = 5.09E-4 )  nlambda = 100, lambda.max = 0.03524, lambda.min = 5.09E-4, lambda.1se = -1.0  161                           28                             6                       py_15_sid_9664\n",
       "glm_grid_model_5    binomial  logit   Elastic Net (alpha = 0.54, lambda = 7.8E-4 )   nlambda = 100, lambda.max = 0.06135, lambda.min = 7.8E-4, lambda.1se = -1.0   161                           30                             6                       py_15_sid_9664\n",
       "glm_grid_model_76   binomial  logit   Elastic Net (alpha = 0.41, lambda = 9.56E-4 )  nlambda = 100, lambda.max = 0.0808, lambda.min = 9.56E-4, lambda.1se = -1.0   161                           30                             6                       py_15_sid_9664\n",
       "glm_grid_model_50   binomial  logit   Elastic Net (alpha = 0.41, lambda = 9.93E-4 )  nlambda = 100, lambda.max = 0.0808, lambda.min = 9.93E-4, lambda.1se = -1.0   161                           30                             6                       py_15_sid_9664\n",
       "glm_grid_model_32   binomial  logit   Elastic Net (alpha = 0.6, lambda = 8.16E-4 )   nlambda = 100, lambda.max = 0.05521, lambda.min = 8.16E-4, lambda.1se = -1.0  161                           28                             7                       py_15_sid_9664\n",
       "glm_grid_model_97   binomial  logit   Elastic Net (alpha = 0.55, lambda = 8.76E-4 )  nlambda = 100, lambda.max = 0.06023, lambda.min = 8.76E-4, lambda.1se = -1.0  161                           28                             6                       py_15_sid_9664\n",
       "glm_grid_model_57   binomial  logit   Elastic Net (alpha = 0.84, lambda = 6.74E-4 )  nlambda = 100, lambda.max = 0.03944, lambda.min = 6.74E-4, lambda.1se = -1.0  161                           25                             6                       py_15_sid_9664\n",
       "glm_grid_model_91   binomial  logit   Elastic Net (alpha = 0.52, lambda = 9.45E-4 )  nlambda = 100, lambda.max = 0.06371, lambda.min = 9.45E-4, lambda.1se = -1.0  161                           28                             7                       py_15_sid_9664\n",
       "glm_grid_model_69   binomial  logit   Elastic Net (alpha = 0.57, lambda = 9.53E-4 )  nlambda = 100, lambda.max = 0.05812, lambda.min = 9.53E-4, lambda.1se = -1.0  161                           26                             6                       py_15_sid_9664\n",
       "glm_grid_model_56   binomial  logit   Elastic Net (alpha = 0.65, lambda = 9.38E-4 )  nlambda = 100, lambda.max = 0.05097, lambda.min = 9.38E-4, lambda.1se = -1.0  161                           25                             6                       py_15_sid_9664\n",
       "glm_grid_model_85   binomial  logit   Elastic Net (alpha = 0.95, lambda = 7.41E-4 )  nlambda = 100, lambda.max = 0.03487, lambda.min = 7.41E-4, lambda.1se = -1.0  161                           21                             6                       py_15_sid_9664\n",
       "glm_grid_model_39   binomial  logit   Elastic Net (alpha = 0.86, lambda = 8.06E-4 )  nlambda = 100, lambda.max = 0.03852, lambda.min = 8.06E-4, lambda.1se = -1.0  161                           21                             6                       py_15_sid_9664\n",
       "glm_grid_model_99   binomial  logit   Elastic Net (alpha = 0.92, lambda = 8.6E-4 )   nlambda = 100, lambda.max = 0.03601, lambda.min = 8.6E-4, lambda.1se = -1.0   161                           19                             7                       py_15_sid_9664\n",
       "glm_grid_model_15   binomial  logit   Elastic Net (alpha = 0.74, lambda = 9.87E-4 )  nlambda = 100, lambda.max = 0.04477, lambda.min = 9.87E-4, lambda.1se = -1.0  161                           21                             6                       py_15_sid_9664\n",
       "glm_grid_model_23   binomial  logit   Elastic Net (alpha = 0.88, lambda = 9.21E-4 )  nlambda = 100, lambda.max = 0.03765, lambda.min = 9.21E-4, lambda.1se = -1.0  161                           19                             6                       py_15_sid_9664\n",
       "glm_grid_model_7    binomial  logit   Elastic Net (alpha = 0.88, lambda = 9.48E-4 )  nlambda = 100, lambda.max = 0.03765, lambda.min = 9.48E-4, lambda.1se = -1.0  161                           19                             6                       py_15_sid_9664"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "glm_grid.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "sorted_glm_grid = glm_grid.get_grid(sort_by='auc',decreasing=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'model_id': 'glm_grid_model_61',\n",
       " 'training_frame': 'py_15_sid_9664',\n",
       " 'validation_frame': 'py_16_sid_9664',\n",
       " 'nfolds': 0,\n",
       " 'seed': 202,\n",
       " 'keep_cross_validation_models': True,\n",
       " 'keep_cross_validation_predictions': False,\n",
       " 'keep_cross_validation_fold_assignment': False,\n",
       " 'fold_assignment': 'AUTO',\n",
       " 'fold_column': None,\n",
       " 'response_column': 'DELINQUENT',\n",
       " 'ignored_columns': ['PRODUCT_TYPE',\n",
       "  'PREPAYMENT_PENALTY_MORTGAGE_FLAG',\n",
       "  'PREPAID'],\n",
       " 'random_columns': None,\n",
       " 'ignore_const_cols': True,\n",
       " 'score_each_iteration': False,\n",
       " 'offset_column': None,\n",
       " 'weights_column': None,\n",
       " 'family': 'binomial',\n",
       " 'rand_family': None,\n",
       " 'tweedie_variance_power': 0.0,\n",
       " 'tweedie_link_power': 1.0,\n",
       " 'theta': 1e-10,\n",
       " 'solver': 'COORDINATE_DESCENT',\n",
       " 'alpha': [0.87],\n",
       " 'lambda': [4.9999999999999996e-06],\n",
       " 'lambda_search': True,\n",
       " 'early_stopping': True,\n",
       " 'nlambdas': 100,\n",
       " 'standardize': True,\n",
       " 'missing_values_handling': 'MeanImputation',\n",
       " 'plug_values': None,\n",
       " 'compute_p_values': False,\n",
       " 'remove_collinear_columns': False,\n",
       " 'intercept': True,\n",
       " 'non_negative': False,\n",
       " 'max_iterations': 1000,\n",
       " 'objective_epsilon': 0.0001,\n",
       " 'beta_epsilon': 0.0001,\n",
       " 'gradient_epsilon': 1.0000000000000002e-06,\n",
       " 'link': 'logit',\n",
       " 'rand_link': None,\n",
       " 'startval': None,\n",
       " 'calc_like': False,\n",
       " 'HGLM': False,\n",
       " 'prior': -1.0,\n",
       " 'lambda_min_ratio': 0.0001,\n",
       " 'beta_constraints': None,\n",
       " 'max_active_predictors': 5000,\n",
       " 'interactions': None,\n",
       " 'interaction_pairs': None,\n",
       " 'obj_reg': 2.854956775954412e-06,\n",
       " 'export_checkpoints_dir': None,\n",
       " 'balance_classes': False,\n",
       " 'class_sampling_factors': None,\n",
       " 'max_after_balance_size': 5.0,\n",
       " 'max_confusion_matrix_size': 20,\n",
       " 'max_hit_ratio_k': 0,\n",
       " 'max_runtime_secs': 179.878,\n",
       " 'custom_metric_func': None}"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sorted_glm_grid[0].actual_params"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[0.13993235618594693, 0.29243426359090174]]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[[0.1388919375261923, 0.29188911043931304]]"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(sorted_glm_grid[0].F1())\n",
    "sorted_glm_grid[1].F1()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "ModelMetricsBinomialGLM: glm\n",
      "** Reported on test data. **\n",
      "\n",
      "MSE: 0.031143376101575086\n",
      "RMSE: 0.17647485968708146\n",
      "LogLoss: 0.12199693111453563\n",
      "Null degrees of freedom: 74897\n",
      "Residual degrees of freedom: 74754\n",
      "Null deviance: 23061.156287645877\n",
      "Residual deviance: 18274.652293232975\n",
      "AIC: 18562.652293232975\n",
      "AUC: 0.8524158062119054\n",
      "AUCPR: 0.20258611034476104\n",
      "Gini: 0.7048316124238108\n",
      "\n",
      "Confusion Matrix (Act/Pred) for max f1 @ threshold = 0.13069466877003805: \n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>FALSE</th>\n",
       "      <th>TRUE</th>\n",
       "      <th>Error</th>\n",
       "      <th>Rate</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>FALSE</td>\n",
       "      <td>68851.0</td>\n",
       "      <td>3375.0</td>\n",
       "      <td>0.0467</td>\n",
       "      <td>(3375.0/72226.0)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>TRUE</td>\n",
       "      <td>1676.0</td>\n",
       "      <td>996.0</td>\n",
       "      <td>0.6272</td>\n",
       "      <td>(1676.0/2672.0)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Total</td>\n",
       "      <td>70527.0</td>\n",
       "      <td>4371.0</td>\n",
       "      <td>0.0674</td>\n",
       "      <td>(5051.0/74898.0)</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            FALSE    TRUE   Error               Rate\n",
       "0  FALSE  68851.0  3375.0  0.0467   (3375.0/72226.0)\n",
       "1   TRUE   1676.0   996.0  0.6272    (1676.0/2672.0)\n",
       "2  Total  70527.0  4371.0  0.0674   (5051.0/74898.0)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Maximum Metrics: Maximum metrics at their respective thresholds\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>metric</th>\n",
       "      <th>threshold</th>\n",
       "      <th>value</th>\n",
       "      <th>idx</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>max f1</td>\n",
       "      <td>0.130695</td>\n",
       "      <td>0.282834</td>\n",
       "      <td>202.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>max f2</td>\n",
       "      <td>0.064974</td>\n",
       "      <td>0.386713</td>\n",
       "      <td>264.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>max f0point5</td>\n",
       "      <td>0.206442</td>\n",
       "      <td>0.283731</td>\n",
       "      <td>151.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>max accuracy</td>\n",
       "      <td>0.939516</td>\n",
       "      <td>0.964311</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>max precision</td>\n",
       "      <td>0.602416</td>\n",
       "      <td>0.438596</td>\n",
       "      <td>32.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>max recall</td>\n",
       "      <td>0.000945</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>398.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>max specificity</td>\n",
       "      <td>0.939516</td>\n",
       "      <td>0.999986</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>max absolute_mcc</td>\n",
       "      <td>0.072018</td>\n",
       "      <td>0.263347</td>\n",
       "      <td>255.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>max min_per_class_accuracy</td>\n",
       "      <td>0.038326</td>\n",
       "      <td>0.773336</td>\n",
       "      <td>305.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>max mean_per_class_accuracy</td>\n",
       "      <td>0.032702</td>\n",
       "      <td>0.777303</td>\n",
       "      <td>315.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>max tns</td>\n",
       "      <td>0.939516</td>\n",
       "      <td>72225.000000</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>max fns</td>\n",
       "      <td>0.939516</td>\n",
       "      <td>2672.000000</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>max fps</td>\n",
       "      <td>0.000566</td>\n",
       "      <td>72226.000000</td>\n",
       "      <td>399.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>max tps</td>\n",
       "      <td>0.000945</td>\n",
       "      <td>2672.000000</td>\n",
       "      <td>398.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>max tnr</td>\n",
       "      <td>0.939516</td>\n",
       "      <td>0.999986</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>max fnr</td>\n",
       "      <td>0.939516</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>max fpr</td>\n",
       "      <td>0.000566</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>399.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>max tpr</td>\n",
       "      <td>0.000945</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>398.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                         metric  threshold         value    idx\n",
       "0                        max f1   0.130695      0.282834  202.0\n",
       "1                        max f2   0.064974      0.386713  264.0\n",
       "2                  max f0point5   0.206442      0.283731  151.0\n",
       "3                  max accuracy   0.939516      0.964311    0.0\n",
       "4                 max precision   0.602416      0.438596   32.0\n",
       "5                    max recall   0.000945      1.000000  398.0\n",
       "6               max specificity   0.939516      0.999986    0.0\n",
       "7              max absolute_mcc   0.072018      0.263347  255.0\n",
       "8    max min_per_class_accuracy   0.038326      0.773336  305.0\n",
       "9   max mean_per_class_accuracy   0.032702      0.777303  315.0\n",
       "10                      max tns   0.939516  72225.000000    0.0\n",
       "11                      max fns   0.939516   2672.000000    0.0\n",
       "12                      max fps   0.000566  72226.000000  399.0\n",
       "13                      max tps   0.000945   2672.000000  398.0\n",
       "14                      max tnr   0.939516      0.999986    0.0\n",
       "15                      max fnr   0.939516      1.000000    0.0\n",
       "16                      max fpr   0.000566      1.000000  399.0\n",
       "17                      max tpr   0.000945      1.000000  398.0"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Gains/Lift Table: Avg response rate:  3.57 %, avg score:  3.60 %\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>group</th>\n",
       "      <th>cumulative_data_fraction</th>\n",
       "      <th>lower_threshold</th>\n",
       "      <th>lift</th>\n",
       "      <th>cumulative_lift</th>\n",
       "      <th>response_rate</th>\n",
       "      <th>score</th>\n",
       "      <th>cumulative_response_rate</th>\n",
       "      <th>cumulative_score</th>\n",
       "      <th>capture_rate</th>\n",
       "      <th>cumulative_capture_rate</th>\n",
       "      <th>gain</th>\n",
       "      <th>cumulative_gain</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td></td>\n",
       "      <td>1</td>\n",
       "      <td>0.010000</td>\n",
       "      <td>0.323153</td>\n",
       "      <td>9.954824</td>\n",
       "      <td>9.954824</td>\n",
       "      <td>0.355140</td>\n",
       "      <td>0.464324</td>\n",
       "      <td>0.355140</td>\n",
       "      <td>0.464324</td>\n",
       "      <td>0.099551</td>\n",
       "      <td>0.099551</td>\n",
       "      <td>895.482400</td>\n",
       "      <td>895.482400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td></td>\n",
       "      <td>2</td>\n",
       "      <td>0.020001</td>\n",
       "      <td>0.238585</td>\n",
       "      <td>7.859072</td>\n",
       "      <td>8.906948</td>\n",
       "      <td>0.280374</td>\n",
       "      <td>0.275583</td>\n",
       "      <td>0.317757</td>\n",
       "      <td>0.369954</td>\n",
       "      <td>0.078593</td>\n",
       "      <td>0.178144</td>\n",
       "      <td>685.907158</td>\n",
       "      <td>790.694779</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td></td>\n",
       "      <td>3</td>\n",
       "      <td>0.030001</td>\n",
       "      <td>0.194006</td>\n",
       "      <td>6.362106</td>\n",
       "      <td>8.058667</td>\n",
       "      <td>0.226969</td>\n",
       "      <td>0.214539</td>\n",
       "      <td>0.287494</td>\n",
       "      <td>0.318149</td>\n",
       "      <td>0.063623</td>\n",
       "      <td>0.241766</td>\n",
       "      <td>536.210556</td>\n",
       "      <td>705.866705</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td></td>\n",
       "      <td>4</td>\n",
       "      <td>0.040001</td>\n",
       "      <td>0.164906</td>\n",
       "      <td>4.939988</td>\n",
       "      <td>7.278997</td>\n",
       "      <td>0.176235</td>\n",
       "      <td>0.178489</td>\n",
       "      <td>0.259680</td>\n",
       "      <td>0.283234</td>\n",
       "      <td>0.049401</td>\n",
       "      <td>0.291168</td>\n",
       "      <td>393.998785</td>\n",
       "      <td>627.899725</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td></td>\n",
       "      <td>5</td>\n",
       "      <td>0.050001</td>\n",
       "      <td>0.143184</td>\n",
       "      <td>4.378626</td>\n",
       "      <td>6.698923</td>\n",
       "      <td>0.156208</td>\n",
       "      <td>0.153711</td>\n",
       "      <td>0.238985</td>\n",
       "      <td>0.257329</td>\n",
       "      <td>0.043787</td>\n",
       "      <td>0.334955</td>\n",
       "      <td>337.862559</td>\n",
       "      <td>569.892292</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td></td>\n",
       "      <td>6</td>\n",
       "      <td>0.100003</td>\n",
       "      <td>0.087476</td>\n",
       "      <td>3.428052</td>\n",
       "      <td>5.063488</td>\n",
       "      <td>0.122296</td>\n",
       "      <td>0.111145</td>\n",
       "      <td>0.180641</td>\n",
       "      <td>0.184237</td>\n",
       "      <td>0.171407</td>\n",
       "      <td>0.506362</td>\n",
       "      <td>242.805217</td>\n",
       "      <td>406.348754</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td></td>\n",
       "      <td>7</td>\n",
       "      <td>0.150004</td>\n",
       "      <td>0.062147</td>\n",
       "      <td>2.492448</td>\n",
       "      <td>4.206475</td>\n",
       "      <td>0.088919</td>\n",
       "      <td>0.073591</td>\n",
       "      <td>0.150067</td>\n",
       "      <td>0.147355</td>\n",
       "      <td>0.124626</td>\n",
       "      <td>0.630988</td>\n",
       "      <td>149.244841</td>\n",
       "      <td>320.647450</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td></td>\n",
       "      <td>8</td>\n",
       "      <td>0.200005</td>\n",
       "      <td>0.047494</td>\n",
       "      <td>1.639178</td>\n",
       "      <td>3.564650</td>\n",
       "      <td>0.058478</td>\n",
       "      <td>0.054272</td>\n",
       "      <td>0.127170</td>\n",
       "      <td>0.124084</td>\n",
       "      <td>0.081961</td>\n",
       "      <td>0.712949</td>\n",
       "      <td>63.917779</td>\n",
       "      <td>256.465032</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td></td>\n",
       "      <td>9</td>\n",
       "      <td>0.300008</td>\n",
       "      <td>0.030300</td>\n",
       "      <td>1.205058</td>\n",
       "      <td>2.778119</td>\n",
       "      <td>0.042991</td>\n",
       "      <td>0.037927</td>\n",
       "      <td>0.099110</td>\n",
       "      <td>0.095365</td>\n",
       "      <td>0.120509</td>\n",
       "      <td>0.833458</td>\n",
       "      <td>20.505764</td>\n",
       "      <td>177.811943</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td></td>\n",
       "      <td>10</td>\n",
       "      <td>0.399997</td>\n",
       "      <td>0.020667</td>\n",
       "      <td>0.591381</td>\n",
       "      <td>2.231489</td>\n",
       "      <td>0.021098</td>\n",
       "      <td>0.025088</td>\n",
       "      <td>0.079609</td>\n",
       "      <td>0.077797</td>\n",
       "      <td>0.059132</td>\n",
       "      <td>0.892590</td>\n",
       "      <td>-40.861947</td>\n",
       "      <td>123.148945</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td></td>\n",
       "      <td>11</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.014547</td>\n",
       "      <td>0.389211</td>\n",
       "      <td>1.863024</td>\n",
       "      <td>0.013885</td>\n",
       "      <td>0.017407</td>\n",
       "      <td>0.066464</td>\n",
       "      <td>0.065719</td>\n",
       "      <td>0.038922</td>\n",
       "      <td>0.931512</td>\n",
       "      <td>-61.078884</td>\n",
       "      <td>86.302395</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td></td>\n",
       "      <td>12</td>\n",
       "      <td>0.600003</td>\n",
       "      <td>0.010226</td>\n",
       "      <td>0.336817</td>\n",
       "      <td>1.608651</td>\n",
       "      <td>0.012016</td>\n",
       "      <td>0.012272</td>\n",
       "      <td>0.057389</td>\n",
       "      <td>0.056811</td>\n",
       "      <td>0.033683</td>\n",
       "      <td>0.965195</td>\n",
       "      <td>-66.318265</td>\n",
       "      <td>60.865053</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td></td>\n",
       "      <td>13</td>\n",
       "      <td>0.699992</td>\n",
       "      <td>0.007069</td>\n",
       "      <td>0.157202</td>\n",
       "      <td>1.401321</td>\n",
       "      <td>0.005608</td>\n",
       "      <td>0.008589</td>\n",
       "      <td>0.049992</td>\n",
       "      <td>0.049923</td>\n",
       "      <td>0.015719</td>\n",
       "      <td>0.980913</td>\n",
       "      <td>-84.279758</td>\n",
       "      <td>40.132057</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td></td>\n",
       "      <td>14</td>\n",
       "      <td>0.799995</td>\n",
       "      <td>0.004617</td>\n",
       "      <td>0.108530</td>\n",
       "      <td>1.239716</td>\n",
       "      <td>0.003872</td>\n",
       "      <td>0.005802</td>\n",
       "      <td>0.044227</td>\n",
       "      <td>0.044408</td>\n",
       "      <td>0.010853</td>\n",
       "      <td>0.991766</td>\n",
       "      <td>-89.146996</td>\n",
       "      <td>23.971636</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td></td>\n",
       "      <td>15</td>\n",
       "      <td>0.899997</td>\n",
       "      <td>0.002619</td>\n",
       "      <td>0.063621</td>\n",
       "      <td>1.109035</td>\n",
       "      <td>0.002270</td>\n",
       "      <td>0.003584</td>\n",
       "      <td>0.039565</td>\n",
       "      <td>0.039872</td>\n",
       "      <td>0.006362</td>\n",
       "      <td>0.998129</td>\n",
       "      <td>-93.637894</td>\n",
       "      <td>10.903523</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td></td>\n",
       "      <td>16</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000105</td>\n",
       "      <td>0.018712</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000668</td>\n",
       "      <td>0.001628</td>\n",
       "      <td>0.035675</td>\n",
       "      <td>0.036047</td>\n",
       "      <td>0.001871</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>-98.128792</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      group  cumulative_data_fraction  lower_threshold      lift  \\\n",
       "0         1                  0.010000         0.323153  9.954824   \n",
       "1         2                  0.020001         0.238585  7.859072   \n",
       "2         3                  0.030001         0.194006  6.362106   \n",
       "3         4                  0.040001         0.164906  4.939988   \n",
       "4         5                  0.050001         0.143184  4.378626   \n",
       "5         6                  0.100003         0.087476  3.428052   \n",
       "6         7                  0.150004         0.062147  2.492448   \n",
       "7         8                  0.200005         0.047494  1.639178   \n",
       "8         9                  0.300008         0.030300  1.205058   \n",
       "9        10                  0.399997         0.020667  0.591381   \n",
       "10       11                  0.500000         0.014547  0.389211   \n",
       "11       12                  0.600003         0.010226  0.336817   \n",
       "12       13                  0.699992         0.007069  0.157202   \n",
       "13       14                  0.799995         0.004617  0.108530   \n",
       "14       15                  0.899997         0.002619  0.063621   \n",
       "15       16                  1.000000         0.000105  0.018712   \n",
       "\n",
       "    cumulative_lift  response_rate     score  cumulative_response_rate  \\\n",
       "0          9.954824       0.355140  0.464324                  0.355140   \n",
       "1          8.906948       0.280374  0.275583                  0.317757   \n",
       "2          8.058667       0.226969  0.214539                  0.287494   \n",
       "3          7.278997       0.176235  0.178489                  0.259680   \n",
       "4          6.698923       0.156208  0.153711                  0.238985   \n",
       "5          5.063488       0.122296  0.111145                  0.180641   \n",
       "6          4.206475       0.088919  0.073591                  0.150067   \n",
       "7          3.564650       0.058478  0.054272                  0.127170   \n",
       "8          2.778119       0.042991  0.037927                  0.099110   \n",
       "9          2.231489       0.021098  0.025088                  0.079609   \n",
       "10         1.863024       0.013885  0.017407                  0.066464   \n",
       "11         1.608651       0.012016  0.012272                  0.057389   \n",
       "12         1.401321       0.005608  0.008589                  0.049992   \n",
       "13         1.239716       0.003872  0.005802                  0.044227   \n",
       "14         1.109035       0.002270  0.003584                  0.039565   \n",
       "15         1.000000       0.000668  0.001628                  0.035675   \n",
       "\n",
       "    cumulative_score  capture_rate  cumulative_capture_rate        gain  \\\n",
       "0           0.464324      0.099551                 0.099551  895.482400   \n",
       "1           0.369954      0.078593                 0.178144  685.907158   \n",
       "2           0.318149      0.063623                 0.241766  536.210556   \n",
       "3           0.283234      0.049401                 0.291168  393.998785   \n",
       "4           0.257329      0.043787                 0.334955  337.862559   \n",
       "5           0.184237      0.171407                 0.506362  242.805217   \n",
       "6           0.147355      0.124626                 0.630988  149.244841   \n",
       "7           0.124084      0.081961                 0.712949   63.917779   \n",
       "8           0.095365      0.120509                 0.833458   20.505764   \n",
       "9           0.077797      0.059132                 0.892590  -40.861947   \n",
       "10          0.065719      0.038922                 0.931512  -61.078884   \n",
       "11          0.056811      0.033683                 0.965195  -66.318265   \n",
       "12          0.049923      0.015719                 0.980913  -84.279758   \n",
       "13          0.044408      0.010853                 0.991766  -89.146996   \n",
       "14          0.039872      0.006362                 0.998129  -93.637894   \n",
       "15          0.036047      0.001871                 1.000000  -98.128792   \n",
       "\n",
       "    cumulative_gain  \n",
       "0        895.482400  \n",
       "1        790.694779  \n",
       "2        705.866705  \n",
       "3        627.899725  \n",
       "4        569.892292  \n",
       "5        406.348754  \n",
       "6        320.647450  \n",
       "7        256.465032  \n",
       "8        177.811943  \n",
       "9        123.148945  \n",
       "10        86.302395  \n",
       "11        60.865053  \n",
       "12        40.132057  \n",
       "13        23.971636  \n",
       "14        10.903523  \n",
       "15         0.000000  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": []
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sorted_glm_grid[0].model_performance(test) # should give AUC of 0.8524 compared to the untuned version of 0.8523"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Random Forest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "drf Grid Build progress: |████████████████████████████████████████████████| 100%\n",
      "CPU times: user 2.33 s, sys: 432 ms, total: 2.76 s\n",
      "Wall time: 6min 45s\n"
     ]
    }
   ],
   "source": [
    "# Grid Search/ Cartesian Search by default or not specified\n",
    "rf_grid = h2o.grid.H2OGridSearch (\n",
    "    H2ORandomForestEstimator(nfolds=10),\n",
    "    \n",
    "    hyper_params = {\n",
    "        \"ntrees\": [50,100],\n",
    "        \"max_depth\": [10,20],\n",
    "        },\n",
    "    \n",
    "     search_criteria = {\n",
    "        \"strategy\":\"RandomDiscrete\", # Random Search \n",
    "        \"max_models\":100,\n",
    "        \"max_runtime_secs\":300,\n",
    "        \"seed\":42\n",
    "        },\n",
    "    \n",
    "    grid_id = \"rf_grid_2\",\n",
    "    \n",
    ")\n",
    "%time rf_grid.train(x=x, y=y, training_frame=train, validation_frame = valid)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "    max_depth ntrees          model_ids                auc\n",
      "0          20     28  rf_grid_2_model_1  0.818864830103598\n"
     ]
    },
    {
     "data": {
      "text/plain": []
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rf_grid.get_grid(sort_by='auc', decreasing=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Get the best model and train on top of that"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "drf Model Build progress: |███████████████████████████████████████████████| 100%\n",
      "CPU times: user 347 ms, sys: 109 ms, total: 456 ms\n",
      "Wall time: 39.7 s\n"
     ]
    }
   ],
   "source": [
    "best_model = rf_grid.get_grid(sort_by=\"auc\", decreasing=True)[0]\n",
    "\n",
    "rf = H2ORandomForestEstimator (seed=42, model_id='default_random_forest', checkpoint=best_model.model_id)\n",
    "%time rf.train(x=x, y=y, training_frame=train, validation_frame=valid)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Model Summary: \n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>number_of_trees</th>\n",
       "      <th>number_of_internal_trees</th>\n",
       "      <th>model_size_in_bytes</th>\n",
       "      <th>min_depth</th>\n",
       "      <th>max_depth</th>\n",
       "      <th>mean_depth</th>\n",
       "      <th>min_leaves</th>\n",
       "      <th>max_leaves</th>\n",
       "      <th>mean_leaves</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td></td>\n",
       "      <td>50.0</td>\n",
       "      <td>50.0</td>\n",
       "      <td>7675073.0</td>\n",
       "      <td>20.0</td>\n",
       "      <td>20.0</td>\n",
       "      <td>8.8</td>\n",
       "      <td>9881.0</td>\n",
       "      <td>11724.0</td>\n",
       "      <td>4780.06</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     number_of_trees  number_of_internal_trees  model_size_in_bytes  \\\n",
       "0               50.0                      50.0            7675073.0   \n",
       "\n",
       "   min_depth  max_depth  mean_depth  min_leaves  max_leaves  mean_leaves  \n",
       "0       20.0       20.0         8.8      9881.0     11724.0      4780.06  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": []
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rf.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}