{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!pip install fastai"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from fastai.tabular.all import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       ""
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "path = untar_data(URLs.ADULT_SAMPLE)\n",
    "df = pd.read_csv(path/'adult.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  Building wheel for bayesian-optimization (setup.py) ... \u001b[?25l\u001b[?25hdone\n"
     ]
    }
   ],
   "source": [
    "!pip install bayesian-optimization -q"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from bayes_opt import BayesianOptimization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def fit_with(lr:float, wd:float, dp:float):\n",
    "  # create a Learner\n",
    "  config = tabular_config(embed_p=dp, ps=wd)\n",
    "  learn = tabular_learner(data, layers=[200,100], metrics=accuracy, config=config)\n",
    "  \n",
    "  # Train for x epochs\n",
    "  with learn.no_bar():\n",
    "    learn.fit_one_cycle(3, lr)\n",
    "    \n",
    "  # Save, print, and return the overall accuracy\n",
    "  acc = float(learn.validate()[1])\n",
    "  \n",
    "  return acc"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's adjust this further to show how we would go about adjusting the learning rate, embedded weight decay, drop out, and layer size:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def fit_with(lr:float, wd:float, dp:float, n_layers:float, layer_1:float, layer_2:float, layer_3:float):\n",
    "\n",
    "  print(lr, wd, dp)\n",
    "  if int(n_layers) == 2:\n",
    "    layers = [int(layer_1), int(layer_2)]\n",
    "  elif int(n_layers) == 3:\n",
    "    layers = [int(layer_1), int(layer_2), int(layer_3)]\n",
    "  else:\n",
    "    layers = [int(layer_1)]\n",
    "  config = tabular_config(embed_p=float(dp),\n",
    "                          ps=float(wd))\n",
    "  learn = tabular_learner(dls, layers=layers, metrics=accuracy, config = config)\n",
    "\n",
    "  with learn.no_bar() and learn.no_logging():\n",
    "    learn.fit(5, lr=float(lr))\n",
    "\n",
    "  acc = float(learn.validate()[1])\n",
    "\n",
    "  return acc"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's try it out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cat_names = ['workclass', 'education', 'marital-status', 'occupation', 'relationship', 'race']\n",
    "cont_names = ['age', 'fnlwgt', 'education-num']\n",
    "procs = [Categorify, FillMissing, Normalize]\n",
    "y_names = 'salary'\n",
    "y_block = CategoryBlock()\n",
    "splits = RandomSplitter()(range_of(df))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "to = TabularPandas(df, procs=procs, cat_names=cat_names, cont_names=cont_names,\n",
    "                   y_names=y_names, y_block=y_block, splits=splits)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dls = to.dataloaders(bs=512)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We'll declare our hyper-parameters:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "hps = {'lr': (1e-05, 1e-01),\n",
    "      'wd': (4e-4, 0.4),\n",
    "      'dp': (0.01, 0.5),\n",
    "       'n_layers': (1,3),\n",
    "       'layer_1': (50, 200),\n",
    "       'layer_2': (100, 1000),\n",
    "       'layer_3': (200, 2000)}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "And now we build the optimizer:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "optim = BayesianOptimization(\n",
    "    f = fit_with, # our fit function\n",
    "    pbounds = hps, # our hyper parameters to tune\n",
    "    verbose = 2, # 1 prints out when a maximum is observed, 0 for silent\n",
    "    random_state=1\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "And now we can search!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%time optim.maximize(n_iter=10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We can grab the best results:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'target': 0.8404483795166016, 'params': {'dp': 0.1710613610736308, 'layer_1': 57.63154958927875, 'layer_2': 100.1567384765859, 'layer_3': 1930.4092799350558, 'lr': 0.0721697277771627, 'n_layers': 2.868052690189961, 'wd': 0.035039066808375346}}\n"
     ]
    }
   ],
   "source": [
    "print(optim.max)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "And with a few conversions we see:\n",
    "\n",
    "* The best number of layers was 2\n",
    "* The first layer a size of 57\n",
    "* The second layer a size of 100\n",
    "And then of course our other hyper paramters"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}