{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU",
    "gpuClass": "premium"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "source": [
        "<table align=\"left\"><tr><td>\n",
        "<a href=\"https://colab.research.google.com/github/rickiepark/deep-learning-with-python-2nd/blob/master/keras_tuner.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>\n",
        "</td></tr></table>"
      ],
      "metadata": {
        "id": "mIZ5RPgx6Ld1"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "M9-QNSLgRuAR",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "bed18075-d879-4125-a9e8-45bf75c9e3a9"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\u001b[K     |████████████████████████████████| 135 kB 5.2 MB/s \n",
            "\u001b[K     |████████████████████████████████| 1.6 MB 43.9 MB/s \n",
            "\u001b[?25h"
          ]
        }
      ],
      "source": [
        "!pip install -q keras-tuner"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import keras_tuner as kt\n",
        "from tensorflow import keras"
      ],
      "metadata": {
        "id": "Mc6porlUhN45"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "(x_train, y_train), (x_test, y_test) = keras.datasets.fashion_mnist.load_data()\n",
        "x_train, x_val = x_train[:-10000], x_train[-10000:]\n",
        "y_train, y_val = y_train[:-10000], y_train[-10000:]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "KHo1NSQDhecq",
        "outputId": "044f04c8-90a6-44f8-ad27-92350121e827"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Downloading data from https://storage.googleapis.com/tensorflow/tf-keras-datasets/train-labels-idx1-ubyte.gz\n",
            "32768/29515 [=================================] - 0s 0us/step\n",
            "40960/29515 [=========================================] - 0s 0us/step\n",
            "Downloading data from https://storage.googleapis.com/tensorflow/tf-keras-datasets/train-images-idx3-ubyte.gz\n",
            "26427392/26421880 [==============================] - 0s 0us/step\n",
            "26435584/26421880 [==============================] - 0s 0us/step\n",
            "Downloading data from https://storage.googleapis.com/tensorflow/tf-keras-datasets/t10k-labels-idx1-ubyte.gz\n",
            "16384/5148 [===============================================================================================] - 0s 0us/step\n",
            "Downloading data from https://storage.googleapis.com/tensorflow/tf-keras-datasets/t10k-images-idx3-ubyte.gz\n",
            "4423680/4422102 [==============================] - 0s 0us/step\n",
            "4431872/4422102 [==============================] - 0s 0us/step\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "x_train = x_train.reshape((-1, 28, 28, 1)) / 255.0\n",
        "x_test = x_test.reshape((-1, 28, 28, 1)) / 255.0\n",
        "x_val = x_val.reshape((-1, 28, 28, 1)) / 255.0"
      ],
      "metadata": {
        "id": "sWNh2TYYj_f9"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def build_model(hp):\n",
        "    inputs = keras.Input(shape=(28, 28, 1))\n",
        "    filters = hp.Int(\"filters_1\", min_value=16, max_value=64, step=16)\n",
        "    x = keras.layers.Conv2D(filters, 3, activation=\"relu\")(inputs)\n",
        "    x = keras.layers.MaxPooling2D(pool_size=2)(x)\n",
        "    filters = hp.Int(\"filters_2\", min_value=32, max_value=512, step=16)\n",
        "    x = keras.layers.Conv2D(filters, 3, activation=\"relu\")(x)\n",
        "    x = keras.layers.MaxPooling2D(pool_size=2)(x)\n",
        "    filters = hp.Int(\"filters_3\", min_value=32, max_value=512, step=16)\n",
        "    x = keras.layers.Conv2D(filters, 3, activation=\"relu\")(x)\n",
        "    x = keras.layers.MaxPooling2D(pool_size=2)(x)\n",
        "    x = keras.layers.Flatten()(x)\n",
        "    outputs = keras.layers.Dense(10, activation=\"softmax\")(x)\n",
        "    model = keras.Model(inputs, outputs)\n",
        "    optimizer = hp.Choice(\"optimizer\", values=[\"rmsprop\", \"adam\"])\n",
        "    lr = hp.Float(\"lr\", min_value=1e-5, max_value=1e-2, sampling=\"log\")\n",
        "    if optimizer == \"rmsprop\":\n",
        "        opt = keras.optimizers.RMSprop(learning_rate=lr)\n",
        "    else:\n",
        "        opt = keras.optimizers.Adam(learning_rate=lr)\n",
        "    model.compile(opt, loss=\"sparse_categorical_crossentropy\",\n",
        "                  metrics=[\"accuracy\"])\n",
        "    return model"
      ],
      "metadata": {
        "id": "yKb1a1ZziJJ6"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "model = build_model(kt.HyperParameters())\n",
        "model.summary()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "EOFqkQOEJJ1j",
        "outputId": "a00a6044-6e01-4f01-b481-6371f6b11d23"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Model: \"model\"\n",
            "_________________________________________________________________\n",
            " Layer (type)                Output Shape              Param #   \n",
            "=================================================================\n",
            " input_1 (InputLayer)        [(None, 28, 28, 1)]       0         \n",
            "                                                                 \n",
            " conv2d (Conv2D)             (None, 26, 26, 16)        160       \n",
            "                                                                 \n",
            " max_pooling2d (MaxPooling2D  (None, 13, 13, 16)       0         \n",
            " )                                                               \n",
            "                                                                 \n",
            " conv2d_1 (Conv2D)           (None, 11, 11, 32)        4640      \n",
            "                                                                 \n",
            " max_pooling2d_1 (MaxPooling  (None, 5, 5, 32)         0         \n",
            " 2D)                                                             \n",
            "                                                                 \n",
            " conv2d_2 (Conv2D)           (None, 3, 3, 32)          9248      \n",
            "                                                                 \n",
            " max_pooling2d_2 (MaxPooling  (None, 1, 1, 32)         0         \n",
            " 2D)                                                             \n",
            "                                                                 \n",
            " flatten (Flatten)           (None, 32)                0         \n",
            "                                                                 \n",
            " dense (Dense)               (None, 10)                330       \n",
            "                                                                 \n",
            "=================================================================\n",
            "Total params: 14,378\n",
            "Trainable params: 14,378\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "tuner = kt.RandomSearch(build_model, objective=\"val_accuracy\", \n",
        "                        max_trials=100, seed=42)\n",
        "tuner.search_space_summary()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "g8aibsjDn5Er",
        "outputId": "c6841a07-f5b8-4825-a380-03ecda7712e3"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Search space summary\n",
            "Default search space size: 5\n",
            "filters_1 (Int)\n",
            "{'default': None, 'conditions': [], 'min_value': 16, 'max_value': 64, 'step': 16, 'sampling': None}\n",
            "filters_2 (Int)\n",
            "{'default': None, 'conditions': [], 'min_value': 32, 'max_value': 512, 'step': 16, 'sampling': None}\n",
            "filters_3 (Int)\n",
            "{'default': None, 'conditions': [], 'min_value': 32, 'max_value': 512, 'step': 16, 'sampling': None}\n",
            "optimizer (Choice)\n",
            "{'default': 'rmsprop', 'conditions': [], 'values': ['rmsprop', 'adam'], 'ordered': False}\n",
            "lr (Float)\n",
            "{'default': 1e-05, 'conditions': [], 'min_value': 1e-05, 'max_value': 0.01, 'step': None, 'sampling': 'log'}\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "callbacks = [keras.callbacks.EarlyStopping(monitor=\"val_loss\", patience=5)]\n",
        "tuner.search(x_train, y_train, batch_size=64, epochs=100,\n",
        "             validation_data=(x_val, y_val), callbacks=callbacks, verbose=2)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "HDUbnPXqrE_g",
        "outputId": "a5b17bf2-4cf4-4bca-be34-e6bb50031020"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Trial 100 Complete [00h 01m 56s]\n",
            "val_accuracy: 0.8949000239372253\n",
            "\n",
            "Best val_accuracy So Far: 0.9114999771118164\n",
            "Total elapsed time: 02h 36m 50s\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "tuner.results_summary()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "x6kCGlMJJSlO",
        "outputId": "e4b1b35a-45b6-43b1-c074-d54ead5d5a7b"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Results summary\n",
            "Results in ./untitled_project\n",
            "Showing 10 best trials\n",
            "<keras_tuner.engine.objective.Objective object at 0x7fe880404c10>\n",
            "Trial summary\n",
            "Hyperparameters:\n",
            "filters_1: 64\n",
            "filters_2: 480\n",
            "filters_3: 464\n",
            "optimizer: rmsprop\n",
            "lr: 0.0005531280941472172\n",
            "Score: 0.9114999771118164\n",
            "Trial summary\n",
            "Hyperparameters:\n",
            "filters_1: 64\n",
            "filters_2: 512\n",
            "filters_3: 128\n",
            "optimizer: adam\n",
            "lr: 0.0002906286580403997\n",
            "Score: 0.9071999788284302\n",
            "Trial summary\n",
            "Hyperparameters:\n",
            "filters_1: 32\n",
            "filters_2: 416\n",
            "filters_3: 352\n",
            "optimizer: rmsprop\n",
            "lr: 0.00030092742988110965\n",
            "Score: 0.9067000150680542\n",
            "Trial summary\n",
            "Hyperparameters:\n",
            "filters_1: 16\n",
            "filters_2: 272\n",
            "filters_3: 336\n",
            "optimizer: rmsprop\n",
            "lr: 0.0007343120079351318\n",
            "Score: 0.9061999917030334\n",
            "Trial summary\n",
            "Hyperparameters:\n",
            "filters_1: 32\n",
            "filters_2: 352\n",
            "filters_3: 192\n",
            "optimizer: adam\n",
            "lr: 0.0010464958375249103\n",
            "Score: 0.9056000113487244\n",
            "Trial summary\n",
            "Hyperparameters:\n",
            "filters_1: 48\n",
            "filters_2: 480\n",
            "filters_3: 432\n",
            "optimizer: adam\n",
            "lr: 0.0009447404851439078\n",
            "Score: 0.9050999879837036\n",
            "Trial summary\n",
            "Hyperparameters:\n",
            "filters_1: 32\n",
            "filters_2: 272\n",
            "filters_3: 272\n",
            "optimizer: adam\n",
            "lr: 0.0005335174447761023\n",
            "Score: 0.9049999713897705\n",
            "Trial summary\n",
            "Hyperparameters:\n",
            "filters_1: 32\n",
            "filters_2: 256\n",
            "filters_3: 80\n",
            "optimizer: adam\n",
            "lr: 0.0008284417575775176\n",
            "Score: 0.9046000242233276\n",
            "Trial summary\n",
            "Hyperparameters:\n",
            "filters_1: 48\n",
            "filters_2: 416\n",
            "filters_3: 96\n",
            "optimizer: adam\n",
            "lr: 0.00032947407039943624\n",
            "Score: 0.9042999744415283\n",
            "Trial summary\n",
            "Hyperparameters:\n",
            "filters_1: 48\n",
            "filters_2: 496\n",
            "filters_3: 480\n",
            "optimizer: adam\n",
            "lr: 0.0010813378402855828\n",
            "Score: 0.9032999873161316\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "best_hps = tuner.get_best_hyperparameters(1)\n",
        "best_hps[0].values"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "_LW-15BCtW7t",
        "outputId": "023e99a9-9322-4a1c-8a43-6653f4409b41"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'filters_1': 64,\n",
              " 'filters_2': 480,\n",
              " 'filters_3': 464,\n",
              " 'optimizer': 'rmsprop',\n",
              " 'lr': 0.0005531280941472172}"
            ]
          },
          "metadata": {},
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "best_models = tuner.get_best_models(1)\n",
        "final_model = best_models[0]"
      ],
      "metadata": {
        "id": "QyKLqlzIvaLV"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "final_model.evaluate(x_test, y_test)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "S20v6V8evnmM",
        "outputId": "93960774-a3db-4ef6-bb63-fdba1212a46b"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "313/313 [==============================] - 1s 2ms/step - loss: 0.3429 - accuracy: 0.9049\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[0.3428693115711212, 0.9049000144004822]"
            ]
          },
          "metadata": {},
          "execution_count": 12
        }
      ]
    }
  ]
}