{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "\n",
    "import keras.backend as K\n",
    "from keras.models import Sequential\n",
    "from keras.datasets import mnist\n",
    "from keras.layers import Dense\n",
    "from keras.utils import np_utils\n",
    "from keras.wrappers.scikit_learn import KerasClassifier\n",
    "\n",
    "np.random.seed(13)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "(x_train, y_train), (x_test, y_test) = mnist.load_data()\n",
    "\n",
    "num_classes = 10 # class size\n",
    "input_unit_size = 28*28 # input vector size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "x_train = x_train.reshape(x_train.shape[0], input_unit_size)\n",
    "x_test  = x_test.reshape(x_test.shape[0], input_unit_size)\n",
    "x_train = x_train.astype('float32')\n",
    "x_test  = x_test.astype('float32')\n",
    "x_train /= 255\n",
    "x_test  /= 255"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def create_model(activation='relu', nb_hidden=10):\n",
    "    model = Sequential()\n",
    "    model.add(Dense(nb_hidden, input_dim=784, activation=activation))\n",
    "    model.add(Dense(num_classes, activation='softmax'))\n",
    "    model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "activations = [K.cos, 'softplus', 'softsign', 'relu', 'tanh', 'sigmoid', 'hard_sigmoid', 'linear', 'elu']\n",
    "nb_hiddens = np.array([100, 1000])\n",
    "\n",
    "param_grid = dict(activation=activations, nb_hidden=nb_hiddens)\n",
    "model = KerasClassifier(build_fn=create_model, epochs=30, batch_size=256, verbose=0)\n",
    "\n",
    "clf = GridSearchCV(estimator=model, param_grid=param_grid, cv=4, scoring='accuracy')\n",
    "res = clf.fit(x_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.981216666667 {'activation': <function cos at 0x7f8222852268>, 'nb_hidden': 1000}\n",
      "mean: 0.97265, std: 0.00067, params: {'activation': <function cos at 0x7f8222852268>, 'nb_hidden': 100}\n",
      "mean: 0.98122, std: 0.00057, params: {'activation': <function cos at 0x7f8222852268>, 'nb_hidden': 1000}\n",
      "mean: 0.97147, std: 0.00086, params: {'activation': 'softplus', 'nb_hidden': 100}\n",
      "mean: 0.97555, std: 0.00081, params: {'activation': 'softplus', 'nb_hidden': 1000}\n",
      "mean: 0.96892, std: 0.00146, params: {'activation': 'softsign', 'nb_hidden': 100}\n",
      "mean: 0.97753, std: 0.00077, params: {'activation': 'softsign', 'nb_hidden': 1000}\n",
      "mean: 0.97358, std: 0.00127, params: {'activation': 'relu', 'nb_hidden': 100}\n",
      "mean: 0.98088, std: 0.00104, params: {'activation': 'relu', 'nb_hidden': 1000}\n",
      "mean: 0.97217, std: 0.00100, params: {'activation': 'tanh', 'nb_hidden': 100}\n",
      "mean: 0.97780, std: 0.00124, params: {'activation': 'tanh', 'nb_hidden': 1000}\n",
      "mean: 0.96852, std: 0.00087, params: {'activation': 'sigmoid', 'nb_hidden': 100}\n",
      "mean: 0.97577, std: 0.00142, params: {'activation': 'sigmoid', 'nb_hidden': 1000}\n",
      "mean: 0.96715, std: 0.00093, params: {'activation': 'hard_sigmoid', 'nb_hidden': 100}\n",
      "mean: 0.97403, std: 0.00080, params: {'activation': 'hard_sigmoid', 'nb_hidden': 1000}\n",
      "mean: 0.91723, std: 0.00290, params: {'activation': 'linear', 'nb_hidden': 100}\n",
      "mean: 0.91645, std: 0.00459, params: {'activation': 'linear', 'nb_hidden': 1000}\n",
      "mean: 0.97150, std: 0.00075, params: {'activation': 'elu', 'nb_hidden': 100}\n",
      "mean: 0.97487, std: 0.00355, params: {'activation': 'elu', 'nb_hidden': 1000}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/nzw/.pyenv/versions/miniconda3-latest/lib/python3.6/site-packages/sklearn/model_selection/_search.py:667: DeprecationWarning: The grid_scores_ attribute was deprecated in version 0.18 in favor of the more elaborate cv_results_ attribute. The grid_scores_ attribute will not be available from 0.20\n",
      "  DeprecationWarning)\n"
     ]
    }
   ],
   "source": [
    "print(res.best_score_, res.best_params_)\n",
    "for i in res.grid_scores_:\n",
    "    print(i)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}