{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "_cell_guid": "84d4608d-4cc3-fcbb-57fb-61f07ad7d020",
    "_uuid": "6407080d145a62b4803d7f159c00118a056a7b5f"
   },
   "source": [
    "# Deep Neural Network training on MNIST\n",
    "\n",
    "This notebook is based on this [Kaggle project](https://www.kaggle.com/kernels/scriptcontent/4482867/download), adapted to fit into Cloudera [AI to EDGE demo](https://github.com/paulvid/ai_to_edge)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "_cell_guid": "654456b6-e648-0379-0d66-1cc97af6d00d",
    "_uuid": "6b48ce0e361bdb67689dd2f254ecedd9ade1f5ff"
   },
   "source": [
    "**Import all required libraries**\n",
    "==============================="
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "_cell_guid": "e5b02688-c589-5a89-e11c-837c6a99eb6e",
    "_uuid": "f043e48097bfd98e41710142dd8aac41fa88a801"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "test.csv\n",
      "test.csv.gz\n",
      "train.csv.gz\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# This Python 3 environment comes with many helpful analytics libraries installed\n",
    "# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n",
    "# For example, here's several helpful packages to load in \n",
    "\n",
    "import numpy as np # linear algebra\n",
    "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense , Dropout , Lambda, Flatten\n",
    "from keras.optimizers import Adam ,RMSprop\n",
    "from sklearn.model_selection import train_test_split\n",
    "from keras import  backend as K\n",
    "from keras.preprocessing.image import ImageDataGenerator\n",
    "\n",
    "\n",
    "import gzip\n",
    "import shutil\n",
    "\n",
    "# Input data files are available in the \"./input/\" directory.\n",
    "# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n",
    "\n",
    "\n",
    "from subprocess import check_output\n",
    "print(check_output([\"ls\", \"./input\"]).decode(\"utf8\"))\n",
    "\n",
    "# adding Open Neural Network Exchange (ONNX) \n",
    "import onnxruntime"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "_cell_guid": "22a7fd70-ab61-432d-24cb-93e558414495",
    "_uuid": "62fbd0fe9c338b7ac0b04e688c8ee7947e6170f7"
   },
   "source": [
    "**Load Train and Test data**\n",
    "============================"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# unzipping files\n",
    "          \n",
    "with gzip.open('./input/train.csv.gz', 'rb') as f_in:\n",
    "    with open('./input/train.csv', 'wb') as f_out:\n",
    "        shutil.copyfileobj(f_in, f_out)\n",
    "        \n",
    "with gzip.open('./input/test.csv.gz', 'rb') as f_in:\n",
    "    with open('./input/test.csv', 'wb') as f_out:\n",
    "        shutil.copyfileobj(f_in, f_out)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "_cell_guid": "05226b08-226a-1a00-044d-a0e6b2101388",
    "_uuid": "4eff577bcd43479a3b7e91180393cbad9fcfca33"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(42000, 785)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>label</th>\n",
       "      <th>pixel0</th>\n",
       "      <th>pixel1</th>\n",
       "      <th>pixel2</th>\n",
       "      <th>pixel3</th>\n",
       "      <th>pixel4</th>\n",
       "      <th>pixel5</th>\n",
       "      <th>pixel6</th>\n",
       "      <th>pixel7</th>\n",
       "      <th>pixel8</th>\n",
       "      <th>...</th>\n",
       "      <th>pixel774</th>\n",
       "      <th>pixel775</th>\n",
       "      <th>pixel776</th>\n",
       "      <th>pixel777</th>\n",
       "      <th>pixel778</th>\n",
       "      <th>pixel779</th>\n",
       "      <th>pixel780</th>\n",
       "      <th>pixel781</th>\n",
       "      <th>pixel782</th>\n",
       "      <th>pixel783</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 785 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   label  pixel0  pixel1  pixel2  pixel3  pixel4  pixel5  pixel6  pixel7  \\\n",
       "0      1       0       0       0       0       0       0       0       0   \n",
       "1      0       0       0       0       0       0       0       0       0   \n",
       "2      1       0       0       0       0       0       0       0       0   \n",
       "3      4       0       0       0       0       0       0       0       0   \n",
       "4      0       0       0       0       0       0       0       0       0   \n",
       "\n",
       "   pixel8  ...  pixel774  pixel775  pixel776  pixel777  pixel778  pixel779  \\\n",
       "0       0  ...         0         0         0         0         0         0   \n",
       "1       0  ...         0         0         0         0         0         0   \n",
       "2       0  ...         0         0         0         0         0         0   \n",
       "3       0  ...         0         0         0         0         0         0   \n",
       "4       0  ...         0         0         0         0         0         0   \n",
       "\n",
       "   pixel780  pixel781  pixel782  pixel783  \n",
       "0         0         0         0         0  \n",
       "1         0         0         0         0  \n",
       "2         0         0         0         0  \n",
       "3         0         0         0         0  \n",
       "4         0         0         0         0  \n",
       "\n",
       "[5 rows x 785 columns]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# create the training & test sets, skipping the header row with [1:]\n",
    "train = pd.read_csv(\"./input/train.csv\")\n",
    "print(train.shape)\n",
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "_cell_guid": "2ec570a6-b41a-2139-5e0e-4941c4f0a9d0",
    "_uuid": "67f0854ad0d812a1395130144a0adef9966fec88"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(28000, 784)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>pixel0</th>\n",
       "      <th>pixel1</th>\n",
       "      <th>pixel2</th>\n",
       "      <th>pixel3</th>\n",
       "      <th>pixel4</th>\n",
       "      <th>pixel5</th>\n",
       "      <th>pixel6</th>\n",
       "      <th>pixel7</th>\n",
       "      <th>pixel8</th>\n",
       "      <th>pixel9</th>\n",
       "      <th>...</th>\n",
       "      <th>pixel774</th>\n",
       "      <th>pixel775</th>\n",
       "      <th>pixel776</th>\n",
       "      <th>pixel777</th>\n",
       "      <th>pixel778</th>\n",
       "      <th>pixel779</th>\n",
       "      <th>pixel780</th>\n",
       "      <th>pixel781</th>\n",
       "      <th>pixel782</th>\n",
       "      <th>pixel783</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 784 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   pixel0  pixel1  pixel2  pixel3  pixel4  pixel5  pixel6  pixel7  pixel8  \\\n",
       "0       0       0       0       0       0       0       0       0       0   \n",
       "1       0       0       0       0       0       0       0       0       0   \n",
       "2       0       0       0       0       0       0       0       0       0   \n",
       "3       0       0       0       0       0       0       0       0       0   \n",
       "4       0       0       0       0       0       0       0       0       0   \n",
       "\n",
       "   pixel9  ...  pixel774  pixel775  pixel776  pixel777  pixel778  pixel779  \\\n",
       "0       0  ...         0         0         0         0         0         0   \n",
       "1       0  ...         0         0         0         0         0         0   \n",
       "2       0  ...         0         0         0         0         0         0   \n",
       "3       0  ...         0         0         0         0         0         0   \n",
       "4       0  ...         0         0         0         0         0         0   \n",
       "\n",
       "   pixel780  pixel781  pixel782  pixel783  \n",
       "0         0         0         0         0  \n",
       "1         0         0         0         0  \n",
       "2         0         0         0         0  \n",
       "3         0         0         0         0  \n",
       "4         0         0         0         0  \n",
       "\n",
       "[5 rows x 784 columns]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test= pd.read_csv(\"./input/test.csv\")\n",
    "print(test.shape)\n",
    "test.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "_cell_guid": "1ae10fe0-dde9-7659-f53d-1a1bd625cfb1",
    "_uuid": "bdffbed77ce62da528c60e43f2b1bea9f57fcdbc"
   },
   "outputs": [],
   "source": [
    "X_train = (train.iloc[:,1:].values).astype('float32') # all pixel values\n",
    "y_train = train.iloc[:,0].values.astype('int32') # only labels i.e targets digits\n",
    "X_test = test.values.astype('float32')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "_cell_guid": "250b1126-ce1d-6d3f-9736-2504f7a1e098",
    "_uuid": "5e3e1e3574c3e019eadfd14e4dda41fd15b4de2a"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0., 0., 0., ..., 0., 0., 0.],\n",
       "       [0., 0., 0., ..., 0., 0., 0.],\n",
       "       [0., 0., 0., ..., 0., 0., 0.],\n",
       "       ...,\n",
       "       [0., 0., 0., ..., 0., 0., 0.],\n",
       "       [0., 0., 0., ..., 0., 0., 0.],\n",
       "       [0., 0., 0., ..., 0., 0., 0.]], dtype=float32)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "_cell_guid": "e0f15f8a-ac08-540a-58db-dab989cc687c",
    "_uuid": "4c96cf1c9cdc364ae3faff6b8c3c97aa7fa982d4"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([1, 0, 1, ..., 7, 6, 9], dtype=int32)"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_train"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "_cell_guid": "c2c91588-5547-353a-7f92-39600027438e",
    "_uuid": "f01a969286e62fa5ffe37031ed6d4aea947b59a8"
   },
   "source": [
    "The output variable is an integer from 0 to 9. This is a **multiclass** classification problem."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "_cell_guid": "60957d82-c76f-4822-28ff-def7011a34fa",
    "_uuid": "da0573528e8c6c3dd2b0e0cf33c600ad3f14466d"
   },
   "source": [
    "## Data Visualization\n",
    "Lets look at 3 images from data set with their labels."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "_cell_guid": "1541678d-a08b-d2b2-1e1e-eabf882baaec",
    "_uuid": "7998af1ce3c065c4a54a73cd97fed9afebde7e96"
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAU4AAABvCAYAAACD1ClOAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAQEklEQVR4nO3de4wUZbrH8e+jIip4QxQJgmy8oVFklXg7umK84o2LBiGKaFTUHBNRYxaJxqCeiJooR42u4BLA27oqIBqjKNEoJiAXMbuAuIoLXlBEj4pwlCM854/pt6p6GGampqu7unt+n4RMdVVP10M/02+/9b5vva+5OyIi0no75B2AiEitUcEpIpKSCk4RkZRUcIqIpKSCU0QkJRWcIiIpqeAUEUmp7gtOM/ul0b8tZvZI3nFJ6czsaTNba2Y/m9knZnZ13jFJ6czsHTP7NfGZXZl3TI3VfcHp7p3DP2B/4H+BF3IOS7JxL9Db3fcALgTuMbNjc45JsnFD4rN7WN7BNFb3BWcjFwHrgPfyDkRK5+7L3P238LDw76AcQ5J2or0VnKOA6a77TOuGmT1mZpuAj4G1wGs5hyTZuNfM1pvZ+2Y2IO9gGrP2UoaY2YHAKuBgd/8873gkO2a2I3AiMAC4z93/L9+IpBRmdjywHNgMDAceBfq5+2e5BpbQnmqcI4F5KjTrj7tvcfd5wAHA9XnHI6Vx9wXuvsHdf3P3acD7wLl5x5XUngrOy4FpeQchZbUTauOsRw5Y3kEktYuC08xOAnqg3vS6YWb7mdlwM+tsZjua2dnACGBu3rFJ25nZXmZ2tpntYmY7mdmlwJ+A1/OOLWmnvAOokFHADHffkHcgkhmn4bL8LzRUAFYDY9x9dq5RSak6APcAfYAtNHT6DXb3T3KNqpF20zkkIpKVdnGpLiKSJRWcIiIplVRwmtk5ZrbSzD41s7FZBSX5Ul7rl3KbjTa3cRYGHX8CnAl8CSwERrj78uzCk0pTXuuXcpudUmqcxwGfuvsqd98M/A0YlE1YkiPltX4ptxkpZThSD+CLxOMvgeOb+4WuXbt67969SzhlbVu8ePF6d9837zhaoLymVCN5hZS5VV63n9eyj+M0s9HAaIBevXqxaNGicp+yapnZ6rxjyIryGlNe61NzeS3lUv0roGfi8QGFfUXcfZK793f3/vvuWwtfyu2e8lq/Wsyt8to6pRScC4FDzOwPZrYzDbOY6K6N2qe81i/lNiNtvlR399/N7AbgDWBHYIq7L8ssMsmF8lq/lNvslNTG6e6voYlj647yWr+U22zoziERkZRUcIqIpKSCU0QkJRWcIiIpqeAUEUmpvcwALyI1YsWKFQA88sgjAPz222/RsXXr1gHw6quvFv3OcccdF20PHToUgIEDBwLQt2/fzGNUjVNEJCUVnCIiKelSXarSd999B8SXa/PmzQPg7bff3ua5HTp0AOC8886L9vXp0weAww47rOi5gwcPjrY7d+4MwE476WOQlw0bGtZPHDduXLRv+vTpRceSwvzBZsWrBS9cuHCb7fHjxwMwbNiw6NjUqVMziFo1ThGR1Griq3bGjBkAzJkzB4AhQ4ZEx7p27Vr03F69egGwfv36aN+mTZtaPMe7774LwKxZswA4/PDDo2Ph2zC8tmTj66+/BuKG/hdffDE69uabbxY9t2PHjgA0NT/k1q1bgTh3zbnyyiuj7X79+gEwatQoAG644YbomGqh5bV6dcOMbaeeeioAa9as2eY55557LgA777xztG97Nc6mfPjhhwA8//zz0b4999wTgAceeGCb105DNU4RkZRq4mv1448/BmDSpEkATJ48OTrW+BuoqRrnxo0bi56TXGep8b7wOJwTittfJDuhTXLp0qXbHLvwwgsBOPnkk4seN26zBJg/fz4AAwYMiPY9/PDDQPEwFYAFCxZE28899xwAN910EwDffvttdOzee+9N8T+R1gpDi0aMGAHENc9kDXL48OEAPP300wDssEPb6ne//PILAM8++2y0L1y9hqtQ1ThFRCqkxYLTzKaY2Toz+2diXxcze9PM/lX4uXd5w5SsKa/1S7ktvxaXBzazPwG/ANPd/cjCvvuBH9x9QmFt5r3d/c8tnax///7eljVM7rnnHgD2228/AE455ZTo2HvvvZf69ZLCMJennnoKiC8ZxowZEz3nwQcfLOkcgZktdvf+mbxYiaohr+FS7PvvvweKhxMdfPDBrX6d119/HShunrnsssta/L1wKXfkkUcCsMcee0THFi9eDMRDnZpTTXmF7HLb1rw257rrrgPiZrdQ/owcOTJ6zsSJEwHo0qVLpudOq7m8tljjdPd3gR8a7R4ETCtsTwMGIzVFea1fym35tbVzqJu7ry1sfwN0yyieJr388ssAXHPNNUDxUKHkdlvMnDkTiGuaRxxxBNBuO4QqmtfW1Apb45xzzmnxOUuWLAHiDiGIOxl/+uknAObOnRsda01Ns8ZUNLfb89JLLwFxTTMMD3vooYei54QhQ9Ws5M4hb3gHtnu9b2ajzWyRmS0Kd4NI9VNe61dzuVVeW6etNc5vzay7u681s+7Auu090d0nAZOgoc2kjecDiocIlSIMT4J4OET4Bhw7diyw7cD6diKXvGYlOYtOaJd+8sknAVi1ahUAnTp1ip5zzDHHAPDKK68AtVHTKUGrcluOvL72WrzEUajdhyu8UNNs7r3/8ccfo+3ff/+96Pf32WefLEJMra01ztnAqML2KODlbMKRnCmv9Uu5zVCLNU4zew4YAHQ1sy+BO4EJwN/N7CpgNTBs+6/QNmFOPohrmqGNs1TJmuvKlSuBeA6/8LPe5ZXXpvz6669AXDuEuGbRWPfu3aPttWsbmuzC7XrJORrDvrPPPhuAJ554Aohvs4T6vaqoltyGK4C777472tc4r03VNENeH3/88aKfEI+c2GWXXQAYPXo0EN9CCW0f1J5GiwWnu4/YzqHTM45FKkh5rV/KbfnpziERkZRq4l71rC+pksNgQqdQuKTbbbfdMj2XtCzMhJS80eDzzz9v9e+H+Qluu+22aN9pp50GNH1vu1RGmE8zOT9AcMEFFwBx88x9990XHQu9+T///PN2Xzs0A4T5WpNlxB133FFK2K2iGqeISEpVW+NMDmxPzu6chdAhBK2b10/KK9Q+Tj89boILi3I1Z8qUKQC88MILADzzzDPRsRNPPDHLEKUNQsdPU7dIhyFgs2fPBpr+HIaZrY466qhtjoW5W8PwpmQHUugw6tatfGP8VeMUEUmpamucSVm1cYZZ3pua2CT5rSj5SLYvNzXTe2N33XUXALfffjtQXOsIt2GGmmeYBbwOb6WsWuG9DpP0AJxxxhkAbN68GYgnVrn00kuj54SbUJpbcSFMzhMGx4chTACfffYZoBqniEhVUcEpIpJSTVyqZyXcMZRsiL7ooouA0mdZktb56KOPou2ePXsCpc+7GO4UufHGG6N9YXhZuDQ8/vjjgbgjCeCggw4q6bzSOslmsGXLlgGwZcsWAHbddVeg7Qshhs9ysjmvR48ebXqtNFTjFBFJqV3VOMNQiGTn0KBBg/IKp10Jw4vOPPPMaN8777wDlGem7z59+gDxsJWrr74aiAfGA7z11lsAHHrooZmfX5qWZmb/poQ5LMLS0sGxxx4bbR944IElnaM1VOMUEUmpXdU4m2rjDDO+S3mFORnPP//8aF8l3vsTTjih6Pyh7RPg+uuvB+JZlUJ7m1SvK664Aohv5wyGDBlS0ThU4xQRSak183H2BKbTsEaJA5Pc/b/NrAvwPNAb+DcwzN3/p3yhtl1YsTD8bGllz/Ygr7zutddeWb1UKqHXdvz48dG+Sy65BID3338fiHvga1k9fF4bS07+8sEHHwDxVWNouw5rF1VKa2qcvwO3uPsRwAnAf5rZEcBYYK67HwLMLTyW2qG81ifltQJaszzwWndfUtjeAKwAeqDlRmua8lqflNfKSNU5ZGa9gT8CC6iS5UbT0ExITatEXsOSF4899li0L8xsU8lF0gYPjsuLMGQpLFlbD5fqSbX+eQ1zS9xyyy3bHNt9992BeJ6CSs9B0OrOITPrDLwEjHH3ohlGtdxo7VJe65PyWl6tqnGaWQcakvCMu88o7M5tudG2Cp1C6hxqUMm8htvuvvjii2jfG2+8AcDFF18MwA47lH+QR3Ihr/333x+A+fPnl/28lVSLn9dNmzZF248++igQL8CWvFIMNcv7778faPutmqVq8S/VGqL+K7DC3R9MHNJyozVMea1PymtltKbG+R/ASOAfZra0sG8cOS0lW4rwzRXathpvtzMVzWuYazPUFAAuv/xyIJ74Ydy4cdGxjh07ZnHabSSXkV26tOG/feedd5blXDmpus9rcs2hcKtkGLA+adIkIF47COK/h6bcfPPNAFx77bWZx5lGa5YHngdsr1dFy43WKOW1PimvlaE7h0REUmoX96pPnjwZiDuFklP5azngyho5cmS0HfIRFteaNWtWdGzChAlA3KnUuXPnNp1v+fLlQLysRnI41K233grkf9lX77755ptoOzTPhHkBQs99U0MFw6xVV111VbQv5CxvqnGKiKTULmqcM2fOBOJvtaFDh+YZjhSE2kffvn0BmDhxYnQsdAKExbgGDhwIxEOXIL5aWLNmDRDfcw4wZ84cAL766isgnu09DHWBeHYkKa/k/Jhh5vf169cXPefoo4+OtkPHUahpVmJG97RU4xQRSalua5zJux7C7OO65bI69evXD4CpU6dG+zZu3AjEw5fC7P1hPkaIa5yrV68Gite2GTFiBAAnnXQSAGeddRZQPABeKiPkF4oHutcy1ThFRFKq2xpnsnYZtjXbe+3o1KkTUDx/pki1UI1TRCQlFZwiIinV7aV6coH6rVu35hiJiNQb1ThFRFKySs5NaWbfARuB9S09twp1pfS4D3T3fbMIppoor8prFSprXitacAKY2SJ371/Rk2agVuOulFp9f2o17kqp1fen3HHrUl1EJCUVnCIiKeVRcE7K4ZxZqNW4K6VW359ajbtSavX9KWvcFW/jFBGpdbpUFxFJqWIFp5mdY2YrzexTMxtbqfOmZWY9zextM1tuZsvM7MbC/i5m9qaZ/avwc++8Y60WtZBb5TU95bWZ81biUt3MdgQ+Ac4EvgQWAiPcfXnZT55SYc3p7u6+xMx2BxYDg4ErgB/cfULhj2hvd/9zjqFWhVrJrfKajvLavErVOI8DPnX3Ve6+GfgbMKhC507F3de6+5LC9gZgBdCDhninFZ42jYbkSI3kVnlNTXltRqUKzh7AF4nHXxb2VTUz6w38EVgAdHP3tYVD3wDdcgqr2tRcbpXXVlFem6HOoe0ws87AS8AYd/85ecwb2jc0HKEGKa/1qdJ5rVTB+RXQM/H4gMK+qmRmHWhIwjPuPqOw+9tCe0poV1mXV3xVpmZyq7ymorw2o1IF50LgEDP7g5ntDAwHZlfo3KlYw3TxfwVWuPuDiUOzgVGF7VHAy5WOrUrVRG6V19SU1+bOW6kB8GZ2LjAR2BGY4u7/VZETp2RmJwPvAf8AwkSe42hoN/k70AtYDQxz9x9yCbLK1EJuldf0lNdmzqs7h0RE0lHnkIhISio4RURSUsEpIpKSCk4RkZRUcIqIpKSCU0QkJRWcIiIpqeAUEUnp/wHCI2BIq3BQBAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 3 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "#Convert train datset to (num_images, img_rows, img_cols) format \n",
    "X_train = X_train.reshape(X_train.shape[0], 28, 28)\n",
    "\n",
    "for i in range(6, 9):\n",
    "    plt.subplot(330 + (i+1))\n",
    "    plt.imshow(X_train[i], cmap=plt.get_cmap('gray_r'))\n",
    "    plt.title(y_train[i]);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "_cell_guid": "6be2f3e9-42eb-85b6-9162-c25e4d706155",
    "_uuid": "4051a0e6612b8e6d4b8aef6a4d131be621cd3a14"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(42000, 28, 28, 1)"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#expand 1 more dimention as 1 for colour channel gray\n",
    "X_train = X_train.reshape(X_train.shape[0], 28, 28,1)\n",
    "X_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "_cell_guid": "6949468c-fd27-19c5-15c7-0b357a961003",
    "_uuid": "6d4c1323f1fa3f89a16532fed893ec5d72051bcb"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(28000, 28, 28, 1)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_test = X_test.reshape(X_test.shape[0], 28, 28,1)\n",
    "X_test.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "_cell_guid": "1232c385-3cb2-56fd-4d1d-f027df7bc78e",
    "_uuid": "185d620525e041eb61aabce19e8536614ab50870"
   },
   "source": [
    "**Preprocessing the digit images**\n",
    "=================================="
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "_cell_guid": "6fcc1f9e-1586-e393-49ba-50c73564e0ed",
    "_uuid": "b8847f48f7408c93ce795db16f30c1b7c6a8cf89"
   },
   "source": [
    "**Feature Standardization**\n",
    "-------------------------------------\n",
    "\n",
    "It is important preprocessing step.\n",
    "It is used to centre the data around zero mean and unit variance."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "_cell_guid": "a3f837ef-0373-8d91-46e6-30992cf73166",
    "_uuid": "528a370b381c91b73131a8c7a4217968278696c8"
   },
   "outputs": [],
   "source": [
    "mean_px = X_train.mean().astype(np.float32)\n",
    "std_px = X_train.std().astype(np.float32)\n",
    "\n",
    "def standardize(x): \n",
    "    return (x-mean_px)/std_px"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "_cell_guid": "725c55fc-9742-a63c-9822-c67ab0c773ee",
    "_uuid": "532d3f3bd26b0dfb42bc0c96e9710269234fae9b"
   },
   "source": [
    "*One Hot encoding of labels.*\n",
    "-----------------------------\n",
    "\n",
    "A one-hot vector is a vector which is 0 in most dimensions, and 1 in a single dimension. In this case, the nth digit will be represented as a vector which is 1 in the nth dimension. \n",
    "\n",
    "For example, 3 would be [0,0,0,1,0,0,0,0,0,0]."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "_cell_guid": "c879f076-b3dd-6cb1-e2d9-2f404f2ed132",
    "_uuid": "41bb3082e71111d73dd0432f9f60261f5be05e15"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "10"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from keras.utils.np_utils import to_categorical\n",
    "y_train= to_categorical(y_train)\n",
    "num_classes = y_train.shape[1]\n",
    "num_classes"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "_cell_guid": "4d76fb04-57fc-e802-6d91-06ece552686b",
    "_uuid": "429e528f5bf36152cd9e0b2acaa457525a202171"
   },
   "source": [
    "Lets plot 10th label."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "_cell_guid": "1c927e75-08d2-d539-54f3-71ab0308fec1",
    "_uuid": "b3ad8362611417de16de730cc55a6ff6309766f4"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.7/site-packages/matplotlib/text.py:1150: FutureWarning: elementwise comparison failed; returning scalar instead, but in the future will perform elementwise comparison\n",
      "  if s != self._text:\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAEICAYAAABPgw/pAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAgAElEQVR4nO3dfZAkd33f8fd3Z59ud+budLe7s+jupL2TdjaWKEDKRcHGUJQliEQSqWwSlwQ4wYWtUIUcbFxJyRCDIRWXiV2OyxUBlgFDwJIsy4BVjkAkQQQqREQnnqwHZvZ0upPupJ3de57ZvX3+5o/pXo32dm5nd2emZ3o+r6opzfR0T3/34T7q/fW3f23ujoiItL6OqAsQEZHaUKCLiMSEAl1EJCYU6CIiMaFAFxGJCQW6iEhMKNBFRGJCgR5zZuZmNm1m/ynqWiR+zOzjwe+Xm1ln1PW0OwV6e3i9u38kfGFmbzCzJ81sJvjvG6r9IDMbMbPHgm1/amY3bWDbXWb21SAAjpnZuzaw7S+b2feC/X672u3Ktv8tM5sws/Nm9nkz69nAtu8K6p02s6+Z2a4NbHtj8H2aCb5vV25g26b/Obn7x4Brq/1sqS8Fepsxs27gb4EvA5cBXwT+NlhejfuBHwK7gY8AD5nZYJXb3gPMA2ng3cCnzazaMDgN/AnwB1Wuv8LM/glwN3AjcCVwAPh4ldteC/wZ8CuU6p4BPlXltgPAV4DfBXYBh4C/qnLbVv05SZTcXY8YPwAHri57/XbgBGBly14Abq7iszLAHJAqW/Zd4P1VbNtPKSQyZcu+BPzBBr+eXwO+vcFt7gN+v+z1jcBEldv+PnBf2eurgq8jVcW2dwLfW/U9uAD8gyq2bZmfEzAS/J511vN3WY/1HzpCbz/XAj/x4F9i4CdU92fztcARdy+ULftxldtmgEV3z21i2626NthX+X7TZrZ7o9u6+3MEgbeJbaeB56j+e91uPyfZIgV6+0kC51YtOwekGrDt+U1uu1Wr6w6fN+JrbsVto/o5yRYp0NtPEdi+atl2oLDGus2y7Vat3nf4vJm/5lbcViKmQG8/TwOvMzMrW/a6YHk12x4ws/KjtddXuW0O6DSz0U1su1VPB/sq32/e3U9tdFszOwD0UPp6NrptP6Ux+Gq/1+32c5KtinoQX4/6Prj4pGg3cAz4IKVguit43V3l5z0O/BHQC/wicBYYrHLbByh1X/QDb6L0p/y1VW6bCPb5fuA7wfOuKre9GZgArgF2At+iypOxlMaOzwNvDur+MvBAldsOBl/jO4N6Pwk8XuW2LfNzQidFm+YReQF61PkHvCrQg2XXAU9S6rj4AXBd2XsfBr5+ic8bAb4dbJsFbip7793A05fYdhfwNWCaUsfGu8reezNQvMS27w2+lvLHF8reLwJvvsT2HwLyQTj/BdBT9t7TwLsvse27gnqnKbUS7ip77+vAhy+x7U3AT4Pv17eBkbL3PgN85hLbtsTPSYHePA8LfiASU2Y2S6mF7U/d/Xejrkfixcw+Rul/lj1Av7svRVxSW1Ogi4jEhE6KiojEhAJdRCQmIpsdbWBgwEdGRqLavYhIS3ryySdPuvua8/JEFugjIyMcOnQoqt2LiLQkMztW6T0NuYiIxIQCXUQkJhToIiIxoUAXEYkJBbqISEysG+jB/RcnzeypCu+bmf2pmR02s5+Y2fW1L1NERNZTzRH6FyjNVlfJLcBo8LgT+PTWyxIRkY1aN9Dd/TuUbtBbyW3Af/OSx4GdZvaaWhUoreEbT73MxLnZqMsQaWu1GEPfA7xY9vp4sOwiZnanmR0ys0NTU1M12LU0g3MXFnj/l3/AZ797JOpSRNpaQ0+Kuvu97n7Q3Q8ODq555aq0oPF86e5k2bzuUiYSpVoE+glgX9nrvcEyaRNhkOcU6CKRqkWgPwz8q6Db5Y3AOXd/uQafKy0iN1EK8vz5Oc7OzEdcjUj7WndyLjO7H3grMGBmx4GPAV0A7v4Z4BHgHcBhYAb41XoVK80pmy+Q6DCWlp1cvsgN+3dFXZJIW1o30N39jnXed+ADNatIWs54vsjPXbWb746fJJcvKNBFIqIrRWVLThbnODU9z1vHhkj2dGocXSRCCnTZknD8fCydYjSdJDuhQBeJigJdtiTscMkMJxlLp8jlC+jG4yLRUKDLluTyBS7r62Iw2UMmneLMzAJTxbmoyxJpSwp02ZJcvkgmncLMGBtOAaWTpCLSeAp02TR3JzdRWAnyTLr0X42ji0RDgS6b9vK5WQpzi4wGQT6Q7Oayvi51uohERIEumxaeEB0LAt3MyKRTmtNFJCIKdNm0sGUxk06uLBsbTjGeL6rTRSQCCnTZtFy+SHp7Dzv7uleWZdIpinOLvKS50UUaToEum5bLF1ZOhIbCE6Q5nRgVaTgFumzK0rIzPnlxoGeGgk4XjaOLNJwCXTblxdMzzC4sr5wQDe3o6yK9vUdH6CIRUKDLpuRWLvlPXfReJp0iN6lAF2k0BbpsShjoo0PJi94bS5c6XZaW1eki0kgKdNmUbL7Ivl3b6O+5eEr9zHCKucVlXjg9E0FlIu1LgS6bkpsorJwAXU1TAIhEQ4EuG7awtMyRk8U1x8/hlWEYTQEg0lgKdNmwoyenWVjyizpcQv09nezbtU2BLtJgCnTZsJWbWlQIdGDlZhci0jgKdNmw3ESBRIdxYLC/4jqZdIojU9PMLy43sDKR9qZAlw3L5gtcubuP3q5ExXUy6RSLy87zJ6cbWJlIe1Ogy4bl8sWK4+ehlU4XDbuINIwCXTZkdmGJY6emLzl+DnBgsJ9EhzGuQBdpGAW6bMjhySLL/sqsipX0diUY2d2nXnSRBlKgy4bkquhwCY0Nq9NFpJEU6LIh2XyB7kQHI7v71l13dCjFsdMzXJhfakBlIqJAlw3JTRQ4MNhPZ2L9X52x4RTupWEaEak/BbpsSC5fXHf8PBQOy2jYRaQxFOhStcLsAifOXqhq/BxgZHcf3YkOBbpIgyjQpWrjwdBJtYHemejgqqGketFFGqSqQDezm80sa2aHzezuNd6/wsweM7MfmtlPzOwdtS9VohbeVm69i4rKZdJJ3Y5OpEHWDXQzSwD3ALcA1wB3mNk1q1b7D8CD7n4dcDvwqVoXKtHL5gts60qw97JtVW+TSad46dws52cX6liZiEB1R+g3AIfd/Yi7zwMPALetWseB7cHzHcBLtStRmsV4vkgmnaSjw6reJjyaH8+r00Wk3qoJ9D3Ai2WvjwfLyv0e8B4zOw48AvzGWh9kZnea2SEzOzQ1NbWJciVK2Xyh6vHzUNgRoxOjIvVXq5OidwBfcPe9wDuAL5nZRZ/t7ve6+0F3Pzg4OFijXUsjnJ6eZ6owt+FA37NzG33dCU0BINIA1QT6CWBf2eu9wbJy7wMeBHD3/wv0AgO1KFCaw8ol/1X2oIc6OozRoaSO0EUaoJpAfwIYNbP9ZtZN6aTnw6vWeQG4EcDMfoZSoGtMJUbCQN5Ih0soo7sXiTTEuoHu7ovAXcCjwLOUulmeNrNPmNmtwWq/Dfy6mf0YuB94r7t7vYqWxsvlC2zv7SS9vWfD244NpzhZnOdUca4OlYlIqLOaldz9EUonO8uXfbTs+TPAm2pbmjST3ETpkn+z6jtcQq9MAVDkZ5Mb/x+CiFRHV4rKutydbL7A6CaGW0CdLiKNokCXdU0W5jh3YWFT4+cAQ6ketvd2agoAkTpToMu6wpbDjbYshsysdLMLtS6K1JUCXdb1yl2Kkpv+jLDTRefKRepHgS7ryuULDCR72L2FE5pjwynOzy6SP69OF5F6UaDLurLBHC5bEQ7XaBxdpH4U6HJJy8vO+CbmcFltpXVR4+gidaNAl0s6cfYCM/NLVd92rpJd/d0MJHt0hC5SRwp0uaRXTohuLdABxoaTjCvQRepGgS6XlK1Bh0uo1OlSZHlZnS4i9aBAl0vKTRS4fEcvqd6uLX/WWDrFhYUljp+5UIPKRGQ1BbpcUjZf3PCUuZWMqtNFpK4U6FLR4tIyz00WN33J/2rhsI3mdBGpDwW6VHTs9AzzS8s1OSEKkOrtYs/ObQp0kTpRoEtFYc/4VlsWy2XSSd2OTqROFOhSUTZfwAyuGtx6h0soM5ziyNQ0C0vLNftMESlRoEtFuXyBK3f1sa07UbPPzAylmF9a5tip6Zp9poiUKNClouzE1i/5Xy0cvslOFGv6uSKiQJcK5haXOHpqpqbj5wBXDyUxU6eLSD0o0GVNR6amWVr2mh+h93YlGNndr0AXqQMFuqyplnO4rJZJJ3VxkUgdKNBlTdmJAp0dxv6B/pp/diad4ujJaWYXlmr+2SLtTIEua8rlixwY7Ke7s/a/Ipl0imUvDeuISO0o0GVNuRrc1KKS8ESrxtFFakuBLheZmV/khdMzNZvDZbWR3f10JUzj6CI1pkCXi4znSz3io3UK9O7ODg4MJHU7OpEaU6DLRcIj51r3oJcbVaeLSM0p0OUi4/kCPZ0dXLGrr277GEunOH7mAtNzi3Xbh0i7UaDLRbL5IqPpJIkOq9s+wptmjE9qCgCRWlGgy0VydZjDZbXwhKvG0UVqp6pAN7ObzSxrZofN7O4K6/yymT1jZk+b2X21LVMa5dzMAhPnZ+se6Pt29dHb1aFxdJEa6lxvBTNLAPcAbwOOA0+Y2cPu/kzZOqPA7wBvcvczZjZUr4KlvnKTwQnROgd6osO4eiipXnSRGqrmCP0G4LC7H3H3eeAB4LZV6/w6cI+7nwFw98nalimNsjKHSx07XEKZdEqBLlJD1QT6HuDFstfHg2XlMkDGzP6PmT1uZjev9UFmdqeZHTKzQ1NTU5urWOoqN1Eg2dPJ5Tt6676vsXSK/Pk5zs7M131fIu2gVidFO4FR4K3AHcCfm9nO1Su5+73uftDdDw4ODtZo11JL2XyBTDqJWf06XEKZlSkA1OkiUgvVBPoJYF/Z673BsnLHgYfdfcHdnwdylAJeWoi71+UuRZWE4/Q6MSpSG9UE+hPAqJntN7Nu4Hbg4VXrfI3S0TlmNkBpCOZIDeuUBjhZnOfMzELDAv01O3pJ9XSqdVGkRtYNdHdfBO4CHgWeBR5096fN7BNmdmuw2qPAKTN7BngM+HfufqpeRUt9jDfgkv9yZsZoWp0uIrWybtsigLs/AjyyatlHy5478KHgIS0qW8e7FFUyNpziG09N4O4NGbcXiTNdKSorcvkCl/V1MZDsbtg+M+kUZ2YWmCrONWyfInGlQJcV4QnRRh4pvzIFgDpdRLZKgS5AqcNlPF9s2Ph5KGxdVKeLyNYp0AWAl8/NUphbbOj4OcBAsodd/d0rJ2RFZPMU6AI05qYWlWR0swuRmlCgC/DKNLaZocYH+lg6RW6iQKlZSkQ2S4EuQOkIPb29hx19XQ3fd2Y4xfT8EifOXmj4vkXiRIEuQKllsdHj56GVThcNu4hsiQJdWFp2Dk8W6z4HeiWjaU3SJVILCnThxdMzzC4sN2QO9LXs2NbF8PZezekiskUKdInkkv/VMsMpdbqIbJECXVaOjEeHkpHVMJZOMj5ZZGlZnS4im6VAF7L5Avt2baO/p6q52uoik04xv7jMsVPTkdUg0uoU6FK65D/C4RZ4ZbhHJ0ZFNk+B3ubmF5d5bqoY6fg5wGi6NNyj1kWRzVOgt7mjp6ZZXPbIA72vu5MrdvXpxKjIFijQ21x2IvoOl1AmmAJARDZHgd7mcvkCiQ7jwGB/1KUwNpzk+ZPTzC8uR12KSEtSoLe5XL7AyO4+ersSUZdCJp1icdl5/qQ6XUQ2Q4He5nIR3NSiknDYR+PoIpujQG9jswtLHD01zWgEU+au5cBgP4kO0zi6yCYp0NvY4cki7tHc1GItPZ0J9g/06whdZJMU6G2smTpcQmPplHrRRTZJgd7GcpMFuhMdjOzui7qUFaPpJC+cnuHC/FLUpYi0HAV6G8tNFLhqKElnonl+DcbSKdxLw0EisjHN8y9ZGi6XL5JJRzfD4lrCOdk1ji6ycQr0NlWYXeDE2QtNNX4OcOWuPro7OzSOLrIJCvQ2NR4MaUQ9y+JqnYkOrh5MKtBFNkGB3qbCXu9maVksl0kn1YsusgkK9DaVzRfo606wZ+e2qEu5SGY4xUvnZjk/uxB1KSItRYHepnL5AqNDSTo6LOpSLhIOA41r2EVkQ6oKdDO72cyyZnbYzO6+xHrvNDM3s4O1K1HqITsR/U0tKlmZ02VCrYsiG7FuoJtZArgHuAW4BrjDzK5ZY70U8EHg+7UuUmrr9PQ8J4tzTTl+DrBn5zb6uxM6MSqyQdUcod8AHHb3I+4+DzwA3LbGev8R+CQwW8P6pA7CoGzWI/SODuNqTQEgsmHVBPoe4MWy18eDZSvM7Hpgn7v/90t9kJndaWaHzOzQ1NTUhouV2giDslmP0AHG0mpdFNmoLZ8UNbMO4I+B315vXXe/190PuvvBwcHBre5aNik7UWB7bydDqZ6oS6kok05xslgaGhKR6lQT6CeAfWWv9wbLQingtcC3zewo8EbgYZ0YbV65fIGx4RRmzdfhEgr/etBRukj1qgn0J4BRM9tvZt3A7cDD4Zvufs7dB9x9xN1HgMeBW939UF0qli1x92AOl+YdboHy1kV1uohUa91Ad/dF4C7gUeBZ4EF3f9rMPmFmt9a7QKmtycIc5y4sNPX4OcBgqocd27o0SZfIBnRWs5K7PwI8smrZRyus+9atlyX10ow3tViLmZVudqEpAESqpitF20yztyyWywwnyeYLuHvUpYi0BAV6m8lOFBhI9rCrvzvqUtY1lk5RmF1k4rwubRCphgK9zeQmi4wNN9dNLSoJ/4rI6cSoSFUU6G1kedkZzxdaYrgFygJd4+giVVGgt5ETZy8wM7/UMoF+WX83g6kedbqIVEmB3kZapcOl3JjmdBGpmgK9jWRXOlxaYwwdSv/zGc8XWV5Wp4vIehTobWQ8X2DPzm2keruiLqVqY8NJLiwscfzMhahLEWl6CvQ2ks0XW+roHGA0vNmFhl1E1qVAbxOLS8s8N9n8c7isNjpU+h+QxtFF1qdAbxNHT80wv7TccoGe6u1iz85tKyd0RaQyBXqbaIWbWlQyNqxOF5FqKNDbRC5fwAyuHmqtMXQodbocmZpmYWk56lJEmpoCvU3k8gVGdvfT25WIupQNy6STzC8tc+zUdNSliDQ1BXqbyE4UVk4wtppw3D87oTldRC5Fgd4GZheWOHpqpiXHz6E0TNRhal0UWY8CvQ0cmZpmadlbrsMl1NuVYGR3vybpElmHAr0NjE+2bodLKJNOkZtUoItcigK9DWQnCnQljJHd/VGXsmmZdJKjJ6eZXViKuhSRpqVAbwO5fIH9A/10d7bujzsznGLZ4bkpnRgVqaR1/4VL1bItdFOLSsZW7l6kYReRShToMTc9t8iLpy+sBGKrGhnopythal0UuQQFeswdniwFYKaFT4gCdCU6uGowybiO0EUqUqDHXNi73epH6FCaSle96CKVKdBjLjdRoKezg327+qIuZcvG0kmOn7lAcW4x6lJEmpICPeay+QKj6SSJDou6lC0LT+xq2EVkbQr0mMvFoMMlFF4YpU4XkbUp0GPs3MwC+fNzsRg/B9h3WR+9XR3k8up0EVmLAj3GwkvlW73DJdTRYYwO6WYXIpUo0GMsvG1bXIZcoPS16HZ0ImurKtDN7GYzy5rZYTO7e433P2Rmz5jZT8zsf5nZlbUvVTYqly+Q7Onk8h29UZdSM2PDSSYLc5yZno+6FJGms26gm1kCuAe4BbgGuMPMrlm12g+Bg+7+OuAh4D/XulDZuOxEgUw6iVnrd7iEMpoCQKSiao7QbwAOu/sRd58HHgBuK1/B3R9z95ng5ePA3tqWKRvl7uTyhZaeMnctK50ukzoxKrJaNYG+B3ix7PXxYFkl7wO+vtYbZnanmR0ys0NTU1PVVykbdrI4z5mZhViNnwMMb+8l1dOpm12IrKGmJ0XN7D3AQeAP13rf3e9194PufnBwcLCWu5ZVwiGJuAW6mZEZ1hQAImupJtBPAPvKXu8Nlr2Kmd0EfAS41d3nalOebFYcO1xCmXSpddHdoy5FpKlUE+hPAKNmtt/MuoHbgYfLVzCz64A/oxTmk7UvUzZqfLLArv5uBpLdUZdSc2PpJGdnFpgq6rhBpNy6ge7ui8BdwKPAs8CD7v60mX3CzG4NVvtDIAn8tZn9yMwervBx0iBx7HAJhRdK5TQ3usirdFazkrs/AjyyatlHy57fVOO6ZAtKHS5F3nn9pc5dt65wGCmbL/DzowMRVyPSPHSlaAy9dG6W4twiozEcPwcYSPawu79bnS4iqyjQYygMurj1oJfL6GYXIhdRoMfQSsviUHwDfWw4xbg6XUReRYEeQ9l8geHtvezo64q6lLrJpFNMzy9x4uyFqEsRaRoK9BjK5QuxmTK3kkw6CWhOF5FyCvSYWVp2xvNFMkPJqEupq/CEb1atiyIrFOgx88LpGeYWl2N/hL5jWxev2dGrI3SRMgr0mAkDLi63nbuUcAoAESlRoMdM2LI4mo73kAsEnS6TRZaW1ekiAgr02MnmC1yxq4++7qouAm5po0NJ5heXOXZqOupSRJqCAj1mcvnCSgdI3K3c7ELDLiKAAj1W5heXOTI1Hcspc9dy9VASM3W6iIQU6DFy9NQ0i8se60v+y/V1d3LFrj5ykzpCFwEFeqzE+aYWlWTSKU3SJRJQoMdILl8g0WEcGOyPupSGyaSTPH9ymrnFpahLEYmcAj1GshMFRnb30dOZiLqUhsmkUywuO8+fVKeLiAI9RnL5QtuMn4fCrzerYRcRBXpczC4scez0TFuNnwMcGEjS2WGM59XpIqJAj4nDk0Xc2+OS/3LdnR3sH+jXzS5EUKDHRnblkv/2CnTQnC4iIQV6TOTyBboTHYzs7ou6lIbLpFO8cHqGmfnFqEsRiZQCPSay+QJXDSXpTLTfj3RsOIl7adhJpJ2137/+mBrPFxlrkzlcVgtPBOd0YlTanAI9BgqzC5w4eyH2N7Wo5Mrd/XR3dmgcXdqeAj0GwiPTzFB7Bnqiw7h6MKledGl7CvQYWLlLUZseoUPpa9cRurQ7BXoMZCcK9HUn2LNzW9SlRCaTTvHyuVnOXViIuhSRyCjQY2B8ssBoOkVHh0VdSmTGhksnhA9rKl1pYwr0GMhOtG+HSyjsdNHNLqSdKdBb3KniHCeLc203h8tqe3Zuo787oXF0aWsK9Ba30uHS5oFuZoymU+p0kbZWVaCb2c1mljWzw2Z29xrv95jZXwXvf9/MRmpdqKxNHS6vGNOcLtLm1g10M0sA9wC3ANcAd5jZNatWex9wxt2vBv4L8MlaFypry+UL7NjWxVCqJ+pSIpcZTnFqep6TxbmoSxGJRGcV69wAHHb3IwBm9gBwG/BM2Tq3Ab8XPH8I+K9mZu7uNawVgAefeJE//+6RWn9sy3rp7AWuvXwHZu3b4RIKpw7+pU99j55OjSZK8/q3N47yz19/ec0/t5pA3wO8WPb6OPCPK63j7otmdg7YDZwsX8nM7gTuBLjiiis2VfDOvi5G27yjo9xoOskvXrc36jKawsGRy7j9H+3j/Kx60aW57djWVZfPrSbQa8bd7wXuBTh48OCmjt7ffu0wb792uKZ1STz0diX4g3e+LuoyRCJTzd+lJ4B9Za/3BsvWXMfMOoEdwKlaFCgiItWpJtCfAEbNbL+ZdQO3Aw+vWudh4F8Hz/8F8K16jJ+LiEhl6w65BGPidwGPAgng8+7+tJl9Ajjk7g8DnwO+ZGaHgdOUQl9ERBqoqjF0d38EeGTVso+WPZ8F/mVtSxMRkY1Qb5eISEwo0EVEYkKBLiISEwp0EZGYsKi6C81sCji2yc0HWHUVakRUx6upjuaqAVTHanGo40p3H1zrjcgCfSvM7JC7H1QdqqNZ62iGGlRH+9WhIRcRkZhQoIuIxESrBvq9URcQUB2vpjpe0Qw1gOpYLdZ1tOQYuoiIXKxVj9BFRGQVBbqISEy0XKCvd8PqBtXweTObNLOnoth/UMM+M3vMzJ4xs6fN7IMR1dFrZv/PzH4c1PHxKOooqydhZj80s7+LsIajZvb3ZvYjMzsUYR07zewhM/upmT1rZj8bQQ1jwfchfJw3s9+MoI7fCn4/nzKz+82st9E1BHV8MKjh6bp8H9y9ZR6Upu99DjgAdAM/Bq6JoI63ANcDT0X4vXgNcH3wPAXkIvpeGJAMnncB3wfeGOH35UPAfcDfRVjDUWAgqv2X1fFF4NeC593AzojrSQATlC6MaeR+9wDPA9uC1w8C743g638t8BTQR2mm2/8JXF3LfbTaEfrKDavdfR4Ib1jdUO7+HUrzvkfG3V929x8EzwvAs5R+cRtdh7t7MXjZFTwiOdNuZnuBfwp8Nor9NxMz20HpwONzAO4+7+5no62KG4Hn3H2zV4hvRSewLbijWh/wUgQ1/AzwfXefcfdF4H8Dv1TLHbRaoK91w+qGh1izMbMR4DpKR8dR7D9hZj8CJoH/4e6R1AH8CfDvgeWI9h9y4Jtm9mRwY/Qo7AemgL8IhqA+a2b9EdUSuh24v9E7dfcTwB8BLwAvA+fc/ZuNroPS0fmbzWy3mfUB7+DVt/fcslYLdFnFzJLA3wC/6e7no6jB3Zfc/Q2U7jd7g5m9ttE1mNk/Aybd/clG73sNP+/u1wO3AB8ws7dEUEMnpWHBT7v7dcA0EMk5J4Dg9pW3An8dwb4vo/SX/H7gcqDfzN7T6Drc/Vngk8A3gW8APwKWarmPVgv0am5Y3TbMrItSmP+lu38l6nqCP+kfA26OYPdvAm41s6OUhuJ+wcy+HEEd4REh7j4JfJXSUGGjHQeOl/219BClgI/KLcAP3D0fwb5vAp539yl3XwC+AvxcBHXg7p9z93/o7m8BzlA691UzrRbo1dywui2YmVEaH33W3f84wjoGzWxn8Hwb8Dbgp42uw91/x933uvsIpd+Lb7l7w4/CzKzfzFLhc+DtlP7Ubih3nwBeNLOxYNGNwDONrqPMHUQw3BJ4AXijmfUF/25upHTOqeHMbCj47xWUxs/vq+XnV3VP0WbhFW5Y3eg6zOx+4K3AgJkdBz7m7p9rcBlvAn4F+Ptg/Brgw9Bn5ZYAAACISURBVF66/2sjvQb4opklKB0gPOjukbUMNoE08NVSbtAJ3Ofu34iolt8A/jI4+DkC/GoURQT/Y3sb8G+i2L+7f9/MHgJ+ACwCPyS6KQD+xsx2AwvAB2p9olqX/ouIxESrDbmIiEgFCnQRkZhQoIuIxIQCXUQkJhToIiIxoUAXEYkJBbqISEz8f0RFX2LIXOJTAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.title(y_train[9])\n",
    "plt.plot(y_train[9])\n",
    "plt.xticks(range(10));"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "_cell_guid": "4e130661-9f09-d9a9-d49b-7274ef13927f",
    "_uuid": "40aecbff9b92269d438c384ac429bfa47ab37dda"
   },
   "source": [
    "Oh its 3 !"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "_cell_guid": "6a89dcdd-7b68-6ed1-2c39-b3a1edb3e7be",
    "_uuid": "dc7ece2b7ee08767b664149d67d922d8c1d0bbb1"
   },
   "source": [
    "**Designing Neural Network Architecture**\n",
    "========================================="
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "_cell_guid": "39107235-d87a-af4d-44fb-80c9c3aa0212",
    "_uuid": "1070353d05490ccec23933c62f11cdfd2d7e5032"
   },
   "outputs": [],
   "source": [
    "# fix random seed for reproducibility\n",
    "seed = 43\n",
    "np.random.seed(seed)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "_cell_guid": "a8b65f54-398b-267f-e31a-313210450f54",
    "_uuid": "62606ecbb1d7e259850aebf8a8514e54263a2a06"
   },
   "source": [
    "*Linear Model*\n",
    "--------------"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "_cell_guid": "5dbe450c-845f-aaa2-dbde-21414a91d8c1",
    "_uuid": "5f54b59d89cd4e43dd129d9b133950ba83b5cad8"
   },
   "outputs": [],
   "source": [
    "from keras.models import  Sequential\n",
    "from keras.layers.core import  Lambda , Dense, Flatten, Dropout\n",
    "from keras.callbacks import EarlyStopping\n",
    "from keras.layers import BatchNormalization, Convolution2D , MaxPooling2D"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "_cell_guid": "5c3f674f-f3fc-9614-f2d4-056c3e3ad633",
    "_uuid": "ff25a88562237e84e44f20b38079f2b44a394d2c"
   },
   "source": [
    "Lets create a simple model from Keras Sequential layer.\n",
    "\n",
    "1. Lambda layer performs simple arithmetic operations like sum, average, exponentiation etc.\n",
    "\n",
    " In 1st layer of the model we have to define input dimensions of our data in (rows,columns,colour channel) format.\n",
    " (In theano colour channel comes first)\n",
    "\n",
    "\n",
    "2. Flatten will transform input into 1D array.\n",
    "\n",
    "\n",
    "3. Dense is fully connected layer that means all neurons in previous layers will be connected to all neurons in fully connected layer.\n",
    " In the last layer we have to specify output dimensions/classes of the model.\n",
    " Here it's 10, since we have to output 10 different digit labels."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "_cell_guid": "a2c27783-3cfa-e907-4749-1e340a513f26",
    "_uuid": "fb79b4558335446a722542c8bc06288e96781423"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING: Logging before flag parsing goes to stderr.\n",
      "W0624 09:59:34.535813 4634289600 deprecation_wrapper.py:119] From /usr/local/lib/python3.7/site-packages/keras/backend/tensorflow_backend.py:74: The name tf.get_default_graph is deprecated. Please use tf.compat.v1.get_default_graph instead.\n",
      "\n",
      "W0624 09:59:34.550372 4634289600 deprecation_wrapper.py:119] From /usr/local/lib/python3.7/site-packages/keras/backend/tensorflow_backend.py:517: The name tf.placeholder is deprecated. Please use tf.compat.v1.placeholder instead.\n",
      "\n",
      "W0624 09:59:34.632287 4634289600 deprecation_wrapper.py:119] From /usr/local/lib/python3.7/site-packages/keras/backend/tensorflow_backend.py:4138: The name tf.random_uniform is deprecated. Please use tf.random.uniform instead.\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "input shape  (None, 28, 28, 1)\n",
      "output shape  (None, 10)\n"
     ]
    }
   ],
   "source": [
    "model= Sequential()\n",
    "model.add(Lambda(standardize,input_shape=(28,28,1)))\n",
    "model.add(Flatten())\n",
    "model.add(Dense(10, activation='softmax'))\n",
    "print(\"input shape \",model.input_shape)\n",
    "print(\"output shape \",model.output_shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "_cell_guid": "260645fb-61b7-68e9-6826-047b97436c14",
    "_uuid": "2dd7f371688dd2590de94a94814799654595e55d"
   },
   "source": [
    "***Compile network***\n",
    "-------------------\n",
    "\n",
    "Before making network ready for training we have to make sure to add below things:\n",
    "\n",
    " 1.  A loss function: to measure how good the network is\n",
    "    \n",
    " 2.  An optimizer: to update network as it sees more data and reduce loss\n",
    "    value\n",
    "    \n",
    " 3.  Metrics: to monitor performance of network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "_cell_guid": "9d1d1af9-b2a8-e3b9-6eaf-100d08fe83aa",
    "_uuid": "4bb75be10b9eec8bcdfe48665f639bf326a5f5fc"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "W0624 09:59:37.087071 4634289600 deprecation_wrapper.py:119] From /usr/local/lib/python3.7/site-packages/keras/optimizers.py:790: The name tf.train.Optimizer is deprecated. Please use tf.compat.v1.train.Optimizer instead.\n",
      "\n",
      "W0624 09:59:37.093498 4634289600 deprecation_wrapper.py:119] From /usr/local/lib/python3.7/site-packages/keras/backend/tensorflow_backend.py:3295: The name tf.log is deprecated. Please use tf.math.log instead.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from keras.optimizers import RMSprop\n",
    "model.compile(optimizer=RMSprop(lr=0.001),\n",
    " loss='categorical_crossentropy',\n",
    " metrics=['accuracy'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "_cell_guid": "db3b4be6-4f72-c6cc-65cd-b45978db2462",
    "_uuid": "51f82558d87e95fa5c146b0469ab6c8b42e13bcf"
   },
   "outputs": [],
   "source": [
    "from keras.preprocessing import image\n",
    "gen = image.ImageDataGenerator()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "_uuid": "841a6f3b78b607e142f3e18d88bd7957202e4dcb"
   },
   "source": [
    "## Cross Validation "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "_cell_guid": "9071d720-da50-8530-e9f3-1f0c37aac7ff",
    "_uuid": "0cff7e02b1ee8894b4ee9080b9268558aaa4e7c5"
   },
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "X = X_train\n",
    "y = y_train\n",
    "X_train, X_val, y_train, y_val = train_test_split(X_train, y_train, test_size=0.10, random_state=42)\n",
    "batches = gen.flow(X_train, y_train, batch_size=64)\n",
    "val_batches=gen.flow(X_val, y_val, batch_size=64)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "_cell_guid": "20e08e2a-a394-bb70-69f1-be0fdab4f9ab",
    "_uuid": "6b23c282e2772b1ee482596131d6f1d3494c3bce"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "W0624 09:59:43.208245 4634289600 deprecation.py:323] From /usr/local/lib/python3.7/site-packages/tensorflow/python/ops/math_grad.py:1250: add_dispatch_support.<locals>.wrapper (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.where in 2.0, which has the same broadcast rule as np.where\n",
      "W0624 09:59:43.251929 4634289600 deprecation_wrapper.py:119] From /usr/local/lib/python3.7/site-packages/keras/backend/tensorflow_backend.py:986: The name tf.assign_add is deprecated. Please use tf.compat.v1.assign_add instead.\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/3\n",
      "37800/37800 [==============================] - 64s 2ms/step - loss: 0.2401 - acc: 0.9342 - val_loss: 0.3294 - val_acc: 0.9145\n",
      "Epoch 2/3\n",
      "37800/37800 [==============================] - 63s 2ms/step - loss: 0.2156 - acc: 0.9417 - val_loss: 0.3547 - val_acc: 0.9072\n",
      "Epoch 3/3\n",
      "37800/37800 [==============================] - 63s 2ms/step - loss: 0.2099 - acc: 0.9437 - val_loss: 0.3724 - val_acc: 0.9072\n"
     ]
    }
   ],
   "source": [
    "history=model.fit_generator(generator=batches, steps_per_epoch=batches.n, epochs=3, \n",
    "                    validation_data=val_batches, validation_steps=val_batches.n)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "_cell_guid": "9f344366-c372-0b04-b7e0-860778d4bfd3",
    "_uuid": "6900e38c62028692b9f101b94730c527129675cc"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict_keys(['val_loss', 'val_acc', 'loss', 'acc'])"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "history_dict = history.history\n",
    "history_dict.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "_cell_guid": "df40f5fc-586a-1fae-025e-ee508a8d9b71",
    "_uuid": "c4b26ff79e0f186212266b60d03611ad58d0d5e3"
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEKCAYAAAAIO8L1AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAZvElEQVR4nO3dfbRddZ3f8feHhCgBFZSrZfJAombWmjAwYM/Eh9HRMkwNQ02wusbEUHHKWilKqi3LKbFxScyUtTqhtS410zG2WHXCZAAHV0bBaBlaxwc0NxDAhIlcIg/JME3wCWmmQPDTP/bvws7Nvsm55O5z7k0+r7XOunv/9m+f872bzf1k798+e8s2ERERI53Q7wIiImJiSkBERESjBERERDRKQERERKMERERENEpAREREo1YDQtJCSTslDUla2bD8ckn3Stom6VuS5pf2EyV9viy7T9KH26wzIiIOpba+ByFpCvBD4HeB3cAWYKntHbU+L7b9eJleBLzf9kJJ7wYW2V4iaTqwA3iL7QdbKTYiIg7R5hHEAmDI9i7bTwEbgcX1DsPhUJwMDKeVgZMlTQVOAp4C6n0jIqJlU1t87xnAI7X53cBrR3aSdAVwJTANOL8030QVJo8C04F/a/snh/uw008/3XPmzDn6qiMijiNbt259zPZA07I2A6IrttcB68pppY8Al1IdfTwD/ApwGvA3kv6n7V31dSUtB5YDzJ49m8HBwZ7WHhEx2Ul6aLRlbZ5i2gPMqs3PLG2j2QhcXKbfDXzN9tO29wLfBjojV7C93nbHdmdgoDEAIyLieWozILYA8yTNlTQNWAJsqneQNK82exFwf5l+mHK6SdLJwOuAv22x1oiIGKG1U0y2D0haAWwGpgDX2d4uaQ0waHsTsELSBcDTwE+pTi8BrAM+J2k7IOBztu9pq9aIiDhUa5e59lqn03HGICIixkbSVtuHnMKHfJM6IiJGkYCIiJjkVq9u530TEBERk9zHPtbO+yYgIiKiUQIiImISWr0apOoFz02P5+mmXMUUETHJSfB8/5TnKqaIiBizBERExCR39dXtvG8CIiJikstlrhER0VMJiIiIaJSAiIiIRgmIiIholICIiIhGCYiIiGiUgIiIiEatBoSkhZJ2ShqStLJh+eWS7pW0TdK3JM2vLTtH0nclbS99XthmrRERcbDWAkLSFKpHh14IzAeW1gOguN722bbPBdYCHy/rTgX+DLjc9lnAW6geSxoRET3S5hHEAmDI9i7bTwEbgcX1DrYfr82eDAzfbuqfAvfYvrv0+7HtZ1qsNSIiRmgzIGYAj9Tmd5e2g0i6QtIDVEcQHyjNvwpY0mZJd0r6d00fIGm5pEFJg/v27Rvn8iMijm99H6S2vc72q4CrgI+U5qnAG4Fl5efbJf1Ow7rrbXdsdwYGBnpWc0TE8aDNgNgDzKrNzyxto9kIXFymdwPftP2Y7f3ALcBrWqkyIiIatRkQW4B5kuZKmgYsATbVO0iaV5u9CLi/TG8GzpY0vQxYvxnY0WKtERExQmsBYfsAsILqj/19wA22t0taI2lR6baiXMa6DbgSuLSs+1OqK5q2ANuAO21/ta1aI9rW1u2YI9qUR45G9MDRPBIyok155GhERIxZAiKiJatXV0cOUjU/PJ3TTTFZ5BRTRA/kFFNMVDnFFBERY5aAiOiBq6/udwURY5eAiOiBjDvEZJSAiIiIRgmIiIholICIiIhGCYiIiGiUgIiIiEYJiIiIaJSAiIiIRgmIiIholICIiIhGrQaEpIWSdkoakrSyYfnlku6VtE3StyTNH7F8tqQnJH2ozTojIuJQrQWEpCnAOuBCYD6wdGQAANfbPtv2ucBaqqfI1X0cuLWtGiMiYnRtHkEsAIZs77L9FLARWFzvYPvx2uzJwLM3RJZ0MfAjYHuLNUZExCjaDIgZwCO1+d2l7SCSrpD0ANURxAdK2ynAVcDHWqwvIiIOo++D1LbX2X4VVSB8pDSvBv6L7ScOt66k5ZIGJQ3u27ev5UojIo4vU1t87z3ArNr8zNI2mo3Afy3TrwXeKWktcCrwS0n/z/an6yvYXg+sh+qJcuNVeEREtBsQW4B5kuZSBcMS4N31DpLm2b6/zF4E3A9g+021PquBJ0aGQ0REtKu1gLB9QNIKYDMwBbjO9nZJa4BB25uAFZIuAJ4Gfgpc2lY9ERExNvIx8iT1TqfjwcHBfpcRETGpSNpqu9O0rO+D1BERMTElICIiolECIiIiGiUgIiKiUQIiIiIaJSCK1av7XUFExMSSgCg+lrs+RUQcJAERERGNjuuAWL0apOoFz03ndFNERL5J/SwJjpFNERHRtXyTOiIixiwBUVx9db8riIiYWBIQRcYdIiIOloCIiIhGCYiIiGjUakBIWihpp6QhSSsbll8u6V5J2yR9S9L80v67kraWZVslnd9mnRERcajWAkLSFGAdcCEwH1g6HAA119s+2/a5wFrg46X9MeBtts+mesrcF9uqMyIimrV5BLEAGLK9y/ZTwEZgcb2D7cdrsycDLu132f670r4dOEnSC1qsNSIiRmjtmdTADOCR2vxu4LUjO0m6ArgSmAY0nUp6B3Cn7SfbKDIiIpr1fZDa9jrbrwKuAj5SXybpLOCPgX/VtK6k5ZIGJQ3u27ev/WIjIo4jbQbEHmBWbX5maRvNRuDi4RlJM4GbgffYfqBpBdvrbXdsdwYGBsah5IiIGNZmQGwB5kmaK2kasATYVO8gaV5t9iLg/tJ+KvBVYKXtb7dYY0REjKK1gLB9AFgBbAbuA26wvV3SGkmLSrcVkrZL2kY1DnHpcDvwauCj5RLYbZJe3latERFxqNzNNSLiOJa7uUZExJglICIiolECIiIiGiUgIiKiUQIiIiIaJSAiIqJRAiIiIholICIiolECIiIiGiUgIiKiUQIiIiIaJSAiIqJRAiIiIholICIiolECIiIiGiUgIiKiUasBIWmhpJ2ShiStbFh+uaR7yxPjviVpfm3Zh8t6OyW9tc06IyLiUK0FhKQpwDrgQmA+sLQeAMX1ts+2fS6wFvh4WXc+1TOszwIWAn9S3i8iInqkzSOIBcCQ7V22nwI2AovrHWw/Xps9GRh+/uliYKPtJ23/CBgq7xcRET0ytcX3ngE8UpvfDbx2ZCdJVwBXAtOA82vr3jFi3RkN6y4HlgPMnj17XIqOiIhK3wepba+z/SrgKuAjY1x3ve2O7c7AwEA7BUZEHKe6CghJr5L0gjL9FkkfkHTqEVbbA8yqzc8sbaPZCFz8PNeNiIhx1u0RxJeAZyS9GlhP9cf7+iOsswWYJ2mupGlUg86b6h0kzavNXgTcX6Y3AUskvUDSXGAe8P0ua42IiHHQ7RjEL20fkPR24FO2PyXprsOtUPqvADYDU4DrbG+XtAYYtL0JWCHpAuBp4KfApWXd7ZJuAHYAB4ArbD/zvH7DiIh4XroNiKclLaX6A/620nbikVayfQtwy4i2j9amP3iYda8BrumyvoiIGGfdnmL6A+D1wDW2f1RO+3yxvbIiIqLfujqCsL0D+ACApNOAF9n+4zYLi4iI/ur2Kqb/JenFkl4K3Al8VtLH2y0tIiL6qdtTTC8p33r+58AXbL8WuKC9siIiot+6DYipks4Afh/4Sov1RETEBNFtQKyhulz1AdtbJL2S576zEBERx6BuB6lvBG6sze8C3tFWURER0X/dDlLPlHSzpL3l9SVJM9suLiIi+qfbU0yfo7r9xa+U11+VtoiIOEZ1GxADtj9n+0B5/Q8gt0+NiDiGdRsQP5Z0iaQp5XUJ8OM2C4uIiP7qNiD+JdUlrn8PPAq8E3hvSzVFRMQE0FVA2H7I9iLbA7ZfbvtichVTRMQx7WieKHfluFURERETztEEhMatioiImHCOJiB8pA6SFkraKWlI0sqG5VdK2iHpHkm3STqztmytpO2S7pP0SUkJpIiIHjpsQEj6haTHG16/oPo+xOHWnQKsAy4E5gNLJc0f0e0uoGP7HOAmYG1Z9w3AbwHnAL8O/Cbw5rH/ehER8Xwd9lYbtl90FO+9ABgqt+VA0kZgMdVjRIff//Za/zuAS4YXAS8EplGdyjoR+D9HUUtERIzR0ZxiOpIZwCO1+d2lbTSXAbcC2P4ucDvVJbWPAptt39dSnRER0aDNgOha+eJdB7i2zL8a+DVgJlWonC/pTQ3rLZc0KGlw3759vSw5IuKY12ZA7AFm1eZnlraDSLoAWAUssv1kaX47cIftJ2w/QXVk8fqR69peb7tjuzMwkDt/RESMpzYDYgswT9JcSdOAJVQ3/HuWpPOAz1CFw97aooeBN0uaKulEqgHqnGKKiOih1gLC9gFgBdWDhu4DbrC9XdIaSYtKt2uBU4AbJW2TNBwgNwEPAPcCdwN32/6rtmqNiIhDyT7i1xkmhU6n48HBwX6XERExqUjaarvTtGxCDFJHRMTEk4CIiIhGCYiIiGiUgIiIiEYJiIiIaJSAiIiIRgmIiIholICIiIhGCYiIiGiUgIiIiEYJiIiIaJSAiIiIRgmIiIholICIiIhGCYiIiGjUakBIWihpp6QhSSsbll8paYekeyTdJunM2rLZkr4u6b7SZ06btUZExMFaCwhJU4B1wIXAfGCppPkjut0FdGyfQ/UUubW1ZV8ArrX9a8ACYC8REdEzbR5BLACGbO+y/RSwEVhc72D7dtv7y+wdwEyAEiRTbX+j9Hui1i8iInqgzYCYATxSm99d2kZzGXBrmf5V4GeS/lLSXZKuLUckERHRIxNikFrSJUAHuLY0TQXeBHwI+E3glcB7G9ZbLmlQ0uC+fft6VG1ExPGhzYDYA8yqzc8sbQeRdAGwClhk+8nSvBvYVk5PHQC+DLxm5Lq219vu2O4MDAyM+y8QEXE8azMgtgDzJM2VNA1YAmyqd5B0HvAZqnDYO2LdUyUN/9U/H9jRYq0RETFCawFR/uW/AtgM3AfcYHu7pDWSFpVu1wKnADdK2iZpU1n3GarTS7dJuhcQ8Nm2ao2IiEPJdr9rGBedTseDg4P9LiMiYlKRtNV2p2nZhBikjoiIiScBERERjRIQERHRKAERERGNEhAREdEoAREREY0SEBER0SgBERERjRIQERHRKAERERGNEhAREdEoAREREY0SEBER0SgBERERjRIQERHRKAERERGNWg0ISQsl7ZQ0JGllw/IrJe2QdI+k2ySdOWL5iyXtlvTptmrcsAHmzIETTqh+btjQ1idFREwurQWEpCnAOuBCYD6wVNL8Ed3uAjq2zwFuAtaOWP5HwDfbqnHDBli+HB56COzq5/LlCYmICGj3CGIBMGR7l+2ngI3A4noH27fb3l9m7wBmDi+T9I+BVwBfb6vAVatg//6D2/bvr9ojIo53bQbEDOCR2vzu0jaay4BbASSdAPxn4EOH+wBJyyUNShrct2/fmAt8+OGxtUdEHE8mxCC1pEuADnBtaXo/cIvt3Ydbz/Z62x3bnYGBgTF/7uzZY2uPiDietBkQe4BZtfmZpe0gki4AVgGLbD9Zml8PrJD0IPCfgPdI+o/jXeA118D06Qe3TZ9etUdEHO+mtvjeW4B5kuZSBcMS4N31DpLOAz4DLLS9d7jd9rJan/dSDWQfchXU0VpWPmXVquq00uzZVTgsW3b49SIijgetBYTtA5JWAJuBKcB1trdLWgMM2t5EdUrpFOBGSQAP217UVk1Nli1LIERENJHtftcwLjqdjgcHB/tdRkTEpCJpq+1O07IJMUgdERETTwIiIiIaJSAiIqJRAiIiIholICIiolECIiIiGiUgIiKiUQIiIiIaJSAiIqJRAiIiIholICIiolECIiIiGiUgIiKiUQIiIiIaJSAiIqJRqwEhaaGknZKGJB3yRDhJV0raIekeSbdJOrO0nyvpu5K2l2XvarPOiIg4VGsBIWkKsA64EJgPLJU0f0S3u6geJ3oOcBOwtrTvB95j+yxgIfAJSae2VWtERByqzSOIBcCQ7V22nwI2AovrHWzfbnt/mb0DmFnaf2j7/jL9d8BeYKDFWiMiYoQ2A2IG8EhtfndpG81lwK0jGyUtAKYBD4xrdRERcVhT+10AgKRLgA7w5hHtZwBfBC61/cuG9ZYDywFmz57dg0ojIo4fbR5B7AFm1eZnlraDSLoAWAUssv1krf3FwFeBVbbvaPoA2+ttd2x3BgZyBioiYjy1GRBbgHmS5kqaBiwBNtU7SDoP+AxVOOyttU8Dbga+YPumFmuMaNWGDTBnDpxwQvVzw4Z+VxTRvdZOMdk+IGkFsBmYAlxne7ukNcCg7U3AtcApwI2SAB62vQj4feC3gZdJem95y/fa3tZWvRHjbcMGWL4c9pfLMB56qJoHWLasf3VFdEu2+13DuOh0Oh4cHOx3GRHPmjOnCoWRzjwTHnyw19VENJO01XanaVm+SR3RkocfHlt7xESTgIhoyWgX1uWCu5gsEhARLbnmGpg+/eC26dOr9ojJIAER0ZJly2D9+mrMQap+rl+fAeqYPBIQES1atqwakP7lL6ufCYcYT21fRj0hvkkdERFj04vLqHMEERExCa1a9Vw4DNu/v2ofLwmIiIhJqBeXUScgIiImoV5cRp2AiIiYhHpxGXUCIiJiEurFZdS5iikiYpJatqzdS6dzBBEREY0SEBER0SgBERERjRIQERHRKAERERGNjpknyknaBzQ8v6trpwOPjVM54yl1jU3qGpvUNTbHYl1n2h5oWnDMBMTRkjQ42mP3+il1jU3qGpvUNTbHW105xRQREY0SEBER0SgB8Zz1/S5gFKlrbFLX2KSusTmu6soYRERENMoRRERENDrmA0LSdZL2SvrBKMsl6ZOShiTdI+k1tWWXSrq/vC7tcV3LSj33SvqOpN+oLXuwtG+TNNjjut4i6efls7dJ+mht2UJJO8u2XNnjuv6wVtMPJD0j6aVlWZvba5ak2yXtkLRd0gcb+vR0H+uypn7tX93U1vN9rMu6er6PSXqhpO9LurvU9bGGPi+Q9Bdlm3xP0pzasg+X9p2S3jrmAmwf0y/gt4HXAD8YZfnvAbcCAl4HfK+0vxTYVX6eVqZP62Fdbxj+PODC4brK/IPA6X3aXm8BvtLQPgV4AHglMA24G5jfq7pG9H0b8Nc92l5nAK8p0y8Cfjjy9+71PtZlTf3av7qpref7WDd19WMfK/vMKWX6ROB7wOtG9Hk/8KdlegnwF2V6ftlGLwDmlm03ZSyff8wfQdj+JvCTw3RZDHzBlTuAUyWdAbwV+Ibtn9j+KfANYGGv6rL9nfK5AHcAM8frs4+mrsNYAAzZ3mX7KWAj1bbtR11LgT8fr88+HNuP2r6zTP8CuA+YMaJbT/exbmrq4/7VzfYaTWv72POoqyf7WNlnniizJ5bXyIHjxcDny/RNwO9IUmnfaPtJ2z8Chqi2YdeO+YDowgzgkdr87tI2Wns/XEb1L9BhBr4uaauk5X2o5/XlkPdWSWeVtgmxvSRNp/oj+6Vac0+2Vzm0P4/qX3l1fdvHDlNTXV/2ryPU1rd97EjbrNf7mKQpkrYBe6n+QTHq/mX7APBz4GWMw/bKA4MmOEn/hOp/4DfWmt9oe4+klwPfkPS35V/YvXAn1Vfzn5D0e8CXgXk9+uxuvA34tu360Ubr20vSKVR/MP6N7cfH872fr25q6tf+dYTa+raPdfnfsaf7mO1ngHMlnQrcLOnXbTeOxY23HEHAHmBWbX5maRutvWcknQP8N2Cx7R8Pt9veU37uBW5mjIeNR8P248OHvLZvAU6UdDoTYHsVSxhx6N/29pJ0ItUflQ22/7KhS8/3sS5q6tv+daTa+rWPdbPNip7vY+W9fwbczqGnIZ/dLpKmAi8Bfsx4bK/xHlSZiC9gDqMPul7EwQOI3y/tLwV+RDV4eFqZfmkP65pNdc7wDSPaTwZeVJv+DrCwh3X9I577/swC4OGy7aZSDbLO5bkBxLN6VVdZ/hKqcYqTe7W9yu/+BeATh+nT032sy5r6sn91WVvP97Fu6urHPgYMAKeW6ZOAvwH+2Yg+V3DwIPUNZfosDh6k3sUYB6mP+VNMkv6c6qqI0yXtBq6mGujB9p8Ct1BdZTIE7Af+oCz7iaQ/AraUt1rjgw8p267ro1TnEf+kGm/igKubcb2C6jATqv9hrrf9tR7W9U7gfZIOAP8ALHG1Nx6QtALYTHW1yXW2t/ewLoC3A1+3/X9rq7a6vYDfAv4FcG85Twzw76n+APdrH+umpr7sX13W1o99rJu6oPf72BnA5yVNoTrjc4Ptr0haAwza3gT8d+CLkoaowmtJqXm7pBuAHcAB4ApXp6u6lm9SR0REo4xBREREowREREQ0SkBERESjBERERDRKQERERKMERMQRlLt2bqu9xvMuonM0yh1qI/rtmP8eRMQ4+Afb5/a7iIheyxFExPNUngGwtjwH4PuSXl3a50j6a1XPW7hN0uzS/gpJN5eb0N0t6Q3lraZI+my53//XJZ1U+n9A1fMJ7pG0sU+/ZhzHEhARR3bSiFNM76ot+7nts4FPA58obZ8CPm/7HGAD8MnS/kngf9v+DapnWwx/C3gesM72WcDPgHeU9pXAeeV9Lm/rl4sYTb5JHXEEkp6wfUpD+4PA+bZ3lRu9/b3tl0l6DDjD9tOl/VHbp0vaB8y0/WTtPeZQ3cJ5Xpm/CjjR9n+Q9DXgCaq7mX7Zzz0XIKIncgQRcXQ8yvRYPFmbfobnxgYvAtZRHW1sKXfqjOiZBETE0XlX7ed3y/R3KDdMA5ZR3YET4DbgffDsQ2BeMtqbSjoBmGX7duAqqruIHnIUE9Gm/Isk4shOqt3hE+BrtocvdT1N0j1URwFLS9u/Bj4n6Q+BfZS7twIfBNZLuozqSOF9wKOjfOYU4M9KiAj4pKvnAUT0TMYgIp6nMgbRsf1Yv2uJaENOMUVERKMcQURERKMcQURERKMERERENEpAREREowREREQ0SkBERESjBERERDT6/7Yd6P5saIdQAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "loss_values = history_dict['loss']\n",
    "val_loss_values = history_dict['val_loss']\n",
    "epochs = range(1, len(loss_values) + 1)\n",
    "\n",
    "# \"bo\" is for \"blue dot\"\n",
    "plt.plot(epochs, loss_values, 'bo')\n",
    "# b+ is for \"blue crosses\"\n",
    "plt.plot(epochs, val_loss_values, 'b+')\n",
    "plt.xlabel('Epochs')\n",
    "plt.ylabel('Loss')\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "_cell_guid": "1ed6b756-00c2-d08c-c596-0ce496ec3d04",
    "_uuid": "fc9be5b885360ca9972e0d6b5da1ea36dc12cb5f"
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY4AAAEICAYAAABI7RO5AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAcWElEQVR4nO3df5RX9X3n8ecLEKNVo8JIrYNAV/a0Y2M1+ZZqNgarzYqJEcVsxXD8kfWEY4xpujm4wtJUQ0I9tvQka+OmhyRYSEmRuKth0yhJEZKcTTR8iYAiBUcSZQa6jjFECV0N+N4/7mfw8mVm+F743u93YF6Pc+6Zez/3cz/f9/1ymffcz+f+UERgZmZWr2GtDsDMzI4uThxmZlaIE4eZmRXixGFmZoU4cZiZWSFOHGZmVkipiUPSFElbJHVKmt3H+nGSVknaKGmNpPaa9adI6pL0xVzZmtTm+jSdUeY+mJnZgUaU1bCk4cD9wPuALmCtpBUR8Wyu2gJgSUQslnQpcA9wQ279Z4Hv99H8jIio1hvL6NGjY/z48UV3wcxsSFu3bt3LEdFWW15a4gAmAZ0RsQ1A0jJgKpBPHB3Ap9L8auCR3hWS3gWMAR4DKkcSyPjx46lW684zZmYGSHqhr/Iyu6rOArbnlrtSWd4GYFqavwY4WdIoScOAvwFm9dP2A6mb6tOS1MigzcxsYK0eHJ8FTJb0FDAZ6Ab2AbcB346Irj62mRER7wAuTtMNfdRB0kxJVUnVnp6ecqI3MxuCyuyq6gbG5pbbU9l+EbGDdMYh6STg2ojYJeki4GJJtwEnASMl7Y6I2RHRnbZ9TdLXybrEltR+eEQsBBYCVCoVP5DLzKxBykwca4GJkiaQJYzpwIfzFSSNBl6JiDeBOcAigIiYkatzM1CJiNmSRgCnRsTLko4DrgT+ucR9MDOzGqV1VUXEXuB2YCWwGVgeEZskzZN0Vap2CbBF0laygfD5h2j2eGClpI3AerKE9OUy4jczs75pKDxWvVKphK+qMrOhYulSmDsXXnwRzj4b5s+HGTMOvV0tSesi4qCrWsvsqjIzsyZbuhRmzoQ9e7LlF17IluHwkkdfWn1VlZmZNdDcuW8ljV579mTljeLEYWZ2DHnxxWLlh8OJw8zsGHL22cXKD4cTh5nZMWT+fDjxxAPLTjwxK28UJw4zs2PIjBmwcCGMGwdS9nPhwsYNjIOvqjIzO+bMmNHYRFHLZxxmZlaIE4eZmRXixGFmZoU4cZi1wNKlMH48DBuW/Vy6tNURmdXPg+NmTdaMR0KYlclnHGZN1oxHQpiVyYnDrMma8UgIszI5cZg1WTMeCWFWJicOsyZrxiMhzMrkxGHWZM14JIRZmXxVlVkLlP1ICLMylXrGIWmKpC2SOiXN7mP9OEmrJG2UtEZSe836UyR1Sfpiruxdkp5Obd4nSWXug5mZHai0xCFpOHA/cAXQAVwvqaOm2gJgSUScB8wD7qlZ/1ng+zVlXwI+CkxM05QGh25mZgMo84xjEtAZEdsi4g1gGTC1pk4H8HiaX51fL+ldwBjgO7myM4FTIuKJiAhgCXB1ebtgZma1ykwcZwHbc8tdqSxvAzAtzV8DnCxplKRhwN8As/pos+sQbZqZWYlafVXVLGCypKeAyUA3sA+4Dfh2RHQNtPFAJM2UVJVU7enpaUy0ZmZW6lVV3cDY3HJ7KtsvInaQzjgknQRcGxG7JF0EXCzpNuAkYKSk3cB/T+3022au7YXAQoBKpRIN2SMzMys1cawFJkqaQPbLfTrw4XwFSaOBVyLiTWAOsAggImbk6twMVCJidlp+VdKFwJPAjcDflrgPZmZWo7SuqojYC9wOrAQ2A8sjYpOkeZKuStUuAbZI2ko2EF7PvbO3AV8BOoHngUcbHbuZmfVP2cVJx7ZKpRLVarXVYZiZHVUkrYuISm15qwfHzczsKOPEYWZmhThxmJlZIU4cZmZWiBOHmZkV4sRhZmaFOHGYmVkhThxmZlaIE4eZmRXixGFmZoU4cZiZWSFOHGZmVogTh5mZFeLEYWZmhThxmJlZIU4cZmZWiBOHmZkV4sRhZmaFlJo4JE2RtEVSp6TZfawfJ2mVpI2S1khqz5X/RNJ6SZsk3ZrbZk1qc32azihzH8zM7EAjympY0nDgfuB9QBewVtKKiHg2V20BsCQiFku6FLgHuAHYCVwUEa9LOgl4Jm27I203IyL8EnEzsxYo84xjEtAZEdsi4g1gGTC1pk4H8HiaX927PiLeiIjXU/nxJcdpZmYFlPkL+Sxge265K5XlbQCmpflrgJMljQKQNFbSxtTGvbmzDYAHUjfVpyWpnPDNzKwvrf5LfhYwWdJTwGSgG9gHEBHbI+I84BzgJklj0jYzIuIdwMVpuqGvhiXNlFSVVO3p6Sl7P8zMhowyE0c3MDa33J7K9ouIHRExLSIuAOamsl21dYBnyJIEEdGdfr4GfJ2sS+wgEbEwIioRUWlraysc/NKlMH48DBuW/Vy6tHATZmbHpDITx1pgoqQJkkYC04EV+QqSRkvqjWEOsCiVt0s6Ic2fBrwH2CJphKTRqfw44EqypNJQS5fCzJnwwgsQkf2cOdPJw8wMSkwcEbEXuB1YCWwGlkfEJknzJF2Vql1ClhC2AmOA+an8d4EnJW0AvgcsiIinyQbKV6axj/VkZzBfbnTsc+fCnj0Hlu3Zk5WbmQ11iohWx1C6SqUS1Wr9V+8OG5adadSS4M03GxiYmdkgJmldRFRqy1s9OD4onX12sXIzs6HEiaMP8+fDiSceWHbiiVm5mdlQ58TRhxkzYOFCGDcu654aNy5bnjGj1ZGZmbVeaY8cOdrNmOFEYWbWF59xmJlZIU4cZmZWiBOHmZkV4sRhZmaFOHGYmVkhThxmZlaIE4eZmRXixGFmZoU4cZiZWSFOHGZmVogTh5mZFeLEYWZmhThxmJlZIU4cZmZWSKmJQ9IUSVskdUqa3cf6cZJWSdooaY2k9lz5TyStl7RJ0q25bd4l6enU5n2SVOY+mJnZgUpLHJKGA/cDVwAdwPWSOmqqLQCWRMR5wDzgnlS+E7goIs4H/hCYLem30rovAR8FJqZpSln7YGZmByvzjGMS0BkR2yLiDWAZMLWmTgfweJpf3bs+It6IiNdT+fG9cUo6EzglIp6IiACWAFeXuA9mZlajzMRxFrA9t9yVyvI2ANPS/DXAyZJGAUgaK2ljauPeiNiRtu86RJtmZlaiVg+OzwImS3oKmAx0A/sAImJ76sI6B7hJ0pgiDUuaKakqqdrT09PouM3MhqwyE0c3MDa33J7K9ouIHRExLSIuAOamsl21dYBngIvT9u0DtZnbbmFEVCKi0tbWdqT7YmZmSZmJYy0wUdIESSOB6cCKfAVJoyX1xjAHWJTK2yWdkOZPA94DbImIncCrki5MV1PdCHyzxH0wM7MapSWOiNgL3A6sBDYDyyNik6R5kq5K1S4BtkjaCowB5qfy3wWelLQB+B6wICKeTutuA74CdALPA4+WtQ9mZnYwZRcnDVBB+gTwDxHxi+aE1HiVSiWq1WqrwzAzO6pIWhcRldryes44xgBrJS1PN/T5hjszsyHskIkjIv6c7Ea7rwI3A89J+ktJ/67k2MzMbBCqa4wj3Wz3r2naC5wGPCTpr0qMzczMBqERh6og6ZNkVy+9TDYofUdE/DpdDfUc8F/LDdHMzAaTQyYO4HRgWkS8kC+MiDclXVlOWGZmNljV01X1KPBK74KkUyT9IUBEbC4rMDMzG5zqSRxfAnbnlnenMjMzG4LqSRyK3M0eEfEm9XVxmZnZMaiexLFN0p9KOi5NnwS2lR2YmZkNTvUkjluBd5M9TLCL7MVKM8sMyszMBq9DdjlFxEtkDyg0MzOr6z6OtwG3AOcCb+stj4j/XGJcZmY2SNXTVfU14DeBy8meVNsOvFZmUGZmNnjVkzjOiYhPA7+KiMXAB8jGOczMbAiqJ3H8Ov3cJen3gLcDZ5QXkpmZDWb13I+xML2F78/J3uB3EvDpUqMyM7NBa8DEkR5k+Gp6idP3gd9uSlRmZjZoDdhVle4S99Nvzcxsv3rGOP5Z0ixJYyWd3juVHpmZmQ1K9SSO64CPk3VVrUtTXS/wTq+a3SKpU9LsPtaPk7RK0kZJayS1p/LzJf1I0qa07rrcNn8v6aeS1qfp/HpiMTOzxqjnzvEJh9OwpOHA/cD7yB5VslbSioh4NldtAbAkIhZLuhS4B7gB2APcGBHPSfotYJ2klRGxK213R0Q8dDhxmZnZkannzvEb+yqPiCWH2HQS0BkR21I7y4CpQD5xdACfSvOrgUdS21tzn7ND0ktAG7ALMzNrqXq6qv4gN10M3A1cVcd2ZwHbc8tdqSxvAzAtzV8DnCxpVL6CpEnASOD5XPH81IX1eUnH9/XhkmZKqkqq9vT01BGumZnV45CJIyI+kZs+CryT7F6ORpgFTJb0FDCZ7Am8+3pXSjqT7JEnH0lXeAHMAX6HLJGdDtzZT9wLI6ISEZW2trYGhWtmZofzQqZfAfWMe3QDY3PL7alsv4jYQTrjkHQScG3vOIakU4B/AuZGxBO5bXam2dclPUCWfMzMrEnqGeP430DvGwCHkY1LLK+j7bXAREkTyBLGdODDNW2PBl5JZxNzgEWpfCTwMNnA+UM125wZETslCbgaeKaOWMzMrEHqOeNYkJvfC7wQEV2H2igi9kq6HVgJDAcWRcQmSfOAakSsAC4B7pEUZJf7fjxt/ifAe4FRkm5OZTdHxHpgqaQ2QMB6shdNmZlZkyj3OvG+K2RnDDsj4v+l5ROAMRHxs/LDa4xKpRLVal23npiZWSJpXURUasvruarqG8CbueV9qczMzIagehLHiIh4o3chzY8sLyQzMxvM6kkcPZL237chaSrwcnkhmZnZYFbP4PitZAPSX0zLXUCfd5Obmdmxr55nVT0PXJjusyAidpcelZmZDVqH7KqS9JeSTo2I3RGxW9Jpkj7XjODMzGzwqWeM44rcU2lJbwN8f3khmZnZYFZP4hief5Bguo+jzwcLmpnZsa+ewfGlwKr0XCgBNwOLywzKzMwGr3oGx++VtAH4Y7JnVq0ExpUdmJmZDU71dFUB/F+ypPGfgEuBzaVFZGZmg1q/ZxyS/j1wfZpeBh4ke7bVHzUpNjMzG4QG6qr6F+AHwJUR0Qkg6b80JSozMxu0BuqqmgbsBFZL+rKky8gGx83MbAjrN3FExCMRMZ3sNa2rgT8DzpD0JUn/sVkBmpnZ4FLPO8d/FRFfj4gPkr3+9Sn6ec+3mZkd++q9qgrI7hqPiIURcVlZAZmZ2eBWKHGYmZmVmjgkTZG0RVKnpNl9rB8naZWkjZLWSGpP5edL+pGkTWnddbltJkh6MrX5oCS/VMrMrIlKSxyShgP3A1cAHcD1kjpqqi0AlkTEecA84J5Uvge4MSLOBaYAX5B0alp3L/D5iDgH+AVwS1n7YGZmByvzjGMS0BkR29LrZpcBU2vqdACPp/nVvesjYmtEPJfmdwAvAW2SRHbn+kNpm8XA1SXug5mZ1SgzcZwFbM8td6WyvA1k94sAXAOcLGlUvoKkSWTvOH8eGAXsioi9A7TZu91MSVVJ1Z6eniPaETMze0urB8dnAZMlPQVMBrqBfb0rJZ0JfA34SES8WaThdPVXJSIqbW1tjYzZzGxIq+ex6oerGxibW25PZfulbqhpAOnVtNf2vjRK0inAPwFzI+KJtMnPgVMljUhnHQe1aWZm5SrzjGMtMDFdBTUSmA6syFeQNFpSbwxzgEWpfCTwMNnAee94BhERZGMhH0pFNwHfLHEfzMysRmmJI50R3E72/o7NwPKI2CRpnqSrUrVLgC2StgJjgPmp/E+A9wI3S1qfpvPTujuBT0nqJBvz+GpZ+2BmZgdT9kf8sa1SqUS1Wm11GGZmRxVJ6yKiUlve6sFxMzM7yjhxmJlZIU4cZmZWiBOHmZkV4sRhZmaFOHGYmVkhThxmZlaIE4eZmRXixGFmZoU4cZiZWSFOHGZmVogTh5mZFeLEYWZmhThxmJlZIU4cZmZWiBOHmZkV4sRhZmaFOHGYmVkhpSYOSVMkbZHUKWl2H+vHSVolaaOkNZLac+sek7RL0rdqtvl7ST/t413kZmbWBKUlDknDgfuBK4AO4HpJHTXVFgBLIuI8YB5wT27dXwM39NP8HRFxfprWNzh0MzMbQJlnHJOAzojYFhFvAMuAqTV1OoDH0/zq/PqIWAW8VmJ8ZmZ2GMpMHGcB23PLXaksbwMwLc1fA5wsaVQdbc9P3Vufl3T8kYfav7vvLrN1M7OjT6sHx2cBkyU9BUwGuoF9h9hmDvA7wB8ApwN39lVJ0kxJVUnVnp6eww7wM5857E3NzI5JZSaObmBsbrk9le0XETsiYlpEXADMTWW7Bmo0InZG5nXgAbIusb7qLYyISkRU2trajmQ/zMwsp8zEsRaYKGmCpJHAdGBFvoKk0ZJ6Y5gDLDpUo5LOTD8FXA0809CoybqnpGzKPiub3G1lZlZi4oiIvcDtwEpgM7A8IjZJmifpqlTtEmCLpK3AGGB+7/aSfgB8A7hMUpeky9OqpZKeBp4GRgOfa3Tsd98NEdmU7Us2OXGYmYGi97fjMaxSqUS1Wj2sbaW3EoiZ2VAiaV1EVGrLWz04PujddVerIzAzG1ycOA7B3VNmZgdy4jAzs0KcOMzMrBAnDjMzK8SJw8zMCnHiMDOzQpw4zMysECcOMzMrxInDzMwKceIwM7NCnDjMzKwQJw4zMyvEicPMzApx4jAzs0KcOMzMrBAnDjMzK8SJw8zMCik1cUiaImmLpE5Js/tYP07SKkkbJa2R1J5b95ikXZK+VbPNBElPpjYflDSyzH0wM7MDlZY4JA0H7geuADqA6yV11FRbACyJiPOAecA9uXV/DdzQR9P3Ap+PiHOAXwC3NDp2MzPrX5lnHJOAzojYFhFvAMuAqTV1OoDH0/zq/PqIWAW8lq8sScClwEOpaDFwdeNDNzOz/pSZOM4CtueWu1JZ3gZgWpq/BjhZ0qgB2hwF7IqIvQO0aWZmJWr14PgsYLKkp4DJQDewrxENS5opqSqp2tPT04gmzcyMchNHNzA2t9yeyvaLiB0RMS0iLgDmprJdA7T5c+BUSSP6azPX9sKIqEREpa2t7XD3wczMapSZONYCE9NVUCOB6cCKfAVJoyX1xjAHWDRQgxERZGMhH0pFNwHfbGjUZmY2oNISRxqHuB1YCWwGlkfEJknzJF2Vql0CbJG0FRgDzO/dXtIPgG8Al0nqknR5WnUn8ClJnWRjHl8tax/MzOxgyv6IP7ZVKpWoVqutDsPM7KgiaV1EVGrLWz04bmZmRxknDjMzK8SJw8zMCnHiMDOzQpw4zMysECcOMzMrxInDzMwKceIwa6G77251BHYsK+v48g2AZi0kwRD4L2gtcqTHl28ANDOzhnDiMGuyu+/O/hKUsuXeeXdbWSM04/hyV5VZC7mrysrkriozMxsUnDjMWuiuu1odgR3Lyjq+3FVlZmZ9cleVmZk1hBOHmZkV4sRhZmaFOHGYmVkhThxmZlbIkLiqSlIP8MJhbj4aeLmB4TSK4yrGcRXjuIo5VuMaFxFttYVDInEcCUnVvi5HazXHVYzjKsZxFTPU4nJXlZmZFeLEYWZmhThxHNrCVgfQD8dVjOMqxnEVM6Ti8hiHmZkV4jMOMzMrZMgmDkmLJL0k6Zl+1kvSfZI6JW2U9M7cupskPZemm5oc14wUz9OSfijp93PrfpbK10tq6FMd64jrEkm/TJ+9XtJf5NZNkbQlfZezmxzXHbmYnpG0T9LpaV2Z39dYSaslPStpk6RP9lGn6cdYnXE1/RirM66mH2N1xtX0Y0zS2yT9WNKGFNdn+qhzvKQH03fypKTxuXVzUvkWSZcXDiAihuQEvBd4J/BMP+vfDzwKCLgQeDKVnw5sSz9PS/OnNTGud/d+HnBFb1xp+WfA6BZ9X5cA3+qjfDjwPPDbwEhgA9DRrLhq6n4QeLxJ39eZwDvT/MnA1tr9bsUxVmdcTT/G6oyr6cdYPXG14hhLx8xJaf444Engwpo6twF/l+anAw+m+Y70HR0PTEjf3fAinz9kzzgi4vvAKwNUmQosicwTwKmSzgQuB74bEa9ExC+A7wJTmhVXRPwwfS7AE0B7oz77SOIawCSgMyK2RcQbwDKy77YVcV0P/GOjPnsgEbEzIn6S5l8DNgNn1VRr+jFWT1ytOMbq/L76U9oxdhhxNeUYS8fM7rR4XJpqB6ynAovT/EPAZZKUypdFxOsR8VOgk+w7rNuQTRx1OAvYnlvuSmX9lbfCLWR/sfYK4DuS1kma2YJ4Lkqnzo9KOjeVDYrvS9KJZL98/2euuCnfV+oiuIDsr8K8lh5jA8SV1/Rj7BBxtewYO9T31exjTNJwSeuBl8j+0Oj3+IqIvcAvgVE04PsacbhBW2tJ+iOy/9TvyRW/JyK6JZ0BfFfSv6S/yJvhJ2SPJ9gt6f3AI8DEJn12PT4I/J+IyJ+dlP59STqJ7BfJn0XEq41s+0jUE1crjrFDxNWyY6zOf8emHmMRsQ84X9KpwMOSfi8i+hzrazSfcfSvGxibW25PZf2VN42k84CvAFMj4ue95RHRnX6+BDxMwdPPIxERr/aeOkfEt4HjJI1mEHxfyXRquhDK/r4kHUf2y2ZpRPyvPqq05BirI66WHGOHiqtVx1g931fS9GMstb0LWM3B3Zn7vxdJI4C3Az+nEd9XowdtjqYJGE//g70f4MCByx+n8tOBn5INWp6W5k9vYlxnk/VJvrum/DeAk3PzPwSmNDGu3+St+4ImAS+m724E2eDuBN4auDy3WXGl9W8nGwf5jWZ9X2nflwBfGKBO04+xOuNq+jFWZ1xNP8bqiasVxxjQBpya5k8AfgBcWVPn4xw4OL48zZ/LgYPj2yg4OD5ku6ok/SPZVRqjJXUBd5ENMBERfwd8m+yql05gD/CRtO4VSZ8F1qam5sWBp6Zlx/UXZP2U/yMb52JvZA8xG0N2ugrZf6SvR8RjTYzrQ8DHJO0F/g2YHtlRulfS7cBKsqtfFkXEpibGBXAN8J2I+FVu01K/L+A/ADcAT6d+aID/RvZLuZXHWD1xteIYqyeuVhxj9cQFzT/GzgQWSxpO1nO0PCK+JWkeUI2IFcBXga9J6iRLatNTzJskLQeeBfYCH4+s26tuvnPczMwK8RiHmZkV4sRhZmaFOHGYmVkhThxmZlaIE4eZmRXixGF2mNJTUNfnpkY+lXW8+nnir1mrDdn7OMwa4N8i4vxWB2HWbD7jMGuw9A6Gv0rvYfixpHNS+XhJjyt718UqSWen8jGSHk4P79sg6d2pqeGSvpzet/AdSSek+n+q7P0QGyUta9Fu2hDmxGF2+E6o6aq6LrfulxHxDuCLwBdS2d8CiyPiPGApcF8qvw/4XkT8Ptm7RXrvep4I3B8R5wK7gGtT+WzggtTOrWXtnFl/fOe42WGStDsiTuqj/GfApRGxLT0g718jYpSkl4EzI+LXqXxnRIyW1AO0R8TruTbGkz0qe2JavhM4LiI+J+kxYDfZ02Efibfey2DWFD7jMCtH9DNfxOu5+X28NSb5AeB+srOTtenJp2ZN48RhVo7rcj9/lOZ/SHrQHDCD7ImmAKuAj8H+l/O8vb9GJQ0DxkbEauBOsqeyHnTWY1Ym/6VidvhOyD0xFeCxiOi9JPc0SRvJzhquT2WfAB6QdAfQQ3oaLvBJYKGkW8jOLD4G7OznM4cD/5CSi4D7Insfg1nTeIzDrMHSGEclIl5udSxmZXBXlZmZFeIzDjMzK8RnHGZmVogTh5mZFeLEYWZmhThxmJlZIU4cZmZWiBOHmZkV8v8BDGjG/NwGOIMAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.clf()   # clear figure\n",
    "acc_values = history_dict['acc']\n",
    "val_acc_values = history_dict['val_acc']\n",
    "\n",
    "plt.plot(epochs, acc_values, 'bo')\n",
    "plt.plot(epochs, val_acc_values, 'b+')\n",
    "plt.xlabel('Epochs')\n",
    "plt.ylabel('Accuracy')\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "_uuid": "64ec304e056ec0c9e33fe94ea2315cbf65a7fbff"
   },
   "source": [
    "## Fully Connected Model\n",
    "\n",
    "Neurons in a fully connected layer have full connections to all activations in the previous layer, as seen in regular Neural Networks. \n",
    "Adding another Dense Layer to model."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "_uuid": "9556f3de5bd370bcddc70a81910eb2104624e3a3"
   },
   "outputs": [],
   "source": [
    "def get_fc_model():\n",
    "    model = Sequential([\n",
    "        Lambda(standardize, input_shape=(28,28,1)),\n",
    "        Flatten(),\n",
    "        Dense(512, activation='relu'),\n",
    "        Dense(10, activation='softmax')\n",
    "        ])\n",
    "    model.compile(optimizer='Adam', loss='categorical_crossentropy',\n",
    "                  metrics=['accuracy'])\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "_uuid": "1901b6805f878ac6ed4efafbaf15bf003d505654"
   },
   "outputs": [],
   "source": [
    "fc = get_fc_model()\n",
    "fc.optimizer.lr=0.01"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "_uuid": "5fb346c542c8920fac61ddc5df44b2136969a6e9"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/1\n",
      "37800/37800 [==============================] - 178s 5ms/step - loss: 0.1598 - acc: 0.9715 - val_loss: 0.5633 - val_acc: 0.9488\n"
     ]
    }
   ],
   "source": [
    "history=fc.fit_generator(generator=batches, steps_per_epoch=batches.n, epochs=1, \n",
    "                    validation_data=val_batches, validation_steps=val_batches.n)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "_uuid": "46b81b17854a98f2b380da694691502c1e583bfb"
   },
   "source": [
    "## Convolutional Neural Network\n",
    "CNNs are extremely efficient for images.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {
    "_uuid": "fd0ab0de6bdbf7addf7515c8d7b59d8d17fef8e7"
   },
   "outputs": [],
   "source": [
    "from keras.layers import Convolution2D, MaxPooling2D\n",
    "\n",
    "def get_cnn_model():\n",
    "    model = Sequential([\n",
    "        Lambda(standardize, input_shape=(28,28,1)),\n",
    "        Convolution2D(32,(3,3), activation='relu'),\n",
    "        Convolution2D(32,(3,3), activation='relu'),\n",
    "        MaxPooling2D(),\n",
    "        Convolution2D(64,(3,3), activation='relu'),\n",
    "        Convolution2D(64,(3,3), activation='relu'),\n",
    "        MaxPooling2D(),\n",
    "        Flatten(),\n",
    "        Dense(512, activation='relu'),\n",
    "        Dense(10, activation='softmax')\n",
    "        ])\n",
    "    model.compile(Adam(), loss='categorical_crossentropy',\n",
    "                  metrics=['accuracy'])\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {
    "_uuid": "b5baeaed50c9f03c68900461555f4b7831a7e7c7"
   },
   "outputs": [],
   "source": [
    "model= get_cnn_model()\n",
    "model.optimizer.lr=0.01"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {
    "_uuid": "a3d15eeead8b26c41b53d084dc111ae9e667a169"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/1\n",
      "42000/42000 [==============================] - 3044s 72ms/step - loss: 0.6667 - acc: 0.9467 - val_loss: 5.9925 - val_acc: 0.6282\n"
     ]
    }
   ],
   "source": [
    "history=model.fit_generator(generator=batches, steps_per_epoch=batches.n, epochs=1, \n",
    "                    validation_data=val_batches, validation_steps=val_batches.n)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "_uuid": "e2891c7e434a2022ee182a0e9bd243a876532dcc"
   },
   "source": [
    "## Data Augmentation\n",
    "It is tehnique of showing slighly different or new images to neural network to avoid overfitting. And  to achieve better generalization.\n",
    "In case you have very small dataset, you can use different kinds of data augmentation techniques to increase your data size. Neural networks perform better if you provide them more data.\n",
    "\n",
    "Different data aumentation techniques are as follows:\n",
    "1. Cropping\n",
    "2. Rotating\n",
    "3. Scaling\n",
    "4. Translating\n",
    "5. Flipping \n",
    "6. Adding Gaussian noise to input images etc.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "_uuid": "daa409b92678202cf7c751371b7ba17fb14aa2ac"
   },
   "outputs": [],
   "source": [
    "gen =ImageDataGenerator(rotation_range=8, width_shift_range=0.08, shear_range=0.3,\n",
    "                               height_shift_range=0.08, zoom_range=0.08)\n",
    "batches = gen.flow(X_train, y_train, batch_size=64)\n",
    "val_batches = gen.flow(X_val, y_val, batch_size=64)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "_uuid": "f21ba7b8d77a37bee6e8238a8f517b654ae3f0a0"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/1\n",
      "37800/37800 [==============================] - 2865s 76ms/step - loss: 7.0282 - acc: 0.5639 - val_loss: 8.1427 - val_acc: 0.4948\n"
     ]
    }
   ],
   "source": [
    "model.optimizer.lr=0.001\n",
    "history=model.fit_generator(generator=batches, steps_per_epoch=batches.n, epochs=1, \n",
    "                    validation_data=val_batches, validation_steps=val_batches.n)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "_uuid": "538f504c44e14d389c70b2f35b7225de61b9015d"
   },
   "source": [
    "## Adding Batch Normalization\n",
    "\n",
    "BN helps to fine tune hyperparameters more better and train really deep neural networks."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "_uuid": "8b72580fbb06f5f4f769c514cb0d7d2f15aa2c2f"
   },
   "outputs": [],
   "source": [
    "from keras.layers.normalization import BatchNormalization\n",
    "\n",
    "def get_bn_model():\n",
    "    model = Sequential([\n",
    "        Lambda(standardize, input_shape=(28,28,1)),\n",
    "        Convolution2D(32,(3,3), activation='relu'),\n",
    "        BatchNormalization(axis=1),\n",
    "        Convolution2D(32,(3,3), activation='relu'),\n",
    "        MaxPooling2D(),\n",
    "        BatchNormalization(axis=1),\n",
    "        Convolution2D(64,(3,3), activation='relu'),\n",
    "        BatchNormalization(axis=1),\n",
    "        Convolution2D(64,(3,3), activation='relu'),\n",
    "        MaxPooling2D(),\n",
    "        Flatten(),\n",
    "        BatchNormalization(),\n",
    "        Dense(512, activation='relu'),\n",
    "        BatchNormalization(),\n",
    "        Dense(10, activation='softmax')\n",
    "        ])\n",
    "    model.compile(Adam(), loss='categorical_crossentropy', metrics=['accuracy'])\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "_uuid": "78e382d0b3de14312e762edc480b5d215be82269"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/1\n",
      "37800/37800 [==============================] - 4002s 106ms/step - loss: 0.0344 - acc: 0.9902 - val_loss: 0.0510 - val_acc: 0.9914\n"
     ]
    }
   ],
   "source": [
    "model= get_bn_model()\n",
    "model.optimizer.lr=0.01\n",
    "history=model.fit_generator(generator=batches, steps_per_epoch=batches.n, epochs=1, \n",
    "                    validation_data=val_batches, validation_steps=val_batches.n)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "_uuid": "8e4b16516a57e152a911f6e7ba7f4d70ff204512"
   },
   "source": [
    "## Saving model to ONNX format"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "_uuid": "0fc055b482971b36561aaf9421c8a9c53df2900b"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/3\n",
      "42000/42000 [==============================] - 4374s 104ms/step - loss: 0.0140 - acc: 0.9981\n",
      "Epoch 2/3\n",
      "42000/42000 [==============================] - 4278s 102ms/step - loss: 0.0214 - acc: 0.9983\n",
      "Epoch 3/3\n",
      "42000/42000 [==============================] - 57188s 1s/step - loss: 0.0288 - acc: 0.9980\n"
     ]
    }
   ],
   "source": [
    "model.optimizer.lr=0.01\n",
    "gen = image.ImageDataGenerator()\n",
    "batches = gen.flow(X, y, batch_size=64)\n",
    "history=model.fit_generator(generator=batches, steps_per_epoch=batches.n, epochs=3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {
    "_cell_guid": "c2841d54-f3dd-1ee8-a30d-4457dec0a67a",
    "_uuid": "4262c6bfb15ec96993e83bd2a2552eadf14fb33d"
   },
   "outputs": [],
   "source": [
    "# Convert into ONNX format with onnxmltools\n",
    "import keras2onnx\n",
    "onnx_model = keras2onnx.convert_keras(model, model.name)\n",
    "\n",
    "import onnx\n",
    "temp_model_file = 'model.onnx'\n",
    "\n",
    "onnx.save_model(onnx_model, temp_model_file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ir_version: 5\n",
      "producer_name: \"keras2onnx\"\n",
      "producer_version: \"1.5.0\"\n",
      "domain: \"onnx\"\n",
      "model_version: 0\n",
      "doc_string: \"\"\n",
      "graph {\n",
      "  node {\n",
      "    input: \"lambda_5_input_01\"\n",
      "    input: \"TFNodes1_lambda_5_sub_y_0\"\n",
      "    output: \"TFNodes1_lambda_5_sub_0\"\n",
      "    name: \"TFNodes1_lambda_5_sub\"\n",
      "    op_type: \"Sub\"\n",
      "    doc_string: \"\"\n",
      "    domain: \"\"\n",
      "  }\n",
      "  node {\n",
      "    input: \"TFNodes1_lambda_5_sub_0\"\n",
      "    input: \"TFNodes1_lambda_5_truediv_y_0\"\n",
      "    output: \"TFNodes1_lambda_5_truediv_0\"\n",
      "    name: \"TFNodes1_lambda_5_truediv\"\n",
      "\n",
      "...\n"
     ]
    }
   ],
   "source": [
    "print(str(onnx_model)[:500] + \"\\n...\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "content = onnx_model.SerializeToString()\n",
    "sess = onnxruntime.InferenceSession('model.onnx')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "onnx.checker.check_model(onnx_model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "_change_revision": 0,
  "_is_fork": false,
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}