{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Classification of Iris"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Package imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "# For building neural networks.\n",
    "import keras as kr\n",
    "\n",
    "# For interacting with data sets.\n",
    "import pandas as pd\n",
    "\n",
    "# For encoding categorical variables.\n",
    "import sklearn.preprocessing as pre\n",
    "\n",
    "# For splitting into training and test sets.\n",
    "import sklearn.model_selection as mod"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load the iris data set from a URL.\n",
    "df = pd.read_csv(\"https://raw.githubusercontent.com/ianmcloughlin/datasets/master/iris.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sepal_length</th>\n",
       "      <th>sepal_width</th>\n",
       "      <th>petal_length</th>\n",
       "      <th>petal_width</th>\n",
       "      <th>class</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>5.1</td>\n",
       "      <td>3.5</td>\n",
       "      <td>1.4</td>\n",
       "      <td>0.2</td>\n",
       "      <td>setosa</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>4.9</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1.4</td>\n",
       "      <td>0.2</td>\n",
       "      <td>setosa</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>4.7</td>\n",
       "      <td>3.2</td>\n",
       "      <td>1.3</td>\n",
       "      <td>0.2</td>\n",
       "      <td>setosa</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4.6</td>\n",
       "      <td>3.1</td>\n",
       "      <td>1.5</td>\n",
       "      <td>0.2</td>\n",
       "      <td>setosa</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5.0</td>\n",
       "      <td>3.6</td>\n",
       "      <td>1.4</td>\n",
       "      <td>0.2</td>\n",
       "      <td>setosa</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>145</th>\n",
       "      <td>6.7</td>\n",
       "      <td>3.0</td>\n",
       "      <td>5.2</td>\n",
       "      <td>2.3</td>\n",
       "      <td>virginica</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>146</th>\n",
       "      <td>6.3</td>\n",
       "      <td>2.5</td>\n",
       "      <td>5.0</td>\n",
       "      <td>1.9</td>\n",
       "      <td>virginica</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>147</th>\n",
       "      <td>6.5</td>\n",
       "      <td>3.0</td>\n",
       "      <td>5.2</td>\n",
       "      <td>2.0</td>\n",
       "      <td>virginica</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>148</th>\n",
       "      <td>6.2</td>\n",
       "      <td>3.4</td>\n",
       "      <td>5.4</td>\n",
       "      <td>2.3</td>\n",
       "      <td>virginica</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>149</th>\n",
       "      <td>5.9</td>\n",
       "      <td>3.0</td>\n",
       "      <td>5.1</td>\n",
       "      <td>1.8</td>\n",
       "      <td>virginica</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>150 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     sepal_length  sepal_width  petal_length  petal_width      class\n",
       "0             5.1          3.5           1.4          0.2     setosa\n",
       "1             4.9          3.0           1.4          0.2     setosa\n",
       "2             4.7          3.2           1.3          0.2     setosa\n",
       "3             4.6          3.1           1.5          0.2     setosa\n",
       "4             5.0          3.6           1.4          0.2     setosa\n",
       "..            ...          ...           ...          ...        ...\n",
       "145           6.7          3.0           5.2          2.3  virginica\n",
       "146           6.3          2.5           5.0          1.9  virginica\n",
       "147           6.5          3.0           5.2          2.0  virginica\n",
       "148           6.2          3.4           5.4          2.3  virginica\n",
       "149           5.9          3.0           5.1          1.8  virginica\n",
       "\n",
       "[150 rows x 5 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Inputs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Separate the inputs from the rest of the variables.\n",
    "inputs = df[['petal_length', 'petal_width', 'sepal_length', 'sepal_width']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>petal_length</th>\n",
       "      <th>petal_width</th>\n",
       "      <th>sepal_length</th>\n",
       "      <th>sepal_width</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.4</td>\n",
       "      <td>0.2</td>\n",
       "      <td>5.1</td>\n",
       "      <td>3.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1.4</td>\n",
       "      <td>0.2</td>\n",
       "      <td>4.9</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1.3</td>\n",
       "      <td>0.2</td>\n",
       "      <td>4.7</td>\n",
       "      <td>3.2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1.5</td>\n",
       "      <td>0.2</td>\n",
       "      <td>4.6</td>\n",
       "      <td>3.1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1.4</td>\n",
       "      <td>0.2</td>\n",
       "      <td>5.0</td>\n",
       "      <td>3.6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>145</th>\n",
       "      <td>5.2</td>\n",
       "      <td>2.3</td>\n",
       "      <td>6.7</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>146</th>\n",
       "      <td>5.0</td>\n",
       "      <td>1.9</td>\n",
       "      <td>6.3</td>\n",
       "      <td>2.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>147</th>\n",
       "      <td>5.2</td>\n",
       "      <td>2.0</td>\n",
       "      <td>6.5</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>148</th>\n",
       "      <td>5.4</td>\n",
       "      <td>2.3</td>\n",
       "      <td>6.2</td>\n",
       "      <td>3.4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>149</th>\n",
       "      <td>5.1</td>\n",
       "      <td>1.8</td>\n",
       "      <td>5.9</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>150 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     petal_length  petal_width  sepal_length  sepal_width\n",
       "0             1.4          0.2           5.1          3.5\n",
       "1             1.4          0.2           4.9          3.0\n",
       "2             1.3          0.2           4.7          3.2\n",
       "3             1.5          0.2           4.6          3.1\n",
       "4             1.4          0.2           5.0          3.6\n",
       "..            ...          ...           ...          ...\n",
       "145           5.2          2.3           6.7          3.0\n",
       "146           5.0          1.9           6.3          2.5\n",
       "147           5.2          2.0           6.5          3.0\n",
       "148           5.4          2.3           6.2          3.4\n",
       "149           5.1          1.8           5.9          3.0\n",
       "\n",
       "[150 rows x 4 columns]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "inputs"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Encoded outputs"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$\n",
    "\\begin{align*}\n",
    "  setosa     & \\rightarrow  [1,0,0] \\\\\n",
    "  versicolor & \\rightarrow  [0,1,0] \\\\\n",
    "  virginica  & \\rightarrow  [0,0,1]\n",
    "\\end{align*}\n",
    "$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[1, 0, 0],\n",
       "       [1, 0, 0],\n",
       "       [1, 0, 0],\n",
       "       [1, 0, 0],\n",
       "       [1, 0, 0],\n",
       "       [1, 0, 0],\n",
       "       [1, 0, 0],\n",
       "       [1, 0, 0],\n",
       "       [1, 0, 0],\n",
       "       [1, 0, 0],\n",
       "       [1, 0, 0],\n",
       "       [1, 0, 0],\n",
       "       [1, 0, 0],\n",
       "       [1, 0, 0],\n",
       "       [1, 0, 0],\n",
       "       [1, 0, 0],\n",
       "       [1, 0, 0],\n",
       "       [1, 0, 0],\n",
       "       [1, 0, 0],\n",
       "       [1, 0, 0],\n",
       "       [1, 0, 0],\n",
       "       [1, 0, 0],\n",
       "       [1, 0, 0],\n",
       "       [1, 0, 0],\n",
       "       [1, 0, 0],\n",
       "       [1, 0, 0],\n",
       "       [1, 0, 0],\n",
       "       [1, 0, 0],\n",
       "       [1, 0, 0],\n",
       "       [1, 0, 0],\n",
       "       [1, 0, 0],\n",
       "       [1, 0, 0],\n",
       "       [1, 0, 0],\n",
       "       [1, 0, 0],\n",
       "       [1, 0, 0],\n",
       "       [1, 0, 0],\n",
       "       [1, 0, 0],\n",
       "       [1, 0, 0],\n",
       "       [1, 0, 0],\n",
       "       [1, 0, 0],\n",
       "       [1, 0, 0],\n",
       "       [1, 0, 0],\n",
       "       [1, 0, 0],\n",
       "       [1, 0, 0],\n",
       "       [1, 0, 0],\n",
       "       [1, 0, 0],\n",
       "       [1, 0, 0],\n",
       "       [1, 0, 0],\n",
       "       [1, 0, 0],\n",
       "       [1, 0, 0],\n",
       "       [0, 1, 0],\n",
       "       [0, 1, 0],\n",
       "       [0, 1, 0],\n",
       "       [0, 1, 0],\n",
       "       [0, 1, 0],\n",
       "       [0, 1, 0],\n",
       "       [0, 1, 0],\n",
       "       [0, 1, 0],\n",
       "       [0, 1, 0],\n",
       "       [0, 1, 0],\n",
       "       [0, 1, 0],\n",
       "       [0, 1, 0],\n",
       "       [0, 1, 0],\n",
       "       [0, 1, 0],\n",
       "       [0, 1, 0],\n",
       "       [0, 1, 0],\n",
       "       [0, 1, 0],\n",
       "       [0, 1, 0],\n",
       "       [0, 1, 0],\n",
       "       [0, 1, 0],\n",
       "       [0, 1, 0],\n",
       "       [0, 1, 0],\n",
       "       [0, 1, 0],\n",
       "       [0, 1, 0],\n",
       "       [0, 1, 0],\n",
       "       [0, 1, 0],\n",
       "       [0, 1, 0],\n",
       "       [0, 1, 0],\n",
       "       [0, 1, 0],\n",
       "       [0, 1, 0],\n",
       "       [0, 1, 0],\n",
       "       [0, 1, 0],\n",
       "       [0, 1, 0],\n",
       "       [0, 1, 0],\n",
       "       [0, 1, 0],\n",
       "       [0, 1, 0],\n",
       "       [0, 1, 0],\n",
       "       [0, 1, 0],\n",
       "       [0, 1, 0],\n",
       "       [0, 1, 0],\n",
       "       [0, 1, 0],\n",
       "       [0, 1, 0],\n",
       "       [0, 1, 0],\n",
       "       [0, 1, 0],\n",
       "       [0, 1, 0],\n",
       "       [0, 1, 0],\n",
       "       [0, 1, 0],\n",
       "       [0, 1, 0],\n",
       "       [0, 1, 0],\n",
       "       [0, 1, 0],\n",
       "       [0, 0, 1],\n",
       "       [0, 0, 1],\n",
       "       [0, 0, 1],\n",
       "       [0, 0, 1],\n",
       "       [0, 0, 1],\n",
       "       [0, 0, 1],\n",
       "       [0, 0, 1],\n",
       "       [0, 0, 1],\n",
       "       [0, 0, 1],\n",
       "       [0, 0, 1],\n",
       "       [0, 0, 1],\n",
       "       [0, 0, 1],\n",
       "       [0, 0, 1],\n",
       "       [0, 0, 1],\n",
       "       [0, 0, 1],\n",
       "       [0, 0, 1],\n",
       "       [0, 0, 1],\n",
       "       [0, 0, 1],\n",
       "       [0, 0, 1],\n",
       "       [0, 0, 1],\n",
       "       [0, 0, 1],\n",
       "       [0, 0, 1],\n",
       "       [0, 0, 1],\n",
       "       [0, 0, 1],\n",
       "       [0, 0, 1],\n",
       "       [0, 0, 1],\n",
       "       [0, 0, 1],\n",
       "       [0, 0, 1],\n",
       "       [0, 0, 1],\n",
       "       [0, 0, 1],\n",
       "       [0, 0, 1],\n",
       "       [0, 0, 1],\n",
       "       [0, 0, 1],\n",
       "       [0, 0, 1],\n",
       "       [0, 0, 1],\n",
       "       [0, 0, 1],\n",
       "       [0, 0, 1],\n",
       "       [0, 0, 1],\n",
       "       [0, 0, 1],\n",
       "       [0, 0, 1],\n",
       "       [0, 0, 1],\n",
       "       [0, 0, 1],\n",
       "       [0, 0, 1],\n",
       "       [0, 0, 1],\n",
       "       [0, 0, 1],\n",
       "       [0, 0, 1],\n",
       "       [0, 0, 1],\n",
       "       [0, 0, 1],\n",
       "       [0, 0, 1],\n",
       "       [0, 0, 1]])"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Encode the classes as above.\n",
    "encoder = pre.LabelBinarizer()\n",
    "encoder.fit(df['class'])\n",
    "outputs = encoder.transform(df['class'])\n",
    "\n",
    "outputs"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Idea\n",
    "\n",
    "The neural network will turn four floating point inputs into three \"floating point\" outputs.\n",
    "\n",
    "$$ [5.1, 3.5, 1.4, 0.2] \\rightarrow [0.8, 0.19, 0.01] $$\n",
    "\n",
    "$$ [5.1, 3.5, 1.4, 0.2] \\rightarrow [1, 0, 0] $$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Build model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From C:\\Users\\mclou\\Anaconda3\\lib\\site-packages\\keras\\backend\\tensorflow_backend.py:74: The name tf.get_default_graph is deprecated. Please use tf.compat.v1.get_default_graph instead.\n",
      "\n",
      "WARNING:tensorflow:From C:\\Users\\mclou\\Anaconda3\\lib\\site-packages\\keras\\backend\\tensorflow_backend.py:517: The name tf.placeholder is deprecated. Please use tf.compat.v1.placeholder instead.\n",
      "\n",
      "WARNING:tensorflow:From C:\\Users\\mclou\\Anaconda3\\lib\\site-packages\\keras\\backend\\tensorflow_backend.py:4138: The name tf.random_uniform is deprecated. Please use tf.random.uniform instead.\n",
      "\n",
      "WARNING:tensorflow:From C:\\Users\\mclou\\Anaconda3\\lib\\site-packages\\keras\\optimizers.py:790: The name tf.train.Optimizer is deprecated. Please use tf.compat.v1.train.Optimizer instead.\n",
      "\n",
      "WARNING:tensorflow:From C:\\Users\\mclou\\Anaconda3\\lib\\site-packages\\keras\\backend\\tensorflow_backend.py:3295: The name tf.log is deprecated. Please use tf.math.log instead.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Start a neural network, building it by layers.\n",
    "model = kr.models.Sequential()\n",
    "\n",
    "# Add a hidden layer with x neurons and an input layer with 4.\n",
    "model.add(kr.layers.Dense(units=30, activation='relu', input_dim=4))\n",
    "# Add a three neuron output layer.\n",
    "model.add(kr.layers.Dense(units=3, activation='softmax'))\n",
    "\n",
    "# Build the graph.\n",
    "model.compile(loss='categorical_crossentropy', optimizer='sgd', metrics=['accuracy'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Split the inputs and outputs into training and test sets.\n",
    "inputs_train, inputs_test, outputs_train, outputs_test = mod.train_test_split(inputs, outputs, test_size=0.2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "petal_length    4.8\n",
       "petal_width     1.4\n",
       "sepal_length    6.8\n",
       "sepal_width     2.8\n",
       "Name: 76, dtype: float64"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "inputs_test.iloc[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From C:\\Users\\mclou\\Anaconda3\\lib\\site-packages\\keras\\backend\\tensorflow_backend.py:2741: The name tf.Session is deprecated. Please use tf.compat.v1.Session instead.\n",
      "\n",
      "WARNING:tensorflow:From C:\\Users\\mclou\\Anaconda3\\lib\\site-packages\\keras\\backend\\tensorflow_backend.py:174: The name tf.get_default_session is deprecated. Please use tf.compat.v1.get_default_session instead.\n",
      "\n",
      "WARNING:tensorflow:From C:\\Users\\mclou\\Anaconda3\\lib\\site-packages\\keras\\backend\\tensorflow_backend.py:181: The name tf.ConfigProto is deprecated. Please use tf.compat.v1.ConfigProto instead.\n",
      "\n",
      "WARNING:tensorflow:From C:\\Users\\mclou\\Anaconda3\\lib\\site-packages\\keras\\backend\\tensorflow_backend.py:190: The name tf.global_variables is deprecated. Please use tf.compat.v1.global_variables instead.\n",
      "\n",
      "WARNING:tensorflow:From C:\\Users\\mclou\\Anaconda3\\lib\\site-packages\\keras\\backend\\tensorflow_backend.py:199: The name tf.is_variable_initialized is deprecated. Please use tf.compat.v1.is_variable_initialized instead.\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\mclou\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:1: FutureWarning: Method .as_matrix will be removed in a future version. Use .values instead.\n",
      "  \"\"\"Entry point for launching an IPython kernel.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From C:\\Users\\mclou\\Anaconda3\\lib\\site-packages\\keras\\backend\\tensorflow_backend.py:206: The name tf.variables_initializer is deprecated. Please use tf.compat.v1.variables_initializer instead.\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([[0.02864362, 0.96718645, 0.00416994]], dtype=float32)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.predict(inputs_test.as_matrix()[0:1])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From C:\\Users\\mclou\\Anaconda3\\lib\\site-packages\\tensorflow_core\\python\\ops\\math_grad.py:1424: where (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.where in 2.0, which has the same broadcast rule as np.where\n",
      "WARNING:tensorflow:From C:\\Users\\mclou\\Anaconda3\\lib\\site-packages\\keras\\backend\\tensorflow_backend.py:986: The name tf.assign_add is deprecated. Please use tf.compat.v1.assign_add instead.\n",
      "\n",
      "WARNING:tensorflow:From C:\\Users\\mclou\\Anaconda3\\lib\\site-packages\\keras\\backend\\tensorflow_backend.py:973: The name tf.assign is deprecated. Please use tf.compat.v1.assign instead.\n",
      "\n",
      "Epoch 1/15\n",
      "120/120 [==============================] - 1s 10ms/step - loss: 1.6282 - acc: 0.3417\n",
      "Epoch 2/15\n",
      "120/120 [==============================] - 0s 671us/step - loss: 1.0749 - acc: 0.4250\n",
      "Epoch 3/15\n",
      "120/120 [==============================] - 0s 767us/step - loss: 0.8976 - acc: 0.5167\n",
      "Epoch 4/15\n",
      "120/120 [==============================] - 0s 650us/step - loss: 0.8089 - acc: 0.7500\n",
      "Epoch 5/15\n",
      "120/120 [==============================] - 0s 850us/step - loss: 0.7383 - acc: 0.7750\n",
      "Epoch 6/15\n",
      "120/120 [==============================] - 0s 813us/step - loss: 0.6916 - acc: 0.8500\n",
      "Epoch 7/15\n",
      "120/120 [==============================] - 0s 700us/step - loss: 0.6547 - acc: 0.8583\n",
      "Epoch 8/15\n",
      "120/120 [==============================] - 0s 333us/step - loss: 0.6256 - acc: 0.8167\n",
      "Epoch 9/15\n",
      "120/120 [==============================] - 0s 342us/step - loss: 0.5870 - acc: 0.7667\n",
      "Epoch 10/15\n",
      "120/120 [==============================] - 0s 550us/step - loss: 0.5789 - acc: 0.8667\n",
      "Epoch 11/15\n",
      "120/120 [==============================] - 0s 817us/step - loss: 0.5514 - acc: 0.8500\n",
      "Epoch 12/15\n",
      "120/120 [==============================] - 0s 900us/step - loss: 0.5314 - acc: 0.8917\n",
      "Epoch 13/15\n",
      "120/120 [==============================] - 0s 746us/step - loss: 0.5162 - acc: 0.8417\n",
      "Epoch 14/15\n",
      "120/120 [==============================] - 0s 392us/step - loss: 0.5029 - acc: 0.8583\n",
      "Epoch 15/15\n",
      "120/120 [==============================] - 0s 292us/step - loss: 0.4830 - acc: 0.8417\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x1c4c4343248>"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Train the neural network.\n",
    "model.fit(inputs_train, outputs_train, epochs=15, batch_size=10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Predict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\mclou\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:1: FutureWarning: Method .as_matrix will be removed in a future version. Use .values instead.\n",
      "  \"\"\"Entry point for launching an IPython kernel.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([[0.07707678, 0.5562058 , 0.36671743]], dtype=float32)"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.predict(inputs_test.as_matrix()[0:1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['versicolor', 'virginica', 'setosa', 'setosa', 'setosa',\n",
       "       'virginica', 'setosa', 'virginica', 'versicolor', 'versicolor',\n",
       "       'virginica', 'versicolor', 'virginica', 'setosa', 'setosa',\n",
       "       'versicolor', 'setosa', 'setosa', 'versicolor', 'virginica',\n",
       "       'setosa', 'versicolor', 'versicolor', 'setosa', 'setosa',\n",
       "       'virginica', 'virginica', 'versicolor', 'virginica', 'virginica'],\n",
       "      dtype='<U10')"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Have the network predict the classes of the test inputs.\n",
    "predictions = model.predict(inputs_test)\n",
    "predictions_labels = encoder.inverse_transform(predictions)\n",
    "predictions_labels"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Evaluate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ True,  True,  True,  True,  True, False,  True,  True,  True,\n",
       "        True,  True,  True,  True,  True,  True,  True,  True,  True,\n",
       "        True,  True,  True,  True,  True,  True,  True,  True,  True,\n",
       "        True,  True,  True])"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Compare the predictions to the actual classes.\n",
    "predictions_labels == encoder.inverse_transform(outputs_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "29"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(predictions_labels == encoder.inverse_transform(outputs_test)).sum()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Whitening"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sklearn.decomposition as dec"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>petal_length</th>\n",
       "      <th>petal_width</th>\n",
       "      <th>sepal_length</th>\n",
       "      <th>sepal_width</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.387221</td>\n",
       "      <td>0.484604</td>\n",
       "      <td>1.353335</td>\n",
       "      <td>0.761078</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.394136</td>\n",
       "      <td>-0.687376</td>\n",
       "      <td>-2.114471</td>\n",
       "      <td>-0.958366</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.925573</td>\n",
       "      <td>0.351229</td>\n",
       "      <td>2.526484</td>\n",
       "      <td>-0.225238</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.675388</td>\n",
       "      <td>-0.325813</td>\n",
       "      <td>-0.203693</td>\n",
       "      <td>-0.997615</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.221818</td>\n",
       "      <td>-1.285400</td>\n",
       "      <td>-0.049826</td>\n",
       "      <td>1.718668</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>115</th>\n",
       "      <td>0.015610</td>\n",
       "      <td>-1.118584</td>\n",
       "      <td>-0.788006</td>\n",
       "      <td>0.266962</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>116</th>\n",
       "      <td>-1.387309</td>\n",
       "      <td>0.580464</td>\n",
       "      <td>0.304557</td>\n",
       "      <td>1.042104</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>117</th>\n",
       "      <td>1.109262</td>\n",
       "      <td>0.787130</td>\n",
       "      <td>0.996607</td>\n",
       "      <td>0.446290</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>118</th>\n",
       "      <td>0.611141</td>\n",
       "      <td>-0.274729</td>\n",
       "      <td>0.178736</td>\n",
       "      <td>-0.899852</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>119</th>\n",
       "      <td>1.388267</td>\n",
       "      <td>0.856468</td>\n",
       "      <td>-1.735097</td>\n",
       "      <td>0.208429</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>120 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     petal_length  petal_width  sepal_length  sepal_width\n",
       "0        0.387221     0.484604      1.353335     0.761078\n",
       "1        0.394136    -0.687376     -2.114471    -0.958366\n",
       "2        0.925573     0.351229      2.526484    -0.225238\n",
       "3        0.675388    -0.325813     -0.203693    -0.997615\n",
       "4        0.221818    -1.285400     -0.049826     1.718668\n",
       "..            ...          ...           ...          ...\n",
       "115      0.015610    -1.118584     -0.788006     0.266962\n",
       "116     -1.387309     0.580464      0.304557     1.042104\n",
       "117      1.109262     0.787130      0.996607     0.446290\n",
       "118      0.611141    -0.274729      0.178736    -0.899852\n",
       "119      1.388267     0.856468     -1.735097     0.208429\n",
       "\n",
       "[120 rows x 4 columns]"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pca = dec.PCA(n_components=4, whiten=True)\n",
    "pca.fit(inputs_train)\n",
    "inputs_train_white = pd.DataFrame(pca.transform(inputs_train), columns=inputs_train.columns)\n",
    "inputs_train_white"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Start a neural network, building it by layers.\n",
    "model = kr.models.Sequential()\n",
    "\n",
    "# Add a hidden layer with x neurons and an input layer with 4.\n",
    "model.add(kr.layers.Dense(units=30, activation='relu', input_dim=4))\n",
    "# Add a three neuron output layer.\n",
    "model.add(kr.layers.Dense(units=3, activation='softmax'))\n",
    "\n",
    "# Build the graph.\n",
    "model.compile(loss='categorical_crossentropy', optimizer='sgd', metrics=['accuracy'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/15\n",
      "120/120 [==============================] - 1s 8ms/step - loss: 1.0737 - acc: 0.4500\n",
      "Epoch 2/15\n",
      "120/120 [==============================] - 0s 283us/step - loss: 1.0342 - acc: 0.5667\n",
      "Epoch 3/15\n",
      "120/120 [==============================] - 0s 392us/step - loss: 0.9963 - acc: 0.6083\n",
      "Epoch 4/15\n",
      "120/120 [==============================] - 0s 291us/step - loss: 0.9618 - acc: 0.6333\n",
      "Epoch 5/15\n",
      "120/120 [==============================] - 0s 417us/step - loss: 0.9288 - acc: 0.6500\n",
      "Epoch 6/15\n",
      "120/120 [==============================] - 0s 325us/step - loss: 0.8977 - acc: 0.6667\n",
      "Epoch 7/15\n",
      "120/120 [==============================] - 0s 496us/step - loss: 0.8687 - acc: 0.6833\n",
      "Epoch 8/15\n",
      "120/120 [==============================] - 0s 881us/step - loss: 0.8403 - acc: 0.6917\n",
      "Epoch 9/15\n",
      "120/120 [==============================] - 0s 792us/step - loss: 0.8140 - acc: 0.7333\n",
      "Epoch 10/15\n",
      "120/120 [==============================] - 0s 933us/step - loss: 0.7887 - acc: 0.7333\n",
      "Epoch 11/15\n",
      "120/120 [==============================] - 0s 325us/step - loss: 0.7646 - acc: 0.7583\n",
      "Epoch 12/15\n",
      "120/120 [==============================] - 0s 342us/step - loss: 0.7417 - acc: 0.7500\n",
      "Epoch 13/15\n",
      "120/120 [==============================] - 0s 267us/step - loss: 0.7200 - acc: 0.7917\n",
      "Epoch 14/15\n",
      "120/120 [==============================] - 0s 375us/step - loss: 0.6999 - acc: 0.8000\n",
      "Epoch 15/15\n",
      "120/120 [==============================] - 0s 292us/step - loss: 0.6800 - acc: 0.8000\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x1c4c573b888>"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Train the neural network.\n",
    "model.fit(inputs_train_white, outputs_train, epochs=15, batch_size=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['versicolor', 'virginica', 'setosa', 'setosa', 'setosa',\n",
       "       'virginica', 'setosa', 'virginica', 'versicolor', 'versicolor',\n",
       "       'virginica', 'virginica', 'virginica', 'setosa', 'setosa',\n",
       "       'versicolor', 'setosa', 'setosa', 'virginica', 'virginica',\n",
       "       'setosa', 'virginica', 'versicolor', 'setosa', 'setosa',\n",
       "       'virginica', 'virginica', 'versicolor', 'virginica', 'virginica'],\n",
       "      dtype='<U10')"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Have the network predict the classes of the test inputs.\n",
    "predictions = model.predict(pca.transform(inputs_test))\n",
    "predictions_labels = encoder.inverse_transform(predictions)\n",
    "predictions_labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "26"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(predictions_labels == encoder.inverse_transform(outputs_test)).sum()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## End"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}