{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# CS 20 : TensorFlow for Deep Learning Research\n",
"## Lecture 07 : ConvNet in TensorFlow\n",
"same contents, but different style with [Lec07_ConvNet mnist by high-level.ipynb](https://nbviewer.jupyter.org/github/aisolab/CS20/blob/master/Lec07_ConvNet%20in%20Tensorflow/Lec07_ConvNet%20mnist%20by%20high-level.ipynb)\n",
"\n",
"### ConvNet mnist by high-level\n",
"- Creating the **data pipeline** with `tf.data`\n",
"- Using `tf.keras`, alias `keras`\n",
"- Creating the model as **Class** by subclassing `tf.keras.Model`\n",
"- Training the model with **Drop out** technique by `tf.keras.layers.Dropout`\n",
"- Using tensorboard"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### Setup"
]
},
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"1.12.0\n"
]
}
],
"source": [
"from __future__ import absolute_import, division, print_function\n",
"import os, sys\n",
"import numpy as np\n",
"import pandas as pd\n",
"import matplotlib.pyplot as plt\n",
"import tensorflow as tf\n",
"from tensorflow import keras\n",
"%matplotlib inline\n",
"\n",
"print(tf.__version__)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### Load and Pre-process data"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [],
"source": [
"(x_train, y_train), (x_tst, y_tst) = tf.keras.datasets.mnist.load_data()\n",
"x_train = x_train / 255\n",
"x_train = x_train.reshape(-1, 28, 28, 1).astype(np.float32)\n",
"x_tst = x_tst / 255\n",
"x_tst = x_tst.reshape(-1, 28, 28, 1).astype(np.float32)\n",
"y_tst = y_tst.astype(np.int32)"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"(55000, 28, 28, 1) (55000,)\n",
"(5000, 28, 28, 1) (5000,)\n"
]
}
],
"source": [
"tr_indices = np.random.choice(range(x_train.shape[0]), size = 55000, replace = False)\n",
"\n",
"x_tr = x_train[tr_indices]\n",
"y_tr = y_train[tr_indices].astype(np.int32)\n",
"\n",
"x_val = np.delete(arr = x_train, obj = tr_indices, axis = 0)\n",
"y_val = np.delete(arr = y_train, obj = tr_indices, axis = 0).astype(np.int32)\n",
"\n",
"print(x_tr.shape, y_tr.shape)\n",
"print(x_val.shape, y_val.shape)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### Define SimpleCNN class by high-level api"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {},
"outputs": [],
"source": [
"class SimpleCNN(keras.Model):\n",
" def __init__(self, num_classes):\n",
" super(SimpleCNN, self).__init__()\n",
" self.__conv1 = keras.layers.Conv2D(filters=32, kernel_size=[5,5], padding='same',\n",
" kernel_initializer=keras.initializers.truncated_normal(),\n",
" bias_initializer=keras.initializers.truncated_normal(),\n",
" activation=tf.nn.relu)\n",
" self.__conv2 = keras.layers.Conv2D(filters=64, kernel_size=[5,5], padding='same',\n",
" kernel_initializer=keras.initializers.truncated_normal(),\n",
" bias_initializer=keras.initializers.truncated_normal(),\n",
" activation=tf.nn.relu)\n",
" self.__pool = keras.layers.MaxPooling2D()\n",
" self.__flatten = keras.layers.Flatten()\n",
" self.__dropout = keras.layers.Dropout(rate =.5)\n",
" self.__dense1 = keras.layers.Dense(units=1024, activation=tf.nn.relu, \n",
" kernel_initializer=keras.initializers.truncated_normal(),\n",
" bias_initializer=keras.initializers.truncated_normal())\n",
" self.__dense2 = keras.layers.Dense(units=num_classes,\n",
" kernel_initializer=keras.initializers.truncated_normal(),\n",
" bias_initializer=keras.initializers.truncated_normal(),\n",
" activation='softmax')\n",
" \n",
" def call(self, inputs, training=False):\n",
" conv1 = self.__conv1(inputs)\n",
" pool1 = self.__pool(conv1)\n",
" conv2 = self.__conv2(pool1)\n",
" pool2 = self.__pool(conv2)\n",
" flattened = self.__flatten(pool2)\n",
" fc = self.__dense1(flattened)\n",
" if training:\n",
" fc = self.__dropout(fc, training=training)\n",
" score = self.__dense2(fc)\n",
" return score"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### Create a model of SimpleCNN"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"550\n"
]
}
],
"source": [
"# hyper-parameter\n",
"lr = .001\n",
"epochs = 10\n",
"batch_size = 100\n",
"total_step = int(x_tr.shape[0] / batch_size)\n",
"print(total_step)"
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"<RepeatDataset shapes: ((?, 28, 28, 1), (?,)), types: (tf.float32, tf.int32)>\n",
"<RepeatDataset shapes: ((?, 28, 28, 1), (?,)), types: (tf.float32, tf.int32)>\n",
"<BatchDataset shapes: ((?, 28, 28, 1), (?,)), types: (tf.float32, tf.int32)>\n"
]
}
],
"source": [
"## create input pipeline with tf.data\n",
"# for train\n",
"tr_dataset = tf.data.Dataset.from_tensor_slices((x_tr, y_tr))\n",
"tr_dataset = tr_dataset.batch(batch_size = batch_size).repeat()\n",
"print(tr_dataset)\n",
"\n",
"# for validation\n",
"val_dataset = tf.data.Dataset.from_tensor_slices((x_val,y_val))\n",
"val_dataset = val_dataset.batch(batch_size = batch_size).repeat()\n",
"print(val_dataset)\n",
"\n",
"# for test\n",
"tst_dataset = tf.data.Dataset.from_tensor_slices((x_tst, y_tst))\n",
"tst_dataset = tst_dataset.batch(batch_size=100)\n",
"print(tst_dataset)"
]
},
{
"cell_type": "code",
"execution_count": 7,
"metadata": {},
"outputs": [],
"source": [
"## create model\n",
"cnn = SimpleCNN(num_classes=10)\n",
"\n",
"# creating callbacks for tensorboard\n",
"callbacks = [keras.callbacks.TensorBoard(log_dir='../graphs/lecture07/convnet_mnist_high_kd/',\n",
" write_graph=True, write_images=True)]"
]
},
{
"cell_type": "code",
"execution_count": 8,
"metadata": {},
"outputs": [],
"source": [
"# complile\n",
"cnn.compile(optimizer=tf.train.AdamOptimizer(learning_rate=lr),\n",
" loss=keras.losses.sparse_categorical_crossentropy,\n",
" callbacks=callbacks)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### Train a model"
]
},
{
"cell_type": "code",
"execution_count": 9,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Epoch 1/10\n",
"550/550 [==============================] - 4s 7ms/step - loss: 0.1362 - val_loss: 0.0472\n",
"Epoch 2/10\n",
"550/550 [==============================] - 3s 5ms/step - loss: 0.0381 - val_loss: 0.0508\n",
"Epoch 3/10\n",
"550/550 [==============================] - 3s 5ms/step - loss: 0.0246 - val_loss: 0.0233\n",
"Epoch 4/10\n",
"550/550 [==============================] - 3s 5ms/step - loss: 0.0177 - val_loss: 0.0200\n",
"Epoch 5/10\n",
"550/550 [==============================] - 3s 5ms/step - loss: 0.0149 - val_loss: 0.0160\n",
"Epoch 6/10\n",
"550/550 [==============================] - 3s 5ms/step - loss: 0.0108 - val_loss: 0.0206\n",
"Epoch 7/10\n",
"550/550 [==============================] - 3s 5ms/step - loss: 0.0087 - val_loss: 0.0112\n",
"Epoch 8/10\n",
"550/550 [==============================] - 3s 5ms/step - loss: 0.0082 - val_loss: 0.0133\n",
"Epoch 9/10\n",
"550/550 [==============================] - 3s 5ms/step - loss: 0.0087 - val_loss: 0.0075\n",
"Epoch 10/10\n",
"550/550 [==============================] - 3s 5ms/step - loss: 0.0060 - val_loss: 0.0054\n"
]
},
{
"data": {
"text/plain": [
"<tensorflow.python.keras.callbacks.History at 0x7f53dd160240>"
]
},
"execution_count": 9,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"cnn.fit(tr_dataset, epochs=epochs, steps_per_epoch=total_step,\n",
" validation_data=val_dataset, validation_steps=5000//100)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### Calculate accuracy"
]
},
{
"cell_type": "code",
"execution_count": 10,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Tensor(\"simple_cnn/dense_1/Softmax:0\", shape=(10000, 10), dtype=float32)\n"
]
}
],
"source": [
"sess = keras.backend.get_session()\n",
"x_tst_tensor = tf.convert_to_tensor(x_tst)\n",
"yhat = cnn(x_tst_tensor, training=False)\n",
"print(yhat)"
]
},
{
"cell_type": "code",
"execution_count": 11,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"tst acc : 99.13%\n"
]
}
],
"source": [
"yhat = sess.run(yhat)\n",
"print('tst acc : {:.2%}'.format(np.mean(np.argmax(yhat, axis=-1) == y_tst)))"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.6.8"
},
"varInspector": {
"cols": {
"lenName": 16,
"lenType": 16,
"lenVar": 40
},
"kernels_config": {
"python": {
"delete_cmd_postfix": "",
"delete_cmd_prefix": "del ",
"library": "var_list.py",
"varRefreshCmd": "print(var_dic_list())"
},
"r": {
"delete_cmd_postfix": ") ",
"delete_cmd_prefix": "rm(",
"library": "var_list.r",
"varRefreshCmd": "cat(var_dic_list()) "
}
},
"types_to_exclude": [
"module",
"function",
"builtin_function_or_method",
"instance",
"_Feature"
],
"window_display": false
}
},
"nbformat": 4,
"nbformat_minor": 2
}