{ "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "Deep Learning Models -- A collection of various deep learning architectures, models, and tips for TensorFlow and PyTorch in Jupyter Notebooks.\n", "- Author: Sebastian Raschka\n", "- GitHub Repository: https://github.com/rasbt/deeplearning-models" ] }, { "cell_type": "code", "execution_count": 1, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Sebastian Raschka \n", "\n", "CPython 3.6.1\n", "IPython 6.0.0\n", "\n", "tensorflow 1.2.0\n" ] } ], "source": [ "%load_ext watermark\n", "%watermark -a 'Sebastian Raschka' -v -p tensorflow" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Model Zoo -- Convolutional Neural Network (VGG16)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "The VGG-16 Convolutional Neural Network Architecture [1] implemented in TensorFlow and trained on Cifar-10 [2, 3] images.\n", "\n", "References:\n", "\n", "- [1] Simonyan, K., & Zisserman, A. (2015). [Very Deep Convolutional Networks for Large-Scale Image Recognition](https://arxiv.org/abs/1409.1556). International Conference on Learning Representations (ICRL), 1–14. https://doi.org/10.1016/j.infsof.2008.09.005\n", "- [2] Krizhevsky, A. (2009). [Learning Multiple Layers of Features from Tiny Images](https://doi.org/10.1.1.222.9220 http://citeseerx.ist.psu.edu/viewdoc/download?doi=10.1.1.222.9220&rep=rep1&type=pdf). Science Department, University of Toronto.\n", "- [3] https://www.cs.toronto.edu/~kriz/cifar.html" ] }, { "cell_type": "code", "execution_count": 1, "metadata": {}, "outputs": [ { "ename": "TypeError", "evalue": "insert() takes exactly 2 arguments (1 given)", "output_type": "error", "traceback": [ "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", "\u001b[0;31mTypeError\u001b[0m Traceback (most recent call last)", "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[1;32m 5\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 6\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0msys\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 7\u001b[0;31m \u001b[0msys\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0minsert\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'..'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 8\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 9\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mhelper\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mdownload_and_extract_cifar\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;31mTypeError\u001b[0m: insert() takes exactly 2 arguments (1 given)" ] } ], "source": [ "##########################\n", "### DATASET\n", "##########################\n", "\n", "# load utilities from ../helper.py\n", "import sys\n", "sys.path.insert(0, '..') \n", "from helper import download_and_extract_cifar\n", "from helper import Cifar10Loader\n", "\n", "dest = download_and_extract_cifar('./cifar-10')\n", "cifar = Cifar10Loader(dest, normalize=True, \n", " zero_center=True,\n", " channel_mean_center=False)\n", "cifar.num_train\n", "\n", "X, y = cifar.load_test()\n", "half = cifar.num_test // 2\n", "X_test, X_valid = X[:half], X[half:]\n", "y_test, y_valid = y[:half], y[half:]\n", "\n", "del X, y" ] }, { "cell_type": "code", "execution_count": 3, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "'/gpu:0'" ] }, "execution_count": 3, "metadata": {}, "output_type": "execute_result" } ], "source": [ "import tensorflow as tf\n", "import numpy as np\n", "\n", "tf.test.gpu_device_name()" ] }, { "cell_type": "code", "execution_count": 4, "metadata": { "scrolled": true }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Minibatch: 0200 | Cost: 9.004\n", "Minibatch: 0400 | Cost: 3.877\n", "Minibatch: 0600 | Cost: 3.213\n", "Minibatch: 0800 | Cost: 2.410\n", "Minibatch: 1000 | Cost: 2.384\n", "Minibatch: 1200 | Cost: 2.250\n", "Minibatch: 1400 | Cost: 2.352\n", "Epoch: 001 | AvgCost: 1897.994 | Train/Valid ACC: 0.152/0.145\n", "Minibatch: 0200 | Cost: 2.433\n", "Minibatch: 0400 | Cost: 2.426\n", "Minibatch: 0600 | Cost: 2.202\n", "Minibatch: 0800 | Cost: 2.221\n", "Minibatch: 1000 | Cost: 2.112\n", "Minibatch: 1200 | Cost: 2.182\n", "Minibatch: 1400 | Cost: 2.382\n", "Epoch: 002 | AvgCost: 2.219 | Train/Valid ACC: 0.176/0.168\n", "Minibatch: 0200 | Cost: 2.145\n", "Minibatch: 0400 | Cost: 2.089\n", "Minibatch: 0600 | Cost: 2.221\n", "Minibatch: 0800 | Cost: 1.903\n", "Minibatch: 1000 | Cost: 2.296\n", "Minibatch: 1200 | Cost: 2.169\n", "Minibatch: 1400 | Cost: 2.164\n", "Epoch: 003 | AvgCost: 2.168 | Train/Valid ACC: 0.166/0.173\n", "Minibatch: 0200 | Cost: 2.094\n", "Minibatch: 0400 | Cost: 2.015\n", "Minibatch: 0600 | Cost: 2.217\n", "Minibatch: 0800 | Cost: 2.028\n", "Minibatch: 1000 | Cost: 2.265\n", "Minibatch: 1200 | Cost: 2.103\n", "Minibatch: 1400 | Cost: 2.060\n", "Epoch: 004 | AvgCost: 2.144 | Train/Valid ACC: 0.189/0.188\n", "Minibatch: 0200 | Cost: 2.310\n", "Minibatch: 0400 | Cost: 1.986\n", "Minibatch: 0600 | Cost: 1.844\n", "Minibatch: 0800 | Cost: 2.183\n", "Minibatch: 1000 | Cost: 2.040\n", "Minibatch: 1200 | Cost: 2.004\n", "Minibatch: 1400 | Cost: 1.987\n", "Epoch: 005 | AvgCost: 2.075 | Train/Valid ACC: 0.180/0.181\n", "Minibatch: 0200 | Cost: 1.849\n", "Minibatch: 0400 | Cost: 1.898\n", "Minibatch: 0600 | Cost: 1.996\n", "Minibatch: 0800 | Cost: 1.968\n", "Minibatch: 1000 | Cost: 1.987\n", "Minibatch: 1200 | Cost: 2.003\n", "Minibatch: 1400 | Cost: 2.041\n", "Epoch: 006 | AvgCost: 2.010 | Train/Valid ACC: 0.176/0.176\n", "Minibatch: 0200 | Cost: 2.419\n", "Minibatch: 0400 | Cost: 2.109\n", "Minibatch: 0600 | Cost: 1.902\n", "Minibatch: 0800 | Cost: 1.937\n", "Minibatch: 1000 | Cost: 1.813\n", "Minibatch: 1200 | Cost: 1.990\n", "Minibatch: 1400 | Cost: 1.662\n", "Epoch: 007 | AvgCost: 1.913 | Train/Valid ACC: 0.256/0.251\n", "Minibatch: 0200 | Cost: 2.036\n", "Minibatch: 0400 | Cost: 1.690\n", "Minibatch: 0600 | Cost: 1.866\n", "Minibatch: 0800 | Cost: 1.574\n", "Minibatch: 1000 | Cost: 1.886\n", "Minibatch: 1200 | Cost: 1.720\n", "Minibatch: 1400 | Cost: 1.937\n", "Epoch: 008 | AvgCost: 1.769 | Train/Valid ACC: 0.270/0.267\n", "Minibatch: 0200 | Cost: 1.569\n", "Minibatch: 0400 | Cost: 1.800\n", "Minibatch: 0600 | Cost: 1.541\n", "Minibatch: 0800 | Cost: 1.499\n", "Minibatch: 1000 | Cost: 1.596\n", "Minibatch: 1200 | Cost: 1.994\n", "Minibatch: 1400 | Cost: 1.283\n", "Epoch: 009 | AvgCost: 1.653 | Train/Valid ACC: 0.343/0.337\n", "Minibatch: 0200 | Cost: 1.415\n", "Minibatch: 0400 | Cost: 1.648\n", "Minibatch: 0600 | Cost: 1.487\n", "Minibatch: 0800 | Cost: 1.454\n", "Minibatch: 1000 | Cost: 1.266\n", "Minibatch: 1200 | Cost: 1.267\n", "Minibatch: 1400 | Cost: 1.589\n", "Epoch: 010 | AvgCost: 1.526 | Train/Valid ACC: 0.469/0.459\n", "Minibatch: 0200 | Cost: 1.585\n", "Minibatch: 0400 | Cost: 1.449\n", "Minibatch: 0600 | Cost: 1.340\n", "Minibatch: 0800 | Cost: 1.297\n", "Minibatch: 1000 | Cost: 1.767\n", "Minibatch: 1200 | Cost: 1.088\n", "Minibatch: 1400 | Cost: 1.159\n", "Epoch: 011 | AvgCost: 1.342 | Train/Valid ACC: 0.536/0.517\n", "Minibatch: 0200 | Cost: 1.408\n", "Minibatch: 0400 | Cost: 1.510\n", "Minibatch: 0600 | Cost: 1.424\n", "Minibatch: 0800 | Cost: 1.019\n", "Minibatch: 1000 | Cost: 1.069\n", "Minibatch: 1200 | Cost: 1.465\n", "Minibatch: 1400 | Cost: 1.289\n", "Epoch: 012 | AvgCost: 1.203 | Train/Valid ACC: 0.590/0.576\n", "Minibatch: 0200 | Cost: 0.670\n", "Minibatch: 0400 | Cost: 0.993\n", "Minibatch: 0600 | Cost: 0.998\n", "Minibatch: 0800 | Cost: 1.103\n", "Minibatch: 1000 | Cost: 1.127\n", "Minibatch: 1200 | Cost: 0.969\n", "Minibatch: 1400 | Cost: 1.216\n", "Epoch: 013 | AvgCost: 1.111 | Train/Valid ACC: 0.634/0.603\n", "Minibatch: 0200 | Cost: 1.262\n", "Minibatch: 0400 | Cost: 1.147\n", "Minibatch: 0600 | Cost: 0.660\n", "Minibatch: 0800 | Cost: 0.970\n", "Minibatch: 1000 | Cost: 0.716\n", "Minibatch: 1200 | Cost: 1.017\n", "Minibatch: 1400 | Cost: 1.264\n", "Epoch: 014 | AvgCost: 1.036 | Train/Valid ACC: 0.638/0.617\n", "Minibatch: 0200 | Cost: 0.846\n", "Minibatch: 0400 | Cost: 0.843\n", "Minibatch: 0600 | Cost: 0.792\n", "Minibatch: 0800 | Cost: 0.718\n", "Minibatch: 1000 | Cost: 1.141\n", "Minibatch: 1200 | Cost: 1.024\n", "Minibatch: 1400 | Cost: 0.877\n", "Epoch: 015 | AvgCost: 0.949 | Train/Valid ACC: 0.683/0.654\n", "Minibatch: 0200 | Cost: 0.463\n", "Minibatch: 0400 | Cost: 0.854\n", "Minibatch: 0600 | Cost: 0.820\n", "Minibatch: 0800 | Cost: 0.952\n", "Minibatch: 1000 | Cost: 0.819\n", "Minibatch: 1200 | Cost: 0.648\n", "Minibatch: 1400 | Cost: 1.030\n", "Epoch: 016 | AvgCost: 0.872 | Train/Valid ACC: 0.738/0.697\n", "Minibatch: 0200 | Cost: 0.862\n", "Minibatch: 0400 | Cost: 1.010\n", "Minibatch: 0600 | Cost: 0.991\n", "Minibatch: 0800 | Cost: 0.664\n", "Minibatch: 1000 | Cost: 0.604\n", "Minibatch: 1200 | Cost: 0.879\n", "Minibatch: 1400 | Cost: 0.791\n", "Epoch: 017 | AvgCost: 0.823 | Train/Valid ACC: 0.753/0.698\n", "Minibatch: 0200 | Cost: 0.823\n", "Minibatch: 0400 | Cost: 0.619\n", "Minibatch: 0600 | Cost: 0.764\n", "Minibatch: 0800 | Cost: 1.127\n", "Minibatch: 1000 | Cost: 0.623\n", "Minibatch: 1200 | Cost: 0.796\n", "Minibatch: 1400 | Cost: 0.694\n", "Epoch: 018 | AvgCost: 0.771 | Train/Valid ACC: 0.769/0.709\n", "Minibatch: 0200 | Cost: 0.431\n", "Minibatch: 0400 | Cost: 0.688\n", "Minibatch: 0600 | Cost: 0.406\n", "Minibatch: 0800 | Cost: 0.906\n", "Minibatch: 1000 | Cost: 0.607\n", "Minibatch: 1200 | Cost: 0.445\n", "Minibatch: 1400 | Cost: 0.660\n", "Epoch: 019 | AvgCost: 0.721 | Train/Valid ACC: 0.780/0.707\n", "Minibatch: 0200 | Cost: 0.740\n", "Minibatch: 0400 | Cost: 0.542\n", "Minibatch: 0600 | Cost: 0.435\n", "Minibatch: 0800 | Cost: 0.953\n", "Minibatch: 1000 | Cost: 0.954\n", "Minibatch: 1200 | Cost: 0.544\n", "Minibatch: 1400 | Cost: 0.855\n", "Epoch: 020 | AvgCost: 0.672 | Train/Valid ACC: 0.779/0.717\n", "Minibatch: 0200 | Cost: 0.432\n", "Minibatch: 0400 | Cost: 0.376\n", "Minibatch: 0600 | Cost: 0.477\n", "Minibatch: 0800 | Cost: 0.476\n", "Minibatch: 1000 | Cost: 0.950\n", "Minibatch: 1200 | Cost: 0.468\n", "Minibatch: 1400 | Cost: 0.595\n", "Epoch: 021 | AvgCost: 0.636 | Train/Valid ACC: 0.818/0.737\n", "Minibatch: 0200 | Cost: 1.114\n", "Minibatch: 0400 | Cost: 0.519\n", "Minibatch: 0600 | Cost: 0.623\n", "Minibatch: 0800 | Cost: 0.721\n", "Minibatch: 1000 | Cost: 0.880\n", "Minibatch: 1200 | Cost: 0.654\n", "Minibatch: 1400 | Cost: 0.378\n", "Epoch: 022 | AvgCost: 0.596 | Train/Valid ACC: 0.825/0.742\n", "Minibatch: 0200 | Cost: 0.541\n", "Minibatch: 0400 | Cost: 0.478\n", "Minibatch: 0600 | Cost: 0.642\n", "Minibatch: 0800 | Cost: 0.871\n", "Minibatch: 1000 | Cost: 0.710\n", "Minibatch: 1200 | Cost: 0.682\n", "Minibatch: 1400 | Cost: 0.671\n", "Epoch: 023 | AvgCost: 0.663 | Train/Valid ACC: 0.776/0.712\n", "Minibatch: 0200 | Cost: 0.498\n", "Minibatch: 0400 | Cost: 0.379\n", "Minibatch: 0600 | Cost: 0.535\n", "Minibatch: 0800 | Cost: 0.435\n", "Minibatch: 1000 | Cost: 0.562\n", "Minibatch: 1200 | Cost: 0.777\n", "Minibatch: 1400 | Cost: 0.635\n", "Epoch: 024 | AvgCost: 0.539 | Train/Valid ACC: 0.853/0.759\n", "Minibatch: 0200 | Cost: 0.331\n", "Minibatch: 0400 | Cost: 0.419\n", "Minibatch: 0600 | Cost: 0.497\n", "Minibatch: 0800 | Cost: 0.619\n", "Minibatch: 1000 | Cost: 0.620\n", "Minibatch: 1200 | Cost: 0.196\n", "Minibatch: 1400 | Cost: 0.648\n", "Epoch: 025 | AvgCost: 0.498 | Train/Valid ACC: 0.849/0.738\n", "Minibatch: 0200 | Cost: 0.247\n", "Minibatch: 0400 | Cost: 0.263\n", "Minibatch: 0600 | Cost: 0.183\n", "Minibatch: 0800 | Cost: 0.251\n", "Minibatch: 1000 | Cost: 0.251\n", "Minibatch: 1200 | Cost: 0.447\n", "Minibatch: 1400 | Cost: 0.443\n", "Epoch: 026 | AvgCost: 0.469 | Train/Valid ACC: 0.850/0.744\n", "Minibatch: 0200 | Cost: 0.390\n", "Minibatch: 0400 | Cost: 0.325\n", "Minibatch: 0600 | Cost: 0.404\n", "Minibatch: 0800 | Cost: 0.363\n", "Minibatch: 1000 | Cost: 0.655\n", "Minibatch: 1200 | Cost: 0.195\n", "Minibatch: 1400 | Cost: 0.647\n", "Epoch: 027 | AvgCost: 0.485 | Train/Valid ACC: 0.855/0.751\n", "Minibatch: 0200 | Cost: 0.580\n", "Minibatch: 0400 | Cost: 0.177\n", "Minibatch: 0600 | Cost: 0.317\n", "Minibatch: 0800 | Cost: 0.249\n", "Minibatch: 1000 | Cost: 0.287\n", "Minibatch: 1200 | Cost: 0.282\n", "Minibatch: 1400 | Cost: 0.429\n", "Epoch: 028 | AvgCost: 0.416 | Train/Valid ACC: 0.887/0.759\n", "Minibatch: 0200 | Cost: 0.233\n", "Minibatch: 0400 | Cost: 0.118\n", "Minibatch: 0600 | Cost: 0.218\n", "Minibatch: 0800 | Cost: 0.195\n", "Minibatch: 1000 | Cost: 0.208\n", "Minibatch: 1200 | Cost: 0.188\n", "Minibatch: 1400 | Cost: 0.509\n", "Epoch: 029 | AvgCost: 0.488 | Train/Valid ACC: 0.874/0.767\n", "Minibatch: 0200 | Cost: 0.313\n", "Minibatch: 0400 | Cost: 0.508\n", "Minibatch: 0600 | Cost: 0.553\n", "Minibatch: 0800 | Cost: 0.385\n", "Minibatch: 1000 | Cost: 0.272\n", "Minibatch: 1200 | Cost: 0.366\n", "Minibatch: 1400 | Cost: 0.156\n", "Epoch: 030 | AvgCost: 0.404 | Train/Valid ACC: 0.909/0.776\n" ] } ], "source": [ "##########################\n", "### SETTINGS\n", "##########################\n", "\n", "# Hyperparameters\n", "learning_rate = 0.001\n", "training_epochs = 30\n", "batch_size = 32\n", "\n", "# Other\n", "print_interval = 200\n", "\n", "# Architecture\n", "image_width, image_height, image_depth = 32, 32, 3\n", "n_classes = 10\n", "\n", "\n", "##########################\n", "### WRAPPER FUNCTIONS\n", "##########################\n", "\n", "def conv_layer(input, input_channels, output_channels, \n", " kernel_size, strides, scope, padding='SAME'):\n", " with tf.name_scope(scope):\n", " weights_shape = kernel_size + [input_channels, output_channels]\n", " weights = tf.Variable(tf.truncated_normal(shape=weights_shape,\n", " mean=0.0,\n", " stddev=0.1,\n", " dtype=tf.float32),\n", " name='weights')\n", " biases = tf.Variable(tf.zeros(shape=[output_channels]),\n", " name='biases')\n", " conv = tf.nn.conv2d(input=input,\n", " filter=weights,\n", " strides=strides,\n", " padding=padding,\n", " name='convolution')\n", " out = tf.nn.bias_add(conv, biases, name='logits')\n", " out = tf.nn.relu(out, name='activation')\n", " return out\n", "\n", "\n", "def fc_layer(input, output_nodes, scope,\n", " activation=None, seed=None):\n", " with tf.name_scope(scope):\n", " shape = int(np.prod(input.get_shape()[1:]))\n", " flat_input = tf.reshape(input, [-1, shape])\n", " weights = tf.Variable(tf.truncated_normal(shape=[shape,\n", " output_nodes],\n", " mean=0.0,\n", " stddev=0.1,\n", " dtype=tf.float32,\n", " seed=seed),\n", " name='weights')\n", " biases = tf.Variable(tf.zeros(shape=[output_nodes]),\n", " name='biases')\n", " act = tf.nn.bias_add(tf.matmul(flat_input, weights), biases, \n", " name='logits')\n", "\n", " if activation is not None:\n", " act = activation(act, name='activation')\n", "\n", " return act\n", "\n", "\n", "##########################\n", "### GRAPH DEFINITION\n", "##########################\n", "\n", "g = tf.Graph()\n", "with g.as_default():\n", "\n", " # Input data\n", " tf_x = tf.placeholder(tf.float32, [None, image_width, image_height, image_depth], name='features')\n", " tf_y = tf.placeholder(tf.float32, [None, n_classes], name='targets')\n", " \n", " ##########################\n", " ### VGG16 Model\n", " ##########################\n", "\n", " # =========\n", " # BLOCK 1\n", " # =========\n", " conv_layer_1 = conv_layer(input=tf_x,\n", " input_channels=3,\n", " output_channels=64,\n", " kernel_size=[3, 3],\n", " strides=[1, 1, 1, 1],\n", " scope='conv1')\n", " \n", " conv_layer_2 = conv_layer(input=conv_layer_1,\n", " input_channels=64,\n", " output_channels=64,\n", " kernel_size=[3, 3],\n", " strides=[1, 1, 1, 1],\n", " scope='conv2') \n", " \n", " pool_layer_1 = tf.nn.max_pool(conv_layer_2,\n", " ksize=[1, 2, 2, 1], \n", " strides=[1, 2, 2, 1],\n", " padding='SAME',\n", " name='pool1') \n", " # =========\n", " # BLOCK 2\n", " # =========\n", " conv_layer_3 = conv_layer(input=pool_layer_1,\n", " input_channels=64,\n", " output_channels=128,\n", " kernel_size=[3, 3],\n", " strides=[1, 1, 1, 1],\n", " scope='conv3') \n", " \n", " conv_layer_4 = conv_layer(input=conv_layer_3,\n", " input_channels=128,\n", " output_channels=128,\n", " kernel_size=[3, 3],\n", " strides=[1, 1, 1, 1],\n", " scope='conv4') \n", " \n", " pool_layer_2 = tf.nn.max_pool(conv_layer_4,\n", " ksize=[1, 2, 2, 1], \n", " strides=[1, 2, 2, 1],\n", " padding='SAME',\n", " name='pool2') \n", " # =========\n", " # BLOCK 3\n", " # =========\n", " conv_layer_5 = conv_layer(input=pool_layer_2,\n", " input_channels=128,\n", " output_channels=256,\n", " kernel_size=[3, 3],\n", " strides=[1, 1, 1, 1],\n", " scope='conv5') \n", " \n", " conv_layer_6 = conv_layer(input=conv_layer_5,\n", " input_channels=256,\n", " output_channels=256,\n", " kernel_size=[3, 3],\n", " strides=[1, 1, 1, 1],\n", " scope='conv6') \n", " \n", " conv_layer_7 = conv_layer(input=conv_layer_6,\n", " input_channels=256,\n", " output_channels=256,\n", " kernel_size=[3, 3],\n", " strides=[1, 1, 1, 1],\n", " scope='conv7')\n", " \n", " pool_layer_3 = tf.nn.max_pool(conv_layer_7,\n", " ksize=[1, 2, 2, 1], \n", " strides=[1, 2, 2, 1],\n", " padding='SAME',\n", " name='pool3') \n", " # =========\n", " # BLOCK 4\n", " # =========\n", " conv_layer_8 = conv_layer(input=pool_layer_3,\n", " input_channels=256,\n", " output_channels=512,\n", " kernel_size=[3, 3],\n", " strides=[1, 1, 1, 1],\n", " scope='conv8') \n", " \n", " conv_layer_9 = conv_layer(input=conv_layer_8,\n", " input_channels=512,\n", " output_channels=512,\n", " kernel_size=[3, 3],\n", " strides=[1, 1, 1, 1],\n", " scope='conv9') \n", " \n", " conv_layer_10 = conv_layer(input=conv_layer_9,\n", " input_channels=512,\n", " output_channels=512,\n", " kernel_size=[3, 3],\n", " strides=[1, 1, 1, 1],\n", " scope='conv10') \n", " \n", " pool_layer_4 = tf.nn.max_pool(conv_layer_10,\n", " ksize=[1, 2, 2, 1], \n", " strides=[1, 2, 2, 1],\n", " padding='SAME',\n", " name='pool4') \n", " # =========\n", " # BLOCK 5\n", " # =========\n", " conv_layer_11 = conv_layer(input=pool_layer_4,\n", " input_channels=512,\n", " output_channels=512,\n", " kernel_size=[3, 3],\n", " strides=[1, 1, 1, 1],\n", " scope='conv11') \n", " \n", " conv_layer_12 = conv_layer(input=conv_layer_11,\n", " input_channels=512,\n", " output_channels=512,\n", " kernel_size=[3, 3],\n", " strides=[1, 1, 1, 1],\n", " scope='conv12') \n", "\n", " conv_layer_13 = conv_layer(input=conv_layer_12,\n", " input_channels=512,\n", " output_channels=512,\n", " kernel_size=[3, 3],\n", " strides=[1, 1, 1, 1],\n", " scope='conv13') \n", " \n", " pool_layer_5 = tf.nn.max_pool(conv_layer_12,\n", " ksize=[1, 2, 2, 1], \n", " strides=[1, 2, 2, 1],\n", " padding='SAME',\n", " name='pool5') \n", " # ===========\n", " # CLASSIFIER\n", " # ===========\n", " \n", " fc_layer_1 = fc_layer(input=pool_layer_5, \n", " output_nodes=4096,\n", " activation=tf.nn.relu,\n", " scope='fc1')\n", " \n", " fc_layer_2 = fc_layer(input=fc_layer_1, \n", " output_nodes=4096,\n", " activation=tf.nn.relu,\n", " scope='fc2')\n", "\n", " out_layer = fc_layer(input=fc_layer_2, \n", " output_nodes=n_classes,\n", " activation=None,\n", " scope='output_layer')\n", " \n", " # Loss and optimizer\n", " loss = tf.nn.softmax_cross_entropy_with_logits(logits=out_layer, labels=tf_y)\n", " cost = tf.reduce_mean(loss, name='cost')\n", " \n", " optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate)\n", " train = optimizer.minimize(cost, name='train')\n", "\n", " # Prediction\n", " correct_prediction = tf.equal(tf.argmax(tf_y, 1), tf.argmax(out_layer, 1), \n", " name='correct_predictions')\n", " accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32), name='accuracy')\n", "\n", " # Saver to save session for reuse\n", " saver = tf.train.Saver()\n", "\n", " \n", "##########################\n", "### TRAINING & EVALUATION\n", "##########################\n", "\n", "with tf.Session(graph=g) as sess:\n", " sess.run(tf.global_variables_initializer())\n", "\n", " for epoch in range(training_epochs):\n", " \n", " avg_cost = 0.\n", " mbatch_cnt = 0\n", " for batch_x, batch_y in cifar.load_train_epoch(shuffle=True, batch_size=batch_size):\n", " \n", " mbatch_cnt += 1\n", " _, c = sess.run(['train', 'cost:0'], feed_dict={'features:0': batch_x,\n", " 'targets:0': batch_y})\n", " avg_cost += c\n", "\n", " if not mbatch_cnt % print_interval:\n", " print(\"Minibatch: %04d | Cost: %.3f\" % (mbatch_cnt, c))\n", " \n", "\n", " # ===================\n", " # Training Accuracy\n", " # ===================\n", " n_predictions, n_correct = 0, 0\n", " for batch_x, batch_y in cifar.load_train_epoch(batch_size=batch_size):\n", " \n", " p = sess.run('correct_predictions:0', \n", " feed_dict={'features:0': batch_x,\n", " 'targets:0': batch_y})\n", " n_correct += np.sum(p)\n", " n_predictions += p.shape[0]\n", " train_acc = n_correct / n_predictions\n", " \n", " \n", " # ===================\n", " # Validation Accuracy\n", " # ===================\n", " #valid_acc = sess.run('accuracy:0', feed_dict={'features:0': X_valid,\n", " # 'targets:0': y_valid})\n", " # ---------------------------------------\n", " # workaround for GPUs with <= 4 Gb memory\n", " n_predictions, n_correct = 0, 0\n", " indices = np.arange(y_valid.shape[0])\n", " chunksize = 500\n", " for start_idx in range(0, indices.shape[0] - chunksize + 1, chunksize):\n", " index_slice = indices[start_idx:start_idx + chunksize]\n", " p = sess.run('correct_predictions:0', \n", " feed_dict={'features:0': X_valid[index_slice],\n", " 'targets:0': y_valid[index_slice]})\n", " n_correct += np.sum(p)\n", " n_predictions += p.shape[0]\n", " valid_acc = n_correct / n_predictions\n", " # ---------------------------------------\n", " \n", " print(\"Epoch: %03d | AvgCost: %.3f\" % (epoch + 1, avg_cost / mbatch_cnt), end=\"\")\n", " print(\" | Train/Valid ACC: %.3f/%.3f\" % (train_acc, valid_acc))\n", " \n", " saver.save(sess, save_path='./convnet-vgg16.ckpt')" ] }, { "cell_type": "code", "execution_count": 5, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow:Restoring parameters from ./convnet-vgg16.ckpt\n", "Test ACC: 0.772\n" ] } ], "source": [ "##########################\n", "### RELOAD & TEST\n", "##########################\n", "\n", "with tf.Session(graph=g) as sess:\n", " saver.restore(sess, save_path='./convnet-vgg16.ckpt')\n", " \n", " # test_acc = sess.run('accuracy:0', feed_dict={'features:0': X_test,\n", " # 'targets:0': y_test})\n", " # ---------------------------------------\n", " # workaround for GPUs with <= 4 Gb memory\n", " n_predictions, n_correct = 0, 0\n", " indices = np.arange(y_test.shape[0])\n", " chunksize = 500\n", " for start_idx in range(0, indices.shape[0] - chunksize + 1, chunksize):\n", " index_slice = indices[start_idx:start_idx + chunksize]\n", " p = sess.run('correct_predictions:0', \n", " feed_dict={'features:0': X_test[index_slice],\n", " 'targets:0': y_test[index_slice]})\n", " n_correct += np.sum(p)\n", " n_predictions += p.shape[0]\n", " test_acc = n_correct / n_predictions\n", " # ---------------------------------------\n", "\n", " print('Test ACC: %.3f' % test_acc)" ] } ], "metadata": { "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.7.1" } }, "nbformat": 4, "nbformat_minor": 2 }