{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "# Convolutional Neural Network Example\n",
    "\n",
    "Build a convolutional neural network with TensorFlow.\n",
    "\n",
    "- Author: Aymeric Damien\n",
    "- Project: https://github.com/aymericdamien/TensorFlow-Examples/"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## CNN Overview\n",
    "\n",
    "![CNN](http://personal.ie.cuhk.edu.hk/~ccloy/project_target_code/images/fig3.png)\n",
    "\n",
    "## MNIST Dataset Overview\n",
    "\n",
    "This example is using MNIST handwritten digits. The dataset contains 60,000 examples for training and 10,000 examples for testing. The digits have been size-normalized and centered in a fixed-size image (28x28 pixels) with values from 0 to 1. For simplicity, each image has been flattened and converted to a 1-D numpy array of 784 features (28*28).\n",
    "\n",
    "![MNIST Dataset](http://neuralnetworksanddeeplearning.com/images/mnist_100_digits.png)\n",
    "\n",
    "More info: http://yann.lecun.com/exdb/mnist/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting /tmp/data/train-images-idx3-ubyte.gz\n",
      "Extracting /tmp/data/train-labels-idx1-ubyte.gz\n",
      "Extracting /tmp/data/t10k-images-idx3-ubyte.gz\n",
      "Extracting /tmp/data/t10k-labels-idx1-ubyte.gz\n"
     ]
    }
   ],
   "source": [
    "from __future__ import division, print_function, absolute_import\n",
    "\n",
    "import tensorflow as tf\n",
    "\n",
    "# Import MNIST data\n",
    "from tensorflow.examples.tutorials.mnist import input_data\n",
    "mnist = input_data.read_data_sets(\"/tmp/data/\", one_hot=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Training Parameters\n",
    "learning_rate = 0.001\n",
    "num_steps = 500\n",
    "batch_size = 128\n",
    "display_step = 10\n",
    "\n",
    "# Network Parameters\n",
    "num_input = 784 # MNIST data input (img shape: 28*28)\n",
    "num_classes = 10 # MNIST total classes (0-9 digits)\n",
    "dropout = 0.75 # Dropout, probability to keep units\n",
    "\n",
    "# tf Graph input\n",
    "X = tf.placeholder(tf.float32, [None, num_input])\n",
    "Y = tf.placeholder(tf.float32, [None, num_classes])\n",
    "keep_prob = tf.placeholder(tf.float32) # dropout (keep probability)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Create some wrappers for simplicity\n",
    "def conv2d(x, W, b, strides=1):\n",
    "    # Conv2D wrapper, with bias and relu activation\n",
    "    x = tf.nn.conv2d(x, W, strides=[1, strides, strides, 1], padding='SAME')\n",
    "    x = tf.nn.bias_add(x, b)\n",
    "    return tf.nn.relu(x)\n",
    "\n",
    "\n",
    "def maxpool2d(x, k=2):\n",
    "    # MaxPool2D wrapper\n",
    "    return tf.nn.max_pool(x, ksize=[1, k, k, 1], strides=[1, k, k, 1],\n",
    "                          padding='SAME')\n",
    "\n",
    "\n",
    "# Create model\n",
    "def conv_net(x, weights, biases, dropout):\n",
    "    # MNIST data input is a 1-D vector of 784 features (28*28 pixels)\n",
    "    # Reshape to match picture format [Height x Width x Channel]\n",
    "    # Tensor input become 4-D: [Batch Size, Height, Width, Channel]\n",
    "    x = tf.reshape(x, shape=[-1, 28, 28, 1])\n",
    "\n",
    "    # Convolution Layer\n",
    "    conv1 = conv2d(x, weights['wc1'], biases['bc1'])\n",
    "    # Max Pooling (down-sampling)\n",
    "    conv1 = maxpool2d(conv1, k=2)\n",
    "\n",
    "    # Convolution Layer\n",
    "    conv2 = conv2d(conv1, weights['wc2'], biases['bc2'])\n",
    "    # Max Pooling (down-sampling)\n",
    "    conv2 = maxpool2d(conv2, k=2)\n",
    "\n",
    "    # Fully connected layer\n",
    "    # Reshape conv2 output to fit fully connected layer input\n",
    "    fc1 = tf.reshape(conv2, [-1, weights['wd1'].get_shape().as_list()[0]])\n",
    "    fc1 = tf.add(tf.matmul(fc1, weights['wd1']), biases['bd1'])\n",
    "    fc1 = tf.nn.relu(fc1)\n",
    "    # Apply Dropout\n",
    "    fc1 = tf.nn.dropout(fc1, dropout)\n",
    "\n",
    "    # Output, class prediction\n",
    "    out = tf.add(tf.matmul(fc1, weights['out']), biases['out'])\n",
    "    return out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Store layers weight & bias\n",
    "weights = {\n",
    "    # 5x5 conv, 1 input, 32 outputs\n",
    "    'wc1': tf.Variable(tf.random_normal([5, 5, 1, 32])),\n",
    "    # 5x5 conv, 32 inputs, 64 outputs\n",
    "    'wc2': tf.Variable(tf.random_normal([5, 5, 32, 64])),\n",
    "    # fully connected, 7*7*64 inputs, 1024 outputs\n",
    "    'wd1': tf.Variable(tf.random_normal([7*7*64, 1024])),\n",
    "    # 1024 inputs, 10 outputs (class prediction)\n",
    "    'out': tf.Variable(tf.random_normal([1024, num_classes]))\n",
    "}\n",
    "\n",
    "biases = {\n",
    "    'bc1': tf.Variable(tf.random_normal([32])),\n",
    "    'bc2': tf.Variable(tf.random_normal([64])),\n",
    "    'bd1': tf.Variable(tf.random_normal([1024])),\n",
    "    'out': tf.Variable(tf.random_normal([num_classes]))\n",
    "}\n",
    "\n",
    "# Construct model\n",
    "logits = conv_net(X, weights, biases, keep_prob)\n",
    "prediction = tf.nn.softmax(logits)\n",
    "\n",
    "# Define loss and optimizer\n",
    "loss_op = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(\n",
    "    logits=logits, labels=Y))\n",
    "optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate)\n",
    "train_op = optimizer.minimize(loss_op)\n",
    "\n",
    "\n",
    "# Evaluate model\n",
    "correct_pred = tf.equal(tf.argmax(prediction, 1), tf.argmax(Y, 1))\n",
    "accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))\n",
    "\n",
    "# Initialize the variables (i.e. assign their default value)\n",
    "init = tf.global_variables_initializer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 1, Minibatch Loss= 63763.3047, Training Accuracy= 0.141\n",
      "Step 10, Minibatch Loss= 26429.6680, Training Accuracy= 0.242\n",
      "Step 20, Minibatch Loss= 12171.8584, Training Accuracy= 0.586\n",
      "Step 30, Minibatch Loss= 6306.6318, Training Accuracy= 0.734\n",
      "Step 40, Minibatch Loss= 5113.7583, Training Accuracy= 0.711\n",
      "Step 50, Minibatch Loss= 4022.2131, Training Accuracy= 0.805\n",
      "Step 60, Minibatch Loss= 3125.4949, Training Accuracy= 0.867\n",
      "Step 70, Minibatch Loss= 2225.4875, Training Accuracy= 0.875\n",
      "Step 80, Minibatch Loss= 1843.3540, Training Accuracy= 0.867\n",
      "Step 90, Minibatch Loss= 1715.7744, Training Accuracy= 0.875\n",
      "Step 100, Minibatch Loss= 2611.2708, Training Accuracy= 0.906\n",
      "Step 110, Minibatch Loss= 4804.0913, Training Accuracy= 0.875\n",
      "Step 120, Minibatch Loss= 1067.5258, Training Accuracy= 0.938\n",
      "Step 130, Minibatch Loss= 2519.1514, Training Accuracy= 0.898\n",
      "Step 140, Minibatch Loss= 2687.9292, Training Accuracy= 0.906\n",
      "Step 150, Minibatch Loss= 1983.4077, Training Accuracy= 0.938\n",
      "Step 160, Minibatch Loss= 2844.6553, Training Accuracy= 0.930\n",
      "Step 170, Minibatch Loss= 3602.2524, Training Accuracy= 0.914\n",
      "Step 180, Minibatch Loss= 175.3922, Training Accuracy= 0.961\n",
      "Step 190, Minibatch Loss= 645.1918, Training Accuracy= 0.945\n",
      "Step 200, Minibatch Loss= 1147.6567, Training Accuracy= 0.938\n",
      "Step 210, Minibatch Loss= 1140.4148, Training Accuracy= 0.914\n",
      "Step 220, Minibatch Loss= 1572.8756, Training Accuracy= 0.906\n",
      "Step 230, Minibatch Loss= 1292.9274, Training Accuracy= 0.898\n",
      "Step 240, Minibatch Loss= 1501.4623, Training Accuracy= 0.953\n",
      "Step 250, Minibatch Loss= 1908.2997, Training Accuracy= 0.898\n",
      "Step 260, Minibatch Loss= 2182.2380, Training Accuracy= 0.898\n",
      "Step 270, Minibatch Loss= 487.5807, Training Accuracy= 0.961\n",
      "Step 280, Minibatch Loss= 1284.1130, Training Accuracy= 0.945\n",
      "Step 290, Minibatch Loss= 1232.4919, Training Accuracy= 0.891\n",
      "Step 300, Minibatch Loss= 1198.8336, Training Accuracy= 0.945\n",
      "Step 310, Minibatch Loss= 2010.5345, Training Accuracy= 0.906\n",
      "Step 320, Minibatch Loss= 786.3917, Training Accuracy= 0.945\n",
      "Step 330, Minibatch Loss= 1408.3556, Training Accuracy= 0.898\n",
      "Step 340, Minibatch Loss= 1453.7538, Training Accuracy= 0.953\n",
      "Step 350, Minibatch Loss= 999.8901, Training Accuracy= 0.906\n",
      "Step 360, Minibatch Loss= 914.3958, Training Accuracy= 0.961\n",
      "Step 370, Minibatch Loss= 488.0052, Training Accuracy= 0.938\n",
      "Step 380, Minibatch Loss= 1070.8710, Training Accuracy= 0.922\n",
      "Step 390, Minibatch Loss= 151.4658, Training Accuracy= 0.961\n",
      "Step 400, Minibatch Loss= 555.3539, Training Accuracy= 0.953\n",
      "Step 410, Minibatch Loss= 765.5746, Training Accuracy= 0.945\n",
      "Step 420, Minibatch Loss= 326.9393, Training Accuracy= 0.969\n",
      "Step 430, Minibatch Loss= 530.8968, Training Accuracy= 0.977\n",
      "Step 440, Minibatch Loss= 463.3909, Training Accuracy= 0.977\n",
      "Step 450, Minibatch Loss= 362.2226, Training Accuracy= 0.977\n",
      "Step 460, Minibatch Loss= 414.0034, Training Accuracy= 0.953\n",
      "Step 470, Minibatch Loss= 583.4587, Training Accuracy= 0.945\n",
      "Step 480, Minibatch Loss= 566.1262, Training Accuracy= 0.969\n",
      "Step 490, Minibatch Loss= 691.1143, Training Accuracy= 0.961\n",
      "Step 500, Minibatch Loss= 282.8893, Training Accuracy= 0.984\n",
      "Optimization Finished!\n",
      "Testing Accuracy: 0.976562\n"
     ]
    }
   ],
   "source": [
    "# Start training\n",
    "with tf.Session() as sess:\n",
    "\n",
    "    # Run the initializer\n",
    "    sess.run(init)\n",
    "\n",
    "    for step in range(1, num_steps+1):\n",
    "        batch_x, batch_y = mnist.train.next_batch(batch_size)\n",
    "        # Run optimization op (backprop)\n",
    "        sess.run(train_op, feed_dict={X: batch_x, Y: batch_y, keep_prob: dropout})\n",
    "        if step % display_step == 0 or step == 1:\n",
    "            # Calculate batch loss and accuracy\n",
    "            loss, acc = sess.run([loss_op, accuracy], feed_dict={X: batch_x,\n",
    "                                                                 Y: batch_y,\n",
    "                                                                 keep_prob: 1.0})\n",
    "            print(\"Step \" + str(step) + \", Minibatch Loss= \" + \\\n",
    "                  \"{:.4f}\".format(loss) + \", Training Accuracy= \" + \\\n",
    "                  \"{:.3f}\".format(acc))\n",
    "\n",
    "    print(\"Optimization Finished!\")\n",
    "\n",
    "    # Calculate accuracy for 256 MNIST test images\n",
    "    print(\"Testing Accuracy:\", \\\n",
    "        sess.run(accuracy, feed_dict={X: mnist.test.images[:256],\n",
    "                                      Y: mnist.test.labels[:256],\n",
    "                                      keep_prob: 1.0}))\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}