{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Convolutional Neural Network in TensorFlow\n",
    "\n",
    "Credits: Forked from [TensorFlow-Examples](https://github.com/aymericdamien/TensorFlow-Examples) by Aymeric Damien\n",
    "\n",
    "## Setup\n",
    "\n",
    "Refer to the [setup instructions](http://nbviewer.ipython.org/github/donnemartin/data-science-ipython-notebooks/blob/master/deep-learning/tensor-flow-examples/Setup_TensorFlow.md)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting /tmp/data/train-images-idx3-ubyte.gz\n",
      "Extracting /tmp/data/train-labels-idx1-ubyte.gz\n",
      "Extracting /tmp/data/t10k-images-idx3-ubyte.gz\n",
      "Extracting /tmp/data/t10k-labels-idx1-ubyte.gz\n"
     ]
    }
   ],
   "source": [
    "# Import MINST data\n",
    "import input_data\n",
    "mnist = input_data.read_data_sets(\"/tmp/data/\", one_hot=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Parameters\n",
    "learning_rate = 0.001\n",
    "training_iters = 100000\n",
    "batch_size = 128\n",
    "display_step = 20"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Network Parameters\n",
    "n_input = 784 # MNIST data input (img shape: 28*28)\n",
    "n_classes = 10 # MNIST total classes (0-9 digits)\n",
    "dropout = 0.75 # Dropout, probability to keep units"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# tf Graph input\n",
    "x = tf.placeholder(tf.float32, [None, n_input])\n",
    "y = tf.placeholder(tf.float32, [None, n_classes])\n",
    "keep_prob = tf.placeholder(tf.float32) #dropout (keep probability)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Create model\n",
    "def conv2d(img, w, b):\n",
    "    return tf.nn.relu(tf.nn.bias_add(tf.nn.conv2d(img, w, strides=[1, 1, 1, 1], \n",
    "                                                  padding='SAME'),b))\n",
    "\n",
    "def max_pool(img, k):\n",
    "    return tf.nn.max_pool(img, ksize=[1, k, k, 1], strides=[1, k, k, 1], padding='SAME')\n",
    "\n",
    "def conv_net(_X, _weights, _biases, _dropout):\n",
    "    # Reshape input picture\n",
    "    _X = tf.reshape(_X, shape=[-1, 28, 28, 1])\n",
    "\n",
    "    # Convolution Layer\n",
    "    conv1 = conv2d(_X, _weights['wc1'], _biases['bc1'])\n",
    "    # Max Pooling (down-sampling)\n",
    "    conv1 = max_pool(conv1, k=2)\n",
    "    # Apply Dropout\n",
    "    conv1 = tf.nn.dropout(conv1, _dropout)\n",
    "\n",
    "    # Convolution Layer\n",
    "    conv2 = conv2d(conv1, _weights['wc2'], _biases['bc2'])\n",
    "    # Max Pooling (down-sampling)\n",
    "    conv2 = max_pool(conv2, k=2)\n",
    "    # Apply Dropout\n",
    "    conv2 = tf.nn.dropout(conv2, _dropout)\n",
    "\n",
    "    # Fully connected layer\n",
    "    # Reshape conv2 output to fit dense layer input\n",
    "    dense1 = tf.reshape(conv2, [-1, _weights['wd1'].get_shape().as_list()[0]]) \n",
    "    # Relu activation\n",
    "    dense1 = tf.nn.relu(tf.add(tf.matmul(dense1, _weights['wd1']), _biases['bd1']))\n",
    "    # Apply Dropout\n",
    "    dense1 = tf.nn.dropout(dense1, _dropout) # Apply Dropout\n",
    "\n",
    "    # Output, class prediction\n",
    "    out = tf.add(tf.matmul(dense1, _weights['out']), _biases['out'])\n",
    "    return out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Store layers weight & bias\n",
    "weights = {\n",
    "    # 5x5 conv, 1 input, 32 outputs\n",
    "    'wc1': tf.Variable(tf.random_normal([5, 5, 1, 32])), \n",
    "    # 5x5 conv, 32 inputs, 64 outputs\n",
    "    'wc2': tf.Variable(tf.random_normal([5, 5, 32, 64])), \n",
    "    # fully connected, 7*7*64 inputs, 1024 outputs\n",
    "    'wd1': tf.Variable(tf.random_normal([7*7*64, 1024])), \n",
    "    # 1024 inputs, 10 outputs (class prediction)\n",
    "    'out': tf.Variable(tf.random_normal([1024, n_classes])) \n",
    "}\n",
    "\n",
    "biases = {\n",
    "    'bc1': tf.Variable(tf.random_normal([32])),\n",
    "    'bc2': tf.Variable(tf.random_normal([64])),\n",
    "    'bd1': tf.Variable(tf.random_normal([1024])),\n",
    "    'out': tf.Variable(tf.random_normal([n_classes]))\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Construct model\n",
    "pred = conv_net(x, weights, biases, keep_prob)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Define loss and optimizer\n",
    "cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(pred, y))\n",
    "optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(cost)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Evaluate model\n",
    "correct_pred = tf.equal(tf.argmax(pred,1), tf.argmax(y,1))\n",
    "accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Initializing the variables\n",
    "init = tf.global_variables_initializer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Iter 2560, Minibatch Loss= 26046.011719, Training Accuracy= 0.21094\n",
      "Iter 5120, Minibatch Loss= 10456.769531, Training Accuracy= 0.52344\n",
      "Iter 7680, Minibatch Loss= 6273.207520, Training Accuracy= 0.71875\n",
      "Iter 10240, Minibatch Loss= 6276.231445, Training Accuracy= 0.64062\n",
      "Iter 12800, Minibatch Loss= 4188.221680, Training Accuracy= 0.77344\n",
      "Iter 15360, Minibatch Loss= 2717.077637, Training Accuracy= 0.80469\n",
      "Iter 17920, Minibatch Loss= 4057.120361, Training Accuracy= 0.81250\n",
      "Iter 20480, Minibatch Loss= 1696.550415, Training Accuracy= 0.87500\n",
      "Iter 23040, Minibatch Loss= 2525.317627, Training Accuracy= 0.85938\n",
      "Iter 25600, Minibatch Loss= 2341.906738, Training Accuracy= 0.87500\n",
      "Iter 28160, Minibatch Loss= 4200.535156, Training Accuracy= 0.79688\n",
      "Iter 30720, Minibatch Loss= 1888.964355, Training Accuracy= 0.89062\n",
      "Iter 33280, Minibatch Loss= 2167.645996, Training Accuracy= 0.84375\n",
      "Iter 35840, Minibatch Loss= 1932.107544, Training Accuracy= 0.89844\n",
      "Iter 38400, Minibatch Loss= 1562.430054, Training Accuracy= 0.90625\n",
      "Iter 40960, Minibatch Loss= 1676.755249, Training Accuracy= 0.84375\n",
      "Iter 43520, Minibatch Loss= 1003.626099, Training Accuracy= 0.93750\n",
      "Iter 46080, Minibatch Loss= 1176.615479, Training Accuracy= 0.86719\n",
      "Iter 48640, Minibatch Loss= 1260.592651, Training Accuracy= 0.88281\n",
      "Iter 51200, Minibatch Loss= 1399.667969, Training Accuracy= 0.86719\n",
      "Iter 53760, Minibatch Loss= 1259.961426, Training Accuracy= 0.89844\n",
      "Iter 56320, Minibatch Loss= 1415.800781, Training Accuracy= 0.89062\n",
      "Iter 58880, Minibatch Loss= 1835.365967, Training Accuracy= 0.85156\n",
      "Iter 61440, Minibatch Loss= 1395.168823, Training Accuracy= 0.90625\n",
      "Iter 64000, Minibatch Loss= 973.283569, Training Accuracy= 0.88281\n",
      "Iter 66560, Minibatch Loss= 818.093811, Training Accuracy= 0.92969\n",
      "Iter 69120, Minibatch Loss= 1178.744263, Training Accuracy= 0.92188\n",
      "Iter 71680, Minibatch Loss= 845.889709, Training Accuracy= 0.89844\n",
      "Iter 74240, Minibatch Loss= 1259.505615, Training Accuracy= 0.90625\n",
      "Iter 76800, Minibatch Loss= 738.037109, Training Accuracy= 0.89844\n",
      "Iter 79360, Minibatch Loss= 862.499146, Training Accuracy= 0.93750\n",
      "Iter 81920, Minibatch Loss= 739.704041, Training Accuracy= 0.90625\n",
      "Iter 84480, Minibatch Loss= 652.880310, Training Accuracy= 0.95312\n",
      "Iter 87040, Minibatch Loss= 635.464600, Training Accuracy= 0.92969\n",
      "Iter 89600, Minibatch Loss= 933.166626, Training Accuracy= 0.90625\n",
      "Iter 92160, Minibatch Loss= 213.874893, Training Accuracy= 0.96094\n",
      "Iter 94720, Minibatch Loss= 609.575684, Training Accuracy= 0.91406\n",
      "Iter 97280, Minibatch Loss= 560.208008, Training Accuracy= 0.93750\n",
      "Iter 99840, Minibatch Loss= 963.577148, Training Accuracy= 0.90625\n",
      "Optimization Finished!\n",
      "Testing Accuracy: 0.960938\n"
     ]
    }
   ],
   "source": [
    "# Launch the graph\n",
    "with tf.Session() as sess:\n",
    "    sess.run(init)\n",
    "    step = 1\n",
    "    # Keep training until reach max iterations\n",
    "    while step * batch_size < training_iters:\n",
    "        batch_xs, batch_ys = mnist.train.next_batch(batch_size)\n",
    "        # Fit training using batch data\n",
    "        sess.run(optimizer, feed_dict={x: batch_xs, y: batch_ys, keep_prob: dropout})\n",
    "        if step % display_step == 0:\n",
    "            # Calculate batch accuracy\n",
    "            acc = sess.run(accuracy, feed_dict={x: batch_xs, y: batch_ys, keep_prob: 1.})\n",
    "            # Calculate batch loss\n",
    "            loss = sess.run(cost, feed_dict={x: batch_xs, y: batch_ys, keep_prob: 1.})\n",
    "            print \"Iter \" + str(step*batch_size) + \", Minibatch Loss= \" + \\\n",
    "                  \"{:.6f}\".format(loss) + \", Training Accuracy= \" + \"{:.5f}\".format(acc)\n",
    "        step += 1\n",
    "    print \"Optimization Finished!\"\n",
    "    # Calculate accuracy for 256 mnist test images\n",
    "    print \"Testing Accuracy:\", sess.run(accuracy, feed_dict={x: mnist.test.images[:256], \n",
    "                                                             y: mnist.test.labels[:256], \n",
    "                                                             keep_prob: 1.})"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}