{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "mnist_cnn.ipynb",
      "version": "0.3.2",
      "views": {},
      "default_view": {},
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "metadata": {
        "id": "0MugGWQcv9rx",
        "colab_type": "code",
        "colab": {
          "autoexec": {
            "startup": false,
            "wait_interval": 0
          },
          "base_uri": "https://localhost:8080/",
          "height": 85
        },
        "outputId": "b6fca165-d7c4-4207-9ed3-d1f67405272a",
        "executionInfo": {
          "status": "ok",
          "timestamp": 1523173555511,
          "user_tz": 420,
          "elapsed": 2619,
          "user": {
            "displayName": "Arun Rajendran",
            "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s128",
            "userId": "100710189298922321078"
          }
        }
      },
      "cell_type": "code",
      "source": [
        "import sys\n",
        "import warnings\n",
        "\n",
        "if not sys.warnoptions:\n",
        "    warnings.simplefilter(\"ignore\")\n",
        "    \n",
        "from tensorflow.examples.tutorials.mnist import input_data\n",
        "import numpy as np\n",
        "mnist = input_data.read_data_sets(\".\", one_hot=True, reshape=False)\n",
        "\n",
        "import tensorflow as tf\n",
        "\n",
        "#tf.set_random_seed(2)\n",
        "\n",
        "# parameters\n",
        "learning_rate = 0.00001\n",
        "epochs = 1000\n",
        "batch_size = 128\n",
        "\n",
        "# number of samples to calculate validation and accuracy\n",
        "# decrease this if you're running out of memory\n",
        "test_valid_size = 256\n",
        "\n",
        "# network Parameters\n",
        "n_classes = 10  # MNIST total classes (0-9 digits)\n",
        "dropout = 0.5  # dropout (probability to keep units)"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Extracting ./train-images-idx3-ubyte.gz\n",
            "Extracting ./train-labels-idx1-ubyte.gz\n",
            "Extracting ./t10k-images-idx3-ubyte.gz\n",
            "Extracting ./t10k-labels-idx1-ubyte.gz\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "00kNYODzv9w9",
        "colab_type": "code",
        "colab": {
          "autoexec": {
            "startup": false,
            "wait_interval": 0
          }
        }
      },
      "cell_type": "code",
      "source": [
        "# store weights & biases\n",
        "weights = {\n",
        "    'wc1': tf.Variable(tf.truncated_normal([5, 5, 1, 32], stddev=0.1)),\n",
        "    'wc2': tf.Variable(tf.truncated_normal([5, 5, 32, 64], stddev=0.1)),\n",
        "    'wd1': tf.Variable(tf.truncated_normal([7*7*64, 1024], stddev=0.1)),\n",
        "    'out': tf.Variable(tf.truncated_normal([1024, n_classes], stddev=0.1))}\n",
        "\n",
        "biases = {\n",
        "    'bc1': tf.Variable(tf.constant(0.1,shape=[32])),\n",
        "    'bc2': tf.Variable(tf.constant(0.1,shape=[64])),\n",
        "    'bd1': tf.Variable(tf.constant(0.1,shape=[1024])),\n",
        "    'out': tf.Variable(tf.constant(0.1,shape=[n_classes]))}"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "rsUuejCCv9y6",
        "colab_type": "code",
        "colab": {
          "autoexec": {
            "startup": false,
            "wait_interval": 0
          }
        }
      },
      "cell_type": "code",
      "source": [
        "def conv2d(x, W, b, strides=1):\n",
        "    x = tf.nn.conv2d(x, W, strides=[1, strides, strides, 1], padding='SAME')\n",
        "    x = tf.nn.bias_add(x, b)\n",
        "    return tf.nn.relu(x)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "8bqTgO_rv91Z",
        "colab_type": "code",
        "colab": {
          "autoexec": {
            "startup": false,
            "wait_interval": 0
          }
        }
      },
      "cell_type": "code",
      "source": [
        "def maxpool2d(x, k=2):\n",
        "    return tf.nn.max_pool(\n",
        "        x,\n",
        "        ksize=[1, k, k, 1],\n",
        "        strides=[1, k, k, 1],\n",
        "        padding='SAME')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "A7rFHNHev93_",
        "colab_type": "code",
        "colab": {
          "autoexec": {
            "startup": false,
            "wait_interval": 0
          }
        }
      },
      "cell_type": "code",
      "source": [
        "def conv_net(x, weights, biases, dropout):\n",
        "    # Layer 1 - 28*28*1 to 14*14*32\n",
        "    conv1 = conv2d(x, weights['wc1'], biases['bc1'])\n",
        "    conv1 = maxpool2d(conv1, k=2)\n",
        "\n",
        "    # Layer 2 - 14*14*32 to 7*7*64\n",
        "    conv2 = conv2d(conv1, weights['wc2'], biases['bc2'])\n",
        "    conv2 = maxpool2d(conv2, k=2)\n",
        "\n",
        "    # Fully connected layer - 7*7*64 to 1024\n",
        "    fc1 = tf.reshape(conv2, [-1, weights['wd1'].get_shape().as_list()[0]])\n",
        "    fc1 = tf.add(tf.matmul(fc1, weights['wd1']), biases['bd1'])\n",
        "    fc1 = tf.nn.relu(fc1)\n",
        "    fc1 = tf.nn.dropout(fc1, dropout)\n",
        "\n",
        "    # Output Layer - class prediction - 1024 to 10\n",
        "    out = tf.add(tf.matmul(fc1, weights['out']), biases['out'])\n",
        "    return out"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "CGyL5lZXv96t",
        "colab_type": "code",
        "colab": {
          "autoexec": {
            "startup": false,
            "wait_interval": 0
          },
          "base_uri": "https://localhost:8080/",
          "height": 374
        },
        "outputId": "8a193712-8611-4913-c7c9-ecf67222551e",
        "executionInfo": {
          "status": "ok",
          "timestamp": 1523185952334,
          "user_tz": 420,
          "elapsed": 12392081,
          "user": {
            "displayName": "Arun Rajendran",
            "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s128",
            "userId": "100710189298922321078"
          }
        }
      },
      "cell_type": "code",
      "source": [
        "# tf Graph input\n",
        "x = tf.placeholder(tf.float32, [None, 28, 28, 1])\n",
        "y = tf.placeholder(tf.float32, [None, n_classes])\n",
        "keep_prob = tf.placeholder(tf.float32)\n",
        "\n",
        "# Model\n",
        "logits = conv_net(x, weights, biases, keep_prob)\n",
        "\n",
        "# Define loss and optimizer\n",
        "cost = tf.reduce_mean(\\\n",
        "    tf.nn.softmax_cross_entropy_with_logits_v2(logits=logits, labels=y))\n",
        "optimizer = tf.train.GradientDescentOptimizer(learning_rate=learning_rate)\\\n",
        "    .minimize(cost)\n",
        "\n",
        "# Accuracy\n",
        "correct_pred = tf.equal(tf.argmax(logits, 1), tf.argmax(y, 1))\n",
        "accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))\n",
        "\n",
        "# Initializing the variables\n",
        "init = tf. global_variables_initializer()\n",
        "\n",
        "# Launch the graph\n",
        "with tf.Session() as sess:\n",
        "    sess.run(init)\n",
        "\n",
        "    for epoch in range(epochs):\n",
        "        train_accuracies = []\n",
        "        val_accuracies = []\n",
        "        for batch in range(mnist.train.num_examples//batch_size):\n",
        "            batch_x, batch_y = mnist.train.next_batch(batch_size)\n",
        "            feed_dict={\n",
        "                x: batch_x,\n",
        "                y: batch_y,\n",
        "                keep_prob: dropout}\n",
        "            _,acc = sess.run([optimizer,accuracy], feed_dict=feed_dict)\n",
        "            train_accuracies.append(acc)\n",
        "            \n",
        "            # Calculate batch loss and accuracy\n",
        "            loss = sess.run(cost, feed_dict={\n",
        "                x: batch_x,\n",
        "                y: batch_y,\n",
        "                keep_prob: 1.})\n",
        "            valid_acc = sess.run(accuracy, feed_dict={\n",
        "                x: mnist.validation.images[:test_valid_size],\n",
        "                y: mnist.validation.labels[:test_valid_size],\n",
        "                keep_prob: 1.})\n",
        "            val_accuracies.append(valid_acc)\n",
        "        train_acc_mean=np.mean(train_accuracies)\n",
        "        val_acc_mean=np.mean(val_accuracies)\n",
        "        if( (epoch+1) % 50 == 0):\n",
        "            print('Epoch {:>2}, '\n",
        "                  'Loss: {:>10.4f} Training Accuracy:{:.6f}, Validation Accuracy: {:.6f}'.format(epoch + 1,loss,train_acc_mean,val_acc_mean))\n",
        "\n",
        "    # Calculate Test Accuracy\n",
        "    test_acc = sess.run(accuracy, feed_dict={\n",
        "        x: mnist.test.images[:test_valid_size],\n",
        "        y: mnist.test.labels[:test_valid_size],\n",
        "        keep_prob: 1.})\n",
        "    print('Testing Accuracy: {}'.format(test_acc))"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch 50, Loss:     1.2460 Training Accuracy:0.339798, Validation Accuracy: 0.636364\n",
            "Epoch 100, Loss:     1.0122 Training Accuracy:0.513822, Validation Accuracy: 0.753906\n",
            "Epoch 150, Loss:     0.7675 Training Accuracy:0.617315, Validation Accuracy: 0.820312\n",
            "Epoch 200, Loss:     0.6523 Training Accuracy:0.679815, Validation Accuracy: 0.848312\n",
            "Epoch 250, Loss:     0.4243 Training Accuracy:0.727309, Validation Accuracy: 0.886719\n",
            "Epoch 300, Loss:     0.4262 Training Accuracy:0.761546, Validation Accuracy: 0.898438\n",
            "Epoch 350, Loss:     0.3318 Training Accuracy:0.784328, Validation Accuracy: 0.902344\n",
            "Epoch 400, Loss:     0.3039 Training Accuracy:0.806181, Validation Accuracy: 0.902344\n",
            "Epoch 450, Loss:     0.3382 Training Accuracy:0.822024, Validation Accuracy: 0.914062\n",
            "Epoch 500, Loss:     0.3123 Training Accuracy:0.833515, Validation Accuracy: 0.917969\n",
            "Epoch 550, Loss:     0.2461 Training Accuracy:0.845061, Validation Accuracy: 0.917969\n",
            "Epoch 600, Loss:     0.3075 Training Accuracy:0.849523, Validation Accuracy: 0.917969\n",
            "Epoch 650, Loss:     0.3382 Training Accuracy:0.859958, Validation Accuracy: 0.921875\n",
            "Epoch 700, Loss:     0.2209 Training Accuracy:0.867461, Validation Accuracy: 0.925781\n",
            "Epoch 750, Loss:     0.2217 Training Accuracy:0.870994, Validation Accuracy: 0.925781\n",
            "Epoch 800, Loss:     0.2504 Training Accuracy:0.876475, Validation Accuracy: 0.925781\n",
            "Epoch 850, Loss:     0.3488 Training Accuracy:0.882230, Validation Accuracy: 0.925781\n",
            "Epoch 900, Loss:     0.1096 Training Accuracy:0.887201, Validation Accuracy: 0.931782\n",
            "Epoch 950, Loss:     0.1928 Training Accuracy:0.890024, Validation Accuracy: 0.933594\n",
            "Epoch 1000, Loss:     0.1055 Training Accuracy:0.894595, Validation Accuracy: 0.933594\n",
            "Testing Accuracy: 0.96875\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}