{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# A Network Tour of Data Science\n",
    "###       Xavier Bresson, Winter 2016/17\n",
    "## Exercise 4 : Introduction to TensorFlow"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Import libraries\n",
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "import time\n",
    "import collections\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting datasets/mnist/train-images-idx3-ubyte.gz\n",
      "Extracting datasets/mnist/train-labels-idx1-ubyte.gz\n",
      "Extracting datasets/mnist/t10k-images-idx3-ubyte.gz\n",
      "Extracting datasets/mnist/t10k-labels-idx1-ubyte.gz\n",
      "(55000, 784)\n",
      "(55000, 10)\n",
      "(10000, 784)\n",
      "(10000, 10)\n"
     ]
    }
   ],
   "source": [
    "# Import MNIST data with TensorFlow\n",
    "from tensorflow.examples.tutorials.mnist import input_data\n",
    "mnist = input_data.read_data_sets(os.path.join('datasets', 'mnist'), one_hot=True) # load data in local folder\n",
    "\n",
    "train_data = mnist.train.images.astype(np.float32)\n",
    "train_labels = mnist.train.labels\n",
    "\n",
    "test_data = mnist.test.images.astype(np.float32)\n",
    "test_labels = mnist.test.labels\n",
    "\n",
    "print(train_data.shape)\n",
    "print(train_labels.shape)\n",
    "print(test_data.shape)\n",
    "print(test_labels.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1st Step: Construct Computational Graph"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Question 1: Prepare the input variables (x,y_label) of the computational graph\n",
    "\n",
    "Hint: You may use the function *tf.placeholder()*"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "x= Tensor(\"Placeholder:0\", shape=(100, 784), dtype=float32) (100, 784)\n",
      "y_label= Tensor(\"Placeholder_1:0\", shape=(100, 10), dtype=float32) (100, 10)\n"
     ]
    }
   ],
   "source": [
    "# computational graph inputs\n",
    "batch_size = 100\n",
    "d = train_data.shape[1]\n",
    "nc = 10\n",
    "x = tf.placeholder(tf.float32,[batch_size,d]); print('x=',x,x.get_shape())\n",
    "y_label = tf.placeholder(tf.float32,[batch_size,nc]); print('y_label=',y_label,y_label.get_shape())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Question 2: Prepare the variables (W,b) of the computational graph\n",
    "\n",
    "Hint: You may use the function *tf.Variable(), tf.truncated_normal()*"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "W= (784, 10)\n",
      "b= (10,)\n"
     ]
    }
   ],
   "source": [
    "# computational graph variables\n",
    "initial = tf.truncated_normal([d,nc], stddev=0.1); W = tf.Variable(initial); print('W=',W.get_shape())\n",
    "b = tf.Variable(tf.zeros([nc],tf.float32)); print('b=',b.get_shape())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Question 3: Compute the classifier such that\n",
    "$$\n",
    "y=softmax(Wx +b)\n",
    "$$\n",
    "\n",
    "Hint: You may use the function *tf.matmul(), tf.nn.softmax()*"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "y1= Tensor(\"MatMul:0\", shape=(100, 10), dtype=float32) (100, 10)\n",
      "y2= Tensor(\"add:0\", shape=(100, 10), dtype=float32) (100, 10)\n",
      "y3= Tensor(\"Softmax:0\", shape=(100, 10), dtype=float32) (100, 10)\n"
     ]
    }
   ],
   "source": [
    "# Construct CG / output value\n",
    "y = tf.matmul(x, W); print('y1=',y,y.get_shape())\n",
    "y += b; print('y2=',y,y.get_shape())\n",
    "y = tf.nn.softmax(y); print('y3=',y,y.get_shape())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Question 4: Construct the loss of the computational graph such that\n",
    "$$\n",
    "loss = cross\\ entropy(y_{label},y) =  mean_{all\\ data} \\ \\sum_{all\\ classes} -\\ y_{label}.\\log(y)\n",
    "$$\n",
    "\n",
    "Hint: You may use the function *tf.Variable(), tf.truncated_normal()*"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Loss\n",
    "cross_entropy = tf.reduce_mean(-tf.reduce_sum(y_label * tf.log(y), 1))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Question 5: Construct the L2 regularization of (W,b) to the computational graph such that\n",
    "$$\n",
    "R(W) = \\|W\\|_2^2\\\\\n",
    "R(b) = \\|b\\|_2^2\n",
    "$$\n",
    "\n",
    "Hint: You may use the function *tf.nn.l2_loss()*"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "reg_loss = tf.nn.l2_loss(W)\n",
    "reg_loss += tf.nn.l2_loss(b)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Question 6: Form the total loss\n",
    "$$\n",
    "total\\ loss = cross\\ entropy(y_{label},y) + reg\\_par* (R(W) + R(b))\n",
    "$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "reg_par = 1e-3\n",
    "total_loss = cross_entropy + reg_par* reg_loss"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Question 7: Perform optimization of the total loss for learning weight variables of the computational graph\n",
    "\n",
    "Hint: You may use the function *tf.train.GradientDescentOptimizer(learning_rate).minimize(total_loss)*"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Update CG variables / backward pass\n",
    "train_step = tf.train.GradientDescentOptimizer(0.25).minimize(total_loss)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Question 8: Evaluate the accuracy\n",
    "\n",
    "Hint: You may use the function *tf.equal(tf.argmax(y,1), tf.argmax(y_label,1))* and *tf.reduce_mean()*"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Accuracy\n",
    "correct_prediction = tf.equal(tf.argmax(y,1), tf.argmax(y_label,1))\n",
    "accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2nd Step: Run the Computational Graph with batches of training data\n",
    "Check out the accuracy of test set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Create test set \n",
    "idx = np.random.permutation(test_data.shape[0]) # rand permutation\n",
    "idx = idx[:batch_size]\n",
    "test_x, test_y = test_data[idx,:], test_labels[idx]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Iteration i= 0 , train accuracy= 0.1 , loss= 2.61576\n",
      "test accuracy= 0.16\n",
      "\n",
      "Iteration i= 1 , train accuracy= 0.12 , loss= 2.25159\n",
      "test accuracy= 0.33\n",
      "\n",
      "Iteration i= 2 , train accuracy= 0.3 , loss= 2.16757\n",
      "test accuracy= 0.36\n",
      "\n",
      "Iteration i= 3 , train accuracy= 0.38 , loss= 1.88768\n",
      "test accuracy= 0.51\n",
      "\n",
      "Iteration i= 4 , train accuracy= 0.5 , loss= 1.70732\n",
      "test accuracy= 0.55\n",
      "\n",
      "Iteration i= 5 , train accuracy= 0.59 , loss= 1.52506\n",
      "test accuracy= 0.62\n",
      "\n",
      "Iteration i= 6 , train accuracy= 0.53 , loss= 1.57437\n",
      "test accuracy= 0.68\n",
      "\n",
      "Iteration i= 7 , train accuracy= 0.62 , loss= 1.34377\n",
      "test accuracy= 0.69\n",
      "\n",
      "Iteration i= 8 , train accuracy= 0.64 , loss= 1.36525\n",
      "test accuracy= 0.72\n",
      "\n",
      "Iteration i= 9 , train accuracy= 0.68 , loss= 1.25514\n",
      "test accuracy= 0.72\n",
      "\n",
      "Iteration i= 10 , train accuracy= 0.63 , loss= 1.22931\n",
      "test accuracy= 0.68\n",
      "\n",
      "Iteration i= 11 , train accuracy= 0.66 , loss= 1.17098\n",
      "test accuracy= 0.74\n",
      "\n",
      "Iteration i= 12 , train accuracy= 0.68 , loss= 1.24049\n",
      "test accuracy= 0.75\n",
      "\n",
      "Iteration i= 13 , train accuracy= 0.71 , loss= 1.10969\n",
      "test accuracy= 0.84\n",
      "\n",
      "Iteration i= 14 , train accuracy= 0.86 , loss= 0.891327\n",
      "test accuracy= 0.79\n",
      "\n",
      "Iteration i= 15 , train accuracy= 0.78 , loss= 0.980348\n",
      "test accuracy= 0.76\n",
      "\n",
      "Iteration i= 16 , train accuracy= 0.74 , loss= 1.00763\n",
      "test accuracy= 0.79\n",
      "\n",
      "Iteration i= 17 , train accuracy= 0.75 , loss= 0.942338\n",
      "test accuracy= 0.83\n",
      "\n",
      "Iteration i= 18 , train accuracy= 0.84 , loss= 0.807404\n",
      "test accuracy= 0.83\n",
      "\n",
      "Iteration i= 19 , train accuracy= 0.82 , loss= 0.828357\n",
      "test accuracy= 0.86\n",
      "\n",
      "Iteration i= 20 , train accuracy= 0.82 , loss= 0.817\n",
      "test accuracy= 0.83\n",
      "\n",
      "Iteration i= 21 , train accuracy= 0.78 , loss= 0.844171\n",
      "test accuracy= 0.86\n",
      "\n",
      "Iteration i= 22 , train accuracy= 0.81 , loss= 0.837174\n",
      "test accuracy= 0.85\n",
      "\n",
      "Iteration i= 23 , train accuracy= 0.78 , loss= 0.974055\n",
      "test accuracy= 0.81\n",
      "\n",
      "Iteration i= 24 , train accuracy= 0.81 , loss= 0.809938\n",
      "test accuracy= 0.85\n",
      "\n",
      "Iteration i= 25 , train accuracy= 0.9 , loss= 0.642673\n",
      "test accuracy= 0.85\n",
      "\n",
      "Iteration i= 26 , train accuracy= 0.74 , loss= 0.901251\n",
      "test accuracy= 0.87\n",
      "\n",
      "Iteration i= 27 , train accuracy= 0.71 , loss= 1.01265\n",
      "test accuracy= 0.85\n",
      "\n",
      "Iteration i= 28 , train accuracy= 0.83 , loss= 0.750725\n",
      "test accuracy= 0.86\n",
      "\n",
      "Iteration i= 29 , train accuracy= 0.85 , loss= 0.667404\n",
      "test accuracy= 0.88\n",
      "\n",
      "Iteration i= 30 , train accuracy= 0.8 , loss= 0.719768\n",
      "test accuracy= 0.81\n",
      "\n",
      "Iteration i= 31 , train accuracy= 0.82 , loss= 0.680213\n",
      "test accuracy= 0.84\n",
      "\n",
      "Iteration i= 32 , train accuracy= 0.75 , loss= 0.878949\n",
      "test accuracy= 0.85\n",
      "\n",
      "Iteration i= 33 , train accuracy= 0.8 , loss= 0.787001\n",
      "test accuracy= 0.84\n",
      "\n",
      "Iteration i= 34 , train accuracy= 0.87 , loss= 0.620717\n",
      "test accuracy= 0.87\n",
      "\n",
      "Iteration i= 35 , train accuracy= 0.8 , loss= 0.723095\n",
      "test accuracy= 0.83\n",
      "\n",
      "Iteration i= 36 , train accuracy= 0.83 , loss= 0.719377\n",
      "test accuracy= 0.85\n",
      "\n",
      "Iteration i= 37 , train accuracy= 0.8 , loss= 0.716804\n",
      "test accuracy= 0.84\n",
      "\n",
      "Iteration i= 38 , train accuracy= 0.88 , loss= 0.591883\n",
      "test accuracy= 0.85\n",
      "\n",
      "Iteration i= 39 , train accuracy= 0.82 , loss= 0.74946\n",
      "test accuracy= 0.86\n",
      "\n",
      "Iteration i= 40 , train accuracy= 0.84 , loss= 0.657294\n",
      "test accuracy= 0.86\n",
      "\n",
      "Iteration i= 41 , train accuracy= 0.83 , loss= 0.613971\n",
      "test accuracy= 0.84\n",
      "\n",
      "Iteration i= 42 , train accuracy= 0.86 , loss= 0.646873\n",
      "test accuracy= 0.86\n",
      "\n",
      "Iteration i= 43 , train accuracy= 0.9 , loss= 0.529555\n",
      "test accuracy= 0.84\n",
      "\n",
      "Iteration i= 44 , train accuracy= 0.83 , loss= 0.648258\n",
      "test accuracy= 0.85\n",
      "\n",
      "Iteration i= 45 , train accuracy= 0.83 , loss= 0.608415\n",
      "test accuracy= 0.85\n",
      "\n",
      "Iteration i= 46 , train accuracy= 0.89 , loss= 0.620664\n",
      "test accuracy= 0.85\n",
      "\n",
      "Iteration i= 47 , train accuracy= 0.83 , loss= 0.653725\n",
      "test accuracy= 0.85\n",
      "\n",
      "Iteration i= 48 , train accuracy= 0.83 , loss= 0.686351\n",
      "test accuracy= 0.85\n",
      "\n",
      "Iteration i= 49 , train accuracy= 0.84 , loss= 0.608361\n",
      "test accuracy= 0.85\n"
     ]
    }
   ],
   "source": [
    "n = train_data.shape[0]\n",
    "indices = collections.deque()\n",
    "\n",
    "# Running Computational Graph\n",
    "init = tf.initialize_all_variables()\n",
    "sess = tf.Session()\n",
    "sess.run(init)\n",
    "for i in range(50):\n",
    "    \n",
    "    # Batch extraction\n",
    "    if len(indices) < batch_size:\n",
    "        indices.extend(np.random.permutation(n)) # rand permutation\n",
    "    idx = [indices.popleft() for i in range(batch_size)] # extract n_batch data\n",
    "    batch_x, batch_y = train_data[idx,:], train_labels[idx]\n",
    "    \n",
    "    # Run CG for variable training\n",
    "    _,acc_train,total_loss_o = sess.run([train_step,accuracy,total_loss], feed_dict={x: batch_x, y_label: batch_y})\n",
    "    print('\\nIteration i=',i,', train accuracy=',acc_train,', loss=',total_loss_o)\n",
    "    \n",
    "    # Run CG for testset\n",
    "    acc_test = sess.run(accuracy, feed_dict={x: test_x, y_label: test_y})\n",
    "    print('test accuracy=',acc_test)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}