{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# TensorFlow Workflow\n",
    "\n",
    "after: https://www.tensorflow.org/versions/r0.12/get_started/index.html\n",
    "\n",
    "_Just to get a feeling for the basic code layout_ ..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Create a dataset that is perfect for linear regression ;-)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(20, 2)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([[ -2.27427525e-01,  -1.88098182e+02],\n",
       "       [  5.20758979e+00,   2.70275824e+00],\n",
       "       [ -1.01162704e+00,  -4.41401868e+00],\n",
       "       [ -4.25334323e+00,  -6.72107608e+01],\n",
       "       [ -9.56918746e+00,   6.15759214e+01],\n",
       "       [ -1.75206095e+00,  -3.32513814e+01],\n",
       "       [ -9.93876659e+00,  -9.52794991e+00],\n",
       "       [  8.08328957e+00,   1.63652934e+02],\n",
       "       [  9.42527934e+00,  -2.19506162e+01],\n",
       "       [  8.98195920e+00,   1.40256440e+02],\n",
       "       [ -5.44693058e+00,   4.09508713e+01],\n",
       "       [  4.06423553e+00,  -1.34027047e+02],\n",
       "       [ -7.73125119e+00,  -1.14416933e+02],\n",
       "       [  2.50624979e+00,  -1.46120478e+02],\n",
       "       [ -7.30325605e+00,  -4.73212155e+01],\n",
       "       [ -2.64987532e+00,   6.00638338e+01],\n",
       "       [ -5.01904881e+00,   1.08949820e+02],\n",
       "       [ -7.23765976e-02,   8.06592591e+01],\n",
       "       [ -6.43722239e+00,  -5.61475588e+01],\n",
       "       [  4.75940242e+00,  -1.36225077e+02]])"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# X has 20 rows and 2 columns\n",
    "X_data = np.random.uniform(-10,10,(20,2))\n",
    "X_data[:,1] = X_data[:,1] * 20 + 2\n",
    "print(X_data.shape)\n",
    "X_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(20,)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([ 217.41589985,   42.92001113,   31.37913754,   84.45073117,\n",
       "        -60.28348374,   57.99519853,    9.71165013, -109.40306502,\n",
       "         80.22645418,  -83.31056283,  -27.29166308,  176.21975389,\n",
       "        121.22317906,  183.63922757,   55.41144738,  -38.01345978,\n",
       "        -94.00696616,  -50.87638893,   66.83589158,  180.5032847 ])"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Our y happens to be perfectly correlated...\n",
    "y_data = 30 + 3 * X_data[:,0] - X_data[:,1]\n",
    "print(y_data.shape)\n",
    "y_data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### First, define the whole computational graph.\n",
    "\n",
    "Declare placeholders for the data:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(<tf.Tensor 'Placeholder:0' shape=(?, 2) dtype=float64>,\n",
       " <tf.Tensor 'Placeholder_1:0' shape=(?,) dtype=float64>)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X = tf.placeholder(tf.float64, [None, 2])\n",
    "y = tf.placeholder(tf.float64, [None])\n",
    "X, y"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Declare a weight vector and the bias, a scalar:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(TensorShape([Dimension(2), Dimension(1)]), TensorShape([Dimension(1)]))"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# the weights will be randomly initialized to small values\n",
    "W = tf.Variable(tf.truncated_normal([2,1], stddev=1,dtype='float64'))\n",
    "b = tf.Variable(tf.zeros([1.0], dtype='float64'))\n",
    "W.get_shape(), b.get_shape()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "If we now tried to evaluate W, we'd get an error, as we don't have created a Session yet:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "ename": "ValueError",
     "evalue": "Cannot evaluate tensor using `eval()`: No default session is registered. Use `with sess.as_default()` or pass an explicit session to `eval(session=sess)`",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-6-c380b204c25c>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mW\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0meval\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m/home/key/python/anaconda2/envs/tensorflow2/lib/python3.5/site-packages/tensorflow/python/ops/variables.py\u001b[0m in \u001b[0;36meval\u001b[0;34m(self, session)\u001b[0m\n\u001b[1;32m    443\u001b[0m       \u001b[0mA\u001b[0m \u001b[0mnumpy\u001b[0m\u001b[0;31m \u001b[0m\u001b[0;31m`\u001b[0m\u001b[0mndarray\u001b[0m\u001b[0;31m`\u001b[0m \u001b[0;32mwith\u001b[0m \u001b[0ma\u001b[0m \u001b[0mcopy\u001b[0m \u001b[0mof\u001b[0m \u001b[0mthe\u001b[0m \u001b[0mvalue\u001b[0m \u001b[0mof\u001b[0m \u001b[0mthis\u001b[0m \u001b[0mvariable\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    444\u001b[0m     \"\"\"\n\u001b[0;32m--> 445\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_variable\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0meval\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msession\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0msession\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    446\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    447\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0minitialized_value\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/key/python/anaconda2/envs/tensorflow2/lib/python3.5/site-packages/tensorflow/python/framework/ops.py\u001b[0m in \u001b[0;36meval\u001b[0;34m(self, feed_dict, session)\u001b[0m\n\u001b[1;32m    557\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    558\u001b[0m     \"\"\"\n\u001b[0;32m--> 559\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0m_eval_using_default_session\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgraph\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msession\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    560\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    561\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/key/python/anaconda2/envs/tensorflow2/lib/python3.5/site-packages/tensorflow/python/framework/ops.py\u001b[0m in \u001b[0;36m_eval_using_default_session\u001b[0;34m(tensors, feed_dict, graph, session)\u001b[0m\n\u001b[1;32m   3745\u001b[0m     \u001b[0msession\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mget_default_session\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3746\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0msession\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 3747\u001b[0;31m       raise ValueError(\"Cannot evaluate tensor using `eval()`: No default \"\n\u001b[0m\u001b[1;32m   3748\u001b[0m                        \u001b[0;34m\"session is registered. Use `with \"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3749\u001b[0m                        \u001b[0;34m\"sess.as_default()` or pass an explicit session to \"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mValueError\u001b[0m: Cannot evaluate tensor using `eval()`: No default session is registered. Use `with sess.as_default()` or pass an explicit session to `eval(session=sess)`"
     ]
    }
   ],
   "source": [
    "W.eval()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We need to define the equation for predicting y from X:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor 'Add:0' shape=(?, 1) dtype=float64>"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Construct a linear model\n",
    "y_hat = tf.add(tf.matmul(X, W), b)\n",
    "y_hat"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now, we need to define our loss function, choosing MSE."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor 'Mean:0' shape=<unknown> dtype=float64>"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "loss = tf.reduce_mean(tf.square(y - tf.squeeze(y_hat)))\n",
    "loss"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We need to choose an optimization method and a learning rate:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#optimizer = tf.train.GradientDescentOptimizer(1e-5)\n",
    "optimizer = tf.train.AdamOptimizer(1e-2)\n",
    "train = optimizer.minimize(loss)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Lastly, before executing the computation, define a graph node to initialize the variables:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "init = tf.initialize_all_variables()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Now we can run the computation:\n",
    "\n",
    "First, create a TensorFlow Session to execute the graph:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sess = tf.Session()\n",
    "sess.run(init)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's have a look at the initialized weights vector and the bias:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 0.85059504],\n",
       "       [ 0.6539813 ]])"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "W.eval(sess)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 0.])"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "b.eval(sess)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now, train the model:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 0: W = [[ 0.84059504]\n",
      " [ 0.6439813 ]], b = [ 0.01], loss = 27866.63059475203\n",
      "Step 1000: W = [[ 2.50593987]\n",
      " [-1.03109144]], b = [ 8.07585589], loss = 460.6994656903801\n",
      "Step 2000: W = [[ 2.67625564]\n",
      " [-1.02035829]], b = [ 15.59202743], loss = 198.96517221566342\n",
      "Step 3000: W = [[ 2.81728831]\n",
      " [-1.01148616]], b = [ 21.8586233], loss = 63.52821889884941\n",
      "Step 4000: W = [[ 2.92038501]\n",
      " [-1.00500435]], b = [ 26.45060758], loss = 12.07477075081085\n",
      "Step 5000: W = [[ 2.97831337]\n",
      " [-1.0013631 ]], b = [ 29.03301844], loss = 0.8962046788134147\n",
      "Step 6000: W = [[ 2.99744354]\n",
      " [-1.00016068]], b = [ 29.88600818], loss = 0.012454266970805195\n",
      "Step 7000: W = [[ 2.99992855]\n",
      " [-1.00000449]], b = [ 29.99681428], loss = 9.727156840375604e-06\n",
      "Step 8000: W = [[ 2.99999983]\n",
      " [-1.00000001]], b = [ 29.99999251], loss = 5.379535438319684e-11\n",
      "Step 9000: W = [[ 3.]\n",
      " [-1.]], b = [ 30.], loss = 4.4295179684599636e-20\n"
     ]
    }
   ],
   "source": [
    "for i in range(10000):\n",
    "#for i in range(100000):\n",
    "    sess.run(train, feed_dict={X:X_data, y:y_data})\n",
    "    if i % 1000 == 0:\n",
    "    #if i % 1000 == 0:    \n",
    "        print('Step {}: W = {}, b = {}, loss = {}'.format(i,\n",
    "                sess.run(W), sess.run(b), sess.run(loss, feed_dict={X:X_data, y:y_data})))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [conda env:tensorflow2]",
   "language": "python",
   "name": "conda-env-tensorflow2-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}