{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "![book](https://raw.githubusercontent.com/ageron/tensorflow-safari-course/master/images/intro_to_tf_course.png)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Try not to peek at the solutions when you go through the exercises. ;-)**"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "First let's make sure this notebook works well in both Python 2 and Python 3:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from __future__ import absolute_import, division, print_function, unicode_literals"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "tf.__version__"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "*__From notebook 4 linear regression__*"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "\n",
    "data = np.loadtxt(\"data/life_satisfaction.csv\",\n",
    "                  dtype=np.float32,\n",
    "                  delimiter=\",\",\n",
    "                  skiprows=1,\n",
    "                  usecols=[1, 2])\n",
    "X_train = data[:, 0:1] / 10000 # feature scaling\n",
    "y_train = data[:, 1:2]\n",
    "\n",
    "learning_rate = 0.01"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "plt.rcParams['axes.labelsize'] = 14\n",
    "plt.rcParams['xtick.labelsize'] = 12\n",
    "plt.rcParams['ytick.labelsize'] = 12"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_life_satisfaction(X_train, y_train):\n",
    "    plt.plot(X_train * 10000, y_train, \"bo\")\n",
    "    plt.axis([0, 60000, 0, 10])\n",
    "    plt.xlabel(\"GDP per capita ($)\")\n",
    "    plt.ylabel(\"Life Satisfaction\")\n",
    "    plt.grid()\n",
    "\n",
    "def plot_life_satisfaction_with_linear_model(X_train, y_train, w, b):\n",
    "    plot_life_satisfaction(X_train, y_train)\n",
    "    plt.plot([0, 60000], [b, w[0][0] * (60000 / 10000) + b])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Using autodiff Instead"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "graph = tf.Graph()\n",
    "with graph.as_default():\n",
    "    X = tf.constant(X_train, dtype=tf.float32, name=\"X\")\n",
    "    y = tf.constant(y_train, dtype=tf.float32, name=\"y\")\n",
    "\n",
    "    b = tf.Variable(0.0, name=\"b\")\n",
    "    w = tf.Variable(tf.zeros([1, 1]), name=\"w\")\n",
    "    y_pred = tf.add(tf.matmul(X, w), b, name=\"y_pred\")  # X @ w + b\n",
    "    \n",
    "    mse = tf.reduce_mean(tf.square(y_pred - y), name=\"mse\")\n",
    "\n",
    "    gradients_w, gradients_b = tf.gradients(mse, [w, b])  # <= IT'S AUTODIFF MAGIC!\n",
    "\n",
    "    tweak_w_op = tf.assign(w, w - learning_rate * gradients_w)\n",
    "    tweak_b_op = tf.assign(b, b - learning_rate * gradients_b)\n",
    "    training_op = tf.group(tweak_w_op, tweak_b_op)\n",
    "\n",
    "    init = tf.global_variables_initializer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_iterations = 2000\n",
    "\n",
    "with tf.Session(graph=graph) as sess:\n",
    "    init.run()\n",
    "    for iteration in range(n_iterations):\n",
    "        if iteration % 100 == 0:\n",
    "            print(\"Iteration {:5}, MSE: {:.4f}\".format(iteration, mse.eval()))\n",
    "        training_op.run()\n",
    "    w_val, b_val = sess.run([w, b])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure(figsize=(10, 5))\n",
    "plot_life_satisfaction_with_linear_model(X_train, y_train, w_val, b_val)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Using Optimizers "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "graph = tf.Graph()\n",
    "with graph.as_default():\n",
    "    X = tf.constant(X_train, dtype=tf.float32, name=\"X\")\n",
    "    y = tf.constant(y_train, dtype=tf.float32, name=\"y\")\n",
    "\n",
    "    b = tf.Variable(0.0, name=\"b\")\n",
    "    w = tf.Variable(tf.zeros([1, 1]), name=\"w\")\n",
    "    y_pred = tf.add(tf.matmul(X, w), b, name=\"y_pred\")  # X @ w + b\n",
    "    \n",
    "    mse = tf.reduce_mean(tf.square(y_pred - y), name=\"mse\")\n",
    "\n",
    "    optimizer = tf.train.GradientDescentOptimizer(learning_rate=learning_rate)\n",
    "    training_op = optimizer.minimize(mse)  # <= MOAR AUTODIFF MAGIC!\n",
    "\n",
    "    init = tf.global_variables_initializer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_iterations = 2000\n",
    "\n",
    "with tf.Session(graph=graph) as sess:\n",
    "    init.run()\n",
    "    for iteration in range(n_iterations):\n",
    "        if iteration % 100 == 0:\n",
    "            print(\"Iteration {:5}, MSE: {:.4f}\".format(iteration, mse.eval()))\n",
    "        training_op.run()\n",
    "    w_val, b_val = sess.run([w, b])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure(figsize=(10, 5))\n",
    "plot_life_satisfaction_with_linear_model(X_train, y_train, w_val, b_val)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Faster Optimizers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "learning_rate = 0.01\n",
    "momentum = 0.8\n",
    "\n",
    "graph = tf.Graph()\n",
    "with graph.as_default():\n",
    "    X = tf.constant(X_train, dtype=tf.float32, name=\"X\")\n",
    "    y = tf.constant(y_train, dtype=tf.float32, name=\"y\")\n",
    "\n",
    "    b = tf.Variable(0.0, name=\"b\")\n",
    "    w = tf.Variable(tf.zeros([1, 1]), name=\"w\")\n",
    "    y_pred = tf.add(tf.matmul(X, w), b, name=\"y_pred\")  # X @ w + b\n",
    "    \n",
    "    mse = tf.reduce_mean(tf.square(y_pred - y), name=\"mse\")\n",
    "\n",
    "    optimizer = tf.train.MomentumOptimizer(learning_rate, momentum)\n",
    "    training_op = optimizer.minimize(mse)\n",
    "\n",
    "    init = tf.global_variables_initializer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_iterations = 500\n",
    "\n",
    "with tf.Session(graph=graph) as sess:\n",
    "    init.run()\n",
    "    for iteration in range(n_iterations):\n",
    "        if iteration % 100 == 0:\n",
    "            print(\"Iteration {:5}, MSE: {:.4f}\".format(iteration, mse.eval()))\n",
    "        training_op.run()\n",
    "    w_val, b_val = sess.run([w, b])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure(figsize=(10, 5))\n",
    "plot_life_satisfaction_with_linear_model(X_train, y_train, w_val, b_val)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "How does the optimizer know which variables to tweak? Answer: the `TRAINABLE_VARIABLES` collection."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "coll = graph.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES)\n",
    "[var.op.name for var in coll]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Making Predictions Outside of TensorFlow"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cyprus_gdp_per_capita = 22000\n",
    "cyprus_life_satisfaction = w_val[0][0] * cyprus_gdp_per_capita / 10000 + b_val\n",
    "cyprus_life_satisfaction"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Using placeholders "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "graph = tf.Graph()\n",
    "with graph.as_default():\n",
    "    X = tf.placeholder(tf.float32, shape=[None, 1], name=\"X\") # <= None allows for any\n",
    "    y = tf.placeholder(tf.float32, shape=[None, 1], name=\"y\") #    training batch size\n",
    "\n",
    "    b = tf.Variable(0.0, name=\"b\")\n",
    "    w = tf.Variable(tf.zeros([1, 1]), name=\"w\")\n",
    "    y_pred = tf.add(tf.matmul(X, w), b, name=\"y_pred\")  # X @ w + b\n",
    "    \n",
    "    mse = tf.reduce_mean(tf.square(y_pred - y), name=\"mse\")\n",
    "\n",
    "    optimizer = tf.train.MomentumOptimizer(learning_rate, momentum)\n",
    "    training_op = optimizer.minimize(mse)\n",
    "\n",
    "    init = tf.global_variables_initializer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_iterations = 500\n",
    "\n",
    "X_test = np.array([[22000]], dtype=np.float32) / 10000\n",
    "\n",
    "with tf.Session(graph=graph) as sess:\n",
    "    init.run()\n",
    "    for iteration in range(n_iterations):\n",
    "        feed_dict = {X: X_train, y: y_train}\n",
    "        if iteration % 100 == 0:\n",
    "            print(\"Iteration {:5}, MSE: {:.4f}\".format(\n",
    "                iteration, \n",
    "                mse.eval(feed_dict))) # <= FEED TRAINING DATA\n",
    "        training_op.run(feed_dict)    # <= FEED TRAINING DATA\n",
    "    # make the prediction:\n",
    "    y_pred_val = y_pred.eval(feed_dict={X: X_test}) # <= FEED TEST DATA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_pred_val"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Exercise 5"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "![Exercise](https://c1.staticflickr.com/9/8101/8553474140_c50cf08708_b.jpg)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "5.1) Create a simple graph that computes the function $f(x) = x^2 - 3x + 1$. Define $x$ as a placeholder for a simple scalar value of type float32 value (i.e., `shape=[], dtype=tf.float32`). Create a session and evaluate $f(5)$. You should find 11.0."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "5.2) Add an operation that computes the derivative of $f(x)$ with regards to $x$, noted $f'(x)$. Create a session and evaluate $f'(5)$. You should find 7.0.\n",
    "\n",
    "Hint: use `tf.gradients()`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "5.3) Using a `MomentumOptimizer`, find the value of $x$ that minimizes $f(x)$. You should find $\\hat{x}=1.5$.\n",
    "\n",
    "Hint: you need to change `x` into a `Variable`. Moreover, the `MomentumOptimizer` has its own variables that need to be initialized, so don't forget to create an `init` operation using a `tf.global_variables_initializer()`, and call it at the start of the session."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Try not to peek at the solution below before you have done the exercise! :)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "![thinking](https://upload.wikimedia.org/wikipedia/commons/0/06/Filos_segundo_logo_%28flipped%29.jpg)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Exercise 5 - Solution"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "5.1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "graph = tf.Graph()\n",
    "with graph.as_default():\n",
    "    x = tf.placeholder(tf.float32, shape=[], name=\"x\")\n",
    "    f = tf.square(x) - 3 * x + 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with tf.Session(graph=graph):\n",
    "    print(f.eval(feed_dict={x: 5.0}))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "5.2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with graph.as_default():\n",
    "    [fp] = tf.gradients(f, [x])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with tf.Session(graph=graph):\n",
    "    print(fp.eval(feed_dict={x: 5.0}))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "5.3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "learning_rate = 0.01\n",
    "momentum = 0.8\n",
    "\n",
    "graph = tf.Graph()\n",
    "with graph.as_default():\n",
    "    x = tf.Variable(0.0, name=\"x\")\n",
    "    f = tf.square(x) - 3 * x + 1    \n",
    "    optimizer = tf.train.MomentumOptimizer(learning_rate, momentum)\n",
    "    training_op = optimizer.minimize(f)\n",
    "    init = tf.global_variables_initializer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_iterations = 70\n",
    "with tf.Session(graph=graph):\n",
    "    init.run()\n",
    "    for iteration in range(n_iterations):\n",
    "        training_op.run()\n",
    "        if iteration % 10 == 0:\n",
    "            print(\"x={:.2f}, f(x)={:.2f}\".format(x.eval(), f.eval()))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Note that it's possible to replace the output value of any operation, not just placeholders. So, for example, even though `x` is now a `Variable`, you can use a `feed_dict` to use any value you want, for example to compute `f(5.0)`. **Important**: this does _not_ affect the variable!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with tf.Session(graph=graph):\n",
    "    init.run()\n",
    "    print(x.eval()) # x == 0.0\n",
    "    print(f.eval()) # f(0) == 1.0\n",
    "    print(f.eval(feed_dict={x: 5.0})) # use 5.0 instead of the value of x, to compute f(5)\n",
    "    print(x.eval()) # x is still 0.0\n",
    "    print(f.eval()) # f(0) is still 1.0"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Saving and Restoring a Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "graph = tf.Graph()\n",
    "with graph.as_default():\n",
    "    X = tf.placeholder(tf.float32, shape=[None, 1], name=\"X\")\n",
    "    y = tf.placeholder(tf.float32, shape=[None, 1], name=\"y\")\n",
    "\n",
    "    b = tf.Variable(0.0, name=\"b\")\n",
    "    w = tf.Variable(tf.zeros([1, 1]), name=\"w\")\n",
    "    y_pred = tf.add(tf.matmul(X, w), b, name=\"y_pred\")  # X @ w + b\n",
    "    \n",
    "    mse = tf.reduce_mean(tf.square(y_pred - y), name=\"mse\")\n",
    "\n",
    "    optimizer = tf.train.MomentumOptimizer(learning_rate, momentum)\n",
    "    training_op = optimizer.minimize(mse)\n",
    "\n",
    "    init = tf.global_variables_initializer()\n",
    "    saver = tf.train.Saver() # <= At the very end of the construction phase"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_iterations = 500\n",
    "\n",
    "with tf.Session(graph=graph) as sess:\n",
    "    init.run()\n",
    "    for iteration in range(n_iterations):\n",
    "        if iteration % 100 == 0:\n",
    "            print(\"Iteration {:5}, MSE: {:.4f}\".format(\n",
    "                iteration, \n",
    "                mse.eval(feed_dict={X: X_train, y: y_train})))\n",
    "        training_op.run(feed_dict={X: X_train, y: y_train}) # <= FEED THE DICT\n",
    "    saver.save(sess, \"./my_life_satisfaction_model\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with tf.Session(graph=graph) as sess:\n",
    "    saver.restore(sess, \"./my_life_satisfaction_model\")\n",
    "    # make the prediction:\n",
    "    y_pred_val = y_pred.eval(feed_dict={X: X_test})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_pred_val"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Restoring a Graph"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_path = \"./my_life_satisfaction_model\"\n",
    "graph = tf.Graph()\n",
    "with tf.Session(graph=graph) as sess:\n",
    "    # restore the graph\n",
    "    saver = tf.train.import_meta_graph(model_path + \".meta\")\n",
    "    saver.restore(sess, model_path)\n",
    "\n",
    "    # get references to the tensors we need\n",
    "    X = graph.get_tensor_by_name(\"X:0\")\n",
    "    y_pred = graph.get_tensor_by_name(\"y_pred:0\")\n",
    "\n",
    "    # make the prediction:\n",
    "    y_pred_val = y_pred.eval(feed_dict={X: X_test})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_pred_val"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}