{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import tensorflow.contrib.eager as tfe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "tfe.enable_eager_execution()  # 注释本行就可以关闭 Eager Execution"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "class MNISTModel(tfe.Network):\n",
    "    def __init__(self):\n",
    "        super(MNISTModel, self).__init__()\n",
    "        self.layer1 = self.track_layer(tf.layers.Dense(units=10))\n",
    "        self.layer2 = self.track_layer(tf.layers.Dense(units=10))\n",
    "    def call(self, input):\n",
    "        \"\"\"Actually runs the model.\"\"\"\n",
    "        result = self.layer1(input)\n",
    "        result = self.layer2(result)\n",
    "        return result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1, 1, 784)\n",
      "tf.Tensor([[[ 0.  0.  0.  0.  0.  0.  0.  0.  0.  0.]]], shape=(1, 1, 10), dtype=float32)\n"
     ]
    }
   ],
   "source": [
    "# Let's make up a blank input image\n",
    "model = MNISTModel()\n",
    "batch = tf.zeros([1, 1, 784])\n",
    "print(batch.shape)\n",
    "# (1, 1, 784)\n",
    "result = model(batch)\n",
    "print(result)\n",
    "# tf.Tensor([[[ 0.  0., ...., 0.]]], shape=(1, 1, 10), dtype=float32)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def loss_function(model, x, y):\n",
    "    y_ = model(x)\n",
    "    return tf.nn.softmax_cross_entropy_with_logits(labels=y, logits=y_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'dataset' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-6-8fd83e3a491a>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0moptimizer\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mGradientDescentOptimizer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlearning_rate\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m0.001\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0;32mfor\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mtfe\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mIterator\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdataset\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      3\u001b[0m   \u001b[0mgrads\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtfe\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mimplicit_gradients\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mloss_function\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m   \u001b[0moptimizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mapply_gradients\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mgrads\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'dataset' is not defined"
     ]
    }
   ],
   "source": [
    "# if not sure we have GPU or not, CPU is always safety\n",
    "optimizer = tf.train.GradientDescentOptimizer(learning_rate=0.001)\n",
    "for (x, y) in tfe.Iterator(dataset):\n",
    "    grads = tfe.implicit_gradients(loss_function)(model, x, y)\n",
    "    optimizer.apply_gradients(grads)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# if we have GPU\n",
    "with tf.device(\"/gpu:0\"):\n",
    "    for (x, y) in tfe.Iterator(dataset):\n",
    "        optimizer.minimize(lambda: loss_function(model, x, y))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}