{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/anaconda3/lib/python3.6/site-packages/h5py/__init__.py:34: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n"
     ]
    }
   ],
   "source": [
    "# %load /Users/facai/Study/book_notes/preconfig.py\n",
    "%matplotlib inline\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "sns.set(color_codes=True)\n",
    "sns.set(font='SimHei', font_scale=2.5)\n",
    "plt.rcParams['axes.grid'] = False\n",
    "\n",
    "import tensorflow as tf\n",
    "\n",
    "def show_image(filename, figsize=None, res_dir=True):\n",
    "    if figsize:\n",
    "        plt.figure(figsize=figsize)\n",
    "\n",
    "    if res_dir:\n",
    "        filename = './res/{}'.format(filename)\n",
    "\n",
    "    plt.imshow(plt.imread(filename))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "TensorFlow图相关知识简介\n",
    "============\n",
    "\n",
    "参考: https://www.tensorflow.org/programmers_guide/graphs\n",
    "\n",
    "1. tf.Graph\n",
    "\n",
    "2. op, tensor\n",
    "\n",
    "3. variable\n",
    "\n",
    "4. name_scope, variable_scop, collection\n",
    "\n",
    "5. save and restore"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "![](https://www.tensorflow.org/images/tensors_flowing.gif)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 0. tf.Graph\n",
    "\n",
    "tf.Graph: GraphDef => *.pb文件\n",
    "+ Graph structure: Operator, Tensor-like object, 连接关系\n",
    "+ Graph collections: metadata\n",
    "\n",
    "tf.Session():\n",
    "+ 本地\n",
    "+ 分布式：master (worker_0)\n",
    "\n",
    "```python\n",
    "with tf.Session(\"grpc://example.org:2222\"):\n",
    "  pass\n",
    "```\n",
    "\n",
    "状态（Variable) => *.ckpt文件"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1. 算子与Tensor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor 'mul_1:0' shape=() dtype=int32>"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a = tf.constant(1)\n",
    "b = a * 2\n",
    "b"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Operation 'mul_1' type=Mul>"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "b.op"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "b.consumers()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Operation 'Const_1' type=Const>"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a.op"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<tf.Operation 'mul_1' type=Mul>]"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a.consumers()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "tensorflow/python/framework/ops.py\n",
    "+ Tensor:\n",
    "    - device\n",
    "    - graph\n",
    "    - op\n",
    "    - consumers\n",
    "    - _override_operator:\n",
    "      数学算子：math_op.add 重载 `__add__`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<tf.Tensor 'mul_1:0' shape=() dtype=int32>]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "b.op.outputs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<tf.Tensor 'Const_1:0' shape=() dtype=int32>,\n",
       " <tf.Tensor 'mul_1/y:0' shape=() dtype=int32>]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "list(b.op.inputs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tensor(\"Const_1:0\", shape=(), dtype=int32)\n",
      "Tensor(\"Const_1:0\", shape=(), dtype=int32)\n"
     ]
    }
   ],
   "source": [
    "print(b.op.inputs[0])\n",
    "print(a)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[]"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "list(a.op.inputs)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "+ Operator: NodeDef\n",
    "   - device\n",
    "   - inputs\n",
    "   - outputs\n",
    "   - graph\n",
    "   - node_def\n",
    "   - op_def\n",
    "   - **run**\n",
    "   - traceback\n",
    "   \n",
    "Operator和Tensor构成无向图\n",
    "\n",
    "```python\n",
    "# run\n",
    "sess.run([b])\n",
    "```\n",
    "\n",
    "参考:\n",
    "+ tf.Tensor: https://www.tensorflow.org/versions/master/api_docs/python/tf/Tensor\n",
    "+ tf.Operator: https://www.tensorflow.org/versions/master/api_docs/python/tf/Operation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2. 变量"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor 'add:0' shape=(1,) dtype=int32>"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "v = tf.Variable([0])\n",
    "c = b + v\n",
    "c"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<tf.Tensor 'mul_1:0' shape=() dtype=int32>,\n",
       " <tf.Tensor 'Variable/read:0' shape=(1,) dtype=int32>]"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "list(c.op.inputs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Operation 'Variable/read' type=Identity>"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "c.op.inputs[1].op"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<tf.Tensor 'Variable:0' shape=(1,) dtype=int32_ref>]"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "list(c.op.inputs[1].op.inputs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Variable 'Variable:0' shape=(1,) dtype=int32_ref>"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "v"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "实际上，对变量的读是通过`tf.identity`算子得到：\n",
    "```python\n",
    "c = tf.add(b, tf.identity(v))\n",
    "```\n",
    "\n",
    "\n",
    "+ Variable: act like Tensor\n",
    "   - [ops](https://stackoverflow.com/questions/40817665/whats-the-difference-between-variable-and-resourcevariable-in-tensorflow)\n",
    "     1. VariableV2\n",
    "     2. ResourceVariable\n",
    "   - _AsTensor -> g.as_graph_element\n",
    "   - value: Identity(variable) -> Tensor\n",
    "   - assign\n",
    "   - init_op: Assign(self, init_value)\n",
    "   - to_proto: VariableDef\n",
    "\n",
    "参考：https://www.tensorflow.org/versions/master/api_docs/python/tf/Variable"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "3. collections\n",
    "===============\n",
    "\n",
    "+ collections: 按作用分组\n",
    "  - Variable: global_varialbe\n",
    "  - 更多见[tf.GraphKeys](https://www.tensorflow.org/versions/master/api_docs/python/tf/GraphKeys)\n",
    "+ name_scope: Operator, Tensor\n",
    "+ variable_scope: Variable\n",
    "  - 伴生name_scope\n",
    "  \n",
    "```python\n",
    "class Layer:\n",
    "  def build(self):\n",
    "    pass\n",
    "  def call(self, inputs):\n",
    "    pass\n",
    "```\n",
    "\n",
    "![](https://www.tensorflow.org/images/mnist_deep.png)\n",
    "\n",
    "参考：https://www.tensorflow.org/programmers_guide/summaries_and_tensorboard"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 4. 保存与恢复"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<tf.Variable 'v1:0' shape=(3,) dtype=float32_ref>\n",
      "Model saved in path: ./tmp/model.ckpt\n",
      "Graph saved in path: ./tmp/graph.pbtxt\n"
     ]
    }
   ],
   "source": [
    "graph_a = tf.Graph()\n",
    "with graph_a.as_default():\n",
    "    v1 = tf.get_variable(\"v1\", shape=[3], initializer = tf.zeros_initializer)\n",
    "    print(v1)\n",
    "    inc_v1 = v1.assign(v1+1)\n",
    "      \n",
    "    init_op = tf.global_variables_initializer()\n",
    "    saver = tf.train.Saver()\n",
    "  \n",
    "    with tf.Session() as sess:\n",
    "        sess.run(init_op)\n",
    "        inc_v1.op.run()\n",
    "        save_path = saver.save(sess, \"./tmp/model.ckpt\", write_meta_graph=True)\n",
    "        print(\"Model saved in path: %s\" % save_path)\n",
    "        \n",
    "        pb_path = tf.train.write_graph(graph_a.as_graph_def(), \"./tmp/\", \"graph.pbtxt\", as_text=True)\n",
    "        print(\"Graph saved in path: %s\" % pb_path)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "graph.pbtxt部份示意：`v1 + 1`:\n",
    "\n",
    "```bash\n",
    "node {\n",
    "  name: \"add\"\n",
    "  op: \"Add\"\n",
    "  input: \"v1/read\"\n",
    "  input: \"add/y\"\n",
    "  attr {\n",
    "    key: \"T\"\n",
    "    value {\n",
    "      type: DT_FLOAT\n",
    "    }\n",
    "  }\n",
    "}\n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from ./tmp/model.ckpt\n",
      "[<tf.Operation 'v1/Initializer/zeros' type=Const>, <tf.Operation 'v1' type=VariableV2>, <tf.Operation 'v1/Assign' type=Assign>, <tf.Operation 'v1/read' type=Identity>, <tf.Operation 'add/y' type=Const>, <tf.Operation 'add' type=Add>, <tf.Operation 'Assign' type=Assign>, <tf.Operation 'init' type=NoOp>, <tf.Operation 'save/Const' type=Const>, <tf.Operation 'save/SaveV2/tensor_names' type=Const>, <tf.Operation 'save/SaveV2/shape_and_slices' type=Const>, <tf.Operation 'save/SaveV2' type=SaveV2>, <tf.Operation 'save/control_dependency' type=Identity>, <tf.Operation 'save/RestoreV2/tensor_names' type=Const>, <tf.Operation 'save/RestoreV2/shape_and_slices' type=Const>, <tf.Operation 'save/RestoreV2' type=RestoreV2>, <tf.Operation 'save/Assign' type=Assign>, <tf.Operation 'save/restore_all' type=NoOp>]\n",
      "------------------\n",
      "v1 : [1. 1. 1.]\n"
     ]
    }
   ],
   "source": [
    "graph_b = tf.Graph()\n",
    "with graph_b.as_default():\n",
    "    with tf.Session() as sess:\n",
    "        saver = tf.train.import_meta_graph('./tmp/model.ckpt.meta')\n",
    "        saver.restore(sess, \"./tmp/model.ckpt\")\n",
    "        print(graph_b.get_operations())\n",
    "        \n",
    "        v1 = graph_b.get_tensor_by_name(\"v1:0\")\n",
    "        print(\"------------------\")\n",
    "        print(\"v1 : %s\" % v1.eval(session=sess))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "总结：\n",
    "+ `tf.train.Saver`会保存GraphDef和Variable信息，用它可以直接恢复图。\n",
    "  - [tf.train.import_meta_graph](https://www.tensorflow.org/versions/master/api_docs/python/tf/train/import_meta_graph)\n",
    "  - [Exporting and Importing a MetaGraph](https://www.tensorflow.org/versions/master/api_guides/python/meta_graph)\n",
    "  - 缺点：无法指input_tensor。\n",
    "+ `tf.train.write_graph`、`tf.GraphDef`和`tf.import_graph_def`，主要用于固化模型（只有GraphDef信息）。\n",
    "\n",
    "\n",
    "参考：\n",
    "+ https://stackoverflow.com/questions/38641887/how-to-save-a-trained-tensorflow-model-for-later-use-for-application\n",
    "+ https://www.tensorflow.org/programmers_guide/saved_model#overview_of_saving_and_restoring_models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}