{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# CS 20 : TensorFlow for Deep Learning Research\n",
"## Lecture 02 : Operations"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### Setup"
]
},
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"1.12.0\n"
]
}
],
"source": [
"from __future__ import absolute_import, division, print_function\n",
"import numpy as np\n",
"import tensorflow as tf\n",
"\n",
"print(tf.__version__)\n",
"sess_config = tf.ConfigProto(gpu_options=tf.GPUOptions(allow_growth=True))"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### 1. Fun with TensorBoard"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Tensor(\"Const:0\", shape=(), dtype=int32) Tensor(\"Const_1:0\", shape=(), dtype=int32) Tensor(\"Add:0\", shape=(), dtype=int32)\n",
"5\n"
]
}
],
"source": [
"# Nodes: operators, variables, and constants \n",
"# Edges: tensors\n",
"a = tf.constant(2)\n",
"b = tf.constant(3)\n",
"x = tf.add(a,b)\n",
"print(a, b, x)\n",
"with tf.Session(config = sess_config) as sess:\n",
" print(sess.run(x))"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Tensor(\"Const:0\", shape=(), dtype=int32) Tensor(\"Const_1:0\", shape=(), dtype=int32) Tensor(\"Add:0\", shape=(), dtype=int32)\n",
"5\n"
]
}
],
"source": [
"# To visualizer above program with Tensorboard\n",
"tf.reset_default_graph()\n",
"\n",
"a = tf.constant(2)\n",
"b = tf.constant(3)\n",
"x = tf.add(a,b)\n",
"print(a, b, x)\n",
"\n",
"# Create the summary writer after graph definition and before running your\n",
"# session\n",
"writer = tf.summary.FileWriter(logdir = '../graphs/lecture02/add_example',\n",
" graph = tf.get_default_graph())\n",
"writer.close()\n",
"with tf.Session(config = sess_config) as sess:\n",
" print(sess.run(x))"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### 2. Constant op"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"It’s straightforward to create a constant in TensorFlow"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Tensor(\"vector:0\", shape=(2,), dtype=int32)\n",
"[2 2]\n"
]
}
],
"source": [
"# constant of 1d tensor (vector)\n",
"a = tf.constant([2, 2], name = 'vector')\n",
"print(a)\n",
"with tf.Session(config = sess_config) as sess:\n",
" print(sess.run(a))"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Tensor(\"matrix:0\", shape=(2, 2), dtype=int32)\n",
"[[0 1]\n",
" [2 3]]\n"
]
}
],
"source": [
"# constant of 2x2 tensor (matrix)\n",
"b = tf.constant([[0,1], [2,3]], name = 'matrix')\n",
"print(b)\n",
"with tf.Session(config = sess_config) as sess:\n",
" print(sess.run(b))"
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Tensor(\"zeros:0\", shape=(2, 3), dtype=int32)\n",
"[[0 0 0]\n",
" [0 0 0]]\n"
]
}
],
"source": [
"# create a tensor of shape and all elements are zeros\n",
"print(tf.zeros(shape = [2,3], dtype = tf.int32))\n",
"with tf.Session(config = sess_config) as sess:\n",
" print(sess.run(tf.zeros(shape = [2,3], dtype = tf.int32)))"
]
},
{
"cell_type": "code",
"execution_count": 7,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Tensor(\"zeros_like:0\", shape=(3, 2), dtype=int32)\n",
"[[0 0]\n",
" [0 0]\n",
" [0 0]]\n"
]
}
],
"source": [
"# create a tensor of shape and type (unless type is specified) as the input_tensor but all\n",
"# elements are zeros.\n",
"# input_tensor [[0, 1], [2, 3], [4, 5]]\n",
"input_tensor = [[0, 1], [2, 3], [4, 5]]\n",
"print(tf.zeros_like(input_tensor))\n",
"with tf.Session(config = sess_config) as sess:\n",
" print(sess.run(tf.zeros_like(input_tensor)))"
]
},
{
"cell_type": "code",
"execution_count": 8,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Tensor(\"ones:0\", shape=(2, 3), dtype=int32)\n",
"[[1 1 1]\n",
" [1 1 1]]\n"
]
}
],
"source": [
"# create a tensor of shape and all elements are ones\n",
"print(tf.ones(shape = [2,3], dtype = tf.int32))\n",
"with tf.Session(config = sess_config) as sess:\n",
" print(sess.run(tf.ones(shape = [2,3], dtype = tf.int32)))"
]
},
{
"cell_type": "code",
"execution_count": 9,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Tensor(\"ones_like:0\", shape=(3, 2), dtype=int32)\n",
"[[1 1]\n",
" [1 1]\n",
" [1 1]]\n"
]
}
],
"source": [
"# create a tensor of shape and type (unless type is specified) as the input_tensor but all\n",
"# elements are ones.\n",
"# input_tensor is [[0, 1], [2, 3], [4, 5]]\n",
"print(tf.ones_like(input_tensor))\n",
"with tf.Session(config = sess_config) as sess:\n",
" print(sess.run(tf.ones_like(input_tensor)))"
]
},
{
"cell_type": "code",
"execution_count": 10,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Tensor(\"Fill:0\", shape=(2, 3), dtype=int32)\n",
"[[8 8 8]\n",
" [8 8 8]]\n"
]
}
],
"source": [
"# create a tensor filled with a scalar value.\n",
"print(tf.fill([2,3], 8))\n",
"with tf.Session(config = sess_config) as sess:\n",
" print(sess.run(tf.fill([2,3], 8)))"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"You can create constants that are sequences"
]
},
{
"cell_type": "code",
"execution_count": 11,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Tensor(\"LinSpace:0\", shape=(4,), dtype=float32)\n",
"[10. 11. 12. 13.]\n"
]
}
],
"source": [
"'''\n",
"create a sequence of num evenly-spaced values are generated beginning at start. If num >\n",
"1, the values in the sequence increase by (stop - start) / (num - 1), so that the last one\n",
"is exactly stop. comparable to but slightly different from numpy.linspace\n",
"'''\n",
"print(tf.lin_space(10., 13., 4))\n",
"with tf.Session(config = sess_config) as sess:\n",
" print(sess.run(tf.lin_space(10., 13., 4)))"
]
},
{
"cell_type": "code",
"execution_count": 12,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Tensor(\"range:0\", shape=(5,), dtype=int32)\n",
"[0 1 2 3 4]\n"
]
}
],
"source": [
"# create a sequence of numbers that begins at start and extends by increments of delta up to\n",
"# but not including limit\n",
"print(tf.range(5)) \n",
"with tf.Session(config = sess_config) as sess:\n",
" print(sess.run(tf.range(5))) # TensorFlow sequences are not iterable"
]
},
{
"cell_type": "code",
"execution_count": 13,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"[ 3 6 9 12 15]\n",
"[3. 2.5 2. 1.5]\n",
"[0 1 2 3 4]\n"
]
}
],
"source": [
"with tf.Session(config = sess_config) as sess:\n",
" print(sess.run(tf.range(3, 18, 3)))\n",
" print(sess.run(tf.range(3, 1, -.5)))\n",
" print(sess.run(tf.range(5)))"
]
},
{
"cell_type": "code",
"execution_count": 14,
"metadata": {},
"outputs": [
{
"ename": "TypeError",
"evalue": "Tensor objects are only iterable when eager execution is enabled. To iterate over this tensor use tf.map_fn.",
"output_type": "error",
"traceback": [
"\u001b[0;31m--------------------------------------\u001b[0m",
"\u001b[0;31mTypeError\u001b[0mTraceback (most recent call last)",
"\u001b[0;32m<ipython-input-14-b1a040fb85ad>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0;32mfor\u001b[0m \u001b[0m_\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m4\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 2\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0m_\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m/usr/local/var/pyenv/versions/3.6.6/envs/tensorflow/lib/python3.6/site-packages/tensorflow/python/framework/ops.py\u001b[0m in \u001b[0;36m__iter__\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 457\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mcontext\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexecuting_eagerly\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 458\u001b[0m raise TypeError(\n\u001b[0;32m--> 459\u001b[0;31m \u001b[0;34m\"Tensor objects are only iterable when eager execution is \"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 460\u001b[0m \"enabled. To iterate over this tensor use tf.map_fn.\")\n\u001b[1;32m 461\u001b[0m \u001b[0mshape\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_shape_tuple\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;31mTypeError\u001b[0m: Tensor objects are only iterable when eager execution is enabled. To iterate over this tensor use tf.map_fn."
]
}
],
"source": [
"for _ in tf.range(4):\n",
" print(_)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### 3. Math Operations\n",
"recommend reading the lecture notes"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### 4. Data Types\n",
"recommend reading the lecture notes"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### 5. Variables"
]
},
{
"cell_type": "code",
"execution_count": 15,
"metadata": {},
"outputs": [],
"source": [
"tf.reset_default_graph()"
]
},
{
"cell_type": "code",
"execution_count": 16,
"metadata": {},
"outputs": [],
"source": [
"# Create variables\n",
"# Note that when we use tf.constant as an initializer,\n",
"# we don't need to provide shape\n",
"s = tf.get_variable(name = 'scalar',\n",
" initializer = tf.constant(2))\n",
"m = tf.get_variable(name = 'matrix',\n",
" initializer = tf.constant([[0, 1], [2, 3]]))\n",
"W = tf.get_variable(name = 'big_matrix', shape = [784, 10],\n",
" initializer = tf.zeros_initializer())"
]
},
{
"cell_type": "code",
"execution_count": 17,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"[b'scalar' b'matrix' b'big_matrix']\n"
]
}
],
"source": [
"# Initialize variables\n",
"with tf.Session(config = sess_config) as sess:\n",
" print(sess.run(tf.report_uninitialized_variables()))"
]
},
{
"cell_type": "code",
"execution_count": 18,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"[]\n"
]
}
],
"source": [
"with tf.Session(config = sess_config) as sess:\n",
" sess.run(tf.global_variables_initializer())\n",
" print(sess.run(tf.report_uninitialized_variables()))"
]
},
{
"cell_type": "code",
"execution_count": 19,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"10\n"
]
}
],
"source": [
"# Interesting Assign example\n",
"tf.reset_default_graph()\n",
"W = tf.Variable(10)\n",
"W.assign(100)\n",
"\n",
"with tf.Session(config = sess_config) as sess:\n",
" sess.run(W.initializer)\n",
" print(W.eval())"
]
},
{
"cell_type": "code",
"execution_count": 20,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"100\n"
]
}
],
"source": [
"tf.reset_default_graph()\n",
"W = tf.Variable(10)\n",
"assign_op = W.assign(100)\n",
"\n",
"with tf.Session(config = sess_config) as sess:\n",
" sess.run(assign_op)\n",
" print(W.eval())"
]
},
{
"cell_type": "code",
"execution_count": 21,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"<tf.Variable 'weights:0' shape=(2, 2) dtype=float32_ref> [[1. 1.]\n",
" [2. 2.]]\n",
"<tf.Variable 'weights:0' shape=(2, 2) dtype=float32_ref> [[1. 1.]\n",
" [2. 2.]]\n"
]
}
],
"source": [
"tf.reset_default_graph()\n",
"W = tf.get_variable(name = 'weights', initializer = tf.constant([[1.,1.],[2.,2.]]))\n",
"tf.get_variable_scope().reuse_variables()\n",
"w = tf.get_variable(name = 'weights')\n",
"writer = tf.summary.FileWriter(logdir = '../graphs/lecture02/get_variables',\n",
" graph = tf.get_default_graph())\n",
"writer.close()\n",
"\n",
"with tf.Session(config = sess_config) as sess:\n",
" sess.run(tf.global_variables_initializer())\n",
" print(W, sess.run(W))\n",
" print(w, sess.run(w))"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### 6. Interactive Session\n",
"recommend reading the lecture notes"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### 7. Control Dependencies\n",
"recommend reading the lecture notes"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### 8. Importing Data\n",
"tf.placeholder"
]
},
{
"cell_type": "code",
"execution_count": 22,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"[6. 7. 8.]\n"
]
}
],
"source": [
"tf.reset_default_graph()\n",
"a = tf.placeholder(dtype = tf.float32, shape = [3])\n",
"b = tf.constant(value = [5, 5, 5], dtype = tf.float32)\n",
"c = a + b\n",
"\n",
"writer = tf.summary.FileWriter(logdir = '../graphs/lecture02/placeholder',\n",
" graph = tf.get_default_graph())\n",
"\n",
"with tf.Session(config = sess_config) as sess:\n",
" print(sess.run(c, feed_dict = {a : [1.,2.,3.]}))"
]
},
{
"cell_type": "code",
"execution_count": 23,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"21\n",
"45\n"
]
}
],
"source": [
"# You can feed values to tensors that aren't placeholders.\n",
"# Any tensors that are feedable can be fed\n",
"tf.reset_default_graph()\n",
"a = tf.add(2,5)\n",
"b = tf.multiply(a, 3)\n",
"\n",
"with tf.Session(config = sess_config) as sess:\n",
" print(sess.run(b))\n",
" print(sess.run(b, feed_dict = {a : 15}))"
]
},
{
"cell_type": "code",
"execution_count": 24,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"True True\n"
]
}
],
"source": [
"with tf.Session(config = sess_config) as sess:\n",
" print(sess.graph.is_feedable(a), sess.graph.is_feedable(b))"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### 9. The trap of lazy loading\n",
"Lazy loading is a term that refers to a programming pattern when you defer declaring/initializing an object until it is loaded"
]
},
{
"cell_type": "code",
"execution_count": 25,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"[<tf.Operation 'Const' type=Const>, <tf.Operation 'x' type=VariableV2>, <tf.Operation 'x/Assign' type=Assign>, <tf.Operation 'x/read' type=Identity>, <tf.Operation 'Const_1' type=Const>, <tf.Operation 'y' type=VariableV2>, <tf.Operation 'y/Assign' type=Assign>, <tf.Operation 'y/read' type=Identity>, <tf.Operation 'Add' type=Add>, <tf.Operation 'init' type=NoOp>]\n"
]
}
],
"source": [
"# normal loading\n",
"tf.reset_default_graph()\n",
"x = tf.get_variable(name = 'x', initializer = tf.constant(10))\n",
"y = tf.get_variable(name = 'y', initializer = tf.constant(20))\n",
"z = tf.add(x,y)\n",
"\n",
"with tf.Session(config = sess_config) as sess:\n",
" sess.run(tf.global_variables_initializer())\n",
" writer = tf.summary.FileWriter(logdir = '../graphs/lecture02/normal_loading',\n",
" graph = tf.get_default_graph())\n",
" \n",
" for _ in range(3):\n",
" sess.run(z)\n",
" else:\n",
" print(tf.get_default_graph().get_operations())\n",
" writer.close()"
]
},
{
"cell_type": "code",
"execution_count": 26,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"[<tf.Operation 'Const' type=Const>, <tf.Operation 'x' type=VariableV2>, <tf.Operation 'x/Assign' type=Assign>, <tf.Operation 'x/read' type=Identity>, <tf.Operation 'Const_1' type=Const>, <tf.Operation 'y' type=VariableV2>, <tf.Operation 'y/Assign' type=Assign>, <tf.Operation 'y/read' type=Identity>, <tf.Operation 'init' type=NoOp>, <tf.Operation 'Add' type=Add>, <tf.Operation 'Add_1' type=Add>, <tf.Operation 'Add_2' type=Add>]\n"
]
}
],
"source": [
"# lazy loading\n",
"tf.reset_default_graph()\n",
"x = tf.get_variable(name = 'x', initializer = tf.constant(10))\n",
"y = tf.get_variable(name = 'y', initializer = tf.constant(20))\n",
"\n",
"with tf.Session(config = sess_config) as sess:\n",
" sess.run(tf.global_variables_initializer())\n",
" writer = tf.summary.FileWriter(logdir = '../graphs/lecture02/lazy_loading',\n",
" graph = sess.graph)\n",
" for _ in range(3):\n",
" sess.run(tf.add(x,y))\n",
" else:\n",
" print(tf.get_default_graph().get_operations())\n",
" writer.close()"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.6.6"
},
"varInspector": {
"cols": {
"lenName": 16,
"lenType": 16,
"lenVar": 40
},
"kernels_config": {
"python": {
"delete_cmd_postfix": "",
"delete_cmd_prefix": "del ",
"library": "var_list.py",
"varRefreshCmd": "print(var_dic_list())"
},
"r": {
"delete_cmd_postfix": ") ",
"delete_cmd_prefix": "rm(",
"library": "var_list.r",
"varRefreshCmd": "cat(var_dic_list()) "
}
},
"types_to_exclude": [
"module",
"function",
"builtin_function_or_method",
"instance",
"_Feature"
],
"window_display": false
}
},
"nbformat": 4,
"nbformat_minor": 2
}