{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# CS 20 : TensorFlow for Deep Learning Research\n",
"## Lecture 05 : Variable sharing and managing experiments\n",
"### Variable sharing"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### Setup"
]
},
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"1.12.0\n"
]
}
],
"source": [
"from __future__ import absolute_import, division, print_function\n",
"import os, sys\n",
"import numpy as np\n",
"import tensorflow as tf\n",
"\n",
"print(tf.__version__)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### name scope\n",
"TensorFlow doesn't know what nodes should be grouped together, unless you tell it to\n",
"```python\n",
"with tf.name_scope(name_of_that_scope):\n",
" # declare op_1\n",
" # declare op_2\n",
" # ...\n",
"```"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"#### Example 1\n",
"~~Don't use tf.Variable function.. Just see below~~ \n",
"`tf.get_variable` and `tf.variable_scope` are deprecated in tensorflow 2.0"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"name : y/a:0, value : 1.0\n",
"name : y/b:0, value : 2.0\n",
"name : y_1/c:0, value : 3.0\n"
]
}
],
"source": [
"tf.reset_default_graph()\n",
"\n",
"with tf.name_scope('y'):\n",
" a1 = tf.Variable(initial_value = 1., name = 'a')\n",
" a2 = tf.Variable(initial_value = 2., name = 'b')\n",
"\n",
"with tf.name_scope('y'):\n",
" a3 = tf.Variable(initial_value = 3., name = 'c')\n",
"\n",
"with tf.Session() as sess:\n",
" sess.run(tf.global_variables_initializer())\n",
" print('name : {}, value : {}'.format(a1.name, a1.eval()))\n",
" print('name : {}, value : {}'.format(a2.name, a2.eval()))\n",
" print('name : {}, value : {}'.format(a3.name, a3.eval()))"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"name : y/a:0, value : 1.0\n",
"name : y/b:0, value : 2.0\n",
"name : y_1/c:0, value : 3.0\n"
]
}
],
"source": [
"tf.reset_default_graph()\n",
"\n",
"with tf.variable_scope('y'):\n",
" a1 = tf.Variable(initial_value = 1., name = 'a')\n",
" a2 = tf.Variable(initial_value = 2., name = 'b')\n",
"\n",
"with tf.variable_scope('y'):\n",
" a3 = tf.Variable(initial_value = 3., name = 'c')\n",
"\n",
"with tf.Session() as sess:\n",
" sess.run(tf.global_variables_initializer())\n",
" print('name : {}, value : {}'.format(a1.name, a1.eval()))\n",
" print('name : {}, value : {}'.format(a2.name, a2.eval()))\n",
" print('name : {}, value : {}'.format(a3.name, a3.eval()))"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"#### Example 2\n",
"using `tf.get_variable` instead of the `tf.Variable` before tensorflow 2.0"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"name : a:0, value : 1\n",
"name : b:0, value : 2\n",
"name : c:0, value : 3\n"
]
}
],
"source": [
"tf.reset_default_graph()\n",
"\n",
"with tf.name_scope('y'):\n",
" a1 = tf.get_variable(name = 'a', initializer = tf.constant(1))\n",
" a2 = tf.get_variable(name = 'b', initializer = tf.constant(2))\n",
"\n",
"with tf.name_scope('y'):\n",
" a3 = tf.get_variable(name = 'c', initializer = tf.constant(3))\n",
"\n",
"with tf.Session() as sess:\n",
" sess.run(tf.global_variables_initializer())\n",
" print('name : {}, value : {}'.format(a1.name, a1.eval()))\n",
" print('name : {}, value : {}'.format(a2.name, a2.eval()))\n",
" print('name : {}, value : {}'.format(a3.name, a3.eval()))"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"name : y/a:0, value : 1\n",
"name : y/b:0, value : 2\n",
"name : y/c:0, value : 3\n"
]
}
],
"source": [
"tf.reset_default_graph()\n",
"\n",
"with tf.variable_scope('y'):\n",
" a1 = tf.get_variable(name = 'a', initializer = tf.constant(1))\n",
" a2 = tf.get_variable(name = 'b', initializer = tf.constant(2))\n",
"\n",
"with tf.variable_scope('y'):\n",
" a3 = tf.get_variable(name = 'c', initializer = tf.constant(3))\n",
"\n",
"with tf.Session() as sess:\n",
" sess.run(tf.global_variables_initializer())\n",
" print('name : {}, value : {}'.format(a1.name, a1.eval()))\n",
" print('name : {}, value : {}'.format(a2.name, a2.eval()))\n",
" print('name : {}, value : {}'.format(a3.name, a3.eval()))"
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Variable already exists\n",
"name : y/a:0, value : 1\n",
"name : y/b:0, value : 2\n"
]
},
{
"ename": "ValueError",
"evalue": "Cannot use the default session to evaluate tensor: the tensor's graph is different from the session's graph. Pass an explicit session to `eval(session=sess)`.",
"output_type": "error",
"traceback": [
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
"\u001b[0;31mValueError\u001b[0m Traceback (most recent call last)",
"\u001b[0;32m<ipython-input-6-72d5dd726ec6>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m 14\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'name : {}, value : {}'\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ma1\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0ma1\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0meval\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 15\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'name : {}, value : {}'\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ma2\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0ma2\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0meval\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 16\u001b[0;31m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'name : {}, value : {}'\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ma3\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0ma3\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0meval\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
"\u001b[0;32m/opt/conda/lib/python3.6/site-packages/tensorflow/python/ops/variables.py\u001b[0m in \u001b[0;36meval\u001b[0;34m(self, session)\u001b[0m\n\u001b[1;32m 1649\u001b[0m \u001b[0mA\u001b[0m \u001b[0mnumpy\u001b[0m\u001b[0;31m \u001b[0m\u001b[0;31m`\u001b[0m\u001b[0mndarray\u001b[0m\u001b[0;31m`\u001b[0m \u001b[0;32mwith\u001b[0m \u001b[0ma\u001b[0m \u001b[0mcopy\u001b[0m \u001b[0mof\u001b[0m \u001b[0mthe\u001b[0m \u001b[0mvalue\u001b[0m \u001b[0mof\u001b[0m \u001b[0mthis\u001b[0m \u001b[0mvariable\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1650\u001b[0m \"\"\"\n\u001b[0;32m-> 1651\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_variable\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0meval\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msession\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0msession\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1652\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1653\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0minitialized_value\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m/opt/conda/lib/python3.6/site-packages/tensorflow/python/framework/ops.py\u001b[0m in \u001b[0;36meval\u001b[0;34m(self, feed_dict, session)\u001b[0m\n\u001b[1;32m 711\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 712\u001b[0m \"\"\"\n\u001b[0;32m--> 713\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0m_eval_using_default_session\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgraph\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msession\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 714\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 715\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m/opt/conda/lib/python3.6/site-packages/tensorflow/python/framework/ops.py\u001b[0m in \u001b[0;36m_eval_using_default_session\u001b[0;34m(tensors, feed_dict, graph, session)\u001b[0m\n\u001b[1;32m 5146\u001b[0m \"`eval(session=sess)`\")\n\u001b[1;32m 5147\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0msession\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgraph\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mgraph\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 5148\u001b[0;31m raise ValueError(\"Cannot use the default session to evaluate tensor: \"\n\u001b[0m\u001b[1;32m 5149\u001b[0m \u001b[0;34m\"the tensor's graph is different from the session's \"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 5150\u001b[0m \u001b[0;34m\"graph. Pass an explicit session to \"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;31mValueError\u001b[0m: Cannot use the default session to evaluate tensor: the tensor's graph is different from the session's graph. Pass an explicit session to `eval(session=sess)`."
]
}
],
"source": [
"tf.reset_default_graph()\n",
"\n",
"with tf.variable_scope('y'):\n",
" a1 = tf.get_variable(name = 'a', initializer = tf.constant(1))\n",
" a2 = tf.get_variable(name = 'b', initializer = tf.constant(2))\n",
"try:\n",
" with tf.variable_scope('y'):\n",
" a3 = tf.get_variable(name = 'a', initializer = tf.constant(3))\n",
"except ValueError:\n",
" print('Variable already exists')\n",
" \n",
"with tf.Session() as sess:\n",
" sess.run(tf.global_variables_initializer())\n",
" print('name : {}, value : {}'.format(a1.name, a1.eval()))\n",
" print('name : {}, value : {}'.format(a2.name, a2.eval()))\n",
" print('name : {}, value : {}'.format(a3.name, a3.eval()))"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"#### Example 3"
]
},
{
"cell_type": "code",
"execution_count": 7,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"name : y/a:0, value : 1.0\n",
"name : y/a_1:0, value : 2.0\n",
"name : y_1/a:0, value : 3.0\n",
"name : y_1/a_1:0, value : 4.0\n"
]
}
],
"source": [
"tf.reset_default_graph()\n",
"\n",
"with tf.variable_scope('y') as scope:\n",
" a1 = tf.Variable(1., name = 'a')\n",
" scope.reuse_variables()\n",
" a2 = tf.Variable(2., name = 'a')\n",
"\n",
"with tf.variable_scope('y', reuse = tf.AUTO_REUSE):\n",
" a3 = tf.Variable(3., name = 'a')\n",
" a4 = tf.Variable(4., name = 'a')\n",
"\n",
"with tf.Session() as sess:\n",
" sess.run(tf.global_variables_initializer())\n",
" print('name : {}, value : {}'.format(a1.name, a1.eval()))\n",
" print('name : {}, value : {}'.format(a2.name, a2.eval()))\n",
" print('name : {}, value : {}'.format(a3.name, a3.eval()))\n",
" print('name : {}, value : {}'.format(a4.name, a4.eval()))"
]
},
{
"cell_type": "code",
"execution_count": 8,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"name : y/a:0, value : 1.0\n",
"name : y/a:0, value : 1.0\n",
"name : y/a:0, value : 1.0\n",
"name : y/a:0, value : 1.0\n"
]
}
],
"source": [
"tf.reset_default_graph()\n",
"\n",
"with tf.variable_scope('y') as scope:\n",
" a1 = tf.get_variable(initializer = tf.constant(1.), name = 'a')\n",
" scope.reuse_variables()\n",
" a2 = tf.get_variable(initializer = tf.constant(2.), name = 'a')\n",
"\n",
"with tf.variable_scope('y', reuse = tf.AUTO_REUSE):\n",
" a3 = tf.get_variable(initializer = tf.constant(3.), name = 'a')\n",
" a4 = tf.get_variable(initializer = tf.constant(4.), name = 'a')\n",
"\n",
"with tf.Session() as sess:\n",
" sess.run(tf.global_variables_initializer())\n",
" print('name : {}, value : {}'.format(a1.name, a1.eval()))\n",
" print('name : {}, value : {}'.format(a2.name, a2.eval()))\n",
" print('name : {}, value : {}'.format(a3.name, a3.eval()))\n",
" print('name : {}, value : {}'.format(a4.name, a4.eval()))"
]
},
{
"cell_type": "code",
"execution_count": 9,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"name : y/a:0, value : 1.0\n",
"name : y/a:0, value : 1.0\n",
"name : y/a:0, value : 1.0\n",
"name : y/a:0, value : 1.0\n"
]
}
],
"source": [
"tf.reset_default_graph()\n",
"\n",
"with tf.variable_scope('y') as scope:\n",
" a1 = tf.get_variable(initializer = tf.constant(1.), name = 'a')\n",
" scope.reuse_variables()\n",
" a2 = tf.get_variable(initializer = tf.constant(2.), name = 'a')\n",
"\n",
"with tf.variable_scope('y', reuse = tf.AUTO_REUSE):\n",
" a3 = tf.get_variable(initializer = tf.constant(3.), name = 'a')\n",
" a4 = tf.get_variable(initializer = tf.constant(4.), name = 'a')\n",
"\n",
"with tf.Session() as sess:\n",
" sess.run(tf.global_variables_initializer())\n",
" print('name : {}, value : {}'.format(a1.name, a1.eval()))\n",
" print('name : {}, value : {}'.format(a2.name, a2.eval()))\n",
" print('name : {}, value : {}'.format(a3.name, a3.eval()))\n",
" print('name : {}, value : {}'.format(a4.name, a4.eval()))"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### variable scope\n",
"**So what's the difference between name_scope and variable_scope?** \n",
"--> While both create namespaces, the main thing variable_scope does is to facilitate variable sharing."
]
},
{
"cell_type": "code",
"execution_count": 10,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"name : a:0, value : 1.0\n",
"name : a:0, value : 1.0\n"
]
}
],
"source": [
"# when using name scope, variable sharing ....\n",
"tf.reset_default_graph()\n",
"\n",
"with tf.name_scope('y'):\n",
" a1 = tf.get_variable(name = 'a', initializer = tf.constant(1.))\n",
"\n",
"tf.get_variable_scope().reuse_variables()\n",
" \n",
"with tf.name_scope('y'):\n",
" a2 = tf.get_variable(name = 'a', initializer = tf.constant(2.))\n",
"\n",
"with tf.Session() as sess:\n",
" sess.run(tf.global_variables_initializer())\n",
" print('name : {}, value : {}'.format(a1.name, a1.eval()))\n",
" print('name : {}, value : {}'.format(a2.name, a2.eval()))"
]
},
{
"cell_type": "code",
"execution_count": 11,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"name : y/a:0, value : 1.0\n",
"name : y/a:0, value : 1.0\n"
]
}
],
"source": [
"tf.reset_default_graph()\n",
"\n",
"with tf.variable_scope('y'):\n",
" a1 = tf.get_variable(name = 'a', initializer = tf.constant(1.))\n",
"\n",
"with tf.variable_scope('y') as scope:\n",
" scope.reuse_variables()\n",
" a2 = tf.get_variable(name = 'a', initializer = tf.constant(2.))\n",
"\n",
"with tf.Session() as sess:\n",
" sess.run(tf.global_variables_initializer())\n",
" print('name : {}, value : {}'.format(a1.name, a1.eval()))\n",
" print('name : {}, value : {}'.format(a2.name, a2.eval()))"
]
},
{
"cell_type": "code",
"execution_count": 12,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"name : y/a:0, value : 1.0\n",
"name : y/a:0, value : 1.0\n"
]
}
],
"source": [
"tf.reset_default_graph()\n",
"\n",
"with tf.variable_scope('y'):\n",
" a1 = tf.get_variable(name = 'a', initializer = tf.constant(1.))\n",
"\n",
"with tf.variable_scope('y', reuse = tf.AUTO_REUSE):\n",
" a2 = tf.get_variable(name = 'a', initializer = tf.constant(2.))\n",
"\n",
"with tf.Session() as sess:\n",
" sess.run(tf.global_variables_initializer())\n",
" print('name : {}, value : {}'.format(a1.name, a1.eval()))\n",
" print('name : {}, value : {}'.format(a2.name, a2.eval()))"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### non-variable shariang"
]
},
{
"cell_type": "code",
"execution_count": 13,
"metadata": {},
"outputs": [],
"source": [
"def two_hidden_layers(x):\n",
" assert x.shape.as_list() == [200, 100]\n",
" w1 = tf.Variable(name = 'h1_weights', initial_value = tf.random_normal([100, 50]))\n",
" b1 = tf.Variable(name = 'h1_biases', initial_value = tf.zeros(50))\n",
" h1 = tf.matmul(x, w1) + b1\n",
" \n",
" assert h1.shape.as_list() == [200, 50]\n",
" w2 = tf.Variable(name = 'h2_weights', initial_value = tf.random_normal([50, 10]))\n",
" b2 = tf.Variable(name = 'h2_biases', initial_value = tf.zeros(10))\n",
" logits = tf.matmul(h1, w2) + b2\n",
" return logits"
]
},
{
"cell_type": "code",
"execution_count": 14,
"metadata": {},
"outputs": [],
"source": [
"# result : Two sets of variables are created.\n",
"tf.reset_default_graph()\n",
"\n",
"x1 = tf.truncated_normal(shape = [200, 100], name = 'x1')\n",
"x2 = tf.truncated_normal(shape = [200, 100], name = 'x2')\n",
"\n",
"logits1 = two_hidden_layers(x1)\n",
"logits2 = two_hidden_layers(x2)"
]
},
{
"cell_type": "code",
"execution_count": 15,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"False\n"
]
}
],
"source": [
"with tf.Session() as sess:\n",
" sess.run(tf.global_variables_initializer())\n",
" l1 = sess.run(fetches = logits1, feed_dict = {x1 : np.ones(shape = [200, 100],)})\n",
" l2 = sess.run(fetches = logits2, feed_dict = {x2 : np.ones(shape = [200, 100])})\n",
"\n",
"print(np.all(l1 == l2))"
]
},
{
"cell_type": "code",
"execution_count": 16,
"metadata": {},
"outputs": [],
"source": [
"writer = tf.summary.FileWriter(logdir = '../graphs/lecture05/var_sharing_wo',\n",
" graph = tf.get_default_graph())\n",
"writer.close()"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### variable sharing\n",
"When using variable sharing, replace tf.Variable with tf.get_variable"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"#### example 1\n",
"```python\n",
"with tf.variable_socpe(name) as scope:\n",
" ... \n",
" scope.reuse_variables()\n",
" ...\n",
"```"
]
},
{
"cell_type": "code",
"execution_count": 17,
"metadata": {},
"outputs": [],
"source": [
"def two_hidden_layers(x):\n",
" assert x.shape.as_list() == [200, 100]\n",
" w1 = tf.get_variable(name = 'h1_weights',\n",
" shape = [100, 50],\n",
" initializer = tf.random_normal_initializer())\n",
" b1 = tf.get_variable(name = 'h1_biases', \n",
" shape = 50,\n",
" initializer = tf.zeros_initializer())\n",
" h1 = tf.matmul(x, w1) + b1\n",
" \n",
" assert h1.shape.as_list() == [200, 50]\n",
" w2 = tf.get_variable(name = 'h2_weights',\n",
" shape = [50, 10],\n",
" initializer = tf.random_normal_initializer())\n",
" b2 = tf.get_variable(name = 'h2_biases',\n",
" shape = 10,\n",
" initializer = tf.random_normal_initializer())\n",
" logits = tf.matmul(h1, w2) + b2\n",
" return logits"
]
},
{
"cell_type": "code",
"execution_count": 18,
"metadata": {},
"outputs": [],
"source": [
"tf.reset_default_graph()\n",
"\n",
"x1 = tf.truncated_normal(shape = [200, 100], name = 'x1')\n",
"x2 = tf.truncated_normal(shape = [200, 100], name = 'x2')\n",
"\n",
"with tf.variable_scope('two_layers') as scope:\n",
" logits1 = two_hidden_layers(x1)\n",
" scope.reuse_variables()\n",
" logits2 = two_hidden_layers(x2)"
]
},
{
"cell_type": "code",
"execution_count": 19,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"name : two_layers/add_1:0\n",
"name : two_layers/add_3:0\n",
"True\n"
]
}
],
"source": [
"with tf.Session() as sess:\n",
" sess.run(tf.global_variables_initializer())\n",
" l1 = sess.run(fetches = logits1, feed_dict = {x1 : np.ones(shape = [200, 100])})\n",
" l2 = sess.run(fetches = logits2, feed_dict = {x2 : np.ones(shape = [200, 100])})\n",
"\n",
"print('name : {}'.format(logits1.name))\n",
"print('name : {}'.format(logits2.name)) \n",
"print(np.all(l1 == l2))"
]
},
{
"cell_type": "code",
"execution_count": 20,
"metadata": {},
"outputs": [],
"source": [
"writer = tf.summary.FileWriter(logdir = '../graphs/lecture05/var_sharing_w1',\n",
" graph = tf.get_default_graph())\n",
"writer.close()"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"#### example 2\n",
"Graph is the same of example 1's result\n",
"```python\n",
"with tf.variable_scope(name, reuse = tf.AUTO_REUSE):\n",
" ...\n",
"```"
]
},
{
"cell_type": "code",
"execution_count": 21,
"metadata": {},
"outputs": [],
"source": [
"tf.reset_default_graph()\n",
"\n",
"x1 = tf.truncated_normal(shape = [200, 100], name = 'x1')\n",
"x2 = tf.truncated_normal(shape = [200, 100], name = 'x2')\n",
"\n",
"with tf.variable_scope('two_layers', reuse = tf.AUTO_REUSE):\n",
" logits1 = two_hidden_layers(x1)\n",
" logits2 = two_hidden_layers(x2)"
]
},
{
"cell_type": "code",
"execution_count": 22,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"name : two_layers/add_1:0\n",
"name : two_layers/add_3:0\n",
"True\n"
]
}
],
"source": [
"with tf.Session() as sess:\n",
" sess.run(tf.global_variables_initializer())\n",
" l1 = sess.run(fetches = logits1, feed_dict = {x1 : np.ones(shape = [200, 100])})\n",
" l2 = sess.run(fetches = logits2, feed_dict = {x2 : np.ones(shape = [200, 100])})\n",
"\n",
"print('name : {}'.format(logits1.name))\n",
"print('name : {}'.format(logits2.name))\n",
"print(np.all(l1 == l2))"
]
},
{
"cell_type": "code",
"execution_count": 23,
"metadata": {},
"outputs": [],
"source": [
"writer = tf.summary.FileWriter(logdir = '../graphs/lecture05/va_sharing_w2',\n",
" graph = tf.get_default_graph())\n",
"writer.close()"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"#### example 3"
]
},
{
"cell_type": "code",
"execution_count": 24,
"metadata": {},
"outputs": [],
"source": [
"def fully_connected(x, output_dim, scope):\n",
" with tf.variable_scope(scope, reuse = tf.AUTO_REUSE) as scope:\n",
" w = tf.get_variable('weights', [x.shape[1], output_dim], initializer=tf.random_normal_initializer())\n",
" b = tf.get_variable('biases', [output_dim], initializer=tf.constant_initializer(0.0))\n",
" return tf.matmul(x, w) + b\n",
" \n",
"def two_hidden_layers(x):\n",
" h = fully_connected(x, 50, 'h1')\n",
" h = fully_connected(h, 10, 'h2')\n",
" return h"
]
},
{
"cell_type": "code",
"execution_count": 25,
"metadata": {},
"outputs": [],
"source": [
"tf.reset_default_graph()\n",
"\n",
"x1 = tf.truncated_normal(shape = [200, 100], name = 'x1')\n",
"x2 = tf.truncated_normal(shape = [200, 100], name = 'x2')\n",
"\n",
"logits1 = two_hidden_layers(x1)\n",
"logits2 = two_hidden_layers(x2)"
]
},
{
"cell_type": "code",
"execution_count": 26,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"name : h2/add:0\n",
"name : h2_1/add:0\n",
"True\n"
]
}
],
"source": [
"with tf.Session() as sess:\n",
" sess.run(tf.global_variables_initializer())\n",
" l1 = sess.run(fetches = logits1, feed_dict = {x1 : np.ones(shape = [200, 100])})\n",
" l2 = sess.run(fetches = logits2, feed_dict = {x2 : np.ones(shape = [200, 100])})\n",
"\n",
"print('name : {}'.format(logits1.name))\n",
"print('name : {}'.format(logits2.name))\n",
"print(np.all(l1 == l2))"
]
},
{
"cell_type": "code",
"execution_count": 27,
"metadata": {},
"outputs": [],
"source": [
"writer = tf.summary.FileWriter(logdir = '../graphs/lecture05/va_sharing_w3',\n",
" graph = tf.get_default_graph())\n",
"writer.close()"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"#### example 4"
]
},
{
"cell_type": "code",
"execution_count": 28,
"metadata": {},
"outputs": [],
"source": [
"def fully_connected(x, output_dim, scope):\n",
" with tf.variable_scope(scope) as scope:\n",
" w = tf.get_variable('weights', [x.shape[1], output_dim], initializer=tf.random_normal_initializer())\n",
" b = tf.get_variable('biases', [output_dim], initializer=tf.constant_initializer(0.0))\n",
" return tf.matmul(x, w) + b\n",
"\n",
"def two_hidden_layers(x):\n",
" with tf.variable_scope('two_hidden_layers', reuse = tf.AUTO_REUSE):\n",
" h = fully_connected(x, 50, 'h1')\n",
" h = fully_connected(h, 10, 'h2')\n",
" return h"
]
},
{
"cell_type": "code",
"execution_count": 29,
"metadata": {},
"outputs": [],
"source": [
"tf.reset_default_graph()\n",
"\n",
"x1 = tf.truncated_normal(shape = [200, 100], name = 'x1')\n",
"x2 = tf.truncated_normal(shape = [200, 100], name = 'x2')\n",
"\n",
"logits1 = two_hidden_layers(x1)\n",
"logits2 = two_hidden_layers(x2)"
]
},
{
"cell_type": "code",
"execution_count": 30,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"name : two_hidden_layers/h2/add:0\n",
"name : two_hidden_layers_1/h2/add:0\n",
"True\n"
]
}
],
"source": [
"with tf.Session() as sess:\n",
" sess.run(tf.global_variables_initializer())\n",
" l1 = sess.run(fetches = logits1, feed_dict = {x1 : np.ones(shape = [200, 100])})\n",
" l2 = sess.run(fetches = logits2, feed_dict = {x2 : np.ones(shape = [200, 100])})\n",
"\n",
"print('name : {}'.format(logits1.name))\n",
"print('name : {}'.format(logits2.name))\n",
"print(np.all(l1 == l2))"
]
},
{
"cell_type": "code",
"execution_count": 31,
"metadata": {},
"outputs": [],
"source": [
"writer = tf.summary.FileWriter(logdir = '../graphs/lecture05/va_sharing_w4',\n",
" graph = tf.get_default_graph())\n",
"writer.close()"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.6.8"
},
"varInspector": {
"cols": {
"lenName": 16,
"lenType": 16,
"lenVar": 40
},
"kernels_config": {
"python": {
"delete_cmd_postfix": "",
"delete_cmd_prefix": "del ",
"library": "var_list.py",
"varRefreshCmd": "print(var_dic_list())"
},
"r": {
"delete_cmd_postfix": ") ",
"delete_cmd_prefix": "rm(",
"library": "var_list.r",
"varRefreshCmd": "cat(var_dic_list()) "
}
},
"types_to_exclude": [
"module",
"function",
"builtin_function_or_method",
"instance",
"_Feature"
],
"window_display": false
}
},
"nbformat": 4,
"nbformat_minor": 2
}