{ "cells": [ { "cell_type": "code", "execution_count": 1, "metadata": { "collapsed": true }, "outputs": [], "source": [ "# http://www.wildml.com/2016/08/rnns-in-tensorflow-a-practical-guide-and-undocumented-features/\n", "# http://learningtensorflow.com/index.html\n", "# http://suriyadeepan.github.io/2016-12-31-practical-seq2seq/\n", "\n", "import tensorflow as tf\n", "import numpy as np\n", "from tensorflow.contrib import rnn\n", "import pprint\n", "pp = pprint.PrettyPrinter(indent=4)\n", "sess = tf.InteractiveSession()" ] }, { "cell_type": "code", "execution_count": 2, "metadata": { "collapsed": true }, "outputs": [], "source": [ "# One hot encoding for each char in 'hello'\n", "h = [1, 0, 0, 0]\n", "e = [0, 1, 0, 0]\n", "l = [0, 0, 1, 0]\n", "o = [0, 0, 0, 1]" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "![image](https://cloud.githubusercontent.com/assets/901975/23348727/cc981856-fce7-11e6-83ea-4b187473466b.png)\n" ] }, { "cell_type": "code", "execution_count": 3, "metadata": { "collapsed": false }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "2 2\n", "array([[[ 1., 0., 0., 0.]]], dtype=float32)\n", "array([[[ 0.28406101, 0.53163123]]], dtype=float32)\n" ] } ], "source": [ "with tf.variable_scope('one_cell') as scope:\n", " # One cell RNN input_dim (4) -> output_dim (2)\n", " hidden_size = 2\n", " cell = tf.contrib.rnn.BasicRNNCell(num_units=hidden_size)\n", " print(cell.output_size, cell.state_size)\n", "\n", " x_data = np.array([[h]], dtype=np.float32) # x_data = [[[1,0,0,0]]]\n", " pp.pprint(x_data)\n", " outputs, _states = tf.nn.dynamic_rnn(cell, x_data, dtype=tf.float32)\n", "\n", " sess.run(tf.global_variables_initializer())\n", " pp.pprint(outputs.eval())" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "![image](https://cloud.githubusercontent.com/assets/901975/23383634/649efd0a-fd82-11e6-925d-8041242743b0.png)" ] }, { "cell_type": "code", "execution_count": 4, "metadata": { "collapsed": false }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "(1, 5, 4)\n", "array([[[ 1., 0., 0., 0.],\n", " [ 0., 1., 0., 0.],\n", " [ 0., 0., 1., 0.],\n", " [ 0., 0., 1., 0.],\n", " [ 0., 0., 0., 1.]]], dtype=float32)\n", "array([[[ 0.50641137, 0.55783308],\n", " [-0.61545879, -0.04334207],\n", " [-0.77109283, -0.23411733],\n", " [-0.76478487, -0.07172935],\n", " [-0.72683465, 0.5266667 ]]], dtype=float32)\n" ] } ], "source": [ "with tf.variable_scope('two_sequances') as scope:\n", " # One cell RNN input_dim (4) -> output_dim (2). sequence: 5\n", " hidden_size = 2\n", " cell = tf.contrib.rnn.BasicRNNCell(num_units=hidden_size)\n", " x_data = np.array([[h, e, l, l, o]], dtype=np.float32)\n", " print(x_data.shape)\n", " pp.pprint(x_data)\n", " outputs, states = tf.nn.dynamic_rnn(cell, x_data, dtype=tf.float32)\n", " sess.run(tf.global_variables_initializer())\n", " pp.pprint(outputs.eval())" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "![image](https://cloud.githubusercontent.com/assets/901975/23383681/9943a9fc-fd82-11e6-8121-bd187994e249.png)" ] }, { "cell_type": "code", "execution_count": 5, "metadata": { "collapsed": false }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "array([[[ 1., 0., 0., 0.],\n", " [ 0., 1., 0., 0.],\n", " [ 0., 0., 1., 0.],\n", " [ 0., 0., 1., 0.],\n", " [ 0., 0., 0., 1.]],\n", "\n", " [[ 0., 1., 0., 0.],\n", " [ 0., 0., 0., 1.],\n", " [ 0., 0., 1., 0.],\n", " [ 0., 0., 1., 0.],\n", " [ 0., 0., 1., 0.]],\n", "\n", " [[ 0., 0., 1., 0.],\n", " [ 0., 0., 1., 0.],\n", " [ 0., 1., 0., 0.],\n", " [ 0., 1., 0., 0.],\n", " [ 0., 0., 1., 0.]]], dtype=float32)\n", "array([[[ 0.05147979, -0.12499733],\n", " [-0.03096316, 0.0606184 ],\n", " [ 0.07608285, -0.03678102],\n", " [ 0.13522112, -0.0920362 ],\n", " [ 0.0235186 , 0.02936662]],\n", "\n", " [[-0.08701393, 0.11020429],\n", " [-0.10396396, 0.17681563],\n", " [ 0.04137699, 0.01251686],\n", " [ 0.11497379, -0.06123064],\n", " [ 0.15778907, -0.10736831]],\n", "\n", " [[ 0.09156778, -0.06834684],\n", " [ 0.14449464, -0.11151826],\n", " [ 0.0351325 , 0.04766054],\n", " [-0.06127799, 0.13465708],\n", " [ 0.06125095, 0.00202556]]], dtype=float32)\n" ] } ], "source": [ "with tf.variable_scope('3_batches') as scope:\n", " # One cell RNN input_dim (4) -> output_dim (2). sequence: 5, batch 3\n", " # 3 batches 'hello', 'eolll', 'lleel'\n", " x_data = np.array([[h, e, l, l, o],\n", " [e, o, l, l, l],\n", " [l, l, e, e, l]], dtype=np.float32)\n", " pp.pprint(x_data)\n", " \n", " hidden_size = 2\n", " cell = rnn.BasicLSTMCell(num_units=hidden_size, state_is_tuple=True)\n", " outputs, _states = tf.nn.dynamic_rnn(\n", " cell, x_data, dtype=tf.float32)\n", " sess.run(tf.global_variables_initializer())\n", " pp.pprint(outputs.eval())" ] }, { "cell_type": "code", "execution_count": 6, "metadata": { "collapsed": false }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "array([[[ 1., 0., 0., 0.],\n", " [ 0., 1., 0., 0.],\n", " [ 0., 0., 1., 0.],\n", " [ 0., 0., 1., 0.],\n", " [ 0., 0., 0., 1.]],\n", "\n", " [[ 0., 1., 0., 0.],\n", " [ 0., 0., 0., 1.],\n", " [ 0., 0., 1., 0.],\n", " [ 0., 0., 1., 0.],\n", " [ 0., 0., 1., 0.]],\n", "\n", " [[ 0., 0., 1., 0.],\n", " [ 0., 0., 1., 0.],\n", " [ 0., 1., 0., 0.],\n", " [ 0., 1., 0., 0.],\n", " [ 0., 0., 1., 0.]]], dtype=float32)\n", "array([[[-0.05008912, -0.01069659],\n", " [-0.12182344, -0.10044969],\n", " [-0.07028138, -0.16675197],\n", " [-0.04712334, -0.20867857],\n", " [-0.08468042, -0.22286232]],\n", "\n", " [[-0.06402468, -0.09570512],\n", " [-0.10435093, -0.13225234],\n", " [-0.06158386, -0.17562068],\n", " [ 0. , 0. ],\n", " [ 0. , 0. ]],\n", "\n", " [[-0.03898201, -0.07288275],\n", " [-0.0454888 , -0.13435912],\n", " [-0.07696934, -0.17394406],\n", " [-0.08638199, -0.20610151],\n", " [ 0. , 0. ]]], dtype=float32)\n" ] } ], "source": [ "with tf.variable_scope('3_batches_dynamic_length') as scope:\n", " # One cell RNN input_dim (4) -> output_dim (5). sequence: 5, batch 3\n", " # 3 batches 'hello', 'eolll', 'lleel'\n", " x_data = np.array([[h, e, l, l, o],\n", " [e, o, l, l, l],\n", " [l, l, e, e, l]], dtype=np.float32)\n", " pp.pprint(x_data)\n", " \n", " hidden_size = 2\n", " cell = rnn.BasicLSTMCell(num_units=hidden_size, state_is_tuple=True)\n", " outputs, _states = tf.nn.dynamic_rnn(\n", " cell, x_data, sequence_length=[5,3,4], dtype=tf.float32)\n", " sess.run(tf.global_variables_initializer())\n", " pp.pprint(outputs.eval())" ] }, { "cell_type": "code", "execution_count": 7, "metadata": { "collapsed": false }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "array([[[ 1., 0., 0., 0.],\n", " [ 0., 1., 0., 0.],\n", " [ 0., 0., 1., 0.],\n", " [ 0., 0., 1., 0.],\n", " [ 0., 0., 0., 1.]],\n", "\n", " [[ 0., 1., 0., 0.],\n", " [ 0., 0., 0., 1.],\n", " [ 0., 0., 1., 0.],\n", " [ 0., 0., 1., 0.],\n", " [ 0., 0., 1., 0.]],\n", "\n", " [[ 0., 0., 1., 0.],\n", " [ 0., 0., 1., 0.],\n", " [ 0., 1., 0., 0.],\n", " [ 0., 1., 0., 0.],\n", " [ 0., 0., 1., 0.]]], dtype=float32)\n", "array([[[-0.10699597, -0.06735446],\n", " [-0.01214837, -0.03093243],\n", " [-0.11962769, 0.10190663],\n", " [-0.20870663, 0.20675965],\n", " [-0.0284863 , 0.1948234 ]],\n", "\n", " [[ 0.06502327, 0.01373457],\n", " [ 0.11152857, 0.07884337],\n", " [-0.00791237, 0.20213564],\n", " [-0.14327727, 0.27671731],\n", " [-0.2389248 , 0.32179514]],\n", "\n", " [[-0.11509674, 0.13784541],\n", " [-0.20916083, 0.231289 ],\n", " [-0.12755771, 0.12080902],\n", " [-0.03355049, 0.0995911 ],\n", " [-0.14717762, 0.24482173]]], dtype=float32)\n" ] } ], "source": [ "with tf.variable_scope('initial_state') as scope:\n", " batch_size = 3\n", " x_data = np.array([[h, e, l, l, o],\n", " [e, o, l, l, l],\n", " [l, l, e, e, l]], dtype=np.float32)\n", " pp.pprint(x_data)\n", " \n", " # One cell RNN input_dim (4) -> output_dim (5). sequence: 5, batch: 3\n", " hidden_size=2\n", " cell = rnn.BasicLSTMCell(num_units=hidden_size, state_is_tuple=True)\n", " initial_state = cell.zero_state(batch_size, tf.float32)\n", " outputs, _states = tf.nn.dynamic_rnn(cell, x_data,\n", " initial_state=initial_state, dtype=tf.float32)\n", " sess.run(tf.global_variables_initializer())\n", " pp.pprint(outputs.eval())" ] }, { "cell_type": "code", "execution_count": 8, "metadata": { "collapsed": false }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "array([[[ 0., 1., 2.],\n", " [ 3., 4., 5.],\n", " [ 6., 7., 8.],\n", " [ 9., 10., 11.],\n", " [ 12., 13., 14.]],\n", "\n", " [[ 15., 16., 17.],\n", " [ 18., 19., 20.],\n", " [ 21., 22., 23.],\n", " [ 24., 25., 26.],\n", " [ 27., 28., 29.]],\n", "\n", " [[ 30., 31., 32.],\n", " [ 33., 34., 35.],\n", " [ 36., 37., 38.],\n", " [ 39., 40., 41.],\n", " [ 42., 43., 44.]]], dtype=float32)\n" ] } ], "source": [ "# Create input data\n", "batch_size=3\n", "sequence_length=5\n", "input_dim=3\n", "\n", "x_data = np.arange(45, dtype=np.float32).reshape(batch_size, sequence_length, input_dim)\n", "pp.pprint(x_data) # batch, sequence_length, input_dim" ] }, { "cell_type": "code", "execution_count": 9, "metadata": { "collapsed": false }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "array([[[ 2.22847834e-01, -1.12753026e-02, -2.09515661e-01,\n", " 1.33116916e-01, 2.59324769e-03],\n", " [ 4.09052998e-01, 1.67620078e-01, -6.23385966e-01,\n", " 2.95646906e-01, 6.76166117e-02],\n", " [ 3.84651929e-01, 2.06338793e-01, -8.70478570e-01,\n", " 2.97074020e-01, 2.39257380e-01],\n", " [ 3.59520555e-01, 1.37896106e-01, -9.57255721e-01,\n", " 2.04702541e-01, 4.43462312e-01],\n", " [ 3.53975475e-01, 7.35354945e-02, -9.84238148e-01,\n", " 1.21178836e-01, 5.96657693e-01]],\n", "\n", " [[ 3.25354218e-01, 1.92702170e-02, -6.57426417e-01,\n", " 5.44418097e-02, 4.50136989e-01],\n", " [ 3.43781590e-01, 1.43233798e-02, -8.98284197e-01,\n", " 3.75912562e-02, 6.67491138e-01],\n", " [ 3.27331871e-01, 7.51791801e-03, -9.64996517e-01,\n", " 2.11958308e-02, 7.30534494e-01],\n", " [ 3.09874654e-01, 3.57595109e-03, -9.84839082e-01,\n", " 1.17445569e-02, 7.47434497e-01],\n", " [ 2.92880654e-01, 1.65894395e-03, -9.91725981e-01,\n", " 6.53389702e-03, 7.52497494e-01]],\n", "\n", " [[ 2.50953138e-01, 4.36680217e-04, -7.03740895e-01,\n", " 3.34404502e-03, 6.93815947e-01],\n", " [ 2.70643830e-01, 3.10635078e-04, -9.09156263e-01,\n", " 2.06257636e-03, 7.49875069e-01],\n", " [ 2.49837011e-01, 1.57007758e-04, -9.63405490e-01,\n", " 1.15697947e-03, 7.56247699e-01],\n", " [ 2.32801229e-01, 7.37999653e-05, -9.80127990e-01,\n", " 6.45897642e-04, 7.57673502e-01],\n", " [ 2.17864171e-01, 3.41164960e-05, -9.86101806e-01,\n", " 3.62177641e-04, 7.58471072e-01]]], dtype=float32)\n" ] } ], "source": [ "with tf.variable_scope('generated_data') as scope:\n", " # One cell RNN input_dim (3) -> output_dim (5). sequence: 5, batch: 3\n", " cell = rnn.BasicLSTMCell(num_units=5, state_is_tuple=True)\n", " initial_state = cell.zero_state(batch_size, tf.float32)\n", " outputs, _states = tf.nn.dynamic_rnn(cell, x_data,\n", " initial_state=initial_state, dtype=tf.float32)\n", " sess.run(tf.global_variables_initializer())\n", " pp.pprint(outputs.eval())" ] }, { "cell_type": "code", "execution_count": 10, "metadata": { "collapsed": false }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "dynamic rnn: Tensor(\"MultiRNNCell/rnn/transpose:0\", shape=(3, 5, 5), dtype=float32)\n", "array([[[ 1.20540499e-03, 1.09513826e-03, 2.20310478e-03,\n", " 3.41569167e-03, -3.82219645e-04],\n", " [ 2.57245242e-03, 1.25761738e-03, 4.36156709e-03,\n", " 6.17466075e-03, 4.01454419e-03],\n", " [ 5.22803096e-03, -1.61505232e-04, 7.24773575e-03,\n", " 8.18122737e-03, 1.67929009e-02],\n", " [ 9.85822361e-03, -2.62762001e-03, 1.19020101e-02,\n", " 1.00588454e-02, 3.70351225e-02],\n", " [ 1.61787979e-02, -5.48204128e-03, 1.84861515e-02,\n", " 1.19717196e-02, 6.11032583e-02]],\n", "\n", " [[ 6.90252695e-04, -1.99922113e-04, 8.86004767e-04,\n", " 2.22551913e-04, 5.27152233e-03],\n", " [ 2.40178546e-03, -4.28301457e-04, 3.41633009e-03,\n", " 8.02060647e-04, 1.65986698e-02],\n", " [ 5.17574837e-03, -4.83547512e-04, 7.89437629e-03,\n", " 1.74584519e-03, 3.25243883e-02],\n", " [ 8.80666822e-03, -2.78592808e-04, 1.41408108e-02,\n", " 2.94073927e-03, 5.08417375e-02],\n", " [ 1.29447393e-02, 1.63523509e-04, 2.16512978e-02,\n", " 4.22920240e-03, 6.95042834e-02]],\n", "\n", " [[ 1.66354628e-04, 1.25368184e-04, 4.29028471e-04,\n", " 8.79739600e-05, 2.20423983e-03],\n", " [ 5.88878815e-04, 4.49586252e-04, 1.61941827e-03,\n", " 3.03511770e-04, 7.01069506e-03],\n", " [ 1.31306029e-03, 9.97182331e-04, 3.73112643e-03,\n", " 6.62667328e-04, 1.40041607e-02],\n", " [ 2.34889123e-03, 1.75245479e-03, 6.76015578e-03,\n", " 1.16268732e-03, 2.24791896e-02],\n", " [ 3.67631158e-03, 2.67310557e-03, 1.05806263e-02,\n", " 1.78660743e-03, 3.17095295e-02]]], dtype=float32)\n" ] } ], "source": [ "with tf.variable_scope('MultiRNNCell') as scope:\n", " # Make rnn\n", " cell = rnn.BasicLSTMCell(num_units=5, state_is_tuple=True)\n", " cell = rnn.MultiRNNCell([cell] * 3, state_is_tuple=True) # 3 layers\n", "\n", " # rnn in/out\n", " outputs, _states = tf.nn.dynamic_rnn(cell, x_data, dtype=tf.float32)\n", " print(\"dynamic rnn: \", outputs)\n", " sess.run(tf.global_variables_initializer())\n", " pp.pprint(outputs.eval()) # batch size, unrolling (time), hidden_size" ] }, { "cell_type": "code", "execution_count": 11, "metadata": { "collapsed": false }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "dynamic rnn: Tensor(\"dynamic_rnn/rnn/transpose:0\", shape=(3, 5, 5), dtype=float32)\n", "array([[[ 2.78908219e-02, -1.48126215e-01, -7.39237070e-02,\n", " 2.88070235e-02, 6.32265732e-02],\n", " [ 0.00000000e+00, 0.00000000e+00, 0.00000000e+00,\n", " 0.00000000e+00, 0.00000000e+00],\n", " [ 0.00000000e+00, 0.00000000e+00, 0.00000000e+00,\n", " 0.00000000e+00, 0.00000000e+00],\n", " [ 0.00000000e+00, 0.00000000e+00, 0.00000000e+00,\n", " 0.00000000e+00, 0.00000000e+00],\n", " [ 0.00000000e+00, 0.00000000e+00, 0.00000000e+00,\n", " 0.00000000e+00, 0.00000000e+00]],\n", "\n", " [[ 4.35886432e-06, 2.38276646e-03, 3.29953309e-06,\n", " 1.10525981e-01, 1.76965340e-08],\n", " [ 6.48097910e-07, 1.61049131e-03, 2.29394141e-06,\n", " 1.04240455e-01, 8.04429634e-10],\n", " [ 7.62259091e-08, 8.23478971e-04, 1.28897921e-06,\n", " 8.51678029e-02, 3.12711766e-11],\n", " [ 0.00000000e+00, 0.00000000e+00, 0.00000000e+00,\n", " 0.00000000e+00, 0.00000000e+00],\n", " [ 0.00000000e+00, 0.00000000e+00, 0.00000000e+00,\n", " 0.00000000e+00, 0.00000000e+00]],\n", "\n", " [[ 6.94665575e-11, 7.26437429e-05, 5.22463635e-11,\n", " 3.71553451e-02, 1.31825178e-15],\n", " [ 1.02408203e-11, 3.06548645e-05, 2.59286290e-11,\n", " 3.11325025e-02, 6.17112537e-17],\n", " [ 0.00000000e+00, 0.00000000e+00, 0.00000000e+00,\n", " 0.00000000e+00, 0.00000000e+00],\n", " [ 0.00000000e+00, 0.00000000e+00, 0.00000000e+00,\n", " 0.00000000e+00, 0.00000000e+00],\n", " [ 0.00000000e+00, 0.00000000e+00, 0.00000000e+00,\n", " 0.00000000e+00, 0.00000000e+00]]], dtype=float32)\n" ] } ], "source": [ "with tf.variable_scope('dynamic_rnn') as scope:\n", " cell = rnn.BasicLSTMCell(num_units=5, state_is_tuple=True)\n", " outputs, _states = tf.nn.dynamic_rnn(cell, x_data, dtype=tf.float32,\n", " sequence_length=[1, 3, 2])\n", " # lentgh 1 for batch 1, lentgh 2 for batch 2\n", " \n", " print(\"dynamic rnn: \", outputs)\n", " sess.run(tf.global_variables_initializer())\n", " pp.pprint(outputs.eval()) # batch size, unrolling (time), hidden_size" ] }, { "cell_type": "code", "execution_count": 12, "metadata": { "collapsed": false }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "( array([[[ -1.00286074e-01, 2.11599812e-01, 4.26371992e-02,\n", " 3.00979782e-02, 9.23170671e-02],\n", " [ 4.75897230e-02, 3.76924574e-01, -4.24038619e-02,\n", " 1.01732194e-01, 4.52822745e-02],\n", " [ 0.00000000e+00, 0.00000000e+00, 0.00000000e+00,\n", " 0.00000000e+00, 0.00000000e+00],\n", " [ 0.00000000e+00, 0.00000000e+00, 0.00000000e+00,\n", " 0.00000000e+00, 0.00000000e+00],\n", " [ 0.00000000e+00, 0.00000000e+00, 0.00000000e+00,\n", " 0.00000000e+00, 0.00000000e+00]],\n", "\n", " [[ 1.03152019e-03, 2.08642017e-02, -8.34622793e-03,\n", " 3.03512835e-03, 7.79041613e-07],\n", " [ 3.73781077e-04, 3.19686793e-02, -4.44590999e-03,\n", " 1.57183059e-03, 2.03600507e-07],\n", " [ 1.10292378e-04, 3.78062688e-02, -1.95636135e-03,\n", " 6.04417117e-04, 4.30081180e-08],\n", " [ 0.00000000e+00, 0.00000000e+00, 0.00000000e+00,\n", " 0.00000000e+00, 0.00000000e+00],\n", " [ 0.00000000e+00, 0.00000000e+00, 0.00000000e+00,\n", " 0.00000000e+00, 0.00000000e+00]],\n", "\n", " [[ 2.00289901e-06, 7.53075059e-04, -1.73141569e-04,\n", " 1.94623790e-05, 1.57498038e-12],\n", " [ 0.00000000e+00, 0.00000000e+00, 0.00000000e+00,\n", " 0.00000000e+00, 0.00000000e+00],\n", " [ 0.00000000e+00, 0.00000000e+00, 0.00000000e+00,\n", " 0.00000000e+00, 0.00000000e+00],\n", " [ 0.00000000e+00, 0.00000000e+00, 0.00000000e+00,\n", " 0.00000000e+00, 0.00000000e+00],\n", " [ 0.00000000e+00, 0.00000000e+00, 0.00000000e+00,\n", " 0.00000000e+00, 0.00000000e+00]]], dtype=float32),\n", " array([[[ -6.70132190e-02, -9.79832560e-02, 4.76102352e-01,\n", " -4.57281440e-01, 1.74056783e-01],\n", " [ -6.16685301e-03, -8.18507746e-03, 6.10428035e-01,\n", " -4.55797821e-01, 3.88450697e-02],\n", " [ 0.00000000e+00, 0.00000000e+00, 0.00000000e+00,\n", " 0.00000000e+00, 0.00000000e+00],\n", " [ 0.00000000e+00, 0.00000000e+00, 0.00000000e+00,\n", " 0.00000000e+00, 0.00000000e+00],\n", " [ 0.00000000e+00, 0.00000000e+00, 0.00000000e+00,\n", " 0.00000000e+00, 0.00000000e+00]],\n", "\n", " [[ -1.84493047e-07, -1.42245389e-08, 9.95001912e-01,\n", " -9.34846520e-01, 1.78247530e-04],\n", " [ -1.38226666e-08, -4.68234562e-10, 9.63908970e-01,\n", " -9.22717512e-01, 3.82163526e-05],\n", " [ -1.21463872e-09, -1.61447106e-11, 7.61411786e-01,\n", " -7.34403074e-01, 7.10739550e-06],\n", " [ 0.00000000e+00, 0.00000000e+00, 0.00000000e+00,\n", " 0.00000000e+00, 0.00000000e+00],\n", " [ 0.00000000e+00, 0.00000000e+00, 0.00000000e+00,\n", " 0.00000000e+00, 0.00000000e+00]],\n", "\n", " [[ -4.17174026e-13, -5.10518192e-16, 7.61586428e-01,\n", " -7.53889859e-01, 7.72119435e-08],\n", " [ 0.00000000e+00, 0.00000000e+00, 0.00000000e+00,\n", " 0.00000000e+00, 0.00000000e+00],\n", " [ 0.00000000e+00, 0.00000000e+00, 0.00000000e+00,\n", " 0.00000000e+00, 0.00000000e+00],\n", " [ 0.00000000e+00, 0.00000000e+00, 0.00000000e+00,\n", " 0.00000000e+00, 0.00000000e+00],\n", " [ 0.00000000e+00, 0.00000000e+00, 0.00000000e+00,\n", " 0.00000000e+00, 0.00000000e+00]]], dtype=float32))\n", "( LSTMStateTuple(c=array([[ 3.18704367e-01, 5.48799157e-01, -1.17933072e-01,\n", " 4.72177684e-01, 3.20702165e-01],\n", " [ 2.99980783e+00, 3.81983854e-02, -5.47450874e-03,\n", " 2.26725507e+00, 4.89415321e-03],\n", " [ 1.00000000e+00, 7.54415756e-04, -4.99026908e-04,\n", " 8.86694312e-01, 2.51370238e-05]], dtype=float32), h=array([[ 4.75897230e-02, 3.76924574e-01, -4.24038619e-02,\n", " 1.01732194e-01, 4.52822745e-02],\n", " [ 1.10292378e-04, 3.78062688e-02, -1.95636135e-03,\n", " 6.04417117e-04, 4.30081180e-08],\n", " [ 2.00289901e-06, 7.53075059e-04, -1.73141569e-04,\n", " 1.94623790e-05, 1.57498038e-12]], dtype=float32)),\n", " LSTMStateTuple(c=array([[ -3.15686703e-01, -3.91717076e-01, 7.36252010e-01,\n", " -9.45747852e-01, 7.38465428e-01],\n", " [ -6.19174778e-01, -4.69935462e-02, 2.99472475e+00,\n", " -2.98918104e+00, 2.54934883e+00],\n", " [ -7.41944671e-01, -1.55421533e-03, 9.99981403e-01,\n", " -9.99999642e-01, 9.77244020e-01]], dtype=float32), h=array([[ -6.70132190e-02, -9.79832560e-02, 4.76102352e-01,\n", " -4.57281440e-01, 1.74056783e-01],\n", " [ -1.84493047e-07, -1.42245389e-08, 9.95001912e-01,\n", " -9.34846520e-01, 1.78247530e-04],\n", " [ -4.17174026e-13, -5.10518192e-16, 7.61586428e-01,\n", " -7.53889859e-01, 7.72119435e-08]], dtype=float32)))\n" ] } ], "source": [ "with tf.variable_scope('bi-directional') as scope:\n", " # bi-directional rnn\n", " cell_fw = rnn.BasicLSTMCell(num_units=5, state_is_tuple=True)\n", " cell_bw = rnn.BasicLSTMCell(num_units=5, state_is_tuple=True)\n", "\n", " outputs, states = tf.nn.bidirectional_dynamic_rnn(cell_fw, cell_bw, x_data,\n", " sequence_length=[2, 3, 1],\n", " dtype=tf.float32)\n", "\n", " sess.run(tf.global_variables_initializer())\n", " pp.pprint(sess.run(outputs))\n", " pp.pprint(sess.run(states))" ] }, { "cell_type": "code", "execution_count": 13, "metadata": { "collapsed": false }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "array([[[ 0., 1., 2.],\n", " [ 3., 4., 5.],\n", " [ 6., 7., 8.],\n", " [ 9., 10., 11.],\n", " [ 12., 13., 14.]],\n", "\n", " [[ 15., 16., 17.],\n", " [ 18., 19., 20.],\n", " [ 21., 22., 23.],\n", " [ 24., 25., 26.],\n", " [ 27., 28., 29.]],\n", "\n", " [[ 30., 31., 32.],\n", " [ 33., 34., 35.],\n", " [ 36., 37., 38.],\n", " [ 39., 40., 41.],\n", " [ 42., 43., 44.]]], dtype=float32)\n", "array([[ 0., 1., 2.],\n", " [ 3., 4., 5.],\n", " [ 6., 7., 8.],\n", " [ 9., 10., 11.],\n", " [ 12., 13., 14.],\n", " [ 15., 16., 17.],\n", " [ 18., 19., 20.],\n", " [ 21., 22., 23.],\n", " [ 24., 25., 26.],\n", " [ 27., 28., 29.],\n", " [ 30., 31., 32.],\n", " [ 33., 34., 35.],\n", " [ 36., 37., 38.],\n", " [ 39., 40., 41.],\n", " [ 42., 43., 44.]], dtype=float32)\n", "array([[[ 25., 28., 31., 34., 37.],\n", " [ 70., 82., 94., 106., 118.],\n", " [ 115., 136., 157., 178., 199.],\n", " [ 160., 190., 220., 250., 280.],\n", " [ 205., 244., 283., 322., 361.]],\n", "\n", " [[ 250., 298., 346., 394., 442.],\n", " [ 295., 352., 409., 466., 523.],\n", " [ 340., 406., 472., 538., 604.],\n", " [ 385., 460., 535., 610., 685.],\n", " [ 430., 514., 598., 682., 766.]],\n", "\n", " [[ 475., 568., 661., 754., 847.],\n", " [ 520., 622., 724., 826., 928.],\n", " [ 565., 676., 787., 898., 1009.],\n", " [ 610., 730., 850., 970., 1090.],\n", " [ 655., 784., 913., 1042., 1171.]]], dtype=float32)\n" ] } ], "source": [ "# flattern based softmax\n", "hidden_size=3\n", "sequence_length=5\n", "batch_size=3\n", "num_classes=5\n", "\n", "pp.pprint(x_data) # hidden_size=3, sequence_length=4, batch_size=2\n", "x_data = x_data.reshape(-1, hidden_size)\n", "pp.pprint(x_data)\n", "\n", "softmax_w = np.arange(15, dtype=np.float32).reshape(hidden_size, num_classes)\n", "outputs = np.matmul(x_data, softmax_w)\n", "outputs = outputs.reshape(-1, sequence_length, num_classes) # batch, seq, class\n", "pp.pprint(outputs)" ] }, { "cell_type": "code", "execution_count": 16, "metadata": { "collapsed": false }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Loss: 0.596759\n" ] } ], "source": [ "# [batch_size, sequence_length]\n", "y_data = tf.constant([[1, 1, 1]])\n", "\n", "# [batch_size, sequence_length, emb_dim ]\n", "prediction = tf.constant([[[0.2, 0.7], [0.6, 0.2], [0.2, 0.9]]], dtype=tf.float32)\n", "\n", "# [batch_size * sequence_length]\n", "weights = tf.constant([[1, 1, 1]], dtype=tf.float32)\n", "\n", "sequence_loss = tf.contrib.seq2seq.sequence_loss(logits=prediction, targets=y_data, weights=weights)\n", "sess.run(tf.global_variables_initializer())\n", "print(\"Loss: \", sequence_loss.eval())" ] }, { "cell_type": "code", "execution_count": 18, "metadata": { "collapsed": false }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Loss1: 0.513015 Loss2: 0.371101 Loss3: 1.31326 Loss4: 1.31326\n" ] } ], "source": [ "# [batch_size, sequence_length]\n", "y_data = tf.constant([[1, 1, 1]])\n", "\n", "# [batch_size, sequence_length, emb_dim ]\n", "prediction1 = tf.constant([[[0.3, 0.7], [0.3, 0.7], [0.3, 0.7]]], dtype=tf.float32)\n", "prediction2 = tf.constant([[[0.1, 0.9], [0.1, 0.9], [0.1, 0.9]]], dtype=tf.float32)\n", "\n", "prediction3 = tf.constant([[[1, 0], [1, 0], [1, 0]]], dtype=tf.float32)\n", "prediction4 = tf.constant([[[0, 1], [1, 0], [0, 1]]], dtype=tf.float32)\n", "\n", "# [batch_size * sequence_length]\n", "weights = tf.constant([[1, 1, 1]], dtype=tf.float32)\n", "\n", "sequence_loss1 = tf.contrib.seq2seq.sequence_loss(prediction1, y_data, weights)\n", "sequence_loss2 = tf.contrib.seq2seq.sequence_loss(prediction2, y_data, weights)\n", "sequence_loss3 = tf.contrib.seq2seq.sequence_loss(prediction3, y_data, weights)\n", "sequence_loss4 = tf.contrib.seq2seq.sequence_loss(prediction3, y_data, weights)\n", "\n", "sess.run(tf.global_variables_initializer())\n", "print(\"Loss1: \", sequence_loss1.eval(),\n", " \"Loss2: \", sequence_loss2.eval(),\n", " \"Loss3: \", sequence_loss3.eval(),\n", " \"Loss4: \", sequence_loss4.eval())" ] } ], "metadata": { "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.6.0" } }, "nbformat": 4, "nbformat_minor": 0 }