{ "cells": [ { "cell_type": "code", "execution_count": 31, "metadata": { "collapsed": false }, "outputs": [], "source": [ "from __future__ import absolute_import\n", "from __future__ import division\n", "from __future__ import print_function\n", "\n", "import collections\n", "import math\n", "import os\n", "import random\n", "import sys\n", "import threading\n", "import time\n", "import zipfile\n", "\n", "import numpy as np\n", "from six.moves import urllib\n", "from six.moves import xrange # pylint: disable=redefined-builtin\n", "\n", "import tensorflow as tf\n", "from tensorflow.python.client import device_lib\n", "\n", "%matplotlib inline" ] }, { "cell_type": "code", "execution_count": 2, "metadata": { "collapsed": false }, "outputs": [ { "data": { "text/plain": [ "['/gpu:0']" ] }, "execution_count": 2, "metadata": {}, "output_type": "execute_result" } ], "source": [ "local_device_protos = device_lib.list_local_devices()\n", "[x.name for x in local_device_protos if x.device_type == 'GPU']" ] }, { "cell_type": "code", "execution_count": 3, "metadata": { "collapsed": false }, "outputs": [], "source": [ "word2vec = tf.load_op_library('./word2vec_ops.so')" ] }, { "cell_type": "code", "execution_count": 4, "metadata": { "collapsed": false }, "outputs": [], "source": [ "flags = tf.app.flags\n", "\n", "flags.DEFINE_string(\"save_path\", None, \"Directory to write the model and \"\n", " \"training summaries.\")\n", "flags.DEFINE_string(\"train_data\", None, \"Training text file. \"\n", " \"E.g., unzipped file http://mattmahoney.net/dc/text8.zip.\")\n", "flags.DEFINE_string(\n", " \"eval_data\", None, \"File consisting of analogies of four tokens.\"\n", " \"embedding 2 - embedding 1 + embedding 3 should be close \"\n", " \"to embedding 4.\"\n", " \"See README.md for how to get 'questions-words.txt'.\")\n", "flags.DEFINE_integer(\"embedding_size\", 200, \"The embedding dimension size.\")\n", "flags.DEFINE_integer(\n", " \"epochs_to_train\", 15,\n", " \"Number of epochs to train. Each epoch processes the training data once \"\n", " \"completely.\")\n", "flags.DEFINE_float(\"learning_rate\", 0.2, \"Initial learning rate.\")\n", "flags.DEFINE_integer(\"num_neg_samples\", 100,\n", " \"Negative samples per training example.\")\n", "flags.DEFINE_integer(\"batch_size\", 16,\n", " \"Number of training examples processed per step \"\n", " \"(size of a minibatch).\")\n", "flags.DEFINE_integer(\"concurrent_steps\", 12,\n", " \"The number of concurrent training steps.\")\n", "flags.DEFINE_integer(\"window_size\", 5,\n", " \"The number of words to predict to the left and right \"\n", " \"of the target word.\")\n", "flags.DEFINE_integer(\"min_count\", 5,\n", " \"The minimum number of word occurrences for it to be \"\n", " \"included in the vocabulary.\")\n", "flags.DEFINE_float(\"subsample\", 1e-3,\n", " \"Subsample threshold for word occurrence. Words that appear \"\n", " \"with higher frequency will be randomly down-sampled. Set \"\n", " \"to 0 to disable.\")\n", "flags.DEFINE_boolean(\n", " \"interactive\", False,\n", " \"If true, enters an IPython interactive session to play with the trained \"\n", " \"model. E.g., try model.analogy(b'france', b'paris', b'russia') and \"\n", " \"model.nearby([b'proton', b'elephant', b'maxwell'])\")\n", "flags.DEFINE_integer(\"statistics_interval\", 5,\n", " \"Print statistics every n seconds.\")\n", "flags.DEFINE_integer(\"summary_interval\", 5,\n", " \"Save training summary to file every n seconds (rounded \"\n", " \"up to statistics interval).\")\n", "flags.DEFINE_integer(\"checkpoint_interval\", 600,\n", " \"Checkpoint the model (i.e. save the parameters) every n \"\n", " \"seconds (rounded up to statistics interval).\")\n", "\n", "FLAGS = flags.FLAGS" ] }, { "cell_type": "code", "execution_count": 6, "metadata": { "collapsed": true }, "outputs": [], "source": [ "class Options(object):\n", " \"\"\"Options used by our word2vec model.\"\"\"\n", "\n", " def __init__(self):\n", " # Model options.\n", "\n", " # Embedding dimension.\n", " self.emb_dim = FLAGS.embedding_size\n", "\n", " # Training options.\n", " # The training text file.\n", " self.train_data = FLAGS.train_data\n", "\n", " # Number of negative samples per example.\n", " self.num_samples = FLAGS.num_neg_samples\n", "\n", " # The initial learning rate.\n", " self.learning_rate = FLAGS.learning_rate\n", "\n", " # Number of epochs to train. After these many epochs, the learning\n", " # rate decays linearly to zero and the training stops.\n", " self.epochs_to_train = FLAGS.epochs_to_train\n", "\n", " # Concurrent training steps.\n", " self.concurrent_steps = FLAGS.concurrent_steps\n", "\n", " # Number of examples for one training step.\n", " self.batch_size = FLAGS.batch_size\n", "\n", " # The number of words to predict to the left and right of the target word.\n", " self.window_size = FLAGS.window_size\n", "\n", " # The minimum number of word occurrences for it to be included in the\n", " # vocabulary.\n", " self.min_count = FLAGS.min_count\n", "\n", " # Subsampling threshold for word occurrence.\n", " self.subsample = FLAGS.subsample\n", "\n", " # How often to print statistics.\n", " self.statistics_interval = FLAGS.statistics_interval\n", "\n", " # How often to write to the summary file (rounds up to the nearest\n", " # statistics_interval).\n", " self.summary_interval = FLAGS.summary_interval\n", "\n", " # How often to write checkpoints (rounds up to the nearest statistics\n", " # interval).\n", " self.checkpoint_interval = FLAGS.checkpoint_interval\n", "\n", " # Where to write out summaries.\n", " self.save_path = FLAGS.save_path\n", " if not os.path.exists(self.save_path):\n", " os.makedirs(self.save_path)\n", "\n", " # Eval options.\n", " # The text file for eval.\n", " self.eval_data = FLAGS.eval_data" ] }, { "cell_type": "code", "execution_count": 22, "metadata": { "collapsed": true }, "outputs": [], "source": [ "class Word2Vec(object):\n", " \"\"\"Word2Vec model (Skipgram).\"\"\"\n", "\n", " def __init__(self, options, session):\n", " self._options = options\n", " self._session = session\n", " self._word2id = {}\n", " self._id2word = []\n", " self.build_graph()\n", " self.build_eval_graph()\n", " self.save_vocab()\n", "\n", " def read_analogies(self):\n", " \"\"\"Reads through the analogy question file.\n", " Returns:\n", " questions: a [n, 4] numpy array containing the analogy question's\n", " word ids.\n", " questions_skipped: questions skipped due to unknown words.\n", " \"\"\"\n", " questions = []\n", " questions_skipped = 0\n", " with open(self._options.eval_data, \"rb\") as analogy_f:\n", " for line in analogy_f:\n", " if line.startswith(b\":\"): # Skip comments.\n", " continue\n", " words = line.strip().lower().split(b\" \")\n", " ids = [self._word2id.get(w.strip()) for w in words]\n", "\n", " if None in ids or len(ids) != 4:\n", " questions_skipped += 1\n", " else:\n", " questions.append(np.array(ids))\n", "\n", " print(\"Eval analogy file: \", self._options.eval_data)\n", " print(\"Questions: \", len(questions))\n", " print(\"Skipped: \", questions_skipped)\n", " self._analogy_questions = np.array(questions, dtype=np.int32)\n", "\n", " def forward(self, examples, labels):\n", " \"\"\"Build the graph for the forward pass.\"\"\"\n", " opts = self._options\n", "\n", " # Declare all variables we need.\n", " # Embedding: [vocab_size, emb_dim]\n", " init_width = 0.5 / opts.emb_dim\n", " emb = tf.Variable(\n", " tf.random_uniform(\n", " [opts.vocab_size, opts.emb_dim], -init_width, init_width),\n", " name=\"emb\")\n", " self._emb = emb\n", "\n", " # Softmax weight: [vocab_size, emb_dim]. Transposed.\n", " sm_w_t = tf.Variable(\n", " tf.zeros([opts.vocab_size, opts.emb_dim]),\n", " name=\"sm_w_t\")\n", "\n", " # Softmax bias: [vocab_size].\n", " sm_b = tf.Variable(tf.zeros([opts.vocab_size]), name=\"sm_b\")\n", "\n", " # Global step: scalar, i.e., shape [].\n", " self.global_step = tf.Variable(0, name=\"global_step\")\n", "\n", " # Nodes to compute the nce loss w/ candidate sampling.\n", " labels_matrix = tf.reshape(\n", " tf.cast(labels,\n", " dtype=tf.int64),\n", " [opts.batch_size, 1])\n", "\n", " # Negative sampling.\n", " sampled_ids, _, _ = (tf.nn.fixed_unigram_candidate_sampler(\n", " true_classes=labels_matrix,\n", " num_true=1,\n", " num_sampled=opts.num_samples,\n", " unique=True,\n", " range_max=opts.vocab_size,\n", " distortion=0.75,\n", " unigrams=opts.vocab_counts.tolist()))\n", "\n", " # Embeddings for examples: [batch_size, emb_dim]\n", " example_emb = tf.nn.embedding_lookup(emb, examples)\n", "\n", " # Weights for labels: [batch_size, emb_dim]\n", " true_w = tf.nn.embedding_lookup(sm_w_t, labels)\n", " # Biases for labels: [batch_size, 1]\n", " true_b = tf.nn.embedding_lookup(sm_b, labels)\n", "\n", " # Weights for sampled ids: [num_sampled, emb_dim]\n", " sampled_w = tf.nn.embedding_lookup(sm_w_t, sampled_ids)\n", " # Biases for sampled ids: [num_sampled, 1]\n", " sampled_b = tf.nn.embedding_lookup(sm_b, sampled_ids)\n", "\n", " # True logits: [batch_size, 1]\n", " true_logits = tf.reduce_sum(tf.multiply(example_emb, true_w), 1) + true_b\n", "\n", " # Sampled logits: [batch_size, num_sampled]\n", " # We replicate sampled noise labels for all examples in the batch\n", " # using the matmul.\n", " sampled_b_vec = tf.reshape(sampled_b, [opts.num_samples])\n", " sampled_logits = tf.matmul(example_emb,\n", " sampled_w,\n", " transpose_b=True) + sampled_b_vec\n", " return true_logits, sampled_logits\n", "\n", " def nce_loss(self, true_logits, sampled_logits):\n", " \"\"\"Build the graph for the NCE loss.\"\"\"\n", "\n", " # cross-entropy(logits, labels)\n", " opts = self._options\n", " true_xent = tf.nn.sigmoid_cross_entropy_with_logits(\n", " labels=tf.ones_like(true_logits), logits=true_logits)\n", " sampled_xent = tf.nn.sigmoid_cross_entropy_with_logits(\n", " labels=tf.zeros_like(sampled_logits), logits=sampled_logits)\n", "\n", " # NCE-loss is the sum of the true and noise (sampled words)\n", " # contributions, averaged over the batch.\n", " nce_loss_tensor = (tf.reduce_sum(true_xent) +\n", " tf.reduce_sum(sampled_xent)) / opts.batch_size\n", " return nce_loss_tensor\n", "\n", " def optimize(self, loss):\n", " \"\"\"Build the graph to optimize the loss function.\"\"\"\n", "\n", " # Optimizer nodes.\n", " # Linear learning rate decay.\n", " opts = self._options\n", " words_to_train = float(opts.words_per_epoch * opts.epochs_to_train)\n", " lr = opts.learning_rate * tf.maximum(\n", " 0.0001, 1.0 - tf.cast(self._words, tf.float32) / words_to_train)\n", " self._lr = lr\n", " optimizer = tf.train.GradientDescentOptimizer(lr)\n", " train = optimizer.minimize(loss,\n", " global_step=self.global_step,\n", " gate_gradients=optimizer.GATE_NONE)\n", " self._train = train\n", "\n", " def build_eval_graph(self):\n", " \"\"\"Build the eval graph.\"\"\"\n", " # Eval graph\n", "\n", " # Each analogy task is to predict the 4th word (d) given three\n", " # words: a, b, c. E.g., a=italy, b=rome, c=france, we should\n", " # predict d=paris.\n", "\n", " # The eval feeds three vectors of word ids for a, b, c, each of\n", " # which is of size N, where N is the number of analogies we want to\n", " # evaluate in one batch.\n", " analogy_a = tf.placeholder(dtype=tf.int32) # [N]\n", " analogy_b = tf.placeholder(dtype=tf.int32) # [N]\n", " analogy_c = tf.placeholder(dtype=tf.int32) # [N]\n", "\n", " # Normalized word embeddings of shape [vocab_size, emb_dim].\n", " nemb = tf.nn.l2_normalize(self._emb, 1)\n", "\n", " # Each row of a_emb, b_emb, c_emb is a word's embedding vector.\n", " # They all have the shape [N, emb_dim]\n", " a_emb = tf.gather(nemb, analogy_a) # a's embs\n", " b_emb = tf.gather(nemb, analogy_b) # b's embs\n", " c_emb = tf.gather(nemb, analogy_c) # c's embs\n", "\n", " # We expect that d's embedding vectors on the unit hyper-sphere is\n", " # near: c_emb + (b_emb - a_emb), which has the shape [N, emb_dim].\n", " target = c_emb + (b_emb - a_emb)\n", "\n", " # Compute cosine distance between each pair of target and vocab.\n", " # dist has shape [N, vocab_size].\n", " dist = tf.matmul(target, nemb, transpose_b=True)\n", "\n", " # For each question (row in dist), find the top 4 words.\n", " _, pred_idx = tf.nn.top_k(dist, 4)\n", "\n", " # Nodes for computing neighbors for a given word according to\n", " # their cosine distance.\n", " nearby_word = tf.placeholder(dtype=tf.int32) # word id\n", " nearby_emb = tf.gather(nemb, nearby_word)\n", " nearby_dist = tf.matmul(nearby_emb, nemb, transpose_b=True)\n", " nearby_val, nearby_idx = tf.nn.top_k(nearby_dist,\n", " min(1000, self._options.vocab_size))\n", "\n", " # Nodes in the construct graph which are used by training and\n", " # evaluation to run/feed/fetch.\n", " self._analogy_a = analogy_a\n", " self._analogy_b = analogy_b\n", " self._analogy_c = analogy_c\n", " self._analogy_pred_idx = pred_idx\n", " self._nearby_word = nearby_word\n", " self._nearby_val = nearby_val\n", " self._nearby_idx = nearby_idx\n", "\n", " def build_graph(self):\n", " \"\"\"Build the graph for the full model.\"\"\"\n", " opts = self._options\n", " # The training data. A text file.\n", " (words, counts, words_per_epoch, self._epoch, self._words, examples,\n", " labels) = word2vec.skipgram_word2vec(filename=opts.train_data,\n", " batch_size=opts.batch_size,\n", " window_size=opts.window_size,\n", " min_count=opts.min_count,\n", " subsample=opts.subsample)\n", " (opts.vocab_words, opts.vocab_counts,\n", " opts.words_per_epoch) = self._session.run([words, counts, words_per_epoch])\n", " opts.vocab_size = len(opts.vocab_words)\n", "\n", " print(\"Data file: \", opts.train_data)\n", " print(\"Vocab size: \", opts.vocab_size - 1, \" + UNK\")\n", " print(\"Words per epoch: \", opts.words_per_epoch)\n", "\n", " self._examples = examples\n", " self._labels = labels\n", " self._id2word = opts.vocab_words\n", "\n", " for i, w in enumerate(self._id2word):\n", " self._word2id[w] = i\n", "\n", " true_logits, sampled_logits = self.forward(examples, labels)\n", " loss = self.nce_loss(true_logits, sampled_logits)\n", " tf.summary.scalar(\"NCE loss\", loss)\n", " self._loss = loss\n", " self.optimize(loss)\n", "\n", " # Properly initialize all variables.\n", " tf.global_variables_initializer().run()\n", "\n", " self.saver = tf.train.Saver()\n", "\n", " def save_vocab(self):\n", " \"\"\"Save the vocabulary to a file so the model can be reloaded.\"\"\"\n", " opts = self._options\n", "\n", " with open(os.path.join(opts.save_path, \"vocab.txt\"), \"w\") as f:\n", "\n", " for i in xrange(opts.vocab_size):\n", " vocab_word = tf.compat.as_text(opts.vocab_words[i]).encode(\"utf-8\")\n", " f.write(\"%s %d\\n\" % (vocab_word,\n", " opts.vocab_counts[i]))\n", "\n", " def _train_thread_body(self):\n", " initial_epoch, = self._session.run([self._epoch])\n", " while True:\n", " _, epoch = self._session.run([self._train, self._epoch])\n", "\n", " if epoch != initial_epoch:\n", " break\n", "\n", " def train(self):\n", " \"\"\"Train the model.\"\"\"\n", " opts = self._options\n", "\n", " initial_epoch, initial_words = self._session.run([self._epoch, self._words])\n", "\n", " summary_op = tf.summary.merge_all()\n", " summary_writer = tf.summary.FileWriter(opts.save_path, self._session.graph)\n", " workers = []\n", " for _ in xrange(opts.concurrent_steps):\n", " t = threading.Thread(target=self._train_thread_body)\n", " t.start()\n", " workers.append(t)\n", "\n", " last_words, last_time, last_summary_time = initial_words, time.time(), 0\n", " last_checkpoint_time = 0\n", "\n", " while True:\n", " time.sleep(opts.statistics_interval) # Reports our progress once a while.\n", " (epoch, step, loss, words, lr) = self._session.run(\n", " [self._epoch, self.global_step, self._loss, self._words, self._lr])\n", "\n", " now = time.time()\n", " last_words, last_time, rate = words, now, (words - last_words) / (\n", " now - last_time)\n", "\n", " print(\"Epoch %4d Step %8d: lr = %5.3f loss = %6.2f words/sec = %8.0f\\r\" %\n", " (epoch, step, lr, loss, rate), end=\"\")\n", " sys.stdout.flush()\n", "\n", " if now - last_summary_time > opts.summary_interval:\n", " summary_str = self._session.run(summary_op)\n", " summary_writer.add_summary(summary_str, step)\n", " last_summary_time = now\n", "\n", " if now - last_checkpoint_time > opts.checkpoint_interval:\n", " self.saver.save(self._session,\n", " os.path.join(opts.save_path, \"model.ckpt\"),\n", " global_step=step.astype(int))\n", " last_checkpoint_time = now\n", "\n", " if epoch != initial_epoch:\n", " break\n", "\n", " for t in workers:\n", " t.join()\n", "\n", " return epoch\n", "\n", " def _predict(self, analogy):\n", " \"\"\"Predict the top 4 answers for analogy questions.\"\"\"\n", " idx, = self._session.run([self._analogy_pred_idx], {\n", " self._analogy_a: analogy[:, 0],\n", " self._analogy_b: analogy[:, 1],\n", " self._analogy_c: analogy[:, 2]\n", " })\n", " return idx\n", "\n", " def eval(self):\n", " \"\"\"Evaluate analogy questions and reports accuracy.\"\"\"\n", "\n", " # How many questions we get right at precision@1.\n", " correct = 0\n", "\n", " try:\n", " total = self._analogy_questions.shape[0]\n", " except AttributeError as e:\n", " raise AttributeError(\"Need to read analogy questions.\")\n", "\n", " start = 0\n", " while start < total:\n", " limit = start + 2500\n", " sub = self._analogy_questions[start:limit, :]\n", " idx = self._predict(sub)\n", " start = limit\n", "\n", " for question in xrange(sub.shape[0]):\n", " for j in xrange(4):\n", " if idx[question, j] == sub[question, 3]:\n", " # Bingo! We predicted correctly. E.g., [italy, rome, france, paris].\n", " correct += 1\n", " break\n", " elif idx[question, j] in sub[question, :3]:\n", " # We need to skip words already in the question.\n", " continue\n", " else:\n", " # The correct label is not the precision@1\n", " break\n", " print()\n", " print(\"Eval %4d/%d accuracy = %4.1f%%\" % (correct, total,\n", " correct * 100.0 / total))\n", "\n", "\n", " def analogy(self, w0, w1, w2):\n", " \"\"\"Predict word w3 as in w0:w1 vs w2:w3.\"\"\"\n", " wid = np.array([[self._word2id.get(w, 0) for w in [w0, w1, w2]]])\n", " idx = self._predict(wid)\n", "\n", " for c in [self._id2word[i] for i in idx[0, :]]:\n", " if c not in [w0, w1, w2]:\n", " print(c)\n", " break\n", " print(\"unknown\")\n", "\n", " def nearby(self, words, num=20):\n", " \"\"\"Prints out nearby words given a list of words.\"\"\"\n", " ids = np.array([self._word2id.get(x, 0) for x in words])\n", " vals, idx = self._session.run(\n", " [self._nearby_val, self._nearby_idx], {self._nearby_word: ids})\n", " for i in xrange(len(words)):\n", "\n", " print(\"\\n%s\\n=====================================\" % (words[i]))\n", "\n", " for (neighbor, distance) in zip(idx[i, :num], vals[i, :num]):\n", " print(\"%-20s %6.4f\" % (self._id2word[neighbor], distance))" ] }, { "cell_type": "code", "execution_count": 23, "metadata": { "collapsed": true }, "outputs": [], "source": [ "def _start_shell(local_ns=None):\n", " # An interactive shell is useful for debugging/development.\n", " import IPython\n", " user_ns = {}\n", " \n", " if local_ns:\n", " user_ns.update(local_ns)\n", " user_ns.update(globals())\n", " \n", " IPython.start_ipython(argv=[], user_ns=user_ns)" ] }, { "cell_type": "code", "execution_count": 32, "metadata": { "collapsed": true }, "outputs": [], "source": [ "def main(_):\n", " \"\"\"Train a word2vec model.\"\"\"\n", " \n", " FLAGS.train_data = \"text8\"\n", " FLAGS.eval_data = \"questions-words.txt\"\n", " FLAGS.save_path=\"/tmp\"\n", " \n", " if not FLAGS.train_data or not FLAGS.eval_data or not FLAGS.save_path:\n", " print(\"--train_data --eval_data and --save_path must be specified.\")\n", " sys.exit(1)\n", " \n", " opts = Options()\n", " \n", " with tf.Graph().as_default(), tf.Session() as session:\n", " with tf.device(\"/cpu:0\"):\n", " model = Word2Vec(opts, session)\n", " model.read_analogies() # Read analogy questions\n", " \n", " for _ in xrange(opts.epochs_to_train):\n", " model.train() # Process one epoch\n", " model.eval() # Eval analogies.\n", " \n", " # Perform a final save.\n", " model.saver.save(session,\n", " os.path.join(opts.save_path, \"model.ckpt\"),\n", " global_step=model.global_step)\n", " \n", " if FLAGS.interactive:\n", " # E.g.,\n", " # [0]: model.analogy(b'france', b'paris', b'russia')\n", " # [1]: model.nearby([b'proton', b'elephant', b'maxwell'])\n", " _start_shell(locals()) " ] }, { "cell_type": "code", "execution_count": 33, "metadata": { "collapsed": false }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Data file: text8\n", "Vocab size: 71290 + UNK\n", "Words per epoch: 17005207\n", "INFO:tensorflow:Summary name NCE loss is illegal; using NCE_loss instead.\n", "Eval analogy file: questions-words.txt\n", "Questions: 17827\n", "Skipped: 1717\n", "Epoch 1 Step 4715964: lr = 0.190 loss = 4.08 words/sec = 16828\n", "Eval 1797/17827 accuracy = 10.1%\n", "Epoch 1 Step 5258636: lr = 0.189 loss = 4.51 words/sec = 16877\r" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Exception in thread Thread-40:\n", "Traceback (most recent call last):\n", " File \"/usr/lib/python3.5/threading.py\", line 914, in _bootstrap_inner\n", " self.run()\n", " File \"/usr/lib/python3.5/threading.py\", line 862, in run\n", " self._target(*self._args, **self._kwargs)\n", " File \"\", line 239, in _train_thread_body\n", " _, epoch = self._session.run([self._train, self._epoch])\n", " File \"/home/felipe/auto-tagger/venv3-tf/lib/python3.5/site-packages/tensorflow/python/client/session.py\", line 767, in run\n", " run_metadata_ptr)\n", " File \"/home/felipe/auto-tagger/venv3-tf/lib/python3.5/site-packages/tensorflow/python/client/session.py\", line 903, in _run\n", " raise RuntimeError('Attempted to use a closed Session.')\n", "RuntimeError: Attempted to use a closed Session.\n", "\n", "Exception in thread Thread-43:\n", "Traceback (most recent call last):\n", " File \"/usr/lib/python3.5/threading.py\", line 914, in _bootstrap_inner\n", " self.run()\n", " File \"/usr/lib/python3.5/threading.py\", line 862, in run\n", " self._target(*self._args, **self._kwargs)\n", " File \"\", line 239, in _train_thread_body\n", " _, epoch = self._session.run([self._train, self._epoch])\n", " File \"/home/felipe/auto-tagger/venv3-tf/lib/python3.5/site-packages/tensorflow/python/client/session.py\", line 767, in run\n", " run_metadata_ptr)\n", " File \"/home/felipe/auto-tagger/venv3-tf/lib/python3.5/site-packages/tensorflow/python/client/session.py\", line 903, in _run\n", " raise RuntimeError('Attempted to use a closed Session.')\n", "RuntimeError: Attempted to use a closed Session.\n", "Exception in thread Thread-32:\n", "Traceback (most recent call last):\n", " File \"/usr/lib/python3.5/threading.py\", line 914, in _bootstrap_inner\n", " self.run()\n", " File \"/usr/lib/python3.5/threading.py\", line 862, in run\n", " self._target(*self._args, **self._kwargs)\n", " File \"\", line 239, in _train_thread_body\n", " _, epoch = self._session.run([self._train, self._epoch])\n", " File \"/home/felipe/auto-tagger/venv3-tf/lib/python3.5/site-packages/tensorflow/python/client/session.py\", line 767, in run\n", " run_metadata_ptr)\n", " File \"/home/felipe/auto-tagger/venv3-tf/lib/python3.5/site-packages/tensorflow/python/client/session.py\", line 903, in _run\n", " raise RuntimeError('Attempted to use a closed Session.')\n", "RuntimeError: Attempted to use a closed Session.\n", "Exception in thread Thread-37:\n", "Traceback (most recent call last):\n", " File \"/usr/lib/python3.5/threading.py\", line 914, in _bootstrap_inner\n", " self.run()\n", " File \"/usr/lib/python3.5/threading.py\", line 862, in run\n", " self._target(*self._args, **self._kwargs)\n", " File \"\", line 239, in _train_thread_body\n", " _, epoch = self._session.run([self._train, self._epoch])\n", " File \"/home/felipe/auto-tagger/venv3-tf/lib/python3.5/site-packages/tensorflow/python/client/session.py\", line 767, in run\n", " run_metadata_ptr)\n", " File \"/home/felipe/auto-tagger/venv3-tf/lib/python3.5/site-packages/tensorflow/python/client/session.py\", line 903, in _run\n", " raise RuntimeError('Attempted to use a closed Session.')\n", "RuntimeError: Attempted to use a closed Session.\n", "Exception in thread Thread-42:\n", "Traceback (most recent call last):\n", " File \"/usr/lib/python3.5/threading.py\", line 914, in _bootstrap_inner\n", " self.run()\n", " File \"/usr/lib/python3.5/threading.py\", line 862, in run\n", " self._target(*self._args, **self._kwargs)\n", " File \"\", line 239, in _train_thread_body\n", " _, epoch = self._session.run([self._train, self._epoch])\n", " File \"/home/felipe/auto-tagger/venv3-tf/lib/python3.5/site-packages/tensorflow/python/client/session.py\", line 767, in run\n", " run_metadata_ptr)\n", " File \"/home/felipe/auto-tagger/venv3-tf/lib/python3.5/site-packages/tensorflow/python/client/session.py\", line 903, in _run\n", " raise RuntimeError('Attempted to use a closed Session.')\n", "RuntimeError: Attempted to use a closed Session.\n", "Exception in thread Thread-33:\n", "Traceback (most recent call last):\n", " File \"/home/felipe/auto-tagger/venv3-tf/lib/python3.5/site-packages/tensorflow/python/client/session.py\", line 1022, in _do_call\n", " return fn(*args)\n", " File \"/home/felipe/auto-tagger/venv3-tf/lib/python3.5/site-packages/tensorflow/python/client/session.py\", line 1004, in _run_fn\n", " status, run_metadata)\n", " File \"/usr/lib/python3.5/contextlib.py\", line 66, in __exit__\n", " next(self.gen)\n", " File \"/home/felipe/auto-tagger/venv3-tf/lib/python3.5/site-packages/tensorflow/python/framework/errors_impl.py\", line 466, in raise_exception_on_not_ok_status\n", " pywrap_tensorflow.TF_GetCode(status))\n", "tensorflow.python.framework.errors_impl.CancelledError: Run call was cancelled\n", "\n", "During handling of the above exception, another exception occurred:\n", "\n", "Traceback (most recent call last):\n", " File \"/usr/lib/python3.5/threading.py\", line 914, in _bootstrap_inner\n", " self.run()\n", " File \"/usr/lib/python3.5/threading.py\", line 862, in run\n", " self._target(*self._args, **self._kwargs)\n", " File \"\", line 239, in _train_thread_body\n", " _, epoch = self._session.run([self._train, self._epoch])\n", " File \"/home/felipe/auto-tagger/venv3-tf/lib/python3.5/site-packages/tensorflow/python/client/session.py\", line 767, in run\n", " run_metadata_ptr)\n", " File \"/home/felipe/auto-tagger/venv3-tf/lib/python3.5/site-packages/tensorflow/python/client/session.py\", line 965, in _run\n", " feed_dict_string, options, run_metadata)\n", " File \"/home/felipe/auto-tagger/venv3-tf/lib/python3.5/site-packages/tensorflow/python/client/session.py\", line 1015, in _do_run\n", " target_list, options, run_metadata)\n", " File \"/home/felipe/auto-tagger/venv3-tf/lib/python3.5/site-packages/tensorflow/python/client/session.py\", line 1035, in _do_call\n", " raise type(e)(node_def, op, message)\n", "tensorflow.python.framework.errors_impl.CancelledError: Run call was cancelled\n", "Exception in thread Thread-41:\n", "Traceback (most recent call last):\n", " File \"/usr/lib/python3.5/threading.py\", line 914, in _bootstrap_inner\n", " self.run()\n", " File \"/usr/lib/python3.5/threading.py\", line 862, in run\n", " self._target(*self._args, **self._kwargs)\n", " File \"\", line 239, in _train_thread_body\n", " _, epoch = self._session.run([self._train, self._epoch])\n", " File \"/home/felipe/auto-tagger/venv3-tf/lib/python3.5/site-packages/tensorflow/python/client/session.py\", line 767, in run\n", " run_metadata_ptr)\n", " File \"/home/felipe/auto-tagger/venv3-tf/lib/python3.5/site-packages/tensorflow/python/client/session.py\", line 903, in _run\n", " raise RuntimeError('Attempted to use a closed Session.')\n", "RuntimeError: Attempted to use a closed Session.\n", "Exception in thread Thread-36:\n", "Traceback (most recent call last):\n", " File \"/usr/lib/python3.5/threading.py\", line 914, in _bootstrap_inner\n", " self.run()\n", " File \"/usr/lib/python3.5/threading.py\", line 862, in run\n", " self._target(*self._args, **self._kwargs)\n", " File \"\", line 239, in _train_thread_body\n", " _, epoch = self._session.run([self._train, self._epoch])\n", " File \"/home/felipe/auto-tagger/venv3-tf/lib/python3.5/site-packages/tensorflow/python/client/session.py\", line 767, in run\n", " run_metadata_ptr)\n", " File \"/home/felipe/auto-tagger/venv3-tf/lib/python3.5/site-packages/tensorflow/python/client/session.py\", line 903, in _run\n", " raise RuntimeError('Attempted to use a closed Session.')\n", "RuntimeError: Attempted to use a closed Session.\n", "Exception in thread Thread-39:\n", "Traceback (most recent call last):\n", " File \"/usr/lib/python3.5/threading.py\", line 914, in _bootstrap_inner\n", " self.run()\n", " File \"/usr/lib/python3.5/threading.py\", line 862, in run\n", " self._target(*self._args, **self._kwargs)\n", " File \"\", line 239, in _train_thread_body\n", " _, epoch = self._session.run([self._train, self._epoch])\n", " File \"/home/felipe/auto-tagger/venv3-tf/lib/python3.5/site-packages/tensorflow/python/client/session.py\", line 767, in run\n", " run_metadata_ptr)\n", " File \"/home/felipe/auto-tagger/venv3-tf/lib/python3.5/site-packages/tensorflow/python/client/session.py\", line 903, in _run\n", " raise RuntimeError('Attempted to use a closed Session.')\n", "RuntimeError: Attempted to use a closed Session.\n", "Exception in thread Thread-35:\n", "Traceback (most recent call last):\n", " File \"/home/felipe/auto-tagger/venv3-tf/lib/python3.5/site-packages/tensorflow/python/client/session.py\", line 1022, in _do_call\n", " return fn(*args)\n", " File \"/home/felipe/auto-tagger/venv3-tf/lib/python3.5/site-packages/tensorflow/python/client/session.py\", line 1004, in _run_fn\n", " status, run_metadata)\n", " File \"/usr/lib/python3.5/contextlib.py\", line 66, in __exit__\n", " next(self.gen)\n", " File \"/home/felipe/auto-tagger/venv3-tf/lib/python3.5/site-packages/tensorflow/python/framework/errors_impl.py\", line 466, in raise_exception_on_not_ok_status\n", " pywrap_tensorflow.TF_GetCode(status))\n", "tensorflow.python.framework.errors_impl.CancelledError: Run call was cancelled\n", "\n", "During handling of the above exception, another exception occurred:\n", "\n", "Traceback (most recent call last):\n", " File \"/usr/lib/python3.5/threading.py\", line 914, in _bootstrap_inner\n", " self.run()\n", " File \"/usr/lib/python3.5/threading.py\", line 862, in run\n", " self._target(*self._args, **self._kwargs)\n", " File \"\", line 239, in _train_thread_body\n", " _, epoch = self._session.run([self._train, self._epoch])\n", " File \"/home/felipe/auto-tagger/venv3-tf/lib/python3.5/site-packages/tensorflow/python/client/session.py\", line 767, in run\n", " run_metadata_ptr)\n", " File \"/home/felipe/auto-tagger/venv3-tf/lib/python3.5/site-packages/tensorflow/python/client/session.py\", line 965, in _run\n", " feed_dict_string, options, run_metadata)\n", " File \"/home/felipe/auto-tagger/venv3-tf/lib/python3.5/site-packages/tensorflow/python/client/session.py\", line 1015, in _do_run\n", " target_list, options, run_metadata)\n", " File \"/home/felipe/auto-tagger/venv3-tf/lib/python3.5/site-packages/tensorflow/python/client/session.py\", line 1035, in _do_call\n", " raise type(e)(node_def, op, message)\n", "tensorflow.python.framework.errors_impl.CancelledError: Run call was cancelled\n", "Exception in thread Thread-38:\n", "Traceback (most recent call last):\n", " File \"/home/felipe/auto-tagger/venv3-tf/lib/python3.5/site-packages/tensorflow/python/client/session.py\", line 1022, in _do_call\n", " return fn(*args)\n", " File \"/home/felipe/auto-tagger/venv3-tf/lib/python3.5/site-packages/tensorflow/python/client/session.py\", line 1004, in _run_fn\n", " status, run_metadata)\n", " File \"/usr/lib/python3.5/contextlib.py\", line 66, in __exit__\n", " next(self.gen)\n", " File \"/home/felipe/auto-tagger/venv3-tf/lib/python3.5/site-packages/tensorflow/python/framework/errors_impl.py\", line 466, in raise_exception_on_not_ok_status\n", " pywrap_tensorflow.TF_GetCode(status))\n", "tensorflow.python.framework.errors_impl.CancelledError: Run call was cancelled\n", "\n", "During handling of the above exception, another exception occurred:\n", "\n", "Traceback (most recent call last):\n", " File \"/usr/lib/python3.5/threading.py\", line 914, in _bootstrap_inner\n", " self.run()\n", " File \"/usr/lib/python3.5/threading.py\", line 862, in run\n", " self._target(*self._args, **self._kwargs)\n", " File \"\", line 239, in _train_thread_body\n", " _, epoch = self._session.run([self._train, self._epoch])\n", " File \"/home/felipe/auto-tagger/venv3-tf/lib/python3.5/site-packages/tensorflow/python/client/session.py\", line 767, in run\n", " run_metadata_ptr)\n", " File \"/home/felipe/auto-tagger/venv3-tf/lib/python3.5/site-packages/tensorflow/python/client/session.py\", line 965, in _run\n", " feed_dict_string, options, run_metadata)\n", " File \"/home/felipe/auto-tagger/venv3-tf/lib/python3.5/site-packages/tensorflow/python/client/session.py\", line 1015, in _do_run\n", " target_list, options, run_metadata)\n", " File \"/home/felipe/auto-tagger/venv3-tf/lib/python3.5/site-packages/tensorflow/python/client/session.py\", line 1035, in _do_call\n", " raise type(e)(node_def, op, message)\n", "tensorflow.python.framework.errors_impl.CancelledError: Run call was cancelled\n", "Exception in thread Thread-34:\n", "Traceback (most recent call last):\n", " File \"/usr/lib/python3.5/threading.py\", line 914, in _bootstrap_inner\n", " self.run()\n", " File \"/usr/lib/python3.5/threading.py\", line 862, in run\n", " self._target(*self._args, **self._kwargs)\n", " File \"\", line 239, in _train_thread_body\n", " _, epoch = self._session.run([self._train, self._epoch])\n", " File \"/home/felipe/auto-tagger/venv3-tf/lib/python3.5/site-packages/tensorflow/python/client/session.py\", line 767, in run\n", " run_metadata_ptr)\n", " File \"/home/felipe/auto-tagger/venv3-tf/lib/python3.5/site-packages/tensorflow/python/client/session.py\", line 903, in _run\n", " raise RuntimeError('Attempted to use a closed Session.')\n", "RuntimeError: Attempted to use a closed Session.\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n" ] }, { "ename": "KeyboardInterrupt", "evalue": "", "output_type": "error", "traceback": [ "\u001b[0;31m--------------------------------------------------------------\u001b[0m", "\u001b[0;31mKeyboardInterrupt\u001b[0m Traceback (most recent call last)", "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[0;31m# sys.argv = [sys.argv[0], 'argument1', 'argument2', 'argument2']\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mapp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", "\u001b[0;32m/home/felipe/auto-tagger/venv3-tf/lib/python3.5/site-packages/tensorflow/python/platform/app.py\u001b[0m in \u001b[0;36mrun\u001b[0;34m(main, argv)\u001b[0m\n\u001b[1;32m 42\u001b[0m \u001b[0;31m# Call the main function, passing through any arguments\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 43\u001b[0m \u001b[0;31m# to the final program.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 44\u001b[0;31m \u001b[0m_sys\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmain\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0m_sys\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0margv\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mflags_passthrough\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 45\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 46\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;32m\u001b[0m in \u001b[0;36mmain\u001b[0;34m(_)\u001b[0m\n\u001b[1;32m 18\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 19\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0m_\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mxrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mopts\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mepochs_to_train\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 20\u001b[0;31m \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;31m# Process one epoch\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 21\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0meval\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;31m# Eval analogies.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 22\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;32m\u001b[0m in \u001b[0;36mtrain\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 260\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 261\u001b[0m \u001b[0;32mwhile\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 262\u001b[0;31m \u001b[0mtime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msleep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mopts\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstatistics_interval\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;31m# Reports our progress once a while.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 263\u001b[0m (epoch, step, loss, words, lr) = self._session.run(\n\u001b[1;32m 264\u001b[0m [self._epoch, self.global_step, self._loss, self._words, self._lr])\n", "\u001b[0;31mKeyboardInterrupt\u001b[0m: " ] } ], "source": [ "# sys.argv = [sys.argv[0], 'argument1', 'argument2', 'argument2']\n", "tf.app.run()" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": true }, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": true }, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": true }, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.5.2" } }, "nbformat": 4, "nbformat_minor": 2 }