{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# CS 20 : TensorFlow for Deep Learning Research\n",
    "## Lecture 05 : Variable sharing and managing experiments\n",
    "### Word2vec (skip-gram) for simple example\n",
    "\n",
    "- Creating the **input pipeline** with `tf.data`\n",
    "- Creating the model as **Class**\n",
    "\n",
    "Ref  \n",
    "- https://github.com/golbin/TensorFlow-Tutorials/blob/master/04%20-%20Neural%20Network%20Basic/03%20-%20Word2Vec.py  \n",
    "- https://github.com/aisolab/TF_code_examples_for_Deep_learning/blob/master/Tutorial%20of%20implementing%20Word2Vec.ipynb"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Setup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1.12.0\n"
     ]
    }
   ],
   "source": [
    "from __future__ import absolute_import, division, print_function\n",
    "import os, sys\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import tensorflow as tf\n",
    "%matplotlib inline\n",
    "\n",
    "print(tf.__version__)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Data-preprocessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 단어 벡터를 분석해볼 임의의 문장들\n",
    "sentences = [\"나 고양이 좋다\",\n",
    "             \"나 강아지 좋다\",\n",
    "             \"나 동물 좋다\",\n",
    "             \"강아지 고양이 동물\",\n",
    "             \"여자친구 고양이 강아지 좋다\",\n",
    "             \"고양이 생선 우유 좋다\",\n",
    "             \"강아지 생선 싫다 우유 좋다\",\n",
    "             \"강아지 고양이 눈 좋다\",\n",
    "             \"나 여자친구 좋다\",\n",
    "             \"여자친구 나 싫다\",\n",
    "             \"여자친구 나 영화 책 음악 좋다\",\n",
    "             \"나 게임 만화 애니 좋다\",\n",
    "             \"고양이 강아지 싫다\",\n",
    "             \"강아지 고양이 좋다\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['나', '고양이', '좋다', '나', '강아지', '좋다', '나', '동물', '좋다', '강아지', '고양이', '동물', '여자친구', '고양이', '강아지', '좋다', '고양이', '생선', '우유', '좋다', '강아지', '생선', '싫다', '우유', '좋다', '강아지', '고양이', '눈', '좋다', '나', '여자친구', '좋다', '여자친구', '나', '싫다', '여자친구', '나', '영화', '책', '음악', '좋다', '나', '게임', '만화', '애니', '좋다', '고양이', '강아지', '싫다', '강아지', '고양이', '좋다']\n",
      "['강아지', '게임', '고양이', '나', '눈', '동물', '만화', '생선', '싫다', '애니', '여자친구', '영화', '우유', '음악', '좋다', '책']\n"
     ]
    }
   ],
   "source": [
    "# 문장을 전부 합친 후, 공백으로 단어들을 나누고 고유한 단어들로 리스트를 만듭니다.\n",
    "word_sequence = ' '.join(sentences).split()\n",
    "word_list = list(set(word_sequence))\n",
    "word_list.sort()\n",
    "print(word_sequence)\n",
    "print(word_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'강아지': 0, '게임': 1, '고양이': 2, '나': 3, '눈': 4, '동물': 5, '만화': 6, '생선': 7, '싫다': 8, '애니': 9, '여자친구': 10, '영화': 11, '우유': 12, '음악': 13, '좋다': 14, '책': 15}\n"
     ]
    }
   ],
   "source": [
    "# 문자열로 분석하는 것 보다, 숫자로 분석하는 것이 훨씬 용이하므로\n",
    "# 리스트에서 문자들의 인덱스를 뽑아서 사용하기 위해,\n",
    "# 이를 표현하기 위한 연관 배열과, 단어 리스트에서 단어를 참조 할 수 있는 인덱스 배열을 만듭합니다.\n",
    "word_dic = {w: i for i, w in enumerate(word_list)}\n",
    "print(word_dic)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Define preprocessor function for skip-gram of Word2Vec"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def preprocessor(sequences, word_dic, window_size):\n",
    "    context = []\n",
    "    for idx in range(window_size, len(sequences) - window_size):\n",
    "        center_word = word_dic.get(sequences[idx])\n",
    "        context_words = [word_dic.get(sequences[idx + _]) for _ in range(-window_size, window_size + 1) if _ != 0]\n",
    "    \n",
    "        for token in context_words:\n",
    "            context.append([center_word, token])\n",
    "    else:\n",
    "        return context"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "batch = preprocessor(sequences = word_sequence, word_dic = word_dic, window_size = 2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "center_words = np.array(batch)[:,0]\n",
    "target_words = np.array(batch)[:,[1]]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Define Word2vec class"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Word2vec:\n",
    "    def __init__(self, center_words, target_words, vocab_size,\n",
    "                embedding_dim = 2, num_sampled = 10):\n",
    "        \n",
    "        self._center_words = center_words\n",
    "        self._target_words = target_words\n",
    "        \n",
    "        with tf.variable_scope('embeddings'):\n",
    "            self._embeddings = tf.get_variable(name = 'lookup_table', shape = [vocab_size, embedding_dim],\n",
    "                                               dtype = tf.float32, initializer = tf.truncated_normal_initializer())\n",
    "            self._selected_embed = tf.nn.embedding_lookup(params = self._embeddings, ids = self._center_words)\n",
    "            \n",
    "\n",
    "        with tf.variable_scope('nce'):\n",
    "            nce_weights = tf.get_variable('weights', shape = [vocab_size, embedding_dim], dtype = tf.float32,\n",
    "                                         initializer = tf.truncated_normal_initializer())\n",
    "            nce_biases = tf.get_variable('biases', initializer = tf.zeros(shape = [vocab_size]))\n",
    "            \n",
    "            self.nce_loss = tf.reduce_mean(tf.nn.nce_loss(weights = nce_weights, biases = nce_biases,\n",
    "                                            labels = self._target_words, inputs = self._selected_embed,\n",
    "                                            num_sampled = num_sampled, num_classes = vocab_size))\n",
    "    \n",
    "    def get_wordvector(self, sess, word_dic, word):\n",
    "        idx = word_dic.get(word)\n",
    "        feed_get_wordvector = {self._center_words : [idx]}\n",
    "        return sess.run(self._selected_embed, feed_dict = feed_get_wordvector)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Create a model of Word2vec"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "24\n"
     ]
    }
   ],
   "source": [
    "# hyper-parameter\n",
    "epochs = 200\n",
    "batch_size = 8\n",
    "learning_rate = .001\n",
    "total_step = int(len(batch) / batch_size)\n",
    "print(total_step)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "## create input pipeline with tf.data\n",
    "dataset = tf.data.Dataset.from_tensor_slices((center_words, target_words))\n",
    "dataset = dataset.shuffle(buffer_size = 32)\n",
    "dataset = dataset.batch(batch_size = batch_size)\n",
    "iterator = dataset.make_initializable_iterator()\n",
    "x_data, y_data = iterator.get_next()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "sgram = Word2vec(center_words = x_data, target_words = y_data, vocab_size = len(word_dic))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Create training op and train model "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# create training op\n",
    "opt = tf.train.AdamOptimizer(learning_rate = learning_rate)\n",
    "\n",
    "# equal to 'var_list = None'\n",
    "training_op = opt.minimize(loss = sgram.nce_loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch :   0, tr_loss : 11.64\n",
      "epoch :  20, tr_loss : 8.82\n",
      "epoch :  40, tr_loss : 6.86\n",
      "epoch :  60, tr_loss : 5.09\n",
      "epoch :  80, tr_loss : 4.15\n",
      "epoch : 100, tr_loss : 3.39\n",
      "epoch : 120, tr_loss : 3.11\n",
      "epoch : 140, tr_loss : 3.16\n",
      "epoch : 160, tr_loss : 3.00\n",
      "epoch : 180, tr_loss : 3.04\n"
     ]
    }
   ],
   "source": [
    "sess_config = tf.ConfigProto(gpu_options=tf.GPUOptions(allow_growth=True))\n",
    "sess = tf.Session(config = sess_config)\n",
    "sess.run(tf.global_variables_initializer())\n",
    "\n",
    "tr_loss_hist = []\n",
    "\n",
    "for epoch in range(epochs):\n",
    "    sess.run(iterator.initializer)\n",
    "    avg_tr_loss = 0\n",
    "    total_step = 0\n",
    "\n",
    "    try:\n",
    "        while True:\n",
    "            _, tr_loss = sess.run(fetches = [training_op, sgram.nce_loss])\n",
    "            avg_tr_loss += tr_loss\n",
    "            total_step += 1\n",
    "\n",
    "    except tf.errors.OutOfRangeError:\n",
    "        pass\n",
    "    \n",
    "    avg_tr_loss /= total_step\n",
    "    tr_loss_hist.append(avg_tr_loss)\n",
    "    \n",
    "    if epoch % 20 == 0:\n",
    "        print('epoch : {:3}, tr_loss : {:.2f}'.format(epoch, avg_tr_loss))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x7fe0f00b1198>]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD8CAYAAABn919SAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzt3Xl4lOWh/vHvM9n3hYQkZCHsi+xGBKGohSpVFLXVat2oPdW2ttVuam1P7WltT/uz9rTWVsSlolVqXavYulb2TfawE3aSQBIgC9kz8/z+mCEFJBBIMu/M5P5cF1cmL28yN28mN88872astYiISPBzOR1AREQ6hwpdRCREqNBFREKECl1EJESo0EVEQoQKXUQkRKjQRURCxBkL3RjzrDGmzBiz4bhljxhjthhj1htj3jDGJHdtTBEROZP2jNCfA6aetOwDYJi1dgSwDfhRJ+cSEZGzFH6mFay1C4wx+Scte/+4T5cBX2zPk6Wlpdn8/PwzriciIv+xatWqCmtt+pnWO2Oht8MdwMvtWTE/P5+VK1d2wlOKiHQfxpg97VmvQztFjTE/BlqAF0+zzp3GmJXGmJXl5eUdeToRETmNcy50Y8wMYBpwsz3NFb6stbOstQXW2oL09DO+YxARkXN0TlMuxpipwH3Axdbaus6NJCIi56I9hy3OAZYCg4wx+40xXwUeBxKAD4wxa40xM7s4p4iInEF7jnK56RSLn+mCLCIi0gE6U1REJESo0EVEQkRQFPrC7eX8eV6R0zFERAJaUBT6ou0V/O79bZTXNDodRUQkYAVFoV9fkEuLx/L66v28ta6Eb7202ulIIiIBpzNO/e9y/XvGc37vFJ5fuofKuiZqm9z88tpmkmIinI4mIhIwgmKEDvClglyKK+upbXIDsP+IzmcSETle0BT6lSOyGJSRwIyL8gHYf6Te2UAiIgEmaAo9Liqc9747iXsmDwBg32GN0EVEjhc0hX5McmwE8VHhGqGLiJwk6ArdGENOSozm0EVEThJ0hQ6QkxKrEbqIyEmCtNBj2He4Dmstm0urufz/FvDi8nbd0ENEJGQFZaHnpsZS2+TmvY0Hue7PS9h6sIbVeyqdjiUi4qigOLHoZDkpMQDc/9p60hIiCTOGw7W6LICIdG/BOUJPiQWgqr6ZH1w2iLwecRyubXI4lYiIs4Ky0HNSvSP0wZkJXDWiFz3iIjmkQheRbi4op1wSoyO4Z/IALh3cE5fLkBoXyRFfof9jbTF5qbGMzktxOKWIiH8F5Qgd4LufG8io3GQAUuMiqW1y09Ds5qG3NjJz/g6H04mI+F/QFvrxUuMiASiurKeyrpl9h3WMuoh0PyFV6BtLqgFaj1EXEelOQqLQexwr9OIqAGoaW6isa3YykoiI34VEoaecNEIH2KurMYpINxMShX5shL6hpKp1mQpdRLqbkCj0xOgIwlyGyrpmEqK8R2Kq0EWkuwmJQne5DCmx3lF6flocafFRugGGiHQ7QXli0amkxkVQcbSRrKRoIsKMRugi0u2ExAgd/nPoYq/kGPJSY1XoItLthEyh94iLAqBXcjS5qbGUVNbT7PY4nEpExH9CptCPjdCzkmLITY3FY3UjaRHpXkKm0FNap1yiGZqVCMCm0urTfYmISEgJmULPSorGGO/9RgdmJBAZ5qJwf9WZv1BEJESEzFEu147Opn/PeDISowEYkpXAehW6iHQjITNCj44I44L81NbPh2UnsaG4Co9HF+kSke4hZAr9ZCNykqhpbGGPdoyKSDcRsoU+PNt784v1+ysdTiIi4h8hW+gDMuKJCteOURHpPs5Y6MaYZ40xZcaYDcctSzXGfGCM2e77GHA38IwIczE8O4kVuw87HUVExC/aM0J/Dph60rIHgI+stQOAj3yfB5xJA9MpLK7isO8G0iIioeyMhW6tXQCcPMydDsz2PZ4NXNPJuTrFpIHpWAsLt5c7HUVEpMud6xx6hrW21Pf4AJDRSXk61fDsJFJiI5i/TYUuIqGvwztFrfduzG0e7G2MudMYs9IYs7K83L/FGuYyfGZAOgu2VVBW3YBbx6SLSAg710I/aIzJAvB9LGtrRWvtLGttgbW2ID09/Ryf7txdPDCdiqONjP3VR3zhiSXUN7n9nkFExB/OtdDfAm73Pb4d+EfnxOl800Zm8evrhvO9zw1k3f5KHnh9Pd43FSIioeWM13IxxswBLgHSjDH7gYeAXwN/N8Z8FdgD3NCVITsiKjyMG8fmAd4pmEfe28qwXkl8bVJfh5OJiHSuMxa6tfamNv5qcidn6XLfvKQfG0uq+N9/bWZQZgKTBvp/CkhEpKuE7Jmip2KM4ZEvjmRgRgJ3v7SaLQd0vXQRCR3dqtAB4qLCeWbGBcRGhjHj2U/Yf0QX7xKR0NDtCh0gOzmGv8wYS21TC196chm7K2qdjiQi0mHdstABhvZKZM7XxlHf7Obul1Y7HUdEpMO6baGD9yYYt4/PZ1NpNXVNLU7HERHpkG5d6OC9VZ21sPVAjdNRREQ6RIWelQjAplId8SIiwa3bF3pOSgwJ0eFsLq2mxe3haKOmXkQkOHX7QjfGMCQzkc2lNTz01kau+uMipyOJiJyTM54p2h0MyUrg5ZX7WL+/kma3paHZTXREmNOxRETOSrcfoYN3Hr2h2UOz23vRrtKqBocTiYicPRU6/9kxmp0cA0BJZb2TcUREzokKHTivVyJ3TOjD/143HIBiFbqIBCHNoQPhYS5+etVQGlvcGKMRuogEJ43QjxMVHkZ6fJQKXUSCkgr9JL2SYyip1E5REQk+KvSTZCfHaIQuIkFJhX6SXsnRFFfW676jIhJ0VOgnyUqKobHFw5G6ZqejiIicFRX6SXrpWHQRCVI6bPEkx04uenH5HhKiI+iTFse0EVkkREc4nExE5PRU6CfJTvEW+pwV+wh3GVo8lvc2HuAvMy7AGONwOhGRtqnQT5IaF8lfZlxAj/hIhvVK4plFu/jlPzfz1roSpo/KdjqeiEibNId+CpcO7smInGRcLsMdE/swMjeZX8zdRLPb43Q0EZE2qdDPIMxl+PLYXCqONnFAV2EUkQCmQm+H3JRYAPYdqXM4iYhI21To7ZDjK/T9R3Qoo4gELhV6O2QmReMyKnQRCWwq9HaIDHeRmRjN/uOmXD7ZfZh1+yodTCUiciIdtthOOSmxrSP0uqYWbn92BXVNbqaP6sXvbhhFmEvHqIuIszRCb6eclBiKfYX+4eYy6prcXDIonX+sLaGwuMrhdCIiKvR2y0mJobSqnma3h7fWlpCZGM39UwcDsPewjn4REeep0NspJyUWj4UtpTXM31bGVSOz6N3DdzijCl1EAoAKvZ1yfNd4efSDrTS7LVePzCY2Mpy0+Cj2HvIWutuja6iLiHNU6O107Fj0eVvLmTy4J8OyEwHo3SOWvYfrWLuvkqE/fZcNmk8XEYeo0Nvp2LHoCVHhPHztsNYrL+alegt9/tZyGls8PLNol8NJRaS76lChG2O+a4zZaIzZYIyZY4yJ7qxggSYy3MUdE/rwyPUjyEqKaV2emxpLSVU9y3cdAmDu+hLKanTNFxHxv3MudGNMNvAdoMBaOwwIA27srGCB6CfThjJ1WNYJy3qnxmItLNt5iHF9U2l2W15cttehhCLSnXV0yiUciDHGhAOxQEnHIwWXPN+RLh4L143OYUL/Hry9rtttBhEJAOdc6NbaYuC3wF6gFKiy1r7fWcGCRV5qbOvjMb2TuXRQT3ZW1FKse5KKiJ91ZMolBZgO9AF6AXHGmFtOsd6dxpiVxpiV5eXl5540QKXHRxEV7iIxOpy+afFMHJAGwKLtofdvFZHA1pEplynALmttubW2GXgduOjklay1s6y1BdbagvT09A48XWByuQwDMuK5ID8Vl8swKCOB9IQoFm6vcDqaiHQzHbk4115gnDEmFqgHJgMrOyVVkHny1gKiwr3/Nxpj+Ez/NOZtK8fjsbh00S4R8ZOOzKEvB14FVgOFvu81q5NyBZXs5BjS4qNaP584II3DtU2s26/L64qI/3ToKBdr7UPW2sHW2mHW2luttY2dFSyYTR6cQUJUOE8t3Ol0FBHpRnSmaBdIio3gKxPy+WfhAbYcqHY6joh0Eyr0LnLHxD4kRIXz+L+LWpc1NLv5zpw1bD1Q42AyEQlVKvQukhwbyTWjs/n3ljKaWjwALN1xiLfWlTBnhc4kFZHOp0LvQhP6p1HX5Gat796j87eVn/BRRKQzqdC70Pi+PXAZWFzkPSZ9wbZywlyGXRW17DlUy5IdFdQ0NDucUkRChQq9CyXFRjA8O4klOyrYd7iOnRW13HxhHgA/eXMDX35qObOX7HY2pIiEDBV6F7uofxpr9lbyyqr9ANw2Pp+81NjWM0nX7dcNMUSkc6jQu9jE/mm0eCyPfbSd/B6x9EuPY+qwTNLiI7moXw8KVegi0kk6cuq/tMP4vj34+fTzSImN5MK+qRhjuO/yQdwzeQBzVuxlyY7NlNc0kp4QdeZvJiJyGir0LuZyGW4bn3/CsvAwF+FhLoZnJwGwobiKSwf3pKnFw97DdfTvGe9AUhEJdppycdB52UkYA4W+G0v/9v2tXPGHhTryRUTOiQrdQfFR4fRJi6OwuIqq+mZeXLaHJreHorKjTkcTkSCkQnfY8OwkVu85wu/e30ptkxtAhS4i50SF7rAbCnJpbPEwe+kexvVNJTLMRVG5Cl1Ezp4K3WET+qex8L5LuW/qIB6+Zjh90+MoOqhCF5Gzp6NcAkBKXCTfvKQ/AP16xrOhWMemi8jZ0wg9wPRPj2fv4Toamt1ORxGRIKNCDzADMuKxFnaW1zodRUSCjAo9wBw7qWh7mW6CISJnR4UeYPqkxeEy8MGmg603xhARaQ8VeoCJCg9jxkV9mLu+lOufXKpSF5F2U6EHoJ9eNZSfXDmEdfsqdZNpEWk3FXqAmjwkA4DNpSp0EWkfFXqA6p0aS2xkGJtLtXNURNpHhR6gXC7DoMwEjdBFpN1U6AFscGYim0ursdY6HUVEgoAKPYANzUqguqGF0qoGp6OISBBQoQewwVmJwH92jM6cv4PXfDebFhE5mQo9gA3KTABgy4EaGprd/N8H23hywQ6HU4lIoFKhB7DE6Aj6pMWxZEcFS3ceorHFw7aDR6msa3I6mogEIBV6gLtmVDaLiw7x16V7Wpet3H3EwUQiEqhU6AHu+oIcjIGPtpQxoX8PIsIMn+w+7HQsEQlAKvQA1ys5hosHpgMwdVgWI3KSVegickoq9CBwx4Q+JMVEMGVITy7IT6WwuIpLHvmYu15YecJ6FUcbHUooIoFAhR4EJg1MZ91Dl5GVFMOkgWk0uy2NLR7e23iQJTsqANhRfpSxv/yQJ+frKBiR7kqFHmQu6pfGigcn8/EPLiEzMZrfvrcVay2LtlfgsfDIe1tZvVc7TUW6IxV6EOqZGE10RBj3TBnA6r2VLC46xPJdh+iZEEVmUjQPvl7odEQRcUCHCt0Yk2yMedUYs8UYs9kYM76zgsmZXTs6m8TocF5ZtY8Vuw4zsX8aMy7KZ8uBGkoq652OJyJ+1tER+h+Ad621g4GRwOaOR5L2io4I46qRvZi7vpSKo01c2DeViQPSAFhcVEGL28PRxhaHU4qIv5xzoRtjkoBJwDMA1toma21lZwWT9rluTA5uj/dqjBf26cGgjATS4iNZXFTBg28UMu2xhQ4nFBF/Ce/A1/YByoG/GGNGAquAe6y1tZ2STNplTF4yfdLiqGtqoXePWIwxTOifxoeby6htasFaqG9yExMZ5nRUEeliHZlyCQfGAE9Ya0cDtcADJ69kjLnTGLPSGLOyvLy8A08np2KM4dEbRvK7G0ZhjAFgQv80jjZ6yxyguLLOwYQi4i8dKfT9wH5r7XLf56/iLfgTWGtnWWsLrLUF6enpHXg6acuYvBQm9E9r/XzSgHQiw1xcOSILgH1HtINUpDs45ykXa+0BY8w+Y8wga+1WYDKwqfOiybnKTIpm2YOTaWrx8M76Uvar0EW6hY7MoQN8G3jRGBMJ7AS+0vFI0hlS4yLxeCyRYS72H9GUi0h30KFCt9auBQo6KYt0MpfLkJ0SoxG6SDehM0VDXI6v0GsamltvZScioUmFHuJyUmIoPlLHb97dwtWPL6JYZ5CKhCwVeojLSYml4mgTb68rpdlteWrBTqcjiUgXUaGHuJyUGACq6pvJ7xHLnBV7dd10kRClQg9x2cneQk+IDmfmrefT5Pbwl8W7HE4lIl1BhR7iclJiAbj8vEwGZyZyxbAsnl+yh+qGZoeTiUhnU6GHuIzEKO6bOohvXdofgG9c0o+axhZeWLrH4WQi0tk6emKRBDhjDN+8pH/r58Oyk7hkUDoz5++grqmF28fn0zMx2sGEItJZNELvhh666jxG5SbzxLwdXP/kUvYcquWFZXvYUFzldDQR6QBjj12Szw8KCgrsypUrz7yi+MXqvUe45enl1DW5AZg8uCfPzLjA4VQicjJjzCpr7RnPyteUSzc2Ji+FZ2dcwOwluymradSZpCJBTlMu3dy4vj144pbz+dzQDEqqGqisa3I6koicIxW6ADAkKxGAzaU1DicRkXOlQhcAhmQlALROu/yrsJTJj85j9d4jTsYSkbOgQhcAeiZEkxYfyabSauauL+Fbc9awo7yWu15YRYku6CUSFLRTVFoNyUpk0fYK3l5Xwpi8ZH50xRBufXo5F/363yRGhxMVEcaVw7P42dXnOR1VRE5BI3RpNTQrkQPVDcRFhfOnm8cwJi+Fl+8azw8uG8h1Y3LoERfJG2uK8eehriLSfhqhS6vReckA/Pq64fRM8J49Oiw7iWHZSQDMXrKbh97ayMHqRjKTdHapSKBRoUury4Zmsuj+S1sv6HWyQZneHadbDlSr0EUCkKZcpJXLZdosc4DBvkLfekCHNooEIhW6tFtybCQZiVFsOVCDtVZz6SIBRoUuZ2VwZiJbDtTw4BuFXPnYIlrcHhYXVfDEvB1ORxPp9jSHLmdlcGYCi4oq2HKgGmvhr8v28Kd5OyivaeTqUb1a75AkIv6nEbqclUGZCbg9lsToCAZnJvCztzdRXuO9R+ncdSUOpxPp3lToclZG5HgPYfz2Z/tz/+cHA3DjBbmMzE3m7fUqdBEnacpFzkr/ngm8d+8kBmbEA/C3O8cxKjeZvy7bw8PvbGZn+VH6psc7nFKke9IIXc7aoMwEjDEYYxjXtwfREWFMG9ELl4GZ87VzVMQpGqFLp8hMiubrF/fjz/N2MDwnmcZmN5cNzSSvR9vHtYtI51KhS6e5d8pAPt5azn+/uQHwXlv90RtGOpxKpPtQoUuniQx38dRt57Ok6BDvbzrAx1vLcHssYS7jdDSRbkGFLp0qJyWWGy6IJToyjA83l7F67xE+3lLGhP5pTOif5nQ8kZCmnaLSJS4emE64y3D/q+v587wd3PfqeppaPE7HEglpKnTpEkkxEYztk8rOilp694iluLKe11bvdzqWSEhToUuXuXZ0Nmnxkbz0tXGMzEni8X8X6UqNIl3I+POKeQUFBXblypV+ez5xnrUWYwzLdx7iq7NXcrSxhYv69eDa0dlcX5DrdDyRoGCMWWWtLTjTehqhS5cyxnuEy4V9e7Do/ku5d8oADlQ18MNX17OhuMrhdCKhRYUufpMcG8m9UwYy+46xAKzdV3nC3xdX1jP98UWfWi4i7dPhQjfGhBlj1hhj5nZGIAl9OSkxJMdGULj/xBH6G6v3s25/FXe/uJrKuiaH0okEr84Yod8DbO6E7yPdhDGG4dlJFJ405fJO4QFyU2Moq2ngpqeW8/dP9ulQR5Gz0KFCN8bkAFcCT3dOHOkuhmcnse1gDQ3NbgB2VdSyubSa28fn8/svjaah2c19r63npqeWcbC6weG0IsGho2eK/h64D0hoawVjzJ3AnQB5eXkdfDoJFcOzk2jxWN5cU8zrq4tx+YYWVwzPoldyDFcMz+Tt9aU88Np6bnhyKe/dO4noiDBnQ4sEuHMeoRtjpgFl1tpVp1vPWjvLWltgrS1IT08/16eTEDPcd6OMB98oZENJFct3HWZsn1R6+W5hZ4zh6pG9ePq2AvYcquPJ+TudjCsSFDoyQp8AXG2MuQKIBhKNMX+11t7SOdEklGUnx5ASG0F1QwvPfWUsfdLiiIr49Pjiov5pTBuRxZ/nFXHN6F707hHnQFqR4HDOI3Rr7Y+stTnW2nzgRuDfKnNpL2MM3/3cQH593XDG9kklPSGKxOiIU6774yuHEBXu4tZnVnCgSvPpIm3RcejimNvG57frbNGspBhm3zGWw7VN3PLMcuqaWvyQTiT4dEqhW2vnWWundcb3EjmV0XkpzLzlfHaUH+WX73iPkl2z9whX/GEhL3+y1+F0IoFB10OXoDFxQBpf+0xfZi3Yydp9lWw9UEOLx/LwO5uZMiSDHvFRTkcUcZQKXYLK9y8bSEOzm/1H6hmTl8I1o3txw5PL+PacNVTWNVPf7GZoViKPXD+C2Ei9vKV70StegkpUeBg/nz7shGW3juvNc0t2Mzw7iSFZCbxTWMqgzAS+M3nAab9XQ7Mbj7UqfgkZeiVL0PvxlUO4ZVwe/dLjMcbwjb+uYub8HUwZkkFiTDg5KbGt667eewSPx3J+7xS+OvsT6pvcvP7NCQ6mF+k8KnQJehFhLvr3/M/JyvdPHcwHmw5yxWMLAXj8y6OZNqIX1lq+9/Jaymsa+cHlg1hcdAiA8ppG0hOisNby3sYDJERHMCo3mbgo/XpIcNErVkJOflocM285n9Kqel5bXcz9r65nSFYiALsP1QHwP29vIikmgqr6ZhZuL+e6MTl8uLmMr/91tfd79Ijl/e9eTGS4juyV4KFXq4SkKUMzuHV8Pn++eQyR4S7uf3U9H28pA+DBKwYT7jL8+rrh9IiLZMG2clrcHn7z7hb6psXxi+nnsftQHf/aUOrwv0Lk7KjQJaT1So7h+5cNYuWeIzwxbwcDM+K5c1I/1vz0c3x+eBafGZDGwu0VzFq4k6Kyo9w3dRA3X9ibPmlxPLdkN2v2HmHOir1Ya1lSVMGdz6/k4y1l+PPWjSLtpSkXCXk3FOQya8FO9h6uaz0zNcF3mYFJA9N5c20J/+/drVw8MJ3Lz8vEGMOt43rz87mbuO6JJVjrnWd/fuluDtU28f6mg0we3JPHbhqteXYJKBqhS8iLDHfx/csGAnD5eRkn/N3kwRl8ZkAav7hmGM/OuKD1HqhfLMihX3ocN5yfyyWD0vndB9uorm/h7W9N5CdXDuHjrWV8+allNLa4T/vc1lp+/EYhX3xiCR9v1cheupbx5wusoKDArly50m/PJ3K84sp6sn2X5z0blXVN3PnCKq4dnc1NY73X9H9t1X6+/8o6nrz1fCb0T+PdDQeYPqoXEWEnjpGeXriTh9/Z3LoD9qaxufx8+rBPrXdMWU0DyTGRRIa7qKprJiYy7LQ7Zvf6dvLm9Yhtcx0JfsaYVdbagjOtp/eL0m2cS5mD9+bWf79r/AnLrh7Vi1/9czNvrS1hxa7DPLNoF1tKq/nJtKGAt8ifW7Kb4sp6Lj8vg8duGs0fPtzOn+ft4GB1I7NuPZ/aJjeHjjbSNz0egKKyGqb9cREDeibw5QvzeHjuJuKiwvn6xf34yoT81ncPG0uqSIyOIDc1lm++tIraRjcffe9iXC7Tga0joUCFLnIOIsJcXDkii5c/2YcxkBQTwdOLdnF+7xRG5ibzm3e3cF6vJL5UkMtXJvYhKjyM+6YOJispmv/+x0buf62Q5bsOUVbTyOvfuIjBmQl8/5X1RIWHsbP8KD96vZBRuclER7j4+dxNjMxNYkxeCs8u3s0v39nEqNxknrqtgA3F1QAsKqpg0sB09h+p44HXCpkypCc3j+vd5jsB8O4XKCyu5LODM9pcB7zvArYerGFQRoLeCQQ4FbrIOZo+Kpvnl+7BZeDNuydw/2uF/PDV9Uzsn4bHek9oOv4sVYBbx+dTVHaU2Uv30CMukuSYCO5+aTV5qbGs21fJH28azeDMBBZur+DmcXk0uy0XPPwhr64qZmd5Lb+Yu4n0hChW763kjTXFAESGuXh+6R4uyE/lrhdWsbm0mkVFFfxjXQl/v2s8q/Yc4cNNBynITyHJN50T5jLc/eJqiivreeSLI1p3Fi8uqiA1LrL1uP0Xlu3hv9/cAEBCVDh///p4hmQlYq1lcdEhyo82MDgzsXX99rDWtr7bOGZ3RS1zVuzljol9yEiMbvNrqxuaSYgK/9TXi5fm0EXOkbWWy3+/gJE5yTxy/UiKK+uZ9thCjtQ1c+3obP7vS6NO+XXNbg+zl+zmc0MzKKtp5MZZy0iOieCui/ty56R+n1r/ey+v5YPNB4mNDCMrKYbffGEEl/9+AQnR3vHYzRf2ZtaCHWQlxVBSVc8ztxdw6GgTP3x1PTdekMs7haXUNHz6GvJp8ZHkpMSy5UA1D18znE0l1Ty7eBdhLu9RPrmpsfzqn5uZ2N97lcsfvLIOj7XMuXMc87eW8/O5mwDvTudXvz6eRUUVfLyljDG9U7hjwn+Kuaishg83l5GVFM3c9aWs3nOEV74+nozEaN5eV8Kh2iZmzt9BTUMLmYnRPH17Aef1SuSHr66ncH8VXzw/hxvH5rK46BDfemk1V47I4tufHcBfl+2hV3I0V43sRVZSDO9vPMBj/96OtXDZ0Ey+cUm/E/Y/vLuhlCa3ZdrwrHZPTzW1eHj0/a0cqWsiMzGagZkJTBmS0a772za1eDrtxLT2zqGr0EU6oKnFQ5jLEOYriIXby/mftzcx85YxJ1yO4HSKK+vpERfZZkksLqrg5qeXA/D3u8ZzQX4Kkx+dz86KWi4bmsHD1wzj3pfXkhQTwdRhmUwflQ3AvX9bw5trS0iOjeD1b1xEVX0zDc0eGprdlFTVc/HAdCLDXVzz+GJKfHeCunVcb1o8Huas2AfAwIx4XvvGRSRER7DtYA03zVpGs9tDXZObSwb15PuXDeS/Zq/kUG0jDc0e+qbHsfdQHbmpsTw74wI+2nyQR97bSmOLB4DE6HAs0C89nqhwF8t3HQa8Nw2/d8oAfvLmBqrqm7lsaAZvri2hb3ocO8trSYmN4GhjC5lJ0ew7XA9ARJih2W2JCndxx8RpK8hSAAAIOUlEQVQ+PLNoFzkpMaTHR7F812GGZSfy3FfGkhYfxaaSaq5+fBEtHsuAnvE0tLipqmsmKTaC/712BBMHpGGtZeb8ncxdX4LHwoyLerOppJrZS/eQkRhFeU0jHgtThmTw1G3nU1xZT3JsJAb4wSvriI8K5/uXDSIjMYrHPiriT/OKeOLmMUwecvoprfZQoYuECLfH8tlH5zEkM5GZt54PwK//tYWZ83fw8+nncdv4/FN+XWVdE/e/tp4ZF/VhfL8ebX7/hmY3ew7V4fZYhvbyTp1UNzRzoKqBvNTYE/6j2Xuojjtmf4LHY3nj7gkkxURQuL+KW55Zzo1jc3lg6mBW7TnCrc+soL7Ze0jnJYPS+cX0YVQ3NJObGsu8reV8Z84ajIHffnEknx3ck+TYCIwxlFU38F/Pr2T9/iqmjcjijzeNZkNxNY9+sJW6JjdP3VbAgm3lfLL7MN+8pD/1zW4eemsjC7aVk5cay5t3TyA1LpJ3Nxzg3pfX0C89nt9eP5L7Xl1PaVU990wZyJtrislMiiY9PoqF28sprWrgwSuGsHzXYd5eV0JB7xQaWzwUFlcB8F8T+/CTaUNpaHbzzKJdPPLeVq4dnc3c9SUkxUSQlRTDxpIqwl0uPNbSMyGKkqoG4qPCCQ8z/PM7n2m9+fm5UqGLhJCahmYiw11EhXvLdUf5Ue7921pm3XY+WUkdK4uz1eL20OKxJxS922Nb36UArNh1mHc3HODKEVmMyUv+1Jz3nz4uIjc1lqtH9vrU969vcvPPwlKuGJ5FTOSZpzY8HsvcwlLG5CWfsM9i3tYyvvb8Sprd3o7705fHcOWIrBO+tqymgetnLmXPoTpcBu6dMpBvf7Y/Hgsz5+9gR/lRfvOFEa07l90ey42zlvLJ7iNM7J9GXVML6/ZX8fsvjWJUbjIvf7KPorKjFOSnMHlIBtMeW0iz25KTEsOvrhvOuL5t/8d6Oip0Een2NpZUsbO8lrzUWEbmJp9ynbqmFvYeriMnJZb4dpz5W1bTwOKiCq4emY3LQGVdMylxkadcd92+Sv614QB7D9dy75SBDMxo3zTcyVToIiIhor2FrlP/RURChApdRCREqNBFREKECl1EJESo0EVEQoQKXUQkRKjQRURChApdRCRE+PXEImNMObDnHL88DajoxDidJVBzQeBmU66zE6i5IHCzhVqu3tba9DOt5NdC7whjzMr2nCnlb4GaCwI3m3KdnUDNBYGbrbvm0pSLiEiIUKGLiISIYCr0WU4HaEOg5oLAzaZcZydQc0HgZuuWuYJmDl1ERE4vmEboIiJyGkFR6MaYqcaYrcaYImPMAw7myDXGfGyM2WSM2WiMuce3/GfGmGJjzFrfnyscyLbbGFPoe/6VvmWpxpgPjDHbfR9T/Jxp0HHbZK0xptoYc69T28sY86wxpswYs+G4ZafcRsbrMd9rbr0xZoyfcz1ijNnie+43jDHJvuX5xpj647bdTD/navNnZ4z5kW97bTXGXO7nXC8fl2m3MWatb7k/t1db/eC/15i1NqD/AGHADqAvEAmsA4Y6lCULGON7nABsA4YCPwN+4PB22g2knbTs/wEP+B4/APzG4Z/jAaC3U9sLmASMATacaRsBVwD/AgwwDlju51yXAeG+x785Llf+8es5sL1O+bPz/R6sA6KAPr7f2TB/5Trp7x8FfurA9mqrH/z2GguGEfpYoMhau9Na2wT8DZjuRBBrbam1drXvcQ2wGch2Iks7TQdm+x7PBq5xMMtkYIe19lxPLOswa+0C4PBJi9vaRtOB563XMiDZGJNFFzhVLmvt+9baFt+ny4Ccrnjus811GtOBv1lrG621u4AivL+7fs1lvDcvvQGY0xXPfTqn6Qe/vcaCodCzgX3Hfb6fAChRY0w+MBpY7lv0Ld/bpmf9PbXhY4H3jTGrjDF3+pZlWGtLfY8PABkO5DrmRk78JXN6ex3T1jYKpNfdHXhHcsf0McasMcbMN8Z8xoE8p/rZBcr2+gxw0Fq7/bhlft9eJ/WD315jwVDoAccYEw+8Btxrra0GngD6AaOAUrxv+fxtorV2DPB54G5jzKTj/9J63+M5ckiTMSYSuBp4xbcoELbXpzi5jdpijPkx0AK86FtUCuRZa0cD3wNeMsYk+jFSQP7sjnMTJw4c/L69TtEPrbr6NRYMhV4M5B73eY5vmSOMMRF4f1gvWmtfB7DWHrTWuq21HuApuuit5ulYa4t9H8uAN3wZDh57C+f7WObvXD6fB1Zbaw/6Mjq+vY7T1jZy/HVnjJkBTANu9hUBvimNQ77Hq/DOVQ/0V6bT/OwCYXuFA9cBLx9b5u/tdap+wI+vsWAo9E+AAcaYPr6R3o3AW04E8c3PPQNsttb+7rjlx897XQtsOPlruzhXnDEm4dhjvDvUNuDdTrf7Vrsd+Ic/cx3nhFGT09vrJG1to7eA23xHIowDqo5729zljDFTgfuAq621dcctTzfGhPke9wUGADv9mKutn91bwI3GmChjTB9frhX+yuUzBdhird1/bIE/t1db/YA/X2P+2Pvb0T949wZvw/u/648dzDER79ul9cBa358rgBeAQt/yt4AsP+fqi/cIg3XAxmPbCOgBfARsBz4EUh3YZnHAISDpuGWObC+8/6mUAs145yu/2tY2wnvkwZ98r7lCoMDPuYrwzq8ee53N9K37Bd/PeC2wGrjKz7na/NkBP/Ztr63A5/2Zy7f8OeDrJ63rz+3VVj/47TWmM0VFREJEMEy5iIhIO6jQRURChApdRCREqNBFREKECl1EJESo0EVEQoQKXUQkRKjQRURCxP8HDp+FEnuUwVUAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(tr_loss_hist)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD8CAYAAACMwORRAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAHONJREFUeJzt3X90VPWd//HnOz9IECUBQkwEI2AVtZSjnFGxFH+UfoVqMWr9Wjhrg21d9pS2i3bxFLe7mq97ttr9divtsa1f7FKLbcmqlYpVj1bFlSpY4i+qRRTtDwhBEA2gQiTy/v4xlziESWaGzMyduXk9zuEw876f3Pv2Or6887k395q7IyIi0VISdgMiIpJ9CncRkQhSuIuIRJDCXUQkghTuIiIRpHAXEYkghbuISAQp3EVEIkjhLiISQWVhbbimpsbHjBkT1uZFRIrSs88++5a7j0w1LmW4m9kS4HPANnefkGT5ucB9wJ+D0r3ufmOq9Y4ZM4bW1tZUw0REJIGZ/TWdcekcud8B3Aos7WPMKnf/XDobFBGR3EsZ7u7+pJmNyX0rIiLR1dzczJo1aygri8duV1cXkydPprm5OSfby9ac+1lm9iKwBVjg7i9nab0iIpHR0tJCdXU1AB0dHSxatChn28pGuD8HHOfu75rZBcBvgBOSDTSzucBcgIaGhixsWkREkun3pZDuvsvd3w1ePwiUm1lNL2MXu3vM3WMjR6Y82SsiIoep3+FuZnVmZsHrM4J17ujvekVE5PClcynkMuBcoMbMNgM3AOUA7n4bcBnwVTPrAvYAs1yPdxIRCVU6V8vMTrH8VuKXSoqISIEI7TdURUQGgvWrVrKqZSlP/89TVG9+lelz/p6Tp56X8+0q3EVEcmT9qpU8svhWuj7o5MiKQdz+0GP818NPUDtmHEOGDWfGjBk527bCXUQkR1a1LKXrg04ApnxsDFM+NgaAo2pGMvdHP8vptnVXSBGRHNm9462M6tmkcBcRyZGjRiT9lZ9e69mkcBcRyZGps5ooG1RxUK1sUAVTZzXlfNuacxcRyZEDV8WsalnK7h1vcdSIGqbOatLVMiIixe7kqeflJcx70rSMiEgEKdxFRCJI4S4iEkEKdxGRCNIJVREpePl+RF0UKNxFpCjk8xF1UaBpGRGRCNKRu0jIeptySFYDND0haUnnSUxLgM8B29x9Qh/jTgdWE38S0z3Za1Ek+pJNOfQ2DaHpCUlHOtMydwB93nTYzEqB7wKPZKEnERHpp5Th7u5PAm+nGPYN4NfAtmw0JSIi/dPvE6pmNgq4BPhJ/9sREfnIzvvv57VPT2P7rT/i9ZkXsfP++8NuqWhk44TqIuBb7r7fzPocaGZzgbkADQ0NWdi0iETVzvvvp/1fr8f37mVEaSnXPv88JVdcQcXJJ1NaW5vTR9RFQTbCPQa0BMFeA1xgZl3u/pueA919MbAYIBaLeRa2LSIRte2WRfjevQDMHjaM2cOGAVB2xBBOWLEizNaKQr/D3d3HHnhtZncAv00W7CLSP6++8yrPtD/D3b+6m1G1o5g/aT5Thk8Ju62c6Wpvz6guB0vnUshlwLlAjZltBm4AygHc/bacdiehyeTaa11j3T+1tbU0NTVRUhI/BbZ//35mzJhxUK393Xa2jt7Kvsp9bL99O5tsE2ttLScOO5E5l84Js/2cKauvp2vLlqR1SS1luLv77HRX5u5X9qsbKSiZXHsth2/evHnMmzcvaf2A8+85n/ffex+AEdNGdNerhlQx77JDfzYKaq+5unvO/QCrrKT2mqtD7Kp46DdURYrA1ve2ZlSPgqqZM4H43HtXeztl9fXUXnN1d136pnAXKQJ1Q+pof+/Quea6IXUhdJM/VTNnKswPk24cJlIE5k+aT2Vp5UG1ytJK5k+aH1JHUuh05C5SBC4cdyEAP3juB2x9byt1Q+qYP2l+d12kJ4W7pGXD06t49oHfcOu6NdTW1zN1VhP1nzgt7LYGlAvHXagwl7RpWkZSWr9qJY//fDGd778P7ux+azuPLL6VDU+vCrs1EemFjtwlqcRrr/+y7jn2dXYyvm4ky/7wAkb8NhNLfr+Wf7h2Ycidikgy5h7OXQBisZi3traGsm3JzH/OmgnJPidm/FOLbuQkkk9m9qy7x1KN07SMpHTUiJqM6iISPk3LSEpTZzXxyOJb6fqgs7tWNqiCqbOaQuyq8OiWDVJIFO6S0slTzwNgVctSdu94i6NG1DB1VlN3XT6iWzZIoVC4S1pOnnqewlykiGjOXUQkghTuIiIRpHAXEYkgzbmLZKi3q2J+8YtfsHr1asrLy+nq6uLUU0+lsrIyxdpEciOdJzEtAT4HbHP3CUmWNwL/BuwHuoCr3f332W5UpJAkuwLmsssuY+HChVRXV9PR0cE3v/lN2tra2Lt3L0cffTTTpk3Tg+Elb9I5cr8DuBVY2svyx4AV7u5mNhG4CzgpO+2JFI+ampruWzbs2LGD9evXc+aZZ7J8+XLMjNtvv53a2louv/zysFuVASCdx+w9aWZj+lj+bsLbIUA49zMQCdlVV13FggULAPjOd75DaWkpp59+Oqeffnr3mKqqqqSP1BPJtqycUDWzS8zsFeAB4MvZWKdIMdu1a1fS+s6dO/PciQxUWTmh6u7LgeVmdjbx+ffPJBtnZnOBuYDmHiXShg4dmrReVVWV507yS7dgKBxZvVommMIZZ2Y17v5WkuWLgcUQvytkNrctUkjOPvtsHn300YNq5eXlTJs2LaSO8ke3YCgM/Z6WMbOPmZkFrycBFcCO/q5XpFhsffMB/va3JfzPk6fx1FNTad96HxMmTGD8+PHdR+pVVVXMnDmTiRMnhtytDBTpXAq5DDgXqDGzzcANQDmAu98GfB5oMrN9wB7gCx7WTeJF8iDxQSZ797aza/d6YrFBfPfmPZhtw+yLHDHkJC5uvFInTyU06VwtMzvF8u8C381aRyIFbt68ed2h/dRTU9nbORKAxsaP5tMrKyqYMkXBLuHR7QdE+mFvZ3tGdZF8UbiL9ENlRX1GdZF8UbiL9MO44xdQUjL4oFpJyWDGHb8gpI4KzEvLYfWP4ebj4JYJsO6usDsaMHTjMJF+qK9rBOCN17/H3s52KivqGXf8gu76QJN4spldbexvX8eMcSU0Le+ixDbA7U3sr53AjMv1u465ZmFd2BKLxby1tTWUbYtIHtwyAXZuOrRedSxc81L++4kIM3vW3WOpxmlaRkRyY+fmzOqSVQp3EcmNqtGZ1SWrNOcuhy2T+4gAGdV135EImHY93P+PsG/PR7XywfG65JzCXfolk/uIZFqXIjcxuG/9YzfGp2KqRseDfaLuZ58PCncpSLq7YERMvFxhHhKFuxQs3V1Q5PDphKqISAQp3EVEIkjTMhKqV9p3Mf2WJ9nWWcox1YOZ98m6sFsSiQQduUtoXmnfxaPrt9G+cw8OtHXs4f/c/ydeaU/+/FERSZ+O3OWwHXQfEWD//v3MmDEjaQ04pP6SjcUrh/LWA98HC44zfD9dp0zmJN1UUaRf0nkS0xLgc8A2d5+QZPnfAd8CDNgNfNXdX8x2o1J4Eh9a0bPe2/hEYxc+wFHAUZMuPKgev9vR2oNqD//lYe78053c/au7GVU7ivmT5jNl+JR+dC8Sbekcud8B3Aos7WX5n4Fz3P0dM/ss8Qdgn5md9iTKjqkeTFvHnqT12qEffSt48703Wb9jPUdMOILtt29nk21ira3lxGEnMufSOSF0LlL40rorpJmNAX6b7Mi9x7hhwEvuPirVOnVXSPnN821cd+8f2bPvw+7a4PJSbrr0E1x82kcfofPvOZ/29w59slH9kHoeueyRvPQqUijSvStktufcvwI8lOV1SkQdCPD/+/AGtnTs4ZjqwVw7ffxBwQ6w9b2tSX++t7qIZDHczew84uH+qT7GzAXmAjQ0NGRr01LELj5t1CFh3lPdkLqkR+51Q3TZpEhvsnIppJlNBH4KNLr7jt7Guftid4+5e2zkyJHZ2LQMAPMnzaeytPKgWmVpJfMnzQ+pI5HC1+8jdzNrAO4Fvujur/a/JZGDXTgufjXND577AVvf20rdkDrmT5rfXReRQ6VzKeQy4Fygxsw2AzcA5QDufhtwPTAC+LGZAXSlM9kvkokLx12oMBfJQMpwd/fZKZZfBVyVtY5ERKTfdPsBEZEI0u0HClw2HmWnh1mIDDwK9yKQjUfZicjAomkZEZEIUriLiESQwl1EJIIU7iIiEaRwFxGJIF0tExGvtO/iqdd3sKTjYY6tG8m108dz7tghYbclIiFRuBe4dB5lt+Wd9/nbESeyr2QIex74Pm9ZCV/8mXFy3ZFc+YWLw2xfREKS1sM6ckEP68ieKTc/nvSJRqOqB/PUwk+H0JGI5Eq6D+vQnHsEbEkS7H3VRST6FO4RcEz14IzqIhJ9CvcIuHb6eAaXlx5UG1xeyrXTx4fUkYiETSdUIyDdZ5GKyMChcI+IdJ5FKiIDR8ppGTNbYmbbzOylXpafZGarzazTzBZkv0UREclUOnPudwAz+lj+NvCPwPey0ZCIiPRfynB39yeJB3hvy7e5+1pgXzYbExGRw6c5dykYmTx1Sk+XEulbXsPdzOYCcwEaGhryuWkpEpk8dUpEepfXcHf3xcBiiN9+IJ/bFhlo9E1oYNO0jEiE6ZvQwJUy3M1sGXAuUGNmm4EbgHIAd7/NzOqAVmAosN/MrgZOcfddOetaRET6lDLc3X12iuVbgdFZ62gA6e1rs74ii0h/aVomZPqKLCK5oHCXgrWx9U1eeOxv3P7XJ6mtH8FZjcdTO74y7LZEioLCXQpG4lOn3n1nL9s37eakUTGWrryZEjOsxRg+agiXzW4Mu1WRgqdwl4Ixb9485s2bB8DP//kp3n27E4CzP/5RmB85vII586aE0l+x2/nQQ+z4+c/Z8KtlDB89mtprroapU8NuS3JE4S4F6UCwp1uXQyV+E9q3dSt71q/nU5WDue69t7D2LdgVV1B+0knMnDMn7FYlBxTuUpCOHF6RNMiPHF4RQjfFKfGb0GufnkZX/TEAzB42rHtM2ZAjOSEYI9GicA/Jq89s1cnCPpzVeDwrf/kKXR/s766VDSrhrMbjQ+yqeHW1t2dUzxf9Fm3uKNxD8OozW1n5y1eo4CidLOzFiWfWAbD6vtd59+1OjhxewVmNx3fXJTNl9fV0bdmStB42/RZtbijcQ7D6vtfp+mA/Z3+8UScL+3DimXUK8yypveZq2v/1enzv3u6aVVbGT6pKJCncQ6CThZJvVTNnArDtlkV0tbdTVl9P7TVXd9clehTuIdDJQglD1cyZCvMBJJ3H7EmWndV4PGWDDt71OlkoItmkI/cQ6GShiOSawj0kOlkoksRLy2H1j2HvLXD0sTDtemg4P+yuipLCXURCk/hbtOxqY3/7OmaMK6FpeRcltgFub2J/7QRmXP7lsFstOuYeztPuYrGYt7a2hrJtESlAt0yAnZsOrVcdC9e8lP9+CpSZPevusVTjUp5QNbMlZrbNzJLuXYv7oZltNLN1ZjbpcBoWkQFu5+bM6tKndK6WuQOY0cfyzwInBH/mAj/pf1siMuBU9fJAt97q0qeU4e7uTwJv9zGkEVjqcWuAajML/3eaRaS4TLseygcfXCsfHK9LxrJxnfsoIHGibHNQExFJ38TLYeYP43PsWPzvmT+M1yVjeb1axszmEp+6oaGhIZ+bFpFiMPFyhXmWZOPIvQ04NuH96KB2CHdf7O4xd4+NHDkyC5sWEZFkshHuK4Cm4KqZycBOdw/3JtEiIgNcymkZM1sGnAvUmNlm4AagHMDdbwMeBC4ANgLvA1/KVbMiIpKelOHu7rNTLHfga1nrSERE+k13hRQRiSCFu4hIBCncRUQiSOEuIhJBCncRkQhSuIuIRJDCXUQkghTuIiIRpHAXEYkghbuISAQp3EVEIkjhLiISQQp3EZEIUriLiESQwl1EJIIU7iIiEZRWuJvZDDPbYGYbzWxhkuXHmdljZrbOzJ4ws9HZb1VERNKVMtzNrBT4EfBZ4BRgtpmd0mPY94Cl7j4RuBG4KduNiohI+tI5cj8D2Ojub7j7B0AL0NhjzCnA48HrlUmWi4hIHqUT7qOATQnvNwe1RC8ClwavLwGOMrMR/W9PREQOR7ZOqC4AzjGz54FzgDbgw56DzGyumbWaWev27duztGkREekpnXBvA45NeD86qHVz9y3ufqm7nwZ8O6h19FyRuy9295i7x0aOHNmPtkVEpC/phPta4AQzG2tmg4BZwIrEAWZWY2YH1nUdsCS7bYqISCbKUg1w9y4z+zrwMFAKLHH3l83sRqDV3VcA5wI3mZkDTwJfy2HPIlIgmpubWbNmDWVl8Sjp6upi8uTJSWvNzc0hdjrwpAx3AHd/EHiwR+36hNf3APdktzURKQYtLS1UV1cD0NHRwaJFi5LWJL/0G6oiIhGkcBcRiSCFu4hIBCncRUQiSOEuIhJBCncRyZoVb77D4k3bGL/qj8Sefplfb3077JYGrLQuhRQRSaa2tpampiZKSkrYsvcDXtr9PmWnf5LOm/6FnSUlfBH4+JBKvtQ4M+xWBxxz91A2HIvFvLW1NZRti0j2xZ5+mc2d+w6pj64op/WTHw+ho2gys2fdPZZqnKZlRCQr2pIEe191yS2Fu4hkxaiK8ozqklsKdxHJiuvG1TO4xA6qDS4xrhtXH1JHA5tOqIpIVny+bjgAN73RTlvnPkZVlHPduPruuuSXwl1EsubzdcMV5gVC0zIiIhGkcBcRiSCFu4hIBKUV7mY2w8w2mNlGM1uYZHmDma00s+fNbJ2ZXZD9VkVEJF0pw93MSoEfAZ8FTgFmm9kpPYb9C3BX8IDsWcCPs92oiIikL50j9zOAje7+hrt/ALQAjT3GODA0eF0FbMleiyIikql0LoUcBWxKeL8ZOLPHmGbgETP7BjAE+ExWuhMRkcOSrROqs4E73H00cAFwp5kdsm4zm2tmrWbWun379ixtWkREekon3NuAYxPejw5qib4C3AXg7quBSqCm54rcfbG7x9w9NnLkyMPrWEREUkon3NcCJ5jZWDMbRPyE6YoeY/4GTAMws5OJh7sOzUVEQpIy3N29C/g68DCwnvhVMS+b2Y1mdlEw7J+AvzezF4FlwJUe1o3iRUQkvXvLuPuDwIM9atcnvP4TMCW7rYmIyOHSb6iKiESQwl1EJIIU7iIiEaRwFxGJIIW7iEgEKdxFRCJI4S4iEkEKdxGRCFK4i4hEkMJdRCSCFO4iIhGkcBcRiaC0bhwWtubmZtasWUNZWbzdrq4uJk+enLTW3NwcYqciIoWhKMIdoKWlherqagA6OjpYtGhR0pqIiGhaRkQkkhTuIiIRlFa4m9kMM9tgZhvNbGGS5beY2QvBn1fNrCP7rYqISLpSzrmbWSnwI+B/AZuBtWa2Inj6EgDufk3C+G8Ap+WgVxERSVM6R+5nABvd/Q13/wBoARr7GD+b+HNURUQkJOlcLTMK2JTwfjNwZrKBZnYcMBZ4vP+t9e69ddvZ/fs22jqeZk9dDUOnj4Gxg3K5SRGRopLtSyFnAfe4+4fJFprZXGAuQENDQ9orra2tpampiZKSEj7c2Uln227OGXMGVz/w75RYCfYzo7RuMBd+oa8vFCIiA4e5e98DzM4Cmt19evD+OgB3vynJ2OeBr7n706k2HIvFvLW1NeOG22/+Ax92dB5SL62uoH7hGRmvT0SkmJjZs+4eSzUunTn3tcAJZjbWzAYRPzpfkWSDJwHDgNWZNpuJZMHeV11EZCBKGe7u3gV8HXgYWA/c5e4vm9mNZnZRwtBZQIun+irQT6XVFRnVRUQGorTm3N39QeDBHrXre7xvzl5bvRs6fQwd976G79vfXbPykvhJVRERAYro3jIHDDmtFoBdD/+FDzs6Ka2uYOj0Md11EREpwnCHeMArzEVEeqd7y4iIRJDCXUQkghTuIiIRpHAXEYkghbuISAQp3EVEIijlvWVytmGz7cBfe5RrgLdCaKe/irHvYuwZirNv9Zw/xdh3pj0f5+4jUw0KLdyTMbPWdG6IU2iKse9i7BmKs2/1nD/F2Heueta0jIhIBCncRUQiqNDCfXHYDRymYuy7GHuG4uxbPedPMfadk54Las5dRESyo9CO3EVEJAvyHu5m9r/N7GUz229mSc8Qm9mxZrbSzP4UjJ2fsKzZzNrM7IXgzwWF0ncwboaZbTCzjWa2MKE+1syeCer/HTzVKtc9Dzez35nZa8Hfw5KMOS9hX75gZnvN7OJg2R1m9ueEZacWQs/BuA8T+lqRUM/7fk63bzM71cxWB5+jdWb2hYRledvXvX1GE5ZXBPtuY7AvxyQsuy6obzCz6bnq8TB6/maQF+vM7DEzOy5hWdLPSoH0faWZbU/o76qEZXOCz9NrZjYn4427e17/ACcD44EngFgvY+qBScHro4BXgVOC983AggLtuxR4HRgHDAJeTOj7LmBW8Po24Kt56Pk/gIXB64XAd1OMHw68DRwRvL8DuCzP+zmtnoF3e6nnfT+n2zdwInBC8PoYoB2ozue+7uszmjBmHnBb8HoW8N/B61OC8RXA2GA9pQXS83kJn9uvHui5r89KgfR9JXBrkp8dDrwR/D0seD0sk+3n/cjd3de7+4YUY9rd/bng9W7ij/cblY/++ugpZd/AGcBGd3/D3T8AWoBGMzPg08A9wbifAxfnrttujcG20t3mZcBD7v5+TrvqW6Y9dwtxP0Mafbv7q+7+WvB6C7ANSPnLKFmW9DPaY0ziP8s9wLRg3zYSf5Rmp7v/GdgYrC/0nt19ZcLndg0wOg99pZLOvu7NdOB37v62u78D/A6YkcnGC37OPfhKeBrwTEL568HXryW9fW0PyShgU8L7zUFtBNDh8efRJtZz7Wh3bw9ebwWOTjF+FrCsR+3fg319i5nl40G16fZcaWatZrbmwDQS4e1nyHBfm9kZxI/mXk8o52Nf9/YZTTom2Jc7ie/bdH42FzLd7leAhxLeJ/us5EO6fX8++Pd+j5kdm+HP9ionT2Iys0eBuiSLvu3u92WwniOBXwNXu/uuoPwT4N8AD/7+T+DL/eu4e3tZ6Tuf+uo58Y27u5n1emmUmdUDnyD+IPQDriMeVIOIX671LeDGAun5OHdvM7NxwONm9kfiIZQzWd7XdwJz3P3Aw4Bzsq8HGjO7AogB5ySUD/msuPvrydeQd/cDy9y908z+gfg3pk9nY8U5CXd3/0x/12Fm5cSD/Zfufm/Cut9MGHM78Nv+bith3f3tuw04NuH96KC2A6g2s7LgSOhAvd/66tnM3jSzendvDwJlWx+ruhxY7u77EtZ94Ei008x+BiwolJ7dvS34+w0ze4L4t7tfk6P9nK2+zWwo8ADxA4Y1CevOyb5OorfPaLIxm82sDKgi/hlO52dzIa3tmtlniP+P9hx37zxQ7+Wzko9wT9m3u+9IePtT4uduDvzsuT1+9olMNl6Q0zLB/N5/Aevd/fs9ltUnvL0EeCmfvaWwFjghuGJjEPFpjhUeP0OykvicNsAcIB/fBFYE20pnm7PpMSVzYF8H/z4uJj/7OmXPZjbswLSFmdUAU4A/hbifIb2+BwHLgaXufk+PZfna10k/oz3GJP6zXAY8HuzbFcCs4GqascAJwB9y1GdGPZvZacD/Ay5y920J9aSflTz0nG7fiXl2EfHzixD/Bn1+0P8w4HwO/ladWghnkC8hPn/UCbwJPBzUjwEeDF5/ivi0yzrgheDPBcGyO4E/BstWAPWF0nfw/gLiV/e8Tvzo7EB9HPH/EDYCdwMVeeh5BPAY8BrwKDA8qMeAnyaMG0P8SKGkx88/Huzrl4BfAEcWQs/AJ4O+Xgz+/kqY+zmDvq8A9iV8pl8ATs33vk72GSU+BXRR8Loy2Hcbg305LuFnvx383Abgs/nYt2n2/Gjw3+WB/boi1WelQPq+CXg56G8lcFLCz345+HewEfhSptvWb6iKiERQQU7LiIhI/yjcRUQiSOEuIhJBCncRkQhSuIuIRJDCXUQkghTuIiIRpHAXEYmg/w9f/JiQYLDA7AAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "for word in word_list:\n",
    "    tmp = sgram.get_wordvector(sess = sess, word_dic = word_dic, word = word)\n",
    "    x, y = tmp[0][0], tmp[0][1]\n",
    "    plt.scatter(x, y)\n",
    "    plt.annotate(word, xy=(x, y), xytext=(5, 2),\n",
    "                 textcoords='offset points', ha='right', va='bottom')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}