{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Neural Machine Translation Tutorial\n",
    "---\n",
    "\n",
    "Paper Implementation: [Neural Machine Translation by Jointly Learning to Align and Translate](https://arxiv.org/abs/1409.0473) - Dzmitry Bahdanau, Kyunghyun Cho, Yoshua Bengio (v7 2016)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "import numpy as np\n",
    "import matplotlib.pylab as plt \n",
    "import matplotlib.ticker as ticker\n",
    "from torch.nn.utils.rnn import pack_padded_sequence, pad_packed_sequence\n",
    "DEVICE=None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from matplotlib import font_manager, rc\n",
    "font_name = font_manager.FontProperties(fname='/usr/share/fonts/truetype/nanum/NanumGothicLight.ttf').get_name()\n",
    "rc('font', family=font_name)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 신경망 번역기의 목적\n",
    "\n",
    "소스(source)문장을 타겟(target)문장으로 변환하는 것\n",
    "\n",
    "Examples:\n",
    "\n",
    "|source|target|\n",
    "|--|--|\n",
    "| Nice to meet you| 만나서 반갑습니다 |\n",
    "| I am very happy to meet you | 만나서 참 반가워요| "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "flatten = lambda d: [t for s in d for t in s]\n",
    "\n",
    "def build_vocab(data, start_tkn=False):\n",
    "    \"\"\"build vocabulary\"\"\"\n",
    "    if start_tkn:\n",
    "        vocab = {'<unk>': 0, '<pad>': 1, '<s>': 2, '</s>': 3}\n",
    "    else:\n",
    "        vocab = {'<unk>': 0, '<pad>': 1}\n",
    "    \n",
    "    words = set(flatten(data))\n",
    "    for t in words:\n",
    "        if vocab.get(t) is None:\n",
    "            vocab[t] = len(vocab) \n",
    "    return vocab\n",
    "\n",
    "def add_pad(data, start_tkn=False):\n",
    "    \"\"\"add padding of sentences in batch to match lenghts\"\"\"\n",
    "    if start_tkn:\n",
    "        data = [['<s>'] + sent + ['</s>'] for sent in data]\n",
    "    max_len = max([len(sent) for sent in data])\n",
    "    data = [sent + ['<pad>']*(max_len-len(sent)) if len(sent) < max_len else sent \\\n",
    "            for sent in data ]\n",
    "    return data\n",
    "\n",
    "def numericalize(data, vocab):\n",
    "    \"\"\"numericalize and turn them into tensor\"\"\"\n",
    "    f = lambda x: [vocab.get(t) if vocab.get(t) is not None else vocab.get('<unk>') for t in x]\n",
    "    data = list(map(f, data))\n",
    "    return data\n",
    "\n",
    "def preprocess(data, vocab, start_tkn=False):\n",
    "    data = add_pad(data, start_tkn=start_tkn)\n",
    "    data = numericalize(data, vocab)\n",
    "    # 텐서플로우는 아래 부분을 수정해야할겁니다. torch.LongTensor 를 제거하고 data 로만 두시고\n",
    "    # 숫자로 치환된 data 를 numpy array 로 출력하신 다음에 진행하면 될것 같습니다. \n",
    "    return torch.LongTensor(data)  \n",
    "\n",
    "def build_batch(src, trg, src_vocab, trg_vocab, is_sort=False):\n",
    "    if is_sort:\n",
    "        sorted_data = sorted(list(zip(src, trg)), key=lambda x: len(x[0]), reverse=True)\n",
    "        src, trg = list(zip(*sorted_data))\n",
    "    src = preprocess(src, src_vocab, start_tkn=False)\n",
    "    trg = preprocess(trg, trg_vocab, start_tkn=True)\n",
    "    if is_sort:\n",
    "        return (src, src.ne(src_vocab.get('<pad>')).sum(1)), trg\n",
    "    return src, trg"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset = \"\"\"Nice to meet you > 만나서 반가워요 \\n I am very happy to meet you > 만나서 참 반가워요\"\"\".splitlines()\n",
    "dataset = [s.strip().split('>') for s in dataset]\n",
    "src, trg = [[sent.split() for sent in x] for x in zip(*dataset)]\n",
    "src_vocab = build_vocab(src)\n",
    "trg_vocab = build_vocab(trg, start_tkn=True)\n",
    "(inputs, lengths), targets = build_batch(src, trg, src_vocab, trg_vocab, is_sort=True)\n",
    "# 향후에 필요함\n",
    "trg_itos = sorted([(v, k) for k, v in trg_vocab.items()], key=lambda x: x[0])\n",
    "trg_itos = [x[1] for x in trg_itos]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[2, 6, 9, 7, 5, 8, 3],\n",
       "        [4, 5, 8, 3, 1, 1, 1]])"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "inputs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([7, 4])"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lengths"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[2, 4, 5, 6, 3],\n",
       "        [2, 4, 6, 3, 1]])"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "targets"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Model Structure\n",
    "\n",
    "![model_structure](./pics/translation.png)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Encoder"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "![encoder](./pics/translation_encoder.png)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Hyperparameters for Encoder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "EMBED = 20  # embedding_size\n",
    "HIDDEN = 60  # hidden_size\n",
    "ENC_N_LAYER = 3  # encoder number of layers\n",
    "L_NORM = True  # whether to use layernorm"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Encoder Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Encoder(nn.Module):\n",
    "    \"\"\"Encoder\"\"\"\n",
    "    def __init__(self, vocab_size, embed_size, hidden_size, n_layers, layernorm=False, bidirec=False):\n",
    "        super(Encoder, self).__init__()        \n",
    "        self.hidden_size = hidden_size\n",
    "        self.n_layers = n_layers\n",
    "        self.n_direction = 2 if bidirec else 1\n",
    "        self.layernorm = layernorm\n",
    "        self.embedding = nn.Embedding(vocab_size, embed_size)\n",
    "        self.gru = nn.GRU(embed_size, hidden_size, n_layers, bidirectional=bidirec, \n",
    "                          batch_first=True)\n",
    "        if layernorm:\n",
    "            self.l_norm = nn.LayerNorm(embed_size)\n",
    "        \n",
    "    def forward(self, inputs, lengths):\n",
    "        \"\"\"\n",
    "        Inputs:\n",
    "        - inputs: B, T_e\n",
    "        - lengths: B, (list)\n",
    "        Outputs:\n",
    "        - outputs: B, T_e, n_directions*H\n",
    "        - hiddens: 1, B, n_directions*H\n",
    "        \"\"\"\n",
    "        assert isinstance(lengths, list), \"lengths must be a list type\"\n",
    "        # B: batch_size, T_e: enc_length, M: embed_size, H: hidden_size\n",
    "        inputs = self.embedding(inputs) # (B, T_e) > (B, T_e, m)\n",
    "        if self.layernorm:\n",
    "            inputs = self.l_norm(inputs)\n",
    "        \n",
    "        packed_inputs = pack_padded_sequence(inputs, lengths, batch_first=True)\n",
    "        # packed_inputs: (B*T_e, M) + batches: (T_e)\n",
    "        packed_outputs, hiddens = self.gru(packed_inputs)\n",
    "        # packed_outputs: (B*T_e, n_directions*H) + batches: (T_e)\n",
    "        # hiddens: (n_layers*n_directions, B, H)\n",
    "        outputs, outputs_lengths = pad_packed_sequence(packed_outputs, batch_first=True)\n",
    "        # output: (B, T_e, n_directions*H) + lengths (B)\n",
    "        hiddens = torch.cat([h for h in hiddens[-self.n_direction:]], 1).unsqueeze(0)\n",
    "        # hiddens: (1, B, n_directions*H)\n",
    "        return outputs, hiddens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(torch.Size([2, 7, 120]), torch.Size([1, 2, 120]))"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "encoder = Encoder(len(src_vocab), EMBED, HIDDEN, ENC_N_LAYER, L_NORM, bidirec=True)\n",
    "enc_output, enc_hidden = encoder(inputs, lengths.tolist())\n",
    "enc_output.size(), enc_hidden.size()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Decoder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([2, 1, 20])"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dec_embedding = nn.Embedding(len(trg_vocab), EMBED)\n",
    "sos = torch.LongTensor([2]*inputs.size(0)).unsqueeze(1)\n",
    "dec_input = dec_embedding(sos)\n",
    "dec_input.size()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Hyperparemeters for Decoder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "DEC_N_LAYER = 1\n",
    "DROP_RATE = 0.2\n",
    "METHOD = 'general'\n",
    "TF = True\n",
    "RETURN_W = True"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Attention"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "![attention](./pics/translation_attention.png)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Attention(nn.Module):\n",
    "    \"\"\"Attention\"\"\"\n",
    "    def __init__(self, hidden_size, method='general', device='cpu'):\n",
    "        super(Attention, self).__init__()\n",
    "        \"\"\"\n",
    "        * hidden_size: decoder hidden_size(H_d=encoder_gru_direction*H)\n",
    "        methods:\n",
    "        - 'dot': dot product between hidden and encoder_outputs\n",
    "        - 'general': encoder_outputs through a linear layer \n",
    "        - 'concat': concat (hidden, encoder_outputs) ***NOT YET***\n",
    "        - 'paper': concat + tanh ***NOT YET***\n",
    "        \"\"\"\n",
    "        self.method = method\n",
    "        self.device = device\n",
    "        self.hidden_size = hidden_size \n",
    "        if self.method == 'general':\n",
    "            self.linear = nn.Linear(hidden_size, hidden_size)\n",
    "\n",
    "    def forward(self, hiddens, enc_outputs, enc_lengths=None, return_weight=False):\n",
    "        \"\"\"\n",
    "        Inputs:\n",
    "        - hiddens(previous_hiddens): B, 1, H_d\n",
    "        - enc_outputs(enc_outputs): B, T_e, H_d\n",
    "        - enc_lengths: real lengths of encoder outputs\n",
    "        - return_weight = return weights(alphas)\n",
    "        Outputs:\n",
    "        - contexts: B, 1, H_d\n",
    "        - attns: B, 1, T_e\n",
    "        \"\"\"\n",
    "        hid, out = hiddens, enc_outputs\n",
    "        # Batch(B), Seq_length(T)\n",
    "        B, T_d, H = hid.size()\n",
    "        B, T_e, H = out.size()\n",
    "        \n",
    "        score = self.get_score(hid, out)\n",
    "        # score: B, 1, T_e\n",
    "        if enc_lengths is not None:\n",
    "            mask = self.get_mask(B, T_d, T_e, enc_lengths)  # masks: B, 1, T_e\n",
    "            score = score.masked_fill(mask, float('-inf'))\n",
    "        \n",
    "        attns = torch.softmax(score, dim=2)  # attns: B, 1, T_e\n",
    "        contexts = attns.bmm(out)\n",
    "        if return_weight:\n",
    "            return contexts, attns\n",
    "        return contexts\n",
    "            \n",
    "    def get_score(self, hid, out):\n",
    "        \"\"\"\n",
    "        Inputs:\n",
    "        - hid(previous_hiddens): B, 1, H_d \n",
    "        - out(enc_outputs): B, T_e, H_d\n",
    "        Outputs:\n",
    "        - score: B, 1, T_e\n",
    "        \"\"\"\n",
    "        if self.method == 'dot':\n",
    "            # bmm: (B, 1, H_d) * (B, H, T_e) = (B, 1, T_e)\n",
    "            score = hid.bmm(out.transpose(1, 2))\n",
    "            return score\n",
    "        \n",
    "        elif self.method == 'general':\n",
    "            # linear: (B, T_e, H_d) > (B, T_e, H_d)\n",
    "            # bmm: (B, 1, H_d) * (B, H_d, T_e) = (B, 1, T_e)\n",
    "            score = self.linear(out)\n",
    "            score = hid.bmm(score.transpose(1, 2))\n",
    "            return score\n",
    "\n",
    "    def get_mask(self, B, T_d, T_e, lengths):\n",
    "        assert isinstance(lengths, list), \"lengths must be list type\"\n",
    "        mask = torch.zeros(B, T_d, T_e, dtype=torch.uint8).to(self.device)\n",
    "        for i, x in enumerate(lengths):\n",
    "            if x < T_e:\n",
    "                mask[i, :, x:].fill_(1)\n",
    "        return mask"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(torch.Size([2, 1, 120]), torch.Size([2, 1, 7]))"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "attention = Attention(encoder.n_direction*HIDDEN, method='general', device=DEVICE).to(DEVICE)\n",
    "contexts, attns = attention(enc_hidden.transpose(0, 1), enc_output, lengths.tolist(), return_weight=True)\n",
    "contexts.size(), attns.size()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAV8AAADwCAYAAACniGcOAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAEDtJREFUeJzt3X+onmd9x/H3N2ltR3/mh9BN1la2WFidsDUqZiAEIiSbRGchlWUdDEatgozqwIoTN1t1lMn+sy7uj7m0O39IoKko07gWM2oJTW0YKLWpYDeZG01C2Fptac757I/znORpes557qd5zv0jvl/lgXP//nIaPue6r+e6r7uSIElq17quC5CkX0aGryR1wPCVpA4YvpLUAcNXkjpg+EpSBwxfSeqA4StJHTB8JakDl3RdgCRdiJ07d+bEiRON9n3yySe/lWTnGpfUiOEradBOnDjBE0880WjfdevWbV7jchozfCUN3sIA56gxfCUNWoAhThBm+EoauBBmE75VtRe4DZgHHk9y33nb7wcWgI3AN5I8MFr/HeDZsV3vTnJ6tWsZvpKGLTC/cOHhW1VXAbcDu5KkqvZX1ZYkx89eKvnwaN8CDgMPjG27c5rrGb6SBi1M1ee7uaqOji3vS7Jv9PM24FDO9WEcBLYDx3mty4BTY8svVNU9wI3A4SRfmVSI4Stp8Kbo8z2RZOsK2zbx6kA9BWxZYd97gbNdEkneD2dbxPdX1Y+TPLJaIT5kIWnwkjT6THAS2DC2vHG07lWq6i7gqSSPLVNHgK8Db5t0McNX0qAlYaHhZ4IjwI5R6xVgN4v9umdV1UeAF5M8uMp53g1MHHhst4OkwZvFULMkp6tqPzBXVWeAY0meXtpeVduAu4FvVtWXR6s/neT5qvoicCVwOXBkuVbx+QxfSYMWYH5G43yTzAFz4+uq6gCwJ8n3gOtXOO7j017L8JU0eGv5kEWSW9fivIavpMHz8WJJaluzkQy9Y/hKGjTndpCkjswvLHRdwtQMX0kDN7uJddpk+EoatARmMK9O6wxfSYNnn68kdcDwlaSWTTmlZG8YvpKGLXG0gyR1wW4HSWpZwKFmktQFh5pJUgfsdpCkDhi+ktSyONpBkrphy1eSWuZDFpLUEYeaSVIHHGomSS1LwoJfuElS++zzlaQOONpBkjpg+EpSy5LY7SBJXXComSS1LMD8AMeaGb6SBs8+X0nqgH2+ktS2xJavJLUt2O0gSZ2w20GSOmD4SlLLnM9Xkrowwy/cqmovcBswDzye5L7ztt8PLAAbgW8keWC0fgdwF/Ai8NMkH5t0LcNX0uBN0fLdXFVHx5b3JdkHUFVXAbcDu5KkqvZX1ZYkx5d2TvLh0b4FHAYeGP38SeD3k7xcVfdW1XuSHFqtEMNX0qBNOdrhRJKtK2zbBhzKuZMdBLYDx5fZ9zLg1OjntwA/TPLyaPkh4AOA4Svp4jajtxdv4lygMvp5ywr73gssdUksd9ymSRdb9zoKlKQeSeP/JjgJbBhb3jha9ypVdRfwVJLHpjnufIavpEFLmn8mOALsGPXhAuxmsV/3rKr6CPBikgfHVj8LvLWqLhstvw/47qSL2e0gafBmMdQsyemq2g/MVdUZ4FiSp5e2V9U24G7gm1X15dHqTyd5vqruAR6sqheA54FvT7qe4Stp8GY11CzJHDA3vq6qDgB7knwPuH6F4x4FHp3mWr0N30nj7bpSVeuBzwK3JNnZdT3jVhqD2LWq+hJwKXAF8EySv+q2onOq6hLgn4D/S/KhrusBqKqnWLwFBjgDfDQ9mbygqn4D+NRocR74TJL/6rCkNX/IIsmta3HeXoZvk/F2HXov8DDwzq4LOd9yYxC7rWhRko8s/VxVX62qm5L8qMuaxvwl8I/Ano7rGHcyyZ1dF3G+0b+rvwE+lOTUpP1b46vjZ2qa8XatSnIQ4FyffC+Nj0HsjaraALwR+J+uawGoqj8CjgLPdF3LedZX1RdYvMX9WpKHui5o5O3AfwKfHzWQHk3yDx3XtKgfNwZT6Wv4TjPeTq81Pgaxc1X1m8Bfs/hH9a4kpzsuiar6HeC6JP9cVTd2XM6rJNkOUFWXAl+rqh/05K7vRuCtwO4kL1XV/VX1oyT/1nFdZICvEerrULPXNW5Oy45B7FySZ5PsZfEP6N6quq7rmoAPAjeNvrX+HPB7o2FEvZHkFRafkrq561pGfs7iHelLo+WHgVs6rOesGQ01a1Vfw3fieDu91gpjEHsjyRlgPfCGHtTyiSQfGvWtfgp4LMmXuq5rGe8CjnVdxMiTwDvGlt8J/HtHtZy1GKxp9OmTXnY7TBpv1xOvdF3AuNXGIHZYFlX1u8DHgBeAq4EDSf6jy5qWMc/iqIJeqKqvAr8ArgQeSvKTbitalORnVfXtqppjcfaunyR5pOu6YJhvsqghFi1JS9580035zJf+vtG+f7pj+5OrTKzTql62fCWpqaVuh6ExfCUNnuErSV0wfCWpfQPM3t4ONQOgqu7ouoaV9LU265peX2uzroYSFuYXGn36pNfhC/Trf/Kr9bU265peX2uzrgaWXiPkOF9JalnfgrWJmYfvr1xxZa65duLrixq5+pqNXPemG3r5W51lbevWzW6Snms2bOTXfv3G3v3OrtmwaWZ1zXpSo2s2bOJN17/5gmubdQDM8nc2S1fP8N/Yz3763Ikkb7zQ8xi+wDXXbuJP7vzErE97Ubv8issm79SBWtfPXqn1l6zvuoRlzZ+Z77qEFfWtv3PJvX/xZ89d8EkSGODEOnY7SBo8W76S1LIAC7Z8JallPl4sSd0Y4mTqhq+kgevfGN4mDF9Jg2f4SlLLnFJSkjqSecNXklpny1eS2tbDSXOaMHwlDZ7hK0ktW5pScmgMX0nDFkhPJw5ajeEraeDs85WkTgwwe5uFb1XtBW4D5oHHk9y3plVJ0hQuypZvVV0F3A7sSpKq2l9VW5IcX/vyJGl1ycU7sc424FDO/Wk5CGwHzobv6G2md8Di63UkqU2zavlOusuvqvXAZ4FbkuwcW/8d4NmxXe9Ocnq1azUJ303AqbHlU8CW8R2S7AP2Ab1955qki1VYWLjw0Q4N7/LfCzwMvPM1VSR3TnO9JuF7Erh5bHnjaJ0kdW+6iXU2V9XRseV9o8YjNLjLT3IQln2J6wtVdQ9wI3A4yVcmFdIkfI8Af15Vfzcqajfw+QbHSVI7mvf5nkiydYVtE+/yV5Lk/QC1mMr3V9WPkzyy2jETwzfJ6araD8xV1RngWJKnmxQkSWtt8Qm3mZzqgu/yR90VXwfeBqwavo3eDZ5kLskHk/xxkr+dphhJWmsZTa4z6TPBEWBHnetT2A0cfh3lvBt4YtJOPmQhadgSFmbwePGUd/mvjC9U1ReBK4HLgSNJHpt0PcNX0uDNaqhZkjlgbnxdVR0A9iSZH9tv13nHfXzaaxm+kgZtrWc1S3LrWpzX8JU0bDP8xq1Nhq+kgXNWM0nqRIY3na/hK2ngwkweL26b4Stp0HyNkCR1xPCVpNblop3PV5L6a7pZzXrD8JU0fIavJLUrwILdDvCm6zZz7yfvmPVpL9gbLunv35mfv/xy1yUs639feqnrEpb1q9de23UJ6pOL+B1uktRjPuEmSZ0wfCWpA4avJLUsgcxgMvW2Gb6SBm+ADV/DV9LQ+YWbJHXC8JWktvl4sSS1L/iQhSR1IMTJ1CWpZXY7SFI3Bpi9hq+k4bPPV5Ja5jvcJKkL9vlKUhcyyFfHr2uyU1Wtr6rPVdW/rHVBkjStLKTRp08ahS/wXuBhbClL6pvFTt9mnx5pFKZJDgJU1dpWI0lTWsreoZlJS7aq7gDuALj++utncUpJamyIX7g17XZYVZJ9SbYm2bp58+ZZnFKSmklYmF9o9OkT+3AlDd4QW77Thu8ra1KFJL1Os3zIoqr2ArcB88DjSe47b/t64LPALUl2jq3fAdwFvAj8NMnHJl1rqm6HJLum2V+S2pCk0Wc1VXUVcDvwviR/CPx2VW05b7fXjPyqxZEInwQ+kGQP8POqes+kmmfS5ytJ3Wk4zGwxfDdX1dGxzx1jJ9oGHMq5lD4IbH/VlZKDSY6cV8BbgB8meXm0/ND5xy3HPl9JwxZI8+/STiTZusK2TcCpseVTwPkt36bHbZp0kOErafBm9HjxSeDmseWNo3VNjtsw7XF2O0gatKUv3C60zxc4Auyoc0+T7QYONyjhWeCtVXXZaPl9wHcnHWTLV9KwzWhWsySnq2o/MFdVZ4BjSZ5eYfdXxo6br6p7gAer6gXgeeDbk65n+EoauNlNmpNkDpgbX1dVB4A9SebH9tt13nGPAo9Ocy3DV9LwreFDFkluXYvzGr6SBi9c/E+4SVKvJGFhYX7yjj1j+EoavF+GuR0kqXcMX0nqgOErSS1bfICiX3P1NmH4Sho8wxf4/ve/f+KySy99bkan2wycmNG5Zq2vtVnX9Ppa2y9DXTfM4iR2OwBJ3jirc1XV0VVmIOpUX2uzrun1tTbras7wlaTW2ecrSa3LjCbWaVvfw3df1wWsoq+1Wdf0+lqbdTU0xPCtIRYtSUuuvnpT3vH2P2i0778+sv/JvvRX973lK0kTBft8Jal1Q7yDN3wlDZpfuElSJxq9n613DF9Jg+d8vpLUAVu+ktS2xU7frquYmuEradCC73CTpE44t4Mktc7RDpLUiYUFW76S1KrF79sMX0lqmd0OktQNw1eS2udQM0nqgN0OktSyJM7tIEldsOUrSR0wfCWpA4avJLUuMKOHLKpqL3AbMA88nuS+Jtur6ingyGi3M8BHM+EvguEradASWJhB+FbVVcDtwK4kqar9VbUlyfEG208muXOa66274IolqWNJGn2AzVV1dOxzx9hptgGHxlqsB4HtDbevr6ovVNWDVfX+JjXb8pU0cJlmbocTSbausG0TcGps+RSwpcn2JNsBqupS4GtV9YOlFvNKbPlKGrwpWr6rOQlsGFveOFrXdDtJXgEOATdPupjhK2nwZhS+R4AdVVWj5d3A4Sm2L3kXcGzSxex2kDRoi1NKXvhQsySnq2o/MFdVZ4BjSZ5usr2qvgr8ArgSeCjJTyZdr4Y4Pk6Sllx++RW54YbfarTvM88cfXKVPt9lVdUBYE+SmT7DbMtX0uCtZSMyya1rcV7DV9LgDfEO3vCVNHC+yUKSWuc73CSpI7Z8Jal1Ib46XpLa5zvcJKkD9vlKUstm9YRb2wxfSQPnUDNJ6sSCX7hJUvvs85Wkti12+nZdxdQMX0mDFhxqJkmd8As3SeqAfb6S1Lo42kGS2uZDFpLUEcNXkloXsM9XktrnUDNJ6oDdDpLUsiQsLMz0re6tMHwlDZ4tX0nqgOErSR0wfCWpC4avJLUrCQvxCzdJap3dDpLUAcNXklrnCzQlqRPO5ytJLXNKSUnqRGz5SlIXDF9J6sCsuh2qai9wGzAPPJ7kvibbJx23HMNX0tB9K8nmhvteXlVHx5b3JdkHUFVXAbcDu5KkqvZX1ZYkx1fbDvz3asetxPCVNGhJds7oVNuAQznXjD4IbAeOT9j+3ITjlrVuRkVL0tBtAk6NLZ8arZu0fdJxyzJ8JWnRSWDD2PLG0bpJ2ycdtyzDV5IWHQF2VFWNlncDhxtsn3TcsuzzlSQgyemq2g/MVdUZ4FiSp5tsX+24ldQQnwyRpLZU1QFgTzLbeSsNX0nqgH2+ktQBw1eSOmD4SlIHDF9J6oDhK0kd+H+2vUjYm60KNQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "fig = plt.figure()\n",
    "ax = fig.add_subplot(111)\n",
    "cax = ax.matshow(attns.detach().squeeze(1).numpy(), cmap='bone')\n",
    "fig.colorbar(cax)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Decoder model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "![deocder](./pics/translation_decoder.png)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Decoder(nn.Module):\n",
    "    \"\"\"Decoder\"\"\"\n",
    "    def __init__(self, vocab_size, embed_size, hidden_size, n_layers=1, sos_idx=2, drop_rate=0.0, layernorm=False, method='general', teacher_force=False, device='cpu', return_w=False):\n",
    "        super(Decoder, self).__init__()\n",
    "        self.vocab_size = vocab_size\n",
    "        self.n_layers = n_layers\n",
    "        self.hidden_size = hidden_size\n",
    "        self.device = device\n",
    "        self.sos_idx = sos_idx\n",
    "        self.return_w = return_w\n",
    "        self.teacher_force = teacher_force\n",
    "        self.layernorm = layernorm\n",
    "        \n",
    "        self.embedding = nn.Embedding(vocab_size, embed_size)\n",
    "        self.dropout = nn.Dropout(drop_rate)\n",
    "        self.attention = Attention(hidden_size, method=method, device=device)\n",
    "        self.gru = nn.GRU(embed_size+hidden_size, hidden_size, n_layers, bidirectional=False, \n",
    "                          batch_first=True)\n",
    "        self.linear = nn.Linear(2*hidden_size, vocab_size)\n",
    "        if layernorm:\n",
    "            self.l_norm = nn.LayerNorm(embed_size)\n",
    "    \n",
    "    def start_token(self, batch_size):\n",
    "        sos = torch.LongTensor([self.sos_idx]*batch_size).unsqueeze(1).to(self.device)\n",
    "        return sos\n",
    "    \n",
    "    def init_hiddens(self, batch_size):\n",
    "        return torch.zeros(batch_size, self.n_layers, self.hidden_size).to(self.device)\n",
    "    \n",
    "    def forward(self, hiddens, enc_output, enc_lengths=None, max_len=None, targets=None, \n",
    "                is_eval=False, is_test=False, stop_idx=3):\n",
    "        \"\"\"\n",
    "        * H_d: decoder hidden_size = encoder_gru_directions * H\n",
    "        * M_d: decoder embedding_size\n",
    "        Inputs:\n",
    "        - hiddens: last encoder hidden at time 0 = 1, B, H_d \n",
    "        - enc_output: encoder output = B, T_e, H_d\n",
    "        - enc_lengths: encoder lengths = B\n",
    "        - max_len: max lenghts of target = T_d\n",
    "        Outputs:\n",
    "        - scores: results of all predictions = B*T_d, vocab_size\n",
    "        - attn_weights: attention weight for all batches = B, T_d, T_e\n",
    "        \"\"\"\n",
    "        if is_test:\n",
    "            is_eval=True\n",
    "        inputs = self.start_token(hiddens.size(1))  # (B, 1)\n",
    "        inputs = self.embedding(inputs) # (B, 1, M_d)\n",
    "        if self.layernorm:\n",
    "            inputs = self.l_norm(inputs)\n",
    "        inputs = self.dropout(inputs)\n",
    "        # match layer size: (1, B, H_d) > (n_layers, B, H_d)\n",
    "        if hiddens.size(0) != self.n_layers:\n",
    "            hiddens = hiddens.repeat(self.n_layers, 1, 1)\n",
    "        # prepare for whole target sentence scores\n",
    "        scores = []  \n",
    "        attn_weights = []\n",
    "        for i in range(1, max_len):\n",
    "            # contexts[c{i}] = alpha(hiddens[s{i-1}], encoder_output[h])\n",
    "            # select last hidden: (1, B, H_d) > transpose: (B, 1, H_d) > attention\n",
    "            contexts = self.attention(hiddens[-1:, :].transpose(0, 1), enc_output, enc_lengths, \n",
    "                                      return_weight=self.return_w)\n",
    "    \n",
    "            if self.return_w:\n",
    "                attns = contexts[1]  # attns: (B, seq_len=1, T_e) \n",
    "                contexts = contexts[0]  # contexts: (B, seq_len=1, H_d)\n",
    "                attn_weights.append(attns)\n",
    "                \n",
    "            # gru_inputs = concat(embeded_token[y{i-1}], contexts[c{i}]): (B, seq_len=1, H_d+M_d)\n",
    "            gru_inputs = torch.cat((inputs, contexts), 2)\n",
    "            \n",
    "            # gru: s{i} = f(gru_inputs, s{i-1})\n",
    "            # (B, 1, M_d+H_d) > (n_layers, B, H_d)\n",
    "            _, hiddens = self.gru(gru_inputs, hiddens)            \n",
    "            \n",
    "            # scores = g(s{i}, c{i})\n",
    "            # select last hidden: (1, B, H_d) > transpose: (B, 1, H_d) > concat: (B, 1, H_d + H_d) >\n",
    "            # output linear : (B, seq_len=1, vocab_size)\n",
    "            score = self.linear(torch.cat((hiddens[-1:, :].transpose(0, 1), contexts), 2))\n",
    "            scores.append(score)\n",
    "    \n",
    "            if (self.teacher_force and not is_eval):\n",
    "                selected_targets = targets[:, i].unsqueeze(1)\n",
    "            else:\n",
    "                selected_targets = None\n",
    "            \n",
    "            inputs, stop_decode = self.decode(is_tf=self.teacher_force, \n",
    "                                              is_eval=is_eval,\n",
    "                                              is_test=is_test,\n",
    "                                              score=score, \n",
    "                                              targets=selected_targets, \n",
    "                                              stop_idx=stop_idx)\n",
    "            if stop_decode:\n",
    "                break\n",
    "            \n",
    "        scores = torch.cat(scores, 1).view(-1, self.vocab_size)  # (B, T_d, vocab_size) > (B*T_d, vocab_size)\n",
    "        if self.return_w:\n",
    "            return scores, torch.cat(attn_weights, 1)  # (B, T_d, T_e)\n",
    "        return scores\n",
    "    \n",
    "    def decode(self, is_tf, is_eval, is_test, score, targets, stop_idx):\n",
    "        \"\"\"\n",
    "        - for validation: if is_tf, set 'is_eval' True, else False\n",
    "        - for test evaluation: set 'is_tf' False and set 'is_eval' True\n",
    "        \"\"\"\n",
    "        stop_decode = False\n",
    "        if is_test:\n",
    "            # test\n",
    "            preds = score.max(2)[1]\n",
    "            if preds.view(-1).item() == stop_idx:\n",
    "                stop_decode = True\n",
    "            inputs = self.embedding(preds)\n",
    "        else:\n",
    "            # train & valid\n",
    "            if is_tf and not is_eval:\n",
    "                assert targets is not None, \"target must not be None in teacher force mode\"\n",
    "                inputs = self.embedding(targets)\n",
    "            else:\n",
    "                preds = score.max(2)[1]\n",
    "                inputs = self.embedding(preds)\n",
    "\n",
    "        if self.layernorm:\n",
    "            inputs = self.l_norm(inputs)\n",
    "        inputs = self.dropout(inputs)\n",
    "        return inputs, stop_decode"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "decoder = Decoder(len(trg_vocab), EMBED, encoder.n_direction*HIDDEN, n_layers=DEC_N_LAYER,\n",
    "                  drop_rate=DROP_RATE, method=METHOD, layernorm=L_NORM, \n",
    "                  sos_idx=trg_vocab['<s>'], teacher_force=TF, \n",
    "                  return_w=RETURN_W, device=DEVICE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(torch.Size([8, 7]), torch.Size([2, 4, 7]))"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "output, attns = decoder(hiddens=enc_hidden, enc_output=enc_output, max_len=targets.size(1),\n",
    "                targets=targets)\n",
    "output.size(), attns.size()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "LR = 0.01\n",
    "LAMBDA = 0.00001\n",
    "DECLR = 5.0\n",
    "STEP = 5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "loss_function = nn.CrossEntropyLoss(ignore_index=trg_vocab['<pad>'])\n",
    "enc_optimizer = optim.Adam(encoder.parameters(), \n",
    "                           lr=LR, \n",
    "                           weight_decay=LAMBDA)\n",
    "dec_optimizer = optim.Adam(decoder.parameters(), \n",
    "                           lr=LR * DECLR, \n",
    "                           weight_decay=LAMBDA)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "STEP: 0, loss: 1.9032\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAWwAAADwCAYAAAAkTF41AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAFl9JREFUeJzt3X+sHfV95vH3Y5OYGlAW2wkUYoekdVcov2NrqyVSd5FcCUcNoWEXs0tJLSQM7S5bhW4jQgE5JT8oqzZdIkpr1oXYRKa0tEA2bFKytUIgrIsB76Lll52GEJpG+NqBdQw19j3P/nHmmPHNvT5z7Ln3znieFzrCM/Odmc+5wMPc73znO7JNREQ035zZLiAiIqpJYEdEtEQCOyKiJRLYEREtkcCOiGiJBHZEREsksCMiWiKBHRHREgnsiIiWOG62C4iImGnnnHOOx8bGKrV97LHHvmH7nGkuqZIEdkR0ztjYGI8++miltnPmzFk0zeVUlsCOiE7qtXAepQR2RHSOgTZOfJfAjogOMiaBHRHRfIbxXgI7IqLxTH192JIuAlYB48Ajtm+csP0WoAcsAL5m+45i/TeBHaWmV9l++XDnSmBHRCfV0Yct6STgYmClbUvaKGmp7e2l8/xG0VbAg8AdpW2Xj3K+PDgTEZ1ku9IHWCRpa+mzpnSYs4AH/Eb63wucPcUp5wG7S8s/kXR9EfKXVqk5V9gR0Tm2R+kSGbO9fIptCzk0hHcDS6do+1ngYHeJ7fPg4JX3LZK+a/tvD1dIrrAjopNGuMI+nF3AyaXlBcW6Q0j6JPCE7YcnqcPAV4H3DTtZAjsiOsfAuF3pM8QWYEVxlQxwLv1+6oMk/Saw1/ZXDnOcXwKGPnqZLpGI6KQ6bjraflnSRmCTpAPANtvPDLZLOgu4Crhf0p8Uq6+1vVPSHwAnAscDWya7+p4ogR0RnVTXsD7bm4BN5XWS7gYusP0dYMkU+/32qOdKYEdE91Trnz6Kw/v86ThuAjsiOidziUREtMh4rzfbJYysVaNEJC2RtFfS0tK6fyfpYkmHuwM7KyTdKunU2a6jDsfSd4kYTP5U5a8maVVg06/3f9IfgD4wF5hr+6LZKemw3sSx81vMsfRdouNs6FX8NEkb/wN8CXhU0oW27xyslHS/7Y9IWgR8DniVfsD/LrAcuAT4MfCK7eumu0hJq4F/AayVtAG4FNgL/DPgDtv/fZrP/w7gz4DHgUXAd4APAvuB/bb/s6TPAW+hP7Tov9l+SNJ7gU8BY/T/Z/hJ+nMlDL7LTbb/T001/jHwh7Z3SFpVnG8lE35Okq4Fvmn7kWK/+21/pI4aorvShz1zbgXulnR/ad2bi7//PvD7tp+Dg499XgessN2T9AVJy2w/Np0F2r5d0r8G1hY1fdb205LmAF+T9LDtH09jCQIO2P4dAEkvAMuK8Z9fLmYY22P7dyUdR/9Jq5VFrf/W9l5JlwPnlb+L7RdrrHETcCH935g+Tv/fx2sm/pwofosq7ffmnzpSxIgS2DOkCN7fAz4DTAzexYOwLrwVOAX4fPEw0ikc+ijpTDjF9tNwsPZtwM8BW6f5vD8o/Xm77Z3Fn/cApwMfkHRDsW5f8fefB64tflZvAYYO5j8KDwG/I+kU+r/9/PwUP6eJ5k6yLqKyOqdXnUmtDGwA209I+jVgGfC/S5v+UdJ7bT9ZLI8BPwKutj3Tt4XH6f+M/1HSmaUrx/cDX5jhWib+22ngf9n+ownr/x74jO3XJqwffJf6CupPR7mN/hX2BmDNFD+nV4Cfhf6N58GfI46Y3cpRIm0L7PHiM7AWeJJ+P+3rxbpPAf9F0p6i7dXAfwX+XNIu+t0E/3GG6v0W8EfAXwOflvQq/avWm23/v2k+98Sf1f4J2/4K+C1Jf0b/6vqhYq6DzwAbJY3Rv9F4he1XB99F0nrbX62xzo3A14E1wHeBGyb+nCTdCdwq6V8V36OWPvTotjZ2iaiNRcexQ9LPAb9m+zOzXUt0x3ve/37/1Te+XqntP//Z0x47zPSqM6ptV9hxDJH0b4BfBX5rtmuJ7mnakL0qEtgxa2z/JfCXs11HdFMbexcS2BHRSQnsiIgWcEtHibTt0fShJrwgs1GaWlvqGk1T64Lm1tbEump6RdiMOuYCm/7wsKZqam2pazRNrQuaW1uj6ho8OFPl0yTpEomITmraTHxVNCKwFy5a5CVLJn2LzsgWL17MBz/0oVr+SdT969DbFy/mAx/8YC0HnTOnvl+OFi9ezIeWLaulLg1vUtmSJUtYVlNdjz/+eB2HOUhSY/9rb2ptNdY1ZvutR3uQDOs7QkuWLOFbD0/nlBVHZt/+/cMbzZIT5s2b7RImddzcZk7zMe9Nb5rtEqI+3z/aA9im18Kbjo0I7IiImda0/ukqEtgR0UlNGwFSRQI7IjopgR0R0QJu4JC9KhLYEdFJGdYXEdECBsZbOK4vgR0RnZQ+7IiIlkgfdkREGzRwYqcqEtgR0TkmXSIREa2RLpGIiJZIYEdEtMBgPuy2SWBHRPd05aajpLfZfmlIm1Nt/+jIy4qImF51XWFLughYBYwDj9i+ccL2W4AesAD4mu07ivUrgE8Ce4EXbV857FwjzYIv6TTgttLynZIWTtL0FklnjHLsiIiZMhglcrTvdJR0EnAx8DHbvwq8V9LSQ85l/4bt/wD8e+CyYj8BnwY+bvsC4FVJvzys7sqBLel0YD1webG8AJhre9ckzS8Dbpb0zqrHj4iYSeO9XqUPsEjS1tKn/H7Ks4AH/Eay3wucPcUp5wG7iz//AvCU7X3F8j2H2e+gSl0iRVjfClxq+8Vi9YXAXZLmA38M/Bh4xfZa2y9JWg3cJuk/2f77SY65huLFnIsXL65SRkRETTzK5E9jtpdPsW0hb4QwxZ+XTtH2s8Cgu2Sy/SbrrTjE0CvsostjYlgDfBS4D3h7cZwrba8dbLS9E1gN3FR0pRzC9jrby20vX7ho0bAyIiJqY1f/DLELOLm0vKBYdwhJnwSesP3wKPtNVKVLZB79zvRXSidfCrxge5/t5+j3a98kadWEffcCrwPzK5wnImLG9Io5sYd9htgCrCj6pAHOBR4sN5D0m8Be218prd4BvEfS4OWsHwO+NexkQ7tEbP9Q0tXABkm/bnsP8AngjlKbzcBmSf9D0v229xRdJV8GbrC9Y9h5IiJmUh3D+my/LGkjsEnSAWCb7WcG2yWdBVwF3C/pT4rV19reKel64CuSfgLsBP5m2Pkq9WHbflLSNfRDezXwYeC6oqAz6d/tfA14vgjrE4ANwOdtP1bpm0dEzJA6H5yxvQnYVF4n6W7gAtvfAZZMsd9mYPMo56o8Dtv2U5Kuon8389uDu6K2n6Z/xV22Hrje9rZRiomImBE2vf4IkGk6vM+fjuOO9OCM7WclXWL7e0OaXmt7+1HUFRExvbrwpGOFsCZhHRFN57wiLCKiHVp4gZ3Ajoju6Y+xbl9iJ7AjopMS2BERrWB649M3SmS6JLAjonPSJRIR0SIJ7IiItkhgR0S0QwvzOoEdER3k3HSMiGiFwSvC2qYRgT3e6/HKq6/Odhk/5aSf+ZnZLmFK8+fNG95oFrx+4MBslxBRSQI7IqIlEtgREW1gQyZ/iohoh1xhR0S0gIFerrAjIlogj6ZHRLRHXmAQEdEKzhV2RERbJLAjIlog06tGRLSIxxPYERGtkCvsiIg2cG46RkS0RgI7IqIFMr1qRERbGJwXGEREtEE7+7DnjLqDpLdVaHPqkZUTETEz+mOxh3+aZKTAlnQacFtp+U5JCydpeoukM46utIiI6eNipMiwT5NUDmxJpwPrgcuL5QXAXNu7Jml+GXCzpHfWUmVERI3s/uRPVT5NUimwi7C+FbjU9g+K1RcCd0maL+l2SV+UtBbA9kvAauBLkt41xTHXSNoqaevuXZNlfkTE9GnjFfbQm45Fl8cgrP+htOmjwHnAO+gH/5UufTvbOyWtBm6XtMb2D8vHtb0OWAfwvg98oFk/lYg4xpler55RIpIuAlYB48Ajtm+csH0u8HvAMtvnlNZ/E9hRanqV7ZcPd64qo0TmFYW8UjrRUuAF2/uA5yTdBtwk6SHbf17ady/wOjC/wnkiImZGTZM/SToJuBhYaduSNkpaant7qdmvAPcBv/hTZdiXj3K+oV0ixZXx1cCGojiATwB3lNpstn0FsHrQRtJ8YANwg+0dREQ0Sc/VPrBo0H1bfNaUjnIW8ECpd+Fe4OzyaWzfa3vLJBX8RNL1RchfWqXkSuOwbT8p6Rr6ob0a+DBwHYCkM4FPA68Bz9veI+kE+mH9eduPVTlHRMRM6T/pWLn5mO3lU2xbCOwuLe8GllaqwT4PQJLoj6z7ru2/Pdw+lR+csf2UpKuAe4BvD/6PYvtp+lfcZeuB621vq3r8iIiZVNMNxV3Au0vLC4p1o9RhSV8F3gccNrBHGodt+1ngEtvXDWl6bcI6IhrLpjfeq/QZYguworhKBjgXePAIKvol4NFhjUZ+NN329yq02T6sTUTEbKrjCtv2y5I2ApskHQC22X5miub7ywuS/gA4ETge2GL74WHny1wiEdE5dc7WZ3sTsKm8TtLdwAW2x0vtVk7Y77dHPVcCOyK6Z8S7jiMf3j5/Oo6bwI6IDmreU4xVJLAjopPcvumwE9gR0UGmtkfTZ1ICOyI6J68Ii4hokQR2REQrNG+u6yoS2BHRPTXN1jfTEtgR0U0J7IiI5jPQS5fIkTlu7lwWnnjibJfxU0xz/4Hu+ad/mu0SJnVgfHx4o4jZVrzTsW0aEdgRETMrTzpGRLRGAjsioiUS2BERLWCDh7+coHES2BHRSS28wE5gR0QX5aZjRERrJLAjItogj6ZHRLSDyYMzEREtYZwXGEREtEC6RCIi2qOFeZ3AjohuSh92REQL5J2OERFtkT7siIi2ML0WjhKZM+oOkt5Woc2pR1ZORMTMcM+VPk0yUmBLOg24rbR8p6SFkzS9RdIZR1daRMQ06XdiV/s0SOXAlnQ6sB64vFheAMy1vWuS5pcBN0t6Zy1VRkTUqKV5XS2wi7C+FbjU9g+K1RcCd0maL+l2SV+UtBbA9kvAauBLkt41xTHXSNoqaevYzp1H+z0iIkZiu9KnSYbedCy6PAZh/Q+lTR8FzgPeQT/4r3Tp29neKWk1cLukNbZ/WD6u7XXAOoAPLVvWrJ9KRBzbbHotfIFBlSvsecA48MpghaSlwAu299l+jn6/9k2SVk3Ydy/wOjC/pnojImrRxivsoYFdXBlfDWyQdFKx+hPAHaU2m21fAawetJE0H9gA3GB7R+2VR0QcocGDM8dcYAPYfhK4hn5ovwX4MPAQgKQzJW2Q9KfA87b3SDoB2Eg/rP9ummqPiDhidQW2pIsk3SfpryV9apLtcyV9TtLXJ6xfIelrku6S9IdVaq784IztpyRdBdwDfHvQX237afpX3GXrgettb6t6/IiImVPPEJCiR+FiYKVtS9ooaant7aVmvwLcB/xiaT8BnwY+YnufpM9K+mXbDxzufCM96Wj7WUmX2P7ekKbXTig4IqI5DK5+z3GRpK2l5XXFoAmAs4AHSgMu7gXOBg7mn+17AfoZfdAvAE/Z3lcs3wN8HKgvsIuTDwtrEtYR0XQjPJo+Znv5FNsWArtLy7uBpRWOOdl+kz2EeIjMJRIRnVPjbH27gHeXlhcU66rsd/Ko+408l0hEROu5tpuOW4AVeqO/41zgwQoV7ADeI2lesfwx4FvDdsoVdkR0UD0TO9l+WdJGYJOkA8A2289M0Xx/ab9xSdcDX5H0E2An8DfDzpfAjohuqmmMte1NwKbyOkl3AxfYHi+1Wzlhv83A5lHOlcCOiE4y0/dQjO3zp+O4CeyI6Bzb9Hrjwxs2TAI7IjqpaY+dV5HAjohOSmBHRLREAjsiogX6Y6zbNx92AvswXnt9//BGs+S4uXNnu4RJnTBv3vBGEQ2QwI6IaIl0iUREtEQCOyKiFdKHHRHRCnausCMiWiOBHRHRCsbVX2DQGAnsiOgkk8COiGiFdIlERLRAbjpGRLRGpdd/NU4COyI6KfNhR0S0RK6wIyLaoN+JPdtVjCyBHRGdY6b3nY7TJYEdEZ2UuUQiIlqhI6NEJL3N9ktD2pxq+0dHXlZExPTqtfDR9DmjNJZ0GnBbaflOSQsnaXqLpDOOrrSIiOnRv+fYq/RpksqBLel0YD1webG8AJhre9ckzS8Dbpb0zlqqjIiolYv3Og7/NEmlwC7C+lbgUts/KFZfCNwlab6k2yV9UdJagKLLZDXwJUnvmuKYayRtlbR1bOfOo/0eERGjGQztG/ZpkKGBXXR5DML6xdKmjwL3AW8vjnOl7bWDjbZ30g/tm4qulEPYXmd7ue3li9761qP6EhERo3LFv5qkyhX2PGAceGWwQtJS4AXb+2w/R79f+yZJqybsuxd4HZhfU70REbU4JrtEbP8QuBrYIOmkYvUngDtKbTbbvgJYPWgjaT6wAbjB9o7aK4+IOEK26fXGK32apFIftu0ngWvoh/ZbgA8DDwFIOlPSBkl/Cjxve4+kE4CN9MP676ap9oiII9bGK+zK47BtPyXpKuAe4Nsuvontp+lfcZetB663va22SiMiatS0MK5ipAdnbD8r6RLb3xvS9Frb24+iroiIaXXMBzZAhbAmYR0RzWZo2EMxVWQukYjoHBt6NQW2pIuAVfRH0z1i+8Yq2yU9AWwpmh0ArvCQy/4EdkR00ghdIoskbS0tr7O9DqAYFXcxsNK2JW2UtHTQyzBk+y7bl49ScwI7IjrIo8wTMmZ7+RTbzgIeKF0Z3wucDWyvsH2upC8AS4C/sH3PsEIS2BHRSTXddFwI7C4t7waWVtlu+2wASW8C/kLS/x12/2+k2foiIo4VNY3D3gWcXFpeUKyruh3b+4EHgHcPO1kCOyI6pz+vUy2BvQVYIUnF8rnAgyNsH/iXwNDnVtIlEhEdZOyjf+zc9suSNgKbJB0Attl+psp2SV8GXgNOBO6x/fyw8yWwI6KT6npwxvYmYFN5naS7gQtsj0+2vdjv10c9VwI7IjppOp90tH3+dBw3gR0RHdS8iZ2qSGBHROcM3unYNo0I7Ccef3zshOOP/35Nh1sEjNV0rLo1tbbUNZqm1gXNra3Out5Rx0FyhX2EbNf2jjBJWw/zVNKsamptqWs0Ta0Lmltb8+oy7uUKOyKiFZr2vsYqEtgR0Unpw26GdbNdwGE0tbbUNZqm1gXNra1RdQ2edGwbtbHoiIij8eY3H+9TTjmjUtsXX3z2sab0vx+LV9gREUP1ctMxIqId0ocdEdEG/U7s2a5iZAnsiOgck2F9ERGt0cYBFwnsiOik9GFHRLSCM0okIqIN2vrgTAI7IjopgR0R0QqG9GFHRLRDhvVFRLREukQiIlrANr3e+GyXMbIEdkR0Uq6wIyJaIoEdEdESCeyIiLZIYEdENJ9tes5Nx4iIVkiXSERESySwIyJawQnsiIi2yHzYEREtkOlVIyJaw7nCjohoiwR2RERL1NUlIukiYBUwDjxi+8Yq24ftN5kEdkR00TdsL6rY9nhJW0vL62yvA5B0EnAxsNK2JW2UtNT29sNtB350uP2mksCOiM6xfU5NhzoLeMBvXK7fC5wNbB+y/ftD9pvUnJqKjojoooXA7tLy7mLdsO3D9ptUAjsi4sjtAk4uLS8o1g3bPmy/SSWwIyKO3BZghSQVy+cCD1bYPmy/SaUPOyLiCNl+WdJGYJOkA8A2289U2X64/aaiNj7tExHRZJLuBi6w653DNYEdEdES6cOOiGiJBHZEREsksCMiWiKBHRHREgnsiIiW+P/XEhOQlMa3/wAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "STEP: 1, loss: 0.7809\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXgAAADwCAYAAAAKCX+nAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAFj1JREFUeJzt3X2wXXV97/H3h8jDhBIgCSbILaKSWkapbUl7p7RzK3eYuVCVYlFCG6MZRyN1tFanWEvBi08t0jva0qptEIuJND7Rik+jQ31ClAKhpOVWHlsVqCg5iVgELiZnf+4fe21Y7JyHtfdeZ++VdT4vZk/2Wuu3f/u7T8J3/853rfX7yTYREdE+B0w6gIiIWBhJ8BERLZUEHxHRUknwEREtlQQfEdFSSfARES2VBB8R0VJJ8BERLZUEHxHRUk+ZdAAREfur0047zVNTU5Xa3nzzzV+0fdoCh/QkSfAREUOampripptuqtT2gAMOWLnA4ewjCT4iYgSdBs/nlQQfETEkA02esDEJPiJiaMYkwUdEtI9hupMEHxHROiY1+IiI1koNPiKipZLgIyJayHZKNBERbZURfERECxmYToKPiGinjOAjIloqNfiIiDayM4KPiGijzEUTEdFi053OpEOY1aJd0UnSsZIelrSmtO+3JW2QdOUkY5uJpMskrZ50HKNqy+eI6HLl/yZh0SZ4up/9S8A7S/uWAEtsr59MSHM6kHb8xtWWzxGBDZ2Kj0lY7P+jPQDcJOkc2x/t7ZT0edu/IWkl8C7gEbpfCH8MrAVeCfwQ+JHtty50kJI2Ar8MXCRpC/Bq4GHgCOAjtj+7gO/9dOBDwD8DK4FvAr8A7AH22P4DSe8CDgd+Cvig7esknQi8GZii+8X5RmBD6XNcavtfa4rx/cB7bN8taV3xfqfT9zOSdCHwj7avL173edu/UUcMsXilBt9slwFXSfp8ad9BxZ/vBt5t+04ASQLeCpxquyPpTyWdZPvmhQzQ9hWSng9cVMT0Ttu3SToA+Jykb9j+4QK9vYC9ts8DkHQPcJLtnZI+LGk98JDtP5b0FOAzdJPru4GX2n5Y0rnAmeXPYfu+GmPcBpxD97ex36L77/qC/p8RxW9opdcdtE9PEQNKgm+wIlG/HXgb0J+of7qX3AtHAauAP+nmelYBR44l0Cessn0bPB77DuBZwPYFfM97S8/vsr2zeP4QcAzw85IuLvY9Vvx5PHBh8XM6HPjGAsZ3HXCepFV0f7M6fpafUb8lM+yLqCzTBe8HbN8i6WXAScC/lA7dL+lE27cW21PA94HzbY/71Pk03b+v+yWdUBqdPg/40zHG0f+v2cA/2f7zvv3/AbzN9qN9+3ufo76AbBdJ/J3AFmDTLD+jHwFHQ/cke+95xNDsRl9Fs5gT/HTx6LkIuJVurfknxb43A38m6aGi7fnAXwAfk7SLbunidWOK92vAnwP/APyRpEfojozfZ/u/FvB9+39Oe/qO/T3wBkkfojt6v872lXR/I9oqaYruidXX236k9zkkXW77MzXGuRX4ArAJ+Hfg4v6fkaSPApdJ+vXic9RyDiAWtyaXaNTk4CKqkvQs4GW23zbpWGLxeO7znue//+IXKrV99tFPu9n22gUO6UkW8wg+WkLSS4AXA2+YdCyx+NR5CWRx0cI6ur8dX2/7kr7jtwA3FJt76f5mPGsESfCx37P9SeCTk44jFqe6qiCSDqN7KfHpxXmlrZLW2L6r1GyX7XOr9rmYb3SKiBiZiwnH5nsAKyVtLz029XV1MnBNaUR+NXBKX5slxeXZV0o6c77YMoKPiBiSB7uKZmqeGvwKYHdpezewptzA9ikAkg4EPiHp3/pG+E+SEfwcZviGbYTENbimxpa4BtPEuAYYwc9nF0++r2Z5sW+m99wDXAM8Z64Ok+Dn1rh/TIXENbimxpa4BtOouHo3OlV5VHADcGpxxzzAGcC1c7T/FWDHXB2mRBMRMYK6Zoq0/aCkrcA2SXuBHbZvL7eR9GHgUbrzPn3K9nfm6rN1CX7ZEUf4qKPruUFx5erVPOuEE2r523vohw/V0Q0AP3XYETx19X+r7eKs//dI/82mwzn44ENZtmxFbXHt3btn/kYVHXjgwSxduqyW2B59tL6/SwBJjbwZZZHENWX7qFE6qPMySdvb6M6t9DhJVwFn2562/YpB+mtdgj/q6KO5ZMuWSYexjy997CuTDmFWd9xy6/yNJmD37u9NOoQZ7djx5UmHEPX57igvtk1ngacqsH3WsK9tXYKPiBinTDYWEdFSTZ7uJQk+ImIESfARES3k6pdATkQSfETECCa1oHYVSfAREUMyMD2pFbUrSIKPiBhBavARES2VGnxERBtVn0hsIpLgIyKGZFKiiYhorZRoIiJaKgk+IqKFevPBN1USfETEsBp+knXBV3SS9Il5jl8uadmo/URETEKNKzrVrrYEL+kWSV8tHrdL6q0GfmBx/EJJJ87w0iX9cUhaL+mcvnYH1hVrREQdelfR1LQma+3qLNF81/aZAJI2Aof3HV9SPKpYBTRzFYqIiJLpBV7wYxTjqMEfL+kP6Cbtqv47sF3SM4EPFfsOma1xsdL6JugusxcRMR5u9GRjddbgn9kr0QB/COwq9t9t+/8AP5jjtZ+V9HsAko4EDgU22f4P28+3/Xzg+7O92PZm22ttr112xBF1fJaIiHnZ1R+TUNsI3vbPSZJnLzYZOKQ4oXo48DN0VwcHeKHtB4vn/xt4O/CLkjbY3lpXjBERdVtMl0l+AnhJ374vFn9eB7wbeAy4H/gW8JlyQ0m/A+y2fSNwo6S/kvQt2zfXHGdERC2afJlk3Ql+n/5sf6D480vAl/qPSypv/ovtvyu99nXqaxAR0RSL7UanZUUNvszAObZnq8HvBToAtv+t/2Cp5LOnriAjImph01ksV9HY/p9DvOZVFdu9dPCIIiIW2CIawUdELCrOkn0REe3U4AF8EnxExLC617g3N8MnwUdEjCAJPiKilUxnepFcRRMRsZjUXaKRtB5YB0wD19u+ZIY2TwG2AA/Zfs1c/S34fPAREW1W13TBkg4DNgC/afvFwImS1szQ9ALgCirMzpsEHxExiuqzja2UtL302NTX08nANaWbO68GTik3KKZz2Q7cWSW0lGgiIkYwQIVmyvbaOY6vAHaXtncDj4/gJf0CsNr230k6rsobJsFHRAzLtZ5k3QU8p7S9nCemXQc4BzhC0l8Dh9Gdcfe1tt8/W4dJ8BERQ+ot2VeTG4A3SHpvUaY5A/iTx9/L/sPe82IEf8FcyR1amOCPPPRQzvqlX5p0GPv48se/OukQZnX//f8+6RBmtHPnvZMOIWJedSV42w9K2gpsk7QX2GH79lmaT9OdqHFOrUvwERHjVOdlkra3AdvK+yRdBZxte7rU7l7g3Pn6S4KPiBiWDQs82Zjts4Z9bRJ8RMQIMlVBREQLGehkuuCIiBbKbJIREe2VBT8iIlqp2jwzk5IEHxExgiT4iIgWyopOEREt5ukk+IiIVsoIPiKijSou5jEpSfARESNIgo+IaKGapwuuXRJ8RMSwDK5vwY/aJcFHRAyt2TX4oRfdlvSJeY5/UNKyUfuJiGiy6mtuj9+8I3hJH7d9dvH8FcAPbX8aOLDY92dAbwmlFcD5tj9T9H1AqZ83Ay+guxLJHcDv2d7T66fU7uXAbwMqHv8FvNX2bSN8zoiIBdHkEXyVEs1xkt5SPD8JuLJ80PZ5veeSzp2pT0k/Czzb9q8X2xuBVwAf7Gt3CvB84EW29xb7ngZ8Eji50ieKiBgTu9mTjVUp0dxj+2LbFwOfK+3/NUlflfQzpX3/A7huhj4eBFZLOroo25wE3D/L+3Xonpzucd/2PiRtkrRd0vadO3fO93kiImrj4lr4+R6TUGUEv2SW59fZPrO3Iem5dMs35Qz72aLEc6mkdwHvAQ4BPma7/GUBgO2vSPrp4nXlEs2r5grQ9mZgM8DatWub+3UaES1jOp39+yqaH0v6Gt1R9DTwhv4GklYAFwOv7Dv0wmKl8HXAKuBG4FhglaTzgJtmeL/7gb/qfQFIek/q7xHRSPv7ZGO2N0j6pO2X9B36MoCkXwYuAH7f9gOz9PGxou1xwO8Dl9Et97yA7oi+7PC+fcfO+ykiIialwTX4qtfB79PO9qXF0/8FnGP7kapvWrQ9BUDSp4o/Xw2sBw7tbqpXllkt6avF89fZ/r9V3yciYiF172SddBSzq5zgS0m2x3QT+ztmec1euidMyzoz7NsDYPsyuiP7iIj9xn5dogGw/cJBO7a9z4lR2/cAb+rb99JB+46IaASbTqYqiIhop/1+BB8REfvKbJIREW3V8LOsSfAREUNr9mySSfARESNwc8+xJsFHRAzN1DpVgaT1wDq6swZcb/uSvuPvpzsD76HAnbYvmqu/JPiIiCHVeZJV0mHABuB025a0VdIa23c9/n72a0vtPyzp2bbvmK3PoRf8iIiIgWaTXNmb9bZ4bOrr6mTgGj/xjXE1xR3//SQdCRwF/GCu2DKCj4gYmgeZD37K9to5jq8Adpe2dwNryg0kHQ+8je6XwRttPzjXG2YEHxExLNc6H/wu4MjS9vJi3xNvZ99tez3dxL9e0uq5OkyCj4gYRX2Lst4AnFqshQFwBnDtzG/pvXTX5zhorg5ToomIGJKBTk3TBRdrZ2wFtknaC+ywfXvvuKRfpDuX14+BZcBVxfxes2pdgr/v/gc47+3vm3QY+3jGc4+bdAiz2nnvr006hBndfcctkw5hRlNT9006hGiKmtdktb0N2FbeJ+kq4Gzb/wy8bJD+WpfgIyLGZ+HvZLV91rCvTYKPiBhBpiqIiGipJPiIiBaywVnwIyKinRo8gE+Cj4gYXqYLjohorST4iIg2chJ8REQrmXpvdKpbEnxExNCMa1zwo25J8BERw0qJJiKivRqc35PgIyJGkRp8REQL1bkm60JIgo+IGFZq8BERbWU6Db6KZmJL9kl6aoU2c643GBExae640mMSJpLgJT0N+NvS9kclrZih6QckHTeuuCIiBtItwte1Jmvtxp7gJR0DXA6cW2wvB5bY3jVD89cA75P0jDGGGBFRScPz+3gTfJHcLwNebfveYvc5wMclLZV0haT3SroIwPYDwEbgLyU9c45+N0naLmn7Iw//eGE/REREie1Kj0kY20nWogTTS+7/WTr0IuBM4Ol0v3De5NJPw/ZOSRuBKyRtsv29/r5tbwY2A6w+5tjmntKOiHax6TR4wY9xjuAPBqaBH/V2SFoD3GP7Mdt30q3LXyppXd9rHwZ+AiwdV7AREVU0eQQ/tgRfjLzPB7ZIOqzY/XLgI6U2X7H9emBjr42kpcAW4GLbd48r3oiI+fRudFr0CR7A9q3ABXST/OHArwLXAUg6QdIWSX8DfMf2Q5IOBbbSTe43jjPWiIgqmpzgx36jk+1vSXoL8Cng6716u+3b6I7oyy4H3mF7x5jDjIioYIKXyFQwkTtZbd8h6ZW2vz1P0wtt3zWWoCIiBmVwc8+xTm6qggrJnST3iGi6Jk9VkLloIiKGlNkkIyLaKrNJRkS01eQmEqsiCT4iYhQ1juAlrQfW0b0p9Hrbl/Qd/wDQAZYDn7P9kX17eUISfETECEw9Cb64uXMDcLptS9oqaU35YhPbv1u0FXAtpRtFZ5IEHxExJNt0OtNVm6+UtL20vbmYR6vnZOCa0lxcVwOnADNdTXgwsHu+N0yCj4gYwQAnWadsr53j+AqenLR3A2tmaftO4JJZjj0uCT4iYgQ1XkWzC3hOaXt5se9JJL0RuMX2N+brcGJL9kVEtEGNc9HcAJxa1NcBzqBbZ3+cpNcCD9u+skqHGcFHRAypm7zruZPV9oOStgLbJO0Fdti+vXdc0snAW4DPS/rrYveFtnfO1mfrEvxjjzzG3bc0b1bhH3z3B5MOYVYrjl4+6RBmdMzxL5h0CDO6Zcc/TjqEaJC6Eny3L28DtpX3SboKONv2N4FjB+mvdQk+ImKcFvpOVttnDfvaJPiIiBFkqoKIiFaqrwa/EJLgIyKG5Ew2FhHRXknwERGtZJwFPyIi2skkwUdEtFJKNBERLZSTrBERrVV5npmJSIKPiBjBAPPBj10SfETECDKCj4hoo24RftJRzCoJPiJiSKa+NVkXQhJ8RMQIMhdNREQr5SqaGUl6qu0H5mmz2vb3xxVTRMSgOg2eqmAia7JKehrwt6Xtj0paMUPTD0g6blxxRUQMonuOtVPpMQljT/CSjgEuB84ttpcDS2zvs3o48BrgfZKeMcYQIyIqqrbg9qTKOGNN8EVyvwx4te17i93nAB+XtFTSFZLeK+kigKKEsxH4S0nPnKPfTZK2S9r+2GOPLuyHiIgo610qOd9jAsZWgy9KML3k/p+lQy8CzgSeTvcL500ufd3Z3ilpI3CFpE22v9fft+3NwGaAI49c1dwzHhHROk2+THKcI/iDgWngR70dktYA99h+zPaddOvyl0pa1/fah4GfAEvHFWxERBUp0QDFyPt8YIukw4rdLwc+UmrzFduvBzb22khaCmwBLrZ997jijYiYj206nelKj0kYaw3e9q3ABXST/OHArwLXAUg6QdIWSX8DfMf2Q5IOBbbSTe43jjPWiIgqmjyCH/t18La/JektwKeAr/fq7bZvozuiL7sceIftHWMOMyKiktzo1Mf2HZJeafvb8zS90PZdYwkqImIISfAzqJDcSXKPiGYzZC6aiIj2saGTBB8R0U4p0UREtJIzXXBERFs1eQQ/kdkkIyLaos7r4CWtl/RpSf8g6c0zHF8i6V2SvlClvyT4iIghdecRq5zgV/YmRSwem8p9FXfvbwB+0/aLgROL6VzKXgh8morVl5RoIiKGZuzK0xBM2V47x/GTgWtKky1eDZwCPH65uO2rASRVesMk+IiIEdRYg18B7C5t7wb6R/ADSYKPiBhBjQl+F/Cc0vbyYt/QUoOPiBharSs63QCcqifqL2cA144SXUbwERFD6q3JWk9fflDSVmCbpL3ADtu3z9J8T5U+1eRrOIchaSfw3Zq6WwlM1dRXnRLX4JoaW+IaTN1xPd32UcO++OCDl/qYY6qVyb/97X+9eZ6TrDOSdBVwtgc4m9vTuhH8KH9Z/SRtH+YvZKElrsE1NbbENZjmxWXcWdg7WW2fNexrW5fgIyLGqclrsibBR0SMIHPR7L82TzqAWSSuwTU1tsQ1mEbF1buTtalad5I1ImJcDjroEK9adVyltvfdd8dQJ1lHkRF8RMQIOgt8knUUSfARESNIDT4ioo26RfhJRzGrJPiIiCGZXCYZEdFaTb5QJQk+ImIEqcFHRLSScxVNREQbNf1GpyT4iIgRJMFHRLSSITX4iIh2ymWSEREtlRJNREQL2abTGXihpbFJgo+IGEFG8BERLZUEHxHRUknwERFtlQQfEdE+tuk4J1kjIlopJZqIiJZKgo+IaCUnwUdEtFXmg4+IaKFMFxwR0VrOCD4ioq2S4CMiWqrOEo2k9cA6YBq43vYlgxzvlwQfETG8L9peWbHtIZK2l7Y3297c25B0GLABON22JW2VtMb2XVWOzyQJPiJiSLZPq7G7k4Fr/MSvBFcDpwB3VTy+jwNqDC4iIoa3Athd2t5d7Kt6fB9J8BERzbALOLK0vbzYV/X4PpLgIyKa4QbgVEkqts8Arh3g+D5Sg4+IaADbD0raCmyTtBfYYfv2qsdnoibfhRURsdhJugo42x58XuIk+IiIlkoNPiKipZLgIyJaKgk+IqKlkuAjIloqCT4ioqX+PwAsOGMJPi96AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "STEP: 2, loss: 3.3569\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAW8AAADwCAYAAADPe+U2AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAExtJREFUeJzt3X+QXlV9x/HPZwNROzIKhPLDCpnRWDpK0bLVDjqWdHAKaiOUFrQx044/AjhUawvU8kOh4i/sUKr4owsoEnCtChTRjEjboTSISKKoM4pAbRGqQjYxlAIK2f30j+du8rB5nt3ngbvPfc7u+5W5s3vPuXvud5fwzdlzzz3HSQQAKMtI0wEAAPpH8gaAApG8AaBAJG8AKBDJGwAKRPIGgAKRvAGgQCRvACgQyRsACrRb0wEAwDA66qijMjEx0dO1mzZtuj7JUfMc0hOQvAGgg4mJCd122209XTsyMrJsnsPZBckbALqYGuK1n0jeANBBJA3zwn0kbwDoKIpI3gBQlkiTUyRvAChKxJg3ABSJMW8AKBDJGwAKk4RhEwAoET1vAChMJE2SvAGgPPS8AaBAjHkDQGkSet4AUBrWNgGAQk1OTTUdQlcLcicd2wfaftj2irayN9heY/vKJmPrxPbFtvdrOo6naqF8H0BLev7ThAWZvNX6vv5V0nltZUskLUmyupmQZrW7FsZvQQvl+wCUSFM9Hk1YyP+jPSDpNtuvT/K56ULb65O82vYySe+T9Ihayf5MSaOS3iTp55IeTPLu+Q7S9p9Jeqmkc2xfLumtkh6W9GxJVyT58jze+yBJn5L0LUnLJH1d0kskPS7p8SSn2n6fpGdJeqakS5JssH2IpNMlTaj1j+I7Ja1p+z4+kuS7NcX4cUkXJLnb9gnV/Y7WjJ+R7bMl/UuSW6qvW5/k1XXEgMWLMe/mXCzpKtvr28qWVh8/JOlDSe6UJNuW9G5JRyaZsv0B24cl2TSfASa5zPYRks6pYjovyQ9sj0j6iu2bk/x8nm5vSduTnCZJtn8s6bAkm21/xvZqSQ8lOdP2bpKuUytxfkjSHyd52PZJko5p/z6S3FdjjOOSXq/Wb1F/qNbf2bNm/oxU/WbV9nVLd2kJ6BPJuyFVEv5bSedKmpmEnzuduCv7SNpX0vtbeVz7StpzIIHutG+SH0g7Yr9d0vMkbZzHe97b9vldSTZXnz8k6TmSXmz7g1XZL6uPz5d0dvVzepakm+cxvg2STrO9r1q/ET2/y89opiUdyoCesSRsw5J82/YbJR0m6TttVT+1fUiS71XnE5J+JumMJIN+xDyp1n+Ln9r+jbZe5aGSPjDAOGb+TY2kbyS5cEb5jySdm+TRGeXT30d9ASWpEvR5ki6XtLbLz+hBSftLrQfW058DT1oy1LNNFmrynqyOaedI+p5aY7uPVWWnS/qw7Yeqa8+Q9A+S/sn2FrWGE04ZULz/LulCSddI+hvbj6jVo/1Ykv+dx/vO/Dk9PqPuaknvsP0ptXrdG5JcqdZvMutsT6j1kPLPkzwy/X3YvjTJdTXGuU7SVyWtlfSfkj4482dk+3OSLrb9u9X3UcuYOxa3YR428TAHB0iS7edJemOSc5uOBYvHiw49NFdf/9Werv31/Q/YlGR0nkN6goXa88YCYfuPJB0r6R1Nx4LFZ4i3sCR5Y7gl+aKkLzYdBxanYR6ZIHkDQBckbwAoTIZ8tslCfT1+TrbXNh1DJ8TVv2GNjbj6M4xxpVoWdq6jCYs2eas17WwYEVf/hjU24urPUMU1/ZJOL0cTGDYBgC6aWjGwF0Ul72XLlmX58uW1tHXggQdqdHS0lv8ymzbVu/yJ7aH8GzOscUnDGxtx9afmuCaS7PNUGmCqYE2WL1+ub952W9Nh7GL33XZvOoSupqYm574IWJjueSpfnERTQ/zAsqjkDQCDxMJUAFAg5nkDQIFI3gBQmDQ4DbAXJG8A6IKpggBQmEiaHOK5giRvAOiizjHvak/YE9Ta6OSWJOfPqH+nWjt+PabWNn4nV5ucdLSYX48HgFnV9Xq87T0krZH0uiTHSjrE9oq2+mertfn5G5O8SdL3Jb1qtjZJ3gDQSY+LUlW982W2N7YdM9dpOVzSDdnZlb9W0sq2+gfV2sN2f9vPkHSQWptvd8WwCQB0EPU1bDIxxzZoe0va2na+VdKOnne10fanJb1N0hZJNyfZMtsN6XkDQBc1riq4RdKebed7VWWSJNu/Kem1Sc5OcqGkR22/ZbYGSd4A0EWNyftWSUfadnW+StJNbfX7S3Lb+aOSls/WIMMmANDB9HretbSVbLO9TtK47e2Sbk9yR9slX5P0StuXS/qlpF+R9PbZ2iR5A0AnNe+Sk2Rc0nh7me2rJB2fZFLSmf20N/BhE9sH2L6xOla2lX9h0LEAwGzmeyedJMdVibtvTfS8T9TOKTAvt70xyUOShndRbACLTp+zTQauieR9iVpvDy2RdIWkixqIAQDmxO7xbZLcq9YOF6dKGkuyrap6RTWU8oL2622vnZ74vnnz5kGHC2DRSs9/mtDEmPfBkj4v6W5Jr7G9qqrakOSIJHe2X59kLMloktF99nlK29EBQM+S3o8mDHTYpOpVv1nSKUnut71U0jFV9fpBxgIAc2E970rVqz7N9jW2d7xtZPtkSdcNMhYAmAsPLNvY3k3SkiRHtJWNSLpa0gWDjgcAOqnzJZ350MRsk0lJh9q+sa1sRNIdnS8HgAYkmhri2SYDT97VkogHDfq+ANA3et4AUJ6wDRoAlGeIO94kbwDopDWHe3izN8kbALogeQNAcaKpSWabAEBRGDYBgEKRvAGgRCRvACjPEOdukjcAdBQeWAJAcdgGrWYjdtMh7GKYF68B8OSRvAGgQCRvAChNIrEwFQCUh543ABQmkqboeQNAYXg9HgDKxGYMAFCc0PMGgBKRvAGgMCwJCwCFyiTJGwCKQ88bAEoTHlgCQJFI3gBQGJaEBYASRUqNmzHYXi3pBEmTkm5Jcv6M+udJOrM6nZT0niQ/6dYeyRsAOqpvzNv2HpLWSDo6SWyvs70iyV1VvSV9UNKJSbb20uZILZH1wfYBtm+sjpVt5V8YdCwAMJvWXO+5D0nLbG9sO9bOaOpwSTdk578G10pa2Vb/25LulfR+21fafstcsTXR8z5R0obq85fb3pjkIUm7NxALAHTVR897IsnoLPV7S2rvUW+VtKLtfLmkF0laleQXtj9h+4dJ/qNbg00k70skLamOKyRd1EAMADCrpNaFqbZIemHb+V5V2bRH1OqZ/6I6/5KkwyR1Td4DHzZJcq+keySdKmksybaq6hXVUMoL2q+3vXb6V5HNmzcPOlwAi1iqud5zHT24VdKR1di2JK2SdFNb/SZJL207f5mk787W4MB73rYPlvReSd+Q9BrbW5J8SdKGJMfMvD7JmKQxSRodHR3eeTsAFpjUtrl4km2210kat71d0u1J7mir/6ntr9kel/SwpP9O8m+ztTnQ5F31qt8s6ZQk99teKmk6Ya8fZCwAMKuaF6ZKMi5pvL3M9lWSjk8ymeRiSRf32t5Ak3eSOyWdZvsa23tOl9s+WdJ1g4wFAOY0z5sxJDnuyX5tE8Mmu0lakuSItrIRSVdLumDQ8QBAJ603LJuOorsmZptMSjrU9o1tZSOS7uh8OQA0g9fj21ST1A8a9H0BoC+Jpmp8Pb5uvB4PAF3Q8waAwrCqIACUaMifWJK8AaAjdtIBgCJleJ9XkrwBoKOottfj5wPJGwA64IElABSK5A0AxUmd63nXjuQNAJ3UvKpg3UjeANANyRsAyhJJUwyb1OPH992vU07/u6bD2MXqPz2j6RC6uv4rn2k6hI62bXug6RA62r79saZDwLCodw/L2hWVvAFgcHjDEgCKRPIGgAKRvAGgMIkUNmMAgPIMcceb5A0AnfHAEgCKRPIGgNLwejwAlCfiJR0AKFAUNmMAgMIwbAIAZRri3E3yBoBuGPMGgMKwhyUAlIgxbwAoUTQ1xLNNRgZ9Q9sH2L6xOla2lX9h0LEAwGwylZ6OJjTR8z5R0obq85fb3pjkIUm7NxALAHTWGvRuOoqumkjel0haUh1XSLqogRgAYFZDnrsHP2yS5F5J90g6VdJYkm1V1SuqoZQXtF9ve63tjbY3Pvro/w06XACLWJKejl7YXm37S7avsX16l2t2s/1Z2/84V3tNjHkfLOnzku6W9Brbq6qqDUmOSHJn+/VJxpKMJhl9xjOeOehwASxWiaYmp3o65mJ7D0lrJL0uybGSDrG9osOlZ0m6TK2RiVkNdNik6lW/WdIpSe63vVTSMVX1+kHGAgBz6WOq4DLbG9vOx5KMtZ0fLumG7GzwWkkrJd01fYHtP5G0UdITOrDdDDR5V73q06pfG/acLrd9sqTrBhkLAMymz5d0JpKMzlK/t6StbedbJe3oedt+iaT9knzW9vJebjjwB5a2d5O0JMkRbWUjkq6WdMGg4wGAbmp8SWeLpBe2ne9VlU17vaRn2/6kpD0k/ZbttyX5eLcGm5htMinpUNs3tpWNSLqjgVgAoIvUOd3kVknvsP331dDJKknv33Gn5K+nP6963mfNlrilBpJ3FfhBg74vAPQlUmp6wTLJNtvrJI3b3i7p9iTdOqyTkrbP1SavxwNAF3W+Hp9kXNJ4e5ntqyQdn2Sy7bp7JZ00V3skbwDoYBCrCiY57sl+LckbADphVUEAKFFzi071guQNAN3Q8waA8kQkbwAoShJNTU3OfWFDSN4A0AUPLAGgQCRvACgQyRsACtPaaGF4NyAuKnlPPPA/GrvwrKbD2MVzn3tw0yF09cpXHt90CB0t+7VlTYfQ0dhHzmg6BAwRkjcAFIhhEwAoEMkbAIrDmDcAFCcsTAUAZSJ5A0BxotS4GUPdSN4A0EVE8gaA4jBsAgCF4YElABQpJG8AKBHreQNAgeh5A0BpWoPeTUfRFckbADqI2MMSAIrE2iYAUJwFNtvE9q8meeCp3tj2fkl+9lTbAYD5MjXEr8eP9HOx7QMkfbrfm9heY/sNM4p/x/Z7+m0LAAah9bxyqqejCT0nb9vPkXSppJOexH2WVMcOSf5Z0oTtc59EewAwz1LtYzn30YSekneVuC+W9NYk99q+1Pb+bfXrq4/n2P6o7Ytsf872qhntLLE9ZvvFkpTkY5J+Zvu9s9x7re2NtjcO8/gTgAVoerrgXEcD5kzetvfWzsR9X1U8sye9tO3zbyU5RdIaSX/R3pSkCyRdmuT26cIkn5B0n+2OOwsnGUsymmTUdi/fEwDUIj3+aUIvPe+nSZqU9GCnSttLZhTdJUlJHpeesJ7i2yXtJ+mbHZq5T9JwbicOYNEqetgkyU8knSHpctt7VMUPSpoeNnmZ1NM/PZdJWle1tYPtP5B0rKS/6i1kAJh/STQ1NdnT0YSexryTfE/SWWol8GepNYzyHtsflfT7kjZXl05Wx7TH28q3JvmypMdsv06SbB8j6bWS1iYZ3hVgACxKw9zz7nmed5Lv236XpMuSHKtW0p15zXtnnB9dfVzXVvZhSbL9e5JeJemk8CQSwBAa5tTU10s6SX5o+y9ruvePJZ1C4gYwrOpMT7ZXSzpBrZGIW5KcP6P+E2o9J9xL0leSXDFbe32/YZnkv/r9mi7t3F1HOwAwPyL1/gLOMtsb287HkoxNn1TPC9dIOjpJbK+zvSLJXTvulpxcXWtJN0mqN3kDwGKQSFO9J++JJKOz1B8u6Ya2kYZrJa1UNTtvhqdJ2jrXDUneANBFjcMme+uJCXmrpBVdrj1P0vld6nYgeQNAR6lz3ZItkl7Ydr5XVfYEtt8p6dtJbp6rwb4WpgKAxaTGqYK3SjrSO18TX6XWuPYOtt8m6eEkV/bSID1vAOiirmGTJNtsr5M0bnu7pNuT3DFdb/twSe+StN72J6vis5Ns7tCcJJI3AHTUWnOqvqmCScYljbeX2b5K0vFJvi7pwH7aI3kDQEfRfL/4neS4J/u1JG8A6GKY3yEkeQNAFyRvACjOAtuAGAAWg+k9LIdVUck7ycTjj//ynpqaWyZpoo6GfvSj79TRzLTa4pJqja3WuGo2rLERV3/qjuugp9oAPe+aJNmnrraqPTFnW4ugEcTVv2GNjbj6M3xxRZmi5w0AxWlqf8pekLwBoAvGvIfT2NyXNIK4+jessRFXf4YqrrrfsKybhzk4AGjK0qVPz777Lu/p2vvu++GmQY/XL+aeNwDMaooHlgBQHsa8AaA0rUHvpqPoiuQNAB1ETBUEgCIN84QOkjcAdMGYNwAUJ8w2AYDSDPtLOiRvAOiC5A0AxYnEmDcAlIepggBQIIZNAKAwSTQ1Ndl0GF2RvAGgC3reAFAgkjcAFIjkDQAlInkDQFmSaCo8sASA4jBsAgAFInkDQHFC8gaAErGeNwAUhiVhAaBIoecNACUieQNAgeocNrG9WtIJkiYl3ZLk/H7qZyJ5A0Bn1ydZ1uO1T7e9se18LMnY9IntPSStkXR0ktheZ3tFkrt6qe+E5A0AHSQ5qsbmDpd0Q3Z25a+VtFLSXT3W72KkxuAAAJ3tLWlr2/nWqqzX+l2QvAFg/m2RtGfb+V5VWa/1uyB5A8D8u1XSkbZdna+SdFMf9btgzBsA5lmSbbbXSRq3vV3S7Unu6LW+Ew/zG0QAsJDZvkrS8Un/a8+SvAGgQIx5A0CBSN4AUCCSNwAUiOQNAAUieQNAgf4f1CA9Bjl2vwsAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "STEP: 3, loss: 0.1916\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXgAAADwCAYAAAAKCX+nAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAFKdJREFUeJzt3XuwpHV95/H3d8YBSheRGchwKS6rjiYVDUmYaC1JJbDF1kI0LFkVMDgJ5cbxUibGlBhXwWiiBsmWuq6J2Rk0LoOON4wEpTSsAckYFhlWorsK4moANxjm4hAWjDLnfPaPfg40PefSp0+f7p7nvF9U1/Rz+/W3zwyf/p3f8/TvqSRIktpn1bgLkCQtDwNeklrKgJekljLgJamlDHhJaikDXpJayoCXpJYy4CWppQx4SWqpJ4y7AEk6WJ111lnZvXt3X/vedtttn09y1jKX9DgGvCQNaPfu3dx666197btq1aqjlrmcAxjwkrQE0xM8n5cBL0kDCjDJEzYa8JI0sBAMeElqn8DUtAEvSa0THIOXpNZyDF6SWsqAl6QWSuIQjSS1lT14SWqhAFMGvCS1kz14SWopx+AlqY0Se/CS1EbORSNJLTY1PT3uEua0Yu/oVFUnVtVDVbWha92Lq2pTVX14nLXNpqq2VtUx465jqdryPqSO9P3fOKzYgKfz3r8AvK1r3WpgdZILx1PSvNbQjt+42vI+JBKY7vMxDiv9f7T7gVur6oIkH51ZWVXXJfnlqjoKeDvwMJ0PhDcBG4GXAt8HHkjy5uUusqouAp4DvKWqrgReBjwEPAW4KslnlvG1TwI+CPxP4Cjgb4GfAR4BHknyuqp6O3AE8C+AK5LsqKpnA68HdtP54HwtsKnrfbw3yVeHVOOfAu9K8q2qOr95vbPp+RlV1aXAf09yc3PcdUl+eRg1aOVyDH6ybQWurqrrutYd0vz5TuCdSb4JUFUFvBk4M8l0Vf1RVZ2a5LblLDDJh6rqdOAtTU1vS/KNqloFfLaqvpTk+8v08gXsT3IxQFXdA5yaZFdV/bequhB4MMmbquoJwLV0wvWdwIuSPFRVrwDO7X4fSb47xBq3AxfQ+W3s39P5d31J78+I5je0ruMOOaAlaZEM+AnWBPUfAG8FeoP6hJlwbxwNrAfe0cl61gNHjqTQx6xP8g14tPbbgacBO5fxNe/ten5Xkl3N8weB44GfrqrLmnU/bP58OnBp83M6AvjSMta3A7i4qtbT+c3q6XP8jHqtnmWd1DenCz4IJPlKVb0EOBX4u65N91XVs5N8rVneDXwPeGOSUZ86n6Lz93VfVf1EV+/0FOCPRlhH77/mAP8jyXt61n8beGuSH/Ssn3kfwysoSRPibwOuBDbP8TN6ADgWOifZZ55LA0sm+iqalRzwU81jxluAr9EZa/5Rs+71wB9X1YPNvm8E/jPwsaraQ2fo4tUjqveLwHuAvwD+Y1U9TKdn/CdJ/mkZX7f35/RIz7ZPAa+pqg/S6b3vSPJhOr8Rbauq3XROrP5Wkodn3kdVfSDJtUOscxvwOWAz8H+Ay3p/RlX1UWBrVf1S8z6Gcg5AK9skD9HUJBcn9auqnga8JMlbx12LVo5nnXJKPvX5z/W17zOPPe62JBuXuaTHWck9eLVEVb0Q+FXgNeOuRSvPBN+S1YDXwS/JJ4FPjrsOrUyTPApiwEvSEhjwktRCmfCraFbyVAULqqrN465hNta1eJNam3UtziTWlWbK4IUe42DAz2/i/jE1rGvxJrU261qciapr5otO/TzGwSEaSVqCcc0U2Y/WBfyRa9fm+BNOGEpbxx1/PM865ZSh/O3d+fU7htEMAKtWrWbNmkOH9q9q//4fLbxTn6pqYv+1T2pt1rU4Q65rd5Kjl9KAl0mO0PEnnMDVn+vviwej9Is//a/GXcKc7r//nnGXMIcJ/j9HbXH3Ug5OwvQEn2RtXcBL0ig52ZgktZTXwUtSSxnwktRCGeMlkP0w4CVpCbxMUpJaKMDUBF8nacBL0hI4Bi9JLeUYvCS10ZAnEquqC4Hz6dwO8+Ykl/dsfy2de0f/iM5N41/Z3ApzVk42JkkDCouaTfKoqtrZ9XjcxGlVdTiwCfh3SX4VeHZVbeja/hTgzCQvSfJS4OvAv5mvPnvwkrQEixii2b3APVlPA67PY78SXAOcAdzVLD8A3FdVxwL7gJOAK+Z7QQNekpZgiGPw64C9Xct7gUd78ElSVX8OvArYA3wpyZ75GnSIRpIGNOT54PcAR3Ytr23WAVBVPwU8P8mlSd4D/KCqfnO+Bg14SRpUn+PvfZ6IvQU4s6qqWT4HuKlr+7FAdS3/ADh5vgaXPeCr6hMLbP9AVT15qe1I0jgMqwefZB+wDdheVVcBX03SfSOJvwKmqurKqtoKvAR493xtDm0Mvqq+QuckAMAxdC7fuQFY02y/FPh0kq/1HLqang+a5lKhqSQf7Vq9Zli1StIwzFxFM7T2ku3A9u51VXU1cF6SKeBNi2lvmCdZ705yblPQRcARPdtXN49+rAd6PwgkaeJMLfMNP5K8YNBjR3EVzdOr6nV0QrtfzwV2VtVTgQ826w6ba+fmetLN0LnNniSNRiZ6srFhjsE/tapurKobgd/jsbO/30ryn4B/nOfYz1TVbwNU1ZHAk4DNSb6d5PQkpwPfm+vgJFuSbEyy8ch164bxXiRpQUn/j3EYWg8+yU9VVWXuAakAhzUnVI8AnkHnLDB0Lv3Z1zz/feAPgJ+tqk1Jtg2rRkkatpU0F80ngBf2rPt88+cO4J3AD4H76HzN9truHavq14C9Sb4MfLmq3ldVX09y25DrlKShWEmzSR7QXpL3N39+AfhC7/bHLvkE4O+SfKTr2FdXzw6SNClmvug0qYYd8E9uxuC7BbggyVxj8PuBaYAk/7t3Y9eQzyPDKlKShiJhepmvolmKoQZ8kn89wDHzftW2a78XLb4iSVpmK6gHL0krSrxlnyS10wR34A14SRpU5xr3yU14A16SlsCAl6RWCtNTK+QqGklaSRyikaQWM+Alqa0MeElqpwnOdwNekgYWT7JKUisN+5Z9w9a6gD9szRqeeeyx4y7jAPfff/e4S5C0DAx4SWopA16S2igBJxuTpHayBy9JLRRg2h68JLWQUxVIUnt5ww9JaqXYg5ektjLgJamFnC5YklosUwa8JLWSPXhJaqN4klWSWsuAl6QWcrpgSWqrQLzhhyS10XDH4KvqQuB8YAq4OcnlPdufBrypWZwCfj/JP8zV3sQFfFUdB3ykWXxrkhua9Z9I8qLxVSZJBxpWvlfV4cAm4OwkqaptVbUhyV3N9gIuA16eZG8/bU5cwAMvB3Y0z3++qnYmeRBYM8aaJGlWi+jBH1VVO7uWtyTZ0rV8GnB9HmvwGuAM4K5m+eeAe4F3NB8GNyS5Yr4XnMSAvwJY3TyuAt433nIkaXbJoiYb251k4zzb1wHdPfO9wIau5ZOBZwHnJPnnqnp/Vd2Z5G/manBVv5WNSpJ7gbuB19H5hNvXbPqFqrqxqp7Re0xVba6qnVW1c9euXaMsV9IKl+Za+IUefdgDHNm1vLZZN+NhOj38f26W/xI4db4GJy7gq+rHgY8D3wKeV1XnNJt2JDk9yTd7j0myJcnGJBuPPvroUZYraUUL09PTfT36cAtwZjPWDnAOcFPX9tuA53QtPxf46nwNTuIQzX8AXp3kH6vqEODcZv11Y6xJkg40xMnGkuyrqm3A9qraD9ye5I6u7fdV1V9V1XbgIeDvk/z1fG1OXMAnubiq/qKqHv1VpapeCVw7xrIkaXZDvOFHku3A9u51VXU1cF6SqSRbga39tjdxAV9VTwBWJzm9a90q4FPAu8ZVlyT16nyTdZlfI3nBoMdOXMDTuXj/lKq6sWvdKuCO2XeXpPFxqoJFaK4BPWncdUjSghKmnapAktrJHrwktZCzSUpSW43iLOsSGPCSNDDv6CRJrZXJPcdqwEvSwEK/0xCMhQEvSQPyJKsktZgBL0mtlMXMBz9yBrwkDWqIs0kuBwNekpbCgJek9gkw7RCNJLXQ4u7JOnIGvCQNzG+ySlJrGfCS1FIGvCS1UALxhh+S1E4T3IE34CVpcJ5klaTWMuAlqY2cqkCS2in4RSdJaqkQb/ghSS3kEI0ktdcE57sBL0lL4Ri8JLWQ92SVpLaa8DH4VYMeWFWfWGD7FVX15KW2I0mTK0xPT/f1GIcFe/BV9fEk5zXPfwP4fpK/BNY06/4Y+Llm93XAG5Nc27S9qqud1wPPA6aAO4HfTvLITDtd+/068GKgmsc/AW9O8o0lvE9JWhYH+xj8yVX1hub5qcCHuzcmuXjmeVW9YrY2q+rHgWcm+aVm+SLgN4ArevY7Azgd+JUk+5t1xwGfBE7r6x1J0qh0BuHHXcWc+hmiuSfJZUkuAz7btf4XqurGqnpG17pfBHbM0sY+4JiqOrYZtjkVuG+O15um82ObkZ5lSZoIM/nez2Mc+unBr57j+Y4k584sVNWz6Azf7Ora5zPNEM97q+rtwLuAw4CPJen+sAAgyQ1VdUJzXPcQzW/OV2BVbQY2A5x44ol9vCVJGo5JPsnaT8D/v6r6Ip0PqyngNb07VNU64DLgpT2bnp9kX1WdD6wHvgycCKyvqouBW2d5vfuA9818AFTVuxYaf0+yBdgCsHHjxsn9aUtql4TpId7wo6ouBM6nk7U3J7l8ln2eAFwJPJjk5fO1t2DAJ9lUVZ9M8sKeTX/dvNhzgEuA30ly/xxtfKzZ92Tgd4CtdIZ7nkenR9/tiJ51dsklTaxF9OCPqqqdXctbms4pAFV1OLAJODtJqmpbVW1IcldPO5cAHwLOW+gF+70O/oD9kry3efpvgQuSPNxnWzT7ngFQVZ9u/nwZcCHwpM5izQzLHFNVNzbPX53kf/X7OpK0nBb5RafdSTbOs/004Po81uA1dHLy0YCvql8DdgLf7OcF+w74rpCdETrB/odzHLOfzgnTbtOzrHsEIMlWOj17STpoDHEMfh2wt2t5L7BhZqGqfgY4JslHmtGQBfUV8Eme33+Njx5zwInRJPcAv9uz7kWLbVuSJsNQL5HZA/xk1/LaZt2MC4CnVNWfAYcDP1tVr0ryp3M16FQFkjSoQIZ3jvUW4DVV9e5mmOYc4B2PvlTyezPPmx78JfOFOxjwkrQkw5qGoLnicBuwvar2A7cnuWOO3afoDIPPy4CXpAENezbJJNuB7d3rqupq4LwkU1373Qu8YqH2DHhJGtQIZpNM8oJBjzXgJWlgOegnG5MkzeUgn6pAkjSHTPBciAa8JA0oCdPTUwvvOCYGvCQtwcE+m6QkaQ4GvCS1lAEvSS2UhAxxroJha13A33nntzn99BePu4wDPPe5i56vbWQeeGD3uEuY1fofO2ncJczqizd9bNwlaIIY8JLUUg7RSFJLGfCS1EqOwUtSK2UEk40thQEvSUtgwEtSK4UM6YYfy8GAl6QlCAa8JLWSQzSS1EKeZJWk1ooBL0lt5XzwktRS9uAlqY06g/DjrmJOBrwkDSh4T1ZJai3nopGkVvIqmllV1Y8luX+BfY5J8r1R1SRJizU9wVMVrBrHi1bVccCfdy1/tKrWzbLr+6vq5FHVJUmL0TnHOt3XYxxGHvBVdTzwAeAVzfJaYHWSPbPs/nLgT6rqX46wREnqU5r7si78GIeRBnwT7luBlyW5t1l9AfDxqnpiVX2oqt5dVW8BaIZwLgL+S1U9dZ52N1fVzqra+cgjP1zeNyFJ3WYulVzoMQYjG4NvhmBmwv3/dm36FeBc4CQ6Hzi/m66PuyS7quoi4ENVtTnJP/S2nWQLsAXg8MPXTu4ZD0mtM8mXSY6yB38oMAU8MLOiqjYA9yT5YZJv0hmXf29Vnd9z7EPAj4AnjqpYSeqHQzRA0/N+I3BlVR3erP514KqufW5I8lvARTP7VNUTgSuBy5J8a1T1StJCkjA9PdXXYxxGOgaf5GvAJXRC/gjg54EdAFX1E1V1ZVX9V+DvkzxYVU8CttEJ9y+PslZJ6sck9+BHfh18kq9X1RuATwN/MzPenuQbdHr03T4A/GGS20dcpiT1xS869UhyZ1W9NMl3Ftj10iR3jaQoSRrAMAO+qi4EzqdzvvLmJJf3bH8/MA2sBT6b5KoDW3nM2L7J2ke4Y7hLmmyBIX2JqTnvuAk4O0mqaltVbejOwSSvbPYt4Ca6zmHOxrloJGlACUz3H/BHVdXOruUtzSXeM04Dru+6TPwa4Axgto7uocDehV7QgJekJVjEEM3uJBvn2b6Ox4f2XmDDHPu+Dbh8jm2PMuAlaWAZ5jwze4Cf7Fpe26x7nKp6LfCVJF9aqMGxTDYmSW0xxMskbwHObMbXAc6hM87+qKp6FfBQkg/306A9eElagmFdRZNkX1VtA7ZX1X7g9iR3zGyvqtOANwDXVdWfNasvTbJrrjYNeEkaUGceseFdJplkO7C9e11VXQ2cl+RvgRMX054BL0kDC8nyTkOQ5AWDHmvAS9IS+E1WSWopA16SWsmbbktSK83ck3VS1SR/+gyiqnYBdw+puaOA3UNqa5isa/EmtTbrWpxh13VSkqMHPfjQQ5+Y44+f68umj/ed73z1tgW+yTp0revBL+Uvq1dV7Rz1X0g/rGvxJrU261qcyasrZHpye/CtC3hJGqVJvierAS9JSzDJY/AG/Py2LLzLWFjX4k1qbda1OBNV17C/yTpsrTvJKkmjcsghh2X9+pP72ve7373Tk6ySdDCZ9iSrJLWTY/CS1EadQfhxVzEnA16SBhS8TFKSWmuSL1Qx4CVpCRyDl6RWilfRSFIbTfoXnQx4SVoCA16SWingGLwktZOXSUpSSzlEI0ktlITp6alxlzEnA16SlsAevCS1lAEvSS1lwEtSWxnwktQ+SZiOJ1klqZUcopGkljLgJamVYsBLUls5H7wktZDTBUtSa8UevCS1lQEvSS01zCGaqroQOB+YAm5Ocvlitvcy4CVpcJ9PclSf+x5WVTu7lrck2TKzUFWHA5uAs5OkqrZV1YYkd/WzfTYGvCQNKMlZQ2zuNOD6PPYrwTXAGcBdfW4/wKohFidJGtw6YG/X8t5mXb/bD2DAS9Jk2AMc2bW8tlnX7/YDGPCSNBluAc6sqmqWzwFuWsT2AzgGL0kTIMm+qtoGbK+q/cDtSe7od/tsapK/hSVJK11VXQ2clyx+XmIDXpJayjF4SWopA16SWsqAl6SWMuAlqaUMeElqqf8PMeSl1pFIgasAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "STEP: 4, loss: 0.1557\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXgAAADwCAYAAAAKCX+nAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAFapJREFUeJzt3X2wXVV9xvHvk4gwQASSaIAWpGp8GUWrpHaGdpR06AgVKa1C0BhhGI3UUQGnWIugWF8asUMttbUNYCmRRgRakZeqVEEMpUAotLYCYl9E21hJQjAGVHLP0z/2PmRz7tu55+57z86+z4c5c89ee521f+fm8jvrrL322rJNRES0z7xhBxARETMjCT4ioqWS4CMiWioJPiKipZLgIyJaKgk+IqKlkuAjIloqCT4ioqWS4CMiWuppww4gImJ3dcwxx3jz5s191b377ru/bPuYGQ7pKZLgIyIGtHnzZu66666+6s6bN2/xDIczShJ8RMQ0dBq8nlcSfETEgAw0ecHGJPiIiIEZkwQfEdE+hpFOEnxEROuYjMFHRLRWxuAjIloqCT4iooVsZ4gmIqKt0oOPiGghAyNJ8BER7ZQefERES2UMPiKijez04CMi2ihr0UREtNhIpzPsEMY1Z+/oJOlQSTskLa2UvVHSKklXDDO2sUi6WNKBw45jutryPiIK7vu/YZizCZ7ivX8V+EilbD4w3/bK4YQ0oT1oxzeutryPCGzo9PkYhrn+P9oPgbsknWz7c91CSTfa/g1Ji4GPAo9RfCC8H1gGnAY8Ajxq+wMzHaSkU4FXAudLuhx4G7AD2B/4rO3rZ/DYzwY+A/wzsBj4R+DlwBPAE7Z/V9JHgf2AfYFLbG+QdDjwXmAzxQfnWcCqyvu4yPa/1hTjnwMX2v6OpBXl8Y6l53ck6TzgH2zfXr7uRtu/UUcMMXdlDL7ZLgaukXRjpezp5c+PAx+3/W0ASQI+ABxtuyPpDyUdYfvumQzQ9mWSjgLOL2P6iO37JM0DbpB0m+1HZujwAnbaPhtA0kPAEbYflvTXklYC222/X9LTgOsokuvHgRNt75B0OnBC9X3Y/n6NMa4HTqb4NvbbFH/X5/b+jii/oVVe9/RRLUVMURJ8g5WJ+g+ADwG9ifqQbnIvPRNYAnysyPUsAQ6YlUB3WWL7Pngy9nuB5wIbZ/CY36s8f9D2w+Xz7cDPAb8oaU1Z9tPy5/OA88rf037AbTMY3wbgbElLKL5ZPW+c31Gv+WOURfQtywXvBmzfI+nNwBHAv1R2bZJ0uO1vltubgR8A59ie7VPnIxT/XpskvajSO30Z8IezGEfvX7OBf7L9yZ7y/wQ+ZPvxnvLu+6gvINtlEv8IcDmwepzf0aPAQVCcZO8+jxiY3ehZNHM5wY+Uj67zgW9SjDX/rCx7L/AJSdvLuucAfwJcKWkLxdDFO2cp3q8DnwT+Dvh9SY9R9Iz/zPaPZvC4vb+nJ3r2/S1whqTPUPTeN9i+guIb0TpJmylOrL7L9mPd9yHpUtvX1RjnOuBLwGrgP4A1vb8jSZ8DLpb06vJ91HIOIOa2Jg/RqMnBRfRL0nOBN9v+0LBjibnjJS97mf/2y1/qq+4LDjr4btvLZjikp5jLPfhoCUlvAH4LOGPYscTc0+BbsibBx+7P9tXA1cOOI+amJo+CJMFHRExDEnxERAu54bNo5vJSBZOStHrYMYwlcU1dU2NLXFPTxLhcLhk82WMYkuAn1rg/plLimrqmxpa4pqZRcXUvdOrnMQwZoomImIZhrRTZj9Yl+L33XeD9Fy6upa39DljEwYf+Qi3/etsfebSOZgDYc8+9WbBgYW1/VT/5yY5a2pk3bz577LFnbXGNjOysqykkMW/e/Fpiq/siZkmNzBBzJK7Ntp85nQYyTXIW7b9wMW8/+/xhhzHKzVf//bBDGNd99//TsEMY0/btW4cdwpgef3z7sEOI+nx3Oi+2TafBJ1lbl+AjImZTFhuLiGipzIOPiGipJPiIiBbyEKdA9iMJPiJiGjJNMiKihQyMNHieZBJ8RMQ0ZAw+IqKl6hyDL29iv4Libmm3276gZ/9ZFLcW/RnFPYV/p7xT2piyFk1ExKD6XGis7OUvlrSx8njKujqSFgCrgN+0/VvA4ZKWVvbvDxxt+822TwO+Bfz6ROGlBx8RMSAzpSGazZPcsu9I4CbvavBaYDnwYLn9KLBJ0kHANuDZwCUTHTAJPiJiGmocolkEVNfn2Ao82YO3bUl/BbwD2ALcZnvLRA1miCYiYhpqXC54C3BAZXthWQaApJcCx9k+z/YngcclvXWiBpPgIyIGVPN68HcAR0tSuX08cGtl/0GAKtuPA4dN1GCGaCIiBlXj3Zpsb5O0DlgvaSdwr+37K1W+ArxK0uXAT4G9gXdP1OaMJ3hJV9k+cYL9lwJn2f7RdNqJiBiGOqdJ2l4PrK+WSboGOMn2CPD+qbRX2xCNpHsk3VI+7pe0vNy1R7n/PEmHj/HS+b1xSFop6eSeenvUFWtERB26s2hm8p6stl9fJvcpq7MH/13bJwBIOhXYr2f//PLRjyXAN+sLLSJiZozM8Rt+PE/S71Ik7X79MrBR0nOAz5Rle41XubxgYDUUt9mLiJgdbvRiY3XOonlOd4gG+D12Te/5ju0/Av5vgtdeL+ndAJIOAPYBVtv+T9tH2T4K+MF4L7a91vYy28v23ndBHe8lImJSdv+PYaitB2/7pZLk8QebDOwl6RkUwzfPp5jmA8Xczm3l8w8CfwC8QtIq2+vqijEiom5zaT34q4A39JR9ufy5Afg4xfSeTRTrKFxXrSjpTcBW23cCd0r6lKRv2b675jgjImoxl1aTHNWe7U+XP78KfLV3/645/QD8i+2/qbz2neqpEBHRFN0LnZqq7gT/jHIMvsrAybbHG4PfCXQAbP97787KkM8TdQUZEVELm85cmUVj+9cGeM2EaylU6uUip4honjnUg4+ImFOcW/ZFRLRTgzvwSfAREYMq5rg3N8MnwUdETEMSfEREK5nOyByZRRMRMZdkiCYiosWS4CMi2ioJPiKinRqc35PgIyIG5pxkjYhope4t+5qqdQn+4Gct4oPvOmXYYYzyoTNOG3YI47Kb2wOJaLok+IiIlkqCj4hoIxuy2FhERDulBx8R0UIGOunBR0S0UJYqiIhor9zwIyKilZwefEREWyXBR0S0UJYLjohoMY8kwUdEtFJ68BERbeScZI2IaK0k+IiIFspywRERbWVwjTf8kLQSWAGMALfbvqBn/3OB95ebI8AHbf/veO0lwUdEDKy+MXhJC4BVwLG2LWmdpKW2Hyz3C1gDvN321n7anDeNYK6aZP8lkp4x3XYiIpqsmAs/+QNYLGlj5bG6p6kjgZu86xPjWmB5Zf8vAd8DPibpCklvnSy2SXvwkj5v+6Ty+SnAI7a/COxRln2iPDDAIuAc29eVbc+rtPNe4LUUXyseAN5t+4luO5V6bwHeCKh8/Aj4gO37Jos1ImK2TaEHv9n2sgn2LwKqPfOtwNLK9mHAS4Djbf9E0qclPWD7G+M12M8QzWGS3lc+PwK4orrT9tnd55JOH6tNSS8EXmD71eX2qcApwCU99ZYDRwGvs72zLDsYuJri0y0iojHsWhcb2wK8uLK9sCzreoyih/+TcvuLFDl53ATfzxDNQ7bX2F4D3FAp/1VJt0h6fqXsVcCGMdrYBhwo6aBy2OYIYNM4x+tQnJzucs/2KJJWd7/2PPzww5O9n4iI2ricCz/Zow93AEeXY+0AxwO3VvbfDbyysv3LwL9O1GA/Pfj54zzfYPuE7oakl1AM31Qz7PXlEM9Fkj4KXAjsBVxpu/phAYDtmyUdUr6uOkQz4ViT7bXAWoBly5Y1d85SRLSM6XTqmUVje5ukdcB6STuBe23fX9m/SdJXJK0HdgD/bftrE7XZT4L/saSvU/SiR4AzeitIWkRxdve0nl3HlUGvAJYAdwKHAksknQ3cNcbxNgGf6n4ASLow4+8R0Ug1LzZmez2wvlom6RrgJNsjti8GLu63vUkTvO1Vkq62/YaeXV8rD/5K4FzgTNs/HKeNK8u6hwFnlgHeQHHSda+e6vv1lB066buIiBiWGb7hh+3XD/rafufBj6pn+6Ly6WuAk20/1u9By7rLASR9ofz5NmAlsE+x+eQUoAMl3VI+f6ftf+v3OBERM6m4knXYUYyv7wRfSbJdpkjsHx7nNTspTphWdcYoewJgql89IiKaYLdfqsD2cVNt2PaoE6O2HwLe01N24lTbjohoBJtOjUsV1C1LFURETMNu34OPiIjRsppkRERbNfwsaxJ8RMTAckeniIjWcnPPsSbBR0QMzNS2VMFMSIKPiBhQTrJGRLRYEnxERCu5zvXga5cEHxExqJpXk6xbEnxExHQkwUdEtI+BToZoZs9Ip8MjO3YMO4xR9tlnv2GHMK4f//iRYYcQsXuq956stWtdgo+ImD25kjUiorWS4CMiWioJPiKihWxwbvgREdFODe7AJ8FHRAwuJ1kjIlorCT4ioo2yVEFERDuZXOgUEdFSxrnhR0REC2WIJiKivRqc35PgIyKmI2PwEREtlHuyRkS0VcPH4OcN+kJJV02y/xJJz5huOxERzWU6nU5fj2GYtAcv6fO2TyqfnwI8YvuLwB5l2SeAXyqrLwLOsX1d2fa8SjvvBV4LjAAPAO+2/US3nUq9twBvBFQ+fgR8wPZ903ifEREzYncfgz9M0vvK50cAV1R32j67+1zS6WO1KemFwAtsv7rcPhU4Bbikp95y4CjgdbZ3lmUHA1cDR/b1jiIiZksxCD/sKMbVzxDNQ7bX2F4D3FAp/1VJt0h6fqXsVcCGMdrYBhwo6aBy2OYIYNM4x+tQ/Nq63LMdEdEI3fzez2MY+unBzx/n+QbbJ3Q3JL2EYvjm4Uqd68shnoskfRS4ENgLuNJ29cMCANs3SzqkfF11iOatEwUoaTWwGuDnDzmkj7cUEVGPOk+ySloJrKAYyr7d9gVj1HkacDmw3fbbJ2qvnwT/Y0lfp/iwGgHOGOOAi4A1wGk9u46zvU3SCmAJcCdwKLBE0tnAXWMcbxPwqe4HgKQLJxt/t70WWAvw8le8Ir39iJgdNp2abvghaQGwCjjWtiWtk7TU9oM9Vc8FLgNOmqzNSRO87VWSrrb9hp5dXyuDemV5wDNt/3CcNq4s6x4GnAlcTDHc81qKHn3Vfj1lh04WY0TEsEyhB79Y0sbK9tqyc9p1JHCTdzV4LbAceDLBS3oTsBH4dj8H7Hce/Kh6ti8qn74GONn2Y322RVl3OYCkL5Q/3wasBPYpNtUdljlQ0i3l83fa/rd+jxMRMZOmeKHTZtvLJti/CNha2d4KLO1uSHo5cKDtvyk7y5PqO8FXkmyXKRL7h8d5zU6KE6ZVnTHKngCwfTFFzz4iYrdR4xj8FuDFle2FZVnXycD+kv4CWAC8QtI7bP/5eA32leBtHzfVSG2POjFq+yHgPT1lJ0617YiIZqh1iswdwBmS/rgcpjke+NiTR7J/r/u87MGfO1FyhyxVEBExOINruki1nJCyDlgvaSdwr+37x6k+QjFKMqEk+IiIaahzGQLb64H11TJJ1wAn2R6p1PsecPpk7SXBR0QMaDZWk7T9+kFfmwQfETGohq8mmQQfETEw7/aLjUVExHjSg4+IaCc3eC3EJPiIiAHZptMZmbzikCTBR0RMQ06yRkS0VBJ8RERLJcFHRLSQbVzXWgUzoHUJ/t577mHhvvsOO4zdjIYdwJia+j9OcbOxiEJT/06hhQk+ImI2ZYgmIqKlkuAjIlopY/AREa3kLDYWEdFeSfAREa1kXOMNP+qWBB8RMQ0mCT4iopUyRBMR0UI5yRoR0VpOgo+IaKusBx8R0VLpwUdEtFExCD/sKMaVBB8RMSCTe7JGRLRW1qKJiGilzKIZk6Rn2f7hJHUOtP2D2YopImKqOg1eqmDeMA4q6WDgryrbn5O0aIyqn5Z02GzFFRExFcU51k5fj2GY9QQv6eeAS4HTy+2FwHzbW8ao/nbgzyT9wiyGGBHRJ5f3ZZ38MQyzmuDL5H4x8Dbb3yuLTwY+L2lvSZdJ+mNJ5wOUQzinAn8q6TkTtLta0kZJG2f2HURE9OhOlZzsMQSzNgZfDsF0k/v/VHa9DjgBeDbFB857XPm4s/2wpFOByySttv2/vW3bXgusLY/T3DMeEdE6TZ4mOZs9+D2BEeDRboGkpcBDtn9q+9sU4/IXSVrR89odwM+AvWcr2IiIfmSIBih73ucAl0taUBa/Bfhspc7Ntt8FnNqtI2lv4HJgje3vzFa8ERGTsU2nM9LXYxhmdQze9jeBcymS/H7ArwAbACS9SNLlkv4S+G/b2yXtA6yjSO53zmasERH9aHIPftbnwdv+lqT3AV8AvtEdb7d9H0WPvupS4MO2753lMCMi+pILnXrYfkDSabb/a5Kq59l+cFaCiogYQJ0JXtJKYAXF+crbbV/Qs//TQAdYCNxg+7OjW9llaFey9pHcSXKPiGYz1HQRU3necRVwrG1LWidpaTUP2v6dsq6AW6mcwxxL1qKJiBiQDZ3+E/zinmt11pZTvLuOBG6qTBO/FlgOjNXR3RPYOtkBk+AjIqZhCkM0m20vm2D/Ip6atLcCS8ep+xHggnH2PSkJPiJiYK5znZktwIsr2wvLsqeQdBZwj+3bJmtwKIuNRUS0RY3TJO8Aji7H1wGOpxhnf5KkdwA7bF/RT4PpwUdETENds2hsb5O0DlgvaSdwr+37u/slHQm8D7hR0l+UxefZfni8NpPgIyIGVKwjVt80SdvrgfXVMknXACfZ/kfg0Km0lwQfETEwY8/sMgS2Xz/oa5PgIyKmIVeyRkS0VBJ8REQr5abbERGt1L0na1O1McFvBr5bU1uLy/aapua4auuB1BrXrunAtZgj/5a1mStxPXu6DaQHP4tsP7OutiRtnOTS4qFIXFPX1NgS19Q0Ly7jTnrwERGt1OR7sibBR0RMQ8bgd19rJ68yFIlr6poaW+KamkbFVfeVrHVTk4OLiGiypz99Ly9Zclhfdb///Qfunu3zB+nBR0RMQycnWSMi2ilj8BERbVQMwg87inElwUdEDMhkmmRERGs1eaJKEnxExDRkDD4iopWcWTQREW3U9AudkuAjIqYhCT4iopUMGYOPiGinTJOMiGipDNFERLSQbTqdkWGHMa4k+IiIaUgPPiKipZLgIyJaKgk+IqKtkuAjItrHNh3nJGtERCtliCYioqWS4CMiWslJ8BERbZX14CMiWijLBUdEtJbTg4+IaKsk+IiIlqpziEbSSmAFMALcbvuCqezvlQQfETG4L9te3GfdvSRtrGyvtb22uyFpAbAKONa2Ja2TtNT2g/3sH0sSfETEgGwfU2NzRwI3eddXgmuB5cCDfe4fZV6NwUVExOAWAVsr21vLsn73j5IEHxHRDFuAAyrbC8uyfvePkgQfEdEMdwBHS1K5fTxw6xT2j5Ix+IiIBrC9TdI6YL2kncC9tu/vd/9Y1OSrsCIi5jpJ1wAn2VNflzgJPiKipTIGHxHRUknwEREtlQQfEdFSSfARES2VBB8R0VL/DxH7ECe4NysJAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "encoder.train()\n",
    "decoder.train()\n",
    "losses = []\n",
    "for i in range(STEP):\n",
    "    encoder.zero_grad()\n",
    "    decoder.zero_grad()\n",
    "    enc_output, enc_hidden = encoder(inputs, lengths.tolist())\n",
    "    outputs, attns = decoder(enc_hidden, enc_output, lengths.tolist(), \n",
    "                     targets.size(1), targets, is_eval=False)\n",
    "    \n",
    "    \n",
    "    loss = loss_function(outputs, targets[:, 1:].contiguous().view(-1))\n",
    "    losses.append(loss.item())\n",
    "    # check training & show attentions\n",
    "    print('STEP: {}, loss: {:.4f}'.format(i, loss.item()))\n",
    "    preds = outputs.max(1)[1]\n",
    "    translated = list(map(lambda x: trg_itos[x], preds.tolist()))\n",
    "    fig = plt.figure()\n",
    "    ax = fig.add_subplot(111)\n",
    "    cax = ax.matshow(attns[1].detach().numpy(), cmap='bone')\n",
    "    fig.colorbar(cax)\n",
    "    ax.set_xticklabels([''] + src[0])\n",
    "    ax.set_yticklabels([''] + translated)\n",
    "    plt.show()\n",
    "    \n",
    "    loss.backward()\n",
    "    \n",
    "    enc_optimizer.step()\n",
    "    dec_optimizer.step()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Test"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### IWSLT 데이터셋\n",
    "\n",
    "IWSLT 2016 TED talk 번역 문제입니다. 독어-영어의 번역이 목적입니다. \n",
    "\n",
    "훈련시간이 오래 걸리기 때문에, 미리 훈련을 시켰습니다. 훈련 하이퍼파라미터는 아래와 같습니다.\n",
    "\n",
    "\n",
    "\n",
    "| 변수명 |1차 훈련 | 2차 훈련 | 3차 훈련 | 설명 | \n",
    "|--|--|--|--|--|\n",
    "| BATCH| 50 | 50 | 50 | 배치 사이즈 | \n",
    "| MAX_LEN | 30 | 30 | 30 | 훈련 시킬 문장의 최대 길이 |\n",
    "| MIN_FREQ | 2 | 2 | 2 | 최소 단어 출현 횟수 |\n",
    "| EMBED | 256 | 256 | 256 | 임베딩 크기 |\n",
    "| HIDDEN | 512 | 512 | 512 | 히든 크기 |\n",
    "| ENC_N_LAYER | 3 | 3 | 3 | 인코더 층 개수 |\n",
    "| DEC_N_LAYER | 1 | 1 | 1 | 디코더 층 개수 |\n",
    "| L_NORM | True | True | True | 임베딩 후, layer normalization |\n",
    "| DROP_RATE | 0.2 | 0.2 | 0.2 | 임베딩 후, dropout 확률 |\n",
    "| METHOD | general | general | general | attention 방법 |\n",
    "| LAMBDA | 0.00001 | 0.00001 | 0.0001 | weight decay rate |\n",
    "| LR | 0.001 | 0.0001 | 1.0 | 학습률 |\n",
    "| DECLR | 5.0 | 5.0 | - | decoder의 학습강도, 학습률에 곱해준다 |\n",
    "| OPTIM | adam | adam | adelta | 최적화 알고리즘 방법 |\n",
    "| STEP | 30 | 20 | 20 | 훈련 step 기준 1/3, 3/4 지점에서 각각 0.1 을 곱해서 학습률을 조정합니다.|\n",
    "| TF | True | True | True | teacher forcing, 모델에게 다음 토큰을 알려주는지 여부 |\n",
    "\n",
    "각 학습 세이브 체크포인트\n",
    "\n",
    "* 1차 훈련: [8/30] (train) loss 2.4335 (valid) loss 5.6971 \n",
    "* 2차 훈련: [1/30] (train) loss 2.3545 (valid) loss 5.6575 \n",
    "* 3차 훈련: [6/20] (train) loss 1.9401 (valid) loss 5.4970 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['runtrain_big.sh', 'runtrain_big2.sh', 'runtrain_loaded2.sh', 'runtrain_loaded.sh', 'runtrain.sh', 'runtrain_wmt.sh']\n"
     ]
    }
   ],
   "source": [
    "import sys\n",
    "import os\n",
    "sys.path.append(os.getcwd() + '/model')\n",
    "print([d for d in os.listdir(path='./model') if '.sh' in d])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "import importlib\n",
    "import torch\n",
    "import numpy as np\n",
    "import matplotlib.pylab as plt \n",
    "import matplotlib.ticker as ticker\n",
    "import settings\n",
    "from train import import_data, build_model, validation\n",
    "\n",
    "%run ./model/build_config.py -r \"./model/runtrain_big2.sh\" -c \"./model/settings.py\" -n\n",
    "importlib.reload(settings)\n",
    "DEVICE = None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Source Language: 51823 words, Target Language: 30980 words\n",
      "Training Examples: 189416, Validation Examples: 946\n",
      "Building Model ...\n"
     ]
    }
   ],
   "source": [
    "SRC, TRG, train, _, test, _, _, test_loader = import_data(config=settings, device=DEVICE, is_test=True)\n",
    "enc, dec, loss_function, *_ = build_model(config=settings, src_field=SRC, trg_field=TRG, device=DEVICE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "test loss is 6.1745\n"
     ]
    }
   ],
   "source": [
    "enc.load_state_dict(torch.load(settings.SAVE_ENC_PATH, map_location={'cuda:0': 'cpu'}))\n",
    "dec.load_state_dict(torch.load(settings.SAVE_DEC_PATH, map_location={'cuda:0': 'cpu'}))\n",
    "print(\"test loss is {:.4f}\".format(validation(settings, enc, dec, test_loader, loss_function)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def evaluate(test_data, max_len, src_field, trg_field, stop_token='</s>'):\n",
    "    f = lambda x: trg_field.vocab.itos[x]\n",
    "    test_ex = np.random.choice(test_data.examples)\n",
    "    src_sent = test_ex.src\n",
    "    trg_sent = test_ex.trg\n",
    "    inputs, lengths = src_field.numericalize(([src_sent], [len(src_sent)]))\n",
    "    enc_output, enc_hidden = enc(inputs, lengths.tolist())\n",
    "    outputs, attns = dec.forward(enc_hidden, enc_output, lengths.tolist(), max_len, \n",
    "                                 targets=None, is_test=True, \n",
    "                                 stop_idx=trg_field.vocab.stoi[stop_token])\n",
    "    preds = outputs.max(1)[1]\n",
    "    translated = list(map(f, preds.tolist()))\n",
    "    \n",
    "    print(\" < input: {}\".format(' '.join(src_sent)))\n",
    "    print(\" > target: {}\".format(' '.join(trg_sent)))\n",
    "    print(\" > predict: {}\".format(' '.join(translated[:-1])))\n",
    "    return attns.squeeze(0).detach(), (src_sent, trg_sent, translated)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def show_attention(input_sentence, output_words, attentions):\n",
    "    # Set up figure with colorbar\n",
    "    fig = plt.figure()\n",
    "    ax = fig.add_subplot(111)\n",
    "    cax = ax.matshow(attentions.numpy(), cmap='bone')\n",
    "    fig.colorbar(cax)\n",
    "\n",
    "    # Set up axes\n",
    "    ax.set_xticklabels([''] + input_sentence + ['</s>'], rotation=90)\n",
    "    ax.set_yticklabels([''] + output_words)\n",
    "\n",
    "    # Show label at every tick\n",
    "    ax.xaxis.set_major_locator(ticker.MultipleLocator(1))\n",
    "    ax.yaxis.set_major_locator(ticker.MultipleLocator(1))\n",
    "\n",
    "    plt.show()\n",
    "    plt.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def eval_and_show(dataset, max_len, src_field, trg_field, stop_token='</s>'):\n",
    "    attns, sents = evaluate(dataset, max_len, src_field, trg_field, stop_token)\n",
    "    show_attention(sents[0], sents[2], attns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " < input: und so würden zwei davon , in den körper implantiert , weniger als einen zehner wiegen .\n",
      " > target: so two of them implanted in the body would weigh less than a dime .\n",
      " > predict: and so , that would two two of them in the body , less than a car .\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAASgAAAErCAYAAABtkx63AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJztnXm4XFWVt98fCcgoqMGJwQAyypwwCSgg0DiBAy3QoCIqNIhKI/igTSMfagtqq6CohMluBAcGJSKTMg8CSZgSEDQElYACYZJ5yP19f+xdSaVSVbfq1qlbp26t9z7nqVO79tln3Vu3Vu299hpkmyAIgjKyRK8FCIIgaEQoqCAISksoqCAISksoqCAISksoqCAISksoqCAISksoqCAISksoqCAISksoqKArSDqrlbYgaEYoqKBbvK36iaRxwKQeyRL0KaGggkKR9CVJTwMbS/pnPp4GHgEu7LF4QZ+hiMULikbSEsBptg/otSxBfxMzqKBwbA8BW/RajqD/CQUVdItbJYWSCjoilnhBV5B0D/BW4K/As4AA2964p4IFfUUoqKArSHpLvXbbfx1tWYL+JZZ4QVfIimg1YKd8/hzx/xa0Scyggq4g6SvAZGBd2+tIejNwru1teyxa0EfEN1rQLT4I7E6yP2H7IWCFnkoU9B2hoIJu8ZLT9NwAkpbrsTxBHxIKKugWv5R0CrCSpE8DvwdO7bFMQZ8RNqiga0jaBdiV5GJwme3f9VikoM8IBRX0DZIErGr7gV7LEowOscQLCkXS9fnx6apg4X9WnncydrZpXVyIoEFfML7XAgRjC9vb5cdu7djdKmkL29O6NH5QImIGFXSFLias2wr4g6T7JN0paaakOwsYNyghMYOqg6SVgU8DE6n6G0X6kLaoTVg3nmIS1v1LAWMEfUIoqPpcCFxH2hqf32NZ+gpJXwK+DCxTZXMS8BIwpdPxbf9V0nbA2rbPzF8my3c67miQs4q+gUW/9P7WO4nKT+zi1UHS7bY37bUc/Yykb9j+UhfG7XkITU7It7XtG9u45rPAV4CHgaHcHNkdhiEUVB0kfQ240XbsGHWApFWAt7DojOHaDse8HdgMuNX2ZrntztH+oEu6rXL/FvvPBray/VgXxRpzxBKvPp8HvizpJdLSpJLL6NW9Fas35BnDnrZ/2cY1xwN7A3ezcJlsoCMFRQ6hkdTrEJorJH0YuMCtfcs/ADzVZZnGHDGDClpC0nTbk9vofy+wse0XC5bjCGBtYBfgG8ABwDm2v1/kfVqQ42lgOZLyfZ5hvsQknQ6sC/wWWPA3sf2d7kvbv8QMqg7ZY3lfYA3bX5W0GvAm27f0WLRe8vusHH5BzlAAYPvxBv3nAEtS9WEsAtvfziE0/yR94I/pRQjNCPy8/paPpfIRtEDMoOog6UckQ+ZOtteX9Brgctsd59iWtC1wLAttM5Vv3jU7HbubSLq/TnNDuSWdD2wCXMGiM4bPdUfC0WWkX2KSlrX93KgIOQYIBVUHSbfa3rzaECrpDtubFDD2PcB/ADOocmEYa8ZTSR+v1277fzsc92lyCpcqngKmA1+wPaeT8duQo60vMUnbAKcDy9teXdImwEG2DxkNefuVWOLV5+Xss1IxxK7Mwq3hTnnK9iUFjTVqSFoWOBxY3faBktYmbfVfVK9/p4qoCd8D5gLnkGafewNrAbcCZwA7dOm+tWxV+RIDsP2EpGZLt++RnEyn5v53SHrHKMjZ10SoS31OAn4FvF7S14Hrgf8uaOyrJH1L0jaSNq8cBY3dTc4k7Wi+PT9/EPhao86S1pZ0nqS7Jc2pHAXIsbvtU2w/bfuftqcA/2L7F8BrOhlY0ock/VnSUy0EOLf9JVYnC0M4AQ9DzKDqYPtsSTOAd5G+pT9g+48FDb9VfqzeETOwU0Hjd4u1bO8laR8A289lO0wjziQ5Jn4X2BH4BMV8IT4n6SPAefn5nsAL+bxTe8U3gfe3+F7XfontCRzdpP8Dkt4OWNKSJFeWov6nxixhg6pC0mubvd5kx2rMI+lGksK+IS9t1gJ+ZnvLBv1n2J4kaabtjarbOpRjTeBEYBuSQrqJZNN7EJhk+/oOxr6hHY90Seux8EvsimaKTdKELPfOuf/lwOfHmu2xaEJBVZF3qkz6B1odeCKfrwT8zfYaBdxjRdLMomJ/uAY4znapnfjy1v7RwAakD9e2wP62r27Q/0ZgO9JM50qSAjne9rqjIvAIkHQi8Ebg1yy683hBg/4RW9dlQkHVQdKpwK8qoS6S3k1a5h1UwNjnA7OAihH5o8Amtj/U6djdRtLrgK1JSvsm2/Oa9N2CtIRZCfgqsCLwTds3jfDeX7T9TUnfp85Srgj3BUln1ml2vSwWNbF18xmmcrKkk+o0PwVMt33hyKUe24SCqkP1sqRRm6SdbF8pqa5iafKtu1ggcr8EJ+ffdTuSgrje9q9G8d7vt/2bbrkvjECetmLrJE0B1gPOzU0fBu4HXgfMsX1YVwTtc8JIXp+HJB0N/DQ/3xd4qKbPO0lLl/fXud5AXQUFPC9pu4qtJDtuPt+5yN1F0g+BtwI/y00HSdrZ9mdq+v2GJsZq27uP5P62f5Mfu6aIJK0D/Ah4g+0NJW1M2jWst1vZbmzdxsC2tufne/2IlNJnO2BmZ5KPXWIGVYdsLK/YiSoBrscVYSSXtClpebdibnoC+LjtUmeFzA6m61cCY3MA8V2216/p985m49i+pkM51gGOYPFkgh3vgkq6BjgSOKXKQXeW7Q3r9G0rti7HJm5ZsTVmW+QtttdVm5kRBom+nEEN5zdk+9YOxh4HfNn251vs/waSj9Sbbb9b0gbANrZPb3DJTNubSHp1lrWjQgKjyGzSxsFf8/PVctsiVCug7Li4HknJ32v7pQLkOBf4MXAaxfsRLWv7lhrviVca9G03tu6bwO2SribZq94B/LdSNobfj1jiMU5fzqAkXZVPlyb5E91BetM3Jhkdt+lw/Jtsb91i30tIPj//mRXPeOC2WhtWVf+/AZeSgm6vdI/egGxPOgF4PelvN1w0/jXAFsAtJIWzJSm85ClYfOkm6b0kRXJfHnsNUmhHR170RbgqNBn7EuBQUgK8zSXtCXzS9rsLGv9NpL8bwDSncvBBE/pSQVWQdAHwFdsz8/MNgWNt79nhuD8CViF9W1dH7i9mV5I0zfYWNXF7DY3eOWTkfaQQjc2Bi4Cfd+K/MxKykbdVp8S2l255Sfg+27Pz87WA39peb4QiV8Y9FniE5CRZvbQqYvm9Jikt8dtJS+/7gf1s/6VO37aWmtmpdV9gTdvHSVodeKMHO0PGsPTlEq+KdSvKCcD2LEnrN7ugRZYGHmNR7+5Ghu9n8/Z7xTazNU2Mp06R7L8klQZ/Dcl57xpgXAFyt8PDbSincSTFv2Mb4z9dUU6ZOcDT7QjYgMou3pFVbQY6zgbhFGi8c152LWG7mbztLjV/SA4uBo4j/S3OJ81Kgwb0u4K6U9JpLLrb1rGx2fYn2uj+BVIA6FqSbgBWJoU9NCTPRvYCdiMtkz4yQlE7YbqkX9CCU6Lt+ZKGJK3YhkPpdEkXk5SxgX8FplXcMhq5YQxHEc6yjZD0KtL2/0RgfMUWZfu4Ot1fsf2jNoZvN7g4oP8V1CeAg0lxTZB229r5p6lLdtir5wy4mMOe7RlZ4axLsrXca/vlJmP/BbiN9ME90vazjfp2mVcDzwG7VrU1c494Bpgp6Xcsuuxt5CC5NMmJsbI0fBRYhuSW0ew+TVGbWRUk/Stwqe2ns+vI5sDXGmykXEia/c6gQaK9qnCo30g6hNaXmt3MkDFm6WsbVNFI+p7tw5RyTVdYGvgg8FC9D6NSqe9rSD4tNwyzLEDSq/to524BJXKQ/AVJgXws+yotSypw0cjmd6ftjZVKVX0N+BYpC+dWdfrWdSmo6VMdDlWL3TiB376kWfPmJDeTPYGjbZ9br3+Q6GsFpcWzUwLQ6J+khfEm2Z5Rp30Jkuf02+u8tgawfT62Jn2bXmf7PxrcY2ngk6TClktXyTyqRUHbdEqsXLMUsE5+OtxMcQ3gsyxuRB6Ro2bVuNNtT1aLyQQr/SR9g+TicU4jv6Ps7f39artmkaiN4OIg0e9LvNOpk52yHkopLg5m0SDdH9d8yA4CDqxz+dqk7fjFsH2/pBdIuZJeIqUWaWaoPwu4h5S87DiS3awX/6inkp0SAWzfKekcanI8SVrP9j2SdiB98/+F9AFbTdLH3biM1K9J789vKHYp85KkZVi4VFqL5nnPH5R0CqnIwgnZztQo7ct2wP55lvQiTeLrWl1qVmbMeWn4CAs98ZH02iJ2H8c0tvv2AG5uo+9ppA/YTvk4Ezitps/6+fFpUlL+yvEn4MMNxr0PuJlkB9uctPvTTI7b8uOd+XFJUuDtaP/tplXLk89vr9NvSn6cQfoAVtrXAWYU8d60KfcupC+XR4GzSQpzhyb9lwU+RKpEDPAmYNcGfd9S72jQ9xfAF4FZVfep9/e7KD/eT9rJnJPP7yfF4PX8c1Tmo99nUFdJ+hbJ4FptqKxnAN3Ciy4DrpR0R3UHL5xyX0gyuF/n4afhJ5G+efchFZS8RtK1tu9r0L8yY3sy+239gwazs5GglBRtIosuq/6vTtd5efZRmYnsCfy9Tr8f58clbd9bNeaf8qy0EScqVQG+nOHfm3b4OCm85DzSh/3zrpNVQYvm9rq6qu1F0s5pdd+KXbAdN4iWEvjZfl8+vYFsq7R9Txv3GWj6XUFVDJ0Vz2LRODvlfElrVRRHdsprtCw8nWRTOil/iG8l/WOdWNsxt50oaXnSruKxwKo09muakv2fjia5JywP/FezX7JVJJ1Fys99O4sWy6ynoD5DckpcT9KDpG/0fev0+xjp959e49KxHzCtiTgbkVLJ7ERVqW86zxxaeW92If2ut+UvhNr3ZgaLGrMrxtbK/0i1nfIckvPsjJp+1Olbod2lZkXu7w/3PxUspN+N5F+p02zX8VuRtBPwE9K3LqRZxidsX1XbN/cfR3Ki2xH4d+B51/GClvRt0j/e8sCNpPzl17mmuoikw+vdpkrmjgs4SvojsIGbvKl15FiGZJN5NgvSKNh1O1KYxna56Trgz268vT87y1JE/F3t2C29N1X9X0uyI1ZvSiwWtJw3QyqlpCre3m+yfXOdvm0l8BuJ3EH/z6CeqTpfmvQt2GhJ9jpgQ5Ji+gApZWxdp0NJV5Cqxv6B9EHcwvYjDcY9HPio7bOrrp/C4sb2SqHHdUn/pFPz8/eT4tuKYBYpI2S9pVojOS4kKcqPDiPHSaSsC98ByEub/yKF6jSSZSWSYbgw2nxvkPQpkn1wVdLMcmvSF8m76nQ/mda9vY8izUCfJP39DgO+Tl5Odip3kOm1EazIA3gVcHWD1ypG6e2Aq4D30sCQS0r0fy3wO9KSbSdgmQZ955BsC8dUtd3aRMZrgRWqnq8AXFvQ738VKYbsMpICnApMLUIO0jJnBkmxfZr0IVuxSf+rgcdbkaXN37Hl9yb3n0n68ro9P18PuKBB31vzY/XGwR3DvO9fafF9b0vuONLR7zOoWpYlfVPWo2KTeS9wqu3fSqrr8+PswyRpBWB/0o7fG0kKsJYnSd/GJykla9tvGBnfQHJHqPBSbmsJpYj4x23Xs3cc2+o47cphe06eNf2alGZkV9vNEu3VW363RLPfsc33BuAF2y9IQtKrnFwmGuVFb8fbu633fQRyB/T5Ek/STBYaNceR4uDqxU1BG/4wkg4l2ZUmkbaxzyDNGOp2t/0KcIik/Uk2qGb12f4PuEVSJV3uB0i2sVY5ixT3d77tI6pfcHvJ4FqSo+ZvDPBa0t/6Zkm4QQ7uNmWppeHv2OZ7AzBX0kokxfo7SU+wMKdVLe2UkmrrfR+B3PXGeKPtf7RzTb/T70byt1Q9fYUUoV83wVh2rNuN5E385/wtvZHty+v0PYL0zzOj0XhVfQ+yfUrV80nAZ9zEM1wp4d72+em1tm9rdo8614tkgL4rP7/e9nZavCz4cDmehpWj5m+8GLYX+bCPVJY6913kd6xqb/m9qTPmO0mZTC91A+N9q97e7b7vnchdNcZvbb93JNf2K32toIIgGNtE6fMgCErLmFFQkurF0HXct5tjl0WOGLu8coykfy+QdIakRyTNavC6JJ0kabakOzVMXYEF9HobsaiDlIu88L7dHLsscsTY5ZVjJP17cZCC8DcnxybWef09wCUk297WtBirOWZmUEEQ9A6nrBbNMjPsAfyfEzcBK+WNqqb0nZvBhAkTPHHixMXaV199dSZPnryIxX/GjMVSOy1AUlu7A+3071bfGHt0xy6LHE36z7O9cjvjVLPbbrt53ryG1esXYcaMGXcBL1Q1TbE9pY3brUIqdlphbm5rFvXQfwpq4sSJTJ8+ffiOgBYPLg+CsUQjf66WmDdvXjufpRdsT+7kfiOh7xRUEATFke1Do8GDpGKvFVbNbU0plQ1K0jPD9wqCoAgMzB8aaukogKnAx/Ju3tbAU7abLu8gZlBBMMAYL168aERI+hmwAzBB0lxSLOaSALZ/DFxM2smbTaom1FJpt8IVlKRfk6ZySwMn2p6SZ0YnktKhPA/sYfthpcT655ByKV1YtCxBEDTBMFTQCs/2PsO8blKSxLboxhLvANuTgMnA55Sq7i5Hyru9CSnlxKdz3xOBH9neiCbWfEkHSpouafqjjz7aBZGDYDBpw8+pJ3RDQX1OKdf3TaSZ1NqkVB6VxGYzSEnjIGUhrFS5OKvRgLan2J5se/LKK494VzUIgioMDNktHb2i0CWeUmminYFtnJLIX01a6r3shWp4fs19I1o5CHpEL2dHrVD0DGpF4ImsnNYjubQ34wZg73xeL2F/EARdwvZo7uKNiKIV1KXAeKXk/ceTlnnN+DzwmZwUbZWCZQmCYBjKboPqu3xQ7YQItPu7hed50GfM6MS7e7PNN/dV17eW1PM1yy3f0b1GSvhBBcGAkozkvZaiOaGggmCAKfsKKhRUEAwq2UheZkJBBcGAYmIGtQBJywG/JEUxjwO+CswDvp3lmAYc7Pr13oIg6AK9dMJshdHMZrAb8JDtTWxvSHJJ+AmwVw51GQ8cPIryBMHAU3Y3g9FUUDOBXSSdIGl7UrjL/bb/lF//X1Je48WojsUbHVGDYBBwyz+9YtSWeLb/lCs5vAf4GnBlG9dOAaZA+6lSgyCojwvMZtAtRtMG9Wbgcds/lfQkcCgwUdJbbc8GPgp0Ui47CII2GYpdvAVsBHxL0hDwMsnetCJwrqSKkfzHoyhPEAw0lWwGZWY0l3iXAZfVeWmz0ZIhCIJFCTeDHtJubF07b1bE7QV9T49zPbXCmFZQQRA0J2ZQQRCUEgPzQ0EFQVBWYgYVBEFpCQUVBEEpcRjJi0HSgcCBvZYjCMYaMYMqgAh1CYLuEAoqCIJSknbxyh3qMprZDFpC0hWSosJLEIwCQ27t6BWlmkFJWgJ4K/B4r2UJgjFPj3M9tUKpFBSwAXC+7ed7LUgQjHUi5W+b2J4FHN5rOYJgUAg3gyAISkvZZ1CFGMklrSTpkHy+g6SL2rx+/5zQLgiCUcK57FQrR68oahdvJeCQDq7fHwgFFQSjzKDkJD8eWEvS7aRsmc9KOg/YEJgB7Gfbko4B3g8sA9wIHAR8GJgMnC3peWCbMJIHwehQ9pzkRc2gjgLus70pcCQpS+ZhpF25NYFtc78f2N4il51aBnif7fOA6cC+tjetp5yiqksQFE9lF28Qy07dYnuu7SHgdlKJKYAdJd0saSawE/C2VgazPcX2ZNuTuyNuEAwmRSooSbtJulfSbElH1Xl9dUlXSbpN0p2S3jPcmN3axauuDjwfGC9paeCHwGTbD0g6Fli6S/cPgmA4spG8CCSNA04GdgHmAtMkTbV9d1W3o4Ff2v6RpA2Ai1k4ealLUTOop4EVhulTUUbzJC0P7Nnm9UEQFEjBS7wtgdm259h+Cfg5sEedW746n68IPDTcoIXMoGw/JukGSbOA54GH6/R5UtKpwCzgH6QyUxV+Avw4jORBMLq04ag5ocYGPCVnGamwCvBA1fO5wFY1YxwLXC7ps8BywM7D3bSwJZ7tf2vQfmjV+dGkaV5tn/OB84uSJQiC1mjDhWBeATbgfYCf2P4fSdsAZ0naMNuq6xKe5EEwwBS4QfcgsFrV81VzWzWfBHZL9/Ufsl16AvBIo0FLl24lCILRoVJZuJWjBaYBa0taQ9JSwN7A1Jo+fwPeBSBpfZJd+tFmg8YMKggGlQJ38Wy/IulQUvXwccAZtu+SdBww3fZU4AvAqZL+g6Qf9/cwFvhRV1DZveAZ29+uaZ8IXJSdOIMg6DJFp1uxfTHJdaC67Ziq87tZ6LTdEjGDCoIBpu+zGUg6UtLn8vl3JV2Zz3eSdLakfSTNlDRL0glV1z1Tdb6npJ/UGXuSpDsk3QF8pohfKAiC1inQBtUVWjGSXwdsn88nA8tLWjK3/Qk4gRS2simwhaQPtHH/M4HP2t6kWaeIxQuCbtBqLoNyK6gZwCRJryaFsPyBpKi2B54Errb9qO1XgLOBd7RyY0krASvZvjY3ndWob8TiBUHx2K0fvWJYG5TtlyXdT8rZdCNwJ7AjqbjBX4BJjS6tOo+YuyAoIb1MRtcKrfpBXQccAVybz/8duA24BXinpAk5WHAf4Jp8zcOS1s+VWj5YO6DtJ4EnJW2Xm/Yd+a8RBEG7FOwH1RXaUVBvAv5g+2HgBeA6238n5YK6CrgDmGH7wnzNUcBFpFnX3xuM+wng5JzoTiP7FYIgGCllzwfVkpuB7SuAJauer1N1/jPgZ3WuOQ84r077sVXnM4BqA/kXW5GnW0it68h237R2xg6CUSHq4gVBUGpCQQVBUFaG5pdbQRUaLFxdfioIgnKTXAjKbYMqOptBp+WngiAYRQZNQS0oPyXpTEm7A0j6laQz8vkBkr6ezw/PITKzJB1WsCxBEDSlNeU0lhRUdfmpy1gYIrMKqQQVue1aSZNIbgZbAVsDn5a0Wb1BI9QlCLqDh9zS0Su6mbDuOmD7XL3hbpLj5puAbUi+UdsBv7L9rO1ngAtYqNAWIUJdgqB4+sEG1bVdPNsP5ni73Uge6K8FPkLKBfV0+AUFQe/xGAl1aZXa8lE3kSoMV0JkjsiP5McPSFpW0nKkcJjrCIJg1Oj7YOF2qCk/dQlJ4exqe7akv5JmUdflvrfmHFG35MtPs31bkfIEQdAE99a+1AqFL/HqlJ86Pbe/TKqFVd33O8B3ipYhCILWiFCXIAhKSdE5ybtBKKggGGBCQQVBUE5sPH+AdvEiFi8I+ouy+0FFLF4QDDBldzOIWLwgGFAqRvJBmkFFLF4Q9At9EOoSsXhBMLCYoflDLR29ImLxgmCAKbubQcTiBcGAMnDZDCIWLwj6jJLPoCIWb4R0c4kaJa2C0cLl9tMMT/IgGGTKboMKBRUEg4rN0IAlrBsxkj4n6Y+Szu61LEEwCAyio2YnHALsYnvfXgsSBAOBiy2aIGk3SfdKmi3pqAZ9PiLpbkl3STpnuDF7ssSTdDhwQH56GrAesCZwiaQzbH+3F3IFwcBR0OxI0jjgZGAXYC4wTdJU23dX9Vkb+BKwre0nJL1+uHFHXUHVhLgIuBnYj+TQuaPteXWuORA4cDTlDIKxT6HLty2B2bbnAEj6ObAHKYqkwqeBk20/AWD7keEG7cUMakGIC4CkhiEuFWxPAabk/uXedgiCPmKo9ZzkE2piYafkz2WFVYAHqp7PJU1CqlkHQNINwDjgWNuXNrtp7OIFwYDibINqkXkFxMKOB9YGdgBWJSUN2Mj2k40u6IWRPEJcgqAkFLiL9yCwWtXzVXNbNXOBqbZftn0/8CeSwmrIqCso27cCPyGFuNxMhLgEQc8oUEFNA9aWtIakpYC9gak1fX5Nmj0haQJpyTen2aA9WeLVC3GxPbEXsgTB4FKckdz2K5IOJeWBGwecYfsuSccB021Pza/tKuluYD5wpO3Hmo0bNqgS8vL8+W31X2KJcS33HRpqb+xgDONiQ11sXwxcXNN2TNW5gcPz0RKhoIJgQDHg+eXeFG/LBlVdtUXSDpIu6o5YQRCMBmMt1CWqtgTBWKFF5dRPCmpB1RbgW8Dyks6TdI+ks5UTE0maJOkaSTMkXZZzkSPpaknfzQUQ/ihpC0kXSPqzpK8V+6sFQTAcRcbidYN2bVBHARva3lTSDsCFwNuAh4AbgG0l3Qx8H9jD9qOS9gK+zsLYu5dsT5b0+Xz9JOBx4D5J3x3Oqh8EQXGM9XxQt9ieC5BnVROBJ4ENgd/lCdU44O9V11R8I2YCd9n+e75+DsnRazEFFbF4QVA8lXQrZaZTBfVi1fn8PJ5IimebYa4Zqrl+qJE8EYsXBF3AxmMsYV1t1ZZ63AusLGkbAElLSnrbSIQLgqC7eKi1o1e0NYOqqdryPPBwnT4vSdoTOEnSivke3wPuKkLgIAiKY8wt8epUbam0H1p1fjvwjjp9dqg6vxq4ut5rQRCMAgV7kneD8CQvIUuOaz10BSJ8JRgZg2AkD4KgbzFD88ttJA8FFQSDSh8s8XpS1UXSjb24bxAENditHT2iV/mg3t6L+wZBsCgln0D1bAb1TH7cIcfnLRbPFwRBd+mHwp1lsEFtRk08H3B9dYcIdQmCLtBe0YSeUIbKwrfYnmt7CKjE8y2C7Sm2JxdQVSIIggWYoaGhlo5eUYYZVL14viAIRoGy7+KFMgiCQSYUVBAEZaTNwp09oVduBsvnx6tZNB7v0AaXBEHQBUo+gYoZVBnppqdFuzaH8PoYy/TWhaAVQkEFwaBierpD1wpdcTOI8lRBUH5M+YsmdMsPKspTBUEfMKie5NXlqV4GnpV0HqmYwgxgP9uWNAn4DrA8MA/Yv1JEIQiCbtPbQOBW6NYM6ijgPtubAkeSwlkOAzYA1iSVp1qSVJ5qT9uTgDNI5amCIBgNPLgzqFpGUp5qARGLFwTdYWh+uWdQo6WgRlKeagFRdioIiqcfUv52a4kX5amCoOwM6hIvylMFQT8wwI6anZSnCoJgdBhYBRWUk3ZDV9r5B46wmP4jgoWDICgl/ZDNoAwZNYMg6BFFGskl7SbpXkmzJR3VpN+HJVnSsBlyR6Swvbk2AAAMPklEQVSgJE3MBvCRXBuxeUFQClpTTq0oKEnjgJOBd5McsveRtEGdfisAnwdubkXCmEEFwaDiQoOFtwRm255j+yXg58Aedfp9FTgBeKGVQTtRUONzmag/5rJRy0p6l6TbJM2UdIakV8GCqd89km4FPpTblpD0Z0krVz2fXXkeBEH3aWMGNUHS9KqjNrJjFeCBqudzc9sCJG0OrGb7t63K14mCWhf4oe31gX8ChwM/AfayvRHJAH+wpKWBU4H3A5OANwLkKi4/BfbN4+0M3GH70dobSTqw8ofpQN4gCKposy7evEplpXxMaedekpYgJQb4QjvXdaKgHrB9Qz7/KfAu4H7bf8pt/0vycVovt//Z6Tf9adUYZwAfy+cHAGfWu1GUnQqCbmA8NNTS0QIPAqtVPV81t1VYgRR7e7WkvwBbA1OHM5R3oqBqF6ZPtj2A/QDwsKSdSGvYSzqQJwiCdjB4qLWjBaYBa0taQ9JSwN7A1AW3sp+yPcH2RNsTgZuA3W03XRV1oqBWr8TRAf8GTAcmSnprbvsocA1wT25fK7fvUzPOaaRZ1bm253cgTxAEbVLULp7tV4BDgcuAPwK/tH2XpOMk7T5S+Tpx1LwX+IykM4C7gc+RtOK5ksaTNOqPbb+YDWq/lfQccB2LBhJPJS3t6i7vgiDoHkWGuti+GLi4pu2YBn13aGXMESko238h2ZZquYKUnK62/6UN+gNsQjKO3zMSWYIgGBn9kG6lp6Eu2dv0YBbu5AUlo534unb/2SN2r8fYDM0fwKourWL7eNtvsX19L+UIgoHFbu3oEREsHAQDjBfbjC8XoaCCYECxwwYVBEFpMW7RyalX9IWCiqouQdAdYgZVAFHVJQi6w1BrYSw9oy8UVBAExZO8xMutoEqXD0rSFZJWGb5nEAQdE24GrZNTMrwVeLzXsgTBIBBuBu2xAXC+7ed7LUgQDAJhJG8D27NIie+CPqTd0JUNN2y9JOL/nPODtsb+l403bqv/YGKGhsqdQKRUCioIgtEjHDWDICg1A6OgJD1je/mixguCoPsMjIIKgqDf6K0LQSt0xQ9K0pGSpkm6U9L/y23LSfqtpDskzZK0V24/XtLdue+3uyFPEAT1MUMtHb2i8BmUpF2BtUlFEESq3PAOYGXgIdvvzf1WlPQ64IPAerYtaaUGY0YsXhAUjF3+UJduzKB2zcdtwK2kVL9rAzOBXSSdIGl7208BT5EqjJ4u6UPAc/UGjLJTQdANiit93i26YYMS8A3bpyz2Qqos+h7ga5KusH2cpC1JNfX2JFWF2KkLMgVBUIeyx+J1Q0FdBnxV0tm2n8lxdS/nez1u+6eSngQ+JWl5YFnbF0u6AZjTBXmCIGjAwO3i2b5c0vrAH7Jn8TPAfqQYu29JGiIprINJ5acuzOXRRXiRB8GoMjAKqtoHyvaJwIk1Xe4jza5q2bIoGYIgaIMeZypohfCDCnrGrFnXttx314026qIkg4mBoZIX8w4FFQQDS2936FphxG4GklaSdEg+30HSRcWJFQTBaFB2N4NO/KBWAg4pSpAgCEafsiuoTpZ4xwNrSbqdtCv3rKTzgA2BGcB+2Tv8GOD9wDLAjcBBuf1q4GZgR5Ky+6Tt6zqQJwiCNkg28nL7QXUygzoKuM/2psCRwGbAYaSsmGsC2+Z+P7C9he0NSUrqfVVjjLe9Zb7uK41uJOlASdMlTe9A3iAIFsF4aKilo1cUGepyi+25Tir5dmBibt9R0s2SZpK8xN9Wdc0F+XFGVf/FiFCXIOgObvGnVxS5i/di1fl8YHx2wPwhMNn2A5KOBZauc838gmUJgqAFxuwuHvA0yRO8GRVlNC+HtezZwf2CICiUVBevlaNXjHjWYvsxSTdImgU8Dzxcp8+Tkk4FZgH/AKaNWNIgCAqlH3KSq+wC1hKlzweTdv9P260w06fM6MQuu+yyr/a667YWaXb77VcMey9Ju5FC3MYBp9k+vub1w4FPAa8AjwIH2P5rszHD7hP0Be0qnHYU2oAos7oUlbBO0jjgZGAXYC4wTdJU23dXdbuNZI9+TtLBwDeBvZqNW7rS50EQjBYGD7V2DM+WwGzbc2y/BPwc2GORu9lX2a4kpbwJWHW4QUNBBcEA04abwYSKL2I+alNwrwI8UPV8bm5rxCeBS4aTL5Z4QTCgtGkkn1eUH6Kk/YDJwDuH6xsKKggGmAI3yR4EVqt6vmpuWwRJOwP/CbzT9ou1r9fS8yWepF9LmiHprjrTxiAIukahflDTgLUlrSFpKWBvYGp1B0mbAacAu9t+pJVByzCDOsD245KWIVn+z7f9WHWHKDsVBN2hqF08269IOpSUNXcccIbtuyQdB0y3PRX4FrA8cG7eOf2b7d2bjVsGBfU5SR/M56uRSlQtoqBsTwGmQPhBBUFRFO2oafti4OKatmOqzndud8yeKihJOwA7A9tk34irWTRWLwiCrhE5yYdjReCJrJzWA7busTxBMFD0sqx5K/RaQV0K/LukPwL3kpy3giAYJcoe6tZTBZW3Gd/dSxmCYHBxYUbybtHrGVQQdIVuxteNlTi/fkj5GwoqCAaYWOIFQVBaQkG1iaTxtl/ptRxBMPYZcDcDSR8DjiBVWb4T+CVwNLAUyRlzX9sP51zla5GqwfwN2KebcgVBkOhlQYRW6JqCkvQ2kjJ6u+15kl5LUlRb57p4nwK+CHwhX7IBsJ3t5+uMFaEuQVAwNgwNze+1GE3p5gxqJ+Bc2/MAcrzdRsAvJL2JNIu6v6r/1HrKKV8boS5BUDi9rRrcCqOdzeD7pEKeGwEHsWhYy7OjLEsQDDxlL33eTQV1JfCvkl4HkJd4K7IwR8zHu3jvIAhaoOwKqmtLvJxq4evANZLmkxKmH0tKtfAESYGt0a37B0EwPGV31IyyU0HQJiXyJO+o7NRSS77KEyYMW7cAgL//Y05H9xoppfODCoKy047SKXM9PwNDJZ9BhYIKggGm7Eu8UFBBMLCU380gFFQQDDChoIIgKCVF5yTvBqGggmBgMR7gUJfCiFi8IOgOAxssXCQRixcE3SGWeEEQlJayK6ielz6vRtLFkt7cazmCYBBIcXaFlT7vCqWaQdl+T69lCIJBouwzqFIpqCAIRpcoOxUEQXmJGVQQBOXEpS993jUjuaS9Jf1nt8YPgqAzKp7kZU5YV5iCkrSUpOWqmt4NXNpi3yAIesCYV1CS1pf0P8C9wDq5TcCmwK2S3inp9nzcJmkF4DXAXZJOkbRFpzIEQTAyyq6gRmSDyrOfjwCfzE1nAsfafjo/3wy4I5eXOgL4jO0bJC0PvGD7aUnrAh8Evi5p5TzGT20/Xud+EeoSBIXj0pedGlHKX0n/JBXi/JTte+q8/mXgfts/k3QUSRGdDVxge26d/qsDPwB2Bda0/VCTe5d72yEIquhyRs2O0vBKS3j8+CVb6vvKKy/1JOXvSJd4e5Kqs1wg6RhJb6l5fVfgcgDbxwOfApYBbpC0XqWTpNdL+gLwG2Ac8G/AwyOUKQiCdrFbO3rEiBSU7ctt7wVsDzwFXCjp95ImSloRGG/7MQBJa9meafsEYBqwnqQVJf0auJZUG+89tt9r+wLb5Z5zBsGYwS3/tIKk3STdK2l2XjnVvv4qSb/Ir98saeJwY3bkB5WV0InAiZK2BOYDuwC/r+p2mKQdgSHgLuASklI6CbjKZfe1D4IxTFFxdpLGASeTPv9zgWmSptq+u6rbJ4EnbL9V0t7ACcBezcYtzFHT9i1Z0K8Ap1W1f7ZO9xdJdfGCIOghBYa6bAnMtj0HQNLPgT2AagW1B6k2JsB5wA8kqdkkpXBPctufKnrMGuYBf63TPiG/1grt9O3m2GWRI8bukhwNjN5F/Y61tt92uSyP3QpLS5pe9XxKztNWYRXggarnc4GtasZY0Mf2K5KeAl5Hk79F34W62F65Xruk6a3uMrTTt5tjl0WOGLu8coykf6vY3q3oMYumVPmggiDoWx4EVqt6vmpuq9tH0nhgReCxZoOGggqCoAimAWtLWkPSUsDewNSaPlOBj+fzPYErh9sk67slXhOmDN9lRH27OXZZ5IixyyvHSPqPOtmmdCjJrjUOOMP2XZKOA6bbngqcDpwlaTbwOEmJNWVEnuRBEASjQSzxgiAoLaGggiAoLaGggiAoLaGggiAoLaGggiAoLaGggiAoLaGggiAoLf8fyzUxuRXxjIIAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "eval_and_show(train, settings.MAX_LEN, SRC, TRG)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " < input: ich habe das privileg , diesen 20ern wie emma jeden tag mitzuteilen : die 30er sind nicht die neuen 20er , macht euch also auf , erlangt ein identitätskapital , nutzt eure schwachen bande sucht euch eure familie selbst aus .\n",
      " > target: it 's what i now have the privilege of saying to twentysomethings like emma every single day : thirty is not the new 20 , so claim your adulthood , get some identity capital , use your weak ties , pick your family .\n",
      " > predict: i 've been taking the privilege , which is the day that i met every day : every new new , so you listen to a new new , not so much , your own friends .\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAV0AAAFGCAYAAADJmUzvAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzsnXecXGXVx7+/hBBKAhEJvYQSgUgJEJAu/aULAoIgShFFKfoKKAoiUl4LVlC6UhSUIk0MRREInTSSECCABJUiVSAFQsie94/zTPbu3Tu7s8nszt2d893P/ezMM8997nPvzJw59zynyMwIgiAIeoZ+jZ5AEARBMxFCNwiCoAcJoRsEQdCDhNANgiDoQULoBkEQ9CAhdIMgCHqQELpBEAQ9SAjdIAiCHiSEbhAEQQ+ySKMnEAR9GUnnA1XDPs3shB6cTlACQugGQfcyrtETCMqFIvdCEARBzxGabhD0AJKGAt8CRgCLVdrNbMeGTSpoCLGQFgQ9w9XAU8AawPeBF4CxjZxQ0BjCvBAEPYCk8Wa2qaTJZrZhahtrZps1em5BzxLmhSDoGeam/69I2hN4GVimgfMJGkQI3SDoGc6WtDRwInA+sBTw9cZOKWgEIXSDoGf4r5m9A7wD7AAgaevGTiloBGHTDYIeQNIEM9uks7ag7xOabhB0I5K2BLYChkr6RualpYD+jZlV0EhC6AZB97IoMAj/rg3OtL8LHNCQGQUNJcwLQdADSFrdzP7Z6HkEjSeEbhB0I5J+YWZfl/RnChLfmNk+DZhW0EDCvBAE3cvv0v+fNHQWQWkITTcIeghJiwLr4hrvNDP7oMFTChpACN0g6AFSFNpFwD8A4TkYvmxmtzd0YkGPE0I3CHoASU8De5nZc+n5WsBfzGzdxs4s6Gkiy1gQ9AwzKgI38Twwo1GTCRpHaLpB0ANIuhBYHbgOt+keCPwL+BuAmd3YuNkFPUkI3SDoASRd3sHLZmZH9thkgoYSQjcIgqAHCT/dIOgBJC0GHAV8nLblekLDbTJiIS0IeobfASsA/wPcB6xCLKQ1JWFeCIIeQNJEM9u4Uq5H0gDgfjPbotFzC3qWMC80AZJWxlfO57/fZjamcTNqSirlet6WtD7wH2C5Bs4naBAhdPs4kn4EHAQ8CcxLzQaE0O1ZLpH0EeA04FY83ePpCztoqj5xBq0/qsK9IdZc2LGD7iHMC30cSdOADc1sTqPn0sxIGph/DyQtY2ZvLeS4TwP/C4yn9UcVM3tzYcYNuo9YSOv7PA8MaPQkAm5MdlwAJK0A/LUO475jZreb2Wtm9mZlq8O4QTcR5oW+z2zgcUl3A/M1LTM7oXFTakpuBq6TdACwKm5iOKkO494j6VzgRtq+vxPqMHbQDYR5oY8j6QtF7WZ2ZU/PpdmRdCywGzAMzzD2UB3GvKeg2cxsx4UdO+geQug2AZIWB1Yzs2mNnkuzkStGKeDzwGRgIoCZ/awR8woaR9h0+ziS9gYeB+5Iz0dKurWxs2oqBme2QbgZ4LlM20IhaXlJv5F0e3o+QtJRCztu0H2EptvHkTQe2BG418w2Tm1PmNn6jZ1Z8yKpHzDIzN6tw1i3A5cDp5rZRpIWASaa2QYLO3bQPYSm2/eZa2bv5NpaGjKTJkbSNZKWkrQk8ATwpKST6zD0smZ2Hek9NbMPybiOBeUjhG7fZ6qkQ4D+koZLOh9Y6AWcoMuMSJrtvsDteLmew+ow7ixJHyVVGpa0BZD/kQ1KRAjdvs/xeGarOcAfgHeBrzd0Rs3JgOSnuy9wq5nN7WyHGvkG7n62lqQHgavw9zwoKWHTDYIeQNIJwLeAScCewGrA781s2zqMvQiwDu4dMa2OAj3oBkLo9lEk/Zl0y1mEme3Tg9NpeiStYWbTM88FrG1mz9Zh7K1w399sQqOrFnbcoHsIodtHkfTJjl43s/t6ai4BSJpgZpvk2sab2aYLOe7vgLVwt8D5CY0i4rC8RBhwHyWEajmQtC5uU19a0qczLy1FpoLEQjAKX6QL7amXEEK3jyLpOjP7jKQptDUzVFL/bdigqTUb6wB7AUOAvTPtM4Cj6zD+E3hFilfqMFbQA4R5oY8iaUUze0XS6kWvm9k/e3pOzYykLc3s4TqOV7HZDwZGAo/RNuFN2OxLSgjdJiAJ3uFm9reUh2ERM4v6XD1IvQtThs2+9xLmhS6QS15S4R1gvJk93tPzqQVJRwNfApbBF1xWAS4CdmrkvJoFSZeY2ZfwwpRP44UpzwQOBZ5a0HErQlXSGsArZvZ+er44sPzCzjvoPiI4omuMAo4BVk7bl/FUfZdK+mYjJ9YBxwJb40ERJBelqM21kKQySJ22AZUsYmub2XeBWSmt5p7AJ+owletpG9Y9L7UFJSWEbtdYBdjEzE40sxOBTXEBth1weCMn1gFzzOyDypPkSB82pYVnl4K23Qvazkz/84Upl6Y+P36LZN/f9HjROowbdBMhdLvGcmQWK/Av0vJm9l6uvUzcJ+k7wOKSdsG1oD83eE69FklfSR4h60ianNmm43ly83wt/f9NrjDlk0CRZtxVXpc0f9FM0qeAN+owblXkfE7S6en5apI2785j9iViIa0LSPousB9wS2raG/8C/RS4xMwOzfVveOnzlEbwKGBX3F3sTjO7tCfn0JeQtDTwEeAHwCmZl2Z0VGQyH5FWrW0B5rMWcDWwEv7+/hv4vJk9tzDjdnLMC3GTxo5mtl76MbnLzDbrrmP2JULodhFJo3AbKcCDZjauSr/C0uc97coj6Wtm9svO2no7krY2swc7a6vj8ZYpaJ5RLe9Bd0WkZcYaBGBmM+sxXifHmmBmm0iamMnRPMnMNuruY/cFwnuh6ywGvGtml0sa2oG2si+wzsKUPq+Tm9EXgLyAPbygrSFIGgjsT/vcAWdW26cK5wOb1NBWLybgBSb/i2uYQ4D/SHoVONrMxkOPRKQhac90jMU8pQMAPwFOxMs0HS1pOP55vK0Oh5wrqT+t6SSHEjmaayaEbheQ9D3cg2EdPFv/AOD3tGq+WSqlzxfG1rvAbkaSPgscAqyRK88zGKh6G9wAbiG53bEA10rSlsBWwNCcS99SQP+6zLCYvwI3mNmdaR674j8elwMX0OqZ0K0RaZIuApYAdgAuAw7AAyUux6/plqnrS7g9vx5C9zzgJmA5SeekY54m6UAza+M5UdTW7IR5oQtIehzYGJiQua2aXBRSK+lPwEZAu9LntWaFqty+VY6R8rHeb2Zb1DDX1fFE2e1sj8DkVGGg4Sxs6aAUJLA97sp3UealGcCf65HFq8pxp+RL4mTep8fNbGTutbpGpBUcs/J/EJ4kfXEzG9VdJoCkwe+Ea/l3m9lTVUwo7dqandB0u8YHZmaSKrdVS3bQ99a0taFaVig8+XSevJvRf4Bhkiq+tqI1l8JS2R1TmO8/adV0yspDkjYwsykLsnMKErhP0hU9HNr8iqRvAX9Mzw8CXk233fNvtSV908x+DByS7j7aUIdsYO+l/7MlrQS8CawIvJYCJSqf1bWAOQuzoCdpKTN7N9mzX8OT4gPsJOl/gZUlnZfZZSmgFD/uZSKEbte4TtLFwJAU6XUkUOgJYGZXqqD0uaRTqD0r1CVpZfi7uAAflNr3MbNazQwzaPXLXRQ3eczKC+kGsg1weHK5msOCJ+QZKOkS2t9B7FivieY4BPgecHN6/mBq6w98JtOv8j4VLrjWgdskDQHOxe3Mhn8m78crQK8q6WrcBHY48Cfa27lvwH3OO+Ma3FQyPh1H6f9A/HP1XnqtwgzgfxfkpPoyYV7oIsnXNet+9dcq/fbGFzMWNbM1JI3E7bJzgBPMbIGyQkl60MyKbMi17CvgU8AWZnZKZ/17AtUpIY+kSbh5YTyZwoyVBa1G0xV754LaRtOi5GKWCpHKa6dtgX9W38CjKH8MZAtiLgWcbGYfX4DTyh9/QMV7IykLq5pZke9yU9P0QlfdlAxGVUqfA69TY1aopMF8nrba23bANFzDyu5/YxfmNt/O1x1I+p2ZHdZZW+a1bfD34PK0Ej6oq/6r9XS/qvF4HwNOokbNuiv2zi72fQC4D9dsHwSGV5lyxfa9DW3NXjOAP5pZp8VKJXVmm/0ZsA9+PcbjJoiHzCy03QxNbV5QjclgcrfobV6iwJ6amGtm72RceMBtfWd0YYqjgUeAKbTaCUcCs3Ftu4IBhUI356bUD/e+eL8Lc1gQ2mhNyc5ZKBC76BFStH/FX/bPkr6Kr6pXfowWxX/4hrFw7mhFXI9/Vi6jg5LnknYH9qAGe2dX+mY4DNgW95w4Fw/GeQd4Eb+uk/DP6Ya4iWOv/IKepIHyitHD6Pg6/bTaeeKfwaWTzfeLwFVm9j1JoenmaGqhiyeD2Rx4FDwZjKR28fBmNngBxm5T+hw4AXgYOMPMdqhxjMXMLJ/Z7MouziPrpvQh8AJuYqiJJDCvykfbVen7baAScvxupRn4ALikym77kTxCAMzsZUmDJd1T43XK2heh7a3z8njI8wK5o3XCh2Z2YQ39XsaF3T50bu/ssG/RGoGZTZf0Pn6NP8Bdx/6F/zBvUlmgTAuxZwDPycPCh9H6/d8V15LbXafsMTt7PyRNkbQibtM+taO+zUxTmxckPWpmn8i4Zi2Cu4NtmOuXXbVthxWEf0paAv/gzbf/AmcBfwE+XbG7dTK//wVm4r6VlS/DWsAP8ZwP60vaEF9YO7vKGKua2b9zbSuY2X86O36m/wN4yOcHnXb2/j8ws2/X2PcxM9tcrVFOS+I/Tq9T43XqYOyFckfrZOwz8NvnrGZd+FlI/efbO2sY+zDg5qyZS9KZuIdEfo3g47i99hrcxPC4mbVImpq300qaimvB99PW9v1zM1utYB75dYmj8R/xK6pMvT++6PugmX1F0prAuWa2fy3n3Sw0u9D9MfA2bjc9Hvgq8KSZnZrrd5uZ7ZVW2POYma3ZyXH6A0smwX0Lrtn9FZiVGaSd65CkY4Fz0hwrb9QKuH3u4qytuJpwkTQXX50+0jwxT5d9JyVdBayH2wKzc/5ZB/vUlHdC0km4HXIX3Kf4SFyA7EyN1ymNcyxwtZm9nZ5/BNdyv7Kg7mgd0dXPgqStcU2zck1Urb+kt/E7ks9WvFQkzcZdwbJrBFNw88Y2eHTc07h9dwxwNn7dfp+GPRT3flnP2vsQXwKcn79O+XUJSd8HPk2xJ4bZAiZkbzaaXei2SwYDXFajO1dnY1+DO+zPA8bidrlf4tpRO8xzrObHeB7Y3MzeyLSNNbPN1NbpvZ0zfqb/RNyF6IvAgWb2j64upCW7a5bv4D8C1TTfC4CDqTHvRJFHiKQvFPUtuk5pjPnXQK114YbjAu55Mu5ouH35K/iiJLiguqhWTXRBkPQ0bk7Ie1e8WdB3Iv65/B1ujrpe0iwzWzL3vs8PzJEHRRyBL+6tAiyZO8cxwIV4lrOHzGx05jotgl+r/HWabWZbVDtmwbzXxD/jW6T9Hwb+18yeX6CL1kdpaqELIGlRYF38QzKto1toeTjtH4BbzGx2J+M+bmYjJR2K+0WegleY2LDINldljLuAfbPHknQ7cBxwfbodPwA4ysyKcrnO12qTpnUp8C3g+13RdDNjLZGby1l4QcTf4V/UQ3Ft7CBgQ1uIvBNp/JquU+o7JR3T5B4p/fAf0aK8t9/FF+wqAvwwYJ6ZfbGL81sfGEHbvBhFQS7zTVk1jlt5z5bFP2+T8DuA4/DP0f74GsEAXJvdBtdiHwIewKMWn8+NWVkMFi6Q5+A2/oqALbpTOh2PqMwf8xTae9WAr4/8mtagiYOB42s976bBzJp2w7P3/xu4F9d2/gXsnuvzEfzLDH5bfwEe6XUDHnO+WJWxp+If0OuBT6a2SbhNbBowPbWNBG6tMsZNwDPAxXi8+3m4Pe1v+ELJS/iXbFgH5zgx83hF3J43u4vXaUtca/1Xer5Rug6TCvpOwsNQB9U49gy8qsW7uFfFvPS45uuUXj8XuA73PNkpPb4GGJzpsxSeE6Fw3l28Jt8D7gFexb0u/oPnYqjW/4dpjlviP8Kb4AtdRX3/knncL+03Dzc1jU3b2biwPwC37+fH2Bo3zTyDa7DPA89XOd4WVa7TElWO+RDuHnYEnlCpsk3u6LqS+S4189bwCTT05N0Gtnbm+Vqp7d70wVsGmI57N/ws068/rkFdh2ccKxr7hCQUR+PaxOq0LmAsTVth+ATwOeD09Hw1XGv4QtGW+iyZ/aJ0cI5b5Z4vAmzXxev0KG4zzM/5IVy77U+rZvkKHvX0HG1/LM6r4TjCs7P9sNp16mDffvjt9A1p+zIwkXQ3l+kzIW1rZdoPTG0Du3BNpqTxJqXnywN/7aD/PQXb36v0PTD3vD8d/+CsjCf92S6zPY1XslgO+Ghm2w937arsOwQXyEXX6SdVjjch93yZtP0I14KHpc/7N3EFpep3qRm3pjYvVOyjmefCgxYWMV84+CIeVfM9tSYUWRzXwg7CtZXbzOz4Go+3CPCAtbeTvYnH8FdNCq3iopid8Q5wkpmtl5tHlwIJ8l4eqW0S7nr2S1yrMvxO4RqqVC6wKvbYguNNxMsM1WxPrDJOUeKZybht9XKSsME1vd2An1qNZpeM18V43E1rBvCUma1b6/w6GLsoOGKWmbXL9SHphxTYz3Htt91tfZVrMtvMlsi1TSbZdAvGyHvVTEzHLBImK5nZwKLvUrXz7+s0pZ+uWgMGxkkajWushms8Y4Fti/wNJV2Ha6B3AL8C7jOzwjyikpYH/g//0O0uaQR+a1nkv9tiZscmYYOZ/VfSopL2wt3MVsdv9YRrIS/QGlW0N/5DUQl8qJTi+TSuda6RvA8qOQKq5nHtYM7/lmdGM3mms6/hAuYFCnx+k9vX+2Y2Lz3vj+cA+E3B2P/N7JoN3niy4DpVjZpKfX5AWxvr8pJOwBeQwL1Tnjezu1P/dVL7pcBnaR+UAFT1mBgnjxi8FNfKZwIPq0p+YFzrb3dtzew3mXPoKDhiZlpTuJ6MNweuubbL2yzph5LOxYNmsq8VleiaW3Sd8KQ+Rcf8ADd5nIp/b1qo7okRvrs5mkbTzbnsrJh56YaC7qPxxZYHzOyraVX2XNzB/28VYdLJ8W7HtalTzWyjpOVOxG1lef/dXXABNNZ8AWUocBee+/bTwBRLb5SkMcCelnw4JQ3GfX8B9rBUOUDSQbgWOgC/Pb0p9aka9tnBnHdIY+2c5nwXLnj74Tlhh9FWwIwAds7MZRBuO/9ywdhZ96NK8MaluBBr5+dsqdR4wdwfwO2sP8d/iI7AF5dWwd2eDF8U+jpum7/DzGZIOg3XdB/AzROn58fuTEOXNAxYyswmS7qD1vzA2c/JzhRcW8ukh5S0EW67PjM3jxm4IM8v8hr+WT7QchUjJN1TMFXDr+/b+IIXeIDQSulx/joV1XAzPJx4vleN2kY95tkSzwfd5rtkTey720xCt2aXnYJ9dzSzv1f7cFlB3gN1wbUreThUzBVXkpJC4xrHTlltWtI0Mp4BSbOajH8ZNrDWhCMD8UWtI4ALrAYXsa7MOb32EO0d7QG+W+0Wttaxu0rFZKJMnttqZpSMqWgb/E7iJ7iQ+5KZTerkOJ2ZH66yAp/pLn4eFrFO8h1LOh9/z1emSt7mKvstiSsUO6emvwJnm9msov5VxmjjVSPp8g66m4X/bhuaybzwjpndnm2QtJjcqb5NORxc2GXL5Gycbv3bJFJIGMV5D2bJszxVNNSdgGWS8F8utb+GV06oLBxVkkLva54U+gVgtKT7aP1C/QN4TFJFc90XF9T9gUflwRfgfrmX4q49i+dvm6t8KfNz3hpYImlvFW3opTTn3wBLmNm38oNI+oakTcxsQnq+KX4Lmx17C+AdeeKYC2mNsNsqXY/liq6TpeCHAubI/a6flXRcmufakn5b0LfyA7EncKmZ/UXS2cDXlXIl565VVmh0ln+gWn7g/LXdAteIi3i2YB5D8dv8Cmun/7+hIG9zOsb8Mj6ZczmTtkntkXR50XnT6k6WZxbweNKm5+CaeOFnKo3d7j1oZkHc5zXdjGbyGVwwZW1cP8ITyhxC23I4K+Grv/n2B83sui4c93zc//EJ/LbxEuBHlkJwJa2Na7W7pv95/ojfZmcT3oDbbbdNj8eY2cQ03mb4KjZAPzP7uTzIoEiQFFWqyM95A9z96+d4AhXw2/Uv4KvRz5Ic7XPjbJbm/jL+xV0Bv/X/ambsoemcz8fzJVycFi/vxDW3kZnrtAKeC3ZHM8sm+skf8yl8Nf4s3A46Ns0RXPDsl+a0Ji6Ud8HvLt7D7eLZBC/z++eFSRLuW1pB0UtJT+ICcTptAw0Oz13bocABVpD6MAnn7DwOxN/XawvO5du0t58PxF26isr4rEn7z8NQWhMxZce+r+h60DYvRJY7aG+3Phb4e36Mapp4M9AMQrfItlVhlJkNVq4cDu461K5MDu7VMKoLx14EX6wR8CczWyf3eiW8+ANahVpFmzZ89bjoVrVqKkR5wp6s1r48HkE2jNY7G6u2epyb841m9rEq/Z7B7YkVR/u5aR8zs6XSNauc7zQzm5sbu9LW5rY7mU/eq3LbPS1/DTOvjcJtwKvjdux255mE5QP4rfVuuK382bTQs4GZ3ZUbsx9ui9yKHKoS1acO8gMXnX9R3yrn18ZUkjmXfrS3n9+FfyaKyvh8PTPsYrit+EMz+2Zn593R9UivF64J5OzWHY7RDPR584J1kBlJ0mPpYbYcznK0ujzl26+T5wq4lrb5AN7KjFnN/vue3IvgZDN7NbUdJS/58ia+8n6/mT2dGevHknbNCgNVSYUoL/n+U1xLfw339X0af49Ppr22nL0O1eZs8hXwb1XsyulLcyDw37xArjLO+sATkj5ByuaW+Jg87eUb8lIylV//9/CqtstXrpPcq+JwfDGuGld3dp74dVsBFzb3prGXwX80ivIJDMff9yLulrQ//sNkSkmRSLfaGfZKx8mXRv+YpGrrAVm7ccWjI/9drcxtZnYRzcxmypMtFZbxsfZJ3R/MfA/yY+cZjhejLPIUAXjTzK6TZ5vDzD6UlF907uiaNgV9XuhWkPR/wI+tbUKU19L/02gth/NdYF6V9lNw4fDV3PBZV5nt8NupvWl7GzcVNzHcl4SI4dFMt+Kr+iOB85MAmoBr1l8BTkyacEWTXAJ/39qkQsRvqbfAvSs2lrQDHnCxjpkV2vxqmPNkXBP6gjwJC/jt+zjgYLVfVDoo9aukkzQ8ac0auC/v0Ezfym33sbjZZV1JL+G+vg9XuU7ZMjh5Xs+fp6QZOVvl4rjwqQieymvCf2CGZtoqx/0mxXwZX5idJ0+tuIQ8Kc2buXGXTY/H4CaCyq32DrgLXNF6wE8zY1Q8OtZSa7pMcEXgm/jnI2s/H4UL3NFyl7YfZ873MrXNlFcR6KOKxlb7PNKVY15Oq6fIDvhibT9guwK79fqZsTu7pk1BnzcvVCi6HZQvjp2L33oPwN1bDF/Rnd8t/Td8weereKy74YLxIkvZu9KYJ9K2flR2/+XwEN5HstqJpN3SMTfDP8TH4F+cEbg9eQ0zO1PSasDtZvZxtU+F+IF59ddJwMbm6f0mAd/AfVDzq9s3Zo7f0ZxXwwMe/oHnqNgS96jYPWe6yX6Q9qDVV3XbTPtdubHBXZcOSH2XwUOAP4oL2XbXyczuoAD5QmX+PCs/Utnrt4KZPZaEz3DaLjLdV9BuVpwdrR/t35sVcTtyUftZeDThK2n/FYErzOx/Csau3PYPo61i9Hz+XHCt/lrc1gqtuS+exH+0t6X1s3ohbveuvMdzcYF+VjpWfuxxBeeyAnChFXiK4O6D7ezWuOms02vaLDST0J0MbGatrlaL47fho2l1eaokR3kLF4D5AIRFcaFwdWo/BA+pnK+BqTUj1zppjFvwD/hh+ALHPbhW+zUzuyXt8y7+JXkY/3I8YGavSboQ/1JlI9UmpzlXUiEehQvFfXBPhh/g2tVruBYzHReWU2m97bbs6nEHc64swv0T/1HYHL8t3wX3m30W93d9V9J38UWps/DFlLdxbbxSvmY53O+4Mnblmi6b6TsP/0HbGtcC89epakpKSb8vOM8tcM2yTaQfHqjwNXxR8PHU7yE8UUu+/WErKMFT5b25CxdURe2DLBMZmIT2VMtFC6bX7shdE3AhPLFg3B+n92I13Kf7E7Telc2gNbXjIXhY9Q20f88+xD8vtZ7LXPx9uiFd35dwj5PT0lxWTfP9BP7d2q+Wa9o0WAlikXtiw7NrPYALqaPS4/9U6TuGtglABqe2Jwv6tmurMsZUXJiCaxXjcIEC/oEfgwu2M3BBtTgpxp22+Qcm4ULv3LTtnNp/it/iLYILyxNwd6JpXbhGRXMeg2uL7+IBAKS5TSYlOMG/gPfgLliPUpAjoYNr+kSu3xRSspyC6zSxg7m3O88Ort8UXOt6PLWti9/mF7ZXOV61sau1/woXSIen7XY8h23R2EXXr9q41d6Dws9qlf6zajkXPJvcJNw8UAk8uTxduy2qjD271mvaLFtRSGCfxMx+hGdMWi9tZwG3StqgoPvytI3++SC1TUh2KgDS4lCbBRhJK8oDE/JjiGTTNA+f3R7YXdLPcBea7XBN5U38g/w27tvaH7c3PpBsjuvjCWWOSdtNkt7B0/4dY2YfmtmVZnYerrU+JHfdqYX8nOfi/rOzgX+YLxRhbk5pocDfFb8beKjgula7pvm+/SyZFAquUzs/6cz1LjrP+dcv9R2a5v2+pcg2SQPNFy/X6aC96HjVxi5sN7PjcA17o7RdYrmcHblzyV+/aser9h5U+6wW9aeWc8HvdCpmlQG4QD0R9wl/psrY1tk1bTaaxryQRZ0nb74WX7TJBiBciwcarIMv9oDf0k3Db8/M3DXnb3i2shdxX9HKGCcCl1vGP1HuUvNbfMHrejy59gu4ieF+Wu1zbSLVLFOKW9JXcDvzWvgX4pn00mC87tWm6bU2fqNprm3K9kj6AW6Prcz5ZNyv+ExJ/azVg2FpXJN5mWJ/1wG091Udii+i5K/pYbm+a+I/QvM9IzLX6VAz60+GzPVeKp1z9phL47e0+Ui/Q/DFn6/jdxX/TXOeU9RuZnsUHK+SYCY/9qKdvWfVyIw9OJ1PLefFT7ihAAAgAElEQVTyBdq+Bw+kMV6g9bNquCvd0/hnPf+ePYNrsJ2dyxfxH+KhaYzsGoCla5If+7V07KrXNHcNulRKqjfS54WupAfMbJvcSqxo/aC084M196nchFwAgqr4YGb3S8cUvgg2MDPG03gawHYfKEm/wm/dxlsu/FPSurRGqt1tqXxL5vWl8TylP8BNDK+nl2aY2VvV5pzO8S9mtmdmrL/g9sDKnB82s7w7EfLk2ivii2vt/F3xH6IiPkrn13QFPKF4OxcuSVtbcUCC8Gv0bP413BTS0fX7JC7Q7rBMAvtq7ZnjjcCFbruxc+/ZEeaLTnlPgMJK0l09F7l7WPY9GIXfwt9fsD/456PoPftXDedSOeaFZvaV/MAFc2nj/9zRNc2M0eYz2Rfp80I3CIKgTDSNTTcIgqAMNKXQlfSlWtq62l6PMXrr2DG/8o7dW+dXBiT9VtJrkp6o8roknSfpOUmT1XkWuuZxGctuwLha2rraXo8xeuvYMb/yjt1b51eGDY/W3IQqpaLwhefbYX4Fkkc7G7MpNd0gCIJaMI+ce6uDLp/CcyibmT0CDEkLiFXpE7kXJD1knWQtWnbZZW3YsGEArLbaaowaNcoAxo9vzf+h4pyiXWqvxxi9deyYX3nH7k3zM7OivNU1s9tuu9kbbxSW6WvH+PHjp+LloSpcYmaXdOFwK9M2EdOLqe2Vajv0CaHbmcAFGDZsGOPGtU8k5R46QRD0Fd54443C73oRkt63LqRrrQd9QuhKmmlmgxo9jyAIykGyt/YEL+G5Jiqsktqq0qdtupK+JGmcpHGvv/565zsEQdDrMWBeS0tNWx24Ffh88mLYAi8LVtW0AH1E061Gss1cAsy34QZB0NcxrLC0W9eR9Ac8/8eykl7E8wgPADCzi/CMf3sAz+G5KI7obMxSCl1JL5jZsEbPIwiCXohBS51ULDP7bCevG56Iv2ZKKXS7g/HjxxcumhXZfmJxLQh6Nz1o0+0yZbXpvg4g6Y/yMtKk51dIOkBSf0nnShorT07eND8eQRB0jAEtZjVtjaCUQtfMNksPKykWkbQonu3oL3gS8ndSv83wwodrNGKuQRCUjy5EnPU4ZdcQbwd+KU/svBueDvA9SbsCG0o6IPVbGs+LOz27c4rpLm1cdxAE9cfM6uWZ0C2UWuia2fuS7gX+B0+m/Mf0koDjzezOTvaf771QLTomCIK+R9h0E5KGVcvW0wHX4m4Y2wKVSrB3Al+RNCCN+zF5VdwFmVO7rUy3IkEQdB2r8a8RlFrTTdyFV1W4xVqzzV+GFy2cIHc1eB0v/xIEQZPjC2mNnkV1GrGQtoikqyU9JekGSUtI2lTSfZLGS7qzkqVH0lrAn3Fb7dry0iHg9bIG4SWmlwB+bWbvNOBcgiAoIWVeSGuE0F0HuMDM1sPLeh8LnA8cYGab4gL1nNT3Etx2uylwEnBBZpwV8VLPewE/7KG5B0FQdtJCWg+FAXeZRpgX/m2txQV/D3wHLw751xSU0B94RdIgYCvg+kywwsDMODebV6d9UtLyRQcK74UgaD6Mci+kNULo5q/GDGCqmW2ZbZS0FPC2mY2sMs6cbPfCA4X3QhA0JY0KfKiFupsXJA2R9NUOuqwmab/kxXAI8AgwVNKWkkZJ+pWkj5vZu8B0SQemcSVpo3rPNwiCvkez2XSHAB0J3WnA54G1gY+Q7LnAj4DfADvgZgWAQ4GjJE0CpuKlMYIgCDqgVoexvuMy9kNgLUmPA/cAG+LCdQBwmpmtK2kYMNzM9pe0JnA5bntdEjjJzC6VdAawGrAYMBj4hZmdl47xD+AcSV/HS2Wc0Q3nEQRBL8TqmGWsO+gOoXsKsL6ZjZS0CLCEmb0raVngEUm3VjpKWgePMjvczCZJ2j431rq45jsYmCbpQmAksD+wES7IJwDjKSAW0oKgOWlp4jBgAf8naTugBS/YVvE0GArcAnzazJ6ssv9fzGwOMEfSa2nfrfFAifeB9yX9udrBYyEtCJqPSpaxstLdfrqH4sJ10+SF8CpuLgB4B/gX7mtbYRCweno8LO1fYR69I4IuCIIG02wLaTNwcwB49q/XzGyupB1oFagAHwD74fWFDkltg3J9ingQ2FvSYsmXd6/6Td0pyscQORmCoJdQYy7dRmnDddcczexNSQ8ml7CxwLqSpgDjgKdzfWdJ2gsPjJhJWkxLi3BDcfPBDXjwxAppn7GSxgJv4drvrHqfQxAEvZsyK0PdcrtuZod03ov1U9+38UTkpCoQt6VFuO1xm+/XgZdxDXcVSS/hi2ir4wJ3Cp6B7Cd1Po0gCHohBsxrNqFbRx4zsxcBkvY7DHgb2BS3Bys9X7xo5/BeCILmpOk03TqSDfWtLKQJGJcPGy4ivBeCoDkps9Ct20KapDMl7dxJn9GShqTHMwu6ZBfhqjGNFDacxhkg6eMLMucgCPoe1gwLaZL6m9npnfUzsz06eT27CPce7mKW7/NBqo12nqSl8XP4BR4mHARB0Ls13VRi5+mCxOMvSPqRpAnAgZny6LtJuj6z//aSbkuPX0iRafljnJwpp/6sma2fKv0+KmmapAeAjwKVfWcAs4EPcS+G+xf2QgRB0Hcos59urZruOsBRZvagpN/SmtDmTTPbBEDSbqntb8AlkpY0s1m0LSjZDnll3+HA5ri99tYUwfYe1cN9LwGOMbNnJX0CT26+Y43nEgRBH8a9F3p/GHA+8fgJ6fG1+Y5m9qGkO/AAhhuAPYFvdjD2rmmbmJ4PwoXwYArCfWtIbj6f8F4IguakLyS8yZ9C5Xm1wIQ/Asfht/7jzGxGB2ML+IGZXdym0TOIFdGPjpObt04yvBeCoPkoeaRord4Lq1W8BfDE4w900v8+YBPgaDowLSTuBI5MGiySVpa0HFXCfRuZ3DzCg4Og/FTK9ZTVplur0J0GHCvpKTw37oUddTazecBtwO7pf0d97wKuAR5O4cI3AIPNbCxwKzAZuB2PPKtU/I3k5kEQVKXMLmPqTNqnhOO3mdn6PTGh3LEHmdlMSUsAY4AvmdmEBRyr265wtWuYsTkHQVAjZrZQX5z1NtjALr/pppr6bjl8+HgzG7Uwx+sqZY9Iu0TSCDwd5JULKnCDIGgeLJVgLyudCl0ze4GUnKanqTFxTlXCeyEImpNG1T+rhbJrugtFeC8EQXNSZpex7q4cUZUquReQdIykz3ew3/wIt0zb3ZJWrvcca6UrXg1BEHQv9fZeSFG20yQ9J+mUgtdXk3SPpImSJkvqMN1B6TRdM7uoK/0l9cPLub/VPTMKgqC3US8FR1J/4NfALsCLwFhJt+bqOp4GXGdmF6Y1qNF4GtpCuk3TTfkUTkiPfy7p7+nxjpKuTo/PkTRJ0iOSlk9tZ0g6KT1eW9LfUp8JktZKww9KOSCext3K/mRm73XXuQRB0ItIC2m1bDWwOfCcmT1vZh/gcQd5F1UDlkqPl8aLLlSlO80L9+MVHQBG4YJyQGobAywJPGJmG6XnRxeMcTXw69RnK+CV1L4xXlFiBJ4I58aiCUj6kqRxksbV55SCICg7dTYvrAz8O/P8xdSW5Qzgc5JexLXc4zsasDuF7nhgU0lL4cnIH8aF77a4QP6A1sCJ8eTUcUmDgZXN7CYAM3vfzGanlx8zsxfNrAV4PL9vBTO7xMxG9bQfXhAEjaULwRHLVhSztC2It9NngSvMbBVgD+B3yexZSLfZdFMF4OnA4cBDeGTZDrj99SlgrrX+1HSlvPqvgBcyz6M0exAEbeiCy9gbnShlLwGrZp6vktqyHAXsBmBmD0taDE9D+1rRgN3tvXA/cBJuPrgfOAaYaDXo9SlJzouS9gWQNDBFph3XjfOtK5GnIQgag1ltWw2MBYZLWkPSosDB+DpSln8BOwFIWg8P5nq92oA9IXRXBB42s1eB9+lawvHDgBNScvOH8DLstwNIWlHSGDxf79mStq0+TBAEzYJRv9wLZvYhrujdid+hX2dmU+XlyfZJ3U4Ejk65YP4AHN6RYtlp7oWyIWmmmQ2SdCKwmJmdk9w6lugohWRZgiMiT0MQdMzC5l4YPmKE/ezqq2vqu88mm0TuhS4wFvht8oi42cwez3eIMOAgaD4q3gtlpWERaQuLmY0BtsON2lcURbGF90IQNCd9IZ9ujyBpiKSvpsftwn1zfVcHXjWzS4HL8KTpQRAEpc6nWzbzwhC86OUFNfTdHjhZ0lxgJlA1X0OZqGa7Hbjo4oXtu+/x5cL2zffYvF3bd760UEnZgqCPYJFlrAv8EFhL0uPAXGCWvLjl+ngAxefSItqmwJF40MUb+GrhK9UGDYKgeeiCO1hDKJvQPQVY38xGStoeuAX4OB7L/CCwtaRHgfOBT5nZ65IOAs7BhXAQBEHvTmLeYB4zsxcBkvY7DHgb13z/mm7V+9Oak6EN4b0QBM1HxU+3rJRd6M7JPK6E+wqYamZbFu/SSiQxD4LmpMwuY2UTujOAwZLOAJav0mcaMFTSlinOeQDwMTOb2lOT7A7mfFCcmfLmm39Rpb19WwReBAFQ8tD6UgldM3tT0oO4B8McYFJBnw8kHQCcJ2lp/Bx+gZdiD4IgKPVKWmmErqRTgS/gmXnuxr0V3pE0FlgUeA64LqV8vAnXbuem1JGTJF1hZnMbNP0gCEpEy7zyCt1SBEckF7CDgZF4PsrN0ks3mtlmKYn5U8BRKb/CvcCeqc/BqV87gRtJzIOg+XCXsYhI64xtgZvMbLaZvUtr6rT1Jd0vaQpwKO4+Bh6BdkR6fARwedGgEQYcBM1JmYVuacwLVbgC2NfMJkk6HI9Cw8welDQs+fL2N7MnGjbDIAhKRrkX0hqq6WZyLYwBDpU0Otls905dBgOvJA+FQ3O7X4Xbdgvro/U26vFLXK0UfBA0G9ZiNW2NoNHmhSHAV81sAnAPnjXsdjxtI8B3gUfxaLSnc/tejVfgfKRnphoEQW8gbLodk821sBEwDvgPnjFsY+AiM1sDL2C5BTBK0iVy9e0koAW4WNLjkoozxgRB0HRYS0tNWyNotNA9BfiHmY0ETqZtafU1ga1Tv18lL4b1gcXxhbZ9gQnAoWY20szaRReE90IQNCd1rJFWdxotdPNUK62+g6RHkxfDjsCDZrY2UBzGlQjvhSBoQqw2e26z2nTztMu1kMoZXwAcYGYbAJfi1TbnI+mhnptiEARlJ2y61ZmBeyh0REXAviFpEHBAfn8z26o7JteTVPM8KNOHJQh6A5UaaWUVug31063kWpD0BG4qeLWgz9uSLgWewBfZxmZevgK4SNLawKAiu24QBM1HmRWThgdHmFlhjRkzOy7z+DTgtII+fwL+lMqyh8ANgsBtuvMiiXlDiCTmQdCchKbbICKJeRA0JyWWub1b6EoaCazU6HkEQVAeKgtpZaVXC108FWSf9sGtljuh6EMVeRaCALByC91Gu4yRsoU9LekKSc9IulrSzsmr4VlJm0taUtJvJT0maaKkT0laFDgTOAh4LlUFDoKg6TFa5rXUtDWCsmi6awMH4mXUxwKHANsA+wDfAZ4E/m5mR0oaAjwG/A04HRiV9XQIgiAos6ZbFqE73cymAEiaCtxtZpbCfocBqwD7SDop9V8MWK2zQcN7IQiaDyu5eaEsQjcb/tuSed6Cz3EesL+ZTcvuJOkTHQ0a3gtB0KSUWOg23KZbI3cCx6eUjkjaOLXXEkbcJ4mQ4SCojrXUtjWC3iJ0zwIGAJOT+eGs1H4PMCLl042FtCAIgHLnXuh2oSvpc8nr4HFJF0s6VtK5mS7b49V9kfQ5PJfuaZIuBv6dcui+DszEzQ3XAh8CmNlb+ELbdDO7trvPJQiCXoAZLS0tNW2NoFuFrqT1cJeurVOi8nm48Nwv0+0g4I9V+lbqoi0JPGpeiv0sYF1JQ9NrRwC/rXL8SGIeBE1G2bOMdbemuxOwKTA2leTZCVgDeF7SFpI+CqyL10Ar6rtmGmce8CcA8yv1O+BzyX1sS7yuWjsiiXkQNCFW38KUknaTNE3Sc5JOqdLnM5KelDRV0jUdjdfd3gsCrjSzb7dplI4EPoMXm7wpuYfN7yvpDGCmmf0k7fK+mc3LDHE58GfgfeB6M/uwm88jCILeRJ20WEn9gV8DuwAv4krhrWb2ZKbPcODb+F36fyUt19GY3a3p3g0cUJmEpGUkrY6XTv8U8Fngj0V9gcVT33aY2cvAy3i6x8u7cf5BEPQ6ajMt1Ghe2Bx4zsyeN7MPcHn1qVyfo4Ffm9l/AczstY4G7FZN18yelHQacJekfsBc4Fgz+6ekp4ARZvZY6r4fbrt9Hl8om4kL3l3S/z8BhwH9gcm4wB0KvCRpOvAxM5vbnecTBEHvoKX2+mfL5tZ8Lkn+/RVWBv6def4ikI8P+BiApAdx+XSGmd1R7YDdHhyRvAraeRaY2V6Vx5I2BQ7Gw4EXwav8XgRcbmbfSX3OBo4ys/Ml3YuHCl+a9rsxBG4QBJAi0moXum/UYc1nEWA47om1CjBG0gZm9na1zmVgW9y2OxtA0q2pff0kbIcAg/AgCfDFs2WBT+PuZkcXDRphwEHQnNTRM+ElYNXM81VSW5YXce+qucB0Sc/gQngsBZQ9OOIK4DjzKsDfJxWpNLN18RPfEuhvZk8U7RzeC0HQnNTRpjsWGC5pjZTZ8GDg1lyfm3EtF0nL4uaG56sNWBahOwbYV9LikgYDe6f2wcArkgbQ6rNb4SrgGmIhLQiCNtRvIS15Rh2H32U/BVxnZlMlnSlpn9TtTuBNSU/iUbInm9mb1cYshXnBzCZIuhaYBLxGq1r+XeBRPCLtUdrmWbgaOBv4Qw9OtdR0JeF5R/2DoFdT5yxjZjYaGJ1rOz3z2IBvpK1TSiF0AczsHOCcgpcurLLLNsAN1YzVQRA0JwbYvPImeiqN0O0Kks4Hdgf2SM9HA19M/rtBEDQ5Zc6u1yuFrpkdn3u+R1G/8F4Igiak5ClNe6XQrZVIYh4EzUkX/HR7nD4tdIMgaE7KrOmWxWWsZlICinzbaEkrNWI+QRCUi2ZP7dhpEnNJh0v6VZW+/VP7TEk/lTQJOFXSzZn9dwHmxCJaEASA23RbWmraGkEkMQ+CoM9R5hpp3W3TzSYmB1gcD354XtIWwLO0JjE/tkpfyCUxl1RJYn45Hgr8+aKDx0JaEDQnZbbpljKJecE4kcQ8CILaqHNEWr0pbRLzTN92RBLzrlFUrr2rIcBlWogIgo4o+0JaaZKYV+sL/LPK8FcDQ83sqe48hyAIehtGy7wGGWxroKFJzCUNS8L3AWArWtM1roTXJfq1pNnAqOTJ8BxerHJpPMPYuQCSxuAJzp/t7vMJgqDkNLl5oRaG4/WFPg68DeyPL34db2abAicBFySb7jRgBDAOmAXMkjQQWLVI4Ib3QhA0KWa1bQ2gDBFp083s8fR4PDAM13qvz9gdB6b/9wPbATcCj+AVIzajSob28F4IguakxIpuKTTdOZnH84BlgLfNbGRmWy+9PgYv7bM5nt9yCJ6x/f4enG8QBCWm7AtpZRC6ed7F6wwdCCBno/TaY7gW3GJm7wOPA1/GhXHQjdTDA6KrlOmLEvQiUmHKWrZG0GihuwqwtqRLJU3FzQWLAN8GLkiLaDOBo9JC2tN4OeSJkubhFSUGA+dLGt6QMwiCoGQYLS0tNW2NoNFC90VcyFYW0ibhgvV0YCszWwLYEVg/s5B2DF6HaAJumlieKgtpQRA0J2U2L/TWhbQ1gB/gmvF9VFlIiyTmQdCklNgMVQahm19IW560kFbQdwzwFdyP93TgZDpYSAvvhSBoPszKncS80eaFImIhLQiChaLEbrqlFLrgKR2PSvlzp+J5GjCzOfhC2iOp3/34QtqURkwy6F6qeUyUxTYXlJXa7LlNadM1sxeA9TPPf5J5ebcq+2ybeXwNHg4cBEHgGA3zTKiFMth0gyAI6oZRbptunxa64b0QBM1JmU1OfVrohvdCEDQjDVwlq4GyLqR1CUl3S1q50fMIgqAEWARHdCsp4fnawFuNnksQBOWgZV55Nd3SC11JSwLX4Xka+uPVgN8AfoLP/1ngZjN7r2GTDIKgNFSyjJWV0gtd3HXsZTPbE0DS0sATwE5m9oykq4DpRTvGQloQNCFROWKhmQLsIulHkrbFczNMN7Nn0utX4vkY2mFml5jZKDMb1TNTDYKg8URwxEKRtNlNgD2As4G/N3hKQRCUnDJruqUXupJWAt4ys99Lehs4DhgmaW0zew44DM80FjQJXUmeXu3L190J2IPGEsERnSDpTFyw/iI9Pwd4DV88OxBYXtKreNLyy/BqwNdLWgRfXHugIRMPgqB01DvLmKTdgF/isuYyM/thlX77AzcAm5lZ1WK4ZbHp/hb4PMx3ATsYT3A+ElgLt+P2A/bBE5m/aWYbm9kGuLnhwwbMOQiCklIvm26qWPNrYHe8EvlnJY0o6DcY+BrwaGdjlkLopsQ3b0raGNgVmAhsA/zBzOaZ2au4CWGzrowbJdiDoBmp60La5sBzZva8mX0A/JGU9TDHWcCPgPc7G7AUQjdxGXA4cASu+VbjQ9rOe7FqHcN7IQiakPoWplwZTydb4cXUNp+00L+qmf2llgG7XehKmpn+ryTphg66/h03K2wG3Innyj1IUn9JQ3G3sMeAfwIjJA2UNATYqVtPIAiCXkcXNN1lK3fDaeuSX38yh/4MOLHWfXpsIc3MXgYO6KDLksAA4DozmyfpJmBLvFilAd80s/8ASLoOD5CYjpsigqCQ/86aVdi+xBJLFbbPnv1ud04n6AG6GJH2Rid3wi8Bq2aer5LaKgzGc4LfmzxiVgBulbRPtcW0HjMvSBom6Yn0+OOSHpP0uKTJqXz6D4EhwD6SzjW/aq8B7+HXcURlHGBv4F5czR8EXNtT5xEEQdkxrKWlpq0GxgLDJa0haVH8bvzW+Ucye8fMljWzYWY2DK9qU1XgQuNsuscAv0zFJ0fhgnMr3G1shJmdLGlXYDhuyB4JbCqpEnk2nNay7W8D+/f4GQRBUE4MrKW2rdOhzD7EYwPuBJ7C78SnSjpT0j4LMr1G+ek+DJwqaRXgRjObmATqbZk+u9LqyQAumIcD/6K4bHs7IvdCEDQn9YxIM7PRwOhc2+lV+m7f2XgNEbpmdo2kR4E9gdGSvgw8n+sm4AdmdnGbRjcv5Mu2L17lOJHEPAiakDKHATfEvCBpTeB5MzsPuAXYEJiBG6Ur3AkcKWmQpCGSvi1puQZMNwiCXkRlIS0S3rTlM8BhkuYC/wH+z8zekvRgWmy7Pdl118NNEQOA1fC8uvMaNOegF7LMoEFd6l/0RYw8Db0MM1rmlbcasMqshleQVIkCmQb8NTXvjv+onW1mnXovhHkhqIUQuo3HzBbqgn/kI8vbDtsfUlPfm27+xfieDp4qU0RaR5wC/CN5OzyCezNsBOwMnCtpxUZOLgiCcmE1/jWC3iJ0s9SckyFyLwRB82FRmLJxhPdCEDQjhtXihNsgeoumm/VsqJaTIQgWGknttjJpSUFthKa7kJjZm1nPBmAyBTkZgiAIAFpqC/FtCL1C6AKYWZvlyJSvYVXgdElLJ1NCEARNjmuxIXS7gyOTb+/iwFhJfzKzN7MdIgw4CJqUEpt/erPQPUHSfunxqnhehjZCNxbSgqA5aZQ7WC30SqEraXvcR3dLM5st6V46qCARBEFzUeaFzl4pdPFqwP9NAnddYItGTyhYcKp9QcoSCVaWeQS1YrS0lDdbQEOFbsoYdjteQn0rPCP7p4CV8AqcQ4HZwNHAs8BzwJp4PoabJb0API4Xg2tTtygIguakEhxRVsrgp1uUkPwS4Hgz2xQ4CbjAzObhuRdG4InNxwGXAgfhWu/VjZh8EATlI/x0O6YoIflWwPWZ27qB6f/9eDDEGsAPcA34PrykRjvCeyEImpPQdDsmn5B8GeBtMxuZ2dZLr48BtsU13dF4TbXtcWHcjijBHgTNiFVsDJ1vDaAMQjfPu8B0SQcCyNkovfYYrgW3mNn7uD33y7gwDoIgAMBoqWlrBGUUugCHAkdJmgRMxRfXMLM5wL/x9I7gGu5gYEojJhnUh6J8B5HzIFhQzDwMuJatEZTBpttf0qW0914A+BDXfK+T1J9W74WlJc0DdjCzIZLGSDrKzJ5twPyDICgV5f5xLoOmuyDeC9sAE4BtJQ0EVg2BGwRBBbOWmrZGUAZNN7wXgiCoK2XWdMsgdPPeC8uTvBcK+o4BvoKbH04HTqYT7wUi90IQNB1lFrplMC/kCe+FIAgWnFrdxcJlrA3hvRB0yashCCoY0GLzatoaQa8owV4PwrzQd4gy6X2bhS3BPmjQENtww+1r6vvww7f0eAn2Mth0u41YSAuC5qTMymSfFrqxkBYEzUkI3SAIgh7C18jKWyOtrAtpNSHpcEkrSbpbUuTTDYIAMKylpaatEfR2Tfdw3LthbeCtxk4lCIKyEDXSaqSDShLrABcBSwD/AI4EdgJGAdcBi/f8bIMgKCtltumW0bxQlIvhKuBbZrYh7pP7PTO7Aa8ecaCZLWdm7+UHkvQlSeMkjevB+QdB0FAsci90kXwuhrWAIWZ2X2q7Eri+loHCeyEImo+okdZ18rkYhjRqIkEQ9E7qWSNN0m6Spkl6TtIpBa9/Q9KTkianRf3VOxqvjEI3zzvAfyVtm54fhmcWA5iBhwEHTURRaHA1IhF6c1KvJOYpj/evgd3xtLKflTQi120iMCqZP28AftzRmL1B6A7Ak+DcIel9YE/gXkkTgQ3xUuyTJMViWhAEeI20ltq2ztkceM7MnjezD4A/knLBzD+a2T1mNjs9fQRYpaMBS2XTNbMXgPUzz38iaX9gOTPbFkDS0sATwE5m9oykq4AJRQtpQRA0J11wGVs2t9B+SVoLqrAynmSrwovAJzoY7yjcA6sqpRK6VZgC/FTSj4DbSKkfzeyZ9PqVwLHAL/I7Ru6FIGg+uriQ9ka9Et5I+hzuxvrJjtgOpyMAAAmgSURBVPqVXugmbXYTYA/gbODvXdg3vBeCoAmpo93+JWDVzPNVUlsbJO0MnAp8MqWgrUrpbbqSVgJmm9nvgXOBLYFNJB2XumQX1oIgaHrq6qc7FhguaQ1JiwIHA7dmO0jaGLgY2MfMXutswNJrusAGwLmSWoC5eLmec4ATJX0ZvygXNXB+QYnpap7deQUr2v37lV43CXLUq7y6mX2YFLw7gf7Ab81sqqQzgXFmdiuuDA6ita7jv8xsn2pjdrvQTaG9d+CrelvhQvJy4PvAcniViD2AmWb2k7TPE8BeaWFt+cpQwNNmNk7SK3hl4FF4jbS9cVeNIAianHoHR5jZaGB0ru30zOOduzJeT2m6awMH4jkTxgKH4GXU9wG+g9c6a4ekjwOnAVuZ2RuSlsm8vGIaY11c3Q+hGwQB7jJW3iWcnrpvmm5mU8yNKFOBu81/iqbgJdersSNwvZm9AWBm2UxiN5tZi5k9Sas23IbIvRAEzYnRUtPWCHpK082u5rVknrekOXxI2x+Axbo4ZqHhLrwXgqA5KXPUYVlWCF4ANgFI7mFrpPa/AwdK+mh6bZnCvYMgCOZjdQsD7g7K4r3wJ+DzkqYCjwLPAKRVwnOA+yTNw2OcD2/YLIM+TzN4KlTTAvtKReWyl+uJEuxB0GSUXegubAn2gQOXsJVXHl5T3+nTJ0cJ9noSYcBB0JyUWZns00I3FtKCoBkpt8tYnxa6QRA0J1GYspuRdDdwuJn9u9POQdALmPl+cabSQYstfNroxQYusdBjlBkzaGmZ1+hpVKUhS7WSzpT09czzcyR9TdK5kp6QNEXSQem17SXdlun7K0mHp8cvpJSP2+DhwEEQND21lepplN23Uf4xvwU+DyCpH56550VgJLARsDOe5GbFGsZaBK8e/LtummsQBL2MMgvdhpgXzOwFSW+mlGjL4/632wB/MLN5wKuS7gM2w5OWd8R5ZvbPohfCeyEImpPwXijmMjzQYQVc892lSr/OQoRnVTtAeC8EQXNS5uCIRobf3ATshmuzdwL3AwdJ6i9pKLAd8BjwT2CEpIGShgA7NWrCQRD0Asxq3xpAwzRdM/tA0j3A22Y2T9JNeFWISYAB3zSz/wBIug4vRjkdN0UEQZ+mmpdC0W1zVyPJ5nzQt2u4GtBSYk23YUI3LaBtAYxOScvBTQ5XAnPM7FpJPwc2MrMdJd0BHGVmh0qaKWk48A5wm6RPmdmrDTmRIAhKR5gXckgaATyH59PdGy9pvAVwNPAAsG3qOgoYJGlAahuT2pcEHjGzjVLb0T03+yAIyk24jLXDzJ40szWBccBNZjbLzGYCNwKbA5tKWgrPmfswLny3xe2+AB/g5dgBxlMlEXokMQ+C5qTMQreMEWmG224PBx4CJgM74CV/nkp95lrrFZtHlfMI74UgaD7qXSOt3tRF05V0gqSnJF2dax8l6bwOdr0f2FfSEpKWBPZLbfcDJ+Gmg/uBY4A3gfPrMd8gCPoyhrXMq2lrBPXSdL8K7GxmL1YaJC1iZuNwE0IhZjZB0hW4axjAZWY2MVWIOBV42MxmSXqflNg8CJqZrngqlD1vbnfSpxPeSLoIWBO4XdJqeGXeNYF/SboYOMnM9kqa7PnA+sAA4AwzuwV4CxeoSwDHSlrJzL4JDJB0hKRvA28DMzOHPSJ5PMwD3jGz7Rb2PIIg6DuU2byw0ELXzI6RtBtudz0O90bYxszek7R9puupwN/N7MgU5PCYpL+l10YCG+MLZ9MknY9Hon0f2BR3DbuHVh/d04H/MbOX0liFRBhwEDQnfVroFnCrmRV5X+8K7CPppPR8MWC19PhuM3sHQNKTwOrAssC9ZvZ6ar8W+Fjq/yBwRQqauLHaRGIhLQiaD/dMKK+fbncI3Wq5EATsb2bT2jRKn6BtOfWq3ggVknb9CWBPYLykTc3szYWYcxAEfYgya7o96ad7J3C8khU/ZRjriEeBT0r6aAqOOLDygqS1zOxRMzsdeB1YtbsmHQRB7yNKsDtnAb8AJqcQ4OnAXtU6m9krks7AgyPeBh7PvHxuCgMWcDeeryEIggyN8FIojcdEiTXdKMEeBEHdqIfQXdgS7P3797fFFluypr6zZ8+IEuz1JLwXgqD5KHtEWmi6QRDUjTJouv369beBA2sr4Pn++7N6XNNtZBLzQiQdLOnURs8jCILeSyS86QBJiwIDzKziarY7UJivoaBvEPRJSrMg1UXKMT+LEuxFSFpP0k+BaaSgh+RONhKYIOmTkh5P20RJg4GPAFMlXSxps8xYoyWt1IjzCIKgXFRsumXVdHtU6EpaMuVTeAC4FHgS2NDMKuG9GwOTUtrGk4BjzWwknkv3vVQdYh08JPicJIxPAD5nZi/35LkEQVBi6lgjTdJu+v/2zh80iiAK47+PC/6pIxYqqIVwWEswpYgWNmlSRBFsgoWICDaJRQor21hJUJs0BkRQIZIg2CgopgsphCiIBpuA2AWRPIvZwHLZy13MZW+PfD8YmJ0Zvnu7xbvdt/veSJ8lLUsaK5jfL2kmm/8o6UQL29qvsr7TRtpO/R1QbzJ/F7ic9cdICRK3gGNN1m8U2FkDjhTMXydVOVsg1el1c+uJ1oxu21XSue/Uz0St1tdWAxZaaNWAL6QiXvtIOQGnG9bcAB5m/RFgZivNssMLw8AK8FzShKTjDfMXgXmAiLgPjAIHgfeS6huLJB2WdAd4RbooV4BNe6RFxFREnImS304aY7pLxHpbrQ0GgOWI+BoRf4CnwFDDmiHS3o4Az4DzG5m3RZT6Ii0i5oF5Sf3AVeCFpFWSc/0F9EVWQyFL9V0EFrP4bV3ST9LJ1YFp4FJErJR5DsaY6tPBFN+jwPfc8Q/Sno6FayLir6TfQD+wWiTYla8XMsc6CUxKGiAVubkAvMktuy3pHLAOLAGvSZXJHgBvY/tR8FXgW9Y/xOYLUjS23fFOaPSqtu3roHbuRqmS9u2iRuPT7/8wl2m2w4GGPRSnIlUn3D3KjOm2iJ08As6W9Fub4jhFY9sd74RGr2rbvupq96p9VWjAIDCXOx4HxhvWzAGDWb+P9AeiZpqVSY6IiNGI+NBtO4wxJscn4JSkk1mewAjp5X2el8C1rD9M2qyh6ZN415MjjDGmqkSK0d4k3c3WgCcRsSTpHukO/SXwGJiWtEzafmxkK8296nSLYjbN4jjbGe+ERq9q277qaveqfZUgImaB2YaxiVx/jVy971bsmYI3xhhTBSoT0zXGmL2Ana4xxpSIna4xxpSIna4xxpSIna4xxpSIna4xxpSIna4xxpTIPwyXA/7KLPvdAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "eval_and_show(test, settings.MAX_LEN, SRC, TRG)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}