{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from fastai.text import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[PosixPath('/home/stas/.fastai/data/giga-fren/models'),\n",
       " PosixPath('/home/stas/.fastai/data/giga-fren/giga-fren.release2.fixed.en'),\n",
       " PosixPath('/home/stas/.fastai/data/giga-fren/giga-fren.release2.fixed.fr'),\n",
       " PosixPath('/home/stas/.fastai/data/giga-fren/data_save.pkl'),\n",
       " PosixPath('/home/stas/.fastai/data/giga-fren/cc.en.300.bin'),\n",
       " PosixPath('/home/stas/.fastai/data/giga-fren/questions_easy.csv'),\n",
       " PosixPath('/home/stas/.fastai/data/giga-fren/cc.fr.300.bin')]"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Note: You need to run translation.ipynb nb before running this one to get the data set up\n",
    "\n",
    "path = Config().data_path()/'giga-fren'\n",
    "path.ls()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We reuse the same functions as in the translation notebook to load our data."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def seq2seq_collate(samples:BatchSamples, pad_idx:int=1, pad_first:bool=True, backwards:bool=False) -> Tuple[LongTensor, LongTensor]:\n",
    "    \"Function that collect samples and adds padding. Flips token order if needed\"\n",
    "    samples = to_data(samples)\n",
    "    max_len_x,max_len_y = max([len(s[0]) for s in samples]),max([len(s[1]) for s in samples])\n",
    "    res_x = torch.zeros(len(samples), max_len_x).long() + pad_idx\n",
    "    res_y = torch.zeros(len(samples), max_len_y).long() + pad_idx\n",
    "    if backwards: pad_first = not pad_first\n",
    "    for i,s in enumerate(samples):\n",
    "        if pad_first: \n",
    "            res_x[i,-len(s[0]):],res_y[i,-len(s[1]):] = LongTensor(s[0]),LongTensor(s[1])\n",
    "        else:         \n",
    "            res_x[i,:len(s[0]):],res_y[i,:len(s[1]):] = LongTensor(s[0]),LongTensor(s[1])\n",
    "    if backwards: res_x,res_y = res_x.flip(1),res_y.flip(1)\n",
    "    return res_x, res_y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Seq2SeqDataBunch(TextDataBunch):\n",
    "    \"Create a `TextDataBunch` suitable for training an RNN classifier.\"\n",
    "    @classmethod\n",
    "    def create(cls, train_ds, valid_ds, test_ds=None, path:PathOrStr='.', bs:int=32, val_bs:int=None, pad_idx=1,\n",
    "               pad_first=False, device:torch.device=None, no_check:bool=False, backwards:bool=False, **dl_kwargs) -> DataBunch:\n",
    "        \"Function that transform the `datasets` in a `DataBunch` for classification. Passes `**dl_kwargs` on to `DataLoader()`\"\n",
    "        datasets = cls._init_ds(train_ds, valid_ds, test_ds)\n",
    "        val_bs = ifnone(val_bs, bs)\n",
    "        collate_fn = partial(seq2seq_collate, pad_idx=pad_idx, pad_first=pad_first, backwards=backwards)\n",
    "        train_sampler = SortishSampler(datasets[0].x, key=lambda t: len(datasets[0][t][0].data), bs=bs//2)\n",
    "        train_dl = DataLoader(datasets[0], batch_size=bs, sampler=train_sampler, drop_last=True, **dl_kwargs)\n",
    "        dataloaders = [train_dl]\n",
    "        for ds in datasets[1:]:\n",
    "            lengths = [len(t) for t in ds.x.items]\n",
    "            sampler = SortSampler(ds.x, key=lengths.__getitem__)\n",
    "            dataloaders.append(DataLoader(ds, batch_size=val_bs, sampler=sampler, **dl_kwargs))\n",
    "        return cls(*dataloaders, path=path, device=device, collate_fn=collate_fn, no_check=no_check)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Seq2SeqTextList(TextList):\n",
    "    _bunch = Seq2SeqDataBunch\n",
    "    _label_cls = TextList"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Refer to the translation notebook for creation of 'questions_easy.csv'."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv(path/'questions_easy.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "src = Seq2SeqTextList.from_df(df, path = path, cols='fr').split_by_rand_pct().label_from_df(cols='en', label_cls=TextList)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "29.0"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.percentile([len(o) for o in src.train.x.items] + [len(o) for o in src.valid.x.items], 90)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "26.0"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.percentile([len(o) for o in src.train.y.items] + [len(o) for o in src.valid.y.items], 90)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "As before, we remove questions with more than 30 tokens."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "src = src.filter_by_func(lambda x,y: len(x) > 30 or len(y) > 30)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "47389"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(src.train) + len(src.valid)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = src.databunch()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data.save()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Can load from here when restarting."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = load_data(path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>text</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>xxbos xxmaj quelles mesures le gouvernement a - t - il prises pour faire passer les ministères et organismes à risque élevé à une catégorie de risque plus faible ?</td>\n",
       "      <td>xxbos xxmaj what action has the xxmaj government taken to xxunk the \" xxmaj medium to xxmaj high \" risk departments and agencies to a lower risk category ?</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>xxbos xxmaj quels sont les rôles respectifs du personnel juridique et non juridique pour la définition du cadre de rédaction législative ainsi que durant le processus lui - même ?</td>\n",
       "      <td>xxbos xxmaj what are the appropriate roles of the legal and non - legal staff in setting the framework for the legislative drafting process and during the process itself ?</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>xxbos xxmaj comment nos chefs peuvent - ils espérer nous faire observer les règles et les politiques de notre organisation lorsqu’ils renoncent eux - mêmes à les faire respecter ?</td>\n",
       "      <td>xxbos xxmaj when our leaders choose not to enforce the regulations and policies of our organization , how can they expect us to follow them ?</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>xxbos xxmaj quel sera le sort réservé aux suppléments en vertu du projet de loi xxup c-51 , et le dosage fera - t - il l'objet d'une réglementation ?</td>\n",
       "      <td>xxbos xxmaj what is the future under xxmaj bill xxup c-51 for supplements and will there be regulated doses ?</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>xxbos xxmaj quelle serait aujourd'hui la prévalence du tabagisme si les manufacturiers de tabac canadiens n'avaient pas augmenté régulièrement le contenu en nicotine des cigarettes entre 1980 et 1995 ?</td>\n",
       "      <td>xxbos xxmaj what would the prevalence of smoking be today if the xxmaj canadian tobacco manufacturers had not regularly raised the nicotine content of cigarettes between 1980 and 1995 ?</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "data.show_batch()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Transformer model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "![Transformer model](images/Transformer.png)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Shifting"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We add a transform to the dataloader that shifts the targets right and adds a padding at the beginning."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def shift_tfm(b):\n",
    "    x,y = b\n",
    "    y = F.pad(y, (1, 0), value=1)\n",
    "    return [x,y[:,:-1]], y[:,1:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data.add_tfm(shift_tfm)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Embeddings"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The input and output embeddings are traditional PyTorch embeddings (and we can use pretrained vectors if we want to). The transformer model isn't a recurrent one, so it has no idea of the relative positions of the words. To help it with that, they had to the input embeddings a positional encoding which is cosine of a certain frequency:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class PositionalEncoding(nn.Module):\n",
    "    \"Encode the position with a sinusoid.\"\n",
    "    def __init__(self, d:int):\n",
    "        super().__init__()\n",
    "        self.register_buffer('freq', 1 / (10000 ** (torch.arange(0., d, 2.)/d)))\n",
    "    \n",
    "    def forward(self, pos:Tensor):\n",
    "        inp = torch.ger(pos, self.freq)\n",
    "        enc = torch.cat([inp.sin(), inp.cos()], dim=-1)\n",
    "        return enc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x7f9e60e41c18>]"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x7f9e6202fc50>]"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x7f9e62031080>]"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x7f9e62031518>]"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYYAAAD8CAYAAABzTgP2AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzsnXd4W9XdgN+j7b1HbMfxyCR7sMNKSspI2CvMtkDYBdp+hbChQKFQWlpmmGXPFsJIQiCEHTLIDpke8V7ylGzN+/1xdW3Z1pZspY3e5/HjRHeda91zf+e3hSRJxIgRI0aMGAqqaA8gRowYMWIcWMQEQ4wYMWLE6EdMMMSIESNGjH7EBEOMGDFixOhHTDDEiBEjRox+xARDjBgxYsToR0wwxIgRI0aMfsQEQ4wYMWLE6EdMMMSIESNGjH5ooj2AUMjMzJSKioqiPYwYMWLE+K9iw4YNzZIkZfnb779SMBQVFbF+/fpoDyNGjBgx/qsQQlQGsl/MlBQjRowYMfoREwwxYsSIEaMfMcEQI0aMGDH6ERMMMWLEiBGjHzHBECNGjBgx+hERwSCEeFEI0SiE2OZluxBC/EMIsVcIsUUIMcNt22VCiD2un8siMZ4YMWLEiBE6kdIYXgZO8rH9ZGCM62cR8DSAECIduBs4HDgMuFsIkRahMcWIESNGjBCISB6DJElfCyGKfOxyOvCKJPcRXSOESBVCjACOB1ZKkmQEEEKsRBYwb0ZiXOHSZrby5a5GzpiWjxAijBPthz0rIWs8jDoKwjnXEOKUnNR21bKndQ/GHiMmmwmz3YxWpSXNkEaqPpX8xHxKU0vRqP4rU2CGju5W6GwAqwnsPZA3HXTxQ3pJSZJwmsw42tpwdnbg6OjE2dWJ09yNs9uM1NODZLMh2exINlv/g9UqhEaL0GgQeh2quHhUcQZUCQmoEpNQJyWiSklBk5bGjiYzVruT6YX/vWu2drON/UYzkwtSwjuRrRu2/wcSsqHwcNAnRWaABxjDNbvzgSq3/1e7PvP2+SCEEIuQtQ0KCwuHZpQDePSzXby2Zj+F6QnMHBXkpJAkWP8CbHoDajb0fT7ycJj9Oxj7y2EREBsqW7nl/S08c/EMRmcPfojL28tZXbWar6u/5mfjz5hsJr/nNKgNjE8fz4ycGcwbNY9DMg4JT3D+t7P5bVh6PTisfZ9ljIbzXuGeH6E4M4HLjioK6pSSJOFoacFaVYWtthZbTS32+jpsjY3YG5uwNzXhMBqRLJbI3osHenRxtBmSKB83Cl1ODpqcbLR5eWhH5KEtyEc3ciSquLghH0c4/O6dTXyzt5n1d/yCZIM2+BM4nbDtPfj8Xuiolj8TKhgxFebeBaVzIjtgD9S397Dk6zIWnzIerXpo3cPDJRg8vTUkH58P/lCSlgBLAGbNmuVxn0jSarLy3gb5AfhwU03wguGrh2H1nyF3Csy9G8adAhXfwHePw5vnw5Tz4cxnh1Q4VLaYuPKV9RhNVpZtreeGubJgMNlMfLj3Q97e9TZl7WUAjEsbx2mlpzE2bSxj0saQE59DnCaOeG08NoeNdks7RouR8vZytjdvZ3vLdl7Z/govbnuR/MR8Tik+hYXjF5IV7zfbPiAkSeKd9VXsN5rp7LHTY3NwxTEljM05gFZoTiesfhC+fgSKjoFZvwZdIlg6YcVtSM/Npav7Uu52HIfdKXH57OJBp5DsdqyVlVj27MWyby/WfWVYKsqxVe7HaeovpFUpKWizs9FkZ6MvKUGdkY4mLQ11Whqq5GTUScmoEhNRxcejio9D6PWodDqEVgtaba/wliQJHA4ku13+sVhwdvcgdZtxmkw4OrtkDaS9ndqKOj7/egeplk6S2syk12/GXl8/SAPR5OSgGzUKXUkx+pJS9KNL0Y8bhyYjY+j+/gHy/b5mvtjZCMDqXU2cNjUvuBOYjfD6uVCzXhYEpz8hf175PWz/N7xxAVz0LpQcF+GR97F8Wz23/nsLVruTM6fnh6/5+GG4BEM1MNLt/wVArevz4wd8vnqYxuSTN9bup8fmZEpBCh9truXO+YcELqV/ekUWCtMugtOf7Hv5Z4+Hmb+St33zVxgxDY68dkjG32qy8uuX1iFJEiPT4/h+XwsLj0rjha0v8J+9/8FkMzElcwq3HX4bxxccz4jEEV7PpVVpidfGMyJxBBMzJjK/ZD4A7ZZ2Vu1fxYqKFTy/9Xle3v4yC0oXcNnEyyhJKQlr/Hsbu7jl/a2oVYIkg4aObhvxOg33nDYxrPNGDIcd/rMItr0P0y+GU/8GGl3f9uJjaf7XJTxqf5bpKSZu/xjicXBGSg8927fRvW0blp93Ytm7t2/VLwTa/Hx0JcXEz5yFrrAQ7cgCdPn5aPPyUCUkRGToQgjQaBAa1/RPTPS67zPLfuaFzvGMzUnilW4bq//veDQCHC0t2GprsVZVY6vaj7VyP9aKCjo+XYazo6P3eHVmJoZx4zAccgiGSZMwTJyINj9v2DRMp1PiwU9/Jj81DovdwWfb64MTDJIEH1wD9Vvg9Kdg6kJQud4DpSfAEdfAy6fCmwvhkn9D4RERHX+PzcG9H+3gzbX7mVKQwuMXTKc4MzLPgS+GSzAsBa4XQryF7GhulySpTgixAnjQzeE8D1g8TGPyisXu4OXvKzh2bBaXHTmKy/+1nq93NzF3Qo7/g3d/Bh/dBKVzYcHjgzUCtRbm3AlNu+CzO2DEFCiaHfHxL3p1PdVt3bxxxeF8vLWSt3e/zqn//garw8q8onlcPOFiJmdNDus6KfoUzhxzJmeOOZP9Hft5ZccrfLD3Az7Y+wHnjT2P66dfT4o+tJXNlup2AJbfeAxjcpI495nv2VLdFtZ4I8raJbJQmHMnHPP7wd9zYjbPJC7mmA13M7flIwo7dpGytJ4KpwMAdUoK+kMmkLZwIfrx4zCMHYuuuPiAMsk4nRIfb65j9phMLjlCngcfb6nlzOkFaLKy0GRlETd1ar9jJEnC0dyMZe9eenbtwrJrNz07d9Ly0ktgtwOysIibOpW4qVOJnzkDw+TJqHQ6T0MImw8317CtpoO/nT+VH8uMfLylDovdgV6jDuwEa56G3cvhpIdh+kWDt8enwyUfwMunyFrFrz6WtYoI8c76Kt5cu5+rjivh9yeOQ6cZngyDiAgGIcSbyCv/TCFENXKkkRZAkqRngE+BU4C9gBn4tWubUQjxJ2Cd61T3KY7oaPLx5jqaOi389dxijizNIC1ey3821vgXDG1V8O6vIHcSnPcvWQh4Qgg442l4bo68/1VfQ3KQ6q0PvtzZyLqKVv567lTMmm180XUPmswmxiTP5v5jb6EopShi11IoTC7kjiPu4Npp1/L0pqd5Z/c7LKtYxg3TbuDcceeiEsE90Ftr2onXqSnJklezUwpSeW1NJXaHE80Q21f90lELXz4Ao3/RTyjYm5owrfkR049r6F63nnMq5Xplbep4RuY08NHEE2grGsstN541rKvmUNlY1UpNWze/nzeWE8ZlMy4niadX7+P0qfmoVJ7HLoToFRoJRx7Z+7nTYsGyezfdW7fSs3kL3Zs30/XFF/IxOh1xU6YQf/jhJBxxOIapUyMiKHpsDh5dsZtJ+cmcPjWflDgtb62r4od9LRw/Ltv/CWp+gpV3wbhT4fCrvO+XlAOXfgjPnwgfXg+LvurTKsJkS3U7WUl6Fp88ISLnC5RIRSUt9LNdAq7zsu1F4MVIjCMSSJLE89+WMy4niWPGZCKEYMHUPN5eV0Vnj40kX46rrx4Cpw3Of81/tIIhWd7v+bmw9Aa4+P2I3cPPdZ0IdQ8bTM/w0RcfUpIymoa9ZzEl5xdDIhTcSTekc/sRt3PuuHN5eO3D3P/j/Xy+/3PuP/p+chIC0LhcbK1pZ2JeMmrXC2hKQQoWu5PdDV0ckpc8VMMPjOW3gtOOc+4DmL/7HtO332L67lsse/YCyPb+qdN4PnEysxacwAXTuxH/+Q0lOfHcY5zAnQUe4ysOOJZuqkWvUXHiITmoVIKrjy/h5rc38+WuxsC0ZzdUej1xkycTN3kyXHghAPbWVrp/+gnz+g2Y162j+emnaX7ySYTBQPysWSTMPprE2bPRlZaGJERfW1NJTVs3j5wzBZVKcFRpJvE6NSt3NPgXDJZOeO83kJgj+xT8XT+lAE68D/59BWx9B6ZeEPR4PbGzvoPxucPvV4tlPg/gh30t/FzXweXHFPc+jGdMz8did7J8W733A5t2yxFIh14BqQFGTWWPh2P/AHs/h+rIlRHf0PATyaWP80nFR1wx+QreXfA2kzNn8N2+lohdwx9j08by/LznuevIu9jctJmzlp7FZxWfBXSs3eFkR20Hk/L7zFCTXf/eWhNdc5JtzXu0Ll1J1eap7D55IVVXXEHrG2+gycom+w+/p+i99xj7w/dsu/Yu3h9zPJNPPBox9WyYsIBfNr1EYlcFrSar/wtFGbvDySdb65gzPrt3MTR/Sh75qXE8/015RK6hSUsjae5ccm75I8XvvcvYNT9Q8NSTpJ5zDraaGhofepiy+QvY94sTqf/T/XR98y1Oa+B/ux/2tTA2J5GjRmcCYNCqOW5sFit3NOB0+olfWfsctJbDWUtkc1EgTDpb9ht+8Sc5rDVM7A55ITRhxPAvhGKCYQCrdzeh06j6Oaimj0xlVEY8H2yq8X7gl/eDNl42LQTDoVdAXBp89ZcQR9yHJEm8s+sdNtsfQqvW8OrJr3LjjBvRqXUcXZrB1uo2Onps/k8UIYQQnDv2XN6Z/w6FSYX8/qvf89f1f8XhsrN7Y1+TiW6bgylukRdFGQkkGTRsdvkehhNLWRnNzzxL+dlns/dXd1K/PhWL0UHq2WczcsmzjP1xDYUvvkDGFVcQN2kiQq1mTVkLyQZN36Q+5VEktZ7bNK+zu6Fz2O8hWNaUGWnusrLAbR5o1bL2sLm6TY5sijDq5GSS5swh947bKV32KaO/+Jzce+9FP24cbe+/T9WVV7LniCOpvvlm2j/5BEdXl8/zlTWbGJ3d37E+b2IOjZ0WNvvyV9m6Yc1Tcghq0dGB34BKBfPul8NZf3wm8OO8UNEi549EQ2OIZSkNoLzZRFFGPAZtn3NKCMEZ0/L5x6o9NHb2kJ1k6H9Q7UbY8SEcdwskZAZ3QX0SHHkdrLpfPk/e9JDGbXVYefDHB3l/z/s4TOM4vfhWpmRN6d1+ZGkm/1i1lx/LjJx4SHBmgHApSinilVNe4S9r/8LL219mT9se/nLsX0jWeV4Jba2RX/6T3TQGlUowOT+FrcMkGCxlZXQsW0bn8uW9JiJD6QiypnSQdPXD6E64yKd544eyFg4vyeg1hZGUi2XmIub+8FeWlm0b0tDGSLB0cw0JOjVzxvc3uZRkJWC2OmjstJCTbPBydGTQ5ueTdv55pJ1/Hs6eHsw//kjnF6voXLWKzmXLETodCbNnk3zSL0mcMwe1W3SV1e5kv9HM/Cn9o+3mjMtBrRJ8tqPBe8LextfA1BT8Ig+g+BgYexJ88xhMvxQSQg/X3VkvR3eNi5mSok9Fs4mijMHhYEeUZCBJsLvewyrli/vkVf+RHt0o/jlsERhS4KtHQjq8y9rFtZ9fy/t73ufsksswV13G5BH9J8SMUanoNSq+39cc2hjDRKvScvsRt3PnEXfyY+2PXPTJRdR0edbAtla3Ea9TU5zZf7U3pSCVnfUdWOy+NY5QsdXV0fLCC5SdcSZlp5xK8xNPokpJIef22xn95SqKF1jJPKEIvR+hUNvWTWWLmSNK+r8UEo6+EodQkbPzlSEZfyRZV9HK7DGZ/RZIQO/cKG/2nwgZSVQGA4nHHceI++5lzFerGfXG66QtvICe7dup/eMt7Dl6NtU33kTn55/jtFrZbzThcEqUZPWfyynxWo4oSWfljgbPF3LY4ft/QMFhMCoIbcGdX9wL1i749rHQjnexs64TtUoM0nqGg5hgcMPplKg0mj3GCRdmyOUNKo0DJkTNBti3CmbfLL/cQ8GQAkdcB7s+gbotQR3a3N3Mb1b8hg0NG3hw9oPMSrkQUDEmp//DpNeoObQone/3Dp+fwRPnjTuP5+Y9h7HHyKXLLqWsrWzQPltr2pmUl9K32nYxpSAFm0NiZ13kTDFOk4m2Dz6g8rJfsXfOXBofeRSh15Fz22JGr15N0WuvkX7JxWitFVC/FQ670q8j8geXL+fIAYJBJOXyneF4prd8DN0HUOjtABxOiepW8yDBDPTOjeEWDO4ItZr4GTPIWbyY0V+uYtQbr5N6zjmY162j+vob2HPMsRgfuJ/xxkpKPCzyjhubxd7GLoyefD3b3pdL2Bzzu9CTT7PHwyFnyJqHrSe0cyBrDKVZCYGH1kaQmGBwo7a9G6vdSZEHwZCbbECnVrHfaO6/4adXQRMHM38d3sUPvwr0yUGtMmq6arhs2WWUt5fzjzn/YEHpAnY3dKESUJo1eFIfNTqDXQ2dNHUOfRkFX8zKncWLv3wRh9PBZcsvY3vL9t5tdoeTHXX9Hc8Kis9hS0145iRJkjD/tJHa229n9zHHUnfrYmy1tWRedx2lK5ZT/PbbpF96KdocNzPK2iWyAJ98rt/z/1DWQlq81qNteGvBhRikHqSNr4Z1D0NJXXs3NodEYfrgWk95qXHo1CoqoigY3BEqFfEzZpB75x2M+Wo1I59bQuIxxxC/agV/+/qfxC+6kJYXXsDe3KcpK1pP1cC57HTK8y/7EBjzy/AGNvMy6GmDn5eGfIqf6zoZnxudCLyYYHCjoll+UEZlDJ4QapWgID2O/S1uD5PVLK8wDjldDj8Nh7hUmH4J/PyxnILvh7quOi5fcTltljaem/ccxxQcA8Cehk4K0+MHmQCgbwW7riLqqSKMSx/HKye/QrwmnstXXM7mps0A7G3q6s04H0h+ahzpCTq2VIW22na0tWH8178oW7CAygsvpGPZcpJPPolRr79G6WcryLr+OnSjRg0+sKNOnuDTLwGd/6zTNWUtHF6c4THWP6l4Jj86x+Nc86xstjgAURY/ngSDWiUYlRFP2QEiGNwRWi2JxxxD/qOP8MYfn+b5Iy5Em55O4yOPsuf4E6i+4bd0ffsdI1Nl30hV6wDBsO8LaNopa//h5iEUHQtpRXIVhBDo6LFR09bN+BHRKQETEwxulLfID7u3lPPC9Hgq3QXDzo/B0iGXRIgEU8+X8yC2/9vnbg2mBi7/7HI6LB0smbeEadnTerftbuhkjJd6Qsrn/e4hihQmF/Kvk/9FuiGda1Zew88tP/c6lz1pDEIIphSk9DqnA6V782Zqb13MnuOOp+HPD6FOSGTE/X9izNdfk/fAA8TPnOk7Tn7Dy+B0wKzf+L1Wl8VOdWs3U0Z6NiuOzUniRftJqDuqYNenQd3HcKGspD0tkACKMhMOGI3BG7u6JKqP/AVFb7xOyaefkH7ppZjXr6fqiitQX3Yu5+z5kvr9A8LPN78Fcekw8czwB6BSwYxL5fpoLfuCPnx3vWwunRDTGKJPRbMJg1ZFzsCoIxej0uOpMpr7QvU2viqvCkJ1Ug0kd4qsxm5+2+suzd3NXPHZFRh7jDxz4jNMzOirHWSxO6hoMTM2x7OzKlGvISVOS03bgSEYAHITcnl+3vMk6BK4auVVfLd/Owk6NSVehPOU/BR2N3RitvpebTutVto++IDyc8+j4vwL6PzsM1LOPIPiD/5D0dtvkXrOOagTA6g5Y7fChpdgzImQUep395pWOX69IM3zS3VsbhIrnbPoNOTBuuf9Xz8K7DeaUasEI1I8z4OSzAQqW8w4/OUCRJGypq7erHl9SQk5f/w/Rn+1mrxHH0U/YgSXb/+Ew2/9FbW33U739u1g6ZIF9cQzvVcsCJZpF4FQh6Q1/OwSDDGN4QBAiUjylu5fmJFAp8VOq9kGrRVQ/rX85Uco/R0h5Kqr1Ws9rjJMNhPXfn4tDeYGnpr7VL9wVHn88mT1VYE0PzWO6tbwk28iSV5iHs/Pex6VULG6/X7GFNi8fgdTClJxSrCjtsPjdntLC01PPMneE+ZQd+tinCYTOXfeweivv2bEPfdgGD8+uMHtWQFdDXDolQHtrgjd/FTPNY8yE/WkJRhYm3yivJrsagxuPMNAZYuZ/NQ4r6VHijITsDqc1LYdWM+RQqvJSqvZRumAiCSVTkfK/FMZ9eorPLbwHrZMnE3HsmVUnH0OlRecQ0e5E2niOZEbSFKuHLq66Q1wBJc/tLOug2SDhtwhDgn2RkwwuFHe4jlUVUGxuVa2mGDTm4CQqy1Gksnnyufd0l9rsDlt/H7179ndupvHjn+MGTkzBh2qJE6N8dB3QaEgLa53VXsgMSp5FM/84lnskpXG+Cdot3g2Fym+h4GJbpayMmrvuIO9J8yh+YknMEyayMgXnqfkk49Jv+iiwLQDT2z/QDYvBFhvv6ZNjkIpSPNeDG9MTiJL7YeD5AzLOTlUVBnNXs1IcGBEJvmirFkOKR8YquqObvQYnj/0PMZ8tZrsW27BVltLzbfp7LvyboyvvY7THCGteuZlYGqEXcuCOmxnfSfjRyRHrZ5WTDC4sDucVBnNHiOSFJTJsr+lCza9LpfdTR3pdf+QSMmXk5+2vC2X/EWOornvh/v4rvY77j7ybmbne67GuqehE5XwPSHy0+KoaesekszVcFHb8+iuvgSzs5GbV9+MzcMqKzvZQFq8ln1N8uQ3//QTVddeR9kpp9Lx0ceknHUmJZ9+QuGzz5J49NHhTSxbj1xZc8J8UAeWC1rT2o1WLchK1HvdZ1xOEl+0ZCBljpMFzwHGfqOZkR4czwqKYKhoOTAFw75GeVwlHsJtFUamx1PdaobEJDLOPZXSk6rJv/xoNBkZNNx/P3tPmEPj449jN4YZqFE6F5Ly5PdFgDidErvqO5kQhcQ2hZhgcFHb1oPNIVGc6X1CjHTZjR1l30J7lWxGGgqmXCCbqqp+BODZLc/ywd4PuGbqNZw5xrtjbHdDF0UZCR4jkhTyU+MwWx20mYevNEag1LR14zCXcPn4W1lXv467v7/bowAbkWwgbuNaKi6+mMoLL6L7p5/IvO46Rn+5ihH33IO+JLxeEL3sWyUnKh1yelD3MCIlzqspDOQggC6Lg87S+VDxrdwS9ACho8dGq9nmMSJJITtJT7xOTVnTASoYmrvQqoVPrW1kehw2h0RDRw9s/zdCOEi+7PcUvf0Wo954nbhZs2h5+hn2zplL/QMPYqutDW0wag1MPAP2fSn7MQKgpq2bLoud8VGokaQQEwwulIgkX6akOJ2a7CQ9mTWfg1oP404emsFMWCDXXdr8Jqv2r+LJTU+yoGQB10y9xudhuxs7/WZJKpOl5gC0D9e3y2aYs8eexvXTruejso94fmufg1ZyOulYuZI//udBznnnUWzVNeTcdhujV31B1g3Xo0kPsNhZoPy8FAypUBx4+Yratm6v/gUFpcTBz+lzAemAMicp4di+BIMQgqKMhANWYyhrkk3CvsqzK4u8KqMZtrwDOZMhWy5tHT9jBiOffIKSTz4m+aSTaH3zTfbO+yW1d9yBdf/+4Ac0/lRwWORw2AD4uU72n0WjRpJCTDC4UMLv/HVHGpUex7j2b6Hk+IBi2kNCnwjjT2Xv7o9Z/M1iJmVM4u6j7vZpFrHYHVS2mP22vsxPlSfEgeaABqht70ElICtJz6Ipizil+BT+ufGffFv1NR3LllF++hnU3PBb4uwWnj18IaM/W0H6pZegivf+EgsZuxV2fipP6iCiVGpau8n3sVIFGOvyAW3syYWsCXJz+QOEKh85DO4UZyUcuD6Gpi6f5lSg11RmrN4pt+ycMjhxUV9aSt5Df2b0iuWknX8+HUs/Yt/Jp1B7y61YKyoCH9DII2Q/1c5PAtp9T6OsWUSzjW1EBIMQ4iQhxC4hxF4hxK0etv9NCLHJ9bNbCNHmts3hti1qS6fyZhMJOjVZSd5twwCHJTSS46iH8acM6XjaS4/nxhQtcSoNfzvhb+jVvsdV1iTXhhlYCmMg+Qe0xtBNVpIerVqFEIK7j7iLM6pz6b7oGmpu/h2Sw0HeI39hw73P8MGImVjEEJYKKP8KLO1BmZGsdicNnT3k+dEYUuK15CTr5WCBiWfKvYM76sIdcUToTW7z4XwGKM5IoLpVrhRwIGF3yMXzSjxk/ruTl2pACEjc58olmXS21321+fnk3nkHpZ+vJP3ii+lYsYJ9p86n9tbFgWkQao1sXdi9PKDopJq2btITdCToo1fjNGzBIIRQA08CJwOHAAuFEIe47yNJ0s2SJE2TJGka8E/APYOrW9kmSdJp4Y4nVCpaTIzKSPDrrDzasRaAnuITh2wsTsnJ4savqNVo+HvKTHITcv0es9e1yvAVkQSQFq8lTqs+ICOT6tp7yE2JQ5IkOlevpv78S1j4ahVqB7y9MI/cf79FyoIF5KYn9O4/ZOz4QC5RUnJ8wIfUt/cgSVDgRzCA7BitaDbJ9ucDyJxUaTSTGq8l2VdDKmTN2uGUBmcPR5mqVrmch7c8GAW9Rk1usoG8pm/k/KGUAr/n1mZnk7P4Vkav/EwWEMuWse/kU6i7805sdX4E+7hToKcdKr/ze53Gjp4hr1zrj0hoDIcBeyVJKpMkyQq8BfhaZi0E3ozAdSNKRbMpoCbb49u/YZOzlGr70DmGXt7+Mt/UreGPUjrTKjcEdExdu/yiH5nu+6UkhHBFJh1YExrkF+us1jIqF15I9dXX4DSZyPvLwxjeepb/FDfzwLo/A5DrSrxS7jniOGyy2j/uZND41tTcUbQwf6YkkO+hocMCWePkpMYDJDqpymhmlB8zEtAbvXegZUCXNSmhqv4rko5LcVJk3gpj5gV1DU1WFjmLb6V05WekLVxI+wcfsm/eL6l/8EHsLV6KVJbOkWuqBWBOqu/oITc58OduKIiEYMgHqtz+X+36bBBCiFFAMbDK7WODEGK9EGKNEOKMCIwnaGwOJ1Wt3RT5iEgCoLOe9LatrHTMHLKyEpsaN/GPn/7BvFHzuGDM2dCwTe4l7YeGDgvxOjWJAaif+alxB5wpqefnn/nN0r9zzmsPYqurI/feeyn95GNSTjuNo0bO5uopV/NR2Ud8uPdDRqTIL976odIYKr6F7tagzEjQJxj8mZIAspP1NHb2yFFX406RI9DIcggvAAAgAElEQVR6hr8J0UD8haoqlByguQxKpNTA5DZPnKDbjhpn0IJBQZudLTcVWrGc5NNPo/W119l34jya/vkEjq4BfxddvCwcdn7SG4bujfr2oe914Y9ICAZPthdvd34B8J4kSe4F9QslSZoFXAj8XQjhse6AEGKRS4Csb2pqCm/EA6hu7cbhlHxGJAGyjRD43DljSARDW08bf/jqD4xIGME9R92DGHeSvGGP/5aYDS71M5C4/fwDKMnNWl1Dzf/9kfIzz6K0pZKys39D6YrlpJ1/HkLbZ85YNGURh+YeygM/PoAFWW0fMlPSXlfUWckJQR2m/E29lZJwJyfJgM0hyVn0o+eC5ICyr0IabqSwO5zUtHb7dTwDpCXoSInTHniCobmL9AQdqfE6v/seal1Lq5SIJTe05lgK2rw88u6/n5KPPyZh9myan3ySffPmYXztdSSbm09h/KnQUQN1m7yey+Zw0mL63xAM1YB7llcB4C3o9wIGmJEkSap1/S4DVgMevyVJkpZIkjRLkqRZWVlZ4Y65H4FGJLFrGVJqIdXaosHlt8NEkiTu/O5OjD1GHj3+UZJ0SZA5Vq7FtHuF3+MbOyxk+3GcK+SnxtFqtvmtNzSUODo6aHjkEcpOPpnOzz5DddFl/PrE27CfeyEqw+BJoVapeeiYhzCoDdz+/S2kJgyhKWnfKhh1pLzKC4KaNjNZSXqfeSQKijmsoaMHCg4FXVLA4YxDRV17D3an53LbnijOPPAik6pbuwPSeHA6KWlbw9fOKdR2RCanR19STME/HqfonbfRjx5Nw/33U7bgNDpWrpQ1w7EngVD5NCc1dVqQpL7nI1pEQjCsA8YIIYqFEDrkl/8gT5oQYhyQBvzg9lmaEELv+ncmcDSwIwJjCgrl4faV9YzVBGWrEeNOpTAjMeKC4b0977G6ejU3z7y5rzCeEHJd+PKv5BLfPmjoDNxh1ZvLEAWtQbLZML76GvtOnIfxxZdIPvVUSlcsp/GCyzHp4nrNRJ7Ijs/m/tn3s7t1N4acZUNjSuqog8YdAZfAcKe2rcdvDoNCjsuG3NDRI4fDlhwHe1f5NTMMJYFGJCkUH4BVVhs7LOQEskCq24TB2sKXjmkRn8txU6ZQ+K+XKXjmaVCrqbnht1Recgnd5fVy6Oqu5V6Pre+Qn+lo1UhSCFswSJJkB64HVgA/A+9IkrRdCHGfEMI9ymgh8JbUP5V1ArBeCLEZ+BJ4SJKkYRcM+41mEnRqMhJ8qJ9lq8HeA+NOojA9Tq6XFCEqOyp5ZN0jHDHiCC6aMCCbeuwv5etWfOP1eEmSXKakwDUGgOph9DNIkkTXV19RdvoZNDzwAPpDJlD87/fJe+jPaHNze81C/swwxxYcy8UTLsak/4qyro2RH+g+l/srBMFQE0Bym4LSN7yxw9U0afRcaN8PzXuCvm6k8NWHwRN5qQYaOi0HVJXVxs4esgOZB3tWIiH4yjllcMOeCCCEIOn44yn58ANy77kba1k5FeecQ+03GmwV28HkucVug2seBHQPQ0hE8hgkSfpUkqSxkiSVSpL0gOuzuyRJWuq2zz2SJN064LjvJUmaLEnSVNfvFyIxnmBp7OwhJ8WPfb7sKzmqoPAoRmUkUNXajTMCE8LutHPbN7ehVWm5/+j7UYkBX0nRbNAm9Po3PNHRY6fH5gxYY8gfZo3BUlZG1aKrqLrqanA6KXjqKQpffBHDhAm9+9QHMSFunHEjiao8mgyveC22FzL7VkFCNmRP9L+vG06nJAuGACKSoO8+G1wrRErnuq4fPXPSfqMZrVr41NrcyU4y4HBKnltkRgGr3Umr2dYrdH2yZwXkz8KkTh3SkFuh0ZB2wQWUrlhOxhWX07G+gn2fZNP86L04LYM7KTb8r2gM/ws0dVp8Fj0DZHPOqCNBo6MwPb43mSlcntvyHFuat3DXkXeRk5AzeAeNXi7Wt/szr2aGxg7lpRrYw5SdZECjEkMemeTo7KThoYcpO+10ujduJPvWWyhZ+iFJc04YJITr2nvITNQF1N/WoDEwL+smJHUn9695MHIDdjqh7Ev57x1kKfUWkxWr3RmwxqDXqEmL1/aaDkgbBRmjYW8UBUOLmYK0+EG9tr2hJINGu1WsQlOXPA6/vrauJqj5CTF2HvlpcVQbh36BpE5KIvsPf6Dko6Uk5jlpevNzyk6dT+fnn/erB1bfYUGnVpHuy3oxDMQEA/KD7fOl2tkgt/xz1cxRnFtVYT5QO407WbJlCfNL5vPLIh89ZkuOh45qMJZ53NzgMkcEZFtFbs84ItUwZBqD5HTS9sEH7Dv5FIz/+hepZ54hr5h+9SuEzvMDX9/eHZTDbXLWZKzNc1he8SkrKvw75wOifguYW0I2I4H3PgyeyEk29H53gKw1VHwbVgP5cAg0VFVBeQE3RmCBFAn6Fkh+5sG+LwAJxpzIyPT4YU3S0xUVU/DrQymcr0YVF0f19TdQdeUiLGXlgKwxZCfro1ZuWyEmGIBGfxpD+dfy7+JjgT7HYTgrJZvTxp3f3UmqIZVbDxtURaQ/rut68zMo6mcwIW5DlcvQ8/PPVF50MXW3Lkabn0fRO+8w4k9/QpOR4fO4uvYecpMDf6nmpRiwNp9AUeJ4HvzxQVp7WsMdep9/oeT4oA9VhGwgOQwKOcmG/i/V0b8Aezfs/z7o60eCuvZu8lMDf4YONI2h0TWOrEQ/91D+tVy7KHcqI9PihsTH4JOS40lIrKL4hcfIue02ujdtouz002n8619paW6PeqgqxAQDJosds9Xhu0ZS+VdgSIERUwF6hUhTGCulF7e+yE7jTu444g5S9J77A/eSORYSc+TVpAcUk1YwDqv81PiIagyOri7qH3yQ8rPPwVpZyYgHHqDozTeJmzwpoOPr2nsCiv9XkLULNafm3UiHtYOH1j4U4sjd2LcKcibJnbeCpLdzW4A+BpAXGL0+BoCio0Gti4o5ye5w0mKykhWIfd5Fr2DoOrAEg995UPGN7LtTqRiZHk+r2UZnzzCWoS+RLQ+i6lvSL72E0uXLSJk/n5bnnueqFxdzVO3WqPdLOegFg7La8WmXLP8Kio4BlWz/TovXoVaJkCfEntY9PLPlGU4uOpm5hXP9HyCE/CCXf+PRz9DYYSHJoCFeF3jRrYK0OBo6e8IugiZJEh3LllF28im0vvoaqeefR+myT0k9+yxEgHZ6s9VOe7ctKFOS4iB19OSyaPIiPi3/lK+qwkgQs5pg/xrZvxACtW09JLl6agdKTrKBJveoHl0CFB7Zp7kMI0aTFUmCrMTAbdvxOg2Jek1fZFWUaeroQQh8Rxe2VkLbfnk+0xcF109ADzWZYyFpRG9CoyYzk7w/P8ioN16nU2PglHf/TtXVV2Otrh6+MQ3goBcMveqnN8HQWiE/SG41+VUqQUaCLiQV2uF0cNd3d5GsS2bx4YsDP7DoGOiqh5a9gzY1dvYEnNymkJ8WhySFV1bCWlVF1ZWLqLn5d2iysih6521G3H036hQ/GtAA6gMMVXUnTqcmNV5LfXsPV0y+gjFpY7hvzX10WjuDunYvFd+B09YXHRQk1QGU2x5IdrIBpwQt7guMkuPkPAqTl5o7Q4SyyPFXXXgg2Un6A0pjyEjQ++zD0FvErkjugtin/Q9jZJUQ8vuk/Cs54MGF45DJXHfcTZSdeznd69ZTNn8BzUue6589PUwc9IKhyZ/6qZQpUOz8LrKS9DR3Bf8wvbXrLba1bOPWw24lzZAW+IHK9RV/hxsNHcGn0Bf05jIEb1+VbDaan11C2fwFdG/cSM5tt1H07jvETZ4c9LmgTzAEm+2Zm2ygrr0HrVrLfUfdR3N3M49teCykMVDxtWzGKTwipMNr2rqD8i9AX7BAPwf0qKPl3/t/8HDE0KHMg0x/0XkDyEzS03SAaAyNnQFk/1d8C/EZkDUekMcP0Dzcwq3keDnQoXF770cNHT04VGrsZy+k5JOPSZh9NE2PPUb5WWdj3jgEOTs+iAkGl33eq/O5/GvZvp81rt/HWUn6oDWGelM9/9z4T47OP5qTik4KbqDpJXLvWA8O6IYQyvQqq9tgG/Z0b95M+dnn0PS3v5F43HGUfPoJ6ZdeglCH3huhL7ktuBfriBQD9R3y+CdlTuLiCRfz3u732NTovRaNVyq/h/yZoA1uDAqBdG4biPKd9TNj5E0HjUEezzCiLHL+uzWGAJLbKr6Rha/LzKnM++EXDC4LRNnq3o96owuTDWhHjGDkE09Q8NSTOLq6qLzwIurv+xOOrsDag4bLQS8YGjstaFSCNE9FtyRJFgzFx8rqnxuZicELhofXPozD6eCOw+8IPhxNCCg+Rl7xuPkZJEmS6yQFmSmpvJQCvQenyUT9Aw9SccFCHO3tFDz1JAX/eBxtjofciyAJtQzAiNQ46tr6XqrXTbuOnPgc7ltzHzZnEOq3pQvqNsOoo4K6voLJIvtIgtYYXPdb7y4YNHq5dlIAdfsjSagaQ1aSvjdMNNr4rRc2wL8AkBKnRaMSwx9ZlZwn+xrcBEOv5uw2D5LmzKHko49Iu/hiWt98k7JT59Oza9eQD++gFwxNnRYyE/Wem7c37QRTo8eev1lJelpMloCzn7/c/yWf7/+cq6deTUGS/6YgHik6BkxN8rhctJltWB1OcoKIJgEwaNUk6TUBTYiub7+jbMFptL72GmkLZTU3aU7wsf7eqGvvJjVeS5wuOK1jRLKBFpOVHptcrDdeG8/iwxezp3UPr+14LfATVa8Dpx0KQxMMAQUweCAzUYdKMPjFOupoOadiGMtwN3fJZduD7RqWnWTAZHVgskSvICOAwynR3GXxnfU8wL8ALn9hom74NQaQF5z714BD/tvVewk7VycmkHv7bRS9/RaGiRPRFRYO+dBigqHL4l19LneZbYqPGbQpK1GPzSHR3u1/ZWq2mXlw7YOMTh3NpRMvDX2wyjjK+8xJSqhqKLHPGYk6WnyUM3C0t1O7+DaqrrgCodcz6vXXyL3rTtSJ/pugBEN9e09IJQAUn4R7VMzcwrkcP/J4nt78NLVd3or8DmD/D3LVy5GHBT0G6DNDZAYpGDRqFZmJ+v4+BpA1F8kJVWtDGk8oNHX6mAc+OFByGVpMFpySn1DVAf4FhczE0PyFYVN4JFi7ev0MDR09JBs0XhdIcVOmMPKpJ1HFhWbuDIaDXjA0dviYEPt/gOR8SB01aFNmEDHcz219jnpTPXcdeRdaVeDhjINIK4KUQtlR6qLPLhn8pM5M1NPsZUJ3rlpF2fwFtC9dSsZVV1H8wX+InzEjpGH7I9gcBgXFJ1E7oPz24sPkaK8/r/1zYCeq/B5yJ4MhtK58ykvFZ5ikF3KSDYNLqxQcCirNsJqTFM05WLIPkFwGZXHgU2sb4F9QkAVDFMavBDrsXwPIgiHa5bYVDnrB0NTlwy5Z9SOMPHyQfwHcnFZ+VkoV7RW8vP1lTis9jenZ4TUEAVx+hu96w9xCyXpWkDWG/uO3t7ZS84f/o/ra61BnZFD87jtk33wTKv3QVXusd/V6DpYRrizdgSG3eYl5XDXlKlZXreabau9VaQGwW2VTUohmJOjTGEJZcctJbgOeIV085M0YVgd0c1cA9cI8oNxztHMZmnrDzr3MAw/+BYWsJO8LpCElpQBSRvZGoNWHEF04VBzUgsHhlGjxZkpqq5K7LXkJXwwk61OSJB5aKzeXuXnmzREZM6OOgm5jbz6DYp8O5aU0UIXu/OILubHI8uVkXn89xe+8jeGQQyIzbi/02By0mKwhaQyK+clTJ7dLDrmEouQiHl73MFaHDzNB3Sa5rHmIjmeAFtffMJTCZ9nJBs/O21FHQc1PfvtwRIqmLguZSSGMv9eUFF0HtFJaxOsiz4N/QUGZB1HJNi48QtYYJImG9uCjC4eKg1owKHZJjy/Vqh/l3yMP93hsX2KMd8GwqmoV39V+x3XTriMzLjPs8fYbj2t8DR0WUuO1AXUNG0hmop5WsxWLsZXaW26h+rrr0WRmUvzeu2Rdf53XgneRRFlphqJCJ+g1JBs0Hju56dQ6bj3sVio7KnllxyveT6KsyguPDPr6Ci0m+TvQ+kqs8kJOkqG3Mms/Rh0tJ9zVrA95XIFitTtpM9v81xjyQFq8Do1K9CaKRgvlOfK6QKr8DuLSBvkXQA4CsDqcdHRHwYFeeAR01uForaSpyxL1ctsKB7Vg8BlNsn+N3Achx3Otn+Q4DTq1yqvG0GPv4S9r/8KYtDFcMP6CiI2ZjNHyA17VZ5cMNiJJITNRx/T6XVSccQbtH39C5rXXyFrC+MGTZ6hQXuqhTohsV1kJTxydfzRzRs5hyZYl1JvqPZ+g8ntXLarQ28U2d1lC8i8A5KZ40TwLD5cd4sNgTlLMiaFoDCqVCCl0O9I0dlpIifOxQKpaKy+qPJRpiWrNp5GyRaJr97c4nBI5/0s+BiHESUKIXUKIvUKIQaVChRC/EkI0CSE2uX6ucNt2mRBij+vnskiMJ1CafJXDqFoDBbNA7Tl8TwjhM8nt5e0vU2uqZfFhi9GoggsB9IkQ8gPuilhp6Aw+hwHkvISxbzzFAz88hyMugaK33ybrt78dFi3BnVATqxQyEnxHVv3fof+HU3Ly2HoPGdFOh7wACENbAPkeQnHcQl8PjUGlSQwpskPcS+HESNLsKgcRio8BXLkMURcMPsrCmI3QvNtr1FlmtJLcALIngD4Fe4W8APif0RiEEGrgSeBk4BBgoRDCk2H6bUmSprl+nncdmw7cDRwOHAbcLYQIok5EePRWYxy44u7pgIbtfl8YmYme6yU1mBp4cduLnDjqRA7NPTRi4+1l5GHyg2420hhC1nP3pk2UnXUWSZ9/zPujj6PlsSXETQquY1mkMLpWqxlBFG9zJzNR37/W0AAKkgr49aRfs6xi2eCM6MYdYGkPy78A8gslVMGQ09vi04ONvvAoqF4PjqGtldPUJV872HBbhewQqgBEmkZf4bbVLnNcwQEoGFRqGHkY+lp5oRdKdOFQEAmN4TBgryRJZZIkWYG3gNMDPPaXwEpJkoySJLUCK4Ega0WEjtdsz+p1chx5oWf/goK3ekmP//Q4dqed3838XcTG2g+Xn8G5fy2NnZaAHybJZqPx8cepuPAisNnRPf40z09aQHMUOzMqfz+PmecBkO5HYwD49cRfkx2XzcNrH8YpudnyK131iMIUDC1dVjJDFGw5A1t8ujPyMLk/Q8O2cIbnl/8JjcFX1nP1WhBqyPccbh31XIzCI0js2EMKXf87GgOQD1S5/b/a9dlAzhZCbBFCvCeEGBnksUNCU6eFJL2HhJKqH2X7boHv1b4nU9KWpi18VPYRlx5yaegZzv7ImwEqDd3lP8h2yQAeJktZORULL6Tl6WdIOe00ipd+SNYx8guxJRrJPS7CcdyCrGm0mW3YHd7Lh8dr47lp5k1sa9nGJ2Wf9G2oXivXn0oNPZPUanfS3m0jI8SXalq8Dq1a0ODppaQ8f1XrQh5fIIRaWVUhO0mP0eRWPnyYkSRJDjv3Ng+qfoTcSXJZcw+kxmlRq0R0NAbotUwcqt4T8nMUaSIhGDwV/Rn4hHwEFEmSNAX4HPhXEMfKOwqxSAixXgixvqmpKeTButPUaSHL02p7/xrImQj6JJ/HZyb2nxCSJPHwuofJjMvkyilXRmSMHtHFQ+6U3sQYX2UAJEmi9a23KD/rLGxVVeT/43HyHvoz6sREkg2+HejDgdFkDdlxC/ROJKPZt3A7teRUJmZM5O8//R2zzRUCWr1O9iOFgdGlrYRqClOpBNlJBs8aQ0qBXLe/eogFg2uBFEpkG0CWp/Lhw0hHtx2r3elZY3DYoXqD1+hC6Cuj3zycpbfdyZ+BXWiYrd8bcL/toSYSgqEaGOn2/wKgXy0CSZJaJElSnprngJmBHut2jiWSJM2SJGlWVlboESTuNHlq6emwyzbJkf7LL2cl6eUJ4bKTL69YzpamLfx2+m9J0HpenUSMkYdjaNyEBrtXU5K9pYXqa66l/p57iZ85k+KlS0meN693uxBynZhoagzNXVYyEkJfJSlCxd89qISKWw67hUZzIy9vf1luCN9a4Vcr9EdvOYwwVnrZyXrPCWJCyIKremhLY/gsCxMAyhyKljlJyWHweA+NO8Bm8ikYIIrZzwDaOCp1Y5ilGvrieIESCcGwDhgjhCgWQuiAC4Cl7jsIIUa4/fc04GfXv1cA84QQaS6n8zzXZ8OCxwnRsE1+kAKoy9+X/WzF6rDy+E+PMy5tHKeVnjYUw+3PyMNQO3qYIPZ7NCV1ffMNZaefgen778m5bTEjn1uCNid70H5RnRC4NIYQV9vQJxiMfvwMANOzp3NS0Um8tO0lGspd7TMjJhhCv4fsJH3/3s/uFBwmC7CuyGjJngi1HIaCEhUXLc3TaxAJ9OUj+fmeM6NcPnybagJj7XvkTPwDgLAFgyRJduB65Bf6z8A7kiRtF0LcJ4RQ3pC/FUJsF0JsBn4L/Mp1rBH4E7JwWQfc5/psWGjs6Bn8MPlJbHPHPf75zZ1vUtNVw+9n/R61KvTeBAHjGt9M1e5+L1an1UrDn/9M1ZWL0KSlUfTuu6RfeqnXNpuZUdYYWros4QmGICNKbpxxI3bJzpM7X5frEbn6eIeK8rcL58WanqD3LtiUF9oQJro1R0hjiFbDnkZfPc+r10Firl8/UmaiLjplMVxscI5Ghw0atkZtDO5EJI9BkqRPJUkaK0lSqSRJD7g+u0uSpKWufy+WJGmiJElTJUk6QZKknW7HvihJ0mjXz0uRGE8gmCx2TFbH4AlRvU6266aO9HygG8rLoLK1iWe3PMvs/NkcmRdeTHzApOTTpsvhcM1e9BpZEFn27aPivPMx/usV0i66iKJ338EwbqzP02REUWOwO5y0mm2kD4MpSaEgqYCF4xfygbmC3bnjZX9NGCh/u3CchhkJOlrNNs8l3POmyQJsCCutyhpD6MI5qgli+CmgV/UjjDzUY70zd5QIw6iUxQC+7ymS/1G9ISrXH8hBm/nstfBZzQa5k1cAKMd+VvsaJptp6MJTvbBPP5EZqt1IkkTbe+9Rfs652OvrKXjqKXLvvAOVwX+0kpwHEJ0J0Wq2ucYQ+kspxRVRMrAYoC+umnQFiU4nfwvjugotJisGrYqEIHtJuJORqMPh9FLCXRsnJ7oNkQO6x+ags8celsZg0KpJNmii1rCnsdNCnFZN4sBeEl2NshkuEO0/US+XxegZ/rIYFruDvZYUTNqMYSmBEggHrWBo9FQOw2wEY1nAkSoJeg3x8W1s7fiUM0efyZi0MUMxVK9sV40j09JC7Q3XUnfHncRNnUrxhx+SNOeEgM/RWycmChNCeZmHUnxOQaUSpCfoAvIxKKR01LGotZ1vbc2sqVsT8rVBrq6bkaAPviOfG8r9e83HKDhMLqjniPx3FAnnObhKk0TRx5Cd7OE7ULSsAARDNJPcWk02QGBMm9KXjBdlDlrB4LEcRs1P8u8ANQYAQ/ZKQM21066N4OgCo7o+gfIVWXSs+pqsm2+m8IXnPTqYfRHNCWHs7WMQ3kspI0EXXKOV6nUs7OwkLy6Lv67/a/+ktyBpNllDzhhWUO7fa7hnwaFyQETTz563h4HPsjBBkJXoJbJqGGjyVg6j6kdQ6wLyIynzIBpJbsrc686eDsZ98gI1yhz0gqHfA1WzHhByQ/YA2NGyA1vcBjIcvyA7PrgXcjhITictL7zAaUvfQkIw6qYTyLxqEUIdvDlDcfxGwwHdbFIct+GZdOSQ2yAmdPU69IZ0bpj5O3Yad7KsfFnI127utJAZhsYDfRqDdwe0S4MdAj9DuLWqFLKToxfV0+It5LnmJ9kMp/F/b0oBwagskFzfu1NZkNb+NOxjGMhBLRjUKtG/FEPNBrksr5/ENoXHf3ocDQmI9sBNN+FiNxqpuvpqGh95lDV5k4g/M414fWXI54umxqC8zMMxJYG84g7GlET1eiiYxSklpzAubRxPbHwCW4j1iFpM4YV6gptw9nYPaUWQkDUkZgavZWGCJJoag9FkJW3gM+R0QO3GgLX/3nkQBY1BeXZ1hTMBcUA4oA9awdDY2SM3Y1cyDSXJ9cII7EFaU7eG72u/Z7zhTFo6h+fPaF63jvIzzsS85kdSbrud+2ddQlvOFKjZ2NvRLViUCRGNrFWjyYpKQGqIdZIU0hOCCLntboOmnVBwKCqh4sYZN1LdVc27u98N+rpOpySvVsPUeJTFiVfhJoRsThqCRLe+qKrw7iEzSU+3zYHZOry+KqdTotXsIXu+ebdsfgtQMKTF61xlMYZfc1YWBBlpmfLC9ABwQB+0gkEuxeC2SmotlzujBfAgOSUnf9vwN0YkjGBW+nzazLbBjVYiiOR00vzMM1Re9itUcXEUvf0W1lPOBCEwZ00Faye07Anp3GnxWoSApmiYkrqsvRMyHDITdXRa7FjsDv87K2q6yzwzO382s3Jm8eyWZ/tKZQRIR48Nu1MKe7Wt06hIMmh8az0Fh8pd+yJsf25y9TFQQp5DJT0+OibJ9m4bTsmD1lnjWnXnBdanXO0KYoiOKcmCRiVIjtPIC9Pq9fJCNYoctIKhxWTt/zD1Op79RyStrFzJjpYdXDftOkYkJ7nONzQPlN1opOrKRTT9/XGSTz6ZovffxzBhQu8qw6n4Q2pCUz81ahXp8UHa6COE0RRecptCb72kQMxJ1S4/kmsBIITgppk3Yewx+u705gFldRmRe/BXJVapDFq3yfs+IRBucpuCMpda/dSsijTK38yjYNAny42tAiRaDYdaumRTmBBCfv90G+WFahQ5aAVD60DBUL0eNHGQ7bvHsd1p54mNTzA6dTTzS+b3Ok6H4oEyr18vm47WrSP33nvJe/QR1IlyDSblJRiXOx50SSELBpBfbNHxMYRXJ0khPZgkt5oNkDlGboTjYmrWVOYWzuXl7S/T2tMa8HUjFeoJuEJufXwHI6bJv2si65gMN3x5Pc4AACAASURBVLlNId2fn2SIUATRYMHwkys5MPBXXGa05oF7IUkl0CDKfoaDVjAM1hg2yA+Sl45tCh/t+4iKjgqun349apV6SGq5S04nzc89R+Vlv0LEGSh6+y3Szj+vX5y28hJJT4qTxx2GYFCS3IabFpO194USDsqLze+kliTZIenBvHDD9Bvotnfz4rYXA75uJMphKKQn+PkO4lIhvVQefwSRNYbwewAopqTWYRYMyt+s31y29cg1z4IIOwfZgR4NH4PR/V2UNQG08VH3MxyUgsHmcNLZY+/7MuxWqNvs90GyOqw8vflpJmVMYs7IOUBfDHpQUTE+cLS1UX3tdTT99TGSTjyRYpfpaCC9DqsEvTzu+m3yhAiBaJXFaOkKP9QTgvgOOmqhq8Fjw5bS1FLml8znzZ1v0mhuDOi6kXLcgmxK8jv+/BkHvMYQqXkQKEZPpqSGbeC0B+xfUMhyFdIb7ioAciFJ1+JCrZG1wygnuh2UgqF14MPUuB0cFr+C4d3d71JnquO3M37bu3qP5ITo3rqV8rPOpuu778i54w7y//YY6sREj/sau6zEadVyk6H8meC0hdzpKxqF9Kx2Ods6Eo1J0gPNxVBW217yVK6eejUOp4MlW5YEdN2WLgsqEXr3OXfSE3W0mv2UJsmbAZ210Fkf9vVALodhsjrC6oehkKTXoFWLKAgGDyHPivYcpMaQmajHanfSaRneyKqWLkv/76BgJtRviWql1YNSMAxyWPX2hPXueDbbzCzZsoTDcg/jiBF9JbkTdGp0apXfRjG+kCQJ4xtvUHnhRUhIFL3+GukXX+SzzEI/9VOZACGakzIT9XRa7PTYAojqiRBebcMhkKSXGw41+wsAqP1JLkiXO9nj5pFJIzl77Nm8v/t9qjqrPO7jTlOX/B1EorlKRoIOm0PyXZpE0XQipDX0fQfhC2ch5Jyg4RcMNhJ06v5Nhmo2QGIOJOcFdS4lyW04HdDKAqnfPMifCQ6r3EsiShyUgmGQxlC7EeIzIcV7RdU3dr6BscfIDdNv6PfCFsJVqyfEFbfTZKL2//5Iw31/Iv6oIyn597+JmzLF73Et7n0MkvPk0sIhvjAyo+A4jEQfAwWl4ZDf76B2I2RPkAvTeWHRlEWoVWqe2fyM3+u2dIWf3KbgN/sZZIEmVBHzM/SZYbQROV8g/bcjjdFkGeynqvlJfrkGWb8q0mbhQPC4QFI02gj7k4LhoBQMgzSG2o3yaszLg9Rp7eSlbS9xbMGxTMueNmh7WoIupDA9S1kZ5eefT8enn5J1042MfPpp1KmpAR3bT2MQrvDLMDQGGN6szz6nYeRerD5fSr2OZ9/lTrLjs1k4fiEf7fuIfW37fO7bHGYvCXf6BIOP70CXIDsnI1QyQS7eFhlTGMj3MOzOZ5O11/ENyAmMLXuC9i9AgMI5wrT01gtzu4fUURCXFhMMw00/KW01yZmwPl4Yr+54lQ5rB9dNu87j9vQEbdAPU8fyFVSccy6OFiOFzz9H5tVXe22m4wnjwKiq/BnyhOhuC2ocEHyzm0gQbq/kgWQk6n3nYrRWQHdrQC+M30z6DXGaOJ7a9JTP/VpM1ohpDH2F9Pw5oKfLK+IIOEiV3JtICrfhNiW1mgfMAyXPw0OAgT96czGG8R765oHbcyRc9dr+2wWDEOIkIcQuIcReIcStHrb/TgixQwixRQjxhRBilNs2hxBik+tn6cBjhwJl8qXGaeVoHsnpVTC09bTxyo5XOHHUiRyS4TnHwWcHrgFINhsNDz1MzU03oR8zhuL//JuEo44K/h5MAxxWYSRAZUahkF6vKSlCGkOmP41BWWUHUCAxzZDGRRMu4rPKz9hl9N6HN1J5GBBEEEPeDDkBqi30+lgKygswkhpDOL62UDB2Wftrnb0Zz4EVwnRH+TsMpznMa+n5vOmuftXR6XERtmAQQqiBJ4GTgUOAhUKIgW/QjcAsSZKmAO8Bf3Hb1i1J0jTXzzA0S5ZXGanxWjRqN3vtiMEmIoCXt7+M2Wbm2qney2qnxwemMdibmqj89a8xvvwyaRddxKhXX0Gbmxv0+M1WOz02Z/8JoYy/NhTBMPwduIwma18ZgAjgt15S7UZQ6/0mMCpcNvEykrRJPLnpSY/be2wOuiz2XodluPR2ogskZBUi4oA2mm2ICNSqUkhP0NFmtmF3DF15GHckSXLlI7n5SGp+grRiiE8P+nxxOjVxWvWwagweTUkgCwanHRq2D9tY3ImExnAYsFeSpDJJkqzAW8Dp7jtIkvSlJElKIZo1QEEErhsy/eyStRtlx23yiEH7NXc388bONzi5+GRGp3lPrU9P0NPRY8fmY0KYN2yg/Kyz6dm+g7xHHiH3zjsQutAmpMeHKT5d7msbgvpp0KpJ0A3/hEhXygBEgIxEP0XcajZC7iTQBPY3T9GncOnES/my6ku2Nw+enL05DBGIqgL5O4jXqf0vMLInyj0GIuBnMJospLo64EUCZdXb5qkT3RDQbXNgsQ9YINVuCsmMpDDcWo/RZEWtEqTEDQgA6HVAR6cEdyQEQz7gHttX7frMG5cD7gXwDUKI9UKINUKIM7wdJIRY5NpvfVNTU1gD7lcOw4dD8qVtL2FxWLhm6jU+z6esWDw5oCVJwvjKK3IBvPh4it5+i5QF88Mav8ekHpDvI8RaOmnDbB9uMVkiEqqq4LOvhNMp/12CdEhePOFiUvQpPLHpiUHbem3DETIlgaL1+NHaNDrImSQLujBpNdkGl6sOg+F23g5aIJmaoaPaq/YfCMPtJ2kxyYUkVQOFc3K+XGo9BAtAJIiEYPC03PDoGRNCXAzMAh5x+7hQkqRZwIXA34UQpZ6OlSRpiSRJsyRJmpWVlRXWgHvrt1u65PK8/8/eeYe3Vd2N/3O0reG948TZeyfsUcqmBUJbSKGlpewOCt1v+2vfDgotlL7lpYVCodDS8bZlj1LKagJlJiGELJM4O957SbZkS+f3x9WVZVuSr6R7ZafR53n8yL5LR74693u+O4ZgaPW18rddf+P8meczPW96wusVRJxWI1dKIZ9PCUX9yU9xf+hDTH/sURxz56Y1dnX8wNgwvYrlw07WJMn0SklPxy2MY4ppr4VAX9IrSbfNzZWLruT1+tfZ0jJygqr3QM8H67iF9FSmrFQEXSi9vJOO0RE9aZLpCqtj7oH6EK1MXTAUZDiyqmO0r1Blgh3QegiGOiA6AaAKaBh9kBDiTOC7wIVSysiySErZEH7dB6wHkvcaJUmkaFXTVkDG/CI9tP0hhkJDfH7p58e93nDP3uHVXuDgQQ5cehk9zz1HyVe+QtWvfonZo60BkJbxQxy7JKS0yihwZnZC6NHHIJqiRH0lxsl4TsRl8y+j0FHIve/fO2K7ngl6KppXq5UrFUHXviet9xsT2ZYm6kIlUxVWO0bfg0bVXzh+K894FDqtGTclxb0HlSuUdq6B5MrB64EegmEjMEcIMUMIYQMuBUZEFwkhVgC/QREKLVHbC4QQ9vDvxcBJgKHpflJKOlWNIY7jucXXwiO7HuHCWRcyNTd+0ptK4SiNoXf9evZffAlDzc1MfeABij9/fVKhqOMRswwADE+IFMxJE2Fb1dWUlEhjqN8MVhcUJ6+tOa1Orlx0JW82vMl7LcOrt47wvdZXMGiMbqtMPdAgmo7RoZ5pUpjoHhhAx2hTUsMWpdBgVOXcZCl02cdo/kbS3pegkGTlCiVismlbxsajkvbTSko5BNwAvADUAI9IKXcIIW4WQqhRRncAbuDRUWGpC4BNQoj3gXXAbVJKQwVDz8AQQyGpfJkatii2PE/ZiGN+u+23hGSI65Zep+maqgrd0TdA6933UPf5L2CdWsX0xx/HffJJun+Gdm8Am9mE2z4qosdZqLSBTEH9VJKTMjMhIhE9epqSEvkYGrdAxVIwpdaMZu28tRQ6CkdEKHWGnYa5Dn2iqiDcu9o7Tr0kgOJ5YHGk1ZtBXSDpKRgKMlxhdYxJtWFLWmYkUPyFfVqbPunAiJLbo4lEGmbenKTLt1pK+Q/gH6O2fT/q9zPjnPcmELtwjUGMKIcRw/Hc5G3isd2PsWb2Gqo82oKnClw2nIP9VP/iB7Rt30jemgsp/9GPMDnSL2cci45EET0Vy1MWDOqESLeb13jENYWlgdNmIcdqHmtKCgWVFdfKK1K/ttXJVYuv4uebfs67ze+yqmxVxGmoV1QVKP+PwFAIbyA4VuhHY7YoDujG91N+L3WBpKdgsJpN5I7XiU5H2r0BrGaBx26Jcjxfn9Y1o/2F5XnGzoPBYIju/sH4AQy5FeCpmBDBcNRlPqsPpWKbX3FKjjIjPbjtQaSUXLv0Ws3XDO3by69e+yVFOzdT9r3vUXHbbYYJBRjPLrlcSX5KsgXk8GrPeK2hI1YNfR1QV9wjaNsNg760V5Jr562lyFEUyYbuHB0/rwORqB4tztvK5dC4NeVe33ont6lkMqqnM1o46+B4hijtPwOfIeKnSuRrmyAH9FEnGNQJMcW3W9kQpTE0eZt4vPZxLppzEVPciSJuh+l54UX2f/JSXEN+nv7sd8etiqoHIwrojUb9PEmaGdSHXCYmRFukFIN+piSIE9WjPjDSCGEEyLHkcPWSq9nQtIGNTRvp8AV0f6hGzGFa2sRWLFd6fXckrucUj/bRZhidyKRgGNFsqyF9xzNktkVphxbNuWK5srjx9xo+nmiOOsEQuRk9NcqGqBVGRFtYMr62IINBWn5xJ/U33YRjzhzuvey/2Vk805AxjyahxqBOjCQdkxGNIRMTwiCNIWaoYeMWpSNW8Zy0r3/J3Esozinmvvfv090+D8MFBTPhgI6YVA3QGDLmfI7OhWlM3/EMmXWgx+w+N5rK5YDMuAP66BMM4Qefu2ObUmbbVQyM1BYq3YnruAe7ujh8/edpv/9+8teuZdof/4CltCxjK6WEgiGnQCkJkLTGMAEqtN4P1lj9ABq2KOWqU3Q8R+OwOLhq8VVsaNpAe7BGf1NYMg+lkvlKiY8UHdBjQj11IpMVVjt9g1EaQ/qOZ4j2MWRAMGjVGCAtf1IqHH2CwRvAbjFhbt46Qu3Uqi0M7NrN/kvW4n3nHcp/9CMqbv4RJpstYyq0f0iJ6En4ZapM3gGdaRVa74geiFH+XHU8p2lGiuaSuZdQ5ChiwPVPQx6qoFE4m61KiY8UNYa42fNpoobcZqI9ZqTzmQ4Zzyr5OVaEyMwCqaNPg0nVU6aU7MlwBvRRKRimOocQ7XsiKwyt2kLP889z4NJLkQMDTP/jHyj45NrIPjUPwOgJMTyhE3yZKldA16GkHNB54QmRiazVTp/+ET2g3ANfIDjcia59Dwx6dVlJqjgsDi6bdwVm1168ola36wI4bWbsFpP2h1LFcmUlmYIDutMbwGYx4bTpG3lT6LISCCqRVUYyGFQ6nxWoYeeQUgLjaCxmE3k5yZfRT4UObwCTCFd5TkTl8rRCk1PhqBQMK+11yh/hFYaqLVyz5JqY58hgkJb/+R/qv/o1HPPnM/3xx8hZPvJhUxAONfQZPCE02SVTiH9WJ0SmNAY9Q1VVxmg9OjmeR3Nq+QWEhty82/M3Xa8rhFAc6FqFc2XYAd25P+n3Usth6C+cw34SgxcY6j0uGpGoOn7nQy0UOjOT7Bm3TtJoVAd0wGv4mFSOOsHQ7g2wzByeSBXLaPY283jt46yZvSZmJFLEn/DAb8n/5Cepfvj3WEtLxxyXKRu9pgY36gRp2prUtWPa6A1AKd6mb6gnDDvQI5+hcQtYclLKeE6Ez28i0P4h9vVtYXOzvtUvC922xF3cokkjAUrvzHOVSHSbwQ/WEZqzTo5nlUzVS9J8DyqWhTOgtxs+JpWjTjB0egPMDe0DTyW4S3lo+0NxtYWB3bvZv/aTij/h5h9R8aMfxi2Vnan4Z0224ZyCcAZ0kpFJKbYoTRa9SzGojBHODVsUO7xZX19Gh3eQwc7jyLUWaOoNnQzJNH1SHNC2lMwMxt0DNbLK2N4eqkZS4LIq5jQdzYWZ8hdqFgyVmXdAH3WCocMbYHqgFiqW0epr5bHdj3HBrAvGZDn3vPgiBy69jFC/j+qHH6Zg7do4V1TQ3IErTTRnDVcsS/qLpEyIDCS4efXPAYBRuRihkKIx6WxGgnDEirSxdu5neKvxrTGVV9NBc4VVCJfgXpSSYzJSL0xnIpFVBpuSVI2k1OSF7sO63ueMac5ahbOnQinBnUE/w1ElGPxDQYL+PooHDkLlch7a/hBBGRyR5SxDIVruuov6G2/CPmc2Mx57HOfK8Z1amdMY/OGInnFMMRXLFNtzEj2gCzNQYTUYknQZtFodUaunY69SgVTHlaSK+lD6zMLLKLAXcN9W/bSGpMM9K8IZ0EkGPSSs0ZMGBaP9PAahzrPivnA+UpqJbdGomnMmAkk0CWchhgMNMsRRJRg6vYMsEIcQSNqKZvLo7kc5f+b5TPUoFVSDvb3UffFLtN97H3mf+DjVf/wj1rKx/oRYZG5CDGp3WEFSfga1WY+RE6Knf5CQ1L8UAwxHVnX4Bg1zPIMyoR1WE4VON1csuoI36t9gW6s+CUiFLhve6Miq8ahcDv5u6Nin+T0GgyF6B4YMuQcumxmbxWR4gpiqkXg6w931dHI8g6L1DAYlff443QB1IBSSSh6G1ntQsQxaamCw37AxRXNUCYYOb4AlJsXx/PueGgZDgxFtwb9vPwfWfpK+119X6h3dcgumJFpv5josWEzC8AmhZHtqcNymkBiTiVBDoxKrICqyyhtQ1G6zHUrm6f4+0Q1uLpt/Gfn2/DH9GlIl6Qz0FO6zpho9KSKEyIjm2eENkJdjxdz0PuRXK341nSjIQCBJ78AQwZDUbs6rXA4ymLEe0EedYFgs9tPkLOaRA3/nozM+SnVuNb3r1nFg7VqC3d1Me+jBlOodCSEyEs3QGdYYxsVVpGR2J2F/zkTZ5BHVbQ0g0lei8f2w41n/6Kdo+7zT6uSKRVfw7/p/x+wNnSxJ16wqXQAma1KCIRLAYIDGAJlx3nb4wqYwnR3PkJm6YcMLJI3fzzR6raTC0SUYfAEWm/bzcEkl/mCAaxZfTdu991L3xS9hmzaNGY89iuvYY1O+flGGJoTmh2qSDuhMhNwalXGrUui00dU3oHxuA8xIMPYeXDrvUnJtubr4GtSoHs1Vbi12KFuYkmAwImQYMiQY+gJMzfErrWx19C9AZuqGaUpUjSZvKuQUZiwDWhfBIIQ4VwixSwixRwjx7Rj77UKIv4X3vyOEmB617zvh7buEEOfoMZ54dHd3U2xu5HFzL+eXn4nt+3fRetcvyT3/fKr/789YKxPXSBqPggxEMyRVvK1imZL9q7EyY0QwGDgh1MlmRESMel1b7yHw9+j+wFAZfQ/cNjefXfhZ1h9eT017TVrXVleQmiqsqlQsU1aSGn1DqtCJ2wcgTTIhGDp9AZZYDil/6HyfiyIht8ZF6CWttQmRUqRhqqQtGIQQZuAe4DxgIXCZEGLhqMOuBjqllLOBO4Hbw+cuRGkFugg4F/h1+HqGYG7dyV/yXeR1BPnMXTvofeUVSr/1LSp/drsu/ROMbo+pOKySEQzJVWZMqh9AirQbbcZw2qjw7VL+MEgwxAq3/dSCT+GxevjN1t+kde2UzHkVy6C/Uwnb1ICaY2CkxmC489kbYIEMO9x11gwLIqYk43IxOlPR2iqXKw7oIWNzREAfjeFYYI+Ucp+UMgD8FVgz6pg1wMPh3x8DzhCKEX8N8FcppV9KuR/YE76eIZg632NLu5s7/gCirYupD9xP0VVX6lYWwOiVUneyET0Ru6S2VUYmIqs6wxE9OTrX6FEpcNmYHqhFmqxQOnp9kj5qjZ7Rwtlj83D5wst55dAr7OrYlfL185224cgqrSTpgFZXwkZEJYEyD3oHhhgMptZEaDzUtqQzBvdAblWkQrJeuO0WrGZhrMaQShBGxTIIDSrCwWD0EAxTgOilSl14W8xjwj2iu4EijefqgpQS/zsv8vVHJfbyKcx47FHcJ+nbj7nAZaO7f5AhgyZE0l8mtTKjxgeGx65EVhnrY0giRC8FCl1WFrCfUMkCJQFMZxKZwj694NO4rW7u33p/ytc3mwT5amSVVsoWgTBrtj93+gLkOixYzca4GCM1qwz6HqltSSv7dxuiFQohKDA4sqozXOU5x5rEAmn2WfD1XYbk5oxGj29GrOX2aGNnvGO0nKtcQIjrhBCbhBCbWltbkxyicrOdPUPsmWVh3qOPY5s6NelrjEeh04qUysreCFKK6KlcrvmBEYmsMtjHYJR/AaAgx8pi0wH6ixYbcn3VPh9LuOXZ8/jUgk/x0sGX2NO5J+X3KEhW87TmKOUxNC4A2pPxU6WA0b6qTm8AF/3k+Q4a9pA02iysFpJMylphd4On3LAxRaOHYKgDop+yVUBDvGOEEBYgD+jQeC4AUsr7pZSrpZSrS0pKUhroxX/ewEeffg+Ty5XS+eNR6E6iA1cKtKciGCqWQdsuCPg0HW50ZJVRxdtUppjaKBB9dOYvMuT640X0fGbBZ8ix5KTla0ipJEMSDmijymGoGO2ravcGWCgOIpCG+ZGMNgtrznqeIPQQDBuBOUKIGUIIG4oz+ZlRxzwDXBH+/WLgX1JJr30GuDQctTQDmANs0GFMMREWCxadC6pFY3RZjGGHVZKCQYagWVtlRkWFNs62mpTzPAXKvYp9v9mlf2IbRJd7jh3Rk+/I57L5l/HCgRfY1609GzmalLS2yuXgbYXepnEPNaochkomNIbFpuEKyUZgdE5SKkUMQzJEs7fZoBGNJG3BEPYZ3AC8ANQAj0gpdwghbhZCXBg+7EGgSAixB/ga8O3wuTuAR4CdwD+BL0kpjW1oYCBGd0GL+BiSsdEn6ZhUIkqMi3owqoCeSlFPDUPSxGHrDEOuryUH4LOLPovD4kjZ15CS1pZEAlSnwffA6HwYNR8p6CozzLRSZLApKZV78PLBlznvifN0SaQcD128T1LKf0gp50opZ0kpbw1v+76U8pnw7wNSykuklLOllMdKKfdFnXtr+Lx5Usrn9RjPRGF0I/FOb4Acqzm5iJ7cSnAWa/YzFLisdCYTEZMEgSGlRo+RGoOzYwe1cgptA8Y4ViOCIcGkLnQUcum8S3l+//Mc6D6Q9HukVMStbDEgxl0ASCkNK7mtonYkM0wweAMsFgeQ5frVRxpNgdNGl8/AQJIkTaohGeK+rfdR5alifuF8Q8YUzVGV+Ww0kfhnA22rSU9oIcKtATVqDE4bXb4AwZD+hfS6DE5uQ0osze+zQ840bLXX4Q3g0RDR89lFn8VmsvHAtgeSfo9Cp1LErTeZIm52t9KQaJz77A0ECQyFDLVvW8wm8p3Gtcfs6+lmtqjHPMW46Bx1nnUZEEgSL+Q5EesOraO2s5brll6H2WRYqleErGDQEbvFjNtuMVRjSCkpqWIZtNbA4MC4hxa4bISkUgVVb1IyhSVDbyPC28p+62zD7MNafSTFOcVcMu8Sntv3HId7tCWeqUTySVIxJ42jGXZoaQ2rA0b2NLB37MQsJEKHHs/xSPkeaKArrJFrFc5SSu7beh/VudWcO/1c3ccTi6xg0JlCA8M9O3waC+iNpmI5hIagZXzbpJGOQ6Nr9Kir5YacuYaaMbTeg6sWX4XFZElaayhK1UZfsQx6G6CvJe4hqv/ISOczGBvVU9it9mAwTmMoMtAsnGw5jPWH1/NBxwdcu+RaLCbjgmeiyQoGnTFyQnSmGk2ixnpr8DMYmZwUyQEw6qHUsAUQdHrmGyacO33a70FxTjEXz72YZ/c+S11vneb3SLnss4ZM985UMm5TIOlcjCQo931AtylP8Z8ZxJj+4TqSzAJJSsm9799LlbuKj878qO5jiUdWMOhMkctmWFvDlGOf86Yq9eo1RKyoE8KQlZLRD6XG96F4Li5ProHCeTCpe3DV4qswCRO/3fZbzeekHPasNqtJsABQv5tGFdBTMTIfZrq/lvqceYr/zCCK3MbNg2SE82t1r1HTUcN1S6/LmLYAWcGgO0atlPxDQfr8Q6nZ55NoDWisxjB+RE9aNG6BimWGRla1e/1JCbZSZymfmPsJnt7zNPV99ZrOUVeSSWs9jjwonJVwARAxYxjQpCcaw9pjDvZTLQ/T5lmg73VHYWRvEq2l51VtYYp7CufPOl/3cSQiKxh0Ro1/1ntCqA6rlCd0xTJo3jluZUajfQxaInpSorcZehuhcrlhkVX9gSADg6GkBdtVi69CCKFZa3DbLdjMptSKuI1TAqXDG8BmNuEyqIihitoeM6nIKg0MNmzDQojeAmMy21VsFhMeh8WQRZ7WBdK/6//NjvYdXLvkWqwmg/xyccgKBp0pdNkIDOnfHlM1AaQc0VO5XFNlRofVjNNmNiTk1tByGOoquWJ5JLJK75pVSXfdClPuKufjcz7OU3ueorGvcdzjlZpV1tTKPlcsh5468LbF3K3eA70qCsfDqBX3wMF3AfCXGpfDoFJkUPnwdg0LJCkl971/H1PcU7hw1oVxjzOKrGDQmYjjUOcHa9oNbiIZ0Nr8DEZoDJ0+AzNuw45nKpYalnmbjinsmiXXAGiOUFKaPqWoMUBcrcHoWlUqhQbZ6IMNW+iQbhxF1bpeNxZKIIn+VQC0hDy/Xv8629q2cc2Sa7Aa0J52PLKCQWeGw9z0/UKl3RKzYLpig9YQmVTkNsaBbrjGUDwH7B7DWjO29YVDPVMw55W7yvnEnE/w5J4nNWkNKYc9RyKT3ou5u90bSGn8yVJokMZgad7K9tAMCgx2noPSdtOoeZBocaFqCxWuCtbMGt3aJjNkBYPOGFUvKe0wwyRaAxoVUWJojZ6G9yJakVEag3q9VCN6VK1Bi68h5SJujjwonJlQYzCyTpKKIeVhBgdwdu1mu5yREa3HsHkwTsjzGw1vsLVt64RpC5AVDLqjPjT0Xmmo11Pr0KRExTJo3gHBxCaKQpfdmPhtn0GrbYRHEwAAIABJREFU1SjHMxgXWZVuRI+qNTyx5wka+mJWl4+QVhG3BBFoGTMlGXEPWnZikkNsC2VGMBS6jYmsShTyLKXk3i33Uu4q52OzP6br+yZDVjDozHC/WP01hrwcK5Z0InoqlkPQD60fJDysyG2jrc+v64RINaJHE1GOZ4hKTtLdlBTAahZ47KnHk2vVGtIq4la5XOn/7G0fsVkNeTY66xnAaTNjt5j0nQfh+7xNziDfafxKWo2s6hnQN7IqUcizqi1cu+TaCdMWICsYdCcSaqjzQ0mXlZ5aW6Yhtv1ZpdBlwz8UwqdjZJXqc0k2okcTUY5ngBybmRyr2QCNwU+Ry55WRE+0ryGR1pBWEbdIoMHI+xzJPM+Aj0EIoX8VgIYt+Mweeu2VhrUljcaI7OdECyQpJb/e8msqXZUTqi1AVjDoTmRCGBCVlLZgKJgB9vEd0CnX6klAp5EN6Bu3QNFssHsimwoNCDXUywyjJUIprSJuqgN61H3OVJ0kFd0FQ+P7HLLPiXRKNBpVgOoZmZQo5FmNRLp26cRqC5AVDIZgRL2k9j4dnIYmE1QuG1djMKIcgKHlMBq2jOn9W+Cy6q4xtPXp4yNRtYanap+Kmw2dlnDOyVei0EaFJmvpJaEnuvZNHvJD8w5qTbMoyIAZCaIiDHVc5A33bR8p3FRtYYp7yoRFIkWTlmAQQhQKIV4SQtSGXwtiHLNcCPGWEGKHEGKrEOKTUft+L4TYL4TYEv4xrlxiBjFitapoDDpMiMoVSpvPofjjK4w40PVbKaXUllQLvc1KRdFRlTaNcKCrDdz14Jol1yCE4IGtsbWGtM0YFcuhYaQDOhJVlQFTEui8QGreAaFB3g/NGvNQNQojotuGw85HzuV/1/+b7e3bJ9y3oJKuxvBt4BUp5RzglfDfo/EBn5VSLgLOBf5XCJEftf+bUsrl4R9tbcYmOXqX3pZSJl28LS6VKyAYSFiC24iSw+q1dO/FoK6KR9XmLzbMlKTPQ6ncVc4lcy/hqT1PxezXkHZpksrl0H0IfB2RTR1xVqtGUaBnT4aGzQBsGqw2xk8Vg0iEoQGCIVprk1Jyz5Z7lCzn2ZnPco5FuoJhDfBw+PeHgYtGHyCl3C2lrA3/3gC0ACVpvu+kRm8fgzcQJBAM6bNa1eCALnLrv1Jq7/NjMQny0gm3jcUox7OK3kl6A4PhiB4dV9tXL7kai8nCb7b+Zsy+SCG9dDQGGHGfO7wBTAL970Ecilw2egeGCAzp0B6z4T1kTiE7fHkZE2xqEIMxGsPw92jd4XXsbN/J55d9PuM1keKRrmAok1I2AoRfSxMdLIQ4FrABe6M23xo2Md0phMjMHTeYIpeNXv8Q/iF9onpUIaOLbTi/WinBnUAwOG0WHFZ9Qw3b+xTHrcmkc42eGI5ngCK3nf7BIL6APqGGw8lt+gmGUmcpl8y9hL/v+zsHew6O2Kd2A0ypLAZElcYYvs/t3gD5Thtmve9BHFQNt0sP7bn+PYIVKxgMGhTZFge9/YWdPkU45zqUzxCSIX695ddM80zj/JmZraCaiHEFgxDiZSHE9hg/SXlIhBAVwB+BK6WU6hLiO8B84BigEPivBOdfJ4TYJITY1NramsxbZ5zhiBJ9irjp6rgVQtEaxnNAu+yREhB60O71U2RENEkMxzPo7zhMuyRJHK5ecjVWk5X73r9vzL6UC+mBIvwLZ47UGPoyk9ymoptJMuCD1hq8RUuAzDnPQdE89dYYCpzDC6SXD77Mrs5dfH7Z5zPab2E8xhUMUsozpZSLY/w8DTSHH/jqgz9mT0EhRC7wHPA9KeXbUddulAp+4HfAsQnGcb+UcrWUcnVJyeS2ROkd7qm747ZyhVJldbA/7iF6T4i2vgDFejs9exoVx3PlyjG7isNCSC/hlk6dpEQU5xRz6fxL+cf+f7Cva9+IfYVOGx3p9JWoXAn1myN/ZirrWUW3vslN20CGaMtTSm1nynkOxmgM6j0IhoL8esuvmZE3g4/M+Ihu76EH6ZqSngGuCP9+BfD06AOEEDbgSeAPUspHR+1ThYpA8U9sT3M8kwK9oxl0N2NUrlB6QDfHd0DrPSHavf7Iw1o3wg5JpowVDJGQW501BiM6n125+EocZgf3bLlnxPaU6yWpTFmlCM7eJiBckuRI1BjC97nRNR/IrMag9zxo6xvuwvjCgRfY272XLy77ImaTsf0xkiVdwXAbcJYQohY4K/w3QojVQgg1538tcCrwuRhhqX8WQmwDtgHFwC1pjmdSMFxATJ/VaofuGkP4QZrAnFSoc4vS9j4DHkr1m0GYoXxsbX7VbKX3PTAia7jQUcjlCy/nxYMv8kHHcLmStB9KqsAMaw0TpjGk62NoeA88FTQGlWh4o9uSRqP0ZNDPpNrW56fEbWcoNMS979/L7PzZnD39bN2urxdpCQYpZbuU8gwp5Zzwa0d4+yYp5TXh3/8kpbRGhaRGwlKllKdLKZeETVOXSyn70v9IE4/uGoMvgMWUXo2eEeRWgqt0hJlhNHpOCF9gCF8gqL+PoWEzlC4Em3PMLlUItekk3PSok5SIKxZdgcfm4Z73hrWGwnTDPcuXKoKzYTPBkNQnez4J8nOsCKGD1la/GSpXRPUkyaTz2c7AYEi3IIb2sEn12b3PcqDnADesuAGTmHx5xpNvRP8B5DttCKFfZcmOsPqpW9ctDQ7oIrd+EyLSgF7P1baUygNjyoqYux1WJapHP1NS+nWSEpFry+Vziz7H+rr1bG3dCgxHVnlTbY9pc0LpAqjfTJcvgJQGZZ7HwWI2kZdjTU9jGOiB9lqoXEGHdxCrWeA2SDjHQs8ghsBQiO7+QfJdJu59/14WFy3m9Kmnp31dI8gKBgMwmwQFTv0SrNr6DLDPV66Atl3gj62kFeo4IVTHra7O5459MNCl2NHjUOTWT+vJhBnm0ws+TYG9gF+99ytg+P+VlgO9cgU0bKYzUsQwc4JBfb+05oFaPrxyJR3hqqRGtyWNpkBH7V/9Lh4eWkejt5Evr/hyRj9LMmQFg0EUOK26mZIUwaDzhK5cATKkRHzEQM/s54jGoKdtWNV2YkQkqRTp6CfRq05SIlxWF1cvuZq3G99mQ+MGij06RFZNWQn9nfQ1KalDmbTPg2IOS0tzVgMMwhpDJh3PoK9ZuL0vACLA2x2PsKpsFSdUnpD2NY0iKxgMoshl11FjCFBihMYAwxNvFMMTIv0Vd6Sqp54P1vp3wZKjmEriUOTWLxdDzzpJifjkvE9S6izlrs13URx+v9beNL5HYcEpw/6kidAY0nqoNrwH+dPAVaSY8zIYqgr6LpBa+/zYCt6id7CDG1fcOGm1BcgKBsMoTDfUMIyUktY+PyUenQWDpwxyq6BuU8zdqulKH1OSARpD/WalDEaCgmPFbv3MeXrWSUqEw+LgC8u+wNa2rdT2vQOkqTGULQKzHXuLUlPqiBMMYcczQKdvAjQGHUtv13W1Yytez6qSE1hZFl/TnQxkBYNBFOqUINbrV2rN6O5jAKhaDfWxBYOePXvb+wK4bGZybDrFageHFNtzAjMSKIKowxsgFEqvE50RdZIScdHsi6jOreaPu+4DQukJBrMVypeQ16GkCGUyogeGC0qm1A3Q2wZdByP3ua3Pn9E8DACP3YLVLHSZB680PoIw9/OVVTfpMDJjyQoGgyh0KhMi3YdSW2/YcesxYEJUrYauQ9A3NmFdz9aM7V5/xF6uC601MNSf0PEMiukqGJJ0p9IFLQoj6iQlwmKycMPyG9jTtYfcku3pm8OmrKSkr4Y8uwm7JbOJVIXh9pi9qURW1b+rvFYdw8BgkN6BIf0153HQq/FWq6+VLd3PEupdzvKyRTqNzjiygsEgCl02QjLF1oxRtKqCwQiNYcpq5TWGOUkIQbHbrospSffktvr4Gc/R6JXkZlSdpEScPf1s5hfOx1TwAi09vvQuVrkSW6if5c6YFWsMJa3otrqNSh5G5fKIcMy0YAB9env8ZutvCDFEnn/yFMpLRFYwGIRepatV+7whgqFimTLxEpiT9LCttvXpXECvYTM48pQicQnQxXmLcXWSEmESJm5aeRNBczt7/S+ld7GwAF1l2TfOgfqTVs2quo1QthBsrsgCaSIEQ1GaIbeHew7z+O7HKQydQqmjSseRGUdWMBiEXmFuwzkABkwIm1NxTsZxQOvViU73Anr1mxW78zhRHXprDJkO9Typ8iTyxUJaLc/RF0ijKEDRHHqFi6Vyj36D00hprvI/Ux/smgmFlPtcdcyI80vcDl3Hp4V0Hei/2vIrLCYL1p6zjZnHBpAVDAYx3JoxvYdSW58fkzDQjFF1jBISGBrbTEWPPIBQSEayhnUh4IOWneOakUC/QnpG1klKhBCCle5PI019/H7H71O/kMnEduYwd7BGt7FpRQ2zTlowtO0Gf0/E3NnaZ6CvbRzSiTDc0b6D5/c/z2cWfobO3hxKJmD8qZAVDAYReSjpoDEUugxsrlK1WpmAbbvH7NKj9HZX/yAhqaMZpuE9pTLs1OPGPbQgXJok3d7VRtdJSsTcgoUMdi/l4R0P0+pLrQ+JlJJNQ7Mo9+8Hf6/OI0xMQbgxUEvvQHInqubNsMbQ1jsxWpvynqk13pJS8otNv6DAXsAVCz+n7wLJYLKCwSD0KinR2hswVv1UHdAx/AyFrvS7oLVH7PM6fYbDSmy/+sBIhNkkKHTaaEtTuBldJykRxW4b/tazGQwNxmzmo4U+/xAbg7MxERqO9MkQJpOg2G1LXmOo26j4kYpmA9DaN0CB04rNkvlHlqopJtt46/X619nQtIHrl13P0JCdkNS5LIyBZAWDQdgtZvJyrMmvlEZhSJ2kaIpmgz1PmYijd+kg3CLOc71MYXUblTE7CzUdrvR+Tt/HkOnEMJVitx05WMyHK9fweO3j7OtO3oHc1hdgS0h5wMa6z0ZT4rGnIBg2KeHIJuUR1dprQJKnRoazn7V/hmAoyC/e/QVTPVNZO3ft8DyYoM+QLFnBYCClHjstPen7GAxdZZhMir2+buxKUg8H+nA5DB0mhJRweANUxW30N4YiV/oht5mokxQP9f92asllOCwO7tx0Z9LXaOkZoAcXvtxZcDjzgqHU44j4CDTh71P8SFFa4UQKBjXjPZl58MzeZ9jTtYebVt6E1Wwd1pyzpqQsZbkOWpJdKUUhpTReYwBlArbsgIB3xGY9Qm7bI+G2OjxYO/eDrw2mJiEYdCiLkak6SbFQ/29+v5NrllzD+rr1bGjckNQ1msPfwUDlakVjSCULOQ1K3ElqDA3vKQUeowVDJuZBHNQFktaQW9+gj7u33M2S4iWcXa004WmN5GEcBaYkIUShEOIlIURt+LUgznHBqO5tz0RtnyGEeCd8/t/CbUD/YyhNRYWOwhsIMjAYMl79rFqtTMSGLSM2q6ubdDJv28NRVfl61Lg5HH4gJiEYinUopNfe589InaRYROcBXL7gcipcFfx8088JybFRZPFo6VHMmbbq46C/A9r3GjLWeJR47LT1BQhqrQKg+rvCme1SStp6DSgkqZGycMitVu3/4R0P0+Jr4RurvxHxSxmaj2QA6WoM3wZekVLOAV4J/x2L/qjubRdGbb8duDN8fidwdZrjmVSU5CqCIaU6MUSVwzD6yxTJgB5pZijUQWNoC9vndYmqOrwBbB4oma/5lCKXjd6B5CNKVAYGg3gDwQkzJTmsZjx2C219ARwWBzeuvJGajhr+vu/vmq/R2uvHZjGRMzNc5jnDfoYSjz3SQU4TdZugcFbEj+QNBOkfDE6YKcltt+C0mTVp/83eZn6343ecXX32iEJ57X1+LCZBriOztapSJV3BsAZ4OPz7w8BFWk8Uiig9HXgslfOPBEo9DgLBEF2+1MpiGNLgJhauImUiqhE/6mabGVua9ZLa+3QM0avbAFWrIInG6aqNPtXPkOk6SbEo9tgjpoiPzPgIi4oWcdfmu+gf6td0fkuv0mdYlMwHe67yf8wg6gNdk/YspSK4RvkXoq+TaYQQlHrsNPeMH0jyy/d+yVBoiK+s+sqI7Ur2vw2TUWHnOpOuYCiTUjYChF9L4xznEEJsEkK8LYRQH/5FQJeUUo2FrAOmpDmeSUVp+Iucqp/B0Kzn0VSfAIfeHpHoJoSg2GVLq2+ybo5bfx8079CUvxBNuklu6nkTFZUEysJA1R5NwsQ3j/kmLb4Wfrf9d5rOb+4ZUMwhJpNinsmwA7o0GcHQdRD6mhXzZpiJFgwApbmOcU1JO9p38MzeZ7h84eVM9UwdsU/J/j8yzEigQTAIIV4WQmyP8bMmifeZJqVcDXwK+F8hxCwgluiMa3MRQlwXFi6bWltTS/TJNMOCIbWQ1dbwQ6k0ExNi2glh+3PtiM0lHntaIbftetVJqn837JDU7l+A9NtjGtJkKElG+0lWla3i3Onn8tD2h2jsaxz3/JZeP6WecCmJqccqgQYZTHQrSWaBdPAt5XXacHczQwtJaqQs10FzgnkgpeRnG35GoaOQ65ZcN2a/bvMgQ4wrGKSUZ0opF8f4eRpoFkJUAIRfY5ZvlFI2hF/3AeuBFUAbkC+EUNNJq4CGBOO4X0q5Wkq5uqSkJImPOHGU5iqTMdWQ1bZeP8LIchjRqBPx0FsjNpflOjSp0PHQrbKqav6oSlxqezSqGStVjUH97JEH6wSgCIaR4//aqq8B8It3fzHu+S09A5GaRVQdqwjY+tid+4ygOJmyGIfeVBLbShdGNrWGH8gTqTGUhUPP4/kLXzjwAptbNnPDihtw29xj9uteL8xg0jUlPQNcEf79CuDp0QcIIQqEEPbw78XAScBOqfyH1wEXJzr/SEZd6SdaaSSirc9PgdOGxZyBqOLCmeAqGV6xhSnPc9DUndr4BwaD9PqH9JkQhzdC8TzIiRn4Fpfh0iSpCeembuW8styJFQzd/YMEhobNfBXuCq5afBX/PPBP3m2On808MBikZ2BoWOtUBevhzPkZXHYLLptZo2B4G6YeH0lsA+WhajaJjHdvi6Ys10F/+Ps8Gt+gj59v+jkLChfw8dkfH7M/Y2HnOpLuE+c24CwhRC1wVvhvhBCrhRC/DR+zANgkhHgfRRDcJqXcGd73X8DXhBB7UHwOD6Y5nkmFy27BbbekrjEYndwWjRCK1hBDY+gZGKI/kHxUT8Rxm+6EUB2SSYSpqrjtFmwWU8oaQ1PPAMVu24SUYlBRC8eNFm5XLr6SMmcZt2+4nWAo9v1Rv3uq9kpOgbIaP/SmcQOOQUmUAz0u3jalZlf1CSM2t/YqndsMqxemgdJIyOrYRdKD2x+k2dfMd477DuYYgRF9/iH8Q6GjR2OQUrZLKc+QUs4Jv3aEt2+SUl4T/v1NKeUSKeWy8OuDUefvk1IeK6WcLaW8REqpT+f2SUQ6uQytvRleZUw7QXH+9Qxb9MrDD5SmFMxJ7XrFbrfuUvwf045P+tR0HeiK43bitAWIymUY1Vcix5LD11d/nZqOGh6vfTzmuap/aISfqvokOPQOBNNrIpUMSlmMcb5D6qJk2okjNhvS8zxJVFNi86hF3uHew/x+++/56MyPsqJ0Rcxzj7QcBshmPhtOOs7bjEcyqA/eQ29HNpXnhQVDCuakNr0ctwf+rbxWn5TS6UVue8qmpMbugYhwnCgSNbs5d/q5HFN+DHdtvouOgY4x+1WH7wgfyfSTYNCr9M3OEKUeDVUADr4FFgdUjnzATmQ5DBU1yW20v+2OjXdgNpn56sqvxj1X90KSGSArGAymNI2yGBm3S5YvBatrhDlJXS2nItwiGkO6eQwH34DcKiiYntLpRW5bylFJzT0DlOVNrGCI9DSI8RmEEHz3uO/iG/Rx1+a7xuxXH2Tqgw0YXpEffEP/wcZBUyG9Q28qyZaWkQuJjGvOMYgEkkR9htfqXmPd4XVct/Q6ylxlcc/NWD6SjmQFg8GUjhPNEA9fYAhfIJjZxiRmixI/HiUY0tIYwhMirQY3UsKB15VVboplr1MtZugfCtLhDUy8xuBJHHI7K38Wn1n4GZ6ofYL3W0dqAS29SsbtCMetpwyK5sCBzAqG3oEhBgbj+Kr8fdC4dYx/IRSStHsnXmNwh/2FqqDtH+rnJ+/8hJl5M7li4RUJz1XDzieqpEcqZAWDwZR6lJ4GfTGiGRKh2pMzvlKqPlFJJBvoBoYnRCo+hqbuATzh81OmrRa8rTD95JQvUZGXQ2uff0RUjxZUYTLRgsFpU0oyJHKgX7/sekqdpdz69q0jHNEtPcpDdUzG7fSTwgmNqZUKSZZxO7nVbQAZHJG/ANDdP8hgUE6Kh2r0AuOBrQ9Q31fP947/HlZz4jIXqimpYAKTJJMlKxgMJhLNkKQ5KVKNMdMTYtrxSpx7VD2dslxt5QBG09DVT0V+mg/VNP0LAJX5DqQcax8eD1UYTrQpCcYvBuiyuvjmMd+kpqOGv3zwl8j2lt6B2AmS1SeBvxuatxsx3DGMm+R28C0QpjGRZ8NVSSeBYAjPg33d+/jdjt9xwcwLOKZ8/IZRSti5FWsmws514sgZ6RGK6vRL1pSR0XIY0UxZDcI8Ip8h1VyGhu5+KvNz0hvPwTfAU6nkWaSIOoaGLm21hVTUz1wxKQTD+H6Sc6rP4aQpJ/HL934ZyYhu6fEPh6pGowraDJmTxq2XdOgtxcdl94zYPBnKYaio2c8/efsnkYgwLbQZ3YXRALKCwWBSLYsREQyZrt9udyuNe/a/FtlU5nGMCdPTQmPXABV5aQgGHfwLQGQMjUkKt2HH7WQQDPYx4aqjEULwveO+B8Ct79yKlDK+xpA3RXHmZ8gBHamXFEu4DQUUDbX6xDG7JkM5DJWyXAdt8g3eaXqHr6z8CkU5RZrOUwvoHUlkBYPBqKu1ZHMZJrL5OTNPU2oThf0MZXlKWYyQ1nr6KBm37d4AU9IxJbXvVQqqpeFfAMWUBIoGkwyN3QPkWM3kOtLwkehEsUdbX4kqTxVfWv4lXq17lef3v0inbzB+OY/qk+DgmyMKJxpFocuGEHHmQf0mGBoY41+AyaUxuHL6MZc8y9Li5Vw89+LxTwjT2J3mAmkCyAoGg8l1WLBbTEnbt9v6/OTlTEzzc2Z+WHEEHngdUJyvQyGZVCc0dXWelikp4l9ITzA4bRbycqzJm5J6BijPc0SarUwkxW47Hb4AQ8HxH+KfXvBpFhQu4LYNPwVT/8hQ1WiqT1ISB1s/0Hm0Y7GYTRS5bLEFw951in9hxqljdrX1Kb0kJoNwfqfnQRABrp7/LUxC27wcCoZo6hlgSrom1QyTFQwGI4SgNNeetPO5pXdg4lZJVceA1Qn71gPDppRkhJv6EE5rpXTgdXCXQ9Gs1K8RpjI/h8auJE1J3QPxH6oZpiJPcaBriQ6zmCz84MQf0OXvxF723HABvdFMD/sZMmROKnbHyX7et04pB56TP2ZXq9pLYoKF87pD69jW9RqBtjOwhso1n9fUM0AwJKkqyAqGLKMo9Yxfy300dZ39E/dlstiU1WRYMKSSy1AfFgwpr5SkVB5YafoXVCrzHDQk6WNo6pn4rGeVqQVOQPleaGFR0SJOK1+LLX8TDf73Yh+UXw15UyP32WhKcx1jNYb+TsVsOfPDMc+ZDOUwegO93PL2LUz3zCLQ/qGk5rJ6v6ZkBUOW0ZSmUBbjcIdvYlcZM09TCpp116dUL0ldnZflpTip22qhtzFt/4JKRb6DxiR8DFJKWnr8lE8S27D6XTjc4dN8zgr3WoL+Uh784A56Aj1jDxACZp8B+17NSN2kEneM7Of9/1bCo2edHvOcyVAO47YNt9E+0M4PTvgRYE6qWnJ9Z5oLpAkiKxgygCIYtK8yuvsH6RkYiqwSJ4SZpymv+1+l2G3DJJI3JZV47Ngt2ttwjmDPS8rrrDNSO38Ulfk5dPkG8QW0JRp2eAMEgiHKJ4kpqTI/ByG0awwAHd4QgcZL6PC38fONP4990OwzIdCbkTLcaoXVEVUA9q1T+nhHdWyLZqLLYfzr0L94Zu8zXL3kalZXLMPjSK5asqo5px22nWGygiEDlOY6EpcDGEVdp7IqnFo4gYKhbJHSn2HfeixmEyUee1KmpIbufirTif+vfUnpv1BQnfo1oqjMU3MZtH0G1XlePglyGABsFhPluQ4Od2rXGJp7Bii0zObKRVfy5J4nea3utbEHzTgVTBbY87KOo41NicfOYFCO7IG+91+KVhgje9g/FKTDF5gwP0/HQAc/eutHzC+cz+eXfh5Ac+9nlbpOHyUeOw5rigukCSIrGDJAJOtT40rjcIeyyphQjUEIRWvYtx6kpDzXkZQpqaErjeS2gFfxL8w5K7XzY6AmqWk1J02mHAaVqoKcpDSGll4/pbl2vrj8i8zOn8333/g+7f3tIw9y5Cl9tDMkGCAql6FjP3QeiGtGOtzRj5QwbQIWSFJKbnn7FnoDvdx68q2RshdlSRbFrO+aQF9hGmQFQwZItkKpqjFM+Bdq5mlKHkFLTVItPqWUNHYPpC4Y9r8GwYBi5tCJZLOfVSE4WTQGUBYK9ckIhh4/ZR4HNrON20+9nd5ALz948wdjCzrOPgOatkJvs84jHokqnFXzCvvWKa+zYjueD3V4Aaguchk6rlg8u+9ZXjr4El9c/kXmFsyNbE+21W19Z/8R51+ANAWDEKJQCPGSEKI2/Dqm76IQ4sNCiC1RPwNCiIvC+34vhNgftW95OuOZrJSOVydmFHWd/bjtFvKdiYtzGc6MDymv+9YnVRaju38QXyCYeimJ2peU8t8xMmFTRclH0G5Kau4ewCQmV0XMqoIcGrv7GdSQywDDGgPA3IK5fHXVV3m17lUe3f3oyANVAbz3X3oOdwzVRcrK/2Cbd/j9cqugaHbM4w+2+0aclykOdB/glrdvYVXZKq5cdOWIfaW52qslh0KShq6BIy4iCdLXGL4NvCKlnAO8Ev57BFLKdVLK5VLK5cDpgA94MeqQb6r7pZRb0hzPpCQiGDTtTQX4AAAbQ0lEQVSuNNSIpImO3SZ/qlKeec9LSbX4VB++Ka2UpFQEw8wPgUW/h7LVbKLEbddsSlJaetoz029bI1WFTkISTfkYQ8FQuFz1sHD+1IJPcWLlidyx8Q72de0bPrhsCbhKDTcnlbjtuGxmDrT7lKqu+19TtIU43/OD7T5cNjNFGaxKGggG+NZr38JmtnHbKbeNadVZ5nEQCIZG+kni0NrnJxAMUTWRJuEUSfdbvwZ4OPz7w8BF4xx/MfC8lFK7B+0/gAKnDatZ0KTVx9Dpm1jHczTzPwL7X2NqjpL1rMXPEEluS0UwtO2G7kO6mpFUKvJzNNdLaurxT4riedFEQlY1OKDb+gJIObKlp0mYuOWkW5QCcK9+nf6hsJA0mRRz0t5/GVqGWwhBdZGLA+1eqNuklFyJY0YCONjuZVqRK6MLpLs230VNRw03n3gz5a6xiWyqBqYlZDViEj7aTElAmZSyESD8WjrO8ZcCfxm17VYhxFYhxJ1CiLhLRCHEdUKITUKITa2tremNOsOYTIKphU4OqCp0AqSUHO7on1jHczTzL4DQEPN6lXafWuyrak2iylTqJNWGw1R1dDyrTMl3DNu3x0HJep5cgmE4yW18wVDfpRwz+h6UOEu47ZTb2Nu1l1vfvnV4x+wzlfIYDcYq7TOKXco8qHkGTNaEC4CDHT6qM7hAevXwq/xh5x+4dN6lnD4ttkM84i/UsMhTAwUm3FeYAuMKBiHEy0KI7TF+1iTzRkKICmAJ8ELU5u8A84FjgELgv+KdL6W8X0q5Wkq5uqSkJJm3nhTMKnGzt7Vv3OM6vAH6B4OT58s0ZRW4y5nSpJgZNAmGrgGsZpFaS8/aF6FkPuRPS/7ccajIU8piaLEPN3b3TyrHMyjOW7NJaIpM2tuiLEJmlbjH7Dtxyolct/Q6nt77NE/WPqlsnPlhQAznjxjE9GIndZ0+ZM0zirbgyIt5XDAkqevoz5h/4VDPIb7z7++woHBBwnLaZR7tyZ6R7P/JMpeTYFzBIKU8U0q5OMbP00Bz+IGvPvhbElxqLfCklDJinJNSNkoFP/A74Ni4Zx/hzCpxc6DdO24RtMPhST9pTEkmE8z/CK5D67ET0OSAbujqpyIvZ2zXsPHw9yl1+Q0wI4HyYO0fDNLdn9g+3B8I0jMwNOk0Bos5nMugIft5b2sfNrMprn37C8u+wHHlx3HrO7eyq2MXuIqUJjk1f9d72COoLnIxTx5AdB2CBRfEPa6pZ4BAMJSRiKT+oX6+uv6rCCH4xWm/wGGJf9/L8xxYTEKT9l/X2U+B04rTNvEFAJMlXVPSM4Da8PQK4OkEx17GKDNSlFARKP6JzLSTmgBmlrgYDMpxV3vqpJ9aOIlWGfPPRwx6OdO+U9NKqbG7P0Uz0gtKmOrcc1MY5PiozvDxzEmRUNVJJhhA+V5o0hha+5hR7MIcRzibTWZuO/U2cm253LTuJjoHOmHRx6B5G7Tu1nvYEWYUuzjXvAEpzDDvo3GPO9iuhqoau0CSUnLzWzdT21nLbafcRpWnKuHxNouJ6cUualvG1/7rO/uPSMczpC8YbgPOEkLUAmeF/0YIsVoI8Vv1ICHEdGAq8Oqo8/8shNgGbAOKgVvSHM+kRVXpxzMnHY7kMEyiL9T0U8Cex/nWdzWbkipTqTG07THwVOgaphqN6gwfL6qnaZJlPUdTVeDU5Hze09LH7NKxZqRoinOK+d8P/y+tvla+/urXGZx/PiBgxxM6jXYs1UVOzjNtoLFglaKlxOFQOFTV6OS2P9f8mb/v+ztfWP4FTqk6RdM5c0rd7NEiGLqOzBwGSFMwSCnbpZRnSCnnhF87wts3SSmviTrugJRyipQyNOr806WUS8KmqcullOP/t49QZpUoKvG4gqFDUT/d9kmkflpsMPccTg5t5EBLjGJsUQRDkqaeFJLb+jsVx/Oij4PJmPIBlRqzn9WoqskoGKYWOGnu8eMfih895B8KcqjDF/nOJWJpyVJ+eOIP2di0kds/eFgRytufUMKGDaCkfz+zTQ2850r8ED7Y4cNqFobWGHqt7jXu2HQHp089neuXXq/5vDllHg62exOWuJFSUtc5wYUw02DyBGn/h5PvtFHksrGvNbFtsm4yhapGs+B8PKEeCtrfJTAU30/S0qvUn69I1pS08xkIDcIS7Z2xkqXYbcdqFuOW397V3IvNYspoRIxW1AdNokS9g+0+QhJmjaMxqFww6wKuXHwlf9v1N/5SOQvadkHLTl3GOxoR9mG8FIpdNE/lULuPqgJnXFNYuuzq2MU3X/0m8wrm8dNTfqq58Q4oGkNIwv4EfoYOb4CBwdAR6XiGrGDIKFoik+o6J1GoajSzziBosnEmGxJ+hoZUq0lufwwKZ0LlinRGmRCTSVCe5xi3LEZNYw/zyjyTKrlNRUv5bdXMESsiKR43rbiJ06pO46dNr/KyywnbH09voPGoeZq9jkW83534O36g3WuYGanV18qXXvkSbpubu8+4G6c1ufeZU6b8XxP5GYZDVSfhXNbA5Pvm/wczq9TF3gQaQygkFYfVZHI8q9jd9FefwQXmt/igvi3uYepKNikfQ0+jUpd/ySW6NOVJhBqyGg8pJTsbelhQ4TF0HKmiapOJHNB7ww+smRpMSSpmk5mffehnLClZwn+VlrCp5nH9zUkd+6FpGwfLzuRwhy9uhJ6UkkPtPkMcz93+bq5/+Xp6Aj3cffrdlDrHS70ay4xiFyYBtc29cY9Ju1HVBJMVDBlkZrGbDm+Azji9k5t7lRC9SakxADnHX0Wx6EEmCGkc1hiSMCXteBKQsNg4M5JKZV7iJLeWXj/t3gALK3INH0sqlOU6sJpFQgf03tY+puTnJB0mmWPJ4Z7T76HSXsCNziF273423eGO5L0/gTDhm30+QyEZ9z50+gbp9Q/pHqrqG/TxxVe+yIHuA9z14btYULQgpevYLWamF7mobY6vMdQfoZ3bVLKCIYPMKlW+6PvaYn+hJnumpHnOGTSbSplXF9/MsL/NS77TiseRRAHAbY9C+RIomTv+sWkyp8xDfVc/Xb7Ywnlno+JcXzBJBYPZpDhkE2oMrd6ktIVo8h35/OaMe8mRkus23DyyplI6DAVg88Mw91xKq5SiefFs9JFQVR1NSYFggK+s+wrb27Zzx6l3cELlCWldb3apm9qW+BpDXacPj8NCXs4EF8JMkaxgyCCRkNWW2BNiOIdhcmoMmMxsLr6QRf4t0L435iGbD3WyYurYpu5xad8LDZsVM1IGWDFNGdt7h7pi7t/ZEBYMlZNTMIDalyG2xhAKSfa2jh+qmojKkoU8kDMfBvu56oUr9REONc+AtxWOuZrpxeEqq+2xP8OhDn2rqvqDfr62/mu81fgWN594M2dUp98VcG6ZhwPtvriBGEdyqCpkBUNGqSpwYjOb2BtHY1Ab9EzmL1TXvLUMSRPetx4cs6/bN8ju5j5WVY+pvh6ft+9VauYsWavjKOOzrCofs0mw+VBnzP01jT1UFeSQm4zGk2Gq8p2R78pomnoG8AWCSTmeYzHzhK/yUGMTDA1w1QtXpS8cNj4IBTNg5umRKqvxNQb9Fki+QR83vHIDr9a9yn8f/9+smZ1UJZ+4zClzEwxJpSBgDOqO4OQ2yAqGjGI2CaYXO+NrDJ0+ynIndxvA6umzeDm0Cuu2/4OhkYXE1IftqupCbRfztil256WfhNwKvYcaE5fdwvxyT1zBsLOxZ9L6F1SmFubQ1uePWQJdjRhLVzAw/WRmFi/ioW6lR/bn/vk5trVuS+1azTvg0Juw+iowmSJVVg/GeagebPdRnutIex70Bfr4wstfYEPTBm49+VbWztNv8aFqZLtjOKB7BgapbembtAEMWsgKhgwzq8TNvjjhnrUtfRPSxjAZFlbk8n/B07H5O+GDkU7oTQc7MJsEy6bGLow2hg33w1A/nHSjASONz8ppBWw51EUwNDLqxhcYYn+bl4WT2IwEip8E4P26seYwNSIpHVMSoESHnXADM1v28Pv51+K0Orn6xat59fDo4gUa2PQQmO2w4vLIpunFTqUvQwwOdXiZlqYZqcnbxOf++Tm2tm7lZ6f+jAtnXZjW9UYzq8SNEMR0QL+9t51gSHLy7GJd3zOTZAVDhplV4uZgx1jbZFufn611XZw0yb9M+U4be9zH0G4th3fuHxHS+O7BThZV5mqLhgl4FcEw7yNQMs/AEY9lZXU+3kCQXU0jV3u7mnqRcvI6nlVOml2M1SxY98HYmpV7WvvIdVgoduvQ3GbhReCpZPqWv/Gnj/yJGXkzuHHdjTyy6xHt1/D3wvt/hcWfAOewJjm9yBU3ZPVAu4/paQiG7W3buey5y6jvq+dXZ/yKc6afk/K14uGwmplW6IxZGuP1PW04bWZWTEvCpDrJyAqGDDOzxEUwJCMONpV1H7QgJZy5oGyCRqadeRV5/J/5Ijj8dkRrGAyG2HK4S7t/4b0/KWUwTrrJwJHGZtU05QE12pykRiRNdlOS227h2BmFrNs1VjDsbfEyq9StT3Mbiw2Ouw72v0pxdyO/O+d3nFh5Ij9++8f84M0fMDCkoenRO7+BQB8cc82IzdOLXDFDVrt8AVp7/SmHqj6//3mu/OeV2M12/njeHzl5yskpXUcLc0o9MSOTXt/TxnEzCrFZjtzH65E78iOUeMX0XqlpoTzXwaJJbsYAZUV9d8/JhIrnwYv/DUN+djb0MDAY0iYYgoPw5t0w9XiYdrzxAx7F1MIcit02Nh8cKRhqGnvwOCyTNlw4mg/PK2V3c9+Y6KS9rX3p+xeiWfU5pf/2G3fhtDq5+/S7uXbJtTxR+wSfff6zHO49HP/c7jr49//A/POhatWIXTPC4bQ1jSNrbz27tRGAD81NrueKb9DHD9/8Id967VssKFrAnz/yZ2YXxO4lrRdzytzsb/OO6MHd0NXPvlbvpNf8xyMrGDLMzBjF9AYGg7xW28oZC0onvs+zBhZU5OIPmTh8zHehcz9suJ93ww/Z1Vocz5seUtp3ToC2AEqLyRXTCsZqDA09LKjIPSLuwYfnKxm70eaknoFBWnr96fsXoskpgOOuV3JN9ryC2WTmxpU3cs8Z91DfV88lz17CI7seISRjhG2+8F2QITjnJ2N2LavKp8Rj568bRwqWx9+tY365J6kF0gcdH3Dpc5fyRO0TXLvkWh4850GKcuJXbtWLOaVuBoNyRNjt63uUqgCnzDnymolFkxUMGcbjsDIlP4dXd7VGOom9va8dXyB4RJiRgEi0xUbLKqWpzqt3sGvvfqbk54xfkbSlBl76Psw+C+adl4HRxmbltAIOtPto71Miq0IhyQdNvZPejKQys9hFdZGTf0UJhi3h3AxdNQaAD/0XFM+FZ76s9GkGTq06lUcueITFxYv58ds/5poXr+FwT9RDft962PkUnPJ1KKgec0mbxcSnjp3G+l2tkbDVPS29bDncxcWrqjQJ575AHz/b+DMu/ful9AX6uP/s+7lx5Y1YTZkJNZ5TqsyD6NIYr9e2UeKxM7dM53uQYbKCYQK4/kMzeWd/B//c3gQoZqQcq5kTZhm/ytGD6UUu7BaTYgY4+1ZkoI/jDt47vhlpyA+PXwM2N1z0a8PrIiVCHevm8MP0YIcPXyB4xAgGIQQfnlfKm3vbGRgMMhgMcetzNVTmOThR7++R1QEX3Qu9jYoWEGaKewoPnPUAPzzhh9S013DR0xdxx8Y76PS2wD++BQXT4cT4EWefPm4aFpPgj28dBOCxd+sxmwRrlk9JOJxgKMize5/lwqcu5E87/8TH5nyMJy58guMrMmuWnFXqwmISPPN+A1JKQiHJG3vaOHl28RGhdSYiLcEghLhECLFDCBESQsStoyuEOFcIsUsIsUcI8e2o7TOEEO8IIWqFEH8TQugQSjH5+dSx05hf7uGW52roDwR5paaZk+cUT+r8hWgsZhPzyj28uruVels1fcuu4uOhF7ky+FjiE1+5GZq3w5p7wJ188TI9WVqVhyWc6Cal5KWdipCe7BFJ0Xx4fin+oRBv7W3n/tf2sau5l5vXLMZlRC+PqtWK6e+9Pyp9M8IIIfjE3E/w1Jqn+OjMj/Knmj9x3mNnc89QE22nf1cRKnEozXVw3pIKHt10mJ6BQZ58r47T5pZQ4ondK3wwOMiTtU+y5uk1/L/X/x/FOcX8+SN/5gcn/IB8RxLZ9jrhtFn46llzeX57Ew++vp8Pmnpp9waOeP8CpK8xbAc+DrwW7wAhhBm4BzgPWAhcJoRYGN59O3CnlHIO0AlcneZ4jggsZhM/vHAR9V39fO2RLTR0D3Dmgol9UCbLNafMpK7Tx5n/8ypf77qYJ4MnsWLP3bD+9rEHh4Lw1q/hrbuV6JR5xrTuTAaH1czCylxe2N7EBXe/zk/+8QHzyz3MLT9yTADHzSgkx2rmD28d4Jev1HLe4nLOXGigOfK070DJfHjsKtg6MmS1zFXGzcd+hyctsznB28t9BXmc9d5P+dr6r/Fmw5sMhmL32f7cidX0+of4xiPv09zj5+JVI1trSin5oOMDfr7x55zz+Dl8/83v47Q4ufO0O/nr+X9lSckSwz6uFr542izOWVTGT5//gLteUVqiHsn5CyppLS2klDXAeGrTscAeKeW+8LF/BdYIIWqA04FPhY97GPghcG86YzpSOH5mERcsq+TZ9xuAYWfikcKFyypZMTWfH/99Jy/ubOYt2w2sWToV0/qfKDVxFpwP5Uuh6yA8+xVo3AKzzoCzfjzRQ4+wurqQh97YT3WRk59dvJSPrZiCdRL2YIiHw2rmpNnFvFzTjMdu4YcXLjL2DS12+PSjijnwiWth9wtw9i0w6IO+ZnjlZmYeeps7z7udA/PP4bHdj/HU3qd46eBLuKwujis/jhMqT2BOwRyqc6spchSxcloBi6fk8uLOZvKdVo6f7WZv1162tW1jS8sW3m1+lwM9B7CYLJwy5RQunnsxp0w5ZdKYaoQQ/PySZVx0zxu8sKOZ2aXuSdn5L1ky0T9yChAdelAHHAcUAV1SyqGo7YmNi/9h/L+PzOflnc3MLfdQ6jnyvkxTC53c/9nVvLa7lcFgCNO8cxTTwcYHlB8Vdxlc/JDStnOSTGiAL58+m1PmFHPKnOJJ2ZRHC2csKOXlmma+de48ynIz8B3Knwaf+we8fies/6nSYEnFbFPu8+KPMx34xjHf4Msrv8zrda/zRsMbvFH/Bv86/K/I4S6rC6fFSaBI4HIEETYfpz4ynNfgsXlYXrKcyxdczjnTz5kQc5EWPA4rv/nMaj52zxuccYQt8OIh5DjNOIQQLwPlMXZ9V0r5dPiY9cA3pJSbYpx/CXCO2gNaCPEZFC3iZuAtKeXs8PapwD+klDF1QyHEdcB1ANOmTVt18OBBTR9wsrNhfwe5ORbmlx85tu1x8bZB0zblJ+iHY68Dh8YyGVmSIjAU4tXdrZwxvxSTQW0w49KwRYk+cpcqwr9kPuTFX9tJKWnwNrC/ez8Hug9Q11fHwNAAA0N+ttV3smLKFGYUVFLqLGVB4QJm5s9MquXmRNPpDeCyWyZ1YpsQ4l0pZeK+qmgQDBrfbD3xBcMJwA+llOeE//5OeNdtQCtQLqUcGn1cIlavXi03bRrzVlmyZMmSJQFaBUMmRNtGYE44AskGXAo8IxWJtA5Q23ZdATydgfFkyZIlS5YEpBuu+jEhRB1wAvCcEOKF8PZKIcQ/AMI+hBuAF4Aa4BEp5Y7wJf4L+JoQYg+Kz2Fskf8sWbJkyZJRdDElZZqsKSlLlixZkmcymZKyZMmSJcsRRFYwZMmSJUuWEWQFQ5YsWbJkGUFWMGTJkiVLlhFkBUOWLFmyZBnBERmVJIRoBVJNfS4G2nQczpHC0fi5j8bPDEfn585+Zm1USynH7SJ0RAqGdBBCbNISrvWfxtH4uY/GzwxH5+fOfmZ9yZqSsmTJkiXLCLKCIUuWLFmyjOBoFAz3T/QAJoij8XMfjZ8Zjs7Pnf3MOnLU+RiyZMmSJUtijkaNIUuWLFmyJOCoEgxCiHOFELuEEHuEEN+e6PEYgRBiqhBinRCiRgixQwhxU3h7oRDiJSFEbfi1YKLHqjdCCLMQ4j0hxN/Df88QQrwT/sx/C5d9/49CCJEvhHhMCPFB+J6f8J9+r4UQXw1/t7cLIf4ihHD8J95rIcRDQogWIcT2qG0x761Q+GX42bZVCLEynfc+agSDEMIM3AOcBywELhNCLJzYURnCEPB1KeUC4HjgS+HP+W3gFSnlHOCV8N//adyEUtpd5Xb4/+2dTYiNURjHf0+GYiSxEDM0piYWykcWQpqGhY/JWJAFmUSWspBiIws7+ShlMz5GicTErKwoNuRrodhoiMsYymeUUf4W59zm3tvcaNzrzbnPr273Pee+9T6n/3vf//s+59z7cCSO+QOwLZOoqssx4Jqk2cBcwviT1drMGoCdwEJJc4BRhBovKWp9BlhZ0ldO21VAS3ztAE78zYFrxhgI5USfSuqTNAhcADoyjqniSOqX9CBufyFcKBoIY+2Ou3UD67KJsDqYWSOwBuiKbQPagHxR4hTHPAFYRqxjImlQ0kcS15pQq36smdUB44B+EtRa0k3gfUl3OW07gLMK3AYmmtnUkR67loyhAXhZ0M7FvmQxsyZgPnAHmCKpH4J5AGlULR/iKLAH+Bnbk4GPsVAUpKl3M6E87umYQusys3oS1lrSK+AQ8IJgCJ+A+6SvdZ5y2lb0+lZLxjBcpfRkl2SZ2XjgMrBL0ues46kmZtYOvJV0v7B7mF1T07sOWACckDQf+EpCaaPhiDn1DmAmMA2oJ6RRSklN699R0fO9lowhB0wvaDcCrzOKpaqY2WiCKZyT1BO7B/KPlvH9bVbxVYElwFoze05IEbYRniAmxnQDpKl3DshJuhPblwhGkbLWK4Bnkt5J+gH0AItJX+s85bSt6PWtlozhLtASVy+MIUxY9WYcU8WJufWTwBNJhws+6gU643YncPVfx1YtJO2V1CipiaDrdUmbgBvA+rhbUmMGkPQGeGlms2LXcuAxCWtNSCEtMrNx8VzPjzlprQsop20vsCWuTloEfMqnnEZCTf3AzcxWE+4kRwGnJB3MOKSKY2ZLgVvAI4by7fsI8wwXgRmEL9cGSaUTW/89ZtYK7JbUbmbNhCeIScBDYLOk71nGV2nMbB5hwn0M0AdsJdzwJau1mR0ANhJW4D0EthPy6UlpbWbngVbCv6gOAPuBKwyjbTTJ44RVTN+ArZLujfjYtWQMjuM4zu+ppVSS4ziO8we4MTiO4zhFuDE4juM4RbgxOI7jOEW4MTiO4zhFuDE4juM4RbgxOI7jOEW4MTiO4zhF/AJV1So3c5pfFwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "tst_encoding = PositionalEncoding(20)\n",
    "res = tst_encoding(torch.arange(0,100).float())\n",
    "_, ax = plt.subplots(1,1)\n",
    "for i in range(1,5): ax.plot(res[:,i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class TransformerEmbedding(nn.Module):\n",
    "    \"Embedding + positional encoding + dropout\"\n",
    "    def __init__(self, vocab_sz:int, emb_sz:int, inp_p:float=0.):\n",
    "        super().__init__()\n",
    "        self.emb_sz = emb_sz\n",
    "        self.embed = embedding(vocab_sz, emb_sz)\n",
    "        self.pos_enc = PositionalEncoding(emb_sz)\n",
    "        self.drop = nn.Dropout(inp_p)\n",
    "    \n",
    "    def forward(self, inp): \n",
    "        pos = torch.arange(0, inp.size(1), device=inp.device).float()\n",
    "        return self.drop(self.embed(inp) * math.sqrt(self.emb_sz) + self.pos_enc(pos))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Feed forward"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The feed forward cell is easy: it's just two linear layers with a skip connection and a LayerNorm."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def feed_forward(d_model:int, d_ff:int, ff_p:float=0., double_drop:bool=True):\n",
    "    layers = [nn.Linear(d_model, d_ff), nn.ReLU()]\n",
    "    if double_drop: layers.append(nn.Dropout(ff_p))\n",
    "    return SequentialEx(*layers, nn.Linear(d_ff, d_model), nn.Dropout(ff_p), MergeLayer(), nn.LayerNorm(d_model))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Multi-head attention"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "![Multi head attention](images/attention.png)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class MultiHeadAttention(nn.Module):\n",
    "    \"MutiHeadAttention.\"\n",
    "    \n",
    "    def __init__(self, n_heads:int, d_model:int, d_head:int=None, resid_p:float=0., attn_p:float=0., bias:bool=True,\n",
    "                 scale:bool=True):\n",
    "        super().__init__()\n",
    "        d_head = ifnone(d_head, d_model//n_heads)\n",
    "        self.n_heads,self.d_head,self.scale = n_heads,d_head,scale\n",
    "        self.q_wgt = nn.Linear(d_model, n_heads * d_head, bias=bias)\n",
    "        self.k_wgt = nn.Linear(d_model, n_heads * d_head, bias=bias)\n",
    "        self.v_wgt = nn.Linear(d_model, n_heads * d_head, bias=bias)\n",
    "        self.out = nn.Linear(n_heads * d_head, d_model, bias=bias)\n",
    "        self.drop_att,self.drop_res = nn.Dropout(attn_p),nn.Dropout(resid_p)\n",
    "        self.ln = nn.LayerNorm(d_model)\n",
    "        \n",
    "    def forward(self, q:Tensor, k:Tensor, v:Tensor, mask:Tensor=None):\n",
    "        return self.ln(q + self.drop_res(self.out(self._apply_attention(q, k, v, mask=mask))))\n",
    "    \n",
    "    def _apply_attention(self, q:Tensor, k:Tensor, v:Tensor, mask:Tensor=None):\n",
    "        bs,seq_len = q.size(0),q.size(1)\n",
    "        wq,wk,wv = self.q_wgt(q),self.k_wgt(k),self.v_wgt(v)\n",
    "        wq,wk,wv = map(lambda x:x.view(bs, x.size(1), self.n_heads, self.d_head), (wq,wk,wv))\n",
    "        wq,wk,wv = wq.permute(0, 2, 1, 3),wk.permute(0, 2, 3, 1),wv.permute(0, 2, 1, 3)\n",
    "        attn_score = torch.matmul(wq, wk)\n",
    "        if self.scale: attn_score = attn_score.div_(self.d_head ** 0.5)\n",
    "        if mask is not None: \n",
    "            attn_score = attn_score.float().masked_fill(mask, -float('inf')).type_as(attn_score)\n",
    "        attn_prob = self.drop_att(F.softmax(attn_score, dim=-1))\n",
    "        attn_vec = torch.matmul(attn_prob, wv)\n",
    "        return attn_vec.permute(0, 2, 1, 3).contiguous().contiguous().view(bs, seq_len, -1)\n",
    "        \n",
    "    def _attention_einsum(self, q:Tensor, k:Tensor, v:Tensor, mask:Tensor=None):\n",
    "        # Permute and matmul is a little bit faster but this implementation is more readable\n",
    "        bs,seq_len = q.size(0),q.size(1)\n",
    "        wq,wk,wv = self.q_wgt(q),self.k_wgt(k),self.v_wgt(v)\n",
    "        wq,wk,wv = map(lambda x:x.view(bs, x.size(1), self.n_heads, self.d_head), (wq,wk,wv))\n",
    "        attn_score = torch.einsum('bind,bjnd->bijn', (wq, wk))\n",
    "        if self.scale: attn_score = attn_score.mul_(1/(self.d_head ** 0.5))\n",
    "        if mask is not None: \n",
    "            attn_score = attn_score.float().masked_fill(mask, -float('inf')).type_as(attn_score)\n",
    "        attn_prob = self.drop_att(F.softmax(attn_score, dim=2))\n",
    "        attn_vec = torch.einsum('bijn,bjnd->bind', (attn_prob, wv))\n",
    "        return attn_vec.contiguous().view(bs, seq_len, -1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Masking"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The attention layer uses a mask to avoid paying attention to certain timesteps. The first thing is that we don't really want the network to pay attention to the padding, so we're going to mask it. The second thing is that since this model isn't recurrent, we need to mask (in the output) all the tokens we're not supposed to see yet (otherwise it would be cheating)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_padding_mask(inp, pad_idx:int=1):\n",
    "    return None\n",
    "    return (inp == pad_idx)[:,None,:,None]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_output_mask(inp, pad_idx:int=1):\n",
    "    return torch.triu(inp.new_ones(inp.size(1),inp.size(1)), diagonal=1)[None,None].byte()\n",
    "    return ((inp == pad_idx)[:,None,:,None].long() + torch.triu(inp.new_ones(inp.size(1),inp.size(1)), diagonal=1)[None,None] != 0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Example of mask for the future tokens:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[0, 1, 1, 1, 1, 1, 1, 1, 1, 1],\n",
       "        [0, 0, 1, 1, 1, 1, 1, 1, 1, 1],\n",
       "        [0, 0, 0, 1, 1, 1, 1, 1, 1, 1],\n",
       "        [0, 0, 0, 0, 1, 1, 1, 1, 1, 1],\n",
       "        [0, 0, 0, 0, 0, 1, 1, 1, 1, 1],\n",
       "        [0, 0, 0, 0, 0, 0, 1, 1, 1, 1],\n",
       "        [0, 0, 0, 0, 0, 0, 0, 1, 1, 1],\n",
       "        [0, 0, 0, 0, 0, 0, 0, 0, 1, 1],\n",
       "        [0, 0, 0, 0, 0, 0, 0, 0, 0, 1],\n",
       "        [0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], dtype=torch.uint8)"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.triu(torch.ones(10,10), diagonal=1).byte()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Encoder and decoder blocks"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We are now ready to regroup these layers in the blocks we add in the model picture:\n",
    "\n",
    "![Transformer model](images/Transformer.png)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class EncoderBlock(nn.Module):\n",
    "    \"Encoder block of a Transformer model.\"\n",
    "    #Can't use Sequential directly cause more than one input...\n",
    "    def __init__(self, n_heads:int, d_model:int, d_head:int, d_inner:int, resid_p:float=0., attn_p:float=0., ff_p:float=0.,\n",
    "                 bias:bool=True, scale:bool=True, double_drop:bool=True):\n",
    "        super().__init__()\n",
    "        self.mha = MultiHeadAttention(n_heads, d_model, d_head, resid_p=resid_p, attn_p=attn_p, bias=bias, scale=scale)\n",
    "        self.ff  = feed_forward(d_model, d_inner, ff_p=ff_p, double_drop=double_drop)\n",
    "    \n",
    "    def forward(self, x:Tensor, mask:Tensor=None): return self.ff(self.mha(x, x, x, mask=mask))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class DecoderBlock(nn.Module):\n",
    "    \"Decoder block of a Transformer model.\"\n",
    "    #Can't use Sequential directly cause more than one input...\n",
    "    def __init__(self, n_heads:int, d_model:int, d_head:int, d_inner:int, resid_p:float=0., attn_p:float=0., ff_p:float=0.,\n",
    "                 bias:bool=True, scale:bool=True, double_drop:bool=True):\n",
    "        super().__init__()\n",
    "        self.mha1 = MultiHeadAttention(n_heads, d_model, d_head, resid_p=resid_p, attn_p=attn_p, bias=bias, scale=scale)\n",
    "        self.mha2 = MultiHeadAttention(n_heads, d_model, d_head, resid_p=resid_p, attn_p=attn_p, bias=bias, scale=scale)\n",
    "        self.ff   = feed_forward(d_model, d_inner, ff_p=ff_p, double_drop=double_drop)\n",
    "    \n",
    "    def forward(self, x:Tensor, enc:Tensor, mask_in:Tensor=None, mask_out:Tensor=None): \n",
    "        y = self.mha1(x, x, x, mask_out)\n",
    "        return self.ff(self.mha2(y, enc, enc, mask=mask_in))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### The whole model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Transformer(nn.Module):\n",
    "    \"Transformer model\"\n",
    "    \n",
    "    def __init__(self, inp_vsz:int, out_vsz:int, n_layers:int=6, n_heads:int=8, d_model:int=256, d_head:int=32, \n",
    "                 d_inner:int=1024, inp_p:float=0.1, resid_p:float=0.1, attn_p:float=0.1, ff_p:float=0.1, bias:bool=True, \n",
    "                 scale:bool=True, double_drop:bool=True, pad_idx:int=1):\n",
    "        super().__init__()\n",
    "        self.enc_emb = TransformerEmbedding(inp_vsz, d_model, inp_p)\n",
    "        self.dec_emb = TransformerEmbedding(out_vsz, d_model, 0.)\n",
    "        self.encoder = nn.ModuleList([EncoderBlock(n_heads, d_model, d_head, d_inner, resid_p, attn_p, \n",
    "                                                   ff_p, bias, scale, double_drop) for _ in range(n_layers)])\n",
    "        self.decoder = nn.ModuleList([DecoderBlock(n_heads, d_model, d_head, d_inner, resid_p, attn_p, \n",
    "                                                   ff_p, bias, scale, double_drop) for _ in range(n_layers)])\n",
    "        self.out = nn.Linear(d_model, out_vsz)\n",
    "        self.out.weight = self.dec_emb.embed.weight\n",
    "        self.pad_idx = pad_idx\n",
    "        \n",
    "    def forward(self, inp, out):\n",
    "        mask_in  = get_padding_mask(inp, self.pad_idx)\n",
    "        mask_out = get_output_mask (out, self.pad_idx)\n",
    "        enc,out = self.enc_emb(inp),self.dec_emb(out)\n",
    "        for enc_block in self.encoder: enc = enc_block(enc, mask_in)\n",
    "        for dec_block in self.decoder: out = dec_block(out, enc, mask_in, mask_out)\n",
    "        return self.out(out)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Bleu metric (see dedicated notebook)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class NGram():\n",
    "    def __init__(self, ngram, max_n=5000): self.ngram,self.max_n = ngram,max_n\n",
    "    def __eq__(self, other):\n",
    "        if len(self.ngram) != len(other.ngram): return False\n",
    "        return np.all(np.array(self.ngram) == np.array(other.ngram))\n",
    "    def __hash__(self): return int(sum([o * self.max_n**i for i,o in enumerate(self.ngram)]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_grams(x, n, max_n=5000):\n",
    "    return x if n==1 else [NGram(x[i:i+n], max_n=max_n) for i in range(len(x)-n+1)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_correct_ngrams(pred, targ, n, max_n=5000):\n",
    "    pred_grams,targ_grams = get_grams(pred, n, max_n=max_n),get_grams(targ, n, max_n=max_n)\n",
    "    pred_cnt,targ_cnt = Counter(pred_grams),Counter(targ_grams)\n",
    "    return sum([min(c, targ_cnt[g]) for g,c in pred_cnt.items()]),len(pred_grams)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class CorpusBLEU(Callback):\n",
    "    def __init__(self, vocab_sz):\n",
    "        self.vocab_sz = vocab_sz\n",
    "        self.name = 'bleu'\n",
    "    \n",
    "    def on_epoch_begin(self, **kwargs):\n",
    "        self.pred_len,self.targ_len,self.corrects,self.counts = 0,0,[0]*4,[0]*4\n",
    "    \n",
    "    def on_batch_end(self, last_output, last_target, **kwargs):\n",
    "        last_output = last_output.argmax(dim=-1)\n",
    "        for pred,targ in zip(last_output.cpu().numpy(),last_target.cpu().numpy()):\n",
    "            self.pred_len += len(pred)\n",
    "            self.targ_len += len(targ)\n",
    "            for i in range(4):\n",
    "                c,t = get_correct_ngrams(pred, targ, i+1, max_n=self.vocab_sz)\n",
    "                self.corrects[i] += c\n",
    "                self.counts[i]   += t\n",
    "    \n",
    "    def on_epoch_end(self, last_metrics, **kwargs):\n",
    "        precs = [c/t for c,t in zip(self.corrects,self.counts)]\n",
    "        len_penalty = exp(1 - self.targ_len/self.pred_len) if self.pred_len < self.targ_len else 1\n",
    "        bleu = len_penalty * ((precs[0]*precs[1]*precs[2]*precs[3]) ** 0.25)\n",
    "        return add_metrics(last_metrics, bleu)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = Transformer(len(data.train_ds.x.vocab.itos), len(data.train_ds.y.vocab.itos), d_model=256)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn = Learner(data, model, metrics=[accuracy, CorpusBLEU(len(data.train_ds.y.vocab.itos))], \n",
    "                loss_func = CrossEntropyFlat())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LR Finder is complete, type {learner_name}.recorder.plot() to see the graph.\n"
     ]
    }
   ],
   "source": [
    "learn.lr_find()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYkAAAEKCAYAAADn+anLAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3Xl8lNXZ//HPNZksJIQ9LBIg7IjKGhFcALfWhWpdi9Wq1bpXa21r/T2tbR9tra1b1bo8SLVq3Zfauq8VQRYJOwqyS9gDQiBAMtv5/TEDpDGBJGRyz/J9v173KzP3nJm5DpNwzVnuc8w5h4iISG18XgcgIiKJS0lCRETqpCQhIiJ1UpIQEZE6KUmIiEidlCRERKROShIiIlInJQkREamTkoSIiNTJ73UADdWhQwdXVFTkdRgiIkll1qxZm51zBQ19XlyThJn9BLgCMOAx59xfajxuwP3AacAu4FLn3Oz9vWZRURElJSVxilhEJDWZ2VeNeV7cupvM7HCiCWIEMBgYZ2Z9axQ7FegbO64EHolXPCIi0nDxHJM4FJjunNvlnAsBk4CzapQ5E3jKRU0H2phZlzjGJCIiDRDPJLEQGG1m7c0sl2iXUrcaZboCpdXur4mdExGRBBC3MQnn3CIz+xPwPlABzANCNYpZbU+tecLMriTaHUX37t2bOFIREalLXKfAOuf+5pwb5pwbDXwNLK1RZA3/3booBNbV8joTnHPFzrnigoIGD86LiEgjxTVJmFnH2M/uwNnAczWK/Bu42KJGAuXOufXxjElEROov3tdJvGJm7YEgcJ1zbquZXQ3gnHsUeIvoWMUyolNgfxjneEREpAHimiScc8fVcu7RarcdcF08Y9hjycYdvDFvHVl+H9n+DLL8PnIyfeRl+2kZO3Kz/GT4DDPwGZgZfp/hz/CRGfuZ7feRk5lBhq+24RQRkdSSdFdcN9bSjRU88NGyJns9v8/IycwgLzuDvGw/+dl+Wub4aZWTSesW0SM/x09OZsbeIzcrg06tsunUKodOrXLIzNCqKCKS2NImSZw+qAunHXEawbAjEI5QFQxTFYqwsyrEjqoQFZUhdgVCRBxEnMPFfobCjlAkQjDsCIUjVIX2HGF2ByLsCux7fkVViE3bKyjfHaR8d5CqUKTOeMygfV427fIyaZObRdvcTNq0yKJljp+8rGjiaZnjp0PLbArysymI/czJzGjGfzURSXdpkyQg2n2U5Tey/D5aZse/6lWhMJXBaEKqDEbYGQixcXslG8or2bC9ko3bK9m6M8jWXQFWbd7F1l3b2FkVYmcgXOdrZvt9tMmNtlTa5mbRt1NL+nduxaGd8+nfOZ/8nMy410tE0kdaJYnmlu3PINufAS32/cd9aJdWB3xeJOLYHQyzozLE5ooqyiqqKNtRxeaKKsp3Bdm2K8i23QHKdlTxrznr2FG1Goi2Tg7t3IqRvdozslc7jurZnta5Shoi0nhKEgnI5zPysv3kZfvp3Dpnv2Wdc6wrr2Tx+u0sXLudGSu38MyMr3j805WYwcAurRjVqz1H92nPkUXt1NIQkQax6ASj5FFcXOy0Cuz+VQbDzCvdxrQVW5i2fAtzVm8jEI5gBl3btKB3QUt6FeTRt2M+I3u1o2eHPKIL8opIqjKzWc654gY/T0ki9VUGw8xevZWSVVtZtqmC5WUVrCjbye5gdOyja5sWjO7XgWP7FFBc1JZOrfbfehGR5KMkIQ0SiThWf72LKcs2M3lpGVOXbWFHVXRpra5tWjC0exuGdW/LyQM70a1drsfRisjBUpKQgxIKR5i/tpzZX21lTuk25ny1lXXllQAM7taGcUd04bRBXejapoXHkYpIYyhJSJNbvWUXby1cz5vz17NgbTkAx/cv4IfH9OS4vh00jiGSRJQkJK6+2rKTV2ev5ZkZq9lcUUXvgjwuPaYn5w0v1AV+Is2g9OtdtIqt5tAYjU0SWhdC6qVH+zx+enI/Pr3leO773mDysv3c+tpCxt71MU9P/4rAfq4uF5GDd+K9k3j446ZbWqi+lCSkQbL9GZw1tJB/XXcMz10xksK2Lbj1tYWccM/HvFhSSjiSXC1TkWTgnCMQipDtwXpvShLSKGbGqN7teenqUTx52Qja5WVx88vzOevhT1kYG78QkaYRin358mJRUCUJOShmxph+BfzrumO4f/wQ1m2r5Iy/TuH3b3zBzqqau9WKSGPs6c7N8itJSJIyM84c0pUPbxrD+BHdmThlJSffO4lXZq1RF5TIQVKSkJTROjeTO846gleuGUXbvCx+9tI8vnXfJN6Yv46IkoVIowTD0SSh7iZJGcN7tOON64/l0YuG4TPjx8/O4bQHJvPBFxtJtmnXIl6rUktCUpGZccrhXXjnxtHcP34IlcEwP3qqhLMfmcrU5Zu9Dk8kaQRiLYlsJQlJRRm+6HjF+zeN4c6zj2BDeSXff2wGF06czjsLN1AVqnuTJRHxtrtJ+0lIs8nM8DF+RHe+O7Qrz85YzSOTlnP1P2bRukUm3xnchbOGFjKsexst9yFSw96BayUJSQc5mRlcdmxPLh7Vg8nLNvPP2Wt5qWQN/5i+mp4d8jh7aFfOGtaVwrZafVYE9rUkvBiTUJIQz/gzfBzfvyPH9+/Ijsogby/YwCuz13DP+0u45/0ljOrVnh8d15MTBnRU60LS2p6Ba3U3SdrKz8nk/CO7cf6R3Sj9ehf/nLOWF2aWcvmTJRzapRXXn9CHUw7rjM+nZCHpR9dJiFTTrV0uN5zYl49/MZa7zxtMVTDMtc/M5uT7JvHOwg2aQitpJxiO/s5rdpNINZkZPs4dXsj7N43hwQuGYmZc/Y9ZfG/CdBas0fpQkj4CHnY3KUlIwsvwGd8ZfAjv/OQ4fv/dw1m+qYLv/HUKN70wl/Xlu70OTyTuAuHoNHF1N4nshz/Dx0Uje/CfX4zlmrG9eWPBek64exIPfLiUyqCutZDUFQztWQW2+cfklCQk6bTKyeSXpwzgw5vGcPyAAu59fwkn3hNdH0rjFZKKqjycAqskIUmrW7tcHr5wOM9fOZJWLTL58bNzGPfgFN79XIPbklr2jElkZzT/VsFKEpL0RvZqzxvXH8vd5w1mZ1WIq56exWkPTOHtBeu18qykhL3LcvjV3STSKBk+49zhhXxw0xjuPT86bfaaZ2ZzxkNT+GRJmVoWktS8XJZDSUJSij/Dx9nDotNm7zlvMFt3Brn48c+4cOIM5pVu8zo8kUYJhCL4LPr73dzi+o5m9lMz+9zMFprZc2aWU+PxS82szMzmxo4fxTMeSR8ZPuOc4YV89PMx/GbcQBZv2MGZD33Kdc/OZvWWXV6HJ9IgwXDEk2skII5Jwsy6AjcAxc65w4EMYHwtRV9wzg2JHRPjFY+kp2x/dDHBT24+nhtO6MNHizZx4r0fc/sbX7BtV8Dr8ETqpSoU8WRmE8R/7SY/0MLMgkAusC7O7ydSq5bZfm76Vn8uHNmDe99bwhOfruTFklLGDerCtw7rzNG925Ptb/6ZIyL1EQxHPFmSA+KYJJxza83sbmA1sBt4zzn3Xi1FzzGz0cAS4KfOudKaBczsSuBKgO7du8crZEkDnVrl8KdzB/HDY4t46D/L+ffcdTz3WSkts/0cP6AjV43uxeFdW3sdpsh/CYRSs7upLXAm0BM4BMgzs4tqFHsdKHLODQI+AJ6s7bWccxOcc8XOueKCgoJ4hSxpZEDnVjx4wVBm3XoyT1x6JOMGdeGTJWV8569TuPnleWzaUel1iCJ7BcLedTfF811PAlY658qcc0HgVeDo6gWcc1ucc1Wxu48Bw+MYj8g35GRmcPyAjtx5ziA+ufl4fnRsT/45Zy3H3/UxD/1nmbZWlYQQDEc8mf4K8U0Sq4GRZpZr0R1jTgQWVS9gZl2q3T2j5uMizal1i0x+dfpA3vvpGI7p04G73v2Ssx6aytKNO7wOTdJcSnY3OedmAC8Ds4EFsfeaYGa3mdkZsWI3xKbIziM6E+rSeMUjUl89O+Qx4eJiJl5czMbtlYx7cApPTl2lC/LEM17ObrJk+8UvLi52JSUlXochaWLTjkpufnk+H39Zxph+Bfz53EF0apVz4CeKNKHxE6YRicCLV49q9GuY2SznXHFDn6crrkX2o2N+Dk9ceiS3nXkY01ds4aR7J/FiSalaFdKsAh62JJQkRA7AzLh4VBFv/+Q4Du3ciptfns/Fj3/Gmq26cluaR6rObhJJKb0KWvL8lSO57czDmPXVVr593ye8s3CD12FJGgiGnCcbDoGShEiD+HzRVsW7N46mb6d8rn1mFv+Y/pXXYUmKi7YkvFkRQElCpBG6tcvl2SuOYmz/jvz6tYXc+96XGqeQuAmEUvM6CZGUlpvlZ8IPhnN+cSEPfLSMW15ZQCi2OYxIU4q2JLzpbor3An8iKc2f4eNP50SnxT740TIqqkL8ZfwQzy58ktTkZUtCSULkIJkZP/tWf1q3yOT3by4iFInw4AXDPJuNIqknqNlNIsnvR8f14jfjBvLu5xu59pnZWvdJmkxKLsshko4uO7Ynt515GB8s2sg1/5hNZVCJQg5OJOIIRZxaEiKp4uJRRfzhrMP5aPEmrnp6lhKFHJRAbDKEkoRICrnwqB7cefYRTFpSxhVPlShRSKPtTRLqbhJJLeNHdOfP5wxiyrLNXPFUCbsDShTScIGQWhIiKev8I7tx17mDmbJsM5c/OVOJQhosGGtJaOBaJEWdO7yQe84bzLQVW/jf1z/3OhxJMntbEkoSIqnr7GGFXD2mN8/PLNWigNIg6m4SSRM/PakfR3RtzS2vzmfj9kqvw5EkEVB3k0h6yPL7+Mv4IVQFI/z8pXlEIloQUA5sT0siWy0JkdTXu6Alt44byOSlm3n805VehyNJIBiOfplQd5NImrhgRDdOHtiJP7/zJV+s2+51OJLg9rQk1N0kkibMjD+dM4g2uZnc8PwcTYuV/QqEo78fakmIpJF2eVnce/4QlpdVcPubX3gdjiSwQCjW3aSWhEh6ObZvB64c3YtnZ6zWtFip0761m7THtUja+dnJ/RlUGJ0Wu758t9fhSALadzGd9rgWSTtZfh/3jx9KIBThxufnEta0WKkhqFVgRdJbzw553Hbm4cxY+TVPTVvldTiSYPbNblJ3k0jaOmdYV47r24G/fLCUbbsCXocjCUTLcogIZsavTj+UHZVB7v9wqdfhSALRshwiAsCAzq343pHdeXraV6woq/A6HEkQWgVWRPa66eR+5GRmcMdbi70ORRJEMBwhM8Pw+TQmIZL2CvKzufb43nywaCNTl232OhxJAIFQxLOuJlCSEEk4lx3Tk65tWnD7m4s0JVYIhCOeDVpDnJOEmf3UzD43s4Vm9pyZ5dR4PNvMXjCzZWY2w8yK4hmPSDLIyczgllMHsGj9dl6eVep1OOKxYDji2XgExDFJmFlX4Aag2Dl3OJABjK9R7HJgq3OuD3Af8Kd4xSOSTMYN6sLwHm25690lVFSFvA5HPFSV4t1NfqCFmfmBXGBdjcfPBJ6M3X4ZONHMvBmdEUkgZsat4wayuaKKh/+zzOtwxEOBUMSzDYcgjknCObcWuBtYDawHyp1z79Uo1hUojZUPAeVA+3jFJJJMhnRrw9lDuzJxykpKv97ldTjikWCqjkmYWVuiLYWewCFAnpldVLNYLU/9xkidmV1pZiVmVlJWVtb0wYokqF+c0p8MM+58R1Ni01Uqz246CVjpnCtzzgWBV4Gja5RZA3QDiHVJtQa+rvlCzrkJzrli51xxQUFBHEMWSSxdWrfgqjG9eHP+emau+safhqSBVJ7dtBoYaWa5sXGGE4FFNcr8G7gkdvtc4CPnnOb8iVRz1ejedGmdw22vf0FEU2LTTjDkPFvcD+I7JjGD6GD0bGBB7L0mmNltZnZGrNjfgPZmtgy4CbglXvGIJKsWWRn88pQBLFhbzqtz1nodjjSzqnCELL83e0lAdPZR3Djnfgv8tsbp31R7vBI4L54xiKSCMwYfwhNTV3H3u19y+hFdaJHl3X8a0ryCoRS9TkJEmo7PZ9x6+qFs2F7JY5NXeB2ONKPomEQKdjeJSNMqLmrHqYd35tFJy9m0vdLrcKSZBNSSEJH6uuXUAQTDEe55b4nXoUgzSdnrJESk6fVon8clo4p4cVYpX6zb7nU40gxS+ToJEYmD60/oS+sWmdzx1iI0Yzz1BUJqSYhIA7TOzeSGE/oyZdlmJi3RCgSpLpUvphOROLloZA8K27bgvg+WqjWRwpxz0SSR6N1NZtbbzLJjt8ea2Q1m1ia+oYlIXbL8Pq4d24d5pdv4ZKl2sEtVoYjDOe/2t4b6tyReAcJm1ofoVdI9gWfjFpWIHNC5wws5pHUO93+wRK2JFBUMRwCSorspElvK+yzgL865nwJd4heWiBxIlt/HNcf3YfbqbXy6bIvX4UgcBELRJJEMs5uCZnYB0cX43oidy4xPSCJSX+cXF9K5VQ73f6jWRCrakySSoSXxQ2AU8Afn3Eoz6wn8I35hiUh9ZPszuGZsb2au2sr0FVpKPNUE9nQ3JXpLwjn3hXPuBufcc7HNhPKdc3fGOTYRqYfvHdmNjvnZ3P+hrsJONUnTkjCzj82slZm1A+YBT5jZvfENTUTqIyczg6vG9Gb6iq/5bKVaE6kkGI52ISZ8kgBaO+e2A2cDTzjnhhPdeU5EEsD3R3SnXV4Wj05a7nUo0oSSaeDab2ZdgPPZN3AtIgmiRVYGl4wq4qPFm1i6cYfX4UgTCYTDQHK0JG4D3gWWO+dmmlkvYGn8whKRhvrBqB7kZPqY8In2m0gVgVCsuynRWxLOuZecc4Occ9fE7q9wzp0T39BEpCHa5WVxfnE3Xpu7lo3abyIl7J3dlOibDplZoZn908w2mdlGM3vFzArjHZyINMyPju1FOOJ44tNVXociTWDv7KYM77arrW8b5gng38AhQFfg9dg5EUkg3dvncurhXXhmxldUVIW8DkcOUjIty1HgnHvCOReKHX8HCuIYl4g00pWje7GjMsTzn632OhQ5SPtmNyV4dxOw2cwuMrOM2HERoMViRBLQ4G5tOKpnOx6fsnLvN1FJTklzMR1wGdHprxuA9cC5RJfqEJEEdNWYXqwrr+TN+eu9DkUOQiBZupucc6udc2c45wqccx2dc98lemGdiCSgsf060qsgjyemrvI6FDkI+wauEzxJ1OGmJotCRJqUz2dcenQR80q3MWf1Vq/DkUZKpoHr2ng3kiIiB3T2sEJaZvt5Uq2JpJVMy3LURovXiySwltl+zisu5M0F69m0QxfXJaNAOIIZ+H0JOrvJzHaY2fZajh1Er5kQkQR28agigmHHszM0HTYZBcIRsjJ8mCVoknDO5TvnWtVy5Dvn/M0VpIg0Ts8OeYztX8AzM1bv7bqQ5BEIRTwdtIaD624SkSRw6dFFlO2o4u2Fmg6bbAKhiKeD1qAkIZLyRvctoGeHPK3nlISCYSUJEYkzn8+4ZFQP5pZuY27pNq/DkQYIhCKezmyCOCYJM+tvZnOrHdvN7MYaZcaaWXm1Mr+JVzwi6eyc4YXkZ/uZOFl7TSSTQAK0JOI2+Oyc+xIYAmBmGcBa4J+1FJ3snBsXrzhEBPJzMvn+yO489skKVm/ZRff2uV6HJPUQCLm0Gbg+keiudl810/uJSA2XHdOTDJ8xcYpaE8kiEI6QmSZjEuOB5+p4bJSZzTOzt83ssGaKRyTtdGqVw1lDu/JiSSlbKqq8DkfqIRAKk53qLQkzywLOAF6q5eHZQA/n3GDgQeC1Ol7jSjMrMbOSsrKy+AUrkuKuHN2LymCEp6apUZ8MgmHn+ZhEc7z7qcBs59zGmg8457Y75ypit98CMs2sQy3lJjjnip1zxQUF2utIpLH6dMznpEM78dS0VewKaOe6RBed3eTtMnnNkSQuoI6uJjPrbLHrzc1sRCwebWYkEkdXj+nF1l1BXipZ43UocgApf52EmeUCJwOvVjt3tZldHbt7LrDQzOYBDwDjnXNaOFAkjoqL2jG8R1sem7yCkHauS2gpfZ0EgHNul3OuvXOuvNq5R51zj8Zu/9U5d5hzbrBzbqRzbmo84xGRqKtG92LN1t28vXCD16HIflRpWQ4R8cJJh3aiZ4c8Jk5ZiRrviSsYjpCtJCEizc3nMy47Jrpz3WztXJewAuEU724SkcR1zvBCWrfIZOLklV6HInXQUuEi4pncLD8XjOjOu59voPTrXV6HI7VI+dlNIpLYLjm6Bz4z/q59sBNOJOIIhp26m0TEO11at+D0QV14YWYpOyqDXocj1QRi05PVkhART11+bE8qqkK8MLPU61CkmmAsSWh2k4h4alBhG0YUtePvU1fp4roEsmdPcnU3iYjnLju2J2u27uadz3VxXaIIhqPXr6i7SUQ8d/LATvQuyOPBD5cRiejiukSwpyWhKbAi4rkMn3HDiX35cuMOLdWRIALhMEDabDokIglu3KBD6NOxJfd/uEStiQQQCMW6m9SSEJFEsKc1sWRjBW8uWO91OGlv3xTY1N9PQkSSxOlHdKFvx5bc/+FSwmpNeGrfmESGp3EoSYjIXhk+4ycn9WXZJrUmvBbUxXQikohOO7wL/Tvlc/8HS9Sa8NC+6yTU3SQiCcQXa00sL9vJxMkrvA4nbVWF1JIQkQR1ymGdOeWwzvzx7cU8qcX/PKFlOUQkYfl8xgMXDOXkgZ347b8/56lpq7wOKe1oWQ4RSWhZfh8PfX8YJx3aid/863Oenv6V1yGlFQ1ci0jCy/L7ePjCYZx0aEdufW0hb8xf53VIaWPvdRJqSYhIIsvy+3jowmEM6daGW19byOaKKq9DSgt7u5vUkhCRRJftz+CucwexsyrM7/79udfhpAW1JEQkqfTtlM8NJ/bhjfnreUeLAMadVoEVkaRz1ZjeDOzSilv/tZDyXdruNJ4CoQh+n+Hz6WI6EUkSmRk+/nzuIL7eGeD2N7/wOpyUFgxHPJ/ZBEoSItJAh3dtzTVjevPyrDV8tHij1+GkrEAo4vk1EqAkISKNcP2JfRjQOZ+fvTiP9eW7vQ4nJQXUkhCRZJXtz+DhC4cRCEW4/tk5ey/8kqYTCDnPB61BSUJEGqlXQUvuOPsISr7ayt3vfel1OClHLQkRSXpnDunK94/qzv9NWsGHizQ+0ZQCobBaEiKS/H4zbiADu7Tiphfn8eGijewOhL0OKSUEwy4hWhJ+rwMQkeSWkxkdnzjnkalc/mQJWX4fR/Vsx5h+BZx/ZDda5WR6HWJSis5u8vYaCYhjS8LM+pvZ3GrHdjO7sUYZM7MHzGyZmc03s2HxikdE4qeoQx6f3nICT18+gotH9mBDeSW/f3MR5zw8lTVbd3kdXlJK+TEJ59yXzrkhzrkhwHBgF/DPGsVOBfrGjiuBR+IVj4jEV05mBsf1LeDX4wby/k1jePZHR7FheyVnPzyVz9eVex1e0gmEImT5M7wOo9nGJE4Eljvnai5IfybwlIuaDrQxsy7NFJOIxNHRfTrw8tVH4/cZ5z86jUlLyrwOKakEQhGyUrm7qYbxwHO1nO8KlFa7vyZ27r+Y2ZVmVmJmJWVl+kUTSRb9O+fz6rXH0K1dLpf9faZmQDVA2izLYWZZwBnAS7U9XMs5940Tzk1wzhU754oLCgqaOkQRiaPOrXN46epR9OuUzy9fWcDWnQGvQ0oKgXD6LMtxKjDbOVfbV4g1QLdq9wsBbX0lkmLyczK5+7xBbNsV4PY3tDBgfUS7m9IjSVxA7V1NAP8GLo7NchoJlDvn1jdDTCLSzA47pDXXju3Nq3PWamHAekiL7iYzywVOBl6tdu5qM7s6dvctYAWwDHgMuDae8YiIt358Ql/6d8rnf15dyPZK7UexP1XpsAqsc26Xc669c6682rlHnXOPxm4759x1zrnezrkjnHMl8YxHRLyV5fdx13mD2LSjkjveXOR1OAnLOUcgFCE71VsSIiI1DSpsw5Wje/P8zFJNi63D8rKdVIUidG+f63UoShIi0vxuPCna7XTTC3PZtL3S63ASzsdfbgJgTD/vZ3MqSYhIs8vJzOChC4eyKxDmJ8/PJRz5xsz3tDZpSRl9OraksK1aEiKSpvp0zOf27x7OtBVbeODDpV6HkzB2B8LMWPl1QrQiQElCRDx07vBCzhlWyAMfLeXTZZu9DichTF+xhUAowtj+ShIiItz+3cPoXdCSnzw/l007ND4xaUkZLTIzOLKondehAEoSIuKx3Cw/D31/GDsqg9zyygKcS+/xiY+/3MSo3u3JyfR+BVhQkhCRBNC/cz63nDqAjxZv4qWSNV6H45lVm3eyasuuhBmPACUJEUkQl4wqYlSv9tz2xheUfp2eGxV9sjR63YiShIhIDT6f8edzBwFw88vziaThtNiPvyyjqH0uRR3yvA5lLyUJEUkY3drlcuu4Q5m2YgtPTlvldTjNqjIYZtryLQnVigAlCRFJMOcXd+OEAR258+3FfLFuu9fhNJuZq75mdzDM2P4dvQ7lvyhJiEhCMTPuPPsI8nMyOeeRqbwwc3VazHia9GUZWX4fR/VKjKmveyhJiEjC6dgqhzdvOJah3dvwy1cWcP1zc1J6aXHnHB8vKeOonu3IzfJ7Hc5/UZIQkYTUqVUOT19+FL/4dn/eXriB0+6fzKL1qdn99NHiTSzbVMGph3fxOpRvUJIQkYSV4TOuO74PL109ilDYcdHEGawoq/A6rCYVCkf449uL6dkhj/OKC70O5xuUJEQk4Q3r3pZnrzgKgIsmzmDttt0eR9R0XigpZdmmCn55yoCE2ImupsSLSESkFr0KWvLU5SPYURXiBxNnULajyuuQDlpFVYj73l/CkUVt+fZhnbwOp1ZKEiKSNA47pDVPXHok68srufjxz1izNbmvzJ4waTmbKwL8z2mHYmZeh1MrJQkRSSrFRe34vx8MZ/mmCo7903846+FPmTh5RdIljA3llUyYvIJxg7owtHtbr8OpkyXb/OPi4mJXUlLidRgi4rHVW3bx+vx1vLVgPZ/HLrrLz/bTIT+b9nlZFORnc8KAjpwx5BCy/Ymxomp1v3hpHv+au44PfzaGbu3ivwOdmc1yzhU3+HlKEiKS7FZt3skHizayZutuNldUsbmiitKvd7N222465mdjbCESAAAL50lEQVRzydFFXHhUdwBmrtrKZyu3sGBtOScP7MwPjy7C52verp5F67dz2gOTufyYnvx63MBmeU8lCRGRapxzTF66mccmr2Dy0s1k+X0EwxGcgyy/j8K2LVhRtpMx/Qq4+7zBFORnN1tsP/jbDOavKWfSL8bSJjerWd6zsUkisS7tExFpImbG6H4FjO5XwOIN23lhZintcrMY0bMdg7u1Idvv45kZq7n9jS849f5PuPu8wc2ybtInS8qYvHQzvz790GZLEAdDLQkRSWtLNu7ghufmsHjDDr475BBuPKlf3JbqDkccpz8wmZ2BEB/cNKZZx0oa25LQ7CYRSWv9OuXz2nXHcM3Y3rzz+QZOvHcSv3x5/t7ZUjurQny1ZScL1pQTPsg9Ll6ZvYbFG3Zw87cHJORgem3UkhARidm0o5KH/7OcZ2esJuIcWX4fuwLhvY+fclhnHrloWKOuadgdCDP27v/QuXULXrv26Ga/LkJjEiIiB6ljfg6/O+Mwrhzdi6emfUUgFKEgP5sOLbNYuqmCCZ+s4Mmpq7j0mJ4Nfu2/TVnBxu1VPHhB45KMV5QkRERqOKRNC245dcB/nYtEHMs3VfCHtxYxrEdbBhW2qffr/WvuWh78aBnfGtiJET0Ta7+IA9GYhIhIPfh8xj3nD6Zjfg7XPTub8t0H3t8iGI5w+xtf8JPn5zK4sA13nH1EM0TatJQkRETqqU1uFg9cMJT12yq5+eV5+90xr2xHFRdNnMHfpqzk0qOLeOaKo+jQsvmuxWgq6m4SEWmA4T3a8stTBvCHtxZx5B8+JNvvw59h+H1GKOKoDIapCkXYWRXCZ8a95w/m7GGJt09EfcU1SZhZG2AicDjggMucc9OqPT4W+BewMnbqVefcbfGMSUTkYP3ouOjA9fKyCoJhRygSIRR2ZGYY2f4MsjN9tMjM4MwhXRl4SCuPoz048W5J3A+845w718yygNpWsZrsnBsX5zhERJqMmXHF6F5eh9Es4pYkzKwVMBq4FMA5FwAC8Xo/ERFpevEcuO4FlAFPmNkcM5toZrVd6z7KzOaZ2dtmdlgc4xERkQaKZ5LwA8OAR5xzQ4GdwC01yswGejjnBgMPAq/V9kJmdqWZlZhZSVlZWRxDFhGR6uKZJNYAa5xzM2L3XyaaNPZyzm13zlXEbr8FZJpZh5ov5Jyb4Jwrds4VFxQUxDFkERGpLm5Jwjm3ASg1s/6xUycCX1QvY2adLXZ9upmNiMWzJV4xiYhIw8R7dtP1wDOxmU0rgB+a2dUAzrlHgXOBa8wsBOwGxrtkW3FQRCSFaRVYEZE0oP0kRESkySVdS8LMyoCvanmoNVDeyPt7bu/52QHY3MgQa75PQ8skSj0OFOeBHm/KekB8P5OG1KO2c7XFXv226lH/OA9URvVofD16OOcaPvPHOZcSBzChsff33K72s6Sp4mhomUSpR33q0lz1iPdn0pB61Dd21aPx9dhfGdWj6etxoCOVupteP4j7r9dRpiniaGiZRKlHfV4nHetR27naYq9+W/U4cCz1LaN6NH099ivpupuag5mVuEYM8CSaVKkHpE5dVI/EonocWCq1JJrSBK8DaCKpUg9InbqoHolF9TgAtSRERKROakmIiEidUj5JmNnjZrbJzBY24rnDzWyBmS0zswf2LCESe+x6M/vSzD43sz83bdS1xtLk9TCz35nZWjObGztOa/rIvxFLXD6P2OM/NzNX2/pfTS1On8ftZjY/9lm8Z2aHNH3k34glHvW4y8wWx+ryz9jmY3EVp3qcF/v7jphZXMctDib+Ol7vEjNbGjsuqXZ+v39DtYrXtKlEOYjuaTEMWNiI534GjAIMeBs4NXb+eOADIDt2v2OS1uN3wM+T/fOIPdYNeJfoNTQdkrEeQKtqZW4AHk3SenwL8Mdu/wn4U5LW41CgP/AxUJyI8cdiK6pxrh3RZZDaAW1jt9vur677O1K+JeGc+wT4uvo5M+ttZu+Y2Swzm2xmA2o+z8y6EP2jneai/7pPAd+NPXwNcKdzrir2HpviW4u41aPZxbEe9wE3E90mN+7iUQ/n3PZqRfNohrrEqR7vOedCsaLTgbhv8Byneixyzn0Z79gPJv46fBt43zn3tXNuK/A+cEpj/y9I+SRRhwnA9c654cDPgYdrKdOV6HLne6yJnQPoBxxnZjPMbJKZHRnXaOt2sPUA+HGsW+BxM2sbv1D366DqYWZnAGudc/PiHegBHPTnYWZ/MLNS4ELgN3GMdX+a4vdqj8uIfmP1QlPWwwv1ib82XYHSavf31KlRdY33KrAJx8xaAkcDL1XrjsuurWgt5/Z8s/MTbcaNBI4EXjSzXrHs3CyaqB6PALfH7t8O3EP0j7rZHGw9zCwX+BXRLg7PNNHngXPuV8CvzOz/AT8GftvEoe5XU9Uj9lq/AkLAM00ZY300ZT28sL/4zeyHwE9i5/oAb5lZAFjpnDuLuuvUqLqmXZIg2nra5pwbUv2kmWUAs2J3/030P9DqzeRCYF3s9hrg1VhS+MzMIkTXTmnObfMOuh7OuY3VnvcY8EY8A67DwdajN9ATmBf7YyoEZpvZCBfd06S5NMXvVXXPAm/SzEmCJqpHbLB0HHBic355qqapP4/mVmv8AM65J4AnAMzsY+BS59yqakXWAGOr3S8kOnaxhsbUNZ6DMYlyAEVUGxACpgLnxW4bMLiO580k2lrYM8hzWuz81cBtsdv9iDbtLAnr0aVamZ8Czyfj51GjzCqaYeA6Tp9H32plrgdeTtJ6nEJ0g7GC5og/3r9XNMPAdWPjp+6B65VEezvaxm63q09da42rOT9ELw7gOWA9ECSaSS8n+s3zHWBe7Jf5N3U8txhYCCwH/sq+iw+zgH/EHpsNnJCk9XgaWADMJ/qtqksy1qNGmVU0z+ymeHwer8TOzye6Lk/XJK3HMqJfnObGjuaYpRWPepwVe60qYCPwbqLFTy1JInb+stjnsAz4YUP+hmoeuuJaRETqlK6zm0REpB6UJEREpE5KEiIiUiclCRERqZOShIiI1ElJQlKCmVU08/tNNLOBTfRaYYuu/LrQzF4/0KqpZtbGzK5tivcWORBNgZWUYGYVzrmWTfh6frdvkbq4qh67mT0JLHHO/WE/5YuAN5xzhzdHfJLe1JKQlGVmBWb2ipnNjB3HxM6PMLOpZjYn9rN/7PylZvaSmb0OvGdmY83sYzN72aL7IzyzZ/392Pni2O2K2MJ888xsupl1ip3vHbs/08xuq2drZxr7Fi5saWYfmtlsi+4BcGaszJ1A71jr465Y2V/E3me+mf1vE/4zSppTkpBUdj9wn3PuSOAcYGLs/GJgtHNuKNGVVu+o9pxRwCXOuRNi94cCNwIDgV7AMbW8Tx4w3Tk3GPgEuKLa+98fe/8DrpETW1foRKJXvwNUAmc554YR3cPknliSugVY7pwb4pz7hZl9C+gLjACGAMPNbPSB3k+kPtJxgT9JHycBA6utotnKzPKB1sCTZtaX6CqYmdWe875zrvq6/p8559YAmNlcouvrTKnxPgH2LY44Czg5dnsU+9brfxa4u444W1R77VlE1/+H6Po6d8T+w48QbWF0quX534odc2L3WxJNGp/U8X4i9aYkIanMB4xyzu2uftLMHgT+45w7K9a//3G1h3fWeI2qarfD1P43E3T7BvfqKrM/u51zQ8ysNdFkcx3wANE9JQqA4c65oJmtAnJqeb4Bf3TO/V8D31fkgNTdJKnsPaJ7MgBgZnuWXW4NrI3dvjSO7z+daDcXwPgDFXbOlRPdtvTnZpZJNM5NsQRxPNAjVnQHkF/tqe8Cl8X2IMDMuppZxyaqg6Q5JQlJFblmtqbacRPR/3CLY4O5XxBd4h3gz8AfzexTICOOMd0I3GRmnwFdgPIDPcE5N4foqp/jiW7WU2xmJURbFYtjZbYAn8amzN7lnHuPaHfWNDNbALzMfycRkUbTFFiROIntmrfbOefMbDxwgXPuzAM9TySRaExCJH6GA3+NzUjaRjNvDSvSFNSSEBGROmlMQkRE6qQkISIidVKSEBGROilJiIhInZQkRESkTkoSIiJSp/8PrDu+EN2SIcMAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn.recorder.plot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>bleu</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>2.432767</td>\n",
       "      <td>2.478084</td>\n",
       "      <td>0.618113</td>\n",
       "      <td>0.460726</td>\n",
       "      <td>01:03</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>1.959129</td>\n",
       "      <td>1.978793</td>\n",
       "      <td>0.686264</td>\n",
       "      <td>0.505202</td>\n",
       "      <td>01:06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>1.536871</td>\n",
       "      <td>1.656002</td>\n",
       "      <td>0.723556</td>\n",
       "      <td>0.543357</td>\n",
       "      <td>01:07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>1.371858</td>\n",
       "      <td>1.453879</td>\n",
       "      <td>0.749335</td>\n",
       "      <td>0.570762</td>\n",
       "      <td>01:07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>1.113087</td>\n",
       "      <td>1.345275</td>\n",
       "      <td>0.764912</td>\n",
       "      <td>0.590188</td>\n",
       "      <td>01:07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5</td>\n",
       "      <td>0.970879</td>\n",
       "      <td>1.287549</td>\n",
       "      <td>0.774231</td>\n",
       "      <td>0.602932</td>\n",
       "      <td>01:06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6</td>\n",
       "      <td>0.852550</td>\n",
       "      <td>1.280241</td>\n",
       "      <td>0.778895</td>\n",
       "      <td>0.609807</td>\n",
       "      <td>01:07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7</td>\n",
       "      <td>0.771543</td>\n",
       "      <td>1.282661</td>\n",
       "      <td>0.779013</td>\n",
       "      <td>0.610055</td>\n",
       "      <td>01:08</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn.fit_one_cycle(8, 5e-4, div_factor=5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_predictions(learn, ds_type=DatasetType.Valid):\n",
    "    learn.model.eval()\n",
    "    inputs, targets, outputs = [],[],[]\n",
    "    with torch.no_grad():\n",
    "        for xb,yb in progress_bar(learn.dl(ds_type)):\n",
    "            out = learn.model(*xb)\n",
    "            for x,y,z in zip(xb[0],xb[1],out):\n",
    "                inputs.append(learn.data.train_ds.x.reconstruct(x))\n",
    "                targets.append(learn.data.train_ds.y.reconstruct(y))\n",
    "                outputs.append(learn.data.train_ds.y.reconstruct(z.argmax(1)))\n",
    "    return inputs, targets, outputs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "        <style>\n",
       "            /* Turns off some styling */\n",
       "            progress {\n",
       "                /* gets rid of default border in Firefox and Opera. */\n",
       "                border: none;\n",
       "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "                background-size: auto;\n",
       "            }\n",
       "            .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "                background: #F44336;\n",
       "            }\n",
       "        </style>\n",
       "      <progress value='149' class='' max='149', style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      100.00% [149/149 00:16<00:00]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "inputs, targets, outputs = get_predictions(learn)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(Text xxbos xxmaj comment cela a - t - il pu se produire et pourquoi n'y a - t - il pas de mécanismes en place pour empêcher cette situation ?,\n",
       " Text xxbos xxmaj why was this allowed to happen and why are n't there any mechanisms in place to prevent this ?,\n",
       " Text xxbos xxmaj why did this happening to take and why is there there no measures not this to prevent this situation)"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "inputs[10],targets[10],outputs[10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(Text xxbos xxmaj qu’advient - il lorsque les attentes sont xxunk et même xxunk ou qu’elles incitent l’organisation à xxunk de ses activités de base ?,\n",
       " Text xxbos xxmaj what happens when expectations are diverse and even xxunk , or take the organization away from its core business ?,\n",
       " Text xxbos xxmaj what happens when expectations are xxunk and their even or or they - organization 's from its core activities activities)"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "inputs[700],targets[700],outputs[700]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(Text xxbos xxmaj quelles mesures sont prises par les autorités sanitaires provinciales et locales pour assurer la sécurité du public lorsque des insecticides sont utilisés ?,\n",
       " Text xxbos xxmaj what steps do provincial / local health authorities take to ensure public safety when pesticides are used ?,\n",
       " Text xxbos xxmaj what measures are provincial and local health authorities have to ensure the safety and xxunk are used ?)"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "inputs[701],targets[701],outputs[701]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(Text xxbos xxmaj quand les restrictions à la propriété concernant les xxunk aux postes terrestres seront - elles xxunk ?,\n",
       " Text xxbos xxmaj when will land xxmaj border store restrictions on ownership be lifted ?,\n",
       " Text xxbos xxmaj when will the restrictions property xxmaj restrictions be the of xxunk ?)"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "inputs[2500],targets[2500],outputs[2500]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(Text xxbos xxmaj qu’est - ce qu’une résolution du conseil de bande ( xxup xxunk ) ?,\n",
       " Text xxbos xxmaj what is a xxmaj band xxmaj council xxmaj resolution ( xxup xxunk ) ?,\n",
       " Text xxbos xxmaj what is a xxmaj band xxmaj board xxmaj agreement ? xxup xxunk ) ?)"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "inputs[4002],targets[4002],outputs[4002]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Label smoothing"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "They point out in the paper that using label smoothing helped getting a better BLEU/accuracy, even if it made the loss worse."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = Transformer(len(data.train_ds.x.vocab.itos), len(data.train_ds.y.vocab.itos), d_model=256)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn = Learner(data, model, metrics=[accuracy, CorpusBLEU(len(data.train_ds.y.vocab.itos))], \n",
    "                loss_func=FlattenedLoss(LabelSmoothingCrossEntropy, axis=-1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>bleu</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>3.288097</td>\n",
       "      <td>3.361678</td>\n",
       "      <td>0.620848</td>\n",
       "      <td>0.462028</td>\n",
       "      <td>01:07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>2.866131</td>\n",
       "      <td>2.932534</td>\n",
       "      <td>0.690689</td>\n",
       "      <td>0.508931</td>\n",
       "      <td>01:08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>2.617174</td>\n",
       "      <td>2.686622</td>\n",
       "      <td>0.720212</td>\n",
       "      <td>0.535796</td>\n",
       "      <td>01:07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>2.396165</td>\n",
       "      <td>2.509400</td>\n",
       "      <td>0.750236</td>\n",
       "      <td>0.572231</td>\n",
       "      <td>01:07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>2.234806</td>\n",
       "      <td>2.415202</td>\n",
       "      <td>0.764170</td>\n",
       "      <td>0.590421</td>\n",
       "      <td>01:07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5</td>\n",
       "      <td>2.094995</td>\n",
       "      <td>2.375675</td>\n",
       "      <td>0.773611</td>\n",
       "      <td>0.601696</td>\n",
       "      <td>01:07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6</td>\n",
       "      <td>1.979516</td>\n",
       "      <td>2.361270</td>\n",
       "      <td>0.778426</td>\n",
       "      <td>0.607941</td>\n",
       "      <td>01:07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7</td>\n",
       "      <td>1.968844</td>\n",
       "      <td>2.363652</td>\n",
       "      <td>0.778740</td>\n",
       "      <td>0.608949</td>\n",
       "      <td>01:07</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn.fit_one_cycle(8, 5e-4, div_factor=5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>bleu</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>2.005607</td>\n",
       "      <td>2.370556</td>\n",
       "      <td>0.776371</td>\n",
       "      <td>0.606602</td>\n",
       "      <td>01:07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>2.039403</td>\n",
       "      <td>2.376988</td>\n",
       "      <td>0.773496</td>\n",
       "      <td>0.602594</td>\n",
       "      <td>01:08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>1.998095</td>\n",
       "      <td>2.364960</td>\n",
       "      <td>0.776239</td>\n",
       "      <td>0.606155</td>\n",
       "      <td>01:07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>1.948557</td>\n",
       "      <td>2.333933</td>\n",
       "      <td>0.783300</td>\n",
       "      <td>0.617531</td>\n",
       "      <td>01:07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>1.846455</td>\n",
       "      <td>2.334134</td>\n",
       "      <td>0.787573</td>\n",
       "      <td>0.623490</td>\n",
       "      <td>01:07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5</td>\n",
       "      <td>1.729133</td>\n",
       "      <td>2.339518</td>\n",
       "      <td>0.792536</td>\n",
       "      <td>0.632397</td>\n",
       "      <td>01:07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6</td>\n",
       "      <td>1.623762</td>\n",
       "      <td>2.345591</td>\n",
       "      <td>0.794781</td>\n",
       "      <td>0.636248</td>\n",
       "      <td>01:08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7</td>\n",
       "      <td>1.599179</td>\n",
       "      <td>2.351281</td>\n",
       "      <td>0.794569</td>\n",
       "      <td>0.636407</td>\n",
       "      <td>01:08</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn.fit_one_cycle(8, 5e-4, div_factor=5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Quels sont les atouts particuliers du Canada en recherche sur l'obésité sur la scène internationale ?\n",
      "What are Specific strengths canada strengths in obesity - ? are up canada ? from international international stage ?\n",
      "Quelles sont les répercussions politiques à long terme de cette révolution scientifique mondiale ?\n",
      "What are the long the long - term policies implications of this global scientific ? ?\n"
     ]
    }
   ],
   "source": [
    "print(\"Quels sont les atouts particuliers du Canada en recherche sur l'obésité sur la scène internationale ?\")\n",
    "print(\"What are Specific strengths canada strengths in obesity - ? are up canada ? from international international stage ?\")\n",
    "print(\"Quelles sont les répercussions politiques à long terme de cette révolution scientifique mondiale ?\")\n",
    "print(\"What are the long the long - term policies implications of this global scientific ? ?\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(Text xxbos xxmaj comment cela a - t - il pu se produire et pourquoi n'y a - t - il pas de mécanismes en place pour empêcher cette situation ?,\n",
       " Text xxbos xxmaj why was this allowed to happen and why are n't there any mechanisms in place to prevent this ?,\n",
       " Text xxbos xxmaj why did this happening to take and why is there there no measures not this to prevent this situation)"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "inputs[10],targets[10],outputs[10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(Text xxbos xxmaj qu’advient - il lorsque les attentes sont xxunk et même xxunk ou qu’elles incitent l’organisation à xxunk de ses activités de base ?,\n",
       " Text xxbos xxmaj what happens when expectations are diverse and even xxunk , or take the organization away from its core business ?,\n",
       " Text xxbos xxmaj what happens when expectations are xxunk and their even or or they - organization 's from its core activities activities)"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "inputs[700],targets[700],outputs[700]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(Text xxbos xxmaj quelles mesures sont prises par les autorités sanitaires provinciales et locales pour assurer la sécurité du public lorsque des insecticides sont utilisés ?,\n",
       " Text xxbos xxmaj what steps do provincial / local health authorities take to ensure public safety when pesticides are used ?,\n",
       " Text xxbos xxmaj what measures are provincial and local health authorities have to ensure the safety and xxunk are used ?)"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "inputs[701],targets[701],outputs[701]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(Text xxbos xxmaj qui a dit qu'un xxmaj xxunk xxmaj xxunk ne xxunk que des armes ?,\n",
       " Text xxbos xxmaj who said a xxmaj xxunk xxmaj tech would have to work on weapons ?,\n",
       " Text xxbos xxmaj who said xxmaj xxmaj xxunk xxmaj xxunk xxunk not not xxunk in weapons ?)"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "inputs[4001],targets[4001],outputs[4001]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Test leakage"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "If we change a token in the targets at position n, it shouldn't impact the predictions before that."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn.model.eval();"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "xb,yb = data.one_batch(cpu=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "inp1,out1 = xb[0][:1],xb[1][:1]\n",
    "inp2,out2 = inp1.clone(),out1.clone()\n",
    "out2[0,15] = 10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "y1 = learn.model(inp1, out1)\n",
    "y2 = learn.model(inp2, out2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(0., device='cuda:0', grad_fn=<MeanBackward0>)"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(y1[0,:15] - y2[0,:15]).abs().mean()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}