{ "cells": [ { "cell_type": "markdown", "metadata": { "id": "70HL69ki43xO" }, "source": [ "Link to the notebook: https://colab.research.google.com/drive/1EFHqj3IFFS-jlp04HV2kMciuT55uBHn8?usp=sharing\n", "Copy the notebook to your GDrive to edit. (to update)" ] }, { "cell_type": "markdown", "metadata": { "id": "ZBx9cWUD-n3Y" }, "source": [ "# Download prerequisite packages\n", "\n", "First we will install extra packages which aren't included in the default colab environment. In this case, we are installing `gensim` for GloVe embeddings, `bpemb` for word-piece embeddings and tokenization, and some extra packages for tokenization." ] }, { "cell_type": "code", "execution_count": 1, "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "0sqP726GGZOe", "outputId": "714796f0-1b46-4ee1-8c96-7fbd2242487d" }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Requirement already satisfied: bpemb in /usr/local/lib/python3.10/dist-packages (0.3.5)\n", "Requirement already satisfied: gensim in /usr/local/lib/python3.10/dist-packages (from bpemb) (4.3.3)\n", "Requirement already satisfied: numpy in /usr/local/lib/python3.10/dist-packages (from bpemb) (1.26.4)\n", "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from bpemb) (2.32.3)\n", "Requirement already satisfied: sentencepiece in /usr/local/lib/python3.10/dist-packages (from bpemb) (0.1.99)\n", "Requirement already satisfied: tqdm in /usr/local/lib/python3.10/dist-packages (from bpemb) (4.66.5)\n", "Requirement already satisfied: scipy<1.14.0,>=1.7.0 in /usr/local/lib/python3.10/dist-packages (from gensim->bpemb) (1.13.1)\n", "Requirement already satisfied: smart-open>=1.8.1 in /usr/local/lib/python3.10/dist-packages (from gensim->bpemb) (7.0.4)\n", "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests->bpemb) (3.3.2)\n", "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->bpemb) (3.8)\n", "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->bpemb) (2.0.7)\n", "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->bpemb) (2024.8.30)\n", "Requirement already satisfied: wrapt in /usr/local/lib/python3.10/dist-packages (from smart-open>=1.8.1->gensim->bpemb) (1.16.0)\n", "Requirement already satisfied: gensim in /usr/local/lib/python3.10/dist-packages (4.3.3)\n", "Requirement already satisfied: numpy<2.0,>=1.18.5 in /usr/local/lib/python3.10/dist-packages (from gensim) (1.26.4)\n", "Requirement already satisfied: scipy<1.14.0,>=1.7.0 in /usr/local/lib/python3.10/dist-packages (from gensim) (1.13.1)\n", "Requirement already satisfied: smart-open>=1.8.1 in /usr/local/lib/python3.10/dist-packages (from gensim) (7.0.4)\n", "Requirement already satisfied: wrapt in /usr/local/lib/python3.10/dist-packages (from smart-open>=1.8.1->gensim) (1.16.0)\n", "Collecting en-core-web-sm==3.7.1\n", " Downloading https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.7.1/en_core_web_sm-3.7.1-py3-none-any.whl (12.8 MB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m12.8/12.8 MB\u001b[0m \u001b[31m101.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hRequirement already satisfied: spacy<3.8.0,>=3.7.2 in /usr/local/lib/python3.10/dist-packages (from en-core-web-sm==3.7.1) (3.7.6)\n", "Requirement already satisfied: spacy-legacy<3.1.0,>=3.0.11 in /usr/local/lib/python3.10/dist-packages (from spacy<3.8.0,>=3.7.2->en-core-web-sm==3.7.1) (3.0.12)\n", "Requirement already satisfied: spacy-loggers<2.0.0,>=1.0.0 in /usr/local/lib/python3.10/dist-packages (from spacy<3.8.0,>=3.7.2->en-core-web-sm==3.7.1) (1.0.5)\n", "Requirement already satisfied: murmurhash<1.1.0,>=0.28.0 in /usr/local/lib/python3.10/dist-packages (from spacy<3.8.0,>=3.7.2->en-core-web-sm==3.7.1) (1.0.10)\n", "Requirement already satisfied: cymem<2.1.0,>=2.0.2 in /usr/local/lib/python3.10/dist-packages (from spacy<3.8.0,>=3.7.2->en-core-web-sm==3.7.1) (2.0.8)\n", "Requirement already satisfied: preshed<3.1.0,>=3.0.2 in /usr/local/lib/python3.10/dist-packages (from spacy<3.8.0,>=3.7.2->en-core-web-sm==3.7.1) (3.0.9)\n", "Requirement already satisfied: thinc<8.3.0,>=8.2.2 in /usr/local/lib/python3.10/dist-packages (from spacy<3.8.0,>=3.7.2->en-core-web-sm==3.7.1) (8.2.5)\n", "Requirement already satisfied: wasabi<1.2.0,>=0.9.1 in /usr/local/lib/python3.10/dist-packages (from spacy<3.8.0,>=3.7.2->en-core-web-sm==3.7.1) (1.1.3)\n", "Requirement already satisfied: srsly<3.0.0,>=2.4.3 in /usr/local/lib/python3.10/dist-packages (from spacy<3.8.0,>=3.7.2->en-core-web-sm==3.7.1) (2.4.8)\n", "Requirement already satisfied: catalogue<2.1.0,>=2.0.6 in /usr/local/lib/python3.10/dist-packages (from spacy<3.8.0,>=3.7.2->en-core-web-sm==3.7.1) (2.0.10)\n", "Requirement already satisfied: weasel<0.5.0,>=0.1.0 in /usr/local/lib/python3.10/dist-packages (from spacy<3.8.0,>=3.7.2->en-core-web-sm==3.7.1) (0.4.1)\n", "Requirement already satisfied: typer<1.0.0,>=0.3.0 in /usr/local/lib/python3.10/dist-packages (from spacy<3.8.0,>=3.7.2->en-core-web-sm==3.7.1) (0.12.5)\n", "Requirement already satisfied: tqdm<5.0.0,>=4.38.0 in /usr/local/lib/python3.10/dist-packages (from spacy<3.8.0,>=3.7.2->en-core-web-sm==3.7.1) (4.66.5)\n", "Requirement already satisfied: requests<3.0.0,>=2.13.0 in /usr/local/lib/python3.10/dist-packages (from spacy<3.8.0,>=3.7.2->en-core-web-sm==3.7.1) (2.32.3)\n", "Requirement already satisfied: pydantic!=1.8,!=1.8.1,<3.0.0,>=1.7.4 in /usr/local/lib/python3.10/dist-packages (from spacy<3.8.0,>=3.7.2->en-core-web-sm==3.7.1) (2.8.2)\n", "Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from spacy<3.8.0,>=3.7.2->en-core-web-sm==3.7.1) (3.1.4)\n", "Requirement already satisfied: setuptools in /usr/local/lib/python3.10/dist-packages (from spacy<3.8.0,>=3.7.2->en-core-web-sm==3.7.1) (71.0.4)\n", "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.10/dist-packages (from spacy<3.8.0,>=3.7.2->en-core-web-sm==3.7.1) (24.1)\n", "Requirement already satisfied: langcodes<4.0.0,>=3.2.0 in /usr/local/lib/python3.10/dist-packages (from spacy<3.8.0,>=3.7.2->en-core-web-sm==3.7.1) (3.4.0)\n", "Requirement already satisfied: numpy>=1.19.0 in /usr/local/lib/python3.10/dist-packages (from spacy<3.8.0,>=3.7.2->en-core-web-sm==3.7.1) (1.26.4)\n", "Requirement already satisfied: language-data>=1.2 in /usr/local/lib/python3.10/dist-packages (from langcodes<4.0.0,>=3.2.0->spacy<3.8.0,>=3.7.2->en-core-web-sm==3.7.1) (1.2.0)\n", "Requirement already satisfied: annotated-types>=0.4.0 in /usr/local/lib/python3.10/dist-packages (from pydantic!=1.8,!=1.8.1,<3.0.0,>=1.7.4->spacy<3.8.0,>=3.7.2->en-core-web-sm==3.7.1) (0.7.0)\n", "Requirement already satisfied: pydantic-core==2.20.1 in /usr/local/lib/python3.10/dist-packages (from pydantic!=1.8,!=1.8.1,<3.0.0,>=1.7.4->spacy<3.8.0,>=3.7.2->en-core-web-sm==3.7.1) (2.20.1)\n", "Requirement already satisfied: typing-extensions>=4.6.1 in /usr/local/lib/python3.10/dist-packages (from pydantic!=1.8,!=1.8.1,<3.0.0,>=1.7.4->spacy<3.8.0,>=3.7.2->en-core-web-sm==3.7.1) (4.12.2)\n", "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests<3.0.0,>=2.13.0->spacy<3.8.0,>=3.7.2->en-core-web-sm==3.7.1) (3.3.2)\n", "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests<3.0.0,>=2.13.0->spacy<3.8.0,>=3.7.2->en-core-web-sm==3.7.1) (3.8)\n", "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests<3.0.0,>=2.13.0->spacy<3.8.0,>=3.7.2->en-core-web-sm==3.7.1) (2.0.7)\n", "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests<3.0.0,>=2.13.0->spacy<3.8.0,>=3.7.2->en-core-web-sm==3.7.1) (2024.8.30)\n", "Requirement already satisfied: blis<0.8.0,>=0.7.8 in /usr/local/lib/python3.10/dist-packages (from thinc<8.3.0,>=8.2.2->spacy<3.8.0,>=3.7.2->en-core-web-sm==3.7.1) (0.7.11)\n", "Requirement already satisfied: confection<1.0.0,>=0.0.1 in /usr/local/lib/python3.10/dist-packages (from thinc<8.3.0,>=8.2.2->spacy<3.8.0,>=3.7.2->en-core-web-sm==3.7.1) (0.1.5)\n", "Requirement already satisfied: click>=8.0.0 in /usr/local/lib/python3.10/dist-packages (from typer<1.0.0,>=0.3.0->spacy<3.8.0,>=3.7.2->en-core-web-sm==3.7.1) (8.1.7)\n", "Requirement already satisfied: shellingham>=1.3.0 in /usr/local/lib/python3.10/dist-packages (from typer<1.0.0,>=0.3.0->spacy<3.8.0,>=3.7.2->en-core-web-sm==3.7.1) (1.5.4)\n", "Requirement already satisfied: rich>=10.11.0 in /usr/local/lib/python3.10/dist-packages (from typer<1.0.0,>=0.3.0->spacy<3.8.0,>=3.7.2->en-core-web-sm==3.7.1) (13.8.0)\n", "Requirement already satisfied: cloudpathlib<1.0.0,>=0.7.0 in /usr/local/lib/python3.10/dist-packages (from weasel<0.5.0,>=0.1.0->spacy<3.8.0,>=3.7.2->en-core-web-sm==3.7.1) (0.19.0)\n", "Requirement already satisfied: smart-open<8.0.0,>=5.2.1 in /usr/local/lib/python3.10/dist-packages (from weasel<0.5.0,>=0.1.0->spacy<3.8.0,>=3.7.2->en-core-web-sm==3.7.1) (7.0.4)\n", "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->spacy<3.8.0,>=3.7.2->en-core-web-sm==3.7.1) (2.1.5)\n", "Requirement already satisfied: marisa-trie>=0.7.7 in /usr/local/lib/python3.10/dist-packages (from language-data>=1.2->langcodes<4.0.0,>=3.2.0->spacy<3.8.0,>=3.7.2->en-core-web-sm==3.7.1) (1.2.0)\n", "Requirement already satisfied: markdown-it-py>=2.2.0 in /usr/local/lib/python3.10/dist-packages (from rich>=10.11.0->typer<1.0.0,>=0.3.0->spacy<3.8.0,>=3.7.2->en-core-web-sm==3.7.1) (3.0.0)\n", "Requirement already satisfied: pygments<3.0.0,>=2.13.0 in /usr/local/lib/python3.10/dist-packages (from rich>=10.11.0->typer<1.0.0,>=0.3.0->spacy<3.8.0,>=3.7.2->en-core-web-sm==3.7.1) (2.16.1)\n", "Requirement already satisfied: wrapt in /usr/local/lib/python3.10/dist-packages (from smart-open<8.0.0,>=5.2.1->weasel<0.5.0,>=0.1.0->spacy<3.8.0,>=3.7.2->en-core-web-sm==3.7.1) (1.16.0)\n", "Requirement already satisfied: mdurl~=0.1 in /usr/local/lib/python3.10/dist-packages (from markdown-it-py>=2.2.0->rich>=10.11.0->typer<1.0.0,>=0.3.0->spacy<3.8.0,>=3.7.2->en-core-web-sm==3.7.1) (0.1.2)\n", "\u001b[38;5;2m✔ Download and installation successful\u001b[0m\n", "You can now load the package via spacy.load('en_core_web_sm')\n", "\u001b[38;5;3m⚠ Restart to reload dependencies\u001b[0m\n", "If you are in a Jupyter or Colab notebook, you may need to restart Python in\n", "order to load all the package's dependencies. You can do this by selecting the\n", "'Restart kernel' or 'Restart runtime' option.\n" ] } ], "source": [ "!pip install bpemb\n", "!pip install gensim\n", "!python -m spacy download en_core_web_sm" ] }, { "cell_type": "code", "execution_count": 2, "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "Bt0lqVTt7Sn0", "outputId": "3f7584a6-c6ae-4cfb-baed-4b837a96c14a", "pycharm": { "name": "#%%\n" } }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "[nltk_data] Downloading package stopwords to /root/nltk_data...\n", "[nltk_data] Package stopwords is already up-to-date!\n" ] }, { "data": { "text/plain": [ "True" ] }, "execution_count": 2, "metadata": {}, "output_type": "execute_result" } ], "source": [ "import nltk\n", "nltk.download('stopwords')" ] }, { "cell_type": "markdown", "metadata": { "id": "lNjcXpu4_9CP" }, "source": [ "Here we are just using some magic commands to make sure changes to external packages are automatically loaded and plots are displayed in the notebook." ] }, { "cell_type": "code", "execution_count": 3, "metadata": { "id": "vKjNPo57FTJ8" }, "outputs": [], "source": [ "%reload_ext autoreload # comment out when using colab\n", "%autoreload 2 # comment out when using colab\n", "%matplotlib inline" ] }, { "cell_type": "markdown", "metadata": { "id": "3ut_Y8P_BLw8" }, "source": [ "# Ensuring reproducible results\n", "\n", "Before we do anything else, we need to make sure that we seed all of the random number generators and set some flags to ensure that our results are reproducible across runs i.e. that we get the same result every time we run our code." ] }, { "cell_type": "code", "execution_count": 4, "metadata": { "id": "y7aI_mECQqhS" }, "outputs": [], "source": [ "import torch\n", "import random\n", "import numpy as np" ] }, { "cell_type": "code", "execution_count": 5, "metadata": { "id": "JkmVtpi_MxEc" }, "outputs": [], "source": [ "def enforce_reproducibility(seed=42):\n", " # Sets seed manually for both CPU and CUDA\n", " torch.manual_seed(seed)\n", " torch.cuda.manual_seed_all(seed)\n", " # For atomic operations there is currently\n", " # no simple way to enforce determinism, as\n", " # the order of parallel operations is not known.\n", " # CUDNN\n", " torch.backends.cudnn.deterministic = True\n", " torch.backends.cudnn.benchmark = False\n", " # System based\n", " random.seed(seed)\n", " np.random.seed(seed)" ] }, { "cell_type": "code", "execution_count": 6, "metadata": { "id": "jQMckQKCTtpO" }, "outputs": [], "source": [ "enforce_reproducibility()" ] }, { "cell_type": "markdown", "metadata": { "id": "O9rrFIfPg3AS" }, "source": [ "# Representations in NLP\n", "\n", "- We need some way to represent our text numerically when working with ML systems\n", "- Transform our inuput text into a vector or a sequence of vectors depending on the method used\n", "- How to encode individual tokens?\n", "- One way is to have a vocabulary of all possible tokens, assign each token a number, and encode them as a one-hot vector\n", "![](https://raw.githubusercontent.com/copenlu/stat-nlp-book/master/img/sparse_binary.svg)\n", "- Couple of problems\n", " - Inefficient: defines one feature for every word in the vocabulary\n", " - All words are orthogonal to each other, so what model learns about one word doesn't apply to similar words\n", "- Solution: continuous word representations\n", " - Define each token in your vocab to be a $d$-dimensional vector\n", " - Train these vectors such that similar words have closer vectors (e.g. as measured by cosine distance)\n", " - Methods which do this are based on the \"distributional hypothesis\": similar words appear in similar contexts\n", " - E.g. word2vec trains word vectors to be able to predict the surrounding words (or vice-versa)\n", " - word2vec vectors: https://projector.tensorflow.org/\n", "- Here we'll look at how to load/use two types of word embeddings: GloVe and byte-pair encoding (BPE) embeddings\n", " - GloVe is trained on word tokens using global word co-occurence statistics ([description here](https://nlp.stanford.edu/projects/glove/#:~:text=The%20training%20objective%20of%20GloVe,'%20probability%20of%20co%2Doccurrence.&text=For%20this%20reason%2C%20the%20resulting,examined%20in%20the%20word2vec%20package.))\n", " - BPE works by iteratively building a vocabulary of size N via merging the most frequent character n-grams from a large text corpus (e.g. Wikipedia). The resulting tokens aren't necessarily words, but are _word pieces_. As a result, you no longer have a problem where a given token doesn't appear in your vocabulary -- you can always deconstruct a word into one or more word pieces. The embeddings for these word pieces are trained using GloVe in the `bpemb` package. Read more [here](https://github.com/bheinzerling/bpemb)" ] }, { "cell_type": "markdown", "metadata": { "id": "Ta5eJlXDi0vx" }, "source": [ "## GloVe embeddings\n", "\n", "- For this we can use `gensim` which has a large variety of pre-trained word embeddings\n", "- Check out their docs https://radimrehurek.com/gensim/auto_examples/index.html#documentation\n", "- We first load 100-d embeddings trained on Wikipedia and the Gigaword corpus\n", "- We can then examine what are the most similar words to some given words" ] }, { "cell_type": "code", "execution_count": 7, "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "jcl02lErhAQ0", "outputId": "dff4a399-66d5-422c-c72a-1d2283e1dcd7" }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "['fasttext-wiki-news-subwords-300', 'conceptnet-numberbatch-17-06-300', 'word2vec-ruscorpora-300', 'word2vec-google-news-300', 'glove-wiki-gigaword-50', 'glove-wiki-gigaword-100', 'glove-wiki-gigaword-200', 'glove-wiki-gigaword-300', 'glove-twitter-25', 'glove-twitter-50', 'glove-twitter-100', 'glove-twitter-200', '__testing_word2vec-matrix-synopsis']\n" ] } ], "source": [ "import gensim.downloader\n", "\n", "print(list(gensim.downloader.info()['models'].keys()))" ] }, { "cell_type": "code", "execution_count": 8, "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "tksNvGeJLoRS", "outputId": "e9eea69b-57bf-4686-e3e2-2880793a6b9b" }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Package Version\n", "-------------------------------- ---------------------\n", "absl-py 1.4.0\n", "accelerate 0.33.0\n", "aiohappyeyeballs 2.4.0\n", "aiohttp 3.10.5\n", "aiosignal 1.3.1\n", "alabaster 0.7.16\n", "albucore 0.0.14\n", "albumentations 1.4.14\n", "altair 4.2.2\n", "annotated-types 0.7.0\n", "anyio 3.7.1\n", "argon2-cffi 23.1.0\n", "argon2-cffi-bindings 21.2.0\n", "array_record 0.5.1\n", "arviz 0.18.0\n", "asn1crypto 1.5.1\n", "astropy 6.1.3\n", "astropy-iers-data 0.2024.8.27.10.28.29\n", "astunparse 1.6.3\n", "async-timeout 4.0.3\n", "atpublic 4.1.0\n", "attrs 24.2.0\n", "audioread 3.0.1\n", "autograd 1.7.0\n", "babel 2.16.0\n", "backcall 0.2.0\n", "beautifulsoup4 4.12.3\n", "bidict 0.23.1\n", "bigframes 1.15.0\n", "bigquery-magics 0.2.0\n", "bleach 6.1.0\n", "blinker 1.4\n", "blis 0.7.11\n", "blosc2 2.0.0\n", "bokeh 3.4.3\n", "bpemb 0.3.5\n", "bqplot 0.12.43\n", "branca 0.7.2\n", "build 1.2.2\n", "CacheControl 0.14.0\n", "cachetools 5.5.0\n", "catalogue 2.0.10\n", "certifi 2024.8.30\n", "cffi 1.17.0\n", "chardet 5.2.0\n", "charset-normalizer 3.3.2\n", "chex 0.1.86\n", "clarabel 0.9.0\n", "click 8.1.7\n", "click-plugins 1.1.1\n", "cligj 0.7.2\n", "cloudpathlib 0.19.0\n", "cloudpickle 2.2.1\n", "cmake 3.30.2\n", "cmdstanpy 1.2.4\n", "colorcet 3.1.0\n", "colorlover 0.3.0\n", "colour 0.1.5\n", "community 1.0.0b1\n", "confection 0.1.5\n", "cons 0.4.6\n", "contextlib2 21.6.0\n", "contourpy 1.3.0\n", "cryptography 43.0.0\n", "cuda-python 12.2.1\n", "cudf-cu12 24.4.1\n", "cufflinks 0.17.3\n", "cupy-cuda12x 12.2.0\n", "cvxopt 1.3.2\n", "cvxpy 1.5.3\n", "cycler 0.12.1\n", "cymem 2.0.8\n", "Cython 3.0.11\n", "dask 2024.7.1\n", "datascience 0.17.6\n", "db-dtypes 1.3.0\n", "dbus-python 1.2.18\n", "debugpy 1.6.6\n", "decorator 4.4.2\n", "defusedxml 0.7.1\n", "distributed 2024.7.1\n", "distro 1.7.0\n", "dlib 19.24.2\n", "dm-tree 0.1.8\n", "docstring_parser 0.16\n", "docutils 0.18.1\n", "dopamine_rl 4.0.9\n", "duckdb 0.10.3\n", "earthengine-api 0.1.418\n", "easydict 1.13\n", "ecos 2.0.14\n", "editdistance 0.8.1\n", "eerepr 0.0.4\n", "einops 0.8.0\n", "en-core-web-sm 3.7.1\n", "entrypoints 0.4\n", "et-xmlfile 1.1.0\n", "etils 1.7.0\n", "etuples 0.3.9\n", "eval_type_backport 0.2.0\n", "exceptiongroup 1.2.2\n", "fastai 2.7.17\n", "fastcore 1.7.1\n", "fastdownload 0.0.7\n", "fastjsonschema 2.20.0\n", "fastprogress 1.0.3\n", "fastrlock 0.8.2\n", "filelock 3.15.4\n", "fiona 1.9.6\n", "firebase-admin 6.5.0\n", "Flask 2.2.5\n", "flatbuffers 24.3.25\n", "flax 0.8.4\n", "folium 0.17.0\n", "fonttools 4.53.1\n", "frozendict 2.4.4\n", "frozenlist 1.4.1\n", "fsspec 2024.6.1\n", "future 1.0.0\n", "gast 0.6.0\n", "gcsfs 2024.6.1\n", "GDAL 3.6.4\n", "gdown 5.1.0\n", "geemap 0.34.0\n", "gensim 4.3.3\n", "geocoder 1.38.1\n", "geographiclib 2.0\n", "geopandas 0.14.4\n", "geopy 2.4.1\n", "gin-config 0.5.0\n", "glob2 0.7\n", "google 2.0.3\n", "google-ai-generativelanguage 0.6.6\n", "google-api-core 2.19.2\n", "google-api-python-client 2.137.0\n", "google-auth 2.27.0\n", "google-auth-httplib2 0.2.0\n", "google-auth-oauthlib 1.2.1\n", "google-cloud-aiplatform 1.64.0\n", "google-cloud-bigquery 3.25.0\n", "google-cloud-bigquery-connection 1.15.5\n", "google-cloud-bigquery-storage 2.25.0\n", "google-cloud-bigtable 2.26.0\n", "google-cloud-core 2.4.1\n", "google-cloud-datastore 2.19.0\n", "google-cloud-firestore 2.16.1\n", "google-cloud-functions 1.16.5\n", "google-cloud-iam 2.15.2\n", "google-cloud-language 2.13.4\n", "google-cloud-pubsub 2.23.0\n", "google-cloud-resource-manager 1.12.5\n", "google-cloud-storage 2.8.0\n", "google-cloud-translate 3.15.5\n", "google-colab 1.0.0\n", "google-crc32c 1.5.0\n", "google-generativeai 0.7.2\n", "google-pasta 0.2.0\n", "google-resumable-media 2.7.2\n", "googleapis-common-protos 1.65.0\n", "googledrivedownloader 0.4\n", "graphviz 0.20.3\n", "greenlet 3.0.3\n", "grpc-google-iam-v1 0.13.1\n", "grpcio 1.64.1\n", "grpcio-status 1.48.2\n", "gspread 6.0.2\n", "gspread-dataframe 3.3.1\n", "gym 0.25.2\n", "gym-notices 0.0.8\n", "h5netcdf 1.3.0\n", "h5py 3.11.0\n", "holidays 0.55\n", "holoviews 1.18.3\n", "html5lib 1.1\n", "httpimport 1.3.1\n", "httplib2 0.22.0\n", "huggingface-hub 0.24.6\n", "humanize 4.10.0\n", "hyperopt 0.2.7\n", "ibis-framework 8.0.0\n", "idna 3.8\n", "imageio 2.34.2\n", "imageio-ffmpeg 0.5.1\n", "imagesize 1.4.1\n", "imbalanced-learn 0.12.3\n", "imgaug 0.4.0\n", "immutabledict 4.2.0\n", "importlib_metadata 8.4.0\n", "importlib_resources 6.4.4\n", "imutils 0.5.4\n", "inflect 7.3.1\n", "iniconfig 2.0.0\n", "intel-cmplr-lib-ur 2024.2.1\n", "intel-openmp 2024.2.1\n", "ipyevents 2.0.2\n", "ipyfilechooser 0.6.0\n", "ipykernel 5.5.6\n", "ipyleaflet 0.18.2\n", "ipyparallel 8.8.0\n", "ipython 7.34.0\n", "ipython-genutils 0.2.0\n", "ipython-sql 0.5.0\n", "ipytree 0.2.2\n", "ipywidgets 7.7.1\n", "itsdangerous 2.2.0\n", "jax 0.4.26\n", "jaxlib 0.4.26+cuda12.cudnn89\n", "jeepney 0.7.1\n", "jellyfish 1.1.0\n", "jieba 0.42.1\n", "Jinja2 3.1.4\n", "joblib 1.4.2\n", "jsonpickle 3.2.2\n", "jsonschema 4.23.0\n", "jsonschema-specifications 2023.12.1\n", "jupyter-client 6.1.12\n", "jupyter-console 6.1.0\n", "jupyter_core 5.7.2\n", "jupyter-server 1.24.0\n", "jupyterlab_pygments 0.3.0\n", "jupyterlab_widgets 3.0.13\n", "kaggle 1.6.17\n", "kagglehub 0.2.9\n", "keras 3.4.1\n", "keyring 23.5.0\n", "kiwisolver 1.4.5\n", "langcodes 3.4.0\n", "language_data 1.2.0\n", "launchpadlib 1.10.16\n", "lazr.restfulclient 0.14.4\n", "lazr.uri 1.0.6\n", "lazy_loader 0.4\n", "libclang 18.1.1\n", "librosa 0.10.2.post1\n", "lightgbm 4.4.0\n", "linkify-it-py 2.0.3\n", "llvmlite 0.43.0\n", "locket 1.0.0\n", "logical-unification 0.4.6\n", "lxml 4.9.4\n", "malloy 2024.1091\n", "marisa-trie 1.2.0\n", "Markdown 3.7\n", "markdown-it-py 3.0.0\n", "MarkupSafe 2.1.5\n", "matplotlib 3.7.1\n", "matplotlib-inline 0.1.7\n", "matplotlib-venn 0.11.10\n", "mdit-py-plugins 0.4.1\n", "mdurl 0.1.2\n", "miniKanren 1.0.3\n", "missingno 0.5.2\n", "mistune 0.8.4\n", "mizani 0.9.3\n", "mkl 2024.2.1\n", "ml-dtypes 0.4.0\n", "mlxtend 0.23.1\n", "more-itertools 10.3.0\n", "moviepy 1.0.3\n", "mpmath 1.3.0\n", "msgpack 1.0.8\n", "multidict 6.0.5\n", "multipledispatch 1.0.0\n", "multitasking 0.0.11\n", "murmurhash 1.0.10\n", "music21 9.1.0\n", "namex 0.0.8\n", "natsort 8.4.0\n", "nbclassic 1.1.0\n", "nbclient 0.10.0\n", "nbconvert 6.5.4\n", "nbformat 5.10.4\n", "nest-asyncio 1.6.0\n", "networkx 3.3\n", "nibabel 5.0.1\n", "nltk 3.8.1\n", "notebook 6.5.5\n", "notebook_shim 0.2.4\n", "numba 0.60.0\n", "numexpr 2.10.1\n", "numpy 1.26.4\n", "nvidia-nccl-cu12 2.22.3\n", "nvtx 0.2.10\n", "oauth2client 4.1.3\n", "oauthlib 3.2.2\n", "opencv-contrib-python 4.10.0.84\n", "opencv-python 4.10.0.84\n", "opencv-python-headless 4.10.0.84\n", "openpyxl 3.1.5\n", "opt-einsum 3.3.0\n", "optax 0.2.2\n", "optree 0.12.1\n", "orbax-checkpoint 0.6.1\n", "osqp 0.6.7.post0\n", "packaging 24.1\n", "pandas 2.1.4\n", "pandas-datareader 0.10.0\n", "pandas-gbq 0.23.1\n", "pandas-stubs 2.1.4.231227\n", "pandocfilters 1.5.1\n", "panel 1.4.5\n", "param 2.1.1\n", "parso 0.8.4\n", "parsy 2.1\n", "partd 1.4.2\n", "pathlib 1.0.1\n", "patsy 0.5.6\n", "peewee 3.17.6\n", "pexpect 4.9.0\n", "pickleshare 0.7.5\n", "Pillow 9.4.0\n", "pip 24.1.2\n", "pip-tools 7.4.1\n", "platformdirs 4.2.2\n", "plotly 5.15.0\n", "plotnine 0.12.4\n", "pluggy 1.5.0\n", "polars 0.20.2\n", "pooch 1.8.2\n", "portpicker 1.5.2\n", "prefetch_generator 1.0.3\n", "preshed 3.0.9\n", "prettytable 3.11.0\n", "proglog 0.1.10\n", "progressbar2 4.2.0\n", "prometheus_client 0.20.0\n", "promise 2.3\n", "prompt_toolkit 3.0.47\n", "prophet 1.1.5\n", "proto-plus 1.24.0\n", "protobuf 3.20.3\n", "psutil 5.9.5\n", "psycopg2 2.9.9\n", "ptyprocess 0.7.0\n", "py-cpuinfo 9.0.0\n", "py4j 0.10.9.7\n", "pyarrow 14.0.2\n", "pyarrow-hotfix 0.6\n", "pyasn1 0.6.0\n", "pyasn1_modules 0.4.0\n", "pycocotools 2.0.8\n", "pycparser 2.22\n", "pydantic 2.8.2\n", "pydantic_core 2.20.1\n", "pydata-google-auth 1.8.2\n", "pydot 1.4.2\n", "pydot-ng 2.0.0\n", "pydotplus 2.0.2\n", "PyDrive 1.3.1\n", "PyDrive2 1.6.3\n", "pyerfa 2.0.1.4\n", "pygame 2.6.0\n", "Pygments 2.16.1\n", "PyGObject 3.42.1\n", "PyJWT 2.9.0\n", "pymc 5.10.4\n", "pymystem3 0.2.0\n", "pynvjitlink-cu12 0.3.0\n", "PyOpenGL 3.1.7\n", "pyOpenSSL 24.2.1\n", "pyparsing 3.1.4\n", "pyperclip 1.9.0\n", "pyproj 3.6.1\n", "pyproject_hooks 1.1.0\n", "pyshp 2.3.1\n", "PySocks 1.7.1\n", "pytensor 2.18.6\n", "pytest 7.4.4\n", "python-apt 2.4.0\n", "python-box 7.2.0\n", "python-dateutil 2.8.2\n", "python-louvain 0.16\n", "python-slugify 8.0.4\n", "python-utils 3.8.2\n", "pytz 2024.1\n", "pyviz_comms 3.0.3\n", "PyYAML 6.0.2\n", "pyzmq 24.0.1\n", "qdldl 0.1.7.post4\n", "ratelim 0.1.6\n", "referencing 0.35.1\n", "regex 2024.5.15\n", "requests 2.32.3\n", "requests-oauthlib 1.3.1\n", "requirements-parser 0.9.0\n", "rich 13.8.0\n", "rmm-cu12 24.4.0\n", "rpds-py 0.20.0\n", "rpy2 3.4.2\n", "rsa 4.9\n", "safetensors 0.4.4\n", "scikit-image 0.23.2\n", "scikit-learn 1.3.2\n", "scipy 1.13.1\n", "scooby 0.10.0\n", "scs 3.2.7\n", "seaborn 0.13.1\n", "SecretStorage 3.3.1\n", "Send2Trash 1.8.3\n", "sentencepiece 0.1.99\n", "setuptools 71.0.4\n", "shapely 2.0.6\n", "shellingham 1.5.4\n", "simple_parsing 0.1.5\n", "six 1.16.0\n", "sklearn-pandas 2.2.0\n", "smart-open 7.0.4\n", "sniffio 1.3.1\n", "snowballstemmer 2.2.0\n", "snowflake-connector-python 3.12.1\n", "sortedcontainers 2.4.0\n", "soundfile 0.12.1\n", "soupsieve 2.6\n", "soxr 0.5.0\n", "spacy 3.7.6\n", "spacy-legacy 3.0.12\n", "spacy-loggers 1.0.5\n", "Sphinx 5.0.2\n", "sphinxcontrib-applehelp 2.0.0\n", "sphinxcontrib-devhelp 2.0.0\n", "sphinxcontrib-htmlhelp 2.1.0\n", "sphinxcontrib-jsmath 1.0.1\n", "sphinxcontrib-qthelp 2.0.0\n", "sphinxcontrib-serializinghtml 2.0.0\n", "SQLAlchemy 2.0.32\n", "sqlglot 20.11.0\n", "sqlparse 0.5.1\n", "srsly 2.4.8\n", "stanio 0.5.1\n", "statsmodels 0.14.2\n", "StrEnum 0.4.15\n", "sympy 1.13.2\n", "tables 3.8.0\n", "tabulate 0.9.0\n", "tbb 2021.13.1\n", "tblib 3.0.0\n", "tenacity 9.0.0\n", "tensorboard 2.17.0\n", "tensorboard-data-server 0.7.2\n", "tensorflow 2.17.0\n", "tensorflow-datasets 4.9.6\n", "tensorflow-hub 0.16.1\n", "tensorflow-io-gcs-filesystem 0.37.1\n", "tensorflow-metadata 1.15.0\n", "tensorflow-probability 0.24.0\n", "tensorstore 0.1.64\n", "termcolor 2.4.0\n", "terminado 0.18.1\n", "text-unidecode 1.3\n", "textblob 0.17.1\n", "tf_keras 2.17.0\n", "tf-slim 1.1.0\n", "thinc 8.2.5\n", "threadpoolctl 3.5.0\n", "tifffile 2024.8.28\n", "tinycss2 1.3.0\n", "tokenizers 0.19.1\n", "toml 0.10.2\n", "tomli 2.0.1\n", "tomlkit 0.13.2\n", "toolz 0.12.1\n", "torch 2.4.0+cu121\n", "torchaudio 2.4.0+cu121\n", "torchsummary 1.5.1\n", "torchvision 0.19.0+cu121\n", "tornado 6.3.3\n", "tqdm 4.66.5\n", "traitlets 5.7.1\n", "traittypes 0.2.1\n", "transformers 4.44.2\n", "tweepy 4.14.0\n", "typeguard 4.3.0\n", "typer 0.12.5\n", "types-pytz 2024.1.0.20240417\n", "types-setuptools 74.1.0.20240907\n", "typing_extensions 4.12.2\n", "tzdata 2024.1\n", "tzlocal 5.2\n", "uc-micro-py 1.0.3\n", "uritemplate 4.1.1\n", "urllib3 2.0.7\n", "vega-datasets 0.9.0\n", "wadllib 1.3.6\n", "wasabi 1.1.3\n", "wcwidth 0.2.13\n", "weasel 0.4.1\n", "webcolors 24.8.0\n", "webencodings 0.5.1\n", "websocket-client 1.8.0\n", "Werkzeug 3.0.4\n", "wheel 0.44.0\n", "widgetsnbextension 3.6.8\n", "wordcloud 1.9.3\n", "wrapt 1.16.0\n", "xarray 2024.6.0\n", "xarray-einstats 0.7.0\n", "xgboost 2.1.1\n", "xlrd 2.0.1\n", "xyzservices 2024.6.0\n", "yarl 1.9.4\n", "yellowbrick 1.5\n", "yfinance 0.2.43\n", "zict 3.0.0\n", "zipp 3.20.1\n" ] } ], "source": [ "!pip list\n" ] }, { "cell_type": "code", "execution_count": 9, "metadata": { "id": "1gaVG9CDhspf" }, "outputs": [], "source": [ "# Load the vectors\n", "\n", "glove_vectors = gensim.downloader.load('glove-wiki-gigaword-100')" ] }, { "cell_type": "code", "execution_count": 10, "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "CYtrchThipk6", "outputId": "dd40b978-8e3b-4b35-e20c-8ff726e06e58" }, "outputs": [ { "data": { "text/plain": [ "[('greatest', 0.7882647514343262),\n", " ('good', 0.7592797875404358),\n", " ('little', 0.7585746049880981),\n", " ('much', 0.7477047443389893),\n", " ('well', 0.7401014566421509),\n", " ('big', 0.7318841218948364),\n", " ('kind', 0.730876624584198),\n", " ('important', 0.7286974191665649),\n", " ('there', 0.7225556373596191),\n", " ('way', 0.7116187810897827)]" ] }, "execution_count": 10, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# Look at some of the most similar words\n", "glove_vectors.most_similar('great')" ] }, { "cell_type": "code", "execution_count": 11, "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "M-MIkvTUqgyD", "outputId": "2a7a4f5b-f823-44e1-8b58-098e201796e5" }, "outputs": [ { "data": { "text/plain": [ "array([-0.013786 , 0.38216 , 0.53236 , 0.15261 , -0.29694 ,\n", " -0.20558 , -0.41846 , -0.58437 , -0.77355 , -0.87866 ,\n", " -0.37858 , -0.18516 , -0.128 , -0.20584 , -0.22925 ,\n", " -0.42599 , 0.3725 , 0.26077 , -1.0702 , 0.62916 ,\n", " -0.091469 , 0.70348 , -0.4973 , -0.77691 , 0.66045 ,\n", " 0.09465 , -0.44893 , 0.018917 , 0.33146 , -0.35022 ,\n", " -0.35789 , 0.030313 , 0.22253 , -0.23236 , -0.19719 ,\n", " -0.0053125, -0.25848 , 0.58081 , -0.10705 , -0.17845 ,\n", " -0.16206 , 0.087086 , 0.63029 , -0.76649 , 0.51619 ,\n", " 0.14073 , 1.019 , -0.43136 , 0.46138 , -0.43585 ,\n", " -0.47568 , 0.19226 , 0.36065 , 0.78987 , 0.088945 ,\n", " -2.7814 , -0.15366 , 0.01015 , 1.1798 , 0.15168 ,\n", " -0.050112 , 1.2626 , -0.77527 , 0.36031 , 0.95761 ,\n", " -0.11385 , 0.28035 , -0.02591 , 0.31246 , -0.15424 ,\n", " 0.3778 , -0.13599 , 0.2946 , -0.31579 , 0.42943 ,\n", " 0.086969 , 0.019169 , -0.27242 , -0.31696 , 0.37327 ,\n", " 0.61997 , 0.13889 , 0.17188 , 0.30363 , -1.2776 ,\n", " 0.044423 , -0.52736 , -0.88536 , -0.19428 , -0.61947 ,\n", " -0.10146 , -0.26301 , -0.061707 , 0.36627 , -0.95223 ,\n", " -0.39346 , -0.69183 , -1.0426 , 0.28855 , 0.63056 ],\n", " dtype=float32)" ] }, "execution_count": 11, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# Get vector representation of word\n", "glove_vectors['great']" ] }, { "cell_type": "code", "execution_count": 12, "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "HTsS5hx5i739", "outputId": "571b7734-f521-4855-8ce1-0256f6d39b7d" }, "outputs": [ { "data": { "text/plain": [ "[('worse', 0.7929712533950806),\n", " ('good', 0.7702797651290894),\n", " ('things', 0.7653602957725525),\n", " ('too', 0.7630148530006409),\n", " ('thing', 0.7609668374061584),\n", " ('lot', 0.7443646788597107),\n", " ('kind', 0.7408681511878967),\n", " ('because', 0.7398799061775208),\n", " ('really', 0.7376540899276733),\n", " (\"n't\", 0.7336540818214417)]" ] }, "execution_count": 12, "metadata": {}, "output_type": "execute_result" } ], "source": [ "glove_vectors.most_similar('bad')" ] }, { "cell_type": "markdown", "metadata": { "id": "PLs-HGKPi41P" }, "source": [ "## BPEmb embeddings\n", "\n", "- Here we use `bpemb`which has pretrained BPE tokenizers/embeddings for 275 languages\n", "- https://github.com/bheinzerling/bpemb\n", "- First we load the English model with 25,000 word pieces and 100-dimensions\n", "- The package has some similar functionality built in to `gensim`, for example observing the top similar words to a given word" ] }, { "cell_type": "code", "execution_count": 13, "metadata": { "id": "byruqtK7jIms" }, "outputs": [], "source": [ "from bpemb import BPEmb\n", "\n", "# Load english model with 25k word-pieces\n", "bpemb_en = BPEmb(lang='en', dim=100, vs=25000)" ] }, { "cell_type": "code", "execution_count": 14, "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "S3g2K36pjLVq", "outputId": "c8975685-50b5-41fe-c4c1-ce38ff4c8847" }, "outputs": [ { "data": { "text/plain": [ "[('little', 0.5838008522987366),\n", " ('▁great', 0.5713018774986267),\n", " ('grand', 0.4832310974597931),\n", " ('the', 0.47381392121315),\n", " ('▁glorious', 0.4674184322357178),\n", " ('bear', 0.44929906725883484),\n", " ('wh', 0.4481717348098755),\n", " ('mother', 0.44679757952690125),\n", " ('▁splend', 0.44266071915626526),\n", " ('southern', 0.427338182926178)]" ] }, "execution_count": 14, "metadata": {}, "output_type": "execute_result" } ], "source": [ "bpemb_en.most_similar('great')" ] }, { "cell_type": "code", "execution_count": 15, "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "s4yyZCTHqtcy", "outputId": "643b5649-dc23-41a0-8422-c5214689b429" }, "outputs": [ { "data": { "text/plain": [ "array([ 0.188451, -0.271453, -0.28907 , 0.061539, -0.00119 , 0.023874,\n", " -0.067886, -0.331542, 0.216387, 0.357444, -0.37975 , 0.568398,\n", " 0.094106, 0.131808, -0.225529, -0.0577 , -0.471579, 0.270219,\n", " 0.500174, -0.351869, 0.249958, -0.083147, -0.196093, 0.536826,\n", " -0.472552, -0.290486, 0.044731, 0.291949, -0.02748 , 0.833167,\n", " -0.702959, 0.102262, -0.192826, 0.323853, -0.222146, -0.471987,\n", " -0.789 , -0.384075, 0.231821, 0.479966, -0.23674 , -0.233447,\n", " -0.095288, 0.994725, 0.38566 , 0.539499, -0.117075, -0.424706,\n", " -0.35617 , -0.073661, 0.185955, -0.329411, -0.336546, -0.883611,\n", " 0.450815, -0.087026, 0.547418, -0.177824, -0.50549 , 0.423664,\n", " 0.15182 , 0.289324, 0.546515, -0.963773, 0.041765, 0.526582,\n", " -0.336767, 0.128516, 0.307212, 0.049208, -0.314852, 0.343516,\n", " 0.09675 , 0.033849, -0.260953, 0.340189, -0.114852, 0.128735,\n", " 0.403689, -0.154596, 0.303449, 0.855851, 0.089396, 0.229577,\n", " 0.022571, -0.194975, 0.080064, 0.273517, 0.394372, -0.196653,\n", " 0.220832, -0.052137, -0.274584, 0.376419, 0.151644, -0.01065 ,\n", " 0.195909, -0.456158, 0.042357, 0.058726], dtype=float32)" ] }, "execution_count": 15, "metadata": {}, "output_type": "execute_result" } ], "source": [ "bpemb_en['great']" ] }, { "cell_type": "code", "execution_count": 16, "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "-XIFDVmrjbDW", "outputId": "fa9c544a-f13b-49d6-abab-3d226031fcce" }, "outputs": [ { "data": { "text/plain": [ "[('▁bad', 0.7139851450920105),\n", " ('bir', 0.5771084427833557),\n", " ('her', 0.5590746402740479),\n", " ('hol', 0.538629412651062),\n", " ('mad', 0.5319529175758362),\n", " ('wal', 0.5289992690086365),\n", " ('wolf', 0.5198653340339661),\n", " ('hal', 0.5146304368972778),\n", " ('▁luck', 0.5110129117965698),\n", " ('ün', 0.506809651851654)]" ] }, "execution_count": 16, "metadata": {}, "output_type": "execute_result" } ], "source": [ "bpemb_en.most_similar('bad')" ] }, { "cell_type": "markdown", "metadata": { "id": "p13nASL6mnsm" }, "source": [ "# Sentiment classification of movie reviews\n", "\n", "We'll see how to train a basic logistic regression classifier for sentiment analysis. For this, we're using the [IMDB movie reviews dataset](https://www.kaggle.com/datasets/ducanger/imdb-dataset). " ] }, { "cell_type": "code", "execution_count": 17, "metadata": { "id": "z8lzv8D7oz6V" }, "outputs": [], "source": [ "import spacy\n", "import pandas as pd\n", "from sklearn.linear_model import LogisticRegression\n", "from sklearn.feature_extraction.text import CountVectorizer\n", "from sklearn.feature_extraction.text import TfidfVectorizer\n", "from sklearn.model_selection import RandomizedSearchCV\n", "from sklearn.metrics import classification_report" ] }, { "cell_type": "code", "execution_count": 18, "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 198 }, "id": "f0z6eqUMmq0l", "outputId": "8d287d37-108f-4e0b-f3ef-650cc34cc8a2" }, "outputs": [ { "data": { "text/html": [ "\n", " \n", " \n", " Upload widget is only available when the cell has been executed in the\n", " current browser session. Please rerun this cell to enable.\n", " \n", " " ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "Saving val.csv to val.csv\n", "Saving train.csv to train.csv\n", "Saving test.csv to test.csv\n", "User uploaded file \"val.csv\" with length 6672419 bytes\n", "User uploaded file \"train.csv\" with length 46207436 bytes\n", "User uploaded file \"test.csv\" with length 13271381 bytes\n" ] } ], "source": [ "# Upload the data from local computer\n", "from google.colab import files\n", "\n", "uploaded = files.upload()\n", "\n", "for fn in uploaded.keys():\n", " print('User uploaded file \"{name}\" with length {length} bytes'.format(\n", " name=fn, length=len(uploaded[fn])))" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "gTuuSYH88HFz" }, "outputs": [], "source": [ "# or you can load from google drive if you upload the files there (faster in my experience)\n", "from google.colab import drive\n", "drive.mount('/content/drive') # this will trigger permission prompts\n" ] }, { "cell_type": "code", "execution_count": 19, "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 206 }, "id": "cfxyV0umowmM", "outputId": "98685015-de31-4e93-bf9b-32c8592ab945" }, "outputs": [ { "data": { "application/vnd.google.colaboratory.intrinsic+json": { "summary": "{\n \"name\": \"valid_data\",\n \"rows\": 5000,\n \"fields\": [\n {\n \"column\": \"review\",\n \"properties\": {\n \"dtype\": \"string\",\n \"num_unique_values\": 4999,\n \"samples\": [\n \"This was made in 2004 for gods sake, what happened to our state of the art special effects? What happened to our rough around the edges but still good actors? The actors in this movie were unbelievably horrible, there was one or two that weren't bad, but the rest, biggg thumbs down. Couldn't stand listening to the badly written dialogue, I mean, who the heck wrote that script? Please don't ever write again! Special effects? Don't even get me started on the special effects. SURELY they could have come up with better then fully fake looking green balls of light in the eye sockets. It looks so old and..lame frankly.! Even the easiest thing to make look real..the teeth, THEY looked so fake and stupid I would almost wipe a tear from my eye in annoyance. Come onnnn I cant believe this was even shown to the public.!\",\n \"all i can say about this film is to read the back of the video case and then put it back on he shelf and pick anything else, i mean anything, a blank video, would be better than watching this.\",\n \"In the end credits of \\\"Shadows\\\", after we read 'directed by John Cassavetes', some white letters on the screen can be seen: \\\"The film you have just seen is improvised\\\", they say. I am always pursuing the fact that words are so important in movies since filmmakers started using them because, basically, there's no film without a screenplay and many other reasons.

Cassavetes pursued the same goal, and he believed in the freedom of words; \\\"Shadows\\\" is the perfect example. It's a film with no real main characters, with no real main plot lines; it's mostly people in different situations, talking. Yes, some of the situations are connected but Cassavetes, apparently always in a rush to get to the talking, uses a fast forward technique when the characters are going somewhere or escaping from someone and are not speaking.

Appearances are everything in this movie. For example, there's a brilliant score, full of jazz influences and a lot of fantastic solos, and there's one character that says he's a jazz musician and plays the trumpet (Ben, all the characters' names are the same names the actors'). However, we never see him play the trumpet or jam with a band; he doesn't even talk about music and just wanders with his friends around the city. They do talk, a lot, and about anything that's in their minds; going from how intelligent each of them are to the hilarious analysis of a sculpture.

\\\"Shadows\\\" is funny in its intellectual references in parts like the one above, because these friends are not cultured. The only important female character in the film (Lelia), though, wants to be an intellectual. But again, she has one very interesting conversation with an older man at a party, about a book she's trying to write, and about how to confront reality; but nothing to do with being intellectual. At that same party, a woman is actually making an intellectual statement, full of complexity, and asks a guy beside her: \\\"Do you agree?\\\". \\\"Yes\\\", he says, but you can tell he doesn't know what she's talking about.

Another character, a singer (Hugh), talks about his glory days in occasions, and we see him perform only once; but no references to the musical industry there. The focus of Cassavetes is the singer's relationship with his manager (Rupert), which most of the time involves chats about trivial stuff and not real 'musical' talks. So the trumpet player's important deal in \\\"Shadows\\\" is the time he spends with his friends; the intellectual wannabe girl's is her way of handling romantic relationships (one of the movie's strong points) and the singer's is the bond with his manager\\u0085Appearances.

The reason why performances are not important in this movie is simple. Cassavetes needed people who could master improvisation, without mattering if they were actually good. I believe some of them aren't, but they surely know how to improvise in a scene, and you can notice how well they do it. \\\"Shadows\\\" is not about performers; it's about a way of making cinema, based on the magic of conversation; and there you could say that performances mean something.

That's why in every conversation the camera is like a stalker, constantly on the eyes of every character, constantly looking for the expressions that come with natural speech. There's a scene where the trumpet player and his friends are trying to pick up some girls. They are three, so each of them sits beside one girl (the girls are three two) in three different tables. They all talk at the same time and the camera shoots through the table, and sometimes the friends look at each other, while they say whatever they are saying\\u0085It's natural.\"\n ],\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n },\n {\n \"column\": \"sentiment\",\n \"properties\": {\n \"dtype\": \"number\",\n \"std\": 0,\n \"min\": 0,\n \"max\": 1,\n \"num_unique_values\": 2,\n \"samples\": [\n 1,\n 0\n ],\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n }\n ]\n}", "type": "dataframe", "variable_name": "valid_data" }, "text/html": [ "\n", "
\n", "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
reviewsentiment
0A genuinely odd, surreal jumble of visual idea...0
1\"The Snow Queen\" is based on the famous and ve...0
2The quintessential Georgian film of Georgi Dan...1
3I'm a huge comedy show fan. Racial humor is al...0
4Pretty good film from Preminger; labyrinthine ...1
\n", "
\n", "
\n", "\n", "
\n", " \n", "\n", " \n", "\n", " \n", "
\n", "\n", "\n", "
\n", " \n", "\n", "\n", "\n", " \n", "
\n", "\n", "
\n", "
\n" ], "text/plain": [ " review sentiment\n", "0 A genuinely odd, surreal jumble of visual idea... 0\n", "1 \"The Snow Queen\" is based on the famous and ve... 0\n", "2 The quintessential Georgian film of Georgi Dan... 1\n", "3 I'm a huge comedy show fan. Racial humor is al... 0\n", "4 Pretty good film from Preminger; labyrinthine ... 1" ] }, "execution_count": 19, "metadata": {}, "output_type": "execute_result" } ], "source": [ "train_data = pd.read_csv('./train.csv', usecols=[1,2]).fillna('')\n", "valid_data = pd.read_csv('.//val.csv', usecols=[1,2]).fillna('')\n", "test_data = pd.read_csv('./test.csv', usecols=[1,2]).fillna('')\n", "valid_data.head() # in the sentiment column, 1 is positive, 0 is negative" ] }, { "cell_type": "code", "execution_count": 20, "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "2joLhlgU6CPe", "outputId": "52a61e5b-f946-4c52-eb4e-69392e8e33e1" }, "outputs": [ { "data": { "text/plain": [ "array([\"Having avoided seeing the movie in the cinema, but buying the DVD for my wife for Xmas, I had to watch it. I did not expect much, which usually means I get more than I bargained for. But 'Mamma Mia' - utter, utter cr**. I like ABBA, I like the songs, I have the old LPs. But this film is just terrible. The stage show looks like a bit of a musical, but this races along with songs hurriedly following one another, no characterisation, the dance numbers (which were heavily choreographed according to the extras on the DVD) are just thrown away with only half the bodies ever on screen, the dance chorus of north Europeans appear on a small Greek island at will, while the set and set up of numbers would have disgraced Cliff Richard's musicals in the sixties!Meryl (see me I'm acting)Streep can't even make her usual mugging effective in an over-the-top musical! Her grand piece - 'The Winner Takes It All' - is Meryl at the Met! Note to director - it should have been shot in stillness with the camera gradually showing distance growing between Streep and Brosnan! Some of the singing is awful karaoke on amateur night. The camera cannot stop moving like bad MTV. One can never settle down and just enjoy the music, enthusiasm and characters. But what is even worse is how this botched piece of excre**** has become the highest grossing film in the UK and the best selling DVD to boot? Blair, Campbell and New Labour really have reduced the UK to zombies - critical faculties anyone???\",\n", " 0], dtype=object)" ] }, "execution_count": 20, "metadata": {}, "output_type": "execute_result" } ], "source": [ "train_data.values[0]" ] }, { "cell_type": "markdown", "metadata": { "id": "eDSgEyswPPG2" }, "source": [ "This defines some functions for vectorizing our text. For GloVe and BPE embeddings, this involves:\n", "\n", "1. Tokenizing the text\n", "2. Looking up the embedding for each token\n", "3. Pooling the representations to obtain a single vector for the whole document\n", "\n", "There are many options for pooling e.g. max-pooling, average-pooling, summation, etc." ] }, { "cell_type": "code", "execution_count": 21, "metadata": { "id": "p5REmUCZpNJU" }, "outputs": [], "source": [ "from tqdm import tqdm\n", "import nltk\n", "from nltk.corpus import stopwords\n", "stopwords = set(stopwords.words('english'))\n", "\n", "\n", "def get_unigram_features(dataset, vectorizer):\n", " X = vectorizer.transform(dataset[:,0])\n", " y = list(dataset[:,1])\n", " return X,y\n", "\n", "def get_bpemb_features(dataset, bpemb):\n", " # With bpemb we can tokenize and embed an entire document using .embed(x)\n", " X = [bpemb.embed(x).mean(0) for x in tqdm(dataset[:,0])]\n", " y = list(dataset[:,1])\n", " return X,y\n", "\n", "def get_glove_features(dataset, glove, nlp):\n", " X = []\n", " for x in tqdm(dataset[:,0]):\n", " # For glove embeddings, we first tokenize the sentence using spacy, lower-case and remove stopwords, and get the available word vectors\n", " vecs = np.vstack([glove[t.text.lower()] if t.text.lower() in glove else np.zeros(100) for t in nlp(x) if t.text.lower() not in stopwords])\n", " X.append(vecs.mean(0))\n", " y = list(dataset[:,1])\n", " return np.stack(X),y" ] }, { "cell_type": "markdown", "metadata": { "id": "GjI-auvMP0ES" }, "source": [ "This defines how to run a basic logistic regression classifier. Commented out are some code for if you want to run a hyperparameter search." ] }, { "cell_type": "code", "execution_count": 22, "metadata": { "id": "Xjw2KFxnp9zW" }, "outputs": [], "source": [ "def run_classifier(X_train, y_train, X_test, y_test):\n", " # Define hyperparams for search\n", " # C = np.logspace(-6, 2, 50)\n", " # warm_start = [False, True]\n", " # class_weight = ['balanced', None]\n", "\n", " # hp = {\"C\": C, \"warm_start\": warm_start, 'class_weight': class_weight}\n", "\n", " classifier = LogisticRegression(penalty='l2', max_iter=1000)\n", " # classifier_random = RandomizedSearchCV(\n", " # estimator=classifier,\n", " # param_distributions=hp,\n", " # n_iter=100,\n", " # cv=5,\n", " # verbose=2,\n", " # random_state=1000,\n", " # n_jobs=-1,\n", " # scoring='f1'\n", " # )\n", "\n", " # classifier_random.fit(X_train, y_train)\n", " # print(classifier_random.best_params_)\n", " # print(classifier_random.best_score_)\n", " # model = classifier_random.best_estimator_\n", " classifier.fit(X_train,y_train)\n", " preds = classifier.predict(X_test)\n", " print(classification_report(y_test, preds))" ] }, { "cell_type": "markdown", "metadata": { "id": "q8Vu2FpFP9UI" }, "source": [ "First we'll run GloVe" ] }, { "cell_type": "code", "execution_count": 23, "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "-oGRK6sUuYON", "outputId": "322591a3-2411-4b22-db9c-07e49d69fd06" }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "100%|██████████| 5000/5000 [03:35<00:00, 23.16it/s]\n", "100%|██████████| 1000/1000 [00:42<00:00, 23.47it/s]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ " precision recall f1-score support\n", "\n", " 0 0.82 0.79 0.80 524\n", " 1 0.78 0.80 0.79 476\n", "\n", " accuracy 0.80 1000\n", " macro avg 0.80 0.80 0.80 1000\n", "weighted avg 0.80 0.80 0.80 1000\n", "\n" ] } ], "source": [ "#only train and test on part of the dataset as an example\n", "nlp = spacy.load('en_core_web_sm')\n", "X_train,y_train = get_glove_features(train_data.values[:5000], glove_vectors, nlp)\n", "X_test,y_test = get_glove_features(test_data.values[:1000], glove_vectors, nlp)\n", "run_classifier(X_train, y_train, X_test, y_test)" ] }, { "cell_type": "markdown", "metadata": { "id": "3a4105HNQAGz" }, "source": [ "Next BPE" ] }, { "cell_type": "code", "execution_count": 24, "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "srVQR_-6yDQJ", "outputId": "1706c312-ff28-48cc-cdb7-d4d1c9940630" }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "100%|██████████| 5000/5000 [00:06<00:00, 820.91it/s]\n", "100%|██████████| 1000/1000 [00:01<00:00, 956.89it/s]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ " precision recall f1-score support\n", "\n", " 0 0.78 0.79 0.78 524\n", " 1 0.76 0.75 0.76 476\n", "\n", " accuracy 0.77 1000\n", " macro avg 0.77 0.77 0.77 1000\n", "weighted avg 0.77 0.77 0.77 1000\n", "\n" ] } ], "source": [ "X_train,y_train = get_bpemb_features(train_data.values[:5000], bpemb_en)\n", "X_test,y_test = get_bpemb_features(test_data.values[:1000], bpemb_en)\n", "run_classifier(X_train, y_train, X_test, y_test)" ] }, { "cell_type": "markdown", "metadata": { "id": "jn8IBZO0FI1k" }, "source": [ "Just as a note, you can actually get much better performance using simple word counts -- why do you think this is?" ] }, { "cell_type": "code", "execution_count": 25, "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "PDMW99tCVW3i", "outputId": "48946c35-bacd-4b56-f107-a6fb9666a899" }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ " precision recall f1-score support\n", "\n", " 0 0.89 0.84 0.87 524\n", " 1 0.83 0.89 0.86 476\n", "\n", " accuracy 0.86 1000\n", " macro avg 0.86 0.86 0.86 1000\n", "weighted avg 0.87 0.86 0.86 1000\n", "\n" ] } ], "source": [ "# Run on unigram features\n", "vectorizer = CountVectorizer()\n", "vectorizer.fit(train_data.values[:,0])\n", "X_train,y_train = get_unigram_features(train_data.values[:5000], vectorizer)\n", "X_test,y_test = get_unigram_features(test_data.values[:1000], vectorizer)\n", "run_classifier(X_train, y_train, X_test, y_test)" ] }, { "cell_type": "markdown", "metadata": { "id": "ouOJTiDOFSun" }, "source": [ "# Sentiment analysis using a Bi-LSTM" ] }, { "cell_type": "markdown", "metadata": { "id": "ciGgJEDwPaP4" }, "source": [ "# Reading data into a model\n", "\n", "A simple and common way that data is read in PyTorch is to use the two following classes: `torch.utils.data.Dataset` and `torch.utils.data.DataLoader`.\n", "\n", "The `Dataset` class can be extended to read in and store the data you are using for your experiment. The only requirements are to implement the `__len__` and `__getitem__` methods. `__len__` simply returns the size of your dataset and `__getitem__` takes an index and returns that sample from your dataset, processed in whatever way is necessary to be input to your model.\n", "\n", "The `DataLoader` class determines how to iterate through your `Dataset`, including how to shuffle and batch your data.\n", "\n" ] }, { "cell_type": "code", "execution_count": 26, "metadata": { "id": "Ym9eaRzpBFfZ" }, "outputs": [], "source": [ "from torch.utils.data import Dataset, DataLoader\n", "from typing import List, Tuple" ] }, { "cell_type": "markdown", "metadata": { "id": "uspFf7yZFdyO" }, "source": [ "This is a utility function which, given a list of text samples and a tokenizer, will tokenize the text and return the IDs of the tokens in the vocabulary. The tokenizer will split the text into individual word-pieces and convert these to IDs in the tokenizer vocabulary. In addition, it will return the length of each sequence, which is needed when inputting padded data into an RNN in PyTorch." ] }, { "cell_type": "code", "execution_count": 27, "metadata": { "id": "z8gwnwv9hFuz" }, "outputs": [], "source": [ "def text_to_batch_bilstm(text: List, tokenizer, max_len=512) -> Tuple[List, List]:\n", " \"\"\"\n", " Creates a tokenized batch for input to a bilstm model\n", " :param text: A list of sentences to tokenize\n", " :param tokenizer: A tokenization function to use (i.e. fasttext)\n", " :return: Tokenized text as well as the length of the input sequence\n", " \"\"\"\n", " # Some light preprocessing\n", " input_ids = [tokenizer.encode_ids_with_eos(t)[:max_len] for t in text]\n", "\n", " return input_ids, [len(ids) for ids in input_ids]" ] }, { "cell_type": "markdown", "metadata": { "id": "4FvRoD7rGC4K" }, "source": [ "This is another utility function which defines how to combine data into a batch. Importantly, it determines the max length of a sequence in the batch and pads all of the samples with a `[PAD]` ID to this length." ] }, { "cell_type": "code", "execution_count": 28, "metadata": { "id": "LJfzyoT_FJ_V" }, "outputs": [], "source": [ "def collate_batch_bilstm(input_data: Tuple) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]:\n", " \"\"\"\n", " Combines multiple data samples into a single batch\n", " :param input_data: The combined input_ids, seq_lens, and labels for the batch\n", " :return: A tuple of tensors (input_ids, seq_lens, labels)\n", " \"\"\"\n", " input_ids = [i[0][0] for i in input_data]\n", " seq_lens = [i[1][0] for i in input_data]\n", " labels = [i[2] for i in input_data]\n", "\n", " max_length = max([len(i) for i in input_ids])\n", "\n", " # Pad all of the input samples to the max length (25000 is the ID of the [PAD] token)\n", " input_ids = [(i + [25000] * (max_length - len(i))) for i in input_ids]\n", "\n", " # Make sure each sample is max_length long\n", " assert (all(len(i) == max_length for i in input_ids))\n", " return torch.tensor(input_ids), torch.tensor(seq_lens), torch.tensor(labels)" ] }, { "cell_type": "markdown", "metadata": { "id": "0vo7eBpGGpyT" }, "source": [ "This is the class which reads the dataset and processes indivudal samples. It is initialized with a pandas `DataFrame` and a tokenizer." ] }, { "cell_type": "code", "execution_count": 29, "metadata": { "id": "sQ7d_SW1FUkq" }, "outputs": [], "source": [ "# This will load the dataset and process it lazily in the __getitem__ function\n", "class ClassificationDatasetReader(Dataset):\n", " def __init__(self, df, tokenizer):\n", " self.df = df\n", " self.tokenizer = tokenizer\n", "\n", " def __len__(self):\n", " return len(self.df)\n", "\n", " def __getitem__(self, idx):\n", " row = self.df.values[idx]\n", " # Calls the text_to_batch function\n", " input_ids,seq_lens = text_to_batch_bilstm([row[0]], self.tokenizer)\n", " label = row[1]\n", " return input_ids, seq_lens, label" ] }, { "cell_type": "markdown", "metadata": { "id": "7NE1DwDCHLpu" }, "source": [ "Here we can test whether or not our implementation is correct. We first load english `bpemb` embeddings, which will be used to initialize the word embedding layer of our model, and the tokenizer which will be used in the dataset reader." ] }, { "cell_type": "code", "execution_count": 31, "metadata": { "id": "wbmE_EQpPCEY" }, "outputs": [], "source": [ "# Extract the embeddings and add a randomly initialized embedding for our extra [PAD] token\n", "pretrained_embeddings = np.concatenate([bpemb_en.emb.vectors, np.zeros(shape=(1,100))], axis=0)\n", "# Extract the vocab and add an extra [PAD] token\n", "vocabulary = bpemb_en.emb.index_to_key + ['[PAD]']" ] }, { "cell_type": "code", "execution_count": 32, "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "U6beVKUCnJ2A", "outputId": "c8abfede-7a62-4804-aae1-96722a2b58ca" }, "outputs": [ { "data": { "text/plain": [ "([[4,\n", " 406,\n", " 9631,\n", " 354,\n", " 10278,\n", " 24934,\n", " 17672,\n", " 7826,\n", " 955,\n", " 27,\n", " 5245,\n", " 4885,\n", " 192,\n", " 3870,\n", " 9002,\n", " 5590,\n", " 12165,\n", " 989,\n", " 71,\n", " 7,\n", " 6768,\n", " 4016,\n", " 24951,\n", " 1025,\n", " 1294,\n", " 6222,\n", " 1972,\n", " 800,\n", " 1221,\n", " 42,\n", " 7,\n", " 1361,\n", " 24934,\n", " 272,\n", " 437,\n", " 1656,\n", " 2170,\n", " 24935,\n", " 14189,\n", " 99,\n", " 24935,\n", " 1412,\n", " 189,\n", " 788,\n", " 24920,\n", " 215,\n", " 7219,\n", " 9042,\n", " 6381,\n", " 27,\n", " 32,\n", " 4307,\n", " 85,\n", " 6354,\n", " 13280,\n", " 19462,\n", " 2895,\n", " 4,\n", " 779,\n", " 143,\n", " 9009,\n", " 191,\n", " 15850,\n", " 400,\n", " 146,\n", " 3701,\n", " 24951,\n", " 7,\n", " 4794,\n", " 10176,\n", " 365,\n", " 42,\n", " 215,\n", " 2414,\n", " 24940,\n", " 8280,\n", " 24934,\n", " 34,\n", " 2054,\n", " 4719,\n", " 42,\n", " 16572,\n", " 146,\n", " 24180,\n", " 1228,\n", " 24935,\n", " 332,\n", " 1949,\n", " 1701,\n", " 34,\n", " 394,\n", " 24940,\n", " 2767,\n", " 862,\n", " 10726,\n", " 962,\n", " 72,\n", " 424,\n", " 24934,\n", " 7,\n", " 7184,\n", " 27,\n", " 800,\n", " 1221,\n", " 332,\n", " 8099,\n", " 37,\n", " 24934,\n", " 592,\n", " 42,\n", " 332,\n", " 24068,\n", " 24934,\n", " 127,\n", " 1579,\n", " 360,\n", " 332,\n", " 38,\n", " 3691,\n", " 2919,\n", " 86,\n", " 3243,\n", " 42,\n", " 13269,\n", " 4175,\n", " 24935,\n", " 12144,\n", " 22944,\n", " 7595,\n", " 14680,\n", " 2586,\n", " 24934,\n", " 683,\n", " 260,\n", " 7,\n", " 2480,\n", " 24933,\n", " 401,\n", " 9984,\n", " 154,\n", " 1025,\n", " 315,\n", " 35,\n", " 88,\n", " 72,\n", " 7,\n", " 334,\n", " 67,\n", " 771,\n", " 24940,\n", " 18932,\n", " 24951,\n", " 351,\n", " 3811,\n", " 80,\n", " 467,\n", " 1613,\n", " 24935,\n", " 5032,\n", " 1412,\n", " 189,\n", " 73,\n", " 1935,\n", " 32,\n", " 1024,\n", " 22531,\n", " 3473,\n", " 592,\n", " 345,\n", " 493,\n", " 1623,\n", " 4820,\n", " 88,\n", " 34,\n", " 278,\n", " 22754,\n", " 13032,\n", " 220,\n", " 875,\n", " 3532,\n", " 24934,\n", " 637,\n", " 215,\n", " 361,\n", " 24940,\n", " 1828,\n", " 773,\n", " 4338,\n", " 3906,\n", " 810,\n", " 24788,\n", " 24934,\n", " 79,\n", " 7,\n", " 670,\n", " 73,\n", " 216,\n", " 4,\n", " 1026,\n", " 24935,\n", " 7669,\n", " 4939,\n", " 130,\n", " 7949,\n", " 2]],\n", " [205],\n", " 0)" ] }, "execution_count": 32, "metadata": {}, "output_type": "execute_result" } ], "source": [ "reader = ClassificationDatasetReader(valid_data, bpemb_en)\n", "reader[0]" ] }, { "cell_type": "markdown", "metadata": { "id": "F0cjbDzfJ3pE" }, "source": [ "# Creating the model\n", "\n", "Next we will create a BiLSTM model with BPE word-piece embeddings. In this case we will extend the PyTorch class `torch.nn.Module`. To create your own module, you need only define your model architecture in the `__init__` function, and define how tensors are processed by your model in the `__forward__` function." ] }, { "cell_type": "code", "execution_count": 33, "metadata": { "id": "MBX9nXd8DTBA" }, "outputs": [], "source": [ "from torch import nn\n", "\n", "# Define a default lstm_dim\n", "lstm_dim = 100" ] }, { "cell_type": "code", "execution_count": 34, "metadata": { "id": "qRuI--wZOsCY" }, "outputs": [], "source": [ "\n", "# Define the model\n", "class BiLSTMNetwork(nn.Module):\n", " \"\"\"\n", " Basic BiLSTM network\n", " \"\"\"\n", " def __init__(\n", " self,\n", " pretrained_embeddings: torch.tensor,\n", " lstm_dim: int,\n", " dropout_prob: float = 0.1,\n", " n_classes: int = 2\n", " ):\n", " \"\"\"\n", " Initializer for basic BiLSTM network\n", " :param pretrained_embeddings: A tensor containing the pretrained BPE embeddings\n", " :param lstm_dim: The dimensionality of the BiLSTM network\n", " :param dropout_prob: Dropout probability\n", " :param n_classes: The number of output classes\n", " \"\"\"\n", "\n", " # First thing is to call the superclass initializer\n", " super(BiLSTMNetwork, self).__init__()\n", "\n", " # We'll define the network in a ModuleDict, which makes organizing the model a bit nicer\n", " # The components are an embedding layer, a 2 layer BiLSTM, and a feed-forward output layer\n", " self.model = nn.ModuleDict({\n", " 'embeddings': nn.Embedding.from_pretrained(pretrained_embeddings, padding_idx=pretrained_embeddings.shape[0] - 1),\n", " 'bilstm': nn.LSTM(\n", " pretrained_embeddings.shape[1],\n", " lstm_dim,\n", " 1,\n", " batch_first=True,\n", " dropout=dropout_prob,\n", " bidirectional=True),\n", " 'cls': nn.Linear(2*lstm_dim, n_classes)\n", " })\n", " self.n_classes = n_classes\n", " self.dropout = nn.Dropout(p=dropout_prob)\n", "\n", " # Initialize the weights of the model\n", " self._init_weights()\n", "\n", " def _init_weights(self):\n", " all_params = list(self.model['bilstm'].named_parameters()) + \\\n", " list(self.model['cls'].named_parameters())\n", " for n,p in all_params:\n", " if 'weight' in n:\n", " nn.init.xavier_normal_(p)\n", " elif 'bias' in n:\n", " nn.init.zeros_(p)\n", "\n", " def forward(self, inputs, input_lens, labels = None):\n", " \"\"\"\n", " Defines how tensors flow through the model\n", " :param inputs: (b x sl) The IDs into the vocabulary of the input samples\n", " :param input_lens: (b) The length of each input sequence\n", " :param labels: (b) The label of each sample\n", " :return: (loss, logits) if `labels` is not None, otherwise just (logits,)\n", " \"\"\"\n", "\n", " # Get embeddings (b x sl x edim)\n", " embeds = self.model['embeddings'](inputs)\n", "\n", " # Pack padded: This is necessary for padded batches input to an RNN\n", " lstm_in = nn.utils.rnn.pack_padded_sequence(\n", " embeds,\n", " input_lens.cpu(),\n", " batch_first=True,\n", " enforce_sorted=False\n", " )\n", "\n", " # Pass the packed sequence through the BiLSTM\n", " lstm_out, hidden = self.model['bilstm'](lstm_in)\n", "\n", " # Unpack the packed sequence --> (b x sl x 2*lstm_dim)\n", " lstm_out,_ = nn.utils.rnn.pad_packed_sequence(lstm_out, batch_first=True)\n", "\n", " # Max pool along the last dimension\n", " ff_in = self.dropout(torch.max(lstm_out, 1)[0])\n", " # Some magic to get the last output of the BiLSTM for classification (b x 2*lstm_dim)\n", " #ff_in = lstm_out.gather(1, input_lens.view(-1,1,1).expand(lstm_out.size(0), 1, lstm_out.size(2)) - 1).squeeze()\n", "\n", " # Get logits (b x n_classes)\n", " logits = self.model['cls'](ff_in).view(-1, self.n_classes)\n", " outputs = (logits,)\n", " if labels is not None:\n", " # Xentropy loss\n", " loss_fn = nn.CrossEntropyLoss()\n", " loss = loss_fn(logits, labels)\n", " outputs = (loss,) + outputs\n", "\n", " return outputs\n", "\n" ] }, { "cell_type": "markdown", "metadata": { "id": "9d-X3p7ENzoV" }, "source": [ "## Choosing the device to run on\n", "\n", "PyTorch tensors exist on a particular device: in order to use a GPU for example, the tensors in your network need to be moved to a GPU device. The following allows us to check if a GPU device is available and get a reference to that device which we can use to place tensors on it." ] }, { "cell_type": "code", "execution_count": 35, "metadata": { "id": "q-l7gpJFNyfe" }, "outputs": [], "source": [ "device = torch.device(\"cpu\")\n", "if torch.cuda.is_available():\n", " device = torch.device(\"cuda\")" ] }, { "cell_type": "markdown", "metadata": { "id": "Bk5Q2-JdNqGN" }, "source": [ "Here we actually instantiate the model and move it to the GPU." ] }, { "cell_type": "code", "execution_count": 36, "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "2gLT6MUrO5xw", "outputId": "02e8523c-e2d1-439e-f348-36640c6dcdde" }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "/usr/local/lib/python3.10/dist-packages/torch/nn/modules/rnn.py:88: UserWarning: dropout option adds dropout after all but last recurrent layer, so non-zero dropout expects num_layers greater than 1, but got dropout=0.1 and num_layers=1\n", " warnings.warn(\"dropout option adds dropout after all but last \"\n" ] } ], "source": [ "# Create the model\n", "model = BiLSTMNetwork(\n", " pretrained_embeddings=torch.FloatTensor(pretrained_embeddings),\n", " lstm_dim=lstm_dim,\n", " n_classes=2\n", " ).to(device)\n" ] }, { "cell_type": "markdown", "metadata": { "id": "nsDib4zuPkOa" }, "source": [ "# Training and evaluation\n", "\n", "Here we define a metric which we are trying to improve -- in this case the accuracy of the model with respect to the label -- and functions to evaluate and train the model." ] }, { "cell_type": "code", "execution_count": 37, "metadata": { "id": "e43ArMISKwzh" }, "outputs": [], "source": [ "def accuracy(logits, labels):\n", " logits = np.asarray(logits).reshape(-1, len(logits[0]))\n", " labels = np.asarray(labels).reshape(-1)\n", " return np.sum(np.argmax(logits, axis=-1) == labels).astype(np.float32) / float(labels.shape[0])" ] }, { "cell_type": "markdown", "metadata": { "id": "XHkYIaldX7bK" }, "source": [ "A useful utility which gives us a progress bar" ] }, { "cell_type": "code", "execution_count": 38, "metadata": { "id": "ovjgriwYQ1v2" }, "outputs": [], "source": [ "from tqdm.notebook import tqdm" ] }, { "cell_type": "markdown", "metadata": { "id": "fDdHYcJ-QRn9" }, "source": [ "This is a utility function which will take a model and a validation dataloader and return the current accuracy of the model against that dataset. We can use this to know when to save the model and to perform early stopping if desired." ] }, { "cell_type": "code", "execution_count": 39, "metadata": { "id": "z2r3DzpSOq_l" }, "outputs": [], "source": [ "def evaluate(model: nn.Module, valid_dl: DataLoader):\n", " \"\"\"\n", " Evaluates the model on the given dataset\n", " :param model: The model under evaluation\n", " :param valid_dl: A `DataLoader` reading validation data\n", " :return: The accuracy of the model on the dataset\n", " \"\"\"\n", " # VERY IMPORTANT: Put your model in \"eval\" mode -- this disables things like\n", " # layer normalization and dropout\n", " model.eval()\n", " labels_all = []\n", " logits_all = []\n", "\n", " # ALSO IMPORTANT: Don't accumulate gradients during this process\n", " with torch.no_grad():\n", " for batch in tqdm(valid_dl, desc='Evaluation'):\n", " batch = tuple(t.to(device) for t in batch)\n", " input_ids = batch[0]\n", " seq_lens = batch[1]\n", " labels = batch[2]\n", "\n", " _, logits = model(input_ids, seq_lens, labels=labels)\n", " labels_all.extend(list(labels.detach().cpu().numpy()))\n", " logits_all.extend(list(logits.detach().cpu().numpy()))\n", " acc = accuracy(logits_all, labels_all)\n", "\n", " return acc,labels_all,logits_all" ] }, { "cell_type": "markdown", "metadata": { "id": "ykX39dqiQ3qI" }, "source": [ "Here we define the main training loop." ] }, { "cell_type": "code", "execution_count": 40, "metadata": { "id": "7aKieRBZkghS" }, "outputs": [], "source": [ "def train(\n", " model: nn.Module,\n", " train_dl: DataLoader,\n", " valid_dl: DataLoader,\n", " optimizer: torch.optim.Optimizer,\n", " n_epochs: int,\n", " device: torch.device,\n", " patience: int = 10\n", "):\n", " \"\"\"\n", " The main training loop which will optimize a given model on a given dataset\n", " :param model: The model being optimized\n", " :param train_dl: The training dataset\n", " :param valid_dl: A validation dataset\n", " :param optimizer: The optimizer used to update the model parameters\n", " :param n_epochs: Number of epochs to train for\n", " :param device: The device to train on\n", " :return: (model, losses) The best model and the losses per iteration\n", " \"\"\"\n", "\n", " # Keep track of the loss and best accuracy\n", " losses = []\n", " best_acc = 0.0\n", " pcounter = 0\n", "\n", " # Iterate through epochs\n", " for ep in range(n_epochs):\n", "\n", " loss_epoch = []\n", "\n", " #Iterate through each batch in the dataloader\n", " for batch in tqdm(train_dl):\n", " # VERY IMPORTANT: Make sure the model is in training mode, which turns on\n", " # things like dropout and layer normalization\n", " model.train()\n", "\n", " # VERY IMPORTANT: zero out all of the gradients on each iteration -- PyTorch\n", " # keeps track of these dynamically in its computation graph so you need to explicitly\n", " # zero them out\n", " optimizer.zero_grad()\n", "\n", " # Place each tensor on the GPU\n", " batch = tuple(t.to(device) for t in batch)\n", " input_ids = batch[0]\n", " seq_lens = batch[1]\n", " labels = batch[2]\n", "\n", " # Pass the inputs through the model, get the current loss and logits\n", " loss, logits = model(input_ids, seq_lens, labels=labels)\n", " losses.append(loss.item())\n", " loss_epoch.append(loss.item())\n", "\n", " # Calculate all of the gradients and weight updates for the model\n", " loss.backward()\n", "\n", " # Optional: clip gradients\n", " #torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)\n", "\n", " # Finally, update the weights of the model\n", " optimizer.step()\n", " #gc.collect()\n", "\n", " # Perform inline evaluation at the end of the epoch\n", " acc,_,_ = evaluate(model, valid_dl)\n", " print(f'Validation accuracy: {acc}, train loss: {sum(loss_epoch) / len(loss_epoch)}')\n", "\n", " # Keep track of the best model based on the accuracy\n", " if acc > best_acc:\n", " torch.save(model.state_dict(), 'best_model')\n", " best_acc = acc\n", " pcounter = 0\n", " else:\n", " pcounter += 1\n", " if pcounter == patience:\n", " break\n", " #gc.collect()\n", "\n", " model.load_state_dict(torch.load('best_model'))\n", " return model, losses" ] }, { "cell_type": "markdown", "metadata": { "id": "ANhsatD4S4C5" }, "source": [ "Now that we have the basic training and evaluation loops defined, we can create the datasets and optimizer and run it!" ] }, { "cell_type": "code", "execution_count": 41, "metadata": { "id": "BT85gSAXKqYc" }, "outputs": [], "source": [ "from torch.optim import Adam" ] }, { "cell_type": "code", "execution_count": 42, "metadata": { "id": "fY0EQ_YFLMos" }, "outputs": [], "source": [ "# Define some hyperparameters\n", "batch_size = 32\n", "lr = 3e-4\n", "n_epochs = 100\n", "num_workers = 2" ] }, { "cell_type": "code", "execution_count": 43, "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 1000, "referenced_widgets": [ "926f1a8556aa447eaa5d4550248bbe6f", "6cb03ede44a94095a5d522c2be948aaa", "00a5e8fa0b8b491e8d3b2bed083472fa", "0c761efeb75845739087bea0bdfd7a26", "410a7468290a41388a93a44ef4fd96b6", "fd57381b0bcf4ae6a924ee89f7bd46b2", "977d3dfb3e864becad5d61647d11b998", "778e934ba90a4d89b96af137b2780e97", "464e2337aa174471bad730089b8bb337", "1edead9a4045424d8daaf6a9ffe96532", "1ee2aa623891499995d317ed81e7df93", "e41799f69bf64d6e89d8d9eb354b22b2", "74bf51d3c7ef4414ab10b727ec703ada", "1f763beae48c4347a08064c7be6913b1", "2d89108598524c98b1d434e7a40420cc", "26cc99e69e654e76b0925a996dc0db66", "e0c570465c9649dfab19ffc10936f510", "2f16e3d3a1de48f38cea0eaa98ca083b", "cc1c81bff93344128eb7c05f6d386edd", "5789ee4d4a9643a4ad6fa4acda536d1f", "aaef7e2c854548e7b00d9f448dab79be", "4e9237550c9a4a14aa18ac88e3093943", "e45b3352e55b400d82d0389cc8aedd86", "d34b0fb76af141bba4b2a2d0d8381a1c", "a1547880ed1c472abe6474c5a0ace8d3", "7e81a3cecde7463c8c95fbf375ce613b", "0115e2de7f8244eb9833b3a1b3df8ba4", "d752f9eeaf3b453bb1e565f9237acd24", "d53326e784d64fb49a9bcb815a3ac690", "9888a01e0c464821a879b787e195cd59", "7f68a78c8ffb400cb9ff66d5d295d781", "3a6b3193cacf4e2c82fbbb542951660c", "73c1a994af6a4184834c91276cbe4501", "6c4cee2a6ca243ca92e0d57b5c41f7ce", "e9271562df854c7f92e9513c739bf6a2", "aa74de82db624b5d9556508ff69b43f6", "beaf665ba5d743a0a4a5a3f064eb5d2e", "55e56d8f701e4b9db2dcaf404446331f", "6ecfcb76194d45f5a525a7a4c2d7b89f", "2c86a2b2aded4190ba398325634cf08c", "421a6b3277034b8aa08ab21808172b3d", "c9aa15a25aac4072a5e37f0b78cf0bfb", "3011132111684534893a685c96522633", "f9d8374dfc3046f98184c4bcb0be0629", "7c75a4b00c9b42b8a3311f594d7c105b", "2a43e746088548e1857a7d0286978b91", "3c3ef800746643fa930819c849060e8d", "11e69739c34f48a8ae600f66e2f2905b", "a2e2c2cb0a504c098f4cc26c5f00858e", "f8beea27ed9843b99f34eb3aa1867e91", "0fb161d5ea4046d6b071d99f352509ca", "a44b4a44a6d6466782021fc4bbb83273", "38480077462a4186b3634020dad89c47", "d991706803d743f8b8d0368d201822e0", "42d82ab2765546179f782605aaecaca2", "bef975f393a44015888c9952cb98fe90", "187f958ed6824f2b9207db7c26f668b0", "d83786a7506d4c33a84a484131cba521", "54cfaf087bac44259b77c2c8f4ebd6a8", "a6e728653312448f839646be5fb389e4", "284b3e0245f04b9db138df4757b6353c", "1589fe3433d649fbb0b7b4050b5b251e", "80ce1b8c0ae942788681718ad6f5aef9", "8f7bb52f0a274ccdb84bfb719047560c", "a59a7fb387a140f082fdd61ef3ebf27b", "3fa3d396d2da4a5f84d4134fb470740f", "04f958cc5b744f2698c37e0446330a84", "6a31df8c34bd4b54a686ea591cf3fe3c", "57109df47b694303b3bb85c1dcdd2712", "d3d9b995824a4c56b07baffe42909166", "a1c693aad81745c4bc53cef7cf28d933", "315a2d105a4f425d84757f0bae9080e7", "0351366a50544974b5d1722008faccb6", "d9792c39a4704c8691f679bce29cce3e", "076c74f7d3e44672954435a793616400", "13144b04c84a465bb447cb9b1b413ccc", "2a0392f6614f4c0dad192ec03f902832", "2915eb53555347c0bee245e459d15672", "2fc1fbab1e974339bdfb2e98b6d37254", "b2bf8ae7135343c4bcbe6c9332f54031", "51e0333ba0504ce1be29d4903cb2cb3e", "997e8d872d804635b5f84c4adf0ee3d0", "10fbaf330b284e9c934130b02fd11e20", "860db445cd234c458e39683abf79d20d", "14ba4e87a086482386a019f0a3471d7b", "8f1531d90b0d466ba4b1b35b772fbaa2", "89b6dce4d3f94793a12eeb0254e8fede", "24dd948c25324f2798f671b658f023c4", "4cc4a2b9de174aaba232585d262aefcf", "717335f03e104753b32e272420ac363d", "13044fec69894f51b88f128fd2216545", "6c305b5be969450f9fadc8090444e861", "d6ee86b2a95f4a31931f2e61a73f06da", "fe4f975b160f47f0b41272208abea66f", "46774fcb283047b0b38bb634a17e216b", "7a128922a05145c1bf6bea7d42305f52", "c1370527eb614857b9b836d8e62f2d95", "ef298b02f1df42fc87229bd08ecc0361", "dd0dbf193e4d4c10944ccd03b92fe931", "a4089e7c634540be8bc131e1c5461ee8", "f10b8098f3954bd3894ffd7675859e2f", "dbd3412d2e924e45b4ee3235a8bac03f", "aefb18b6aa22468caf74b91476b670c0", "0fe2fe516e954ff2a9e1de295124a4c8", "d5583e9be70449a28eb0bd0760f1b716", "07cac27e2000464db7585ac90fa09698", "9c3288144dc14580bff60b1f6b311568", "73b75df972194c83bf35ba155890c630", "de225802aaed413cb163a6c039d90583", "93b07d248c82483e8b6d928dffb5dd7d", "db8bcecfeace403da61beda6126c0b2b", "8f49aa30f5af47d08624a7dcd8c3716b", "ae8756914539434183bf2fe7b4f9cb72", "8ba69e9872034cd59af36c8aac6d8301", "e8a7ba5ff0d34f10bcd13c5bee452f7e", "5bcb55d604b64cd78e2f9f2405cd6bac", "fe28dac770e54cb3b6ebf2fb5c71afb6", "b928108ae2204b28a1b59f6507e6f00d", "c789a87b1d5d4bf587ff6ccdb034a988", "4bbc91e7f6324fa98b7a9723c2ddba27", "29ae2bce175d4244bec8b3f93fe3a45e", "13a05e893911434087b1eee34b4a3d18", "1443558ae67c4196aa7b7c8a7f8dc33e", "359bd21c532b4229b46e5866a0fc201d", "f1da81f6a88c48ef82332f5783bfa1b0", "e5012e3a63e04577aeba0f601e9e71e3", "8510c3cf77b544afb9332c9939e335fb", "f6fb70615d7f42deb6a5f0bf8e4e9388", "96efd0ea3ab8461aa83bcfc7d87bdef2", "793448e628744fc4ac8a37873d8dd351", "6f55a6a5140b466b883fc6b10da6ea3c", "640b3ae423f14358862c559511cb9667", "e86f39f9529a4ae096768fec7311ae01", "54bd1e6e42c44883bbbf182689ae59f5", "3f59ddf770ed4ee6afb02f27189cfb25", "8ea510a5f971480da98c971828c281ff", "2e4d02c9ce6340928e30ff08abcbdb28", "a5d4ef0e7777423cbc273b6dad481c96", "559ad4d1d5564c80b8d93104336f4f33", "ea6fa70b5a4e49ffa573d5bfd3594759", "b2639fe5a574488bb6cd18dc98bc1f26", "1b74c804631d41d78b0aaca83b99b253", "835e69ac1c2c4d19aa2cafce31332fa0", "f073bb9b740c4195a84df8fcd6d650ca", "23a0e7ba49c346b9a20597cabe225017", "b015bf33cca9493c98c81388598ac400", "ffd629b430f242088633dc63c0d17bf4", "aa66068cef5945c98d6e1e7b48b6ce7b", "a0c177d582374a969be38eb4552a395a", "843dd998195041fe9e4b86bc837331f8", "2619d3ef306644b28ed856eb8a6a7564", "6b11f17aae2f4ec4885ddc31fd550269", "c505c8940d9b4da2844bcaab2250aefa", "61abc725cc954ef8bff37e7cea64386c", "cf8bfad984e1457786a056a8196e6bab", "1b375a5fe4a04e28b8144129fe50dbb0", "f44761488e6c48ac995a9ff3796b09c9", "b1664053f2714f22bc3438e1dbd387b2", "fb1853d35fdf411a9974059e82d88c1b", "2650f402ae1f4e538fd8a35385be7d3b", "5de18f1410ec43749f760c95940563af", "8f737fd915a848e3a6381b1d11662941", "0e41afc8a28949be9d2e71459515351b", "bee3a5b4e07a40feba0b57d94e81e8d9", "0a10e49406bf42bab816de28bfa2f9f1", "264c749235c941c2acb97c8bf5765042", "3e0cce91b84a44e38609199174da6e5f", "eb331ae42cf247ba997ddd062022756b", "5073d285492c47f1a47df7134f3c2031", "100ea04111be4bd7abb76048505ade6e", "0dbbee9da36e4a10ba73662ce5eb3719", "2b8a93a718b14ed2a5eef2706095818c", "cda4f5ac3bb844168a6f9e0897c476fb", "1e8bfb5e4b864819b338901257ab22aa", "c24e6db71b9840669e8f7a05f9f89bb1", "18648d4f24de43a28c135e4d4ab5b7b3", "49b749c89201407e8c71a1bfafd99d69", "a3ff2cc049fd415a8a2a0c81614c0b29", "fbe0f2c61ea64a61b835ee693e3408d5", "0c78747a170041dcbdec58fd301a272a", "40d488202d6940e6acde879db9535cbd", "7d599757da4c41acbf64846512dbab12", "e2a79c1117684a2aafd029dad8031ed1", "0f0ad75cfe3c4da194955795e416f884", "a146eb9cdb7f4c9fb873e82a7b4d6796", "5d7bf6664de34592b3ec8941d4050ac0", "1f66b37289fe44f2a5d56a77d9b27466", "c3b3296b0241445f8a74aec5055467c6", "f9f322fee7cb4bdd80d491a7d73e1a77", "9b5832a50ac34b6a8575689b788c7380", "75ea770b967f47bb9fdd6cdcdbd29af7", "4257084ee88e4f71aa12bfe135ced1a1", "7050eab150a741d7accd1ade692b2469", "ae711a2aa0f24f72a875db70f2523409", "8d11d201dfb74706b07ff2e88e26ca10", "d9c0995134674ff581ae7dd50e5d1ab6", "c9ddb9080e1b471c85c8287d7eea9002", "befb3e0751aa437980cad02836bda2ce", "76d44733bceb4f05a1f358ecaad41759", "337f0624edbc4921ae42015334f89776", "ef31d01242434af18420a227bdacafcf", "86190a84b44842ddab09622c11e17dc6", "3b3510621bfd4851be8caeaf545d15c4", "ffdb44fedbb9441c837ab7e99d768236", "b2fbd5d5b0e44d81904ea7733ee5756a", "a534bbb1de684a46849973f23285e431", "e1c520a6cd2345c7be796ce5a939a1e5", "fda066cb60a34be5b4686ceae8480398", "6689cf0a91b9442eaab0b8b0cc156afa", "eca521bca75143b1bceaef01e14fa94a", "93b71e07793f48968faf615a6f4c27c7", "700b0af97d294637a3a97baf73400783", "cfc88ba3d004439485c24cdd0ba8c199", "78d3332ce02349eaaecab20d05a910e7", "d970ec0c66294f72b578a20a43d9774b", "42d717cc6b4741629a59ed1b5c9a9503", "1cc9484e07f042bf84d0da2a8f7290a0", "dc07a954e9a643c18da6e45ec9b020aa", "25c57eaf54c94949b7dbfe82b57e0f93", "24046e2cc0f54699b8e7b33f04b59fef", "c58478d9854d4f969563aa4945f0ec6e", "05fdfdbc91304fd2a753c56150464a8a", "5419e039cb574b5c980caf966af56f70", "4e86ea319dd842b187f90063a6f8afac", "f58954996c4c48ee8d5421469b611f57", "0b9835ebbfcb4033a048191149c56369", "7ccd3615c0f54450873dd5abe3939263", "8fd38ef199514b18a393bdacc477dbf8", "0c5a6097e48d4ad289227751c88ec278", "40c99f2ff79a4f7da9b76558d5abbd39", "99d7eca1706b4509a3b39b24e3781166", "39d8c9ff3a234c4597013816f8b25ad9", "7635e04aea5d4162a9f738e3878938b3", "03285b7828d14175b48f5dc0dad8eb0d", "6906303d86e14250b81fa4ed00845596", "82747f919c9d4059982a40a6e494d25b", "cdd91aff477c4cda92ba6121672b9c29", "fcef8293ed474dd3a4f148862b42e165", "f61161dac6b34c79bc3151efb8fef0db", "06420f4b43cb486899318292b036bd71", "6b72a6604e2d4661a6eaeba03a915d4b", "09c0bc43372d46398aa80188425c1e71", "91d74e14e93f432cbe9f599f5f28fb32", "da4e076bd8f9468f9cb9422545cb0ed3", "3008fe8659bd4b0e9fa6cb7d963d9f85", "55bb177ae55e4b20ae8152aa535e0a32", "7dd0d38d8d1e447a959472d9c7a608d2", "f1151d172b13404790364cda0488a219", "e9278cfe54c24d37bf89a933fbc57750", "297780c8c7464390a20cc1f1420d0232", "2affb9b5b7d043ce874ffc3802b29876", "c1d4336862cc47329cd4f90ba9cc3f02", "727410f822c745aca36ddbe71a34933b", "c291cef916a5445c9db859e17c49b308", "0e531b5b797e46999cf40acc7335bd8c", "3902448b2a74497897dcd58565063fff", "f21be4a743b745c6bd6dfc80495df788", "10c22310fa4543d18018e43d120a726f", "e68dd03382e04325954fa8e4fe03ea37", "fc50c36f6c5d426cac1b5341712d710a", "1e107b8ff98141f8bee67fdbde076187", "19f589a8fe2c419099e9a807517ffe60", "82f25546407541e69619901f59b67b75", "1f980db77335406c89adfe1ad2c72571", "645941d325a54bd8b5f3bbfb5276a85c", "766f8ec5270e487dba41911660bb41ef", "4dde59398fe147e6b4fa34e2edc5288d", "c048fbe26186488685b584866c810842", "5f9dd1217ac043e19debeca98bc905cf", "d24d2fa8732940dcbf42fcd6e2f78b92", "ac2a0a45514a43dcb91963cadd9182f6", "083fed7981a2405bb9fd3e0d7cad2b53", "d1d03b96d0254460b301ec599478c50b", "77b300181c79414c8f77299143d09050", "34808d6bfaae4581861394a96a283020", "4acc897e5fa44548aebb4da7dfa57919", "70d0bd3e33bc4db988edadb22e424a52", "cf362202f7914f39b458e953ca1cf67f", "5a706e9dd6b7419f824ef92776d37423", "dc990c2f04234bf4a636a55651b0f15f", "61d940dd339f4021993b6d3174fe3a2a", "e6fd29013a964389942bd6df1ffd4ccf", "89fea1ae9c2e4a7da050dbc4049b2664", "3845feadad9e4c07b2bf3d3b288543b6", "514e717601bd4a058038dc592f0ff7a9", "98f6b10b3c24448092e6b46341749367", "728efa2c661447f599d8e30a2747fb56", "be0809b0f26b42aa9480a0fb00777602", "c7011d80b79943ef93a0ef74e8f07303", "1bff28736c7d41e4bce008d0fc7d11b1", "244e96647ef049698f679bb08a62e11d", "a50661b223b6440489201fca644238c5", "144b99ee09cc4857aa9c6682b3ee0475", "33b2898aae0541ca9fcb609843643be4", "ae507da3de9b41cfb20c84ec4df2ec82", "47a5fff8a29b4517a7bcf33b875fb862", "cb3667cad85d4e7f9acc67b71f2f7bfa", "bda7082797ee4684953b812ad37f55d4", "b6693add71854a1d95d4513c5c60d5fc", "bf74f83ad3f74fd1a44c0cf73c3122f2", "69fafaed134b49fc9acbc344bb1a5fad", "12cc25efff3d41ee92afac072f8ff329", "6e588fe4366e4603bf17095968db7b50", "25d14f5f8cef4b68a242266eb92e11da", "2a733f55ff9f49caab105e263c0fe9a8", "8afceee5437a44eca642ab7592c441d0", "adcee41956db481fb8f4251726839fea", "6107c4bbdbec4eae859058e7dc45bb5e", "4f5751ebcf0c4facaf587d420f36867c", "4e1811558da0440b95202fc8c9607d15", "43561a41478f4335aadc4a3087e9a4d1", "0ed51a0abaa84d85a23570d8b0a46448", "b0cae64e399a4c30b4bc9a9b33182f5c", "b666506e437f4e59af9e67a9ba4811fd", "aab25fdb0b184aafbc7b37b9d62515f6", "cfa03a315ef345559091514b6f07bf92", "c109c5785ed74690b6ea8b99cd7cc3cf", "e57b6431ee314748952aec42285d9df4", "6ee0d375d8be44afbca80233404ed22e", "5cad6ae4a1c74fcf81eeb03c9facb741", "b3930aa5c0ed4afd87b524e86081c355", "a8156075a7de4358a5e11b8d477d5de4", "24a6c04737794ae28a727114643cfd50", "0b7b575e995e434599e7411c171a5862", "77c7ba44afdd4b06af5d183058a473b8", "c366fa99f7284dac8c6afd069501798f", "e5e891b0af024cb8b2501f192e47d5bb", "fa7ed7ac76d84534b3fedfe2a2419e86", "7fb5a898cc7742deb868018732b49e9b", "caf06a47058e4b76b1bba8d5d4f3f5e1", "8170aba797ef499ea2255be2341e685d", "8f93d6c971394ac682691eed860b1950", "3b93f43fde7045b08ffc21284d6b65d9", "a888e86d828f4d5b873a4bd10715b647", "2dcee1857504403e878556da85eece12", "f1a5e0bc37734a7591087bb5943bf36c", "237cfe88569041beb4af0f38dbddba12", "7a8d7297ec36447a88f19cfacb60403c", "b2d737703d1d46fea3a6503678877f2d", "7690dce588fd43b8a821334c5293505b", "1489132be9b645228b19d5d9be8554b2", "985b3c5764bd406e8e24be333bef7310", "bd789ccb066541caa194c3f523fec968", "a6899956bd004e459700834365bd192b", "9eef93f7c32c4e7aa141aaf44f1a73b2", "d33ebb3b6ffd4b8bbb088717d5a97d97", "f2181a10cb2648058894e1b39b34c174", "17cd548d7e834c76a8f586f08402a4ea", "8d2aee2bd6034dc6875b13ecc111d530", "cdffc0d797114960837d01853b5c68ed", "0d51a34a66ba43adb4c370a19ce1eb24", "9adfecd13d3e4a9ca80b37ef7f3e7129", "cfd7ea2bf5514222a2c86c033ca059b2", "44513e9b826a4ca2b5f33f596bac9e53", "c1fbcb7f81fc480e89b877b673a9cdfb", "a06eeb5b18fd4a7d9ff21c5d049b48e2", "4e2c761d262c4c1a943928815b632760", "b015b250c9504585a08b9e767b351385", "99fb90d73c214886afef20d39fccc047", "6578b7d418a14a8693cf95df53ac4cb6", "933c6f23627546e788a6e2719264b456", "1d9fce4d18844a4b83941f1cead00311", "4ae8706aa99d469abe873082a38c00a5", "53b05d76c480415dbf0ef87ea7c9736b", "ac3c736373f24d08a7ea1e69298bd87c", "63addf66aa6d44978977106ea9988543", "b8bd4fbdad30407b9a9c480e6966ae94", "4ba1bfa2192c487387756477a1629449", "cc9cf2e8b755462e97a67836b08be7c8", "e58b390f63d24d398c8da2e16a568dae", "5b6876def70d433a951012bad9603b19", "5f7df48c098744ac88d799a2cc3e7add", "2f5da747be8c470da49f8eff6da90025", "815f1fb8adaa470b995ce8dbd3c776a9", "bff3a19ec4504cddbdf9f3c16b7baa6f", "7d12f94898c7467598c3990199d16317", "ebd74d38458948fdbd1d8bff336db452", "c83f5fbcd13d465681ee9a4d8caaa2fc", "5ca1144be12d46e3aca0cdd720d78295", "47c2f6ab68b849b3949fa1b84377d303", "a90026f09eb54575b10e7e0f2b0db737", "0b6ae2fc404f41b099f136f5b07e6926", "1537778cecb94b96ab9e12ce4d657377", "cb220c367ac94ff2863eacbcd40540f6", "d31608b08b984bf9b21bb43c938d6f56", "13d82c832d2741deaab24ee2f246ba9c", "1be0cea26e86489fa16532ff5a0641ed", "45f5375ec6744cfdb6dd5d83f3ca44de", "399b64b25ce540a3bb2b0081c9497198", "bda0ac0859bc4557a5311d370b058836", "02649503ee314f62b5e24be226a12cf4", "09d468be840f488ab44d372b2463be1e", "e8be4cf8be1443c08b6eb4bab2de67b5", "3ce9155eb2624f15a24720a95a2317dd", "0df859a6812b4405b3cb7c5989918e42", "a827f98e8e5c4938bfe5fec333877779", "5a6c393ca7b94fdb80570c672236758d", "8043254f767444fdb26b88baf110407c", "03f1fd87228a4c8caaed5d67f17570ab", "c120b91960624e07ad83f7cfc8bde174", "c10a23ca8ce3482abc153ecc524f2de2", "3d179c45aefc4341adfafd3f393eb2ba", "3e1d558567554d859f7239dafb05c406", "5ed38c6b7b52484d8b81c10daae12f3a", "2177578e276e46eca5d69156d9ef8b45", "5a504757241f4162a473f18d1122d8fe", "875cad29154f4f9385531ef4d41de2f3", "54b585c4955a446cab3a5c0815195d7f", "09d26a8bf0b44c81b3d284bb19c50b97", "f628b2563b9c45b2bd254e78cdf9450f", "84017df4aa394010b514ac10a22dd864", "d5d428227282436b964b3d98fbfa3586", "84ac8f79a19b444b9f64e38d7f2634a9", "b0db4ee21adb4115b143209d013735a0", "d3943306441f4270a8160cea454620b2", "e290c50282be48edb498960a29c332a4", "4376d3e35769428590b6ac06ee20e1a5", "4dcb7eb965634c698aaa915476a4d408", "d33a62c25171410ebe983c918e7238ff", "64775cefab814b2e8ac36fb1d1f23388", "f9477accd29f45f0bf8f5e46a09c1f14", "b2e141503abe4189b74b1be599daeff1", "8d68ad4489e94d67bc2061ba19d97215", "05518e3b2c5546d480cdfb8f4ca45ded", "2e97ef16a5f2474294baa0c0d78417fc", "7aa1079a3d1442ebb52b7ed0c41a4fe4", "101ab481e5b14eafb60a9b0176f5586b", "d1e54aa3d10c4660a21ca8d9bb3a4b78", "bf96422df24f421cb36e5e7b93be209e", "385a8687f64144b8a8f3c1a77b47bb9e", "d490000823be4c2299e357df2f2a1488", "217dc42b2333474fb736b173a0144e9e", "4250a224b3ef4278bff2fa9ac1517b3d", "0725ac8d1cf54c3e9b9089dc42be1ce9", "b46fbe9f682e4f069d05121224c8a39d", "78e487322cd2418990c5896a6bc96948", "2671665b2a084c268b809d903130caa5", "8abdb8a6f994439fb4136069b14bf890", "9fbd7864dee04ced93e51c06f2486dd7", "3052a0b2dc2f4447b1bb17b0eb91e404", "2e751fd288b549b585ce1ab15e936f4c", "451f6311a61b495a8a6306f51844bb11", "e823afb52d79403b94d7d066c16be6d6", "b99d246c3b774607997d8530ea4dde9e", "0c4463f3dea14352beab367bb3da9cd7", "38772c0ad7f845c385fb73113441c16b", "652fe383916f463dbf5b4669e345c1ca", "e79cf09d68f84e31b56d9f4f8df5c7f6", "80a3693e50d1415dbe7c4ab9f933fc48", "fa240abf8e034ba69628acee99a6a303", "1f4b305c231a4ca6ad9cb5e234c2fea2", "c44b29dfe089498194e853462a90f632", "09db54c6acae41248d6cbada318c91c4", "8ce1dcf885354634a080aac999e21bb0", "825bf32476e940f8ad898de8282a0543", "2e80ccb374554a34b67c8f72050b61f6", "67c6fcef2b95402cb761ffef266396fb", "e7327806092a42c8bdce0f004769a809", "5398e8ee75244093b87699f7ac9ea3cc", "d11a69f2abc84878a8021a4f9081ea28", "dd6e824a0ba248e9a5a3ed7a85fb15ce", "4f14011d33064c81af5fccdd4ce6211a", "db85f255fcbb4dd2954367796584653a", "4e35f607f7824366b045b59875032288", "50a51559dad94c16b63332d565b4a838", "aa302cb15d3440b6ac199d328abc2c8d", "911daa3bab964021a2ca998d6f7c14ec", "37b355e896b54efabc32e63b360d2958", "8e51863eeecc4e02924eb958ad6d6104", "6e3c7910405a421581561559c1538f85", "6631a81ee34b4709b1df4cb87658f006", "cba6e1a7c10340a08f770cb2a4b36687", "b4b0db206919447e9f14fca9fba541e8", "44b500eff82e42258b01b8061948d009", "dae5c5534a7743f4bfcd4ff172c3ee58", "d8e51aeb58544fbf8decf931cdceea57", "17b785e4a0144303a2a504d5d9151fe2", "4026252ee2104e30844312a9da208c3a", "6ae4fd44a672446fbe82e48353548064", "5debc4f96bbd4fea97df5464bccf79e2", "df97191fcc8c4f478f337c8e75fc6419", "ef6da07f21644524929e15c08182a665", "d46ceb4e0fcd4d42b25e77eff799304a", "99533b5e1a4e4aa6b83f54c48483c7da", "e46441ada10142b783e277d14484d370", "ccb215a06a7e4d009c9e66e91c6652ac", "9c0cedbf179f4f66a5999a627fda49a3", "6a5045881caa493b9fec950574e5c420", "0e62c91d8afe44ad84f1eade0156e563", "b5295f11e24542b2b781e53dfd23fb3e", "ea75431995d641a4b8c79dd95918d121", "5df9ca5826a14775b642cf424fd9a23b", "786a3cf1352943eda7682c88eae4b205", "5e66fe2343a643b8b7591c9a358867e5", "31e5f16965574338a06cf3f427eb1d19", "8ab9a493408f4c1d8b5b1a261b080aa2", "933756ab525e4de58228b84efeb6afea", "7dcf57dec1184463bd2da4295b5c1f08", "0d98746b01244fb384acd8db70b0abcd", "b6640610dc48447d90123d905324bc44", "f865fc0fc5274467a7d8ad094ec9fc9d", "a0e2f3f79b9a42acb9b850d0d76280fd", "4df4244c89e24f5b9ac0380410f85d2a", "f36712a1352d4a4fb53191fb6df2f7d9", "ecadab9dbb9349aab7653ad6146480ec", "d5372bb6be7b4cf6babc21aaeea97d58", "53ad749ece3d4136824faf06a4059988", "3b5da713e65b40b2883a4d1c6577e7e6", "efec5e833b4e4391be3c7d91473788d4", "71c893df7ff94c4e87a99de0161b2057", "aaa00d85265048a584b98d8c97b8eced", "b1febabd98404e1e8d7a43e3bd3d858c", "6704fd28b5fc447b96a750a9d4b9fe0e", "492c167a97a04f03a4d9500a884716e7", "9c158782f953469385f2ed87cf0722cc", "742373274b484adeb70df5c2a205cdc6", "611b00131ea1456d9c1969dc724b924e", "2af0d5d80d1d4ef38ab0e17f22aaebad", "02e312c6fc8c4fbebd246d76e335e56b", "f697f33b37a84b9e8a6670012f2dcc0d", "b68ffaeb797045c8a2d1c9d98f1c6ec9", "bf6527d9dfc44682980a5251868b4a1f", "499be1adc6054fb688fecb5535b05c98", "a0b61e2a57b140f783bf0943a7b548aa", "e53a1125b08c4e02b5e5fc23a299612b", "f8481351c4fb430e82b685e29cc972ca", "79db07b1df644967b00bd41073ac124d", "a219032f3cd24bec9b71ef63bec8e41f", "5fc908c6458043b8823c04007b996b5c", "7ab06bc5bbd3436b90115f1b680611be", "a0029183c66a4db28d922d1a35574a4d", "ccf17703148d4daca92673ce3ab8fa52", "5771b76aa4d04f62a22ef1cf4d23270d", "1629523aab8f4263bb215f83594d72bf", "86bb41b0ed3d4d37af8ab9240eddc49b", "d51c6c6efc4f4bccbfb82d9f1b03a74b", "fd5badbbc46645a5ab2b6924e61e27f0", "872faa811d50402f8a79d79024c1d680", "b1f958157e124f9fb9bc16b7d2d1bdcc", "7a6e591fa45c4ddeb8cea26bd7553f6a", "7db87f2c9c434ee8a38dea5571899e63", "8f4888dee0834dd98b652a0306f75524", "fdb00147d95d467e86b7f4655f823e73", "304869df519c4c2e978e8d1a309e539d", "2ae099a9648c41d3b4235d069c087b75", "768198b5557f4939bfad5934a9c574cc", "caa2b16e37c74b41a61c9e9940ff91c7", "548780d130f741878660eb4c012e9a53", "0b8c4d644df74fe8ba071b097746476f", "d31be4e017a440249711649bb9316852", "0f1066a89709459b8d7f28210dadc48f", "59b815bf96a34efd99aa05c8f25a01b9", "ca1dc366a3da40ad8d704f0dcca508e2", "b77e0e47333f4f818124430acb214fb4", "fffba7b4e29b4236aff5e761c2fa0a27", "eff6c3a0b85b428cb3e4b811f6763d11", "824bfbb7042d4d888407ced20aeca868", "f8d24497bd974ca0b465782e855df134", "a5389bce24494a8aa7bf75b7aa442c79", "920f952741c44a6487132f1698c118fd", "eede9701859e4c51b95f829ba2082d08", "ae8de9a65ff54d1a88222a27dd7bc2f0", "d6a75f9f60b549388667015024f9bb9d", "3eeef5d3d8194f87ab856db7dce8e650", "d7ebae1b08c84d3e8a5f7dbef80942c0", "21589299bb804c88b15bdc7fe79dd99f", "a966f0cd89084038b89fed2ae7e4fc5b", "9fc788db01354434918024a2bcae8d2f", "42832ea7361f40a7a8ab14a34d53b95c", "32fbb826a3b644a08aa3bd5e212d7044", "1426ab7681514e2babd3db934446a14a", "d98d76161c934e9ea9f685128cde8592", "5a3af39fb2c044e6845bb1e9edd2d0fa", "08ff6b32e40a45ecac46ac0c5a030f5c", "6d0eaa028bd347cdb57df3246a311ce4", "8bb0fbc473d54c3390273d9a6677a16f", "8fc354852cb74a7c9a429e3cf9a55f37", "a231fd9aa67c4b709ef00a63f797d629", "f0c2ab054ff64df586a77fde3f2ca1aa", "19a88715c6004d86abecb4549d48b903", "9a9f9454f30647e1a1ddadd866fe8580", "4827bf92998448ef8c88d8ca33408326", "bfbd470a1af84d64817e8c2abf6e5451", "1718274132ac461ca3b96e68836a76ca", "a52831c69459435c91cd6357e52f42fd", "51de3f52eceb4a07b1d66e00f1845d25", "b62f2ce01e8e4fb6ae5be92459465c92", "eb84922224964072bc11b268b86763d5", "fe4d5b21f18d4148b3ef6773375f0d3c", "3c570cb9b5dc48329cf94be364c75f6c", "070840196df4477789d6af4d58373ab7", "fb70b3d98cd144508e41b38a1db05b45", "3b03f6a558e34d299d90086bf77bbcf2", "16e04d763b324a70aaa436af7b73380a", "4746ff54d8bd4edfa77be3b378861532", "272493a58eb549d3b53ad78ab466145e", "7ccfed68bffd42ac9cf7c93b0035dffa", "31fea2a00fb8480aa52467d1522d960e", "414a7e7aec07440c9a3d6d9cfbb05859", "836e8811049646b8a3cf1796a14f5640", "8511ef3ccf90447ca2a4dd1cffc22020", "1791e9e59ea942eeb9392c03a78cc18a", "4c85457f85e84973908bbf89b347a0f2", "3f3eb87a735e48a8ba72a354de0788a1", "330556f463c344979c39479efd437bbf", "f02d8f12e6e94ff9b3ee7dc1b16d37c1", "c60b780570254e7580c1de2413dadea6", "291295a0fadf49ef8288e7fe3de1e7cf", "551421d522ef48dd9b73a1d35b460d41", "1f6da62f669b46bf8e863e1203723f09", "e6db9f79b2a7491aa8083b4c60c69738", "e6a1049b883d4c97bfba773922048fb3", "60733bb99dcf48b08013081b505c4dd8", "727ac66f25604b828ba265f28a95f385", "ccc6ccdf40cc465d8b17cbd387e8ca4c", "e1987f35adb54c0ea321d3e5b48d2415", "c71c3ef93aa943189d3a9e852ced19bf", "8cf5082accbe4c8e86ffe9b6feea1789", "8d42fc0623d9405a905fe369d4db6ff5", "a0442cc1502645819395b5635ac3ab10", "36c32d8b2bc040e4bcf103a5e31bbd7b", "368b98960aa94aa3a9b9b6df346fbd76", "16537135732a4ddc984bd2be4673c46c", "dfdb00d35b9f4dacae12a009fe862da4", "a2c8fc40705f494997c807c1b543789b", "709da65cc91e4d4da9575a4a2074bc5e", "52f09ae496714ad19769d7fbc019dba3", "3df55501ec30435da03292465927b683", "6e606e39ce1e422ba4d5c815485c61fa", "a3f55bbe815945408cd109757dda24f1", "4f51f64197b64fee8c50021ab3145ace", "ca8975123f1a44af9e06e89858ac3d61", "3f81924f55584a398de343a67b8ec1e5", "bfa521d1a5474410a4bce80be8ba6c5e", "dfdd4cccc5274446a6459a35a48e21b6", "a429d1d03cf341e59010014aa038d4d2", "87ccb5d262c049f78219bb183c5358b0", "80df90d1aecc48fdb7565233802f8f43", "b9104c49569047c88bb7e11b45a71c4a", "a8a795bd11434481b52585646b9057cb", "aa39c34f95d546ba98fec173b633ea15", "a2c78937984e489598dc4e9a043e0948", "d2f0b589813c4f1492da0afdcc3588ef", "201547634cd946d1b84d1922b8e4080a", "257c51ec4f8849aabe67fc07fa8bc510", "0348aa735cd945b1b490352c856df3b1", "3e6fc820ac93464b8cbe529c6e8863b3", "54b3053a68bc40cba799c1f6211729b2", "04d50d0f06f7461f911a3bc54e316f9b", "ae2f2d96efcd45549b3d0231748d3cae", "727edcc2c6e74344b0f75998fe909bad", "798db7ef9d3d451a893ac94b6b04657d", "91c81ee959f74954bcaa0d9725033cc9", "cace9d0ea44e458d9ce3a50a26b091e2", "cf71b636053d494dad5dac53fb21d96b", "a2984685d7aa4f5ca35e17185c41f024", "a33d1a68c856452c80446bf491cbee53", "67adc72d86e04f6b8c0228ea4300e90a", "8f5460a7452d4edba9c059a8e838ad49", "0d954c56ae5b424784cf67ab9a7942a2", "968f12e53d3e41a5bef8effa725d42fd", "f65f21301cdf4a1c84d579f8807be76c", "0e03285f11bd40dd99f7e8dfd3f0dee7", "85aa0978604f453eafd476d3a44b994e", "73dcee079dec414db908ee2bbe155ca1", "3816970f74e344859b58529080edce62", "66999f3e61f34aaf88879d3e68fe7e75", "345aa11de23a43c5b2044f141045f624", "09c25edb039e470abea89b8c06c0bfef", "624f11a621f748b19954155ab0250637", "0aff8ec3c76f44ff89fa315687cd349b", "0174c4af58f547db9c066c53c5937157", "9b8279f7019742d1b96c998c6ba0b14c", "d40a1bee6e8a4f2cb1a2038d99f2d2eb", "c9ddc9d1747547bcbccb34263be20bfd", "d9d2cca602894a17a358f8c3c3372718", "7de06af5aee44286a98f702ab1f99f5b", "eee2c6e7c31f47c2a7f2e41d62b914d1", "cf4261ecfea24fdd9982b5616c2c3855", "8987c37c7f834ff29fd6796d74b3c183", "4c63efb7821c4602aad7dfed923f7c72", "e5ff8e7620fe491fb7adc1600e2bdb29", "49603a255a1240ab8c44d20965e57d60", "2839eed273cb4ac888c698feee20baaa", "bdab1fee268649f199711dd61bac0fc7", "f276b741c9ac4e6dadd4c8af2ec41a6e", "40b3488d9ce14a1da186686923c5bb01", "926a375748574ae3b853031033c189d6", "ef69926e5a8a4ad6aa4ae95b023aec63", "b5f7d7848a33492c90debf1bdf4721fe", "57684c49cc444f1684e0564769f36333", "50dd8a42ddc94c869fcec3c59efe3237", "98f813bc97a549438159951ac1602325", "6e40c8506c0c4589898b53f889828065", "d8b0eb22e4e54fe3965022f6d25768bf", "72d59710c6c745c68984899c7091c979", "5e475092fd304ab492f6105047660f82", "064937203cec47e6a2d00c944658b972", "8e3ba17b783c423698e3b8fd4c53d920", "5221f909fddc4d6cad87b63930068eab", "cc45ed2957af469e9911a5c73f97f8b7", "b4f56ffa28274de498cf772d6498894c", "c6ea94e205634bd6971268dae55ee196", "b229da4dcc8e4d359a5f0827e97984da", "e11d3f163b5a4b968284a232dd61e7eb", "137aa0663ecb41a0afe7f429d6573c1f", "a185399b419b44ecadb7c1d7b7b00949", "4ea599e4f50541f7884d69957dff0b67", "e7c5f8ca08b644cea90a51e04518faa4", "2bab5ddfae59469db50f23ae7f5f0aaf", "002d7d275cf5460a9e299e058e934833", "77dd786c67574ecbbf1d0d6b98d08f84", "44ed8464ed2f4fab885d44d8a3f74109", "ed954992c97c4267af321dd11b235e12", "72ca391b07a740ccbd7c38000afc15bf", "b8ea89cf84f949178c0c9a090fc84994", "3484f0e9730542768e2bb2943d4dd241", "1491e04215824be5b1d4be42375757a1", "f9be9a32f3904d208fe6fc834839c2c4", "1aa18116178e4ccfa09a9175c4afface", "2985c4882302468096fd28b8248b51f7", "97140722f4654d7eb6cdc5d3af061b85", "6fa1d89cf4bd402a8d233a1c9d66115c", "e76853d817eb4aa287112e06776f64b4", "2bc71f474c714166805496a5f7bd9d17", "958a7042a6174bcab47875d148f73597", "a5de57763c4d4c8686222597d1bd9d45", "dd6bf36bfa66427680dbf106223b7e18", "a8f1965f964a4c9e8454fc7f8bfbc2e1", "5604d37741d24ccea39f49cc65dfaa2f", "2e386420b4ab44a4b229ec348021f0e1", "97e31cdb02d24597bc09898816bd218a", "c6837f325ce848f1ad1f0e588895b255", "620ad69774024b8e84f92714284d5ed3", "ff59ad75e99e42518a55a1f5ad843605", "f21943a17e4c437bb5adaca749572e96", "7e685519de8b4c31a57e4644dfc9ba83", "7e06bf0c6fc8485294eaf7c25aaf8dc8", "41873ef9866c4e188623956e7dd8b659", "d8db12edd2ac4c6a93f9ee9b1ca1b3a3", "d76d412b744e4858a7929035ecbe568e", "342894ee98454eada0e4b3cda2e0595f", "7189c070e5dd4adabc207e139972469d", "9d54650269834501b9bb3acdd083f3f5", "5aa7b4de282a4cf69a94945067f7e4be", "1631a853f8304f02ab0cc87a5bc80840", "b55c43d3b29a42f79fc070488433ecc5", "cffc023362fb4a4298eef75e913e9e41", "8b1042b1fc834ef3951773234f16f840", "4849156c7f484bc6b81e7e7ec6edc4a7", "499a5f25adc84b50800f5f7a7e3cf5eb", "aa8127ce4fdf4a869e6cf8e8e64adb0d", "ef0033c9100b4c40827f3f2f4121e468", "54269827120f4f248c676309be6a4e6a", "f9d90a280ebf486e9043bf4a385fccbf", "70caa3aa1a8c4ed4b186bbc20268fff3", "b1cb3815b3554bc3957577ea9ce05975", "fb1a08265619482db9a80c7d89ed0932", "3aa1bfc6960e4ae1a7ad1c359c5ab468", "57a1eb3d686a4153989a385cc7095cb2", "55501712f860436f8f14465955929f08", "dbe4ccaf0f764563911b3a9b5692c926", "0b754fb586174b4287ed56cc9eeaeb56", "b528456340184ebebbc6d5804746a124", "13b6acab5dba4abaa42a8d2988745f83", "5557805192334fb18ea3fe64018da3ef", "71e9c921a9774fdb987579257f8990b5", "b185f3aca70d44f084421a1dbefe2283", "3b4b2ac11ba84a94b972d0c5d91d18d8", "6d8d18aa20054a7dac95bdd990ee1c48", "cf01211abc8d4588b23e7103a28d2760", "845f90c39cca4a8aa3bd8d88ac16b59f", "81cda2bc9a6048e2b16ea220537f2495", "32653f3816a4491aba17136369fcd953", "12ff5e3d64304769ab5190260c8c1e4e", "c33fc51969904c31905b7a2dffff72c7", "9eba214afcb64e649b070b7012b56dfc", "36a7f3345cdd4c888f284ba85a803c70", "5ac7e9f694e14925a0e9639a288c438d", "7bc4dfdd80a24d918fec2736479b4030", "e4b3ec3f539f4ce3befa123dd2109f5e", "41afd9dcb3ea4a16843b6739b708354d", "00b390f06e04496ab2045b5141e138f7", "8fb09fb539ef419d822fbfe0c3a51d7f", "aabbafd9bf0744349c9c4331d9002bea", "46897c7e138e445594c7c6886347ea49", "f432b772f97545e5bdc9ea6458d9b43c", "b6311e15acb34458af1aab5f2ef4afb0", "925199a16c504950bda677c78f16b600", "6b83b07cbcb145f194955ee8770bd1f7", "34ae354bf1954baaaee5beab0d2dc4be", "637ac10e5ce84d58a493fcd48a3f36c5", "f1dcdf646d204bf39d77e7f4f83ef71c", "da0e45f3759049788ab8c6af90b6e35c", "d08251c16ad74c35b0f00ee07f9416de", "927a3092b90d4308b38ecfa29228ba60", "6773b3d844564267839dba93b6328ce2", "10ee445b87cc43da9184ca3b2dd9d8a9", "61955445d4a2429e9c87760e0f23b9dd", "70fa002f2db44ea5a8dd0fb0243229b0", "5845dceabf79465bbec15e93dfa6b152", "fddeb9eedd194235808c30514699fe13", "6678bfed9a584c3daffc1a5250f7665b", "d9bc22e0eff5428c81c92c587bd7d6fa", "e6807616ad39465590ce31c6291b4709", "7524cded64b04066a6e32bac9a578b33", "35180147f2674c71a93ef1cbbda900cf", "298bf24f3d014dc6bba763ba49fef061", "8c5f271d7f3d4635b1dff16898de277b", "cd5a01854212492c8737facbd099c322", "4ec1485401f7441aac2f2158970224f8", "5c83ed4e0f2b4b6aaf9fbb7427a6bc4b", "817cc8e1b5f5476dbcb764e091619d20", "05ec94fc5dc542eab3ca50068454002f" ] }, "id": "FJCaeC67kaWS", "outputId": "766004ca-8447-4eba-eb11-efd60d4bdd2f" }, "outputs": [ { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "926f1a8556aa447eaa5d4550248bbe6f", "version_major": 2, "version_minor": 0 }, "text/plain": [ " 0%| | 0/157 [00:00:78: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature.\n", " model.load_state_dict(torch.load('best_model'))\n" ] } ], "source": [ "# Create the dataset readers\n", "train_dataset = ClassificationDatasetReader(train_data[:5000], bpemb_en)\n", "# dataset loaded lazily with N workers in parallel\n", "train_dl = DataLoader(train_dataset, batch_size=batch_size, shuffle=True, collate_fn=collate_batch_bilstm, num_workers=num_workers)\n", "\n", "valid_dataset = ClassificationDatasetReader(valid_data[:1000], bpemb_en)\n", "valid_dl = DataLoader(valid_dataset, batch_size=len(valid_data[:1000]), collate_fn=collate_batch_bilstm, num_workers=num_workers)\n", "\n", "# Create the optimizer\n", "optimizer = Adam(model.parameters(), lr=lr)\n", "\n", "# Train\n", "model, losses = train(model, train_dl, valid_dl, optimizer, n_epochs, device)" ] }, { "cell_type": "markdown", "metadata": { "id": "s2Jiw-ssYc-u" }, "source": [ "Next we can plot the loss curve" ] }, { "cell_type": "code", "execution_count": 44, "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 447 }, "id": "B-zuTelyYZgb", "outputId": "687f4f01-843c-4c37-d395-d30ad46765f8" }, "outputs": [ { "data": { "text/plain": [ "[]" ] }, "execution_count": 44, "metadata": {}, "output_type": "execute_result" }, { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAAi0AAAGdCAYAAADey0OaAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAABcnElEQVR4nO3dd3wUZf4H8M+mbRJIg0ACIRA60kIPoYhIBBX7qRwiIIfYwBYbqIAdfnpynh7KiSJ2sOsJIhCpEokktNBbCC0JIaSQkD6/PzBLNpndnZmd2Z3Z/bxfr7wku8888+y42fnu92kmQRAEEBEREemcj7sbQERERCQFgxYiIiIyBAYtREREZAgMWoiIiMgQGLQQERGRITBoISIiIkNg0EJERESGwKCFiIiIDMHP3Q2Qora2FqdPn0ZISAhMJpO7m0NEREQSCIKAkpIStG7dGj4+zudJDBG0nD59GrGxse5uBhERESlw4sQJtGnTxul6DBG0hISEALj0okNDQ93cGiIiIpKiuLgYsbGxlvu4swwRtNR1CYWGhjJoISIiMhi1hnZwIC4REREZAoMWIiIiMgQGLURERGQIDFqIiIjIEBi0EBERkSEwaCEiIiJDYNBCREREhsCghYiIiAyBQQsREREZAoMWIiIiMgQGLURERGQIDFqIiIjIEBi0EBF2nijE0t+PQRAEdzeFiMgmQ+zyTETaunnh7wCA5k3NuDG+tZtbQ0QkjpkWIrI4lHfB3U0gIrKJQQsREREZAoMWIiIiMgQGLURERGQIDFqIiIjIEBi0EBERkSEwaCEiIiJDYNBCREREhsCghYiIiAyBQQsREREZAoMWIiIiMgQGLURERGQIDFqIiIjIEBi0EBERkSEwaCEiIiJDYNBCREREhsCghYiIiAyBQQsREREZgqKgZeHChYiLi0NgYCASEhKQlpZmt/xbb72Frl27IigoCLGxsXj88cdRXl6uqMFERETknWQHLcuXL0dycjLmzp2LjIwMxMfHY8yYMcjLyxMt/8UXX2DmzJmYO3cu9u3bhw8//BDLly/Hs88+63TjiYiIyHvIDloWLFiAadOmYcqUKejevTsWLVqE4OBgLFmyRLT8li1bMHToUNx1112Ii4vD6NGjMX78eIfZGSIiIqL6ZAUtlZWVSE9PR1JS0uUKfHyQlJSE1NRU0WOGDBmC9PR0S5By9OhRrFy5Etdff73N81RUVKC4uNjqh4iIiLybn5zC+fn5qKmpQVRUlNXjUVFR2L9/v+gxd911F/Lz8zFs2DAIgoDq6mo88MADdruH5s2bhxdffFFO04hIBSZ3N4CIyA7NZw+tX78er732Gt59911kZGTgu+++w4oVK/Dyyy/bPGbWrFkoKiqy/Jw4cULrZhIREZHOycq0REZGwtfXF7m5uVaP5+bmIjo6WvSY2bNnY+LEibj33nsBAL169UJpaSnuu+8+PPfcc/DxaRw3mc1mmM1mOU0jIhUI7m4AEZEdsjItAQEB6N+/P1JSUiyP1dbWIiUlBYmJiaLHlJWVNQpMfH19AQCCwI9IIiIikkZWpgUAkpOTMXnyZAwYMACDBg3CW2+9hdLSUkyZMgUAMGnSJMTExGDevHkAgBtvvBELFixA3759kZCQgMOHD2P27Nm48cYbLcELERERkSOyg5Zx48bh7NmzmDNnDnJyctCnTx+sWrXKMjg3OzvbKrPy/PPPw2Qy4fnnn8epU6fQokUL3HjjjXj11VfVexVERETk8UyCAfpoiouLERYWhqKiIoSGhrq7OWRD5qkiPPzldsy8rhvG9BAf40T6FDdzBQDgkVGdkXxNFze3hog8hdr3b+49RKq575NtOJZfivs/TXd3U4iIyAMxaCHVlFXVuLsJRETkwRi0EBERkSEwaCEiIiJDYNBCqtH/kG4iIjIyBi1EZMG9h4hIzxi0EBERkSEwaCEiC/bwEZGeMWghIiIiQ2DQQkRERIbAoIWIiIgMgUELERERGQKDFlKNAfbeJCIiA2PQQprJPFWEuT9moqC00t1NISIiD+Dn7gaQ57rhnc0AgLySCrx3d383t4aIiIyOmRaVCYKA3/bn4uT5Mnc3xeVMJvH1VA/klLi4JURE5ImYaVHZb/vzMPXjbQCArPlj3dwaInm4jD8R6RkzLSrbeqzA3U1wGw7EJSIiLXl1pmXXyUIUXaxC91ahaN7U7O7mEBERkR1enWmZ8+MeTPwwDRnZhZbHVmWewaQlaTh3ocJ9DSNyE+bKiEjPvDpo8fW51INfU3v5o/qBzzKw8eBZPP9DpruaRURERCK8O2j5a7aL2FiMU4UXFdXJcR1ERETa8OqgpW6Gbo1IoLHrZJGLW0NERET2eHXQItY95Cxba5UQuVrmqSLM/2U/LlRUu7spRESqYNACwF6Pzp7TRXjiq504rbC7qL6zJRWoVTFAIrLnhnc2Y9GGI3h91X53N4WISBVeHbTUZUXsZVrGvr0Z32acxMNfbnfqXBsPnsXAV9c6XY+eMRzTp31nit3dBCIiVXh10OL7V0/OE1/vxODXUjDqzfU2yx7MlbYUva2BuIs2HAEArNh9RlYbTxVexFfbTqCyutZmmeqaWnz0+zHsz+HNiYiIPJdXLy5X1z0EADnF5fYLuymNMOrN9SivqkVOUTkeGdVZtMxnfxzHi//bC4BbB5BzOCKLiPTMqzMtcgbNSo1Z1B6IW151KcOy+XC+zTK7TnGmExEReT6vDlp8OdOHiIjIMLw6aDlfVmn3+frjSJQsGrdmb67sY5QwMalPRERewKuDFkc7Mnd5/hfLv6WGLPWDm2mfbFPSLNmYMCIiIm/g1UGLHGqvzn/07AUsWHMQRWVVEhug7vmJxPBtRkR65tWzh+QQJHycl1fV4Jv0k5Lqu/bfm1BZXYtj+aV4Z3xfp9rGRAsREXkDZlokEgRgxhcZuGXh7zYXo5v/y36cl5g5qRsvk3H8vLQGGCEy4dd0IiLSEDMtEgkAft51aWG43aeK0Cc23PLc6cKLmPFFBjKyC20er+W4E45pISIib8BMi0T2VqSd8+MeuwELoP6YGF1i8ERERBpi0KJAw3tzcbnEwbRERESkmFcHLY9c3UmdiiRkUZzuwrFzDt2s0+IN2SQPp5N3EhGRKK8OWmZcLb6XjyNKAhCv6B4iIiLSkFcHLQF+yl6+WzIbdk7JgbhEROQNvDpocSXOHiIiInIOgxYFtAwSSsqrFO1zpGsMqtzK095OROS9GLS4ga2g5FBuCXq9sBr3fiyyZ5GRbzxGbjsREekGgxYVSFniv773Nx4VffyzP44DAFL25zndJi1U19Riy+F8lFVWu7spJIOczCDjSyLSMwYtCtzwzmb0fWk1yqtqAAB/Zklciv8vb/x6QP5J7d54XNP/8nbKIdz1wVbc/2m6S85HRERUn9cHLa/d2kvRcefLqvC397Y4LPf51uMYMi8FR/JKFZ1HClcNxP1sazYAYNOhfNeckIiIqB6vD1ruSmir+Ng9p4sdlnnu+0ycLipHTnG55bHqehsuetLMH3YtEBGRlrw+aNETk70Ixu6KuNbySsrx4eZjKJK447QrZZ4qQrrUna1lqKkVPG/WFRERWeEuzzp1KLcEWefKFB179wdbcTD3AlKP5OODyQNVbpkCf0VVgiDghnc2AwAyZl+DZk0CVKm+sroWI/+5Hq3DA/H1A0NUqZOIiPSHmRYAV3dr6e4mNHLNvzZi2iciU59FNEzQHMy9AABYu8+1s5Bs5olEEiD5Fyos/71r8R/4aedpxefdc7oIpwovyh4QTY15UG8lEXkgBi0AXr+9NyYntkOLELPsY38/7KJBqTq4mzjqflHSOfP6qv3YcuQcHvlyu7JGkUPsNSMiT8GgBUBkUzNevLknNj09UvaxL/5vjwYtsu2j349hVeYZl55TS4U6HHdD+lBWWc1xSkRkhUFLPYH+vrKPqeuKUUrO7KF9Z4rx4v/24oHPMmyWySkqt/mcnrjyXvROyiGMenM9zpdWuu6kOmLEGWrHz5Wi+5xfJXeREpF3YNDSwK4XRuOTfwxC2nOj8FhSZ5ee2+7NRbg8DsSeCxWXMxdq36zszm5yI0ftenPNQRw5W4r3N4mvREz680XapTWBXD0ui4j0jUFLA6GB/riySwu0DAnE/Vd2xPDOke5ukl2CIMCkhwEvsDPmxTJ7yHVtEVNTy64GIiIjY9BiR1CALz6dmuDuZtiUfa4M/V9Zi0//2rNIbYIg4Pi5Ut2PK9B7+4yEV5KI9IxBi47YzZiIPPV/v+5HQaNxGuplXV5dsQ8j3liPd9cfAeA4OLDZTaOTOyGDGyIiY2PQQjZ9sPkYAIUbPLqQXsfaEBGRuhi0GIXCJEFVTS2KLlbh620nUFyu7fRivWcydN48IiJygMv4u5mW65QIAtD1+V8Q17wJjuaX4pfMHCy5RwfL+gMQ/orCmCTRHoM1IvIUzLS4WUl5teXfjm7gSmYJ1QrA0fxSAMBv+907fZT3Tv3TTQzJNwsRiWDQIsHSKQMx98buyJo/1q3tEFT4JL9QUe24kA2Kx46IHFYXgKmRBdDNjZaIiDTFoEWCq7q2xJSh7TWr/8O/BrzaJfHO7Ciu6Dn3V5woULZ7tJpjVtQIwC7XRfZ4SxfcCz/twT91PmiciJzDoEUHXv55L/KKHSy/LwAVVbVWDym9F32//ZTDMqoOqrVTlbfcUEkmme+LEwVlWLolC/9Zd5iLCBJ5MAYtMoUF+WtS78WqGrvPH82/gHsb7MNy9Gxpo3JqxRobDp5VpyId4a3Mc1XW1DouRESGx6BFpldv7alZ3fa+XOZfaLzZ394zxZq1ZcuRc5rVLdWh3BJc+9ZGh7taM1lDROQdGLTIlHRFlCb1qrV/kJTuFnfd5OV2OT22fAf255TY3dWaxGm9Jg8RkTsoCloWLlyIuLg4BAYGIiEhAWlpaXbLFxYWYvr06WjVqhXMZjO6dOmClStXKmqwuwX6+2L+bb1Ur1fNgamOvLnmIE4VXrRbRklgo/YrKHVippMYb1mvJP9CBXq/sNrdzXAbvS9ySETKyQ5ali9fjuTkZMydOxcZGRmIj4/HmDFjkJcnvgZIZWUlrrnmGmRlZeGbb77BgQMHsHjxYsTExDjdeHcJ8FM/QVV0scqybL4rvPjTHpedS2lqR2yKtSAIOHehwskGebbNh/KtfpdzD9fN7V43DSEiPZG9Iu6CBQswbdo0TJkyBQCwaNEirFixAkuWLMHMmTMblV+yZAkKCgqwZcsW+PtfGsQaFxfnXKvdrHPLENXrnLdyv6zyzt64Ha7XUi9ecPUMn+qaWvj6mETP+8TXO/FdxqXZTy/f0hMTB7eT3D6jzFQ6d6ECucUV6N461N1NMSTuRUXkuWSlDCorK5Geno6kpKTLFfj4ICkpCampqaLH/PTTT0hMTMT06dMRFRWFnj174rXXXkNNje3ZMhUVFSguLrb60ZNebcJUrzNb5topjy7bIfp42rECScdrkUG3easQOZet85+7UIHeL67Go8t2wEfk5lMXsADA7B8y7dYl9Zx60/+Vtbj+7U3Ye1pf73siIneTFbTk5+ejpqYGUVHWg1GjoqKQk5MjeszRo0fxzTffoKamBitXrsTs2bPx5ptv4pVXXrF5nnnz5iEsLMzyExsbK6eZhiS3H/6Po+Kze2Z9t1va+TTIvzuqUez5hgOQv0k/ibLKGvy087TXzwqy9f9YLjmJB0+45hzTQuS5NJ89VFtbi5YtW+L9999H//79MW7cODz33HNYtGiRzWNmzZqFoqIiy8+JEye0bqZskU0DVK3vdJGDxeUacDYD7uhzvX4wocWNrK799oInsUyL2g7nlWDp78dQWc11PnRF5v96Twi2iMgxWWNaIiMj4evri9zcXKvHc3NzER0dLXpMq1at4O/vD19fX8tjV1xxBXJyclBZWYmAgMY3f7PZDLPZLKdpLudNX+a0GCMg5fpJPW3RRWnTe8UCpKQFGwEA5dW1eGBER2knJN3xoj9HIq8mK9MSEBCA/v37IyUlxfJYbW0tUlJSkJiYKHrM0KFDcfjwYdTWXv4me/DgQbRq1Uo0YDEKo39IOmq/WMDg7GuWu7y6lGCppLwKk5bYn3IvxY7sQqfrICJ5KqtrseVIPiqq7a8ITlRHdvdQcnIyFi9ejI8//hj79u3Dgw8+iNLSUstsokmTJmHWrFmW8g8++CAKCgrw6KOP4uDBg1ixYgVee+01TJ8+Xb1X4Qbu7jdXazE62/XLk1tcjrJKGx88f1W2LC1b5CnbZ/KR0IgDOSVWv7v7/4scxeVV+HVPjuof2K5c80czHvASyLE5P2birsVb8ex3me5uChmE7CnP48aNw9mzZzFnzhzk5OSgT58+WLVqlWVwbnZ2Nnx8LsdCsbGx+PXXX/H444+jd+/eiImJwaOPPopnnnlGvVfhBhVGHwMh46YgJYB5c7Xj3XXTZWYzXDlztVYQMPHDrWgVFojXb493yTnvXboNaVkFuGdIHF64qYcqdR7Ou4DHl+9UpS6jYrxjHMv+vDRe8duMk3jzTtf83ZGxyQ5aAGDGjBmYMWOG6HPr169v9FhiYiL++OMPJafSrfaRTbDHS6akVv/VrWMvhqiqsX2rOHq2FHkl5aiWuamdK5Mme04XW1YJdlXQkpZ1aXr6t+knRYMWJUHblKWNu8oMlHxSjANxibwD9x5SqEOLpu5tgLOzh1z8fXTQqymW4EeP5I63cQUlwcaJAvvbM3gq/f3fIyItMGjxUg6nPGvw1bVW9kBc+eew97rsPefORVR5wyUikkZR9xC5n7PrijicPSSSynH25lo/0/LBpmPYeuwcIoKNO4PME3lCAOUN3WFE3oqZFnKZ+veSbzNO4uT5i9h9qsipOrnNjDWx66HVNcopKsePO06hSuZYJUm4uBwRiWCmhVQh5aahxY1FrW/V7rzp6WmatpzrkLRgAy5UVON0YTkevIoL8xGR9php8VJKbpR2x4tIOV72GeVTeg5P2RnYla+ibqfwDQfz1K9c5v9I/YR9RKQlBi1eSsmKuFKXy7d5Tg0yCp4Qa5TaWJRP0UBkJ9viCTxicT0iEsWgRaEuLd085Vljcu+X7ood1IqD9NRFU0eHTRKlh3Z6QOxKRBIwaFFo2pUd3N0Ep+w+WYSiMumZE+4N4lhNrYDUI+cs3SbuIHbz1kNQQUSkBgYtCgX6+2LGyE7uboZi1bUChr/+m+0CDfomPth0TOMWKdOwC0VpxkSNMS0f/X4M4xf/gQkfbHW6LiJ75q3ch3+vPeTuZhC5HGcPebHicukZgSNnL2jYEtfQugvo620nAQA7TxRqeh6yz9MzS6cLL+K/G48CAB4a2RH+vvzuSd6D73YneMIgUCORu6KuHEr/X+ZfqMDB3BLHBV3EU2ZBkW2G36yVyAkMWshi18lC3LkoFduzzzd+0k68kLx8B77bfsph/XK/ATdclffpb3fJq0AhORmZAa+sxeh/bURWfqnq7VAr/jBkHCOhzWdLKjQNZIlIfxi0kMWd/01FWlYBbn13C/ZK3MG6tKJaUsACAPtz5O2K3XDq6jfpJyUc4x47ThQaMzhQgSbX3EGlW47kY+CrazHtk21anJ2IdIpBC1mUV11OO6/dlyvpGDk3rPwLlTJbJIX6kYKSMRFarA3ijrEZRslbLNmcBQBI2a/BwnYG4unjd4gaYtBCTlEzZDh3oQLzftlfr24ptavzqe2tWRIyHr5VyZsxaHGCN314uOIL3b9TtJ3CqfZrqHHTeIqii1XIKSrXpG5F72l+23cpXm7yZgxaSJKGg1PrflczQ/FJ6nEFR1k3QI10udQqZnyR4fzJHFi3Pw8fbrZeIyf+xdUYPC8F5y5UNCrPxeW8CzOE5G0YtHi5EwVlkso1TCqsyszRoDXOyxe5kWvll3rXQIvAwGQCpiz9Ey//vBfpxwsaPb/3jLyBzZrhjZOIXIRBi5cb/vo68SnODTS8Jx/Ku7TY3EUbm/25y5D5tlf5lRpYqLED9qsr9mKmilO0c4pcF4zJprNMjjdllrzptRIBDFqc4yG52f/tPOOwjK0b+cJ1R9RujsXuU0Wa1d2Q2KDf1CPnsHavtFlUDS3edAzL/jyB7HPSMllEUnnGpw6RMgxaCIDj1WYbPlsXw2SdE19UbUmDcRh6l12vm6zutY5f/Afu/WQbBr+Wgm8lrBEjthptVa3nr16allWAcf9NxenCi+5uChF5OAYtBAECjtkIPuoVkuWln/cqb5DO5BSX44mvd+KaBRvw3w1HsOHgWcnHOpO+Vyv174qE4NZjBXj+h0ztT0R6640jcilumEiS1Da4g3pIz5goW8HCobwLlnVksuaPtT5G60ZJ5M7/L+fLtFg8UD4tFvojIn1g0EKSnCvVxw1JqrLKatz27haM6NrCZhk5GRMpxOMF5TdQRwEIB2Fexo0iibwDgxYneNPH5J9Z1lNuBQE4XXgRv+l0GfVv0k9if04J9udc3oG5/jfwY/mlmLwkzR1N81juDKKUzPgyKm/63CFqiGNaSBKxe8LUj/W7WV11jf2bmK0BxIDa3Qva3WLEkgtis6DceT8/kFOC447GSyniPUEKEV3GTAspYjIB+/SyuJkIV9/SbH/Tl9eSDzYdlXFOsbPp52Z+vrQSY97aCKDxGCAiIiUYtBCKL1bLPkbv2XhnuguU7fIsXXVNLXxsjMF4ZcU++SfXqVNOTIF2fD1tZ7D0/t5Uk56CVCJXYNBCKK/W16q2cmgx/jK7oAxdokJkHbMsLRullY2Dv/o30NpaAb/uycH0LzJwRatQWfVLfZ2Xuoe870bGgbhE3oFjWpzgKZ+TK3adwTqdDqit48y3drle+GmP7GMysgtx3MHqt1+nn8CDn2egVgD2nJbXtVYX/NTfWdpT3n9q8KaBuETejEELAdB/t8T49/9wuo68Ymn791RWa7OK7br9zk2xLimvQsJrKZbfRe/TYoNzRR4rr6rB9uzzDldCdhfGY0QkhkELKfKvtQdder5sibtR27N6by4+dPH2ApU1ygOgF/93eVVhkwlYufuM1S7WAoDnvt+Nj7dkya77oc8zcOu7W7Boo3Z7R7krE6TPMIyI1MCghbzKyy7eXuDd9ZeDArVv4luPnsPnW7Mx96c9OHm+DEkLNkjOEtWtr6Mk4NGHhis0e2duhr1i5G0YtBCJKBMZVKvEhgPqrLordnMqKb/cxhf/txeH8y6oUq9SvH+6hpfGZ0QAGLQ4RWwhLzIGR//nFm2Qvl6Ku9Sf7lpeZXsGmGd+G7f+P+hNA3Fd9VI/++M4bnhnE86WSBsLRuQKDFrIIzn6YHf0uZ9bVK5aW9Twn3WHse9MidVjSm5emaeK8MvuMyq1SjvO3Jf1FMDoqS1yPf9DJjJPFWPBGteOXyOyh+u0kKHla/AtUMptRurNqH45Z9L6+84UN1qBWO7tsLyqBje8s9mpOpQSBEHTcSd6HNOy6dBZzPhiO+bf1gvX9Wrl7uYoZi+LR+RqzLSQod33abq7m2AY9cfA2KLmvd/bu08nfpiGootVePDzDHc3hchjMGghr2TvdqpmSr+upvKqGqzcnaNavYA2YxuM05thmIaqTodJJSKXYfeQE/jhoV/O7Mmy53QxMrIL7dcvs/r/W7VfcXukkNI9ovSaCIKAiR+moYnZF/+dOEBRHXI586flveEMkedj0ELUQIUGK+JK2SahtEL5NGt72SF78YyU4OtEwUVsPpwP4FLGKNDfV27zIAhqB/ne+43BONkwIvWxe0hFwztHursJ9BdnZw+5w3Pf75Z5hGteRf0MjZzAQ80g5Zv0k0g9ck69Cj0EAxjyNgxaVNK3bTgW3d3f3c0gF5F6r5BzU/lhx2lFbQGcmT2j3V3PmRtq/UN3nyzCk1/vxPjFzu8/5QnYLU3ejEGLE+p/dnz34BA0MbO3zShc9bkvQMCuk4XIcrADNCD/ZiQ1KHDZt3E7J3KmCSfPy9t36j+/Hcb+HHm7aBORMTBoUYke14kgfbjpP79LKic3uKiVccDB3BK8+auyRcKUBj1q/UmIn952o97feBTXvrVJnZMTka4wNUAeyd59duvRc9juYHaQw/ol3sm1GNSrxOh/bXR3E0gDzsySIzIiBi3kdca977qxEVp2zbhq8TapGZPCi1VYuzcXw7tEwuwnf4YRcCkY3HLkHPKKy+s9pqgqIvJA7B4iUkAP91E1vmWrGRAcP1eGez/Zhpf+txeAdbAjNTP12dZsTPhgq4RByeyOdRU97Z+UV1KOOxZtwffbT7q7KeQmDFqccOfAWADA6O5Rbm4JeSOp95KCskrJdUrJqpSUV9nd+ffzrdmSz1dfRvZ5zP4hU9Gx5B3m/7Iff2adx+PLd7q7KeQm7B5yQlRoIA68ci0CfBn76Y2Ovhy6RPa5UpvPHT1r+zlHl6m6pha1DQr1emG1jJZJ9226Mb49V1TX4I1VBzCyW0sM7eT5azPpaZJB8UXlCzCSZ+Dd1klmP19d/VGTa+hhgG39WELKlGpJddartKqmFle+vg43NdgZWlG99f4td9dgVw02La+qwc+7TqPQQWbqky3H8cHmY5jwwVaXtMsebwvO9dExS+7ETAuRAos3HnV3EzS/YR05ewGni8odF5Rh06GzmPhhGp4c3QXRYUHoFROGrtEhdo8Rf53qv/h5K/fh49Tj6N0mDD/NGGazXHaBOgGiUVRUywsyibTEoIVIAT3cuLTKQFTX1KKqRpu6Z313aauCf66+vGZM1vyxmpyrvuLyKjz51U7c3CcGY3u3Ei3z485Lg393nSzSvD3OcNWssTpiu5MLgoD7P01HaJA//nlHvMva4n2ZJWqI3UPkkbTuUtDDjAo1bl5ir+PqNzfgijmrUFphnG/Yjq7Ef347jNV7czH9iwzN23I4rwRzf9RuQLEe1mY5ll+K1Xtz8U36SVTXuK6r1P2vnNyNmRYislKXRco85VzGoar68i1GSownb2iYdWFH1edfsD3bSbxG2xwFDWPf3uyyMU/uuonXX43ZSGP6LlbWIC2rAIM7NFO8lhC5FzMt5JF0kAjRnBrZHq0u0/6cYtz4n8YDeJXc39TYR8iVXSp6GKRN4h7+cjsmL0nDqyv2ubsppBCDFiIF1L7Z3/autP2JXMmZL9BvrTmkWjsWrjuiWl32GCVj4OoxLY7ooatUqrX7cgEAn6Qed3NLSCkGLSq7tW8MACAkkD1vJF2Ggr2Q3HGreOgzZWNC1B+Hof6rl3rz1VvQ4B7yr8Havbm4ZeHvOJZve90gR4wUIJE2GLSobMGd8TjwyrV44cYe7m4KaShlX567m6D5B7jYbSllv/tftxhHt9CT59Wb7aWHgbBGdO8n27DjRCEeW7bd3U0hA2PQojKTyQSzny/+1r8NUp4Y4e7mkEYuVHjmypxWPSROdJcoOVRqBmPzoXysbRA0ph8/b/eYrccKHNZ7vqxK0vndrX7QZMTMQ9FFZde5orqG4SJx9pCWOrZo6u4meKV1B/SZDdAjre55Deut+12NrpW7P2y8Eu0TXzu3F82ZootOHU/aKiitRMJra1VbP8ggw5dIBDMt5HGmfPSn3Q39PIXm3zpVjmi2HM7XxaJ8Yj7eYpyBmUYfU6PkXfXzrtOqLnhowAQV/YVBC3mknScL3d0EQ7DbveDE19HfRMa+3KWDvXpsWbRB+gwlsUv2xdZsXPXGOmQ5MchULVuO5OORL7fjnIS1aYiMht1D5JG84ZuUFq/xrbWXpyo7832+0oWrpOrBs99f2p5gtsSVcLccyUdpRQ2u6R7l1HnF3gJ3Lb4cHL49vq9T9dtSP551xZ+asXNLpCZmWsgjeUHMotpr/HVP471lPMFX205g1ne7UVOr7ruhYQKqql6AViUxWLtr8VZM+2QbclTekLK+U4XajdNxJqOkhy8UHNNiXMy0kGfSwyejxtSaOXL/p+mq1KM3T3+zCwBwZedIVetteNnv+SjN5nOO5F+oQHRYoAqtcq2pH29z7QkZZdBfmGkh8mJ6Cu20ui8VKpxiK9Xvh89pWr/eyQ3UFP1/VvlLiBd8p/FYioKWhQsXIi4uDoGBgUhISEBaWprjgwAsW7YMJpMJt9xyi5LTEknGzyR98ZSbxMnzZfh8a7a7m0HktWQHLcuXL0dycjLmzp2LjIwMxMfHY8yYMcjLs782RlZWFp588kkMHz5ccWOJpPKUm6Q9Wr9Eb8/IH84rQWmDRQSTFmxwU2vEGfF9rqjNKr8Zvf29bWSyg5YFCxZg2rRpmDJlCrp3745FixYhODgYS5YssXlMTU0NJkyYgBdffBEdOnRwqsGepE1EkLub4LG41LpE9mY8e/GcjT+zCpC0YCNGvWkdpJRXedesKCK9kRW0VFZWIj09HUlJSZcr8PFBUlISUlNTbR730ksvoWXLlpg6daqk81RUVKC4uNjqh0iOWm+4txgoLpv7k7SpwHrxy+5LM6pyitWb3fNnVgFueGeTwy0HHNFblsAVXxB09pLJjWQFLfn5+aipqUFUlPXaAlFRUcjJEZ82uXnzZnz44YdYvHix5PPMmzcPYWFhlp/Y2Fg5zSTCCRU3yNOrFbvPOF1HiZ09lI6cveB0/XW+2nbSYRmtbkybD+WLPn6+tBIrd59BZbV6Ea692/cdi1KReaoYty/acrm8hvd7vd7omQUlZ2g6e6ikpAQTJ07E4sWLERkpfdrhrFmzUFRUZPk5ceKEhq10ncmJ7dzdBK9RUu6ZGxq60oebj7m7CaqwFdzd+8k2PPR5Bv619qBL2+NsoGJ1vJ26GBqQJ5K1TktkZCR8fX2Rm5tr9Xhubi6io6MblT9y5AiysrJw4403Wh6r/Stv7+fnhwMHDqBjx46NjjObzTCbzXKapnvhwf548eae+Dj18h4nekvzEnmLorIqSzfNF5wN5BTZU551mwMiI5CVaQkICED//v2RkpJieay2thYpKSlITExsVL5bt27YvXs3duzYYfm56aabMHLkSOzYsYPdPkQki1oL6j22fLvqdZJ23PEFb+PBs5j+RQYKSitdf3KySfaKuMnJyZg8eTIGDBiAQYMG4a233kJpaSmmTJkCAJg0aRJiYmIwb948BAYGomfPnlbHh4eHA0Cjxz3VxMHt8Okfx/HMtd3c3RQiXdp7uhitwwMlden9kqnOlgPrDpy1/NtTQxa95jOMMqZl0pJL64+Z/Xyw4M4+7m0MWcgOWsaNG4ezZ89izpw5yMnJQZ8+fbBq1SrL4Nzs7Gz4+HCh3Tov3dwDD43siFZhnN5MJOb6tzchwM9H0oBYsd2jnWaMeygAYH9OMa59a5O7m+FyjrqUisur8MCn6bgpvjX+PqithPqk03J/KJJP0d5DM2bMwIwZM0SfW79+vd1jly5dquSUhmUymWwGLCaYsOuF0ej9wmoXt4pIX6TO4Nl1slDbhjhL4wBo1ne7G5zO9gkNFIs57f0NR7HlyDlsOXJOUtDiTdfG0zAl4mLrnrzK8u/42HCEBvq7rzFEBnMwV71p2HXEbmC1LhrnIkDAodwSXPXGOvyw/ZTD8nI2rJayHsy6A3m4+4Otmu4I3ZAWl7akXNv9pUg/GLS4WPvIJlibfCUeuqojXr65h7ubQ0Qilm7JUnaggoEkyV/tRNa5Mjy2fIeyczphykd/YvPhfMz8dpfLz02khKLuIXJOp5YheJoDc4l0QdXZQwqqKq+qUe/8Cp0tqVB8rCt2eVZ79pBeBymTY8y0EBF5OZML5xRzhjk5g0ELEZEddZmQlH25Dkoq40y8ICcAMPJ6NGqHVMa9EsSghYi8mqMb2MJ1hwEAUz/e1ui5Ujt7N0nx2sp9TnXNSFVWWY0Rb6zHU1/vVL1uo6y7Qp6BQQsReTVHCYh9Z0psPtdj7q9OnfuPowU4X6Z85svX6dL2ZVu5OwfZBWX4Ot3xxpXegGNajItBCxF5NceZAv1mEl5buV+Velx5EzdwLxXpAIMWHWkS4Gv3+d5twlzUEiLvUV5lf2E7WeNGNA5wlAYXjsazHD9XqrBm1wQh3FyW6jBo0ZGHR3XGbX1jbD7Pv1uixvadKZZVfurSPzVqifMe+XI7pnyU5vJBs6WVrpt2zQCEnMGgRUcEAQgPDrBdgH/tRI1c9295e/GkaLF/kUp+2nka6w6cxfFzZZbHBEGw7H+jNJRxR49MTa2AqUv/xOurrLuw2D1EzuDicgbCkIXI9dxxj62/jcAz3+7CV9tOolPLpsgtds/mfScKyhDRJABNzdJvGb8fzkfK/jyk7M/jYpqkGmZadOCqri0AALf2jbHbJ85EC5HrHTl7QZOpwvbU/xT4atulGT+H8y6gpNy5KdZKDX99HQa+slb0OVufWFI3wazvdOFFTPkoDZsP5Vs97miXZ/IeDFp04KN7BmLfS9ciOixQ9PkxPaIAAPdf2cGVzSIiAMfPuW+qcF23kB5cdMF2A09/swvrDpzF3R9u1fxcZEzsHtIBk8mEoL9mDol9o3h3Qn/kFJcjJjzI1U0jIhnUHK/xy+4zePDzDHUqc0Mfl5LM8JkiG7tNq733ENPWhsVMiwH4+pgYsBAZwLbj51WpRxCgXsCiMSNuD2DENtMlzLToDJfEJiKlftxxyt1NcIrSDIij4/IvVOCrbdJWDyZ9Y6bFC/ytXxt3N4GIZJH/5WXXyUI8umyH+k1RSEn8oTQD4ui4hz7LwOurDjisZ+6Pmbj2rY2WTTJJf5hp0RmOkiciJY7l217VVkoGVxAE3Y71cLZVaVkF1vXZeJ0fpx4HAKzcfcbJM5JWmGnxAuxyIjIWLYdclFWKT5ue+GGaovpsNVXJFzC9BE01tfzM1CsGLTrXtlmwzefuGRLnuoYQka5JDXS6z/kVs77b1ejxzYfzRUqrT0kXkNrBDAfiGhe7h3RuxSPDbD7XJSrEhS0hIldR65Zqq8vny7QTGNIxUqWzELkOMy06FxLob/M5nWRSiUhlaiUC4l9cjV/35Ig+9/wPmaqcQw9JC7mZGL10Q5F8DFoMTOqfHTMyRJ5PbOxacXk17v80XTSwqK6Rv8y+LDY+oOwFDEpDCb1093yXcRIzvsjg7CMNsXvIC0wZGoeyyhq8nXLI3U0hIp2odtNgU3sBhu1BvcaQ/NWlPar6xIbj3uHcdkULzLToTNIVLVWra1BcM0wf2RFmP18kX9MFLUPMqtVNRNpRMuPPXrJB7CnVghZ9JDl0pehilaLjyqtqdJM10isGLTozpFMkrusZrUpdXz2QiKfGcEt4IqP5frv2K9vW6vDmWFgmfrPXcghK+vECzPx2F86XVmp3EgnOllSg2+xVuGNRKlbuPsMuJhvYPaRDV3driV8yxQfP1dfwD7lts2C0iQjCliPnNGoZEbnCfzccdXcTnGYrzrAVKqUdK0D+hQqnz1t0sQphQbYnMADW67D87b1UAEBltcZjfByo24Jh2/Hz2Hb8PP4+MBbz/9bbrW3SI2ZadEjJ95+WIWZsfHokOrZoqmpbnhzdBVGh7FYi0ju5iRO9jRP574Yjio+tP7g3/sXVqFIwyPjYOdsrCsulRhLrm/STzlfigRi0eLGmZseJthlXd8Yfs0a5oDVE5Ax790mxm6h6Q1oElFfV4J2UQ9h7ulidSuW2ocELvFAuvuqv/Tou/3vN3lxnm+Q0LTrvjpy9gEeXbceh3BINancNBi0GJrZMttS+3/GD2mLqsPbSzsM1DYh0L6fookvOIzZQdOG6w3hzzUFc//YmTc7p6o+g1TKDlo0Hz2LBmoOW3/X6kTlh8Vb8uOM07vxvqrubohjHtHipNhFBqHBzHy6Rt1ixS/sN+P65+qDjQk5avScHz3zbeAuAzFNFjR7zlC87Ul7GpCXK9m1ytZzicgDAeRsDno2AmRYd8veV9sceFmx/sFlD+psrQOQdpn+R4e4mSFZaUW1zTaf7Pk1vdMOTMn7D2Zk5YlllV21q+PvhxhMbyqtqsGDNQew6WWjzOEEQ8Glqlt0yJB+DFh26vlcr9GsbjgdGdLRZpldMGK65IkrTdoQEXk7EBfn7anouIlIu/fh5u88v33ZCcl2vr9pv1dUhhaPwoe/La2TVV2f9gTz889cDotOzv673mlyd1Vm04QjeTjmEm/7zu+jzggCs2H0Gs3/cg5v+87vLx/oUl1ehpNy42RR72D2kQ2Y/X3z30FC7ZV6/vTd8fETGtMg5kYOvSGuTR1yut0HFA+MicLakAlnnyuSckYg08Lf3tth9fueJQkn11NYK2C6xrFJyZtbc89GfAICrurZo9Nz+HPcNJt1/xvG5D9Rr3/Vvb0LW/LGyzqF0kbnK6lr0fmE1AODIa9fDV+Q+YWTMtBjMjJGdcEf/NugW7dx+QoLgeM3NoIDL2ZUmDWYaff3AEHR1sg1EpC+3vPs7dp1sPD7FnoafI3Xrjah5qzxTWG6/DSrMMfaU7vP6a92UVsqfRaV3DFoM5skxXfHGHfGN0qFa/MGF1AtUPpw8oNHzOlxQk8jr7M9Rr+tBbsAi5tFlO+w+XzcYVA6jjek1mZwP2jxlILPaGLR4kV4xYZZ/Swk46v/R9GwdZqckEbmL1K4fLa0/cLbRY3q55xp1Lx+t2/3R78fw8ZYsTc+hBY5p8RBSPh9ev703Bryy1vK7nL8JvXwAEZE1d9+TC0qdX3ofALIL5I+PEwQBJpMJaVniA5EfW7YdmSoOgl2VeQbfpJ9EiYPF6wSh8Wfm2ZIKFJZVonOUeLf62n3Wa8NoPTnqxf/tBQDc3r9No+5/PTNOS8muuve3vZRiZFN1l+N39Df14k09MPenPaqek4j0paqm8SdBnoIuoEN5F2w+J/a5drrwIka8sR7X9ozGvjPigckPO07Lboc9D3ymfOr6wFcvfWHc9PRIxDYLbvT8H0cLFNftjGqR/396xu4hL+Z4KK5zJg+Jc3rAMBHZt+lwvlvPv1Zk9dhBr6Xg/Y3qbfp4OK/xbJ3Ve3ORXVBm8zyy9x+yk7L6YfspTPjgD3n12fgCuZPrtjiFmRYPIbf3RgDgI6PPx2QyoV/bcGRkF16uw1gBOpFHcsVqu/a8aWNNl02H1AumxLI5jqzbn6fa+R9bvkO1unTHYF3/zLR4CCXxwz+GtkebiCA8dJXtRezqazjfv0VIgIKzEhFpT0mg4wrOfNk7d6ECv+w+o2gXa0/BoMVLCQIQ0SQAm54eiaev7aaojqfGdHM4TobZGCJq6NwFdQbv6pkWCYybF/6OBz/PwKL1RzSo3RgYtHgYP5mrH9oauPvRlIEOj23WJEB0/Ra1dYlqqvk5iMh1+tebxWjL9M+Ns1+THAKAwrJKfLDpqOwByyfPX9rJe9WeHA1aZgwMWjzMg1d1RFzzYDxxTRen6okODWz8WFhQo8c6tGhitx41pkqvfnyEzec+vzfB+RMQke6s2O3cWB2tJxooJQgCnvhqJ15ZsQ+TlqThya934qHP02XWIa1caUU17lyUig83H1PQUn3iQFwP07ypGeufGqlJ3XNu6I6q6lqMT2hreSwk0P5O02p1D/VoHYo9IustDO0Uqc4JiIhcJOWvQcL7c0oU7aEk9WN16ZYspGUVIC2rAFOHtRctY7Q1uBi0eIiuNhYsskXJt5AWIWYsmthf9nFERHqndl5Gy2BA6mq5FytrtGuEm7B7yOBWPDIM4we1xYI7493dFCIiUtE7KYdQXF6l+HhPnAjBoMXgerQOw7zbeqGlyBgUexy9mfX2ZpfSnqGdmuOzqRzjQkT6+wyrI6ddb645iBe4qrgVBi2kmvDgxuNbXDkYzs/HB8M6c4wLEUlTWmF/DyE92GZjXyUpjDZeRQoGLV7KUSih5M0e4OuDm+JbK2rPhqeuUr09RET2vLv+sCb1ChBgUmmlFrEvflKzNXnFjtfDMdpHK4MWEqU0tap0t9DoMHndW/a8dmsv1eoiIm2cL610dxNwpkj+xo7Okpt9dqabS8p6Lslf7cTrq/YrP4mLMWjxMn8fGIumZj/cPbit48Iu5O9j/61Y/w83wNd+2bsS9PXaiKixi1Xundky7r+p+C7jlOV3pcFBichAWXtZFjXG2tgLfORmpdfszcW7Blphl1Oevcz8v/XGK7f0hJ+DG7+S7hiTCWgT0XgBOil8fEzY+uwoVFbX4mJVDUb/a6PNsqFBfsi/0PhbGruQiEiqrccKVKln7NubGz1WUFaJqBD1sseH80pwOO+C5Xd7gY9eByCrhUGLF3IUsADK3/hTh7VHbnE5kq6Ikl1PlJ0ZUAxIiDyLlvdWk0nb+uvLLihr9NgXW7ORbGNVciWfrUkLrL/EeXhcYhe7h8hpPWNCAQC39IlBoL8vXrq5J67s0kKz8z18dWcAwK19YzQ7BxEZl5LAYPepIvUbopG6xeV2nSzElI/ScDBX/qq6RsVMC4kKDvCVXPbzqYORevQcru7WstFzamVI6n8ITUpsh2GdIxHX3P6+R0SkX96aPJUbT9kLwG5e+DsEAdhzuhhpzyUB8PysNIMWamTGyE6Ii5QeEIQF++PantGiz2nRv2oymdCxBXd+JiL90jJ2qPtYrft8zSu5PLXZ08e0sHuIrPRrG44nx3TV9BxhQf6YPrKjpucgIn3TekyLXkndN8h+Jc5XYVQMWkhTY3u3En38qTHd7B7n72v9qaPnDyEikm/GFxma1a3nbINemzbz213qBFQaY9BCmpo+shMW3W29M7SUAGTi4DgAQMcWl7qpDPC3REQybM8udHcTNFVr6zNLp59ly/48gW3HlW8Z4Coc00Ka8vf1aTTepVdMmMPjnr62K+JjwzCsk/S9hJiMISK9+Nfag5rVbS/usfWl8Iftp8SfqKes0r0L/knBTAtZMWnUD/PI1Z0QHOCL8YNiseDOPg7LB/r74uY+MWje1Oyw7KzruqF5kwDMvqG7pLb8MH2opHJEZEy7DDR92RGxLht73Ti2nnps+Q6VWuRezLSQFa36NJNHd0XyaG0G+N4/oiPuu7KDaMA1KK4Z0rKsV74MC2q8GzUReY7JS9Lwxu293d0MUXL3HhKvQ5vPaiNkqxm0kEewlSGKiQgCsi79+7Z+Mbj/yo7w5aheIo/31De73N0EzeSXVGDo/N9Ur1enw22ssHuI3ObzexMQEeyPRXf3c8n5bu/XBl2jQxDbLAhXddVuxV4iIltWZTreebk+sUCitLIGpxvsUL35UD7+vfaQIQIPZzDTQm4ztFMkMmZfo9k4GkA8hWoymbB0yiDEzVyh2nl8fUyosTldgIjoknUHzmpS790fbgUgbzXzhoyQg1aUaVm4cCHi4uIQGBiIhIQEpKWl2Sy7ePFiDB8+HBEREYiIiEBSUpLd8uReWgYQ7jifK8KIuxLa4qN7BrrgTERE9p0433gDR08iO2hZvnw5kpOTMXfuXGRkZCA+Ph5jxoxBXl6eaPn169dj/PjxWLduHVJTUxEbG4vRo0fj1CnH06/I9fS6uNAdA9oAAOLbOJ4uXV+rsCAtmmNlSMfmaGJW/u2GiMgWuR/Jn6Qe16YhOiE7aFmwYAGmTZuGKVOmoHv37li0aBGCg4OxZMkS0fKff/45HnroIfTp0wfdunXDBx98gNraWqSkpDjdePIekxPjsPy+wfhi2mBJ5ZdOGYjb+7fBjKs7adwyIiJyFVlBS2VlJdLT05GUlHS5Ah8fJCUlITU1VVIdZWVlqKqqQrNmzWyWqaioQHFxsdUPeTcfHxMSOjRHE7O0YVhXdW2Jf94Rj6b1y9voiZqU2M7p9rUOd5zR8TEBH0waYLfMZBXaQkSeI6e43HEhLyIraMnPz0dNTQ2ioqKsHo+KikJOjrQR0c888wxat25tFfg0NG/ePISFhVl+YmNj5TSTnODqMS2ewASTpG6og69ch6TuUQ7LERGROJdOeZ4/fz6WLVuG77//HoGBgTbLzZo1C0VFRZafEydOuLCVRJd98o9BmONgpV2po/X9fB3/ufn4MGgkIrJFVtASGRkJX19f5ObmWj2em5uL6OhoG0dd8s9//hPz58/H6tWr0bu3/ZUKzWYzQkNDrX7INfQ6EFdLQzo2t/nclV1a4B/D2uORUZ1Fn78pvjVGdBFf8+WWPq0lt+GhqzqiXfNgTB3W3vJYt+gQyccTETnLCIl2WUFLQEAA+vfvbzWItm5QbWJios3jXn/9dbz88stYtWoVBgyw36dP5Cprk0fgnfF9MaZHNB5xMGA3+Zou6BDZxOqxFiFmvD2+r83syFt/7yu5LY8ldcGGp0aieZPLey01bxog+XgiIm8ge3G55ORkTJ48GQMGDMCgQYPw1ltvobS0FFOmTAEATJo0CTExMZg3bx4A4P/+7/8wZ84cfPHFF4iLi7OMfWnatCmaNm2q4kshNXjymBZTg5G4nVo2RaeWl96Dsc2CHR4f6G/dDaRmUqpuP5KgAF/c1i8G5VU1CPKX9+f5y6PDcd2/N6nXKCIinZEdtIwbNw5nz57FnDlzkJOTgz59+mDVqlWWwbnZ2dnw8bmcwHnvvfdQWVmJ22+/3aqeuXPn4oUXXnCu9UQy2NuoTEqwJrLXqlPtsaVuF+wnv94p+ZgOkU3YnUREHk/RMv4zZszAjBkzRJ9bv3691e9ZWVlKTkHkNsvvG4zZP2bi5Zt7uuycASKDdOXkvH56eJhHZ8mIiABumEhepGH3kC0JHZpj9eMjkNDB9gBdQL3uoSu7tHA64JAz6Whs71ZOnYuIPJPUz0h3YtBCAIAerS/N0LqtX4ybW+IeSv5UlcYs3z00xOr3AF/nPyjkfNi0idB+awMiMh57Xeh6wV2eCQDw1f2J2J9TjL6xEe5uim41nA7eMsRso6R9/dpaX2MfGVmWB6/qiPfWH1F03vpCAv1QUl7tdD1ERK7ETAsBAJqY/dC/XTOvXdxMbu/MkI7NsXBCP1XO7Svjmtsqaa/9DdeRMcGEtckjJJ+TiLwDu4eIPNQX0wajYwt1puzbChTVGlf79vjG68VEhVqvSN29VSjurbewHRGRHrF7iAjAwDjbG3hqzddGdBIa6K9K/WFBjuu5f0QHlFbUqHI+IiKtMGghr2EvcxHbLBgbnroK4cGuX4XWVvfQw1d3xv6cEmw+nO+wDjlZmaGdGs+K4nRpIjICdg8R/aVd8yZ2sxJqroD7cL1tA+7o30a0TFiwPz67N8HqMWdji5du7oHhnRvvlcSQhYiMgJkWIhe4K6Gt1e9PjO6KyUPiUFZRg7bN7W8hMPO6bpj/y368cXtvnC+rFC3jaADdU2O64mxJBSYlxokfr8OoZfygtvgyLdvdzSDyGnr8HGiIQQuRRMM6R+JAbgmaBPg6LtxAQvvGY2Yim5oBCWN5HxjREfdf2QEmkwnlVTXYlnUeSd2j8PQ3u2we09TshwsVl6c0Tx9pf0NIPc4aiAhWZ0yPs9o1D8bxc2XubgYRgd1DRJI9NaYrXrq5B359/EqXn7tuzEmgvy/enzQAdw6ItVu+b9twmfUrbZnnu7pbS3c3gYj+wqCFvIaz9+VAf19MSoxDmwjHO0I31KN1mJNnt8/ZoMME/a2GqZfWDGjnvpllRGSN3UNEGtr09EjklVSgU0t11nSRSsqg4XuGxGHpliwAzLTYc32vaHc3gYj+wkwLkYZimwWjfzvtt0aoizn8/9rHqFt0iMNjIqymdzuOWiKbytu2oG4/KyO7f0QHTgcn0hEGLUQeZOfc0dg++xqESlhQTo52zYPx4eQBDsvNuaG75d9qLY5HRFSHQQuRQb1xe2/Lv+uyAcEBfohoIm2BvPoJhCZmX7sziDY8NRLxseEO6xw38PIAYWcTFF2i5Hep3XdlB+dO2oAeZ1URacUI73YGLUQGdUvfGDRrEoC2zYKhZJ/L+ocM7RgpaSBu91b2u3zqBypmP+UfL00CfHFzfIzs46RsWUBExsWghbyG1mMTXD30wd/XB1ufHYV1T17V6LX1ipE3W0nq7t4/PzzM7vMmmPDk6C4Y3jkS1/dqJasN9f1jWHv4+JiwRub0cjk7ZhOR8XD2EJGB+fuKf++4qmsLvD2+L66QMCBXDkfBjckEzLi6M2YA+HHHKcXnadvs0rTyzlEhiAj2x/myKknHqR2ycAwukb4w00Ieb2yvVujROhT9ZC645oz5t/Vy2bnEmEwm3BTfGp2jbActbZoFubBF0j06qjNu63d5PyZbnVZieza1ax6MPhLG3ojpExuOhXf1s3pMzf2miMh5zLSQx1s4oR8EQdC+ewiXb7B/H9TWXlFduCk+BkfySjEgTvsp2XI8fk0XSeVahjaegj2mRzT+OFqAHScKZZ1zUPtm+M/4vrhYVSPrOCJyLQYt5BW41kZjvj4mPDmmq7uboSop/59DzH4oq6pBTe3lNMpjSZ3RMjQQx8+VNqhP9SZaNG8SgHOl4htgEpE4dg8REQAguMFGkH4KBrVqcZO31UWTdEWU4jr/N2MYbopvrfh4Z8XHhqNpIL8zks4Y4LsdgxYiAgDc0Ls1kq64vDng09d2xRPXdMHaZNdvEClF37YRWK1g80p/Px90bx2Kt8f3bfRcw3VZtPoM//7BIRrVTOTZGLQQEYBLM5E+mDzQ8nsTsx8eHtUZnVqKD+bt3ioUYUH+eDzp8hgUZxdju6Z74+yJYGc0bJeoECy4Mx4A8O+/95F0jnluHiQNXJqFxUG+RPIxP0lEiiR0aIY5N3RH1rky/GvtQcnHvXhTD4zt3QoRwQHo+OxKAMCguGZ4buwV6NZK2hTt+t1Qt/Vrgxt6t0aAxMXsOrawvdKu2d913+P0tqs2kREwaCFSiclk8qo5soJw6TU3q7dtgJTF3SYPibP8OyrUjNziCozpGS1pm4A6b//dumtHasDiSFRooCr1SOFFbxUi1TBoISJR8W3CJZULC/LHD9OHwt/XJHtF2pWPDEdGdiFGdm2hoIXO69yyKY7llzpc2+X5sVfglRX7VD23M9scqO2a7lFYszfX3c0gNzPCXlsMWojIyuZnRuJMUTl6OtgKoH4XjdIF3Zo3NYuOY6lPLCGhVpJi1WNXoqqmFoH+vqLP173Ge4d3UD1oeWd8P9z36Ta0j2yCTYfyVa1bLmZ9CDBGl6V+Qn0ig9P/dxRp2kQEY2BcM3c34zKRz1F7g3PlVOjrY7IZsABAdJj8VYMnJ7aTlHHq3joUm5+5Gg+O6Cj7HETeikELESlihFSys/4+MFb2MS/e3BNHXrtedEo1cGn13foSOzZH7zbyNrhUG9deJMAYf9MMWohUMvO6bgCAe+oNNCXnDesc6bZz29qQUorBHayDk5nXdcPiSQOw/L7BVo+bTCZ8dX+iw/rSnh2F/S9fi3cn9HNY9plru8lqK7uHyCg4poVIJVOHtceYHtFoE6HPjQhdzexnu9tFjvm39UZcZBNEBPvjtZX7ValTiQ6RTXA0v9RxQRvi24QjsWNz0ecC/X1xXc9o/JKZY/P4ln/NbLq+Vyu75zGZgBCutksKGCHjxkwLkUpMJhNimwVzn6O/JF3REiO6tMANve3fZB0JC/bHM9d2w31XXh77ERrk72zzHGoZYr0hY4jMcwYHuC9wsJU4mTI0zpXNIFIdgxYiUsRRbObn64OP/zEIT4+R11Vhz/zbemHi4Ha4qou0KdLThrdXfK67EuTv1B1WL7BpavbDwrsud+XoYWbGY0ldmIUhm4zwdYtBCxFpKraZet1lfx/UFi/f0lNyNmts78ubIsaEB6Gp2Q9tmzVRrT1tmwVb/f5YUucG57+cZQqyM0sJkJea3/Z8ks3nwoL8bQ5SEQRBvfni5HF8FGyS6moMuYlIEakfb3Wr5haUVmraHjF9YsOx4M54tGsejPg24agVlK+eK/Z61yaPwLnSCiTO+w0AMFxk0PDM67rh+LkyxWvZiIlsernrqlt0CPbnlFh+7xUTZhWXfDEtAXct3goAqBWANs2Cse9MsVV9oUG8FZAxMi18pxKRItFhrlvy3hm39WujWd0Bfj6ICA6o90jjj/0HJK7DYm+66Ts2pk/bUlt7OWzpEnV5P6daQcCiu/vh5Z/3Ia+kHLtOFgEAZl13Bb7LOCXrHETuwO4hIpJlyT0DcPfgtpiY2M7dTRF1hcRNFx1RMg1Yq+z6jfGtHReyof44m7Agf7Rr3gQfTB6AvvUyPy1CzKoO0r2iVahqdZHrGGESATMtRCTL1d2icHU3+0vvu9Pt/WNRUl6NhPbi04u15I4P/b5tI/D2+L4Y/a+Nlsfqx1v+vj7YOXc0IFivO9MwJmsYpAX4+qCyplZRmyKC/bH+yatw5OwFTP14m6I6yPUMELMwaCEi7bnys9DXx4R7h3dQvV5bH+iqfdDLrGdt8gis3H0G/xjWHk3N1h/ltQ0CkDAFU8Sfvrar4v2WTCYgLrIJ4iKbICzIH0UXqxTVI2bzMyOxKjMH1/dqhSHzf1OtXjIGBi1ERCKUTLJxZXDWqWVTPDKqs+hzSvZm6q5il0798Tlbnx2FkvJqDHx1rdP1do0KQZuIYE2CUjLGQFyOaSEikqB9c/Gp0vXjA7WyLo52vlZDw7jmul7Rdp93ZJCNTTYD/X3RIsQMs8JZW3IsuDNe83N4Mh8D9A8xaCEiEtElqqnV77Nv6I5xA+xvoKjWhnOLJw1QfKwgSAs4Gi52J3U8TsPrUifYbH8dmvVPXSWpfmdoOVPMqJKukB4AGyBmYdBCRNozwodhnZ9mDMVLN/fA2AZ7/EQ0CcD/3d670fL+amVaGh7qTGaiblG7eBm7RzcJsA46/H3FX8ynUxNEH3f00luFuWZPLgOsj+ZS70/s7+4mqIpBCxFp7u7Bl6ZHiy2+pje924RjUmKc5MyDVsvzL7tvMLq3CsWX0wY7LlyPj48JrcODsOuF0fjuoaE2yzXMxjR8veMGim9jEBXqeH0eJcHbqG4t5R8k4sWbe6pSj6cw0hcGKRi0EJHmHr66M5bfNxjvT1Te7aFXgX6+lqyElBu6LQ2Dhr5tI7Dy0eE2d4ZuaO6N3REdGogXbuwOAAgN9IevwrTDkI7NERRgu7unt0gGp377lexv9OE9A61+7xUT5jBg69s2vNFjEwe3Q9qzo9ArRnqWyZMZYe0VOTh7iIg05+tjQkIH16+booWG9wAfHxN2vzAGtYKgeIsANUwZ2h73DJGeIboroS0+35qNYZ3kZ7+W3TcYB3JK8MBn6cgtrgBg3T00+4busuts6J3xfdGuebDdMrZeacvQQM0yDGN7tcKK3We0qdzNlCyo6GrMtBAROSnQ3xfBAe7/DijnW3WP1mHYPvsafPKPQbLPExzgh75tI2wOPLY1fmXiYGmrKLcKC0RcZBOHr+eZay/vID7vtl5Wz4Vbba+gzNUNuqx6xYQZoovTkzFoISKS4aoul25k0U50BYlxRxI/okmAUzv7yh3Pc3t/abN7pLaofvau4Tozr93aeGzLfVfKW99l9g3d0S06xOp3ci/3fzUgIjKQOTd2R/fWoRjdQ79bGThLyQq6alIyDsOvwWynNhHB2DLzaqtVc5+9/gpEhQbi5Z/3Sqoz0N8HXz+QiD2ni9EnNhyB/r44evaC7LaRehi0EBHJ0MTsh8lD4lSvVw/jJd+d0A8fb8nC3Bt7yD5WSfu7RYcgIjgAt/RVviFkh8gmOFdaiU4tG68f06LB9HQAmJzYDiGBfvjzWAG+Tj/psP6QQH8MVmk81pJ7BuAfS/W7F5MBhrQwaCEi0oNB7Zvhxx2n3dqG63u1wvUN1qex58berfHB5mN/daHIj1pCg/zx5X3ypnQ3tPrxK1ErQPIgaD9fH9w5IBY5ReUOy4qN2VEaXL47oR93v1YBx7QQEenA3we2xYI747HxqZHubopkT47pinfG95W9lkydCQnia8HUDwx2vTDabh1+vj42A5b68cUyBcGRWIDSJSqk8YMi6p9v2vD2NoPBq7q2kN0ub8aghYhIB3x9TLitXxu0dTDNV08C/X1xY3xrRDQJkJSB6PrXoNawIH/8+tiVuClevFuoSb2ZWKGB6oyviVBhNhFwaf2cRXf3w8pHhtstV79LaVjnS4FJgG/jW64gAM+PvUKVtnkDBi1ERGTXdT2j8fLNPfDbEyOcqifQ3xf7XroW255PQtfokEYDbhfe1Q8dWzTBv8f3ceo8dewN6L1jQOOZTG+N64NPpw5CaKAfmgT4onkT8UDn2p6t0L315a6e2Td0x+f3im9vUF/zpmY8cnUnPJ7UxfJYE7Ov23at/u6hIVZdVkp2B3c1jmkhIiK7/Hx9MDExzm4ZqUM97K20O7Z3K8u+SVprFRaE/S9fi02H8jHtk0uDY2/pGwMA+PP5JACXXrdUQyUu0pc8uisAoE1EED5OzbJMo/72wSF45MvtOFV40WEd82/rhZnf7QYAvHxzD8z+cY9oudWPX2m3Hh2M/ZaNmRYiIrJLyjfwmdd1Q2igH5Kv6eKwrKs4uikH+vuiT2w4AOuNFs1+vjD72d+12ll/698GP80YZlmIr3+7CIddTr88Ohx7XxqDvw+qNxbIRjZpxSPDJI+/MRJmWoiIyC4pnQYdWjTFjjmjnVqszh1ahJiR9uwoNDErvx02DOq+e2iIonrCgv0REuiHkvJq0efFZh/5igQt+1661m5Gqz4jdAnVx0wLERGpQm8Bi9TpyS1DA50KWuqsTb4Sn04dhH5tIwAAt/RpjStahWKIxE0vAeDP55LQv12Ew3J1m1Ze0z0Kqx6zztA0DFjusLESccMxP50NkJlh0EJERKKeHN0FkU3NmFlvjx+jcsXifZ1ahmB458tTmN/6e1+sfGQY/GWMjQn098XCu/phbO9W+PqBRJvlvn9oKPa8OAYtQszoFh1qMzABgFdu7Ykvpw3GI1d3snq84SVpqkLgpjX9t5CIiNxixtWdMX1kJ0XL6uuBHtqtpA3RYYFYeFc/u2V8fUxW2aGJie3wdfpJ0V27zX6+SOzYHIPaN8OVXVrg9kWpAID2LZrIbpu7MWghIiKb9HDj798uAunHz2PcwFjFdYQHa7efUj8J3Tla690mHOnPJ9nd3drXx4QBcc2wc+5oVFTXqLYGjisxaCEiIl37bGoC9p4pQt9Y+cHBp1MHobSiBi1D1N2VGwB+n3k1ss+VWcawuFvzpo33WhJzaUNM4wUsAIMWIiLSuaAAX/Rv10zRsfXHmKgtJjwIMeFBmtXvCgabPMSBuERERN7q9dt7I8jf17LInd4x00JEROSl4mPDkfniGPjqbLq6Lcy0EBER6dRDV3XU/BxGCVgAhUHLwoULERcXh8DAQCQkJCAtLc1u+a+//hrdunVDYGAgevXqhZUrVypqLBERkTcJ9Nd2OwGjkR20LF++HMnJyZg7dy4yMjIQHx+PMWPGIC8vT7T8li1bMH78eEydOhXbt2/HLbfcgltuuQWZmZlON56IiMiTiS3d781MgsyNBxISEjBw4ED85z//AQDU1tYiNjYWDz/8MGbOnNmo/Lhx41BaWoqff/7Z8tjgwYPRp08fLFq0SNI5i4uLERYWhqKiIoSG8n8gERF5B0EQ8E36SfRoHYburY13/1P7/i0r01JZWYn09HQkJSVdrsDHB0lJSUhNTRU9JjU11ao8AIwZM8ZmeQCoqKhAcXGx1Q8REZG3MZlMuGNArCEDFi3IClry8/NRU1ODqKgoq8ejoqKQk5MjekxOTo6s8gAwb948hIWFWX5iY5WvgkhERESeQZezh2bNmoWioiLLz4kTJ9zdJCIiInIzWeu0REZGwtfXF7m5uVaP5+bmIjo6WvSY6OhoWeUBwGw2w2yWthwxEREReQdZmZaAgAD0798fKSkplsdqa2uRkpKCxETxLbQTExOtygPAmjVrbJYnIiIiEiN7Rdzk5GRMnjwZAwYMwKBBg/DWW2+htLQUU6ZMAQBMmjQJMTExmDdvHgDg0UcfxYgRI/Dmm29i7NixWLZsGbZt24b3339f3VdCREREHk120DJu3DicPXsWc+bMQU5ODvr06YNVq1ZZBttmZ2fDx+dyAmfIkCH44osv8Pzzz+PZZ59F586d8cMPP6Bnz57qvQoiIiLyeLLXaXEHrtNCRERkPG5dp4WIiIjIXRi0EBERkSEwaCEiIiJDYNBCREREhsCghYiIiAyBQQsREREZgux1WtyhblY2d3smIiIyjrr7tlqrqxgiaCkpKQEA7vZMRERkQCUlJQgLC3O6HkMsLldbW4vTp08jJCQEJpNJtXqLi4sRGxuLEydOcNE6GXjdlOF1k4/XTBleN/l4zZRxdN0EQUBJSQlat25ttVq+UobItPj4+KBNmzaa1R8aGso3qQK8bsrwusnHa6YMr5t8vGbK2LtuamRY6nAgLhERERkCgxYiIiIyBK8OWsxmM+bOnQuz2ezuphgKr5syvG7y8Zopw+smH6+ZMq6+boYYiEtERETk1ZkWIiIiMg4GLURERGQIDFqIiIjIEBi0EBERkSF4ddCycOFCxMXFITAwEAkJCUhLS3N3k1xm48aNuPHGG9G6dWuYTCb88MMPVs8LgoA5c+agVatWCAoKQlJSEg4dOmRVpqCgABMmTEBoaCjCw8MxdepUXLhwwarMrl27MHz4cAQGBiI2Nhavv/661i9NM/PmzcPAgQMREhKCli1b4pZbbsGBAwesypSXl2P69Olo3rw5mjZtir/97W/Izc21KpOdnY2xY8ciODgYLVu2xFNPPYXq6mqrMuvXr0e/fv1gNpvRqVMnLF26VOuXp5n33nsPvXv3tiw+lZiYiF9++cXyPK+ZY/Pnz4fJZMJjjz1meYzXrbEXXngBJpPJ6qdbt26W53nNbDt16hTuvvtuNG/eHEFBQejVqxe2bdtmeV439wTBSy1btkwICAgQlixZIuzZs0eYNm2aEB4eLuTm5rq7aS6xcuVK4bnnnhO+++47AYDw/fffWz0/f/58ISwsTPjhhx+EnTt3CjfddJPQvn174eLFi5Yy1157rRAfHy/88ccfwqZNm4ROnToJ48ePtzxfVFQkREVFCRMmTBAyMzOFL7/8UggKChL++9//uuplqmrMmDHCRx99JGRmZgo7duwQrr/+eqFt27bChQsXLGUeeOABITY2VkhJSRG2bdsmDB48WBgyZIjl+erqaqFnz55CUlKSsH37dmHlypVCZGSkMGvWLEuZo0ePCsHBwUJycrKwd+9e4Z133hF8fX2FVatWufT1quWnn34SVqxYIRw8eFA4cOCA8Oyzzwr+/v5CZmamIAi8Zo6kpaUJcXFxQu/evYVHH33U8jivW2Nz584VevToIZw5c8byc/bsWcvzvGbiCgoKhHbt2gn33HOPsHXrVuHo0aPCr7/+Khw+fNhSRi/3BK8NWgYNGiRMnz7d8ntNTY3QunVrYd68eW5slXs0DFpqa2uF6Oho4Y033rA8VlhYKJjNZuHLL78UBEEQ9u7dKwAQ/vzzT0uZX375RTCZTMKpU6cEQRCEd999V4iIiBAqKiosZZ555hmha9euGr8i18jLyxMACBs2bBAE4dI18vf3F77++mtLmX379gkAhNTUVEEQLgWLPj4+Qk5OjqXMe++9J4SGhlqu09NPPy306NHD6lzjxo0TxowZo/VLcpmIiAjhgw8+4DVzoKSkROjcubOwZs0aYcSIEZaghddN3Ny5c4X4+HjR53jNbHvmmWeEYcOG2XxeT/cEr+weqqysRHp6OpKSkiyP+fj4ICkpCampqW5smT4cO3YMOTk5VtcnLCwMCQkJluuTmpqK8PBwDBgwwFImKSkJPj4+2Lp1q6XMlVdeiYCAAEuZMWPG4MCBAzh//ryLXo12ioqKAADNmjUDAKSnp6OqqsrqunXr1g1t27a1um69evVCVFSUpcyYMWNQXFyMPXv2WMrUr6OujCe8N2tqarBs2TKUlpYiMTGR18yB6dOnY+zYsY1eG6+bbYcOHULr1q3RoUMHTJgwAdnZ2QB4zez56aefMGDAANxxxx1o2bIl+vbti8WLF1ue19M9wSuDlvz8fNTU1Fi9MQEgKioKOTk5bmqVftRdA3vXJycnBy1btrR63s/PD82aNbMqI1ZH/XMYVW1tLR577DEMHToUPXv2BHDpNQUEBCA8PNyqbMPr5uia2CpTXFyMixcvavFyNLd79240bdoUZrMZDzzwAL7//nt0796d18yOZcuWISMjA/PmzWv0HK+buISEBCxduhSrVq3Ce++9h2PHjmH48OEoKSnhNbPj6NGjeO+999C5c2f8+uuvePDBB/HII4/g448/BqCve4Ihdnkm0pvp06cjMzMTmzdvdndTDKFr167YsWMHioqK8M0332Dy5MnYsGGDu5ulWydOnMCjjz6KNWvWIDAw0N3NMYzrrrvO8u/evXsjISEB7dq1w1dffYWgoCA3tkzfamtrMWDAALz22msAgL59+yIzMxOLFi3C5MmT3dw6a16ZaYmMjISvr2+jUeO5ubmIjo52U6v0o+4a2Ls+0dHRyMvLs3q+uroaBQUFVmXE6qh/DiOaMWMGfv75Z6xbtw5t2rSxPB4dHY3KykoUFhZalW943RxdE1tlQkNDDfvBGxAQgE6dOqF///6YN28e4uPj8e9//5vXzIb09HTk5eWhX79+8PPzg5+fHzZs2IC3334bfn5+iIqK4nWTIDw8HF26dMHhw4f5XrOjVatW6N69u9VjV1xxhaVrTU/3BK8MWgICAtC/f3+kpKRYHqutrUVKSgoSExPd2DJ9aN++PaKjo62uT3FxMbZu3Wq5PomJiSgsLER6erqlzG+//Yba2lokJCRYymzcuBFVVVWWMmvWrEHXrl0RERHholejHkEQMGPGDHz//ff47bff0L59e6vn+/fvD39/f6vrduDAAWRnZ1tdt927d1v9ca9ZswahoaGWD43ExESrOurKeNJ7s7a2FhUVFbxmNowaNQq7d+/Gjh07LD8DBgzAhAkTLP/mdXPswoULOHLkCFq1asX3mh1Dhw5ttHzDwYMH0a5dOwA6uydIHrLrYZYtWyaYzWZh6dKlwt69e4X77rtPCA8Ptxo17slKSkqE7du3C9u3bxcACAsWLBC2b98uHD9+XBCES9PbwsPDhR9//FHYtWuXcPPNN4tOb+vbt6+wdetWYfPmzULnzp2tprcVFhYKUVFRwsSJE4XMzExh2bJlQnBwsGGnPD/44INCWFiYsH79eqsplWVlZZYyDzzwgNC2bVvht99+E7Zt2yYkJiYKiYmJlufrplSOHj1a2LFjh7Bq1SqhRYsWolMqn3rqKWHfvn3CwoULDT2lcubMmcKGDRuEY8eOCbt27RJmzpwpmEwmYfXq1YIg8JpJVX/2kCDwuol54oknhPXr1wvHjh0Tfv/9dyEpKUmIjIwU8vLyBEHgNbMlLS1N8PPzE1599VXh0KFDwueffy4EBwcLn332maWMXu4JXhu0CIIgvPPOO0Lbtm2FgIAAYdCgQcIff/zh7ia5zLp16wQAjX4mT54sCMKlKW6zZ88WoqKiBLPZLIwaNUo4cOCAVR3nzp0Txo8fLzRt2lQIDQ0VpkyZIpSUlFiV2blzpzBs2DDBbDYLMTExwvz58131ElUndr0ACB999JGlzMWLF4WHHnpIiIiIEIKDg4Vbb71VOHPmjFU9WVlZwnXXXScEBQUJkZGRwhNPPCFUVVVZlVm3bp3Qp08fISAgQOjQoYPVOYzmH//4h9CuXTshICBAaNGihTBq1ChLwCIIvGZSNQxaeN0aGzdunNCqVSshICBAiImJEcaNG2e11givmW3/+9//hJ49ewpms1no1q2b8P7771s9r5d7gkkQBEFyDomIiIjITbxyTAsREREZD4MWIiIiMgQGLURERGQIDFqIiIjIEBi0EBERkSEwaCEiIiJDYNBCREREhsCghYiIiAyBQQsREREZAoMWIiIiMgQGLURERGQIDFqIiIjIEP4fjCEG0kMq+skAAAAASUVORK5CYII=", "text/plain": [ "
" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "import matplotlib.pyplot as plt\n", "\n", "plt.plot(losses)" ] }, { "cell_type": "code", "execution_count": 46, "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 116, "referenced_widgets": [ "b3a68ff845bd47a8bacc7fa7d026be4f", "bcb49fbb1ce34ef7a82935b57520cd68", "c98b465d074b435b930d485164c24d80", "fa93c96f64fb46fe9f494d71fae52ba8", "b54170d14f584ce3a32614f2bca314e1", "28ff9e63ca194de7a2847d68b749adbf", "f647e9a4109d4150a786383d96bbe080", "d9097819dd4a4c3fb3f99f259974ae76", "5dbc564c55d64c58bc23ba8827c179e7", "1761e7f78d1e48239dfd3e346bc22c1e", "fd5cc6284ac94b76b6e7dc985ec0b9b7", "174dc177226845e68b96377225510332", "c3a07d5ddc634d0f81dc49118cf74004", "9ecefe686e15415eb8237d45b5d59e43", "86b05c06147a4f529b25fb3aabc3781e", "fab0a18ef0144b2b8f0b0543eda8a5f9", "57933b8d292541cab289fb0a81a9a836", "2ff6b0967e024b1894822b4aad082eae", "564fb5e9e0824ac08b6d4ae3190ccad1", "74ac57cfb6174e8e8feac335d7988313", "3fc277f072244f58a87a2c4fb3b4776e", "161fea5c0b8e4f36979609f27a3ae3f8" ] }, "id": "-7DsUVuVUzQ9", "outputId": "43d966d3-dd59-4997-c07d-a9aaa100b3d9" }, "outputs": [ { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "b3a68ff845bd47a8bacc7fa7d026be4f", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Evaluation: 0%| | 0/1 [00:00