{ "cells": [ { "cell_type": "code", "execution_count": null, "metadata": { "id": "8NNfwWXGvn_o" }, "outputs": [], "source": [ "%load_ext autoreload\n", "%autoreload 2" ] }, { "cell_type": "markdown", "metadata": { "id": "YY0f_c_WLNGP" }, "source": [ "## Installation\n", "If you haven't installed Pearl, please make sure you install Pearl with the following cell. Otherwise, you can skip the cell below." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "1uLHbYlegKX-", "colab": { "base_uri": "https://localhost:8080/" }, "outputId": "54a2922c-bdec-47d4-ca3b-a959d9ad6001" }, "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ "\u001b[33mWARNING: Skipping Pearl as it is not installed.\u001b[0m\u001b[33m\n", "\u001b[0mCloning into 'Pearl'...\n", "remote: Enumerating objects: 5987, done.\u001b[K\n", "remote: Counting objects: 100% (2196/2196), done.\u001b[K\n", "remote: Compressing objects: 100% (675/675), done.\u001b[K\n", "remote: Total 5987 (delta 1674), reused 1941 (delta 1503), pack-reused 3791\u001b[K\n", "Receiving objects: 100% (5987/5987), 54.36 MiB | 14.03 MiB/s, done.\n", "Resolving deltas: 100% (4001/4001), done.\n", "/content/Pearl\n", "Processing /content/Pearl\n", " Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n", " Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n", " Installing backend dependencies ... \u001b[?25l\u001b[?25hdone\n", " Preparing metadata (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n", "Requirement already satisfied: gym in /usr/local/lib/python3.10/dist-packages (from Pearl==0.1.0) (0.25.2)\n", "Collecting gymnasium[accept-rom-license,atari,mujoco] (from Pearl==0.1.0)\n", " Downloading gymnasium-0.29.1-py3-none-any.whl (953 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m953.9/953.9 kB\u001b[0m \u001b[31m15.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hRequirement already satisfied: numpy in /usr/local/lib/python3.10/dist-packages (from Pearl==0.1.0) (1.25.2)\n", "Requirement already satisfied: matplotlib in /usr/local/lib/python3.10/dist-packages (from Pearl==0.1.0) (3.7.1)\n", "Requirement already satisfied: pandas in /usr/local/lib/python3.10/dist-packages (from Pearl==0.1.0) (2.0.3)\n", "Collecting parameterized (from Pearl==0.1.0)\n", " Downloading parameterized-0.9.0-py2.py3-none-any.whl (20 kB)\n", "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from Pearl==0.1.0) (2.31.0)\n", "Collecting mujoco (from Pearl==0.1.0)\n", " Downloading mujoco-3.1.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (5.5 MB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m5.5/5.5 MB\u001b[0m \u001b[31m71.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hRequirement already satisfied: torch in /usr/local/lib/python3.10/dist-packages (from Pearl==0.1.0) (2.2.1+cu121)\n", "Requirement already satisfied: torchvision in /usr/local/lib/python3.10/dist-packages (from Pearl==0.1.0) (0.17.1+cu121)\n", "Requirement already satisfied: torchaudio in /usr/local/lib/python3.10/dist-packages (from Pearl==0.1.0) (2.2.1+cu121)\n", "Requirement already satisfied: cloudpickle>=1.2.0 in /usr/local/lib/python3.10/dist-packages (from gym->Pearl==0.1.0) (2.2.1)\n", "Requirement already satisfied: gym-notices>=0.0.4 in /usr/local/lib/python3.10/dist-packages (from gym->Pearl==0.1.0) (0.0.8)\n", "Requirement already satisfied: typing-extensions>=4.3.0 in /usr/local/lib/python3.10/dist-packages (from gymnasium[accept-rom-license,atari,mujoco]->Pearl==0.1.0) (4.11.0)\n", "Collecting farama-notifications>=0.0.1 (from gymnasium[accept-rom-license,atari,mujoco]->Pearl==0.1.0)\n", " Downloading Farama_Notifications-0.0.4-py3-none-any.whl (2.5 kB)\n", "Collecting autorom[accept-rom-license]~=0.4.2 (from gymnasium[accept-rom-license,atari,mujoco]->Pearl==0.1.0)\n", " Downloading AutoROM-0.4.2-py3-none-any.whl (16 kB)\n", "Collecting shimmy[atari]<1.0,>=0.1.0 (from gymnasium[accept-rom-license,atari,mujoco]->Pearl==0.1.0)\n", " Downloading Shimmy-0.2.1-py3-none-any.whl (25 kB)\n", "Requirement already satisfied: imageio>=2.14.1 in /usr/local/lib/python3.10/dist-packages (from gymnasium[accept-rom-license,atari,mujoco]->Pearl==0.1.0) (2.31.6)\n", "Requirement already satisfied: absl-py in /usr/local/lib/python3.10/dist-packages (from mujoco->Pearl==0.1.0) (1.4.0)\n", "Requirement already satisfied: etils[epath] in /usr/local/lib/python3.10/dist-packages (from mujoco->Pearl==0.1.0) (1.7.0)\n", "Collecting glfw (from mujoco->Pearl==0.1.0)\n", " Downloading glfw-2.7.0-py2.py27.py3.py30.py31.py32.py33.py34.py35.py36.py37.py38-none-manylinux2014_x86_64.whl (211 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m211.8/211.8 kB\u001b[0m \u001b[31m33.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hRequirement already satisfied: pyopengl in /usr/local/lib/python3.10/dist-packages (from mujoco->Pearl==0.1.0) (3.1.7)\n", "Requirement already satisfied: contourpy>=1.0.1 in /usr/local/lib/python3.10/dist-packages (from matplotlib->Pearl==0.1.0) (1.2.1)\n", "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.10/dist-packages (from matplotlib->Pearl==0.1.0) (0.12.1)\n", "Requirement already satisfied: fonttools>=4.22.0 in /usr/local/lib/python3.10/dist-packages (from matplotlib->Pearl==0.1.0) (4.51.0)\n", "Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.10/dist-packages (from matplotlib->Pearl==0.1.0) (1.4.5)\n", "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.10/dist-packages (from matplotlib->Pearl==0.1.0) (24.0)\n", "Requirement already satisfied: pillow>=6.2.0 in /usr/local/lib/python3.10/dist-packages (from matplotlib->Pearl==0.1.0) (9.4.0)\n", "Requirement already satisfied: pyparsing>=2.3.1 in /usr/local/lib/python3.10/dist-packages (from matplotlib->Pearl==0.1.0) (3.1.2)\n", "Requirement already satisfied: python-dateutil>=2.7 in /usr/local/lib/python3.10/dist-packages (from matplotlib->Pearl==0.1.0) (2.8.2)\n", "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.10/dist-packages (from pandas->Pearl==0.1.0) (2023.4)\n", "Requirement already satisfied: tzdata>=2022.1 in /usr/local/lib/python3.10/dist-packages (from pandas->Pearl==0.1.0) (2024.1)\n", "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests->Pearl==0.1.0) (3.3.2)\n", "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->Pearl==0.1.0) (3.7)\n", "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->Pearl==0.1.0) (2.0.7)\n", "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->Pearl==0.1.0) (2024.2.2)\n", "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from torch->Pearl==0.1.0) (3.14.0)\n", "Requirement already satisfied: sympy in /usr/local/lib/python3.10/dist-packages (from torch->Pearl==0.1.0) (1.12)\n", "Requirement already satisfied: networkx in /usr/local/lib/python3.10/dist-packages (from torch->Pearl==0.1.0) (3.3)\n", "Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from torch->Pearl==0.1.0) (3.1.4)\n", "Requirement already satisfied: fsspec in /usr/local/lib/python3.10/dist-packages (from torch->Pearl==0.1.0) (2023.6.0)\n", "Collecting nvidia-cuda-nvrtc-cu12==12.1.105 (from torch->Pearl==0.1.0)\n", " Using cached nvidia_cuda_nvrtc_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (23.7 MB)\n", "Collecting nvidia-cuda-runtime-cu12==12.1.105 (from torch->Pearl==0.1.0)\n", " Using cached nvidia_cuda_runtime_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (823 kB)\n", "Collecting nvidia-cuda-cupti-cu12==12.1.105 (from torch->Pearl==0.1.0)\n", " Using cached nvidia_cuda_cupti_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (14.1 MB)\n", "Collecting nvidia-cudnn-cu12==8.9.2.26 (from torch->Pearl==0.1.0)\n", " Using cached nvidia_cudnn_cu12-8.9.2.26-py3-none-manylinux1_x86_64.whl (731.7 MB)\n", "Collecting nvidia-cublas-cu12==12.1.3.1 (from torch->Pearl==0.1.0)\n", " Using cached nvidia_cublas_cu12-12.1.3.1-py3-none-manylinux1_x86_64.whl (410.6 MB)\n", "Collecting nvidia-cufft-cu12==11.0.2.54 (from torch->Pearl==0.1.0)\n", " Using cached nvidia_cufft_cu12-11.0.2.54-py3-none-manylinux1_x86_64.whl (121.6 MB)\n", "Collecting nvidia-curand-cu12==10.3.2.106 (from torch->Pearl==0.1.0)\n", " Using cached nvidia_curand_cu12-10.3.2.106-py3-none-manylinux1_x86_64.whl (56.5 MB)\n", "Collecting nvidia-cusolver-cu12==11.4.5.107 (from torch->Pearl==0.1.0)\n", " Using cached nvidia_cusolver_cu12-11.4.5.107-py3-none-manylinux1_x86_64.whl (124.2 MB)\n", "Collecting nvidia-cusparse-cu12==12.1.0.106 (from torch->Pearl==0.1.0)\n", " Using cached nvidia_cusparse_cu12-12.1.0.106-py3-none-manylinux1_x86_64.whl (196.0 MB)\n", "Collecting nvidia-nccl-cu12==2.19.3 (from torch->Pearl==0.1.0)\n", " Using cached nvidia_nccl_cu12-2.19.3-py3-none-manylinux1_x86_64.whl (166.0 MB)\n", "Collecting nvidia-nvtx-cu12==12.1.105 (from torch->Pearl==0.1.0)\n", " Using cached nvidia_nvtx_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (99 kB)\n", "Requirement already satisfied: triton==2.2.0 in /usr/local/lib/python3.10/dist-packages (from torch->Pearl==0.1.0) (2.2.0)\n", "Collecting nvidia-nvjitlink-cu12 (from nvidia-cusolver-cu12==11.4.5.107->torch->Pearl==0.1.0)\n", " Using cached nvidia_nvjitlink_cu12-12.4.127-py3-none-manylinux2014_x86_64.whl (21.1 MB)\n", "Requirement already satisfied: click in /usr/local/lib/python3.10/dist-packages (from autorom[accept-rom-license]~=0.4.2->gymnasium[accept-rom-license,atari,mujoco]->Pearl==0.1.0) (8.1.7)\n", "Requirement already satisfied: tqdm in /usr/local/lib/python3.10/dist-packages (from autorom[accept-rom-license]~=0.4.2->gymnasium[accept-rom-license,atari,mujoco]->Pearl==0.1.0) (4.66.4)\n", "Collecting AutoROM.accept-rom-license (from autorom[accept-rom-license]~=0.4.2->gymnasium[accept-rom-license,atari,mujoco]->Pearl==0.1.0)\n", " Downloading AutoROM.accept-rom-license-0.6.1.tar.gz (434 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m434.7/434.7 kB\u001b[0m \u001b[31m47.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25h Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n", " Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n", " Preparing metadata (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n", "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.10/dist-packages (from python-dateutil>=2.7->matplotlib->Pearl==0.1.0) (1.16.0)\n", "Collecting ale-py~=0.8.1 (from shimmy[atari]<1.0,>=0.1.0->gymnasium[accept-rom-license,atari,mujoco]->Pearl==0.1.0)\n", " Downloading ale_py-0.8.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.7 MB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.7/1.7 MB\u001b[0m \u001b[31m36.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hRequirement already satisfied: importlib_resources in /usr/local/lib/python3.10/dist-packages (from etils[epath]->mujoco->Pearl==0.1.0) (6.4.0)\n", "Requirement already satisfied: zipp in /usr/local/lib/python3.10/dist-packages (from etils[epath]->mujoco->Pearl==0.1.0) (3.18.1)\n", "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->torch->Pearl==0.1.0) (2.1.5)\n", "Requirement already satisfied: mpmath>=0.19 in /usr/local/lib/python3.10/dist-packages (from sympy->torch->Pearl==0.1.0) (1.3.0)\n", "Building wheels for collected packages: Pearl, AutoROM.accept-rom-license\n", " Building wheel for Pearl (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n", " Created wheel for Pearl: filename=Pearl-0.1.0-py3-none-any.whl size=215044 sha256=c2d11dd3e65c5bb6720538a7d224f13e2f26f857c237ad6f718fa55a99983926\n", " Stored in directory: /tmp/pip-ephem-wheel-cache-mi9lzaly/wheels/83/80/1d/d9211ba70ee392341daf21a07252739e0cb2af9f95439a28cd\n", " Building wheel for AutoROM.accept-rom-license (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n", " Created wheel for AutoROM.accept-rom-license: filename=AutoROM.accept_rom_license-0.6.1-py3-none-any.whl size=446659 sha256=50b64bf3c726e54dfa462a4cd8af9e666df0c9401e6bb1aedd4a0644e4abc2da\n", " Stored in directory: /root/.cache/pip/wheels/6b/1b/ef/a43ff1a2f1736d5711faa1ba4c1f61be1131b8899e6a057811\n", "Successfully built Pearl AutoROM.accept-rom-license\n", "Installing collected packages: glfw, farama-notifications, parameterized, nvidia-nvtx-cu12, nvidia-nvjitlink-cu12, nvidia-nccl-cu12, nvidia-curand-cu12, nvidia-cufft-cu12, nvidia-cuda-runtime-cu12, nvidia-cuda-nvrtc-cu12, nvidia-cuda-cupti-cu12, nvidia-cublas-cu12, gymnasium, ale-py, shimmy, nvidia-cusparse-cu12, nvidia-cudnn-cu12, AutoROM.accept-rom-license, autorom, nvidia-cusolver-cu12, mujoco, Pearl\n", "Successfully installed AutoROM.accept-rom-license-0.6.1 Pearl-0.1.0 ale-py-0.8.1 autorom-0.4.2 farama-notifications-0.0.4 glfw-2.7.0 gymnasium-0.29.1 mujoco-3.1.5 nvidia-cublas-cu12-12.1.3.1 nvidia-cuda-cupti-cu12-12.1.105 nvidia-cuda-nvrtc-cu12-12.1.105 nvidia-cuda-runtime-cu12-12.1.105 nvidia-cudnn-cu12-8.9.2.26 nvidia-cufft-cu12-11.0.2.54 nvidia-curand-cu12-10.3.2.106 nvidia-cusolver-cu12-11.4.5.107 nvidia-cusparse-cu12-12.1.0.106 nvidia-nccl-cu12-2.19.3 nvidia-nvjitlink-cu12-12.4.127 nvidia-nvtx-cu12-12.1.105 parameterized-0.9.0 shimmy-0.2.1\n", "/content\n" ] } ], "source": [ "%pip uninstall Pearl -y\n", "%rm -rf Pearl\n", "!git clone https://github.com/facebookresearch/Pearl.git\n", "%cd Pearl\n", "%pip install .\n", "%cd .." ] }, { "cell_type": "markdown", "metadata": { "id": "nCSJf8nALNGQ" }, "source": [ "## Import Modules" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "vcb70ZC_h3OA" }, "outputs": [], "source": [ "from pearl.utils.functional_utils.experimentation.set_seed import set_seed\n", "from pearl.action_representation_modules.one_hot_action_representation_module import OneHotActionTensorRepresentationModule\n", "from pearl.replay_buffers.sequential_decision_making.fifo_off_policy_replay_buffer import FIFOOffPolicyReplayBuffer\n", "from pearl.utils.functional_utils.train_and_eval.online_learning import online_learning\n", "from pearl.pearl_agent import PearlAgent\n", "from pearl.utils.uci_data import download_uci_data\n", "from pearl.utils.instantiations.environments.contextual_bandit_uci_environment import (\n", " SLCBEnvironment,\n", ")\n", "from pearl.policy_learners.exploration_modules.contextual_bandits.squarecb_exploration import SquareCBExploration\n", "from pearl.policy_learners.exploration_modules.contextual_bandits.ucb_exploration import (\n", " UCBExploration,\n", ")\n", "from pearl.policy_learners.exploration_modules.contextual_bandits.thompson_sampling_exploration import (\n", " ThompsonSamplingExplorationLinear,\n", ")\n", "from pearl.policy_learners.contextual_bandits.neural_bandit import NeuralBandit\n", "from pearl.policy_learners.contextual_bandits.neural_linear_bandit import (\n", " NeuralLinearBandit,\n", ")\n", "import torch\n", "import matplotlib.pyplot as plt\n", "import numpy as np\n", "import os\n", "\n", "set_seed(0)" ] }, { "cell_type": "markdown", "metadata": { "id": "A2756RO8LNGR" }, "source": [ "## Load Environment\n", "The environment which underlies the experiments to follow is a contextual bandit environment we added to Pearl that allows us to use UCI datasets (https://archive.ics.uci.edu/datasets).\n", "\n", "The UCI datasets span a wide variety of prediction tasks. We use these tasks to construct a contexual bandit environment in which an agent receives an expected reward of 1 if it correctly labels a data point and 0 otherwise. Pearl currently supports the following datasets: pendigits, letter, satimage, yeast. Additional ones can be readily added.\n", "\n", "In the following experiment we will test different types of contextual bandits algorithms on the pendigits UCI dataset." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "g1VHtmldi3A2" }, "outputs": [], "source": [ "# load environment\n", "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n", "\n", "# Download UCI dataset if doesn't exist\n", "uci_data_path = \"./utils/instantiations/environments/uci_datasets\"\n", "if not os.path.exists(uci_data_path):\n", " os.makedirs(uci_data_path)\n", " download_uci_data(data_path=uci_data_path)\n", "\n", "# Built CB environment using the pendigits UCI dataset\n", "pendigits_uci_dict = {\n", " \"path_filename\": os.path.join(uci_data_path, \"pendigits/pendigits.tra\"),\n", " \"action_embeddings\": \"discrete\",\n", " \"delim_whitespace\": False,\n", " \"ind_to_drop\": [],\n", " \"target_column\": 16,\n", "}\n", "env = SLCBEnvironment(**pendigits_uci_dict)\n", "\n", "# experiment code\n", "number_of_steps = 10000\n", "record_period = 400" ] }, { "cell_type": "markdown", "metadata": { "id": "UYIoDAGSLNGR" }, "source": [ "## Contextual Bandits learners\n", "The following sections show how to implement the neural versions of SquareCB, LinUCB and LinTS with Pearl.\n", "\n", "## Contextual Bandits learners: SquareCB\n", "\n", "The SquareCB algorithm requires only a regression model with which it learns the reward function. Given the reward model, SquareCB executes the following policy:\n", "$$\n", "\\widehat{a}_*\\in \\arg\\max_a\\widehat{r}(x,a)\\\\\n", "\\widehat{r}_*\\in \\max_a\\widehat{r}(x,a)\\\\\n", "\\text{If $a\\neq \\widehat{a}_*$}: \\pi(a,x)= \\frac{1}{A + \\gamma (\\widehat{r}_* - \\widehat{r}(x,a))}\\\\\n", "\\text{If $a= \\widehat{a}_*$}: \\pi(a,x) = 1-\\sum_{a'\\neq \\widehat{a}_*}\\pi(a',x).\n", "$$\n", "This policy balances exploration and exploitation in an intelligent way.\n", "\n", "To use the SquareCB algrorithm in Pearl we set the policy learner as `NeuralBandit`. `NeuralBandit` is class supportings the estimation of the reward function with a neural architecture. With access to an estimated reward model, we then use an instance of `SquareCBExploration` as an exploration module.\n", "\n", "To further highlight the versatility of the modular design of Pearl, we use the `OneHotActionTensorRepresentationModule` as the action representation module. This module internally converts actions from integers to one-hot-encoded vectors.\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 1000 }, "id": "kulkpFAvnOQx", "outputId": "8c78fd2a-cc1a-4e2f-9688-09e5dc51c8b0" }, "outputs": [ { "output_type": "stream", "name": "stderr", "text": [ "/usr/local/lib/python3.10/dist-packages/ipykernel/ipkernel.py:283: DeprecationWarning: `should_run_async` will not call `transform_cell` automatically in the future. Please pass the result to `transformed_cell` argument and any exception that happen during thetransform in `preprocessing_exc_tuple` in IPython 7.17 and above.\n", " and should_run_async(code)\n" ] }, { "output_type": "stream", "name": "stdout", "text": [ "episode 100, step 100, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: -0.02050408534705639\n", "episode 200, step 200, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: -0.039917074143886566\n", "episode 300, step 300, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 1.0325154066085815\n", "episode 400, step 400, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 1.0192910432815552\n", "episode 500, step 500, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 0.020320570096373558\n", "episode 600, step 600, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 1.0140972137451172\n", "episode 700, step 700, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 1.1774886846542358\n", "episode 800, step 800, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 1.0041024684906006\n", "episode 900, step 900, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 0.9082531929016113\n", "episode 1000, step 1000, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 1.0025124549865723\n", "episode 1100, step 1100, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 0.9679498076438904\n", "episode 1200, step 1200, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 0.9570890069007874\n", "episode 1300, step 1300, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 0.9786309599876404\n", "episode 1400, step 1400, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 1.0437270402908325\n", "episode 1500, step 1500, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 1.1215401887893677\n", "episode 1600, step 1600, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 1.0220673084259033\n", "episode 1700, step 1700, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: -0.05659715086221695\n", "episode 1800, step 1800, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 1.1713285446166992\n", "episode 1900, step 1900, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 0.9805613160133362\n", "episode 2000, step 2000, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 1.0700596570968628\n", "episode 2100, step 2100, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 0.9755151867866516\n", "episode 2200, step 2200, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 1.06023371219635\n", "episode 2300, step 2300, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: -0.2449081689119339\n", "episode 2400, step 2400, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 0.9615944623947144\n", "episode 2500, step 2500, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 1.1294598579406738\n", "episode 2600, step 2600, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 1.0927162170410156\n", "episode 2700, step 2700, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 1.0192159414291382\n", "episode 2800, step 2800, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: -0.14172282814979553\n", "episode 2900, step 2900, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 1.020617127418518\n", "episode 3000, step 3000, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 1.07183039188385\n", "episode 3100, step 3100, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 1.1373060941696167\n", "episode 3200, step 3200, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 0.8844553828239441\n", "episode 3300, step 3300, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 1.1407734155654907\n", "episode 3400, step 3400, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 1.0968198776245117\n", "episode 3500, step 3500, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 1.0457595586776733\n", "episode 3600, step 3600, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 1.0209089517593384\n", "episode 3700, step 3700, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 1.016340732574463\n", "episode 3800, step 3800, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 1.076174259185791\n", "episode 3900, step 3900, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 0.9254880547523499\n", "episode 4000, step 4000, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 0.9435696601867676\n", "episode 4100, step 4100, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 0.9403758645057678\n", "episode 4200, step 4200, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: -0.062246013432741165\n", "episode 4300, step 4300, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: -0.09749681502580643\n", "episode 4400, step 4400, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 0.9377647042274475\n", "episode 4500, step 4500, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 0.9589371681213379\n", "episode 4600, step 4600, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 0.8841699361801147\n", "episode 4700, step 4700, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 1.0646312236785889\n", "episode 4800, step 4800, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 1.1062999963760376\n", "episode 4900, step 4900, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 1.1038718223571777\n", "episode 5000, step 5000, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 0.7176197171211243\n", "episode 5100, step 5100, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 0.8898045420646667\n", "episode 5200, step 5200, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 0.010150366462767124\n", "episode 5300, step 5300, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 0.9760612845420837\n", "episode 5400, step 5400, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 0.06860487163066864\n", "episode 5500, step 5500, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 0.03486515209078789\n", "episode 5600, step 5600, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 1.1883316040039062\n", "episode 5700, step 5700, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 0.9038528800010681\n", "episode 5800, step 5800, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 0.8557829260826111\n", "episode 5900, step 5900, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 1.0221916437149048\n", "episode 6000, step 6000, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 0.8543550968170166\n", "episode 6100, step 6100, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 0.9892317056655884\n", "episode 6200, step 6200, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 0.9139447212219238\n", "episode 6300, step 6300, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: -0.0652153417468071\n", "episode 6400, step 6400, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 1.0828280448913574\n", "episode 6500, step 6500, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 0.9806367754936218\n", "episode 6600, step 6600, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 1.2393121719360352\n", "episode 6700, step 6700, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 1.0103856325149536\n", "episode 6800, step 6800, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 0.8305141925811768\n", "episode 6900, step 6900, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 0.9939833879470825\n", "episode 7000, step 7000, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 0.7439374923706055\n", "episode 7100, step 7100, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 0.9110734462738037\n", "episode 7200, step 7200, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 0.805903434753418\n", "episode 7300, step 7300, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 1.0544805526733398\n", "episode 7400, step 7400, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 1.1555981636047363\n", "episode 7500, step 7500, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 0.9537562727928162\n", "episode 7600, step 7600, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 1.0062040090560913\n", "episode 7700, step 7700, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 0.8801354169845581\n", "episode 7800, step 7800, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 0.7815372943878174\n", "episode 7900, step 7900, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 1.0681519508361816\n", "episode 8000, step 8000, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 0.9116690158843994\n", "episode 8100, step 8100, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 1.01902437210083\n", "episode 8200, step 8200, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 0.9659032225608826\n", "episode 8300, step 8300, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 0.8814241290092468\n", "episode 8400, step 8400, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 0.9224587082862854\n", "episode 8500, step 8500, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 1.2102456092834473\n", "episode 8600, step 8600, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 0.9070967435836792\n", "episode 8700, step 8700, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 1.2006794214248657\n", "episode 8800, step 8800, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 1.0592678785324097\n", "episode 8900, step 8900, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 1.0463409423828125\n", "episode 9000, step 9000, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 0.9944213032722473\n", "episode 9100, step 9100, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 0.9995201230049133\n", "episode 9200, step 9200, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 0.9861416220664978\n", "episode 9300, step 9300, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 0.9023017287254333\n", "episode 9400, step 9400, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 0.9761253595352173\n", "episode 9500, step 9500, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 0.8212168216705322\n", "episode 9600, step 9600, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 0.9000095129013062\n", "episode 9700, step 9700, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 0.9509682655334473\n", "episode 9800, step 9800, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 1.136844277381897\n", "episode 9900, step 9900, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 1.0328410863876343\n", "episode 10000, step 10000, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 0.9436702728271484\n" ] }, { "output_type": "display_data", "data": { "text/plain": [ "
" ], "image/png": "iVBORw0KGgoAAAANSUhEUgAAAkkAAAGwCAYAAAC99fF4AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAABUEklEQVR4nO3deVhU9f4H8PfMAMO+KDsOIijihiAK4VJWJJrXtMXMLM3UyvSq2aJmaVlp6S/Typtd07Tspi1mmaYZae6SuOKOqCA7Igz7MvP9/YGMTo4KODNnBt6v55nnXs6cM/OZQzpvv6tMCCFARERERHrkUhdAREREZIkYkoiIiIgMYEgiIiIiMoAhiYiIiMgAhiQiIiIiAxiSiIiIiAxgSCIiIiIywEbqAsxNq9UiMzMTLi4ukMlkUpdDRERE9SCEQHFxMfz9/SGXm6eNp9mFpMzMTKhUKqnLICIiokZIT09Hq1atzPJezS4kubi4AKi9ya6urhJXQ0RERPWhVquhUql03+Pm0OxCUl0Xm6urK0MSERGRlTHnUBkO3CYiIiIygCGJiIiIyACGJCIiIiIDmt2YpPrSaDSorq6WugwyAltbWygUCqnLICIiK8OQ9A9CCGRnZ6OwsFDqUsiI3N3d4evry7WxiIio3hiS/qEuIHl7e8PR0ZFfqlZOCIGysjLk5uYCAPz8/CSuiIiIrAVD0nU0Go0uILVs2VLqcshIHBwcAAC5ubnw9vZm1xsREdULB25fp24MkqOjo8SVkLHV/U45zoyIiOqLIckAdrE1PfydEhFRQzEkERERERnAkERERERkAEMSERERkQEMSU1IXl4exo8fj8DAQCiVSvj6+iI+Ph67d++WurR6y87Oxr///W8EBwdDqVRCpVJh0KBBSEhI0J0TFBQEmUwGmUwGhUIBf39/jBkzBleuXJGwciKi2xNCoKyqRuoyqJ64BEAT8uijj6KqqgqrVq1CcHAwcnJykJCQgMuXL0taV1VVFezs7G573oULF9CrVy+4u7tjwYIF6NKlC6qrq7FlyxZMmDABp06d0p07Z84cjBs3DhqNBmfOnMFzzz2HSZMm4euvvzblRyEiapSSyhqsO3gJq/ZcwLm8UoS3csOgcH8MDPeDv7uD1OXRTTAk3YYQAuXVGkne28FWUe9ZWYWFhdi5cye2b9+Oe+65BwDQunVrREdH6845e/YsxowZg8TERAQHB2Px4sXo168ffvrpJwwZMgTbt2/HvffeiytXrsDd3R0AcPjwYURGRuL8+fMICgrC5cuXMXHiROzYsQNXrlxBSEgIXn/9dQwfPlz3Pn379kXnzp1hY2OD1atXo0uXLti2bRuSk5Px6quvYufOnXByckK/fv3w0UcfwdPTEwDw4osvQiaTITExEU5OTrrX69SpE5599lm9z+vi4gJfX18AQEBAAEaNGoVvv/224TeZiMiEUvNK8NXei/gh6RJKKq+1IB29VISjl4rw3qaT6BHkgUFd/TGgsx+8XJQSVkv/xJB0G+XVGnSctUWS9z4xJx6OdvX7FTk7O8PZ2Rnr16/HXXfdBaVS/w+aVqvFI488Ah8fH+zfvx9FRUWYMmVKg2uqqKhAVFQUpk2bBldXV2zcuBFPP/00QkJC9ALZqlWrMH78eF1XX2FhIe677z6MHTsWH330EcrLyzFt2jQ8/vjj+PPPP1FQUIDNmzfjvffe0wtIdepCmyEZGRnYsGEDYmJiGvx5iIiMTasV2HY6F6v2XsSOM3m648FeThgVG4S+7b2w40weNhzJQuKFAvx94Qr+vnAFb/1yHD1DPDGoqx/iO/nC3fH2LfBkWgxJTYSNjQ1WrlyJcePGYenSpejWrRvuuecePPHEEwgPD8cff/yBU6dOYcuWLfD39wcAzJ07FwMGDGjQ+wQEBOCVV17R/fzvf/8bW7ZswXfffacXktq1a4f58+frfn733XcRGRmJuXPn6o6tWLECKpUKZ86cQWFhIYQQCAsLq1cd06ZNwxtvvAGNRoOKigrExMRg4cKFDfosRETGVFRWje+T0vHV3otIKygDAMhkwP1h3hjVMwi9Qjwhl9f2Djwd64SnY4OQVVSOjUezsOFoFo6kF2JXSj52peTjjfXJuLudF/7V1Q8PdPSFs5Jf11LgXb8NB1sFTsyJl+y9G+LRRx/FwIEDsXPnTuzbtw+//fYb5s+fjy+++AJFRUVQqVS6gAQAsbGxDa5Jo9Fg7ty5+O6775CRkYGqqipUVlbesEp5VFSU3s9HjhzBtm3b4OzsfMNrnjt3Di1atGhQHa+++iqeeeYZCCGQnp6O119/HQMHDsSOHTu47QgRmdWpbDVW7bmI9YcydMMzXO1tMKyHCk/fFYTAljffxcHPzQFj+wRjbJ9gpF0uw4ajmdhwJBOnsouRcCoXCadyobQ5hvvCvDGoqz/uC/OGfQO/G6jxGJJuQyaT1bvLyxLY29vjgQcewAMPPIA333wTY8eOxezZszF16tTbXiuX1052FELojv1zG48FCxZg8eLFWLRoEbp06QInJydMmTIFVVVVeuf9s8uspKQEgwYNwgcffHDD+/r5+aGyshIymUxvcPateHp6om3btgBqW60WLVqE2NhYbNu2DXFxcfV6DSKyLOVVGixOOIujlwqh0QpohYBGe/UhBDTa2q4szfXHr55XoxU3PKcVAi2dlAj2ckKwpxOCvZzRxtMJwV5O8Hdz0LXqNEaNRoutJ3Kwau8F7Est0B0P83XBqJ5BGBIRAAe7hoWZwJaOmHBvW0y4ty3O5hRjw9Es/HokE6n5pfgtORu/JWfDyU6BBzr6YFBXf/Rp5wU7G05SNyXr+fanRunYsSPWr1+PDh06ID09HVlZWfDz8wMA7Nu3T+9cLy8vAEBWVhY8PDwA1A7cvt7u3bsxePBgPPXUUwBqxzqdOXMGHTt2vGUd3bp1w48//oigoCDY2Nz4n52TkxPi4+OxZMkSTJo06YaQVVhYeMtxSXWtR+Xl5besg4gs07m8Ekz45iBOZRcb9XUzCsuRUViOnWfz9Y4rbeRo4+mkC01tPJ11YepWY4EKSqvwbWIavtl3EZlFFQAAhVyG+E4+GBUbhOg2LYyyDVI7HxdMfcAFL8W1w/FMNX49moUNRzKRUViO9Yczsf5wJtwcbDEkwh/TB3RocCAzFSEElmxLwcBwf7TxvHF8qbVhSGoiLl++jKFDh+LZZ59FeHg4XFxccODAAcyfPx+DBw9GXFwcQkNDMWrUKCxYsABqtRozZ87Ue422bdtCpVLhrbfewnvvvYczZ87gww8/1DunXbt2+OGHH7Bnzx54eHhg4cKFyMnJuW1ImjBhApYtW4bhw4fjtddeQ4sWLZCSkoI1a9bgiy++gEKhwJIlS9CrVy9ER0djzpw5CA8PR01NDbZu3YrPPvsMJ0+e1L1ecXExsrOzdd1tr732Gry8vNCzZ0/j3VQLIoTg/nNmkl9Sie2n8/DXmTzYKeQY3zcYbb1dpC6rSdtwJBPTfzyK0ioNPJ2VmPpAKFwdbKCQySCXy2Ajr/1fhUwGhfzaQy6rfa7u/1//nEImg0wGZKsrcD6vFOfyS3A+rxSp+aW4eLkUlTVanMouNhjKWjjZ1YYnTye08XJCsKczXB1s8GNSBjYczURVjVZ33vBoFUbEtDbZNH6ZTIbOAW7oHOCGaf3b41B6ITYcycTGo1nILa7Eqr0XcSq7GMuf6SH5uCWNVuDNn5Pxv/1p+DYxHVun3m1VPTGGWHf1pOPs7IyYmBh89NFHOHfuHKqrq6FSqTBu3Di8/vrrkMvl+OmnnzBmzBhER0cjKCgIH3/8Mfr37697DVtbW3z77bcYP348wsPD0aNHD7z77rsYOnSo7pw33ngDqampiI+Ph6OjI5577jkMGTIERUVFt6zP398fu3fvxrRp09CvXz9UVlaidevW6N+/v66bLzg4GAcPHsR7772Hl19+GVlZWfDy8kJUVBQ+++wzvdebNWsWZs2aBaC2BaxHjx74/fff0bJlS2PdUslV1Wix/lAG/rszFRXVGqx9PhYBXE/F6IQQOJ6pxp+ncvHnqVwcuVSI63qcsf5wBp7oocKUuFBOzzayyhoN3v31JL7edxEAENOmBT4ZHglvV3ujvYeqhSN6BOmPeazRaJFRWI7U/FKk5pXifH7J1f8tRVZRBQpKq1BQWoWki4YXqA1v5YZRsUEYGO5n1vFBMpkM3QI90C3QA28M7Ihtp3Lx0trD2H++ACO+2I9Vo3tINiOuWqPF1O+OYMORTMhkwMT72lp9QAIAmbh+AEozoFar4ebmhqKiIri6uuo9V1FRgfPnz6NNmzawtzfeH1JLJpPJdOskNWXW9LstrazBt4lp+GLneWSrK3THu7f2wJrn7oKNgmMQ7lRpZQ12peRj29VglFtcqfd85wBX3NveG6eyi7H1RA4AwMlOgfF9QzCmd7DFdG1Ys/SCMrz4zUEcy6j9B9bEe9tiSlw7yf/7LquqwXldeCpFal6JLjz1DGmJkT2DEKlyt5iW3WOXivD0iv0oLKtGmK8Lvh4TY/YwX16lwYvfJGHb6TzYKmT4aFgE/hXuf/sLG+hW39+mYv0xj6iJKCitwso9F7BqzwUUldcOmPdxVeLJ6Nb4YmcqDly8go8TzmJqv/YSV2qdLl4u1bUW7U8tQJVGq3vO0U6B3m09cV+YN+4N84bPdS0Z+1Mv471NJ3H0UhH+7/czWL0vDa/Et8fDkQFQ3MHA3+bs9+PZePn7IyiuqIGHoy0WDovAve29pS4LAOBoZ4NO/m7o5O8mdSn10qWVG9Y+F4unlu/HqexiDPt8L74ZFwM/N/O0OqsrqjF25QEkXiiAva0cS5+KQl8L+V0aA0MSkcQyCsuxbEcq1vydhorq2i/uNp5OeOGeYAyJDIDSRoE2Xk6Y9O0hfLItBbEhnogNaTrdiqZSrdHi7wsFutaic3mles8HtnDEfWHeuC/MGzHBLaC0Mdw6FBPcEutf7IUNRzMxf/NpZBSW45Xvj2DFrvN4/cEO6N3O0xwfp0mo1mjxwW+n8MWu8wCAboHu+PTJbtyW4w6193XBd8/HYsSyfUjNL8XQpXvxzdgYtG5p2oHTl0sqMerLRCRnqOGitMGK0T1u6Nq0duxuu441dclQw1ji7/ZsTjGW/pWKnw9noEZb+8ewS4AbxvcNQXwn3xtaKV774Qi+O3AJPq5K/Db5brRw4mq8/1RcUY0tx3Ow7VQudpzJQ/F120DYyGXoEdRC11oU4uXU4C6TimoNVu25gE+3paC4ova1+7b3wowBHdDel4O7byWzsBwT/3cQB9MKAQBje7fBtAFhsGX3sdFkFJZjxLJ9uHC5DD6uSnwzNsZkkw4yC8vx9PL9OJdXipZOdlj1bDQ6B5i29U2K7jaGpOvUfZEGBQXBwYH/smlKysvLceHCBYsISQfTruCz7ed0Y10AoFfblhh/T1v0atvypl/cZVU1+Ncnu5CaV4q4Dt5YNrK7xYyLsAQJJ3Pw+k/HkKO+Nr6opZMd+ravbS3qE+oJV3tbo7xXQWkVPk44i9X7LqJGKyCXAcN6qPBSXKhRBx3/U7VGi4wr5fB3d7Cq9XG2n64dYHylrBou9jZY8FhX9O/sK3VZTVKuugJPLd+PMzklaOFkh69MEF7O55fiqS/2I6OwHP5u9vh6bAxCvG5cKNjYGJLM4FY3uW5HeW9v7yY1S4pql0jIzc1FaGioJCtyCyGw42w+Ptueolt4TiYD4jv64oW+IYhQudfrdY5nFuHhJXtQpdHirUEd8UyvNias2jpcKa3CnF9P4KdDGQAAVQsHPBwRgHvDvNG1lfsdLRh4O+fzSzF/8yn8lpwNoHZs03N3B+O5u4PveGZPjUaLc3mlOHqpEMcyajdDPZGlRlWNFi2d7PBoVCs80UOFYDN8OTVWjUaLRX+cxZLtKRCidkD8f56MuuUK1HTnrpRWYeSKRBzLKIKLvQ1Wjo5GVGsPo7z2iUw1Rq5IRH5JJYI9nfD12BizzbplSDKD293krKwsFBYWwtvbG46OjvyXupUTQqCsrAy5ublwd3fXLaRpLhqtwG/JWfhs+zkcz1QDAGwVMjwcGYDn7g5BW++Gf8Gt3H0eb204ATuFHD9N6Gk1A0xNYXNyFt5Yfxz5JZWQy4CxfYIx9YFQs2/bcOBCAd7deBKH0wsBAN4uSrzcLxSPRanqNbhbqxVIzS/FsYxCHL1UhGOXinA8U63b4uJ6CrkMGu21v7aj27TA8GgVBnQ273T028lVV2DSmkO6fxQ8dVcg3hjY0aJqbMrUFdUYs/Jv/H3hChztFPhiZHf0bHtn4+eSLhZg9Jd/Q11Rgw5+rvjq2WizzqRjSDKD291kIQSys7NRWFho/uLIZNzd3eHr62u20FtVo8UPSZfw3x3ncOFy7UaXDrYKPBkTiDG929zRQFUhBMZ9lYQ/TuYg2MsJv/67d5NYj6Qh8ksqMfuX49h4NAsA0M7bGfMfC0dkoHH+tdwYQghsOpaN9zefRHpB7crv7X1c8PrADrgn1EvvvIuXy3A0owjHLtWGouSMIpRW3RiInOwU6BzghvBWbujSyh3hAW4I8HDA9tN5WJOYhm2nc1GXl1ztbfBIt1Z4IlqFMF/zfIHczJ5z+Zj07WHkl1TC0U6BeY90weCIAElrao7Kqmrw/NdJ2Hk2H3Y2cix9qhvuC/Np1GvtOJOH579OQnm1Bt1be2D5Mz3g5mCc7uv6Ykgyg/reZI1Gc8O+ZWSdbG1tzdrFVlRe+y+4A1cXonN3tMUzPYMwKjYIHkYabH2ltAoDFu9EtroCQ6NaYcHQrkZ5XUsnhMCGo1l465fjKCitgkIuwwv3BGPS/e1uOjvN3CprNPh670V88meKbimHPu080cnfDccyCnHsUhHUFTU3XOdgq0Anf1d0aXU1FAW4I9jT6ZbdhVlF5fj+wCWs/TsdGYXXtuSJULljeLQK/wr3h5MZV2HWagX+sz0FC7eegVbUhsQlI7o1qsWUjKOyRoOJ/zuErSdyYCOXYfETkRgY3rAW9U3HsjB5zSFUawTuDvXC0qe6SfIPM4YkM5DiJlPzcbmkEiNXJOJ4phqu9jaYHBeK4dEqk/yFsi/1Mp5ctg9aASx+IqLJ/0s9V12BmeuTdQPew3xd8H9Du5p8Rk1jFZZV4dM/U7Bq7wVUa/T/mrWzkaOjn+vVMOSG8FbuCPFyavRCihqtwK6UfKxJTMPWEzm62ZJOdgo8FOGPJ3oEIryVm0lbUgtKqzBl7WHsOJMHABga1QpzBnfmwpsWoFqjxSvfH8HPhzMhlwEfPBqOod1V9br2u7/TMX3dUWgFMLCLHz4aFiHZpAGGJDNgSCJTyVFXYMQX+5GSWwJPZzt8PSYGHfxM+9/Ywq1n8HHCWTgrbbBpUp8mOSBWCIF1BzMw59cTKCqvho1chon3tcWLfdtaxQyvtMtlWL4rFdVagfAAN3Rp5YZQHxeTTX3PK67EjwdrW5fO519bG6qDnyuGR6swOCLgjrpJhBBQV9Qgv6QS+cWVyC+pQo66Av/dkYpsdQXsbeWYM7gzHq/nlzCZh0YrMPOnY1jzdzoAYM7gThgZG3TLa77YmYp3N9bumTmsuwpzH+ki6QKqDElmwJBEppBeUIYRX+xHWkEZ/NzssdpMU2JrNFoMX7YPf1+4gq4qd/zwQmyTWncmq6gcr687hm2na1snugS4Yf5j4SYPn02BEAL7zxdgTWIaNiVn6zZltbeV48EufhgeHYjurT0gk8kghEBReTXySyqRezX41AagukeVXii6frXy6wV7OeE/I7pJPiaKDBNC4J1fT2LF7trFPF/r3x4v9m1r8LyPtp7Bx3+mAADG9WmD1x/sIPlEJoYkM2BIImNLyS3BU1/sR7a6Aq1bOmL1mBioWpivRSejsBwPLt6JovJqPH9PMGYM6GC29zYVIQTW/p2O9zaeRHFlDewUckyOa4fn7w6WfG8va1RYVoWfDmVgTWI6Tudc2/U+wN0BGq3A5dLKG7oEb8dZaQNPZzt4Oivh6axEOx9nPH9PiOQ70dOt/TMATby3LV7uF6oLQFqtwJxfT2DlngsAgFf6hWLCvW0lD0gAQ5JZMCSRMR3PLMLI5Ym4XFqFdt7OWD02Rm/fL3PZnJyNF1YnAQC+ejYad183m8rapBeUYca6Y9iVkg8AiAx0x4LHwk22cnBzIoTAofRCrElMw4YjWTcsMeBqbwNPl9rQ4+WsvBaCrh6r+9nLRcmp/FZu6V/n8P5vpwAAo3sFYda/OkKjFXjth6NYd3XNsfp0yZlTswxJS5YswYIFC5CdnY2uXbvik08+QXR0tMFzq6urMW/ePKxatQoZGRlo3749PvjgA/Tv37/e78eQRMaSdPEKRn+ZCHVFDToHuOKrZ2Mk3SrkjfXHsHpfGjydlfhtch+z7wR+p7RagW/2X8T7v51CaZUGShs5Xo1vj9G92nAjWRMorqjGkfQiuDrYwNNZiZbOdhYzQ5DM4+u9F/Dmz8cB1I45KiirwtYTOVDIZfi/oeF4OLKVxBXqa3Yhae3atRg5ciSWLl2KmJgYLFq0CN9//z1Onz4Nb+8bdxGeNm0aVq9ejWXLliEsLAxbtmzB1KlTsWfPHkRGRtbrPRmSmhYhBL5NTIefmz36tvcyW5PwnpR8jP3qAMqqatcMWTG6h9G2vGisimoNBn+6G6dzinF3qBdWPtPDpKtNG9PFy6V47Yej2H++duHB6KAW+OCxcLTxNO0GnUTN3Q9Jl/DaD0d0623Z2cix5MlueKBj49ZTMqVmF5JiYmLQo0cPfPrppwAArVYLlUqFf//735g+ffoN5/v7+2PmzJmYMGGC7tijjz4KBwcHrF69ul7vyZDUtGw5no3nv67tZuoR5IHpA8IQ1dq0u1AnnMzB+G8OoqpGiz7tPPH501EWs5jjmZxiPPTpLlRUa/H6g2F47u4QqUvS0WgFstUVSC8oq31cKcelgjKkFZThWEYRKmu0cLRTYFr/MDx9V2urCXhE1m7j0dp1kJQ2ciwb1R09Q+5sZW5TkeL7W7K/2auqqpCUlIQZM2bojsnlcsTFxWHv3r0Gr6msrLxhc1IHBwfs2rXrpu9TWVmJysprG16q1eo7rJwsyfars54A4O8LV/DoZ3vRr6MPXusfZpIF7DYcycRLaw+jRivQr6MPPnky0qK6KEJ9XDDrX53w+k/HMH/zacS0aYmu9dwX7k4JIVBQWoX0K+VIvxp+Ll0pQ3pBOdKvlCGzsPyWg4N7hrTEB4+Gm3XQOxEBA8P90FXlBjsbObxdpN0A3NJIFpLy8/Oh0Wjg46PfpOfj44NTp04ZvCY+Ph4LFy7E3XffjZCQECQkJGDdunXQaG5czr/OvHnz8Pbbbxu1drIMQgjsPFsbkuY90gWH0wrxfVI6fj+Rg4RTuXi8uwpT4toZbSD12r/TMH3dMQgBDInwx4KhXS1yuv3waBV2p+Rj47Es/PvbQ9g4qTdcjNwVmFFYjj9P5uBcXqleECozsLXG9WzkMgR4OEDl4QhVC0eoWtT+/zaeTujk72oRM2iImqNWHvzHiSGW0UdQT4sXL8a4ceMQFhYGmUyGkJAQjB49GitWrLjpNTNmzMDUqVN1P6vVaqhUXOSsKbh4uQyXrpTDViHD4Ah/DI8OxNg+bfDB5tP442QOvk1Mw0+HLmFs72A8d0/wHY0ZWrHrPOb8egIA8GRMIN4d3Nliu4NkMhnmPtIFh9MLkVZQhjfWJ2PRsIg7DiBpl8vwW3IWNiVn48jVjVwN8XFVIrCFI1QejmjVwhEqD4ergcgRvq72HIRNRFZDspDk6ekJhUKBnJwcveM5OTnw9fU1eI2XlxfWr1+PiooKXL58Gf7+/pg+fTqCg4Nv+j5KpRJKpXXN8qH6qWtFimrtoRsT1M7HBV+M6o6/LxRg3qaTOJhWiE+3peCb/Rfx7/vaYcRdgQ3qHhNC4NM/U/Dh1jMALGdRtdtxc7DFx8Mj8fjne/Hz4Uz0aeeFx6IaPlMlNa8EvyVn47fkLCRnXOuqlsmAHq1bIDLQXS8IBbg7cGo4ETUZkoUkOzs7REVFISEhAUOGDAFQO3A7ISEBEydOvOW19vb2CAgIQHV1NX788Uc8/vjjZqiYLM3Os7Xr6PRpd+OaQD2CWuDH8T3x+4kcfLD5FFLzSjHn1xP4cs95vNKvPQaF+9+2JUgIgfc3n8Lnf6UCAF6KC8Wk+y1jUbX6iGrtgakPhGLBltOY9XMyugW6I7geq4CfzSnGpmO1wehU9rWFB+UyIDakJQZ09kO/Tj4cu0BETZ7kSwCMGjUKn3/+OaKjo7Fo0SJ89913OHXqFHx8fDBy5EgEBARg3rx5AID9+/cjIyMDERERyMjIwFtvvYXz58/j4MGDcHd3r9d7cnZb01Ct0aLbnK0orqzBLxN7IbyV+03PrdFo8X3SJXy09Qxyi2sH8Xfyd8X0AWEGAxZQu2bP7F+O4+t9FwEAbwzsgLF9bt5iaak0WoGnvtiPvamX0cnfFete7HlDS5oQAqeyi/HbsdqutJTcEt1zNnIZerb1xIOdffFARx+0dGarLBFJo1nNbgOAYcOGIS8vD7NmzUJ2djYiIiKwefNm3WDutLQ0yOXXBsZWVFTgjTfeQGpqKpydnfHggw/i66+/rndAoqbjSHohiitr4OFoi07+t94F3kYhx/DoQAyO8MeXuy9g6fZzOJ6pxtPLE9GnnSem9Q/T20m+RqPFaz8exbqDGZDJgLkPd8Hw6EBTfySTUMhlWPREBAYs3onjmWq8/9spzB7UCUIIJGeo8VtyFn5LztbbCNVOIUfvdp4YcDUYuTtKt0AmEZGUJF9x29zYktQ0LNx6Bh8nnMW/wv3w6ZPdGnRtQWkVPv0zBV/vu6Cbkj44wh+v9GsPb1clJn97GJuPZ0Mhl2Hh410xOCLAFB/BrP48lYNnVx4AADzarRX2n7+MS1fKdc/b2cjRN9QLD3bxw30dvCVfGJOI6J+a3WKSUmBIahoe+c9uHEwrxAePdsGwHo1r5UkvKMOHv5/G+sOZAABbhQzBns44nVMMO4UcS0ZY5qqzjTVnwwnd7t8A4GCrwL1hXhjQ2Q/3hnlzY1IismjNrruNqDGKyqtx+OoU9N43GVNUH6oWjlj0RCTG9gnGB5tPYefZfJzOKYaDrQLLRnZH73aWuepsY00b0B6llTWoqNGgfydf3NPey2JWCiciskT8G5Kszt5z+dAKIMTLCQHuDnf8ep0D3PD1mBjsOpuPnw9n4MmYQEQGehihUsuitFHgg8fCpS6DiMhqMCSR1bnV1P870budZ5NrPSIiosazvD0ViG7jWkhioCEiItNhSCKrcvFyKdIKymCrkOGu4JZSl0NERE0YQxJZlbpWpMhADzhxNhYREZkQQxJZlbr92u5mVxsREZkYQxJZjRqNFntSLgMw/qBtIiKif2JIIqtx5FIRiitr4OZgq7eNCBERkSkwJJHVqOtq693WEwq5TOJqiIioqWNIIqvBqf9ERGRODElkFdQV129FwpBERESmx5BEVmHvucvQaAWCPZ3QysNR6nKIiKgZYEgiq1A3HoldbUREZC4MSWQVdl0dj9SbU/+JiMhMGJLI4qVdLsOFy2WwkctwV3ALqcshIqJmgiGJLN7OlNqutm6BHnCxt5W4GiIiai4YksjiXetq43gkIiIyH4Yksmg1Gi12p3B9JCIiMj+GJLJoRzOKoK6ogau9DcJbuUtdDhERNSMMSWTR6rraenErEiIiMjOGJLJo19ZH4tR/IiIyL4YksljFFdU4mFYIgOORiIjI/BiSyGLVbUUS1NIRqhbcioSIiMyLIYks1i7drDZ2tRERkfkxJJHF2nmWU/+JiEg6DElkkdILynA+vxQKuQx3hbSUuhwiImqGGJLIItV1tUWq3OHKrUiIiEgCDElkkTj1n4iIpMaQRBZHoxXYnXIZAPdrIyIi6TAkkcU5llGEovJquNjboGsrN6nLISKiZoohiSzOzjO1XW29Qjxho+B/okREJA1+A5HFqZv6z642IiKSEkMSWZSSyhocTLsCALibg7aJiEhCDElkUfadu4warUDrlo4IbMmtSIiISDoMSWRR6qb+927LrjYiIpIWQxJZlJ3cr42IiCwEQxJZjEtXypCaV7sVSSy3IiEiIokxJJHF2HV1VlvXVm5wc+BWJEREJC2GJLIY7GojIiJLwpBEFqF2K5LakHR3KAdtExGR9BiSyCIkZxShsKwaLkobdG3lLnU5REREDElkGXZdbUWKDWnJrUiIiMgi8NuILMKOq/u19QnleCQiIrIMDEkkudLrtiLpw0UkiYjIQjAkkeT2n7+Mao2AqoUDWnMrEiIishAMSSS5HWeuTf2XyWQSV0NERFSLIYkkV7dfG7vaiIjIkjAkkaQyC8txLq8UchnQM4QhiYiILIfkIWnJkiUICgqCvb09YmJikJiYeMvzFy1ahPbt28PBwQEqlQovvfQSKioqzFQtGZtuKxKVO9wcuRUJERFZDklD0tq1azF16lTMnj0bBw8eRNeuXREfH4/c3FyD5//vf//D9OnTMXv2bJw8eRLLly/H2rVr8frrr5u5cjKWHexqIyIiCyVpSFq4cCHGjRuH0aNHo2PHjli6dCkcHR2xYsUKg+fv2bMHvXr1wpNPPomgoCD069cPw4cPv23rE1km7XVbkXB9JCIisjSShaSqqiokJSUhLi7uWjFyOeLi4rB3716D1/Ts2RNJSUm6UJSamopNmzbhwQcfvOn7VFZWQq1W6z3IMhzPVONKWTWclTaIULlLXQ4REZEeG6neOD8/HxqNBj4+PnrHfXx8cOrUKYPXPPnkk8jPz0fv3r0hhEBNTQ1eeOGFW3a3zZs3D2+//bZRayfjqOtquyu4JWy5FQkREVkYq/pm2r59O+bOnYv//Oc/OHjwINatW4eNGzfinXfeuek1M2bMQFFRke6Rnp5uxorpVuqm/t8dyvFIRERkeSRrSfL09IRCoUBOTo7e8ZycHPj6+hq85s0338TTTz+NsWPHAgC6dOmC0tJSPPfcc5g5cybk8hszn1KphFKpNP4HoDtSVlWDpItXtyJpx/FIRERkeSRrSbKzs0NUVBQSEhJ0x7RaLRISEhAbG2vwmrKyshuCkEKhAAAIIUxXLBnd/tQCVGsEAtwdEMStSIiIyAJJ1pIEAFOnTsWoUaPQvXt3REdHY9GiRSgtLcXo0aMBACNHjkRAQADmzZsHABg0aBAWLlyIyMhIxMTEICUlBW+++SYGDRqkC0tkHXZc19XGrUiIiMgSSRqShg0bhry8PMyaNQvZ2dmIiIjA5s2bdYO509LS9FqO3njjDchkMrzxxhvIyMiAl5cXBg0ahPfee0+qj0CNoNUK7DhzdX0kdrUREZGFkolm1k+lVqvh5uaGoqIiuLq6Sl1Os5N2uQyv/HAEiecLYCOX4cAbcXB3tJO6LCIisnBSfH9L2pJEzYdWK/DN/ouY99splFVp4GCrwLtDOjMgERGRxWJIIpNLLyjDtB+PYs+5ywCAmDYtsOCxrgjkgG0iIrJgDElkMkII/C8xDXM3nkRplQb2tnJM7x+GkbFBkMs5WJuIiCwbQxKZREZhOab/eBQ7z9buzdYjyAMLHuuKIE8niSsjIiKqH4YkMiohBL47kI53fj2JksoaKG3keK1/GJ7pGQQFW4+IiMiKMCSR0WQVlWP6j8fw19Xp/d0C3bFgaFeEeDlLXBkREVHDMSTRHRNC4IekS5jz6wkUV9TAzkaOV/qFYkzvYLYeERGR1WJIojuSo67AjHXH8OepXABAV5U7PhwajrbeLhJXRkREdGcYkqhRhBBYfzgDs38+DnVFDewUcrz0QCjG9WkDG4VkWwISEREZDUMSNVhucQVm/pSMrSdyAABdAtzw4eNdEerD1iMiImo6GJKo3oQQ+OVIJmb/chyFZdWwVcgwJS4Uz98dzNYjIiJqchiSqN5m/3IcX+29CADo5O+K/xvaFR38uP8dERE1TQxJVC+7zubjq70XIZcBk+8PxYv3hsCWrUdERNSEMSTRbVXWaDDr52QAwMjYIEyOaydxRURERKbHpgC6rWU7UpGaXwovFyWm9guVuhwiIiKzYEiiW0ovKMMnf6YAAN4Y2AGu9rYSV0RERGQeDEl0S29vOI7KGi1ig1vioa7+UpdDRERkNgxJdFNbT+Tgj5O5sFXI8M6QTpDJuMUIERE1HwxJZFB5lQZv/XIcADC2TzC3GSEiomaHIYkM+nTbWWQUliPA3QH/vq+t1OUQERGZHUMS3SAltwT/3ZEKAJg9qCMc7bhSBBERNT8MSaRHCIHZvySjWiNwX5g3HujoI3VJREREkmBIIj0bjmZhd8plKG3keGsQB2sTEVHzxZBEOsUV1Xj31xMAgAn3tkVgS0eJKyIiIpIOQxLpfLT1LHKLK9HG0wnP3R0sdTlERESSYkgiAMDxzCKs3HMeAPD2Q51gb6uQuCIiIiJpMSQRtFqBN9cnQyuAgV38cHeol9QlERERSY4hifBD0iUcTCuEk50Cb/6ro9TlEBERWQSGpGbuSmkV5v12EgDw0gOh8HWzl7giIiIiy8CQ1MzN33IKV8qq0d7HBaN6BkldDhERkcVgSGrGDqZdwZq/0wEA7z7cGbYK/udARERUh9+KzVSNRos31ydDCOCxqFboEdRC6pKIiIgsCkNSM7V630Ucz1TD1d4G0weESV0OERGRxWFIaoZyiyvw4e9nAACv9Q+Dp7NS4oqIiIgsD0NSMzR340kUV9agays3DI8OlLocIiIii8SQ1MzsOZeP9YczIZMB7wzpDIWcG9gSEREZwpDUjFTVaDHr5+MAgKdiWiO8lbu0BREREVkwhqRmZPmu80jJLUFLJzu80q+91OUQERFZNIakZiKjsBwfJ5wFALz+YAe4OdpKXBEREZFlY0hqJuZsOI7yag2ig1rgkW4BUpdDRERk8RiSmoFtp3Kx5XgOFHIZ3hnSGTIZB2sTERHdDkNSE1dRrcHsX2oHa4/p3QbtfV0kroiIiMg6MCQ1cf/dkYq0gjL4utpj8v3tpC6HiIjIajAkNWEllTX4YmcqAOD1gR3gpLSRuCIiIiLrwZDUhP1v/0WoK2oQ7OWEf3Xxk7ocIiIiq8KQ1ERV1mjwxc7zAIAX7gmBnCtrExERNQhDUhP1Y1IGcosr4edmjyERnPJPRETUUAxJTVCNRovPd5wDAIztEww7G/6aiYiIGorfnk3QpuRsXLxcBg9HWwyPVkldDhERkVWyiJC0ZMkSBAUFwd7eHjExMUhMTLzpuX379oVMJrvhMXDgQDNWbLmEEPhse20r0jM928DRjjPaiIiIGkPykLR27VpMnToVs2fPxsGDB9G1a1fEx8cjNzfX4Pnr1q1DVlaW7pGcnAyFQoGhQ4eauXLLtP10Hk5mqeFkp8Conq2lLoeIiMhqSR6SFi5ciHHjxmH06NHo2LEjli5dCkdHR6xYscLg+S1atICvr6/usXXrVjg6OjIkXVXXivRkTCDcHe0kroaIiMh6SRqSqqqqkJSUhLi4ON0xuVyOuLg47N27t16vsXz5cjzxxBNwcnIy+HxlZSXUarXeo6k6cKEAiRcKYKeQY2yfYKnLISIismqShqT8/HxoNBr4+PjoHffx8UF2dvZtr09MTERycjLGjh1703PmzZsHNzc33UOlaroDmf9ztRXp0agA+LjaS1wNERGRdZO8u+1OLF++HF26dEF0dPRNz5kxYwaKiop0j/T0dDNWaD4ns9T481Qu5DLg+btDpC6HiIjI6kk69cnT0xMKhQI5OTl6x3NycuDr63vLa0tLS7FmzRrMmTPnlucplUoolco7rtXS1Y1FerCLH4I8DXc9EhERUf1J2pJkZ2eHqKgoJCQk6I5ptVokJCQgNjb2ltd+//33qKysxFNPPWXqMi3excul+PVoJgBgfF+2IhERERlDo1uSCgsLkZiYiNzcXGi1Wr3nRo4cWe/XmTp1KkaNGoXu3bsjOjoaixYtQmlpKUaPHq17rYCAAMybN0/vuuXLl2PIkCFo2bJlYz9Ck/H5jlRoBdC3vRc6+btJXQ4REVGT0KiQtGHDBowYMQIlJSVwdXWFTHZt81SZTNagkDRs2DDk5eVh1qxZyM7ORkREBDZv3qwbzJ2Wlga5XL/B6/Tp09i1axd+//33xpTfpOSqK/DDgUsAgBf7tpW4GiIioqZDJoQQDb0oNDQUDz74IObOnQtHR0dT1GUyarUabm5uKCoqgqurq9Tl3LF5m07i8x2piGrtgR9eiNULrERERE2FFN/fjRqTlJGRgUmTJlldQGpqisqqsXrfRQDAi31DGJCIiIiMqFEhKT4+HgcOHDB2LdRAX+29gNIqDcJ8XXBfmLfU5RARETUpjRqTNHDgQLz66qs4ceIEunTpAltbW73nH3roIaMURzdXXqXBl3suAKid0cZWJCIiIuNqVEgaN24cABhco0gmk0Gj0dxZVXRba/5OQ0FpFQJbOGJgFz+pyyEiImpyGhWS/jnln8yrqkaLZTtSAQDP3R0MG4VVL5xORERkkRr87VpdXQ0bGxskJyeboh6qh58PZyCzqAJeLko8FtVK6nKIiIiapAaHJFtbWwQGBrJLTSJarcDSv2q3IBnTuw3sbRUSV0RERNQ0NaqfZubMmXj99ddRUFBg7HroNn4/kYNzeaVwtbfBiJhAqcshIiJqsho1JunTTz9FSkoK/P390bp1azg56W+oevDgQaMUR/qEEPhsewoAYGRsEFzsbW9zBRERETVWo0LSkCFDjFwG1ceec5dx5FIR7G3lGN0rSOpyiIiImrRGhaTZs2cbuw6qh/9cbUV6okcgWjorJa6GiIioaePccStxJL0Qu1Muw0Yuw9g+baQuh4iIqMlrVEuSXC6/5QrPnPlmfHWtSA9F+KOVB/fMIyIiMrVGhaSffvpJ7+fq6mocOnQIq1atwttvv22UwuialNxibDmeAwAYf0+IxNUQERE1D40KSYMHD77h2GOPPYZOnTph7dq1GDNmzB0XRtd8tr12de1+HX3QzsdF4mqIiIiaB6OOSbrrrruQkJBgzJds9jIKy/Hz4QwAwIv3tpW4GiIioubDaCGpvLwcH3/8MQICAoz1kgRg2Y5U1GgFeoa0RITKXepyiIiImo1Gdbd5eHjoDdwWQqC4uBiOjo5YvXq10Ypr7i6XVGLN32kAgBf7shWJiIjInBoVkj766CO9kCSXy+Hl5YWYmBh4eHgYrbjm7svdF1BRrUV4Kzf0attS6nKIiIialUaFpPvuuw8qlcrgMgBpaWkIDOSeYnequKIaq/ZeAAC82DfklksuEBERkfE1akxSmzZtkJeXd8Pxy5cvo00bLnRoDN/sT0NxRQ1CvJzQr6Ov1OUQERE1O40KSUIIg8dLSkpgb29/RwURUFGtwfJd5wEAL9wTArmcrUhERETm1qDutqlTpwIAZDIZZs2aBUfHays/azQa7N+/HxEREUYtsDnanJyNvOJK+LvZY3AEZwsSERFJoUEh6dChQwBqW5KOHTsGOzs73XN2dnbo2rUrXnnlFeNW2AydySkGAMR19IGdDbfXIyIikkKDQtK2bdsAAKNHj8bixYvh6upqkqKau8zCcgBAgLuDxJUQERE1X41qpvjyyy/h6uqKlJQUbNmyBeXltV/qNxurRA2TWVgBAPBnSCIiIpJMo0JSQUEB7r//foSGhuLBBx9EVlYWAGDMmDF4+eWXjVpgc5RxtSWJIYmIiEg6jQpJU6ZMga2tLdLS0vQGbw8bNgybN282WnHNkUYrkK2ubUlq5cGQREREJJVGLSb5+++/Y8uWLWjVqpXe8Xbt2uHixYtGKay5ylFXQKMVsFXI4OWslLocIiKiZqtRLUmlpaV6LUh1CgoKoFTyi/1O1A3a9nWz5/pIREREEmpUSOrTpw+++uor3c8ymQxarRbz58/Hvffea7TimiPdeCQ3drURERFJqVHdbQsWLMB9992HAwcOoKqqCq+99hqOHz+OgoIC7N6929g1Nit1M9sCOB6JiIhIUg0OSdXV1Zg0aRI2bNiArVu3wsXFBSUlJXjkkUcwYcIE+Pn5maLOZiOjsAwA10giIiKSWoNDkq2tLY4ePQoPDw/MnDnTFDU1a1wjiYiIyDI0akzSU089heXLlxu7FsK1gdsMSURERNJq1JikmpoarFixAn/88QeioqLg5OSk9/zChQuNUlxzlMEtSYiIiCxCo0JScnIyunXrBgA4c+aM3nMyGaetN5a6ohrFFTUAAH93e4mrISIiat4aFZLqNrol46rravNwtIWjXaN+NURERGQkjRqTRKbB8UhERESWgyHJgmRwZhsREZHFYEiyIJkctE1ERGQxGJIsSMYVhiQiIiJLwZBkQTgmiYiIyHIwJFmQayGJ0/+JiIikxpBkIWo0WmSrubktERGRpWBIshA5xZXQCsBOIYenk1LqcoiIiJo9hiQLUTdo28/dHnI5Vy0nIiKSGkOShdCNR3JjVxsREZElYEiyEBmc2UZERGRRGJIshG4hSQ7aJiIisgiSh6QlS5YgKCgI9vb2iImJQWJi4i3PLywsxIQJE+Dn5welUonQ0FBs2rTJTNWaToZutW1O/yciIrIEkm41v3btWkydOhVLly5FTEwMFi1ahPj4eJw+fRre3t43nF9VVYUHHngA3t7e+OGHHxAQEICLFy/C3d3d/MUbGReSJCIisiyShqSFCxdi3LhxGD16NABg6dKl2LhxI1asWIHp06ffcP6KFStQUFCAPXv2wNbWFgAQFBRkzpJNQgihm93GkERERGQZJOtuq6qqQlJSEuLi4q4VI5cjLi4Oe/fuNXjNL7/8gtjYWEyYMAE+Pj7o3Lkz5s6dC41Gc9P3qayshFqt1ntYGnVFDUqraj8D920jIiKyDJKFpPz8fGg0Gvj4+Ogd9/HxQXZ2tsFrUlNT8cMPP0Cj0WDTpk1488038eGHH+Ldd9+96fvMmzcPbm5uuodKpTLq5zCGulaklk52sLdVSFwNERERARYwcLshtFotvL298d///hdRUVEYNmwYZs6ciaVLl970mhkzZqCoqEj3SE9PN2PF9cPxSERERJZHsjFJnp6eUCgUyMnJ0Tuek5MDX19fg9f4+fnB1tYWCsW11pYOHTogOzsbVVVVsLOzu+EapVIJpdKyt/nILOLGtkRERJZGspYkOzs7REVFISEhQXdMq9UiISEBsbGxBq/p1asXUlJSoNVqdcfOnDkDPz8/gwHJWlyb/u8ocSVERERUR9LutqlTp2LZsmVYtWoVTp48ifHjx6O0tFQ3223kyJGYMWOG7vzx48ejoKAAkydPxpkzZ7Bx40bMnTsXEyZMkOojGEVmYQUAtiQRERFZEkmXABg2bBjy8vIwa9YsZGdnIyIiAps3b9YN5k5LS4Ncfi3HqVQqbNmyBS+99BLCw8MREBCAyZMnY9q0aVJ9BKPIuFIGgDPbiIiILIlMCCGkLsKc1Go13NzcUFRUBFdXV6nLAQDcNTcB2eoK/DyhF7qq3KUuh4iIyOJI8f1tVbPbmqJqjRY5xXXdbWxJIiIishQMSRLLLqqAEICdjRyeztY7+JyIiKipYUiS2LWZbQ6QyWQSV0NERER1GJIkdm0hSc5sIyIisiQMSRLThSQ3jkciIiKyJAxJEsu4ukZSgAdDEhERkSVhSJIY920jIiKyTAxJErt+4DYRERFZDoYkCQkh2JJERERkoRiSJFRUXo2yKg0AwM+Ns9uIiIgsCUOShOq62jydlbC3VUhcDREREV2PIUlCGVfqxiOxFYmIiMjSMCRJiOORiIiILBdDkoQyi7ixLRERkaViSJIQp/8TERFZLoYkCdWNSWJLEhERkeVhSJJQJluSiIiILBZDkkQqazTILa4EAPhzdhsREZHFYUiSSE5RbUCyt5WjhZOdxNUQERHRPzEkSSTjuun/MplM4mqIiIjonxiSJMKZbURERJaNIUkiuoUk3RiSiIiILBFDkkS42jYREZFlY0iSiK67zYMhiYiIyBIxJEnk2sBtTv8nIiKyRAxJEhBCcCFJIiIiC8eQJIErZdWoqNYCAHzd2JJERERkiRiSJFDXiuTtooTSRiFxNURERGQIQ5IEMjizjYiIyOIxJEkg4wrHIxEREVk6hiQJZHJmGxERkcVjSJJAZhG724iIiCwdQ5IEMgorALC7jYiIyJIxJEmgbkwSW5KIiIgsF0OSmVVUa5BfUgmALUlERESWjCHJzLKLarvaHGwVcHe0lbgaIiIiuhmGJDPLvG5jW5lMJnE1REREdDMMSWbGhSSJiIisA0OSmWXoNrblGklERESWjCHJzHQLSbqxJYmIiMiSMSSZWWbdGkkeDElERESWjCHJzDI5JomIiMgqMCSZkRDiujFJDElERESWjCHJjC6XVqGyRguZDPBx5cBtIiIiS8aQZEZ1XW3eLkrY2fDWExERWTJ+U5tRJrvaiIiIrAZDkhld4sa2REREVoMhyYx00/8ZkoiIiCweQ5IZcfo/ERGR9bCIkLRkyRIEBQXB3t4eMTExSExMvOm5K1euhEwm03vY21vHTLHMIo5JIiIishaSh6S1a9di6tSpmD17Ng4ePIiuXbsiPj4eubm5N73G1dUVWVlZusfFixfNWHHjsSWJiIjIekgekhYuXIhx48Zh9OjR6NixI5YuXQpHR0esWLHiptfIZDL4+vrqHj4+PmasuHEqqjXIL6kCwJYkIiIiayBpSKqqqkJSUhLi4uJ0x+RyOeLi4rB3796bXldSUoLWrVtDpVJh8ODBOH78+E3PrayshFqt1ntIoa4VyclOAVcHG0lqICIiovqTNCTl5+dDo9Hc0BLk4+OD7Oxsg9e0b98eK1aswM8//4zVq1dDq9WiZ8+euHTpksHz582bBzc3N91DpVIZ/XPUR93MNn93B8hkMklqICIiovqTvLutoWJjYzFy5EhERETgnnvuwbp16+Dl5YXPP//c4PkzZsxAUVGR7pGenm7mimvpFpL0YFcbERGRNZC038fT0xMKhQI5OTl6x3NycuDr61uv17C1tUVkZCRSUlIMPq9UKqFUKu+41jt1iYO2iYiIrIqkLUl2dnaIiopCQkKC7phWq0VCQgJiY2Pr9RoajQbHjh2Dn5+fqco0Cm5JQkREZF0kH0E8depUjBo1Ct27d0d0dDQWLVqE0tJSjB49GgAwcuRIBAQEYN68eQCAOXPm4K677kLbtm1RWFiIBQsW4OLFixg7dqyUH+O2rk3/t441nYiIiJo7yUPSsGHDkJeXh1mzZiE7OxsRERHYvHmzbjB3Wloa5PJrDV5XrlzBuHHjkJ2dDQ8PD0RFRWHPnj3o2LGjVB+hXq61JDlKXAkRERHVh0wIIaQuwpzUajXc3NxQVFQEV1dXs7ynVisQNmszqmq02DXtXrTyYFAiIiJqCCm+v61udps1yi+tRFWNFnIZ4OPK7jYiIiJrwJBkBnVrJPm42sNWwVtORERkDfiNbQac2UZERGR9GJLMgBvbEhERWR+GJDO4dIUhiYiIyNowJJnBte42DtomIiKyFgxJZpBZxJYkIiIia8OQZAZ1s9u4uS0REZH1YEgysfIqDQpKqwCwJYmIiMiaMCSZWMbV8UguShu42ttKXA0RERHVF0OSiXH6PxERkXViSDIx3cw2jkciIiKyKgxJJnatJYnT/4mIiKwJQ5KJXWJ3GxERkVViSDIx7ttGRERknRiSTKxujSS2JBEREVkXhiQT0moFsorYkkRERGSNGJJMKK+kEtUaAYVcBm8XpdTlEBERUQMwJJlQ3UKSvq72sFHwVhMREVkTfnObEKf/ExERWS+GJBPizDYiIiLrxZBkQpzZRkREZL0Ykkzo0hUuJElERGStGJJMiN1tRERE1oshyYQyi7i5LRERkbViSDKR0soaFJZVAwD83Di7jYiIyNowJJlIXVebq70NXOxtJa6GiIiIGoohyUQyCjlom4iIyJoxJJlI3fR/DtomIiKyTgxJJqKb2cZB20RERFaJIclEMtndRkREZNUYkkzkEkMSERGRVWNIMpFrC0ly+j8REZE1YkgyAY1WILuobuC2o8TVEBERUWMwJJlAXnElarQCNnIZvFyUUpdDREREjcCQZAIZhWUAAF83eyjkMomrISIiosZgSDKBjKtrJHHQNhERkfViSDKBa4O2GZKIiIisFUOSCTAkERERWT+GJBPgQpJERETWjyHJBC5dqQtJXCOJiIjIWjEkmQC724iIiKwfQ5KRFVdUQ11RA4DdbURERNaMIcnIsq6utO3uaAsnpY3E1RAREVFjMSQZWUbdeCQ3tiIRERFZM4YkI8vgzDYiIqImgSHJyOoGbbfyYEgiIiKyZgxJRnZtjSRO/yciIrJmDElGxu42IiKipoEhycgyubktERFRk2ARIWnJkiUICgqCvb09YmJikJiYWK/r1qxZA5lMhiFDhpi2wHqq0WiRra4NSVxIkoiIyLpJHpLWrl2LqVOnYvbs2Th48CC6du2K+Ph45Obm3vK6Cxcu4JVXXkGfPn3MVOnt5RZXQqMVsFXI4OWslLocIiIiugOSh6SFCxdi3LhxGD16NDp27IilS5fC0dERK1asuOk1Go0GI0aMwNtvv43g4GAzVntrdYO2/dwcIJfLJK6GiIiI7oSkIamqqgpJSUmIi4vTHZPL5YiLi8PevXtvet2cOXPg7e2NMWPG3PY9KisroVar9R6mksGZbURERE2GpCEpPz8fGo0GPj4+esd9fHyQnZ1t8Jpdu3Zh+fLlWLZsWb3eY968eXBzc9M9VCrVHdd9M5zZRkRE1HRI3t3WEMXFxXj66aexbNkyeHp61uuaGTNmoKioSPdIT083WX26hSQZkoiIiKyepDuwenp6QqFQICcnR+94Tk4OfH19bzj/3LlzuHDhAgYNGqQ7ptVqAQA2NjY4ffo0QkJC9K5RKpVQKs0ziJrT/4mIiJoOSVuS7OzsEBUVhYSEBN0xrVaLhIQExMbG3nB+WFgYjh07hsOHD+seDz30EO69914cPnzYpF1p9aHb3JYhiYiIyOpJ2pIEAFOnTsWoUaPQvXt3REdHY9GiRSgtLcXo0aMBACNHjkRAQADmzZsHe3t7dO7cWe96d3d3ALjhuBQyOSaJiIioyZA8JA0bNgx5eXmYNWsWsrOzERERgc2bN+sGc6elpUEut/yhU+qKahRX1gDg7DYiIqKmQCaEEFIXYU5qtRpubm4oKiqCq6ur0V73VLYa/RftRAsnOxx88wGjvS4RERGZ7vv7Viy/icZKFJZVw9Xehq1IRERETYTk3W1NxV3BLXH0rXhU1mikLoWIiIiMgC1JRqa0UUhdAhERERkBQxIRERGRAQxJRERERAYwJBEREREZwJBEREREZABDEhEREZEBDElEREREBjAkERERERnAkERERERkAEMSERERkQEMSUREREQGMCQRERERGcCQRERERGQAQxIRERGRATZSF2BuQggAgFqtlrgSIiIiqq+67+2673FzaHYhqbi4GACgUqkkroSIiIgaqri4GG5ubmZ5L5kwZySzAFqtFpmZmXBxcYFMJjPqa6vVaqhUKqSnp8PV1dWor003x/suDd53afC+S4P3XRrX33cXFxcUFxfD398fcrl5Rgs1u5YkuVyOVq1amfQ9XF1d+YdIArzv0uB9lwbvuzR436VRd9/N1YJUhwO3iYiIiAxgSCIiIiIygCHJiJRKJWbPng2lUil1Kc0K77s0eN+lwfsuDd53aUh935vdwG0iIiKi+mBLEhEREZEBDElEREREBjAkERERERnAkERERERkAEOSkSxZsgRBQUGwt7dHTEwMEhMTpS7JasybNw89evSAi4sLvL29MWTIEJw+fVrvnIqKCkyYMAEtW7aEs7MzHn30UeTk5Oidk5aWhoEDB8LR0RHe3t549dVXUVNTo3fO9u3b0a1bNyiVSrRt2xYrV6409cezGu+//z5kMhmmTJmiO8b7bhoZGRl46qmn0LJlSzg4OKBLly44cOCA7nkhBGbNmgU/Pz84ODggLi4OZ8+e1XuNgoICjBgxAq6urnB3d8eYMWNQUlKid87Ro0fRp08f2NvbQ6VSYf78+Wb5fJZIo9HgzTffRJs2beDg4ICQkBC88847evuA8b7fuR07dmDQoEHw9/eHTCbD+vXr9Z435z3+/vvvERYWBnt7e3Tp0gWbNm1q+AcSdMfWrFkj7OzsxIoVK8Tx48fFuHHjhLu7u8jJyZG6NKsQHx8vvvzyS5GcnCwOHz4sHnzwQREYGChKSkp057zwwgtCpVKJhIQEceDAAXHXXXeJnj176p6vqakRnTt3FnFxceLQoUNi06ZNwtPTU8yYMUN3TmpqqnB0dBRTp04VJ06cEJ988olQKBRi8+bNZv28ligxMVEEBQWJ8PBwMXnyZN1x3nfjKygoEK1btxbPPPOM2L9/v0hNTRVbtmwRKSkpunPef/994ebmJtavXy+OHDkiHnroIdGmTRtRXl6uO6d///6ia9euYt++fWLnzp2ibdu2Yvjw4brni4qKhI+PjxgxYoRITk4W3377rXBwcBCff/65WT+vpXjvvfdEy5Ytxa+//irOnz8vvv/+e+Hs7CwWL16sO4f3/c5t2rRJzJw5U6xbt04AED/99JPe8+a6x7t37xYKhULMnz9fnDhxQrzxxhvC1tZWHDt2rEGfhyHJCKKjo8WECRN0P2s0GuHv7y/mzZsnYVXWKzc3VwAQf/31lxBCiMLCQmFrayu+//573TknT54UAMTevXuFELV/MOVyucjOztad89lnnwlXV1dRWVkphBDitddeE506ddJ7r2HDhon4+HhTfySLVlxcLNq1aye2bt0q7rnnHl1I4n03jWnTponevXvf9HmtVit8fX3FggULdMcKCwuFUqkU3377rRBCiBMnTggA4u+//9ad89tvvwmZTCYyMjKEEEL85z//ER4eHrrfQ917t2/f3tgfySoMHDhQPPvss3rHHnnkETFixAghBO+7KfwzJJnzHj/++ONi4MCBevXExMSI559/vkGfgd1td6iqqgpJSUmIi4vTHZPL5YiLi8PevXslrMx6FRUVAQBatGgBAEhKSkJ1dbXePQ4LC0NgYKDuHu/duxddunSBj4+P7pz4+Hio1WocP35cd871r1F3TnP/PU2YMAEDBw684d7wvpvGL7/8gu7du2Po0KHw9vZGZGQkli1bpnv+/PnzyM7O1rtnbm5uiImJ0bvv7u7u6N69u+6cuLg4yOVy7N+/X3fO3XffDTs7O9058fHxOH36NK5cuWLqj2lxevbsiYSEBJw5cwYAcOTIEezatQsDBgwAwPtuDua8x8b6e4ch6Q7l5+dDo9HofUkAgI+PD7KzsyWqynpptVpMmTIFvXr1QufOnQEA2dnZsLOzg7u7u96519/j7Oxsg7+DuududY5arUZ5ebkpPo7FW7NmDQ4ePIh58+bd8Bzvu2mkpqbis88+Q7t27bBlyxaMHz8ekyZNwqpVqwBcu2+3+jslOzsb3t7ees/b2NigRYsWDfrdNCfTp0/HE088gbCwMNja2iIyMhJTpkzBiBEjAPC+m4M57/HNzmno78CmQWcTmdiECROQnJyMXbt2SV1Kk5eeno7Jkydj69atsLe3l7qcZkOr1aJ79+6YO3cuACAyMhLJyclYunQpRo0aJXF1Tdd3332Hb775Bv/73//QqVMnHD58GFOmTIG/vz/vO90UW5LukKenJxQKxQ0zfnJycuDr6ytRVdZp4sSJ+PXXX7Ft2za0atVKd9zX1xdVVVUoLCzUO//6e+zr62vwd1D33K3OcXV1hYODg7E/jsVLSkpCbm4uunXrBhsbG9jY2OCvv/7Cxx9/DBsbG/j4+PC+m4Cfnx86duyod6xDhw5IS0sDcO2+3ervFF9fX+Tm5uo9X1NTg4KCggb9bpqTV199Vdea1KVLFzz99NN46aWXdK2ovO+mZ857fLNzGvo7YEi6Q3Z2doiKikJCQoLumFarRUJCAmJjYyWszHoIITBx4kT89NNP+PPPP9GmTRu956OiomBra6t3j0+fPo20tDTdPY6NjcWxY8f0/nBt3boVrq6uui+k2NhYvdeoO6e5/p7uv/9+HDt2DIcPH9Y9unfvjhEjRuj+P++78fXq1euGJS7OnDmD1q1bAwDatGkDX19fvXumVquxf/9+vfteWFiIpKQk3Tl//vkntFotYmJidOfs2LED1dXVunO2bt2K9u3bw8PDw2Sfz1KVlZVBLtf/ylMoFNBqtQB4383BnPfYaH/vNGiYNxm0Zs0aoVQqxcqVK8WJEyfEc889J9zd3fVm/NDNjR8/Xri5uYnt27eLrKws3aOsrEx3zgsvvCACAwPFn3/+KQ4cOCBiY2NFbGys7vm6qej9+vUThw8fFps3bxZeXl4Gp6K/+uqr4uTJk2LJkiXNeiq6IdfPbhOC990UEhMThY2NjXjvvffE2bNnxTfffCMcHR3F6tWrdee8//77wt3dXfz888/i6NGjYvDgwQanSUdGRor9+/eLXbt2iXbt2ulNky4sLBQ+Pj7i6aefFsnJyWLNmjXC0dGx2UxF/6dRo0aJgIAA3RIA69atE56enuK1117TncP7fueKi4vFoUOHxKFDhwQAsXDhQnHo0CFx8eJFIYT57vHu3buFjY2N+L//+z9x8uRJMXv2bC4BIKVPPvlEBAYGCjs7OxEdHS327dsndUlWA4DBx5dffqk7p7y8XLz44ovCw8NDODo6iocfflhkZWXpvc6FCxfEgAEDhIODg/D09BQvv/yyqK6u1jtn27ZtIiIiQtjZ2Yng4GC996AbQxLvu2ls2LBBdO7cWSiVShEWFib++9//6j2v1WrFm2++KXx8fIRSqRT333+/OH36tN45ly9fFsOHDxfOzs7C1dVVjB49WhQXF+udc+TIEdG7d2+hVCpFQECAeP/9903+2SyVWq0WkydPFoGBgcLe3l4EBweLmTNn6k0j532/c9u2bTP49/moUaOEEOa9x999950IDQ0VdnZ2olOnTmLjxo0N/jwyIa5bbpSIiIiIAHBMEhEREZFBDElEREREBjAkERERERnAkERERERkAEMSERERkQEMSUREREQGMCQRERERGcCQRERERGQAQxIRmdz27dshk8lu2CyXiMiSMSQRkVH17dsXU6ZM0TvWs2dPZGVlwc3NTZqibiIoKAiLFi2SugwislA2UhdARE2fnZ0dfH19pS6DiKhB2JJEREbzzDPP4K+//sLixYshk8kgk8lw4cKFG7rbVq5cCXd3d/z6669o3749HB0d8dhjj6GsrAyrVq1CUFAQPDw8MGnSJGg0Gt3rV1ZW4pVXXkFAQACcnJwQExOD7du337QeIQTeeustBAYGQqlUwt/fH5MmTQJQ2+J18eJFvPTSS7pa6+zatQt9+vSBg4MDVCoVJk2ahNLSUt3zQUFBeOeddzB8+HA4OTkhICAAS5YsMe7NJCLJMSQRkdEsXrwYsbGxGDduHLKyspCVlQWVSmXw3LKyMnz88cdYs2YNNm/ejO3bt+Phhx/Gpk2bsGnTJnz99df4/PPP8cMPP+iumThxIvbu3Ys1a9bg6NGjGDp0KPr374+zZ88afI8ff/wRH330ET7//HOcPXsW69evR5cuXQAA69atQ6tWrTBnzhxdrQBw7tw59O/fH48++iiOHj2KtWvXYteuXZg4caLeay9YsABdu3bFoUOHMH36dEyePBlbt241xm0kIkshiIiM6J577hGTJ0/WO7Zt2zYBQFy5ckUIIcSXX34pAIiUlBTdOc8//7xwdHQUxcXFumPx8fHi+eefF0IIcfHiRaFQKERGRobea99///1ixowZBmv58MMPRWhoqKiqqjL4fOvWrcVHH32kd2zMmDHiueee0zu2c+dOIZfLRXl5ue66/v37650zbNgwMWDAAIPvQ0TWiS1JRCQJR0dHhISE6H728fFBUFAQnJ2d9Y7l5uYCAI4dOwaNRoPQ0FA4OzvrHn/99RfOnTtn8D2GDh2K8vJyBAcHY9y4cfjpp59QU1Nzy7qOHDmClStX6r1HfHw8tFotzp8/rzsvNjZW77rY2FicPHmywfeBiCwXB24TkSRsbW31fpbJZAaPabVaAEBJSQkUCgWSkpKgUCj0zrs+WF1PpVLh9OnT+OOPP7B161a8+OKLWLBgAf76668b3qtOSUkJnn/+ed3YpesFBgbW+/MRkfVjSCIio7Kzs9MbbG0skZGR0Gg0yM3NRZ8+fep9nYODAwYNGoRBgwZhwoQJCAsLw7Fjx9CtWzeDtXbr1g0nTpxA27Ztb/m6+/btu+HnDh061P8DEZHFY0giIqMKCgrC/v37ceHCBTg7O6NFixZGed3Q0FCMGDECI0eOxIcffojIyEjk5eUhISEB4eHhGDhw4A3XrFy5EhqNBjExMXB0dMTq1avh4OCA1q1b62rdsWMHnnjiCSiVSnh6emLatGm46667MHHiRIwdOxZOTk44ceIEtm7dik8//VT32rt378b8+fMxZMgQbN26Fd9//z02btxolM9KRJaBY5KIyKheeeUVKBQKdOzYEV5eXkhLSzPaa3/55ZcYOXIkXn75ZbRv3x5DhgzB33//fdNuMHd3dyxbtgy9evVCeHg4/vjjD2zYsAEtW7YEAMyZMwcXLlxASEgIvLy8AADh4eH466+/cObMGfTp0weRkZGYNWsW/P399V775ZdfxoEDBxAZGYl3330XCxcuRHx8vNE+KxFJTyaEEFIXQURkTYKCgjBlypQbVhYnoqaFLUlEREREBjAkERERERnA7jYiIiIiA9iSRERERGQAQxIRERGRAQxJRERERAYwJBEREREZwJBEREREZABDEhEREZEBDElEREREBjAkERERERnw/3NKis3TodnbAAAAAElFTkSuQmCC\n" }, "metadata": {} } ], "source": [ "# Create a Neural SquareCB pearl agent with 1-hot action representation\n", "action_representation_module = OneHotActionTensorRepresentationModule(\n", " max_number_actions= env.unique_labels_num,\n", ")\n", "\n", "agent = PearlAgent(\n", " policy_learner=NeuralBandit(\n", " feature_dim = env.observation_dim + env.unique_labels_num,\n", " hidden_dims=[64, 16],\n", " training_rounds=10,\n", " learning_rate=0.01,\n", " action_representation_module=action_representation_module,\n", " exploration_module= SquareCBExploration(gamma = env.observation_dim * env.unique_labels_num * number_of_steps)\n", " ),\n", " replay_buffer=FIFOOffPolicyReplayBuffer(100_000),\n", " device_id=-1,\n", ")\n", "\n", "\n", "info = online_learning(\n", " agent=agent,\n", " env=env,\n", " number_of_steps=number_of_steps,\n", " print_every_x_steps=100,\n", " record_period=record_period,\n", " learn_after_episode=True,\n", ")\n", "torch.save(info[\"return\"], \"SquareCB-return.pt\")\n", "plt.plot(record_period * np.arange(len(info[\"return\"])), info[\"return\"], label=\"SquareCB\")\n", "plt.xlabel(\"time step\")\n", "plt.ylabel(\"return\")\n", "plt.legend()\n", "plt.show()" ] }, { "cell_type": "markdown", "metadata": { "id": "cjLo_VzyLNGR" }, "source": [ "## Contextual Bandits learners: LinUCB\n", "\n", "Next, we describe how to use the neural version of the LinUCB algorithm with Pearl, which uses UCB type of exploration with neural architectures. LinUCB and its neural version are generalizations of the seminal Upper Confidence Bound (UCB) algorithm. Both execute a policy of the following form:\n", "$$\n", "\\pi(a,x) \\in \\arg\\max_a \\widehat{r}(x,a) + \\mathrm{score}(x,a),\n", "$$\n", "that is, both use a function that estimates the expected reward with an additional bonus term that quantifies the potential of choosing an action given a certain context. A common way to estimate the score function in the linear case with features $\\phi(x,a)$ is:\n", "$$\n", "\\mathrm{score}(x,a) = \\alpha ||\\phi(x,a) ||_{A^{-1}}\\\\\n", "\\text{where } A= \\lambda I + \\sum_{n\\leq t} \\phi(x_n,a_n)\\phi^T(x_n,a_n).\n", "$$\n", "\n", "To implement the LinUCB algorithm in Pearl, use the `NeuralLinearBandit` policy learner module. This module supports (i) learning a reward model, and (ii) calculating a score function by estimating the uncertainty using the last layer features. Further, we set the exploration module to an instance of `UCBExploration` and set the `alpha` hyper-parameter to enable the agent with the UCB-like update rule.\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "cDauzO74nS4c", "colab": { "base_uri": "https://localhost:8080/", "height": 1000 }, "outputId": "481e1140-da7a-4194-b6f4-073ac2805528" }, "outputs": [ { "output_type": "stream", "name": "stderr", "text": [ "/usr/local/lib/python3.10/dist-packages/ipykernel/ipkernel.py:283: DeprecationWarning: `should_run_async` will not call `transform_cell` automatically in the future. Please pass the result to `transformed_cell` argument and any exception that happen during thetransform in `preprocessing_exc_tuple` in IPython 7.17 and above.\n", " and should_run_async(code)\n" ] }, { "output_type": "stream", "name": "stdout", "text": [ "episode 100, step 100, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 0.20280151069164276\n", "episode 200, step 200, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 0.054456718266010284\n", "episode 300, step 300, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: -0.13178661465644836\n", "episode 400, step 400, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 0.9865397214889526\n", "episode 500, step 500, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 0.14140896499156952\n", "episode 600, step 600, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 1.1187363862991333\n", "episode 700, step 700, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 1.2206898927688599\n", "episode 800, step 800, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: -0.22980380058288574\n", "episode 900, step 900, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 1.0912549495697021\n", "episode 1000, step 1000, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: -0.09148281812667847\n", "episode 1100, step 1100, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 0.9348994493484497\n", "episode 1200, step 1200, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 0.0068538920022547245\n", "episode 1300, step 1300, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 0.9425393342971802\n", "episode 1400, step 1400, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 0.016242675483226776\n", "episode 1500, step 1500, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: -0.09567844122648239\n", "episode 1600, step 1600, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 0.9641198515892029\n", "episode 1700, step 1700, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 1.0396792888641357\n", "episode 1800, step 1800, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 0.9747456908226013\n", "episode 1900, step 1900, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 0.8553079962730408\n", "episode 2000, step 2000, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 0.9961886405944824\n", "episode 2100, step 2100, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 1.0519009828567505\n", "episode 2200, step 2200, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: -0.05198976397514343\n", "episode 2300, step 2300, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 0.960572361946106\n", "episode 2400, step 2400, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 0.9912083745002747\n", "episode 2500, step 2500, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 0.055912960320711136\n", "episode 2600, step 2600, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 0.9400607943534851\n", "episode 2700, step 2700, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 1.0280994176864624\n", "episode 2800, step 2800, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 0.024195637553930283\n", "episode 2900, step 2900, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 1.1863090991973877\n", "episode 3000, step 3000, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 1.147364616394043\n", "episode 3100, step 3100, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 1.197069525718689\n", "episode 3200, step 3200, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 0.9864020347595215\n", "episode 3300, step 3300, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 0.899193286895752\n", "episode 3400, step 3400, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 1.1261299848556519\n", "episode 3500, step 3500, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 1.1639732122421265\n", "episode 3600, step 3600, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 0.833449125289917\n", "episode 3700, step 3700, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 1.0843310356140137\n", "episode 3800, step 3800, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 0.9897075891494751\n", "episode 3900, step 3900, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 1.2469022274017334\n", "episode 4000, step 4000, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 0.991430401802063\n", "episode 4100, step 4100, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 0.936269223690033\n", "episode 4200, step 4200, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 1.0206660032272339\n", "episode 4300, step 4300, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 0.9332488775253296\n", "episode 4400, step 4400, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 0.9900529384613037\n", "episode 4500, step 4500, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 1.0219541788101196\n", "episode 4600, step 4600, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 0.8899850845336914\n", "episode 4700, step 4700, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 0.9463319778442383\n", "episode 4800, step 4800, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 1.121014952659607\n", "episode 4900, step 4900, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 1.040822982788086\n", "episode 5000, step 5000, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 0.984503448009491\n", "episode 5100, step 5100, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 0.9031474590301514\n", "episode 5200, step 5200, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 1.0292550325393677\n", "episode 5300, step 5300, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 0.8698779344558716\n", "episode 5400, step 5400, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 0.999628484249115\n", "episode 5500, step 5500, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 0.9985622763633728\n", "episode 5600, step 5600, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: -0.11453455686569214\n", "episode 5700, step 5700, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 1.1535426378250122\n", "episode 5800, step 5800, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 1.0082354545593262\n", "episode 5900, step 5900, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 1.0808966159820557\n", "episode 6000, step 6000, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 0.9922400712966919\n", "episode 6100, step 6100, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 1.0792633295059204\n", "episode 6200, step 6200, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 0.8470718860626221\n", "episode 6300, step 6300, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 0.8804739117622375\n", "episode 6400, step 6400, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 1.0601600408554077\n", "episode 6500, step 6500, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 1.0142043828964233\n", "episode 6600, step 6600, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 0.948247492313385\n", "episode 6700, step 6700, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 0.9545789957046509\n", "episode 6800, step 6800, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 1.010140061378479\n", "episode 6900, step 6900, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 0.8813512921333313\n", "episode 7000, step 7000, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 0.9689813852310181\n", "episode 7100, step 7100, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 1.060390591621399\n", "episode 7200, step 7200, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 0.9932740330696106\n", "episode 7300, step 7300, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 1.0427846908569336\n", "episode 7400, step 7400, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 1.0150879621505737\n", "episode 7500, step 7500, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 1.0593535900115967\n", "episode 7600, step 7600, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: -0.04487299919128418\n", "episode 7700, step 7700, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 1.0764495134353638\n", "episode 7800, step 7800, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 0.8947196006774902\n", "episode 7900, step 7900, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 1.0651267766952515\n", "episode 8000, step 8000, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 0.015225519426167011\n", "episode 8100, step 8100, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 0.9122466444969177\n", "episode 8200, step 8200, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 1.0724564790725708\n", "episode 8300, step 8300, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 1.0051934719085693\n", "episode 8400, step 8400, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 1.0062308311462402\n", "episode 8500, step 8500, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 0.9350183606147766\n", "episode 8600, step 8600, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 0.920856773853302\n", "episode 8700, step 8700, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 0.9818281531333923\n", "episode 8800, step 8800, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 0.9964724183082581\n", "episode 8900, step 8900, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 1.0790421962738037\n", "episode 9000, step 9000, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 0.0036260781344026327\n", "episode 9100, step 9100, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 0.9669089913368225\n", "episode 9200, step 9200, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 0.9350236058235168\n", "episode 9300, step 9300, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 1.08956778049469\n", "episode 9400, step 9400, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 0.76462721824646\n", "episode 9500, step 9500, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 0.9117434620857239\n", "episode 9600, step 9600, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 0.015561874024569988\n", "episode 9700, step 9700, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 1.0754772424697876\n", "episode 9800, step 9800, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 0.9884140491485596\n", "episode 9900, step 9900, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 0.9845787286758423\n", "episode 10000, step 10000, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 0.9720855951309204\n" ] }, { "output_type": "display_data", "data": { "text/plain": [ "
" ], "image/png": "iVBORw0KGgoAAAANSUhEUgAAAkkAAAGwCAYAAAC99fF4AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAABR0klEQVR4nO3de1xT9f8H8Nc2YDAuA0TuIF7xhoIYiGVqUVh+LbuaWZqZ3eybRn0rs7TsQuU308qy+mn27aZZapmmKaVlmiiIihcUbyByFdm4M7bP7w9iuhwKuO1s8Ho+HnsUZ+fsvHem7OXnfM77yIQQAkRERERkQi51AURERET2iCGJiIiIyAyGJCIiIiIzGJKIiIiIzGBIIiIiIjKDIYmIiIjIDIYkIiIiIjOcpC7A1gwGA86cOQNPT0/IZDKpyyEiIqIWEEKgoqICwcHBkMttM8bT4ULSmTNnEBYWJnUZRERE1AZ5eXkIDQ21yb46XEjy9PQE0HiQvby8JK6GiIiIWkKr1SIsLMz4PW4LHS4kNZ1i8/LyYkgiIiJyMLacKsOJ20RERERmMCQRERERmcGQRERERGRGh5uT1FJ6vR46nU7qMjosZ2dnKBQKqcsgIqIOjCHpH4QQKCwsRHl5udSldHje3t4IDAxkPysiIpIEQ9I/NAUkf39/qFQqfkFLQAiB6upqFBcXAwCCgoIkroiIiDoihqQL6PV6Y0Dq1KmT1OV0aG5ubgCA4uJi+Pv789QbERHZHCduX6BpDpJKpZK4EgLOfw6cG0ZERFJgSDKDp9jsAz8HIiKSEkMSERERkRkMSURERERmSBqSfv/9d4wZMwbBwcGQyWRYs2bNZbfZsmULBg0aBKVSiR49emDZsmVWr9PRtfTYEhER0XmShqSqqioMHDgQixYtatH6J06cwOjRozFy5EhkZmZixowZeOihh7Bx40YrV2r/HnjgAYwdO9bscwUFBbjpppta/FrLli2Dt7e32efMBa7vv/8eI0aMgFqthoeHBwYMGIC5c+eirKzM+Hoymcz48PDwQGxsLFatWtXimoiIyDH8cbQEeoOQugyLkDQk3XTTTXjttddw2223tWj9xYsXo2vXrnjnnXfQp08fPPHEE7jzzjvx7rvvWrlSxxYYGAilUmmV1541axbGjRuHq666Cj///DOysrLwzjvvYO/evfjiiy+M63l5eaGgoAAFBQXYs2cPkpKScPfddyM7O9sqdRERke2lnzqH+5ekIWnB79DpDVKXc8Ucak7Sjh07kJiYaLIsKSkJO3bsaHaburo6aLVak0drCCFQXd8gyUMIyyTxC0d/Tp48CZlMhlWrVmHkyJFQqVQYOHDgJY9hc9LS0vDGG2/gnXfewbx58zB06FBERETghhtuwPfff49JkyaZ1BAYGIjAwED07NkTr732GuRyOfbt22eR90hERNJbvPUYAGBQuDecFQ4VMcxyqGaShYWFCAgIMFkWEBAArVaLmpoaYwPCC6WkpOCVV15p8z5rdHr0nS3N6byDc5OgcrHORzRr1iz897//Rc+ePTFr1iyMHz8eOTk5cHJq+f6++uoreHh44PHHHzf7fHOn7PR6Pf73v/8BAAYNGtTq2omIyP7kFFdi08EiAMDD13aXuBrLcKiQ1BYzZ85EcnKy8WetVouwsDAJK7IPzzzzDEaPHg0AeOWVV9CvXz/k5OSgd+/eLX6No0ePolu3bnB2dr7suhqNBh4eHgCAmpoaODs745NPPkH37u3jLxIRUUf3ye+No0g39A1AD38PiauxDIcKSYGBgSgqKjJZVlRUBC8vL7OjSACgVCqvaD6Om7MCB+cmtXn7K+HmbL1bcQwYMMD4/033RisuLm5VSGrN6UBPT09kZGQAAKqrq7F582Y8+uij6NSpE8aMGdPi1yEiIvtTqKnF6j35AIBHh7eff/w6VEhKSEjA+vXrTZZt2rQJCQkJVtunTCaz2ikvKV04+tPU2dpgaJxk5+XlhaqqKhgMBsjl588pl5eXAwDUajUAoFevXti2bRt0Ot1lR5Pkcjl69Ohh/HnAgAH45Zdf8NZbbzEkERE5uKV/noBOLxAX4YvYLj5Sl2Mxks6qqqysRGZmJjIzMwE0XuKfmZmJ3NxcAI2nyiZOnGhc/9FHH8Xx48fx7LPP4vDhw/jwww/x7bff4qmnnpKi/HYrMjISDQ0Nxs+lSdNIUK9evQAA9957LyorK/Hhhx+afZ2mUNUchUKBmpqaK66XiIiko6nR4eudjd/bj47oJnE1liXpEMnu3bsxcuRI489Nc4cmTZqEZcuWoaCgwBiYAKBr165Yt24dnnrqKSxcuBChoaH4v//7PyQlSXM6zN5oNJqLgk2nTp1a/Tr9+vXDjTfeiAcffBDvvPMOunXrhuzsbMyYMQPjxo1DSEgIACA+Ph7PPvssnn76aeTn5+O2225DcHAwcnJysHjxYlxzzTWYPn06gMZTc4WFhQAa5yRt2rQJGzduxOzZs6/sTRMRkaS+2nkKlXUN6BXggRG9/KUux6IkDUkjRoy45LwWc920R4wYgT179lixKse1ZcsWxMTEmCybMmVKm15rxYoVmDNnDh555BGcOXMGoaGhuO222/DSSy+ZrPfWW28hNjYWixYtwuLFi2EwGNC9e3fceeedJi0AtFqtce6TUqlEly5dMHfuXDz33HNtqo+IiKRXq9Nj6baTAIBHru0Oubx93ZhcJizVjMdBaLVaqNVqaDQaeHl5mTxXW1uLEydOoGvXrnB1dZWoQmrCz4OIyL59k5aLmav2I1jtiq3PjrRqb6RLfX9bi+N3eiIiIiKb0xsEPvn9OADgwWu6tovmkf/U/t4RERERWd0vBwpxorQKajdnjI8Ll7ocq2BIIiIiolYRQhhvQTIxoQvcle2vVQ7AkGRWB5umZbf4ORAR2ae/jpdh72kNlE5yTBoaIXU5VsOQdIGmhojV1dUSV0LA+c+hJbc9ISIi22kaRbprcCj8PNp+Vwt71z7Hx9pIoVDA29sbxcXFAACVSmXsRk22I4RAdXU1iouL4e3tDYXCerdnISKi1jl4RoutR0oglwFTh7Wv5pH/xJD0D4GBgQBgDEokHW9vb+PnQURE9qHpRrY3RQWhSyd3iauxLoakf5DJZAgKCoK/vz90Op3U5XRYzs7OHEEiIrIzeWXVWLuvAADwWDu6kW1zGJKaoVAo+CVNRER0gSXbTkBvELimhx/6h6ilLsfqOHGbiIiILqusqh7LdzXeT/WR4e17LlIThiQiIiK6rP/tOIlanQH9gr1wTQ8/qcuxCYYkIiIiuqTq+gZ8vv0kAODR4d07zJXfDElERER0Sd/uysO5ah3CfVW4qX/HueqYIYmIiIia1aA34NM/TgAApg7rCqd2eCPb5nScd0pERESttm5/AfLLa9DJ3QV3DQ6TuhybYkgiIiIisxpvZHscAPDA0Ai4Ones1jgMSURERGTW70dLcahAC5WLAvcndJG6HJtjSCIiIiKzFm9pvAXJPVeFw1vlInE1tseQRERERBfZm1eOHcfPwkkuw5RhXaUuRxIMSURERHSRxVsbR5FuGRiMEG83iauRBkMSERERmThRWoUNBwoBAI90gBvZNochiYiIiEx88vtxCAFc19sfkYGeUpcjGYYkIiIiMiquqMX3GacBAI9c2zFuZNschiQiIiIyWvbnSdQ3GBAT7o24rr5SlyMphiQiIiICAFTU6vDFX6cAdKwb2TaHIYmIiIgAAN+k5aKitgHdOrvjhj4BUpcjOYYkIiIiQl2DHku2Nd7I9pFru0Eu79ijSABDEhEREQH4IfMMirR1CPBSYmxMiNTl2AWGJCIiog7OYBD4+O/mkQ9e3RVKp451I9vmMCQRERF1cJsPFeFYSRU8lU4YHx8udTl2gyGJiIioA8srq8b7v+YAACYM6QIvV2eJK7IfTlIXQERERLZVU6/Hz1kFWLn7NHYcPwsAcHGS48GrI6QtzM4wJBEREXUAQgjsySvHyt2n8dPeM6ioawAAyGTANT38MG1kD/h7uUpcpX1hSCKiVtuTew6v/nQQA8O8MSOxF9RuHJ4nslfFFbVYnZGPlemnkVNcaVwe5uuGu2LDcEdsKEK83SSs0H4xJBFRq6zfX4CnVmSirsGAjNxyrN17Bs+N6o07BoWyrwqRndDpDfj1cDFW7j6N37KLoTcIAICrsxw39w/CnYNDMaRrJ/6dvQyGJCJqESEEPtp6DG9vyAbQODx/RlOD4yVV+M93+/BNWi7m3tof/UPUEldK1HEdKarAt7vysCYzH6WV9cblMeHeuHtwGEYPCOLE7FZgSCKiy6pvMOClNVlYsTsPAPDA0Ai89K++0BsElv55Au+lHkVGbjlu+WAb7hvSBU/fEAm1ir+I7dkPmfn4Lv00runhh9tiQjgXxYFpanRYu/cMVqafxt68cuNyPw8l7hgUgrsGh6KHv6d0BTowmRBCSF2ELWm1WqjVamg0Gnh5eUldDpHd01Tr8NhX6dh+7CzkMmDOmH6YNDTCZJ0CTQ1eW3cI6/YVAAA6ubvguVG9cWcsT8HZm+r6Bsz+4QC+Sz9tXCaXAdf26ow7BoXihr4BcHVmI0FHkJF7Dp9vP4kNWYWoazAAAJzkMlzX2x93Dw7D8MjOcFa0n04/Unx/Sx6SFi1ahHnz5qGwsBADBw7E+++/j7i4OLPr6nQ6pKSk4PPPP0d+fj4iIyPx1ltvYdSoUS3eH0MSUcvlnq3G5GVpOFZSBXcXBd6/NwbX9W7+ppd/5pRizo8HjJNDY8K98SpPwdmNQwVaPPF1Bo6VVEEuA8bHhSO7sAK7T50zruPl6oR/DQzGnbGhiAnz7vB3gbdHeWXVeGvDYfz09z9KAKBXgAfuHhyGsTEh8PNQSlid9XS4kLRixQpMnDgRixcvRnx8PBYsWICVK1ciOzsb/v7+F63/3HPP4csvv8Snn36K3r17Y+PGjUhOTsb27dsRExPTon0yJBG1TPqpMkz9XzrKquoRpHbFkklXoW/w5f/O1DcY8NmfJ7Aw9Siq6/WQyYAJ8eF45sZIeKtcbFA5/ZMQAl/tzMXcnw6ivsGAAC8lFt4TgyHdOgEATpRWYVXGaXyffhpnNLXG7br5ueOO2FDcFhOCYCtf/VRZ14AD+RocOKOFh9IJSf0DedXkP1TU6vDhlmNYsu0E6hsMkMmA22NCMTGhCwaEqtt9oO1wISk+Ph5XXXUVPvjgAwCAwWBAWFgY/v3vf+P555+/aP3g4GDMmjUL06ZNMy6744474Obmhi+//LJF+2RIIrq8H/eewTMr96K+wYD+IV5YMukqBLRyzkqhphavrz+EtXvPAAB83V3w3KhI3BUbxlNwNqSp0WHmqn1Yv78QADAysjP+e9dAdDIz2mAwCOw4fhbfp5/Gz1mFqNHpATT20bm6ux/ujA1FUr9AuLlc2em4ilodDpzRIitfg/1/P06UVuHCbyMXJzlu6BuAOweFYlhPPzhJeNpIpzfgeEkVIvxUktzTTG8Q+HZ3Ht75Jds4GXto9054cXTfFv3Dpb3oUCGpvr4eKpUK3333HcaOHWtcPmnSJJSXl+OHH364aJtOnTrh7bffxpQpU4zL7rvvPmzbtg0nT540u5+6ujrU1dUZf9ZqtQgLC2NIIjJDCIEPfs3BO5uOAABu6BuAhfdEQ+XS9ms8th8rxZwfDuDo36fgosMaT8FFhfIUnLXtyT2Hf3+zB6fP1cBZIcNzo3rjwau7tiikVtY1YP3+Anyffho7T5QZl3sonTA6Kgh3xIbiqgify45eaGt1OJB/PhBl5WtwvLTK7LpBalf0C1Yjr6wa2UUVxuWdPZUYGx2MO2JD0TvQNr+3NdU6bDlSjE0Hi7D1SAkqahvgo3LG7YNCMT4uzGYTof/MKcWrPx3E4cLG49HVzx0v3NwHiX382/3I0T91qJB05swZhISEYPv27UhISDAuf/bZZ7F161bs3Lnzom3uvfde7N27F2vWrEH37t2RmpqKW2+9FXq93iQIXejll1/GK6+8ctFyhiQiU3UNesxctR+rMvIBAA9d0xUzb+4DhQVGfXR6Az7ffhLvbjqCqr9PwY2PC8d/boyEj7vlTsEZDAJ6IdrVZNW2MBgEPv3jOOZtzEaDQSDM1w0fjB+EgWHebXq93LPVWLXnNL7POI28shrj8i6dVLg9JhS3DwpBmK8KmhodDlwwOpSVr8HJs9VmXzNY7Yr+IWpEhajRP7Txv01zaYQQOHBGi+/ST+PHvWdQVnX+UvZ+wV64Y1Aobo0ONjsadiVOna3CpoNF2HyoCLtOnjP2FgIAZ4UMOv35n6+K8ME9V4Vj9IAgq0x0P15SiTfWH8LmQ8UAGueKTU/shfuHdIGLU8f8882QhEuHpJKSEkydOhVr166FTCZD9+7dkZiYiKVLl6Kmpuai9QGOJBG1RHl1PR7+Ih1pJ8qgkMvwyi39cN+QLhbfT5G2Fm+sP4QfMhtPwfmonPHsqN4YN/jiU3ANegPKa3Qor65HWZUO56rrUV5dj3PVOpyrqse5f/x/ebUO5TU66A0CAV5KhPuqEOarQvg/Hp09le36X+CllXV4+tu92HqkBAAwekAQUm6PskhvHINBYNfJMnyXfhrr9xegql5vfC5Y7Woyn+lCId5uiApRIypUjX7BXogKUbc44NQ3GLAluxjfZ5zGr4eLjUHFSS7DyN7+uGNQKK7r7d+m4KA3CGTmncOmg8XYfKjIpBs10DgZ+vo+AUjsE4CoEDW25ZTgm7Q8/Hr4fHNGT1cn3B4TgnviwtEn6Mq/U8qr67Ew9Si+2HEKDQYBhVyG+4d0wfTre1r0HxSOqEOFpLacbmtSW1uLs2fPIjg4GM8//zx++uknHDhwoEX75ZwkIlMnS6swedkunCitgofSCYsmDMLwXp2tus+/jp/FnB8OGE+p9A3yQoCXEmXVjaHoXFU9tLUNVtm3q7McoT7nQ9OFQSrM1+2KTi1KbXtOKWasyERxRR2UTnK8fEs/3HNVmFVCYXV9AzZkFeL7jNPYfuyscT5RqE9jIDKOEoWo4WuhL/eyqnqs3XsG32ecxr7TGuNyH5UzbhnYeDouKuTSE5ir6hrwx9FSbD5UhF8PF5uMUjnJZYjr6ovEv4NReCeV2dco0tZi5e48LN+Vh9Pnzv8DfWCYN+6NC8O/BgTDXdm6P0c6vQFf/nUKC1OPorxaBwC4rrc/Xri5D3r4e7TqtdqrDhWSgMaJ23FxcXj//fcBNE7cDg8PxxNPPGF24vY/6XQ69OnTB3fffTfeeOONFu2TIYnovLQTZXj4i90or9YhxNsNSx+4CpGBtplrodMb8L8dp/DupiOorGs+EKndnOGjcoa3ygW+7i7wVjnDR+UCH5UzfNxd4KM6v8zX3QUKuQynz9Ugt6waeX8/cv9+nCmvgeEyv/H8PJQI93VDuK8K/YLVGBsTgs6e9n1JdYPegIWpR/HBbzkQAujp74EP7h1ks8/yTHnj8Y4M8LTZaMeRogp8n3Eaa/bko0h7/mxBT38P4xV5TRcbFGhqkHqocbRo+7GzqP+7pxDQeBprRKQ/EvsGYHivzq26os5gENiWU4rlu3Lxy4EiNPz9h8vdRYFbokNwb1z4ZefeCSHw6+FivL7+EI6XNM7VigzwxIv/6oNhPa37jxVH0+FC0ooVKzBp0iR8/PHHiIuLw4IFC/Dtt9/i8OHDCAgIwMSJExESEoKUlBQAwM6dO5Gfn4/o6Gjk5+fj5ZdfxokTJ5CRkQFvb+8W7ZMhiajR6j2n8dx3+1GvN2BgqBqfThoMf0/bd10u1tZi48EiKBVyeKuc/w5CjSFI7eZs0auadHqD8Qu9MUTVmIQoTY3uom2c5DLc0DcA4+PCcU0PP7u7Mq9AU4Pp32Qi7WTj5Opxg8Pw8i39rvgKNEeh/zuofJ9+GhsPnG+qKJcBV/fww7nqemTla0226dJJhcQ+Abi+jz+uivC1yBy2koo6fJ9xGsvTck3mYfUL9sI9ceG4NTr4olOehwu1eO2nQ9iWUwqgsQlr8o29MG5wmKRX89mrDheSAOCDDz4wNpOMjo7Ge++9h/j4eADAiBEjEBERgWXLlgEAtm7disceewzHjx+Hh4cHbr75Zrz55psIDg5u8f4YkqijE0JgweajWJh6FABwU/9AzL87usN8qV6KplqHvHONgenU2Wr8crAQe3LLjc+H+brhnqvCcVdsqF3cxmPzwSI8891elFfr4KF0wuu39cet0SFSlyUZba0O6/Y1XpF3YYNMmQwYFO7z92k0f/Tw97DavDQhBP46XoZv0nKxIasQ9frG0ObmrMC/BgThnrhwhPuqMH/TEazYlQuDAFwUcky+JgLTRvbgfdUuoUOGJFtjSKKOrK5Bj+e+24c1f0+cfnR4dzybFGl3oyP25FCBFsvTcrFqTz4q/p4npZDLkNjHH/fEhePanp0tcgVga9Q16PHWz9lY+ucJAEBUiBrvj49BhJ+7TeuwZydLq/DLwUL4qFwwsre/JF2oy6rqsSrjNJbvyjOZFK6Qy4wTv2+OCsTzo/o0O/+JzmNIsgGGJOqotLU6PLRsN9JOlsFJLsNrY/vjnrhwqctyGDX1eqzfX4Bv0nJNRilCvN0w7qow3D04DIFq648unSytwhPfZBhPIU25piueG9W7w14W7giEEEg/dQ7fpOXhp31nUNdgQFSIGi/9qy/iuvpKXZ7DYEiyAYYk6oi0tTpMXJKGzLxyeLo64aMJsbimp5/UZTmsI0UV+CYtF6sy8o3zmOQy4LreAbg3PgzDe/lf8ehSXYMep85W43hJJY6VVOF4SRWOl1biUIEWtToDvFXOeOeugbi+T/P30iP7o6nRoUBTg17+nhzBbSWGJBtgSKKO5sKA5K1yxpdT4nnDWQup1emxIasQX6flIu2CrtTBalfc/ffo0qXueSaEQJG2rjEIlVbheEmlMQzln2v+Sry4CF8sHB+NILV176dGZE8YkmyAIYk6kn8GpK8eike/YAYka8gprsTytFx8n3Ea56rPjy6NiPTH+LhwBKldccwYghoD0YnSKlRf0JDxnzyUTujW2R3d/NzRrbMHunV2R/fOHugd6NmuG2ISmcOQZAMMSdRRaGp0mLg0DXsZkGyqVqfHxgOFWJ6Whx3Hz152fYVchnBfFbr5uaPrBWGoW2d3dPZo393BiVpDiu9vx20tS0TNYkCSjquzArdGh+DW6BAcL6nEil15WL0nH3qD+HtUqDEENQWicF8VJ10T2SmOJBG1M5oaHSYu2Ym9pzXwUTnjq4eGoG8w/6wTkWOT4vub/3whakcYkIiILIen24jaCU2NDvcv2Yl9DEhERBbBkETUDmiqdbh/aWNA8nV3wVcPxaNPEAMSEdGVYEgicnCaah3uW7IT+/MZkIiILIkhiciB/TMgfT01Hr0DGZCIiCyBIYnIQWmqdZiw5C9k5WsZkIiIrIAhicgBlVfX474lO40B6ZupQxAZ6Cl1WURE7QpDEpGDKa+ux4T/24kDZ7To5O6CrxmQiIisgn2SiBwIAxIRke0wJBE5CAYkIiLb4uk2IgdwrqoxIB0s0MLPozEg9QpgQCIisiaOJBHZOQYkIiJpcCSJyI6dKK3Cw//bjaPFlfDzaLyKrScDEhGRTTAkEdmpTQeLkLwiExV1DejsqcTXD8UzIBER2RBDEpGd0RsE3t10BB/8lgMAGNzFBx9OGAR/L1eJKyMi6lgYkojsyLmqejy5fA/+OFoKAHhgaARmje4DZwWnDxIR2RpDEpGdyMrX4JEv0pFfXgNXZznevH0AxsaESF0WEVGHxZBEZAe+3Z2HF9dkob7BgC6dVFh8Xyz6BPE+bEREUmJIIpJQXYMeL/94EN+k5QIAru/tj/njoqF2c5a4MiIiYkgiksiZ8ho89lUG9uaVQyYDnkrshSdG9oBcLpO6NCIiAkMSkSS255Ti39/swdmqeqjdnLHgnmiMjPSXuiwiIroAQxKRDQkh8Mnvx/HWhsMwCKBvkBcW3xeL8E4qqUsjIqJ/YEgispHKugb8Z+Ve/JxVCAC4Y1AoXr+tP1ydFRJXRkRE5jAkEdlATnElHvliN46VVMFZIcPsMf1wX3w4ZDLOPyIislcMSURW9vP+Ajyzci+q6vUI9HLFh/cNwqBwH6nLIiKiy2BIIrKSBr0B8zZm4+PfjwMAhnTzxfvjB6Gzp1LiyoiIqCUYkois4GxlHf79zR5sP3YWADB1WFc8N6o3nHh7ESIih8GQRGRhBoPAY19mIO1kGVQuCsy7cyBGDwiSuiwiImolhiQiC/t2d54xIK1+/GpEBnpKXRIREbUBx/6JLKikog5vrD8EAEi+oRcDEhGRA2NIIrKg19cdhLa2Af2CvfDA0AipyyEioisgeUhatGgRIiIi4Orqivj4eKSlpV1y/QULFiAyMhJubm4ICwvDU089hdraWhtVS9S8P46WYE3mGchlQMrtUZykTUTk4CT9Lb5ixQokJydjzpw5yMjIwMCBA5GUlITi4mKz63/99dd4/vnnMWfOHBw6dAhLlizBihUr8MILL9i4ciJTtTo9XlyTBQCYNDQCA0K9pS2IiIiumKQhaf78+Zg6dSomT56Mvn37YvHixVCpVFi6dKnZ9bdv346rr74a9957LyIiInDjjTdi/Pjxlx19IrK29389ilNnqxHo5Yqnb4yUuhwiIrIAyUJSfX090tPTkZiYeL4YuRyJiYnYsWOH2W2GDh2K9PR0Yyg6fvw41q9fj5tvvrnZ/dTV1UGr1Zo8iCzpSFEFPt7a2DDylVv7wUPJi0aJiNoDyX6bl5aWQq/XIyAgwGR5QEAADh8+bHabe++9F6WlpbjmmmsghEBDQwMeffTRS55uS0lJwSuvvGLR2omaGAwCL6zajwaDwA19A5DUL1DqkoiIyEIcambpli1b8MYbb+DDDz9ERkYGVq1ahXXr1uHVV19tdpuZM2dCo9EYH3l5eTasmNq75bvysPvUObi7KPDKLf2kLoeIiCxIspEkPz8/KBQKFBUVmSwvKipCYKD5f42/9NJLuP/++/HQQw8BAKKiolBVVYWHH34Ys2bNglx+ceZTKpVQKnmvLLK84opavPlzY0+kp2+MRLC3m8QVERGRJUk2kuTi4oLY2FikpqYalxkMBqSmpiIhIcHsNtXV1RcFIYVCAQAQQlivWCIzXvvpELS1DYgKUWMSeyIREbU7ks4wTU5OxqRJkzB48GDExcVhwYIFqKqqwuTJkwEAEydOREhICFJSUgAAY8aMwfz58xETE4P4+Hjk5OTgpZdewpgxY4xhicgWth4pwY97G3sivXFbFBRymdQlERGRhUkaksaNG4eSkhLMnj0bhYWFiI6OxoYNG4yTuXNzc01Gjl588UXIZDK8+OKLyM/PR+fOnTFmzBi8/vrrUr0F6oBq6vV4cc1+AMADQ7siKlQtcUVERGQNMtHBzlNptVqo1WpoNBp4eXlJXQ45oLc2HMZHW44hWO2KTcnD4c5L/omIrE6K72+HurqNSGqHC7X49Pemnkj9GZCIiNoxhiSiFrqwJ1JSvwDc0Dfg8hsREZHDYkgih6c32OaM8Te7cpGRWw53FwVeZk8kIqJ2jyGJHFZNvR4LNx9F1MsbccdH23GowHq3nCnW1uLNnxs7wT+TFIkgNXsiERG1d5xQQQ5HCIEf957Bmz8fRoGmFgCQfuoc/vX+Njw0rCumX98TKhfL/tGe+9NBVNQ2YECoGhMTIiz62kREZJ84kkQOZU/uOdz+0XZMX56JAk0tQrzdMO/OAbipfyD0BoGPtx7Hje/+jt+yiy22z9+yi/HTvgL2RCIi6mA4kkQOoVBTi7c2HMbqPfkAAJWLAtNG9sCUa7rC1VmBuwaHYfPBIsz58QBOn6vB5M92YfSAIMz5V1/4e7m2eb819Xq8tCYLAPDg1V3RP4Q9kYiIOgqGJLJrNfV6fPL7cSzeegw1Oj0A4M7YUDybFHlR+EnsG4CE7p2wYPMRLNl2Auv2FeD37BI8e1NvTIgLh7wNI0ALUo/g9LkaBKtd8dQNvSzynoiIyDGwmSTZpaZ5R2/9fBhn/p53NLiLD2aP6YsBod6X3T4rX4NZq/dj72kNACAm3Btv3BaFPkEt/8wPFWjxr/e3QW8Q+L+Jg5HIS/6JiCQjxfc3QxLZncy8csxdewAZueUAgBBvN8y8uTdGRwVBJmv5aJDeIPDlX6cwb2M2KusaoJDLWjyx22AQuP2j7cjMK8dN/QPx0X2xV/KWiIjoCjEk2QBDkv0q1NTi7Q2HseqCeUePj+iOh4Z1g6tz229gXKipxStrD+DnrEIAQKiPG14d2x8jI/2b3eaLv07hpTVZ8FA6YXPycASq2z6viYiIrpwU39+ck0SSq6nX49M/juOjLefnHd0xKBTPjopEwBVMum4SqHbFR/fFtnhid5G2Fm//3RPpP0mRDEhERB0UQxJJRgiBtfsK8Ob6Q22ad9RaTRO7F6YeveTE7rlrD6KirgEDQ9W4b0gXi9dBRESOgafbSBJ7cs/htXWHkH7qHIDGeUfP39Qb/xrQunlHbXXgjAYvrM7C3rxyAOcndhdoavDgst1QyGX48Ymr0S+Yl/wTEdkDzkmyAYYk6Qgh8MfRUizeegzbj50FALg5N847mnrtlc07agu9QeCrnafw9obzE7s9XZ1QXq3Dw9d2wws397FpPURE1DzOSaJ2qUFvwPqsQny89RgOnGm8v5qTXIbbYkLwTJJl5h21hUIuw8SECNzYNxBzfzqA9fsLUV6tQ4i3G2Yk9pSkJiIish8MSWQ1tTo9Vu7Ow6d/nEBuWTWAxpGje+LC8NCwbgjxto+bxAaqXfHhhFikHirCd+mn8cjw7ha/9xsRETkefhOQxZVX1+OLHaewbPtJnK2qBwD4urtgUkIEJiZ0gY+7i8QVmnd9nwBc34cNI4mIqBFDElnMmfIaLNl2At+k5aK6vvFS/lAfN0wd1g13Dw6Dm4tt5xwRERFdCYYkumJHiyqweOtx/JCZjwZD43UAfYK88OjwbhgdFQQnhVziComIiFqPIYnabNfJMny89Rg2Hyo2LhvSzRePDu+O4b062+RSfiIiImthSKJWMRgEUg8XY/HWY8YeRzIZkNQ3EI+O6I7oMG9pCyQiIrIQhiRqsa1HSvDaTwdxtLgSAOCikOP2QSGYem03dO/sIXF1RERElsWQRC2SU1yBh/+3G3UNBngqnXDvkHBMubrrRfc9IyIiai8Ykuiy6hr0ePKbTNQ1GHBNDz98eN8geLk6S10WERGRVfGyI7qsd345goMFWvi6u2D+3QMZkIiIqENgSKJL+jOnFJ/8fhwA8ObtUTy9RkREHQZDEjXrXFU9nv52LwDg3vhw3NgvUOKKiIiIbIchicwSQuCF1ftRqK1Ft87ueHF0H6lLIiIisimGJDJr5e7T+DmrEE5yGRaOi+ENX4mIqMNhSKKLnCytwstrDwAAnr4xElGhaokrIiIisj2GJDKh0xswfUUmquv1GNLNFw9f203qkoiIiCTBkEQm3ks9ir155fBydcL8u6OhkPP+a0RE1DExJJHRrpNlWPRbDgAg5fYBCPZ2k7giIiIi6TAkEQBAW6vDjOWZMAjgjkGhGD0gSOqSiIiIJMWQRACA2WuykF9eg3BfFV6+pa/U5RAREUmOIYmwZk8+1mSegUIuw7vjouHJ244QERExJHV0eWXVeGlNFgDgyet6IraLj8QVERER2QeGpA5MbxBI/jYTFXUNiO3ig2kju0tdEhERkd2wi5C0aNEiREREwNXVFfHx8UhLS2t23REjRkAmk130GD16tA0rbh8+2pKDXSfPwUPphHfvjoaTwi7+OBAREdkFyb8VV6xYgeTkZMyZMwcZGRkYOHAgkpKSUFxcbHb9VatWoaCgwPjIysqCQqHAXXfdZePKHVtmXjne3XwUADD31n4I76SSuCIiIiL7InlImj9/PqZOnYrJkyejb9++WLx4MVQqFZYuXWp2fV9fXwQGBhofmzZtgkqlYkhqhaq6BsxYvgd6g8CYgcG4LSZE6pKIiIjsjqQhqb6+Hunp6UhMTDQuk8vlSExMxI4dO1r0GkuWLME999wDd3d3s8/X1dVBq9WaPDq6uWsP4uTZagSrXfHa2P6QydhVm4iI6J8kDUmlpaXQ6/UICAgwWR4QEIDCwsLLbp+WloasrCw89NBDza6TkpICtVptfISFhV1x3Y7s5/0FWLE7DzIZMH9cNNRuvNyfiIjIHKe2blheXo60tDQUFxfDYDCYPDdx4sQrLqwllixZgqioKMTFxTW7zsyZM5GcnGz8WavVdtigVKCpwfOr9gMAHhveHUO6dZK4IiIiIvvVppC0du1aTJgwAZWVlfDy8jI5XSOTyVockvz8/KBQKFBUVGSyvKioCIGBgZfctqqqCsuXL8fcuXMvuZ5SqYRSqWxRPe2ZwSDwzMq90NToMCBUjRmJvaQuiYiIyK616XTb008/jQcffBCVlZUoLy/HuXPnjI+ysrIWv46LiwtiY2ORmppqXGYwGJCamoqEhIRLbrty5UrU1dXhvvvua8tb6HCWbDuBP3POws1ZgQXjouHiJPmcfSIiIrvWppGk/Px8PPnkk1Cprvyy8eTkZEyaNAmDBw9GXFwcFixYgKqqKkyePBlA46m7kJAQpKSkmGy3ZMkSjB07Fp068ZTR5Rw4o8HbGw8DAGaP6YtunT0kroiIiMj+tSkkJSUlYffu3ejWrdsVFzBu3DiUlJRg9uzZKCwsRHR0NDZs2GCczJ2bmwu53HTUIzs7G9u2bcMvv/xyxftv72rq9Zi+PBM6vcCNfQNwz1Udcz4WERFRa8mEEKK1Gy1ZsgRz587F5MmTERUVBWdn0yukbrnlFosVaGlarRZqtRoajQZeXl5Sl2N1r6w9gM/+PAl/TyU2zLgWvu4uUpdERETUalJ8f7cpJP1zZMfkBWUy6PX6KyrKmjpSSMrMK8dtH/4JIYDPH4zD8F6dpS6JiIioTaT4/m7T6bZ/XvJP9kenN+D57/dBCOD2mBAGJCIiolZq9SVOOp0OTk5OyMrKskY9ZCH/98cJHC6sgI/KGbNG95G6HCIiIofT6pDk7OyM8PBwuz6l1tGdOluFBZuPAABeHN0XnTzYJ4qIiKi12tQsZ9asWXjhhRda1ROJbEMIgVmrs1DXYMDVPTrh9kG8eS0REVFbtGlO0gcffICcnBwEBwejS5cuF91cNiMjwyLFUeutyczHtpxSKJ3keH1sFG9eS0RE1EZtCkljx461cBlkCWVV9Xj1p0MAgCev74kIP/fLbEFERETNaVNImjNnjqXrIAt4fd0hlFXVIzLAEw9fe+WNPomIiDoy3sCrnfgzpxTfZ5yGTAak3BEFZwU/WiIioivRppEkuVx+ybkuvPLNtmp1erywej8AYOKQLhgU7iNxRURERI6vTSFp9erVJj/rdDrs2bMHn3/+OV555RWLFEYt9/6vR3HqbDUCvVzxTFKk1OUQERG1C20KSbfeeutFy+68807069cPK1aswJQpU664MGqZw4VafLz1OADglVv7wdPV+TJbEBERUUtYdOLKkCFDkJqaasmXpEvQGwSe/34/GgwCSf0CkNQvUOqSiIiI2g2LhaSamhq89957CAlh80Jb+WrnKWTmlcND6YRXbukvdTlERETtSptOt/n4+JhM3BZCoKKiAiqVCl9++aXFiqPmFWhq8PaGbADAc6MiEah2lbgiIiKi9qVNIendd981CUlyuRydO3dGfHw8fHx4ZZUtzPnhACrrGjAo3BsT4rtIXQ4REVG706aQdN111yEsLMxsG4Dc3FyEh4dfcWHUvA1ZhfjlYBGc5DKk3D4AcjlvPUJERGRpbZqT1LVrV5SUlFy0/OzZs+jatesVF0XNq6jVYc6PWQCAR4Z3Q2Sgp8QVERERtU9tCklCCLPLKysr4erKuTHWNG9jNoq0dYjopMK/r+spdTlERETtVqtOtyUnJwMAZDIZZs+eDZVKZXxOr9dj586diI6OtmiBdF76qXP44q9TAIA3bouCq7NC4oqIiIjar1aFpD179gBoHEnav38/XFxcjM+5uLhg4MCBeOaZZyxbIQEAdHoDXli1H0IAd8aGYmgPP6lLIiIiatdaFZJ+++03AMDkyZOxcOFCeHl5WaUoutgnvx9HdlEFfN1dMOvmPlKXQ0RE1O61aU7SZ599Bi8vL+Tk5GDjxo2oqakB0PxcJboyJ0qrsDD1KADgpX/1gY+7y2W2ICIioivVppBUVlaG66+/Hr169cLNN9+MgoICAMCUKVPw9NNPW7TAjk4IgVmr96O+wYBhPf0wNpodzYmIiGyhTSFpxowZcHZ2Rm5ursnk7XHjxmHDhg0WK46A7zPysf3YWbg6y/H62CizvamIiIjI8trUTPKXX37Bxo0bERoaarK8Z8+eOHXqlEUKI+BsZR1eW3cQADAjsRfCO6kuswURERFZSptGkqqqqkxGkJqUlZVBqVRecVHU6LV1h1BerUPvQE9MuYZNOomIiGypTSFp2LBh+N///mf8WSaTwWAw4O2338bIkSMtVlxH9sfREqzekw+ZDHjzjgFwVrTpoyIiIqI2atPptnnz5uG6667D7t27UV9fj2effRYHDhxAWVkZ/vzzT0vX2OHU1Osxa3XjrUcmJUQgOsxb2oKIiIg6oFaHJJ1OhyeffBJr167Fpk2b4OnpicrKStx+++2YNm0agoKCrFFnh/JDZj5yy6oRpHbFM0mRUpdDRETUIbU6JDk7O2Pfvn3w8fHBrFmzrFFTh3espBIAcHNUEDyUbRrsIyIioivUpoku9913H5YsWWLpWuhveWWNzTnDfNwkroSIiKjjatMwRUNDA5YuXYrNmzcjNjYW7u7uJs/Pnz/fIsV1VKfLqwEAoT685J+IiEgqbQpJWVlZGDRoEADgyJEjJs+x2eGVO32ucSQp1JcjSURERFJpU0hqutEtWV5FrQ7l1ToAQIg3QxIREZFU2HzHzuSXN44ieauc4enqLHE1REREHRdDkp05P2mb85GIiIikxJBkZ06fa5q0zVNtREREUmJIsjPGSdsMSURERJKSPCQtWrQIERERcHV1RXx8PNLS0i65fnl5ubGzt1KpRK9evbB+/XobVWt950eSeLqNiIhISpK2c16xYgWSk5OxePFixMfHY8GCBUhKSkJ2djb8/f0vWr++vh433HAD/P398d133yEkJASnTp2Ct7e37Yu3EuOcJF7+T0REJClJQ9L8+fMxdepUTJ48GQCwePFirFu3DkuXLsXzzz9/0fpLly5FWVkZtm/fDmfnxiu/IiIiLrmPuro61NXVGX/WarWWewNWwJEkIiIi+yDZ6bb6+nqkp6cjMTHxfDFyORITE7Fjxw6z2/z4449ISEjAtGnTEBAQgP79++ONN96AXq9vdj8pKSlQq9XGR1hYmMXfi6VoanTQ1jYAYI8kIiIiqUkWkkpLS6HX6xEQEGCyPCAgAIWFhWa3OX78OL777jvo9XqsX78eL730Et555x289tprze5n5syZ0Gg0xkdeXp5F34cl5f89advX3QXuvLEtERGRpBzqm9hgMMDf3x+ffPIJFAoFYmNjkZ+fj3nz5mHOnDlmt1EqlVAqlTautG14+T8REZH9kCwk+fn5QaFQoKioyGR5UVERAgMDzW4TFBQEZ2dnKBQK47I+ffqgsLAQ9fX1cHFxsWrN1pZ3jo0kiYiI7IVkp9tcXFwQGxuL1NRU4zKDwYDU1FQkJCSY3ebqq69GTk4ODAaDcdmRI0cQFBTk8AEJ4EgSERGRPZG0T1JycjI+/fRTfP755zh06BAee+wxVFVVGa92mzhxImbOnGlc/7HHHkNZWRmmT5+OI0eOYN26dXjjjTcwbdo0qd6CRbGRJBERkf2QdE7SuHHjUFJSgtmzZ6OwsBDR0dHYsGGDcTJ3bm4u5PLzOS4sLAwbN27EU089hQEDBiAkJATTp0/Hc889J9VbsKjzIYmn24iIiKQmE0IIqYuwJa1WC7VaDY1GAy8vL6nLMRJCYMDLv6CirgGbk69FD39PqUsiIiKyG1J8f0t+WxJqpK1pQEVdU48kjiQRERFJjSHJTuT9PWnbz8MFbi6Ky6xNRERE1saQZCea5iOFcD4SERGRXWBIshO8/J+IiMi+MCTZidNsJElERGRXGJLsBEeSiIiI7AtDkp1gI0kiIiL7wpBkB4QQbCRJRERkZxiS7ICmRofKv3skcSSJiIjIPjAk2YG8ssZRpM6eSrg6s0cSERGRPWBIsgOctE1ERGR/GJLsAOcjERER2R+GJDvAkSQiIiL7w5BkB/LYSJKIiMjuMCTZAY4kERER2R+GJImZ9khiSCIiIrIXDEkSO1etQ3W9HgAQ7M2QREREZC8YkiTWdKotwIs9koiIiOwJQ5LEmhpJ8vJ/IiIi+8KQJDFO2iYiIrJPDEkS46RtIiIi+8SQJLHzI0k83UZERGRPGJIkxkaSRERE9okhSUKNPZI4J4mIiMgeMSRJ6GxVPWp1BshkQJC3q9TlEBER0QUYkiTUNGk7wNMVSif2SCIiIrInDEkSajrVFubLU21ERET2hiFJQmwkSUREZL8YkiTESdtERET2iyFJQmwkSUREZL8YkiTERpJERET2iyFJIo09kthIkoiIyF4xJEmkpLIOdQ0GyGVAoJo9koiIiOwNQ5JEmkaRAr1c4eLEj4GIiMje8NtZIucnbfNUGxERkT1iSJKIcdI2G0kSERHZJYYkibCRJBERkX1jSJIIG0kSERHZN7sISYsWLUJERARcXV0RHx+PtLS0ZtddtmwZZDKZycPV1fGuDstnI0kiIiK7JnlIWrFiBZKTkzFnzhxkZGRg4MCBSEpKQnFxcbPbeHl5oaCgwPg4deqUDSu+cgaDwOly9kgiIiKyZ5KHpPnz52Pq1KmYPHky+vbti8WLF0OlUmHp0qXNbiOTyRAYGGh8BAQE2LDiK1dSWYf6BgMUchmC2COJiIjILkkakurr65Geno7ExETjMrlcjsTEROzYsaPZ7SorK9GlSxeEhYXh1ltvxYEDB5pdt66uDlqt1uQhtab5SIFernBSSJ5TiYiIyAxJv6FLS0uh1+svGgkKCAhAYWGh2W0iIyOxdOlS/PDDD/jyyy9hMBgwdOhQnD592uz6KSkpUKvVxkdYWJjF30dr8ca2RERE9s/hhjESEhIwceJEREdHY/jw4Vi1ahU6d+6Mjz/+2Oz6M2fOhEajMT7y8vJsXPHF2EiSiIjI/jlJuXM/Pz8oFAoUFRWZLC8qKkJgYGCLXsPZ2RkxMTHIyckx+7xSqYRSqbziWi2p6XRbGBtJEhER2S1JR5JcXFwQGxuL1NRU4zKDwYDU1FQkJCS06DX0ej3279+PoKAga5VpcWwkSUREZP8kHUkCgOTkZEyaNAmDBw9GXFwcFixYgKqqKkyePBkAMHHiRISEhCAlJQUAMHfuXAwZMgQ9evRAeXk55s2bh1OnTuGhhx6S8m20ChtJEhER2T/JQ9K4ceNQUlKC2bNno7CwENHR0diwYYNxMndubi7k8vMDXufOncPUqVNRWFgIHx8fxMbGYvv27ejbt69Ub6FVDAaB/HJO3CYiIrJ3MiGEkLoIW9JqtVCr1dBoNPDy8rL5/gs1tRiSkgqFXIbsV0exBQAREVELSPH9zW9oG8v7+1RbsDd7JBEREdkzfkvbmHE+kjcnbRMREdkzhiQbO13G+UhERESOgCHJxthIkoiIyDEwJNnY6XI2kiQiInIEDEk2xkaSREREjoEhyYb0BoEz7JFERETkEBiSbKhIW4sGg4CTXIYAL1epyyEiIqJLYEiyoaZJ28HeblDIZRJXQ0RERJfCkGRDeWWctE1EROQoGJJsyHj5PxtJEhER2T2GJBsydtvmpG0iIiK7x5BkQ8aRJJ5uIyIisnsMSTZkbCTJHklERER2jyHJRhr0BpwprwXARpJERESOgCHJRgq1tdAbBJwVMvh7KqUuh4iIiC6DIclGmuYjhXi7Qc4eSURERHaPIclGjJO2eaqNiIjIITAk2QgbSRIRETkWhiQb4UgSERGRY2FIshE2kiQiInIsDEk2cn4kiSGJiIjIETAk2UCD3oBCbWOPJDaSJCIicgwMSTZQoGnskeTiJIefB3skEREROQKGJBvIa5qPxB5JREREDoMhyQaMjSQ5H4mIiMhhMCTZQFNICvPlfCQiIiJHwZBkA7z8n4iIyPEwJNnA6TI2kiQiInI0DEk2wJEkIiIix8OQZGX1Ded7JDEkEREROQ6GJCsr1NTCIAClkxyd2SOJiIjIYTAkWVneBafaZDL2SCIiInIUDElWdn4+EidtExERORKGJCvjjW2JiIgcE0OSlbGRJBERkWNiSLIyXv5PRETkmBiSrCyPjSSJiIgcEkOSFdU16FFUwR5JREREjsguQtKiRYsQEREBV1dXxMfHIy0trUXbLV++HDKZDGPHjrVugW1UUF4LIQA3ZwU6ubtIXQ4RERG1guQhacWKFUhOTsacOXOQkZGBgQMHIikpCcXFxZfc7uTJk3jmmWcwbNgwG1Xaehde2cYeSURERI5F8pA0f/58TJ06FZMnT0bfvn2xePFiqFQqLF26tNlt9Ho9JkyYgFdeeQXdunW75OvX1dVBq9WaPGwlj5O2iYiIHJakIam+vh7p6elITEw0LpPL5UhMTMSOHTua3W7u3Lnw9/fHlClTLruPlJQUqNVq4yMsLMwitbcEG0kSERE5LklDUmlpKfR6PQICAkyWBwQEoLCw0Ow227Ztw5IlS/Dpp5+2aB8zZ86ERqMxPvLy8q647pZiI0kiIiLH5SR1Aa1RUVGB+++/H59++in8/PxatI1SqYRSKc2NZdlIkoiIyHFJGpL8/PygUChQVFRksryoqAiBgYEXrX/s2DGcPHkSY8aMMS4zGAwAACcnJ2RnZ6N79+7WLboV2EiSiIjIcUl6us3FxQWxsbFITU01LjMYDEhNTUVCQsJF6/fu3Rv79+9HZmam8XHLLbdg5MiRyMzMtOl8o8up1elRpK0DwDlJREREjkjy023JycmYNGkSBg8ejLi4OCxYsABVVVWYPHkyAGDixIkICQlBSkoKXF1d0b9/f5Ptvb29AeCi5VI7U954qk3looCPylniaoiIiKi1JA9J48aNQ0lJCWbPno3CwkJER0djw4YNxsncubm5kMsl71TQauyRRERE5NhkQgghdRG2pNVqoVarodFo4OXlZbX9fL0zFy+s3o/re/tjyQNXWW0/REREHYGtvr8v5HhDNA6CjSSJiIgcG0OSlZw/3cZJ20RERI6IIclKePk/ERGRY2NIshI2kiQiInJsDElWUKvTo6SiqUcSR5KIiIgcEUOSFTSNInkonaB2Y48kIiIiR8SQZAUXzkdijyQiIiLHxJBkBRc2kiQiIiLHxJBkBbz8n4iIyPExJFkBG0kSERE5PoYkK+BIEhERkeNjSLKCfI4kEREROTyGJAurqdejtLIeABtJEhEROTKGJAvLL28cRfJ0ZY8kIiIiR8aQZGF5ZZyPRERE1B4wJFkYb2xLRETUPjAkWRgbSRIREbUPDEkW1hSSwni6jYiIyKExJFkYG0kSERG1DwxJFsZGkkRERO0DQ5IFVdU1oKyqsUdSqC9HkoiIiBwZQ5IF5Zc3jiKp3Zzh5coeSURERI6MIcmCePk/ERFR+8GQZEHnG0kyJBERETk6hiQLOj+SxEnbREREjo4hyYLYSJKIiKj9YEiyIDaSJCIiaj8YkizI2EiSl/8TERE5PIYkC6mo1aG8WgcACPFmSCIiInJ0DEkW0tQjyVvlDE/2SCIiInJ4DEkWUl6tg9rNmfORiIiI2gknqQtoL4Z064S9c25ErU4vdSlERERkARxJsjBXZ4XUJRAREZEFMCQRERERmcGQRERERGQGQxIRERGRGQxJRERERGYwJBERERGZYRchadGiRYiIiICrqyvi4+ORlpbW7LqrVq3C4MGD4e3tDXd3d0RHR+OLL76wYbVERETUEUgeklasWIHk5GTMmTMHGRkZGDhwIJKSklBcXGx2fV9fX8yaNQs7duzAvn37MHnyZEyePBkbN260ceVERETUnsmEEELKAuLj43HVVVfhgw8+AAAYDAaEhYXh3//+N55//vkWvcagQYMwevRovPrqq5ddV6vVQq1WQ6PRwMvL64pqJyIiItuQ4vtb0pGk+vp6pKenIzEx0bhMLpcjMTERO3bsuOz2QgikpqYiOzsb1157rdl16urqoNVqTR5ERERElyNpSCotLYVer0dAQIDJ8oCAABQWFja7nUajgYeHB1xcXDB69Gi8//77uOGGG8yum5KSArVabXyEhYVZ9D0QERFR+yT5nKS28PT0RGZmJnbt2oXXX38dycnJ2LJli9l1Z86cCY1GY3zk5eXZtlgiIiJySJLe4NbPzw8KhQJFRUUmy4uKihAYGNjsdnK5HD169AAAREdH49ChQ0hJScGIESMuWlepVEKpVFq0biIiImr/JB1JcnFxQWxsLFJTU43LDAYDUlNTkZCQ0OLXMRgMqKurs0aJRERE1EFJOpIEAMnJyZg0aRIGDx6MuLg4LFiwAFVVVZg8eTIAYOLEiQgJCUFKSgqAxjlGgwcPRvfu3VFXV4f169fjiy++wEcffdSi/TVdzMcJ3ERERI6j6XvblhflSx6Sxo0bh5KSEsyePRuFhYWIjo7Ghg0bjJO5c3NzIZefH/CqqqrC448/jtOnT8PNzQ29e/fGl19+iXHjxrVofxUVFQDACdxEREQOqKKiAmq12ib7krxPkq0ZDAacOXMGnp6ekMlkFn1trVaLsLAw5OXlsQeTDfG4S4PHXRo87tLgcZfGhcfd09MTFRUVCA4ONhk8sSbJR5JsTS6XIzQ01Kr78PLy4l8iCfC4S4PHXRo87tLgcZdG03G31QhSE4dsAUBERERkbQxJRERERGYwJFmQUqnEnDlz2JfJxnjcpcHjLg0ed2nwuEtD6uPe4SZuExEREbUER5KIiIiIzGBIIiIiIjKDIYmIiIjIDIYkIiIiIjMYkixk0aJFiIiIgKurK+Lj45GWliZ1SQ4jJSUFV111FTw9PeHv74+xY8ciOzvbZJ3a2lpMmzYNnTp1goeHB+644w4UFRWZrJObm4vRo0dDpVLB398f//nPf9DQ0GCyzpYtWzBo0CAolUr06NEDy5Yts/bbcxhvvvkmZDIZZsyYYVzG424d+fn5uO+++9CpUye4ubkhKioKu3fvNj4vhMDs2bMRFBQENzc3JCYm4ujRoyavUVZWhgkTJsDLywve3t6YMmUKKisrTdbZt28fhg0bBldXV4SFheHtt9+2yfuzR3q9Hi+99BK6du0KNzc3dO/eHa+++qrJfcB43K/c77//jjFjxiA4OBgymQxr1qwxed6Wx3jlypXo3bs3XF1dERUVhfXr17f+DQm6YsuXLxcuLi5i6dKl4sCBA2Lq1KnC29tbFBUVSV2aQ0hKShKfffaZyMrKEpmZmeLmm28W4eHhorKy0rjOo48+KsLCwkRqaqrYvXu3GDJkiBg6dKjx+YaGBtG/f3+RmJgo9uzZI9avXy/8/PzEzJkzjescP35cqFQqkZycLA4ePCjef/99oVAoxIYNG2z6fu1RWlqaiIiIEAMGDBDTp083Ludxt7yysjLRpUsX8cADD4idO3eK48ePi40bN4qcnBzjOm+++aZQq9VizZo1Yu/eveKWW24RXbt2FTU1NcZ1Ro0aJQYOHCj++usv8ccff4gePXqI8ePHG5/XaDQiICBATJgwQWRlZYlvvvlGuLm5iY8//tim79devP7666JTp07ip59+EidOnBArV64UHh4eYuHChcZ1eNyv3Pr168WsWbPEqlWrBACxevVqk+dtdYz//PNPoVAoxNtvvy0OHjwoXnzxReHs7Cz279/fqvfDkGQBcXFxYtq0acaf9Xq9CA4OFikpKRJW5biKi4sFALF161YhhBDl5eXC2dlZrFy50rjOoUOHBACxY8cOIUTjX0y5XC4KCwuN63z00UfCy8tL1NXVCSGEePbZZ0W/fv1M9jVu3DiRlJRk7bdk1yoqKkTPnj3Fpk2bxPDhw40hicfdOp577jlxzTXXNPu8wWAQgYGBYt68ecZl5eXlQqlUim+++UYIIcTBgwcFALFr1y7jOj///LOQyWQiPz9fCCHEhx9+KHx8fIyfQ9O+IyMjLf2WHMLo0aPFgw8+aLLs9ttvFxMmTBBC8Lhbwz9Dki2P8d133y1Gjx5tUk98fLx45JFHWvUeeLrtCtXX1yM9PR2JiYnGZXK5HImJidixY4eElTkujUYDAPD19QUApKenQ6fTmRzj3r17Izw83HiMd+zYgaioKAQEBBjXSUpKglarxYEDB4zrXPgaTet09M9p2rRpGD169EXHhsfdOn788UcMHjwYd911F/z9/RETE4NPP/3U+PyJEydQWFhocszUajXi4+NNjru3tzcGDx5sXCcxMRFyuRw7d+40rnPttdfCxcXFuE5SUhKys7Nx7tw5a79NuzN06FCkpqbiyJEjAIC9e/di27ZtuOmmmwDwuNuCLY+xpX7vMCRdodLSUuj1epMvCQAICAhAYWGhRFU5LoPBgBkzZuDqq69G//79AQCFhYVwcXGBt7e3yboXHuPCwkKzn0HTc5daR6vVoqamxhpvx+4tX74cGRkZSElJueg5HnfrOH78OD766CP07NkTGzduxGOPPYYnn3wSn3/+OYDzx+1Sv1MKCwvh7+9v8ryTkxN8fX1b9dl0JM8//zzuuece9O7dG87OzoiJicGMGTMwYcIEADzutmDLY9zcOq39DJxatTaRlU2bNg1ZWVnYtm2b1KW0e3l5eZg+fTo2bdoEV1dXqcvpMAwGAwYPHow33ngDABATE4OsrCwsXrwYkyZNkri69uvbb7/FV199ha+//hr9+vVDZmYmZsyYgeDgYB53ahZHkq6Qn58fFArFRVf8FBUVITAwUKKqHNMTTzyBn376Cb/99htCQ0ONywMDA1FfX4/y8nKT9S88xoGBgWY/g6bnLrWOl5cX3NzcLP127F56ejqKi4sxaNAgODk5wcnJCVu3bsV7770HJycnBAQE8LhbQVBQEPr27WuyrE+fPsjNzQVw/rhd6ndKYGAgiouLTZ5vaGhAWVlZqz6bjuQ///mPcTQpKioK999/P5566injKCqPu/XZ8hg3t05rPwOGpCvk4uKC2NhYpKamGpcZDAakpqYiISFBwsochxACTzzxBFavXo1ff/0VXbt2NXk+NjYWzs7OJsc4Ozsbubm5xmOckJCA/fv3m/zl2rRpE7y8vIxfSAkJCSav0bROR/2crr/+euzfvx+ZmZnGx+DBgzFhwgTj//O4W97VV199UYuLI0eOoEuXLgCArl27IjAw0OSYabVa7Ny50+S4l5eXIz093bjOr7/+CoPBgPj4eOM6v//+O3Q6nXGdTZs2ITIyEj4+PlZ7f/aquroacrnpV55CoYDBYADA424LtjzGFvu906pp3mTW8uXLhVKpFMuWLRMHDx4UDz/8sPD29ja54oea99hjjwm1Wi22bNkiCgoKjI/q6mrjOo8++qgIDw8Xv/76q9i9e7dISEgQCQkJxuebLkW/8cYbRWZmptiwYYPo3Lmz2UvR//Of/4hDhw6JRYsWdehL0c258Oo2IXjcrSEtLU04OTmJ119/XRw9elR89dVXQqVSiS+//NK4zptvvim8vb3FDz/8IPbt2yduvfVWs5dJx8TEiJ07d4pt27aJnj17mlwmXV5eLgICAsT9998vsrKyxPLly4VKpeowl6L/06RJk0RISIixBcCqVauEn5+fePbZZ43r8LhfuYqKCrFnzx6xZ88eAUDMnz9f7NmzR5w6dUoIYbtj/OeffwonJyfx3//+Vxw6dEjMmTOHLQCk9P7774vw8HDh4uIi4uLixF9//SV1SQ4DgNnHZ599ZlynpqZGPP7448LHx0eoVCpx2223iYKCApPXOXnypLjpppuEm5ub8PPzE08//bTQ6XQm6/z2228iOjpauLi4iG7dupnsgy4OSTzu1rF27VrRv39/oVQqRe/evcUnn3xi8rzBYBAvvfSSCAgIEEqlUlx//fUiOzvbZJ2zZ8+K8ePHCw8PD+Hl5SUmT54sKioqTNbZu3evuOaaa4RSqRQhISHizTfftPp7s1darVZMnz5dhIeHC1dXV9GtWzcxa9Ysk8vIedyv3G+//Wb29/mkSZOEELY9xt9++63o1auXcHFxEf369RPr1q1r9fuRCXFBu1EiIiIiAsA5SURERERmMSQRERERmcGQRERERGQGQxIRERGRGQxJRERERGYwJBERERGZwZBEREREZAZDEhEREZEZDElEZHVbtmyBTCa76Ga5RET2jCGJiCxqxIgRmDFjhsmyoUOHoqCgAGq1WpqimhEREYEFCxZIXQYR2SknqQsgovbPxcUFgYGBUpdBRNQqHEkiIot54IEHsHXrVixcuBAymQwymQwnT5686HTbsmXL4O3tjZ9++gmRkZFQqVS48847UV1djc8//xwRERHw8fHBk08+Cb1eb3z9uro6PPPMMwgJCYG7uzvi4+OxZcuWZusRQuDll19GeHg4lEolgoOD8eSTTwJoHPE6deoUnnrqKWOtTbZt24Zhw4bBzc0NYWFhePLJJ1FVVWV8PiIiAq+++irGjx8Pd3d3hISEYNGiRZY9mEQkOYYkIrKYhQsXIiEhAVOnTkVBQQEKCgoQFhZmdt3q6mq89957WL58OTZs2IAtW7bgtttuw/r167F+/Xp88cUX+Pjjj/Hdd98Zt3niiSewY8cOLF++HPv27cNdd92FUaNG4ejRo2b38f333+Pdd9/Fxx9/jKNHj2LNmjWIiooCAKxatQqhoaGYO3eusVYAOHbsGEaNGoU77rgD+/btw4oVK7Bt2zY88cQTJq89b948DBw4EHv27MHzzz+P6dOnY9OmTZY4jERkLwQRkQUNHz5cTJ8+3WTZb7/9JgCIc+fOCSGE+OyzzwQAkZOTY1znkUceESqVSlRUVBiXJSUliUceeUQIIcSpU6eEQqEQ+fn5Jq99/fXXi5kzZ5qt5Z133hG9evUS9fX1Zp/v0qWLePfdd02WTZkyRTz88MMmy/744w8hl8tFTU2NcbtRo0aZrDNu3Dhx0003md0PETkmjiQRkSRUKhW6d+9u/DkgIAARERHw8PAwWVZcXAwA2L9/P/R6PXr16gUPDw/jY+vWrTh27JjZfdx1112oqalBt27dMHXqVKxevRoNDQ2XrGvv3r1YtmyZyT6SkpJgMBhw4sQJ43oJCQkm2yUkJODQoUOtPg5EZL84cZuIJOHs7Gzys0wmM7vMYDAAACorK6FQKJCeng6FQmGy3oXB6kJhYWHIzs7G5s2bsWnTJjz++OOYN28etm7detG+mlRWVuKRRx4xzl26UHh4eIvfHxE5PoYkIrIoFxcXk8nWlhITEwO9Xo/i4mIMGzasxdu5ublhzJgxGDNmDKZNm4bevXtj//79GDRokNlaBw0ahIMHD6JHjx6XfN2//vrrop/79OnT8jdERHaPIYmILCoiIgI7d+7EyZMn4eHhAV9fX4u8bq9evTBhwgRMnDgR77zzDmJiYlBSUoLU1FQMGDAAo0ePvmibZcuWQa/XIz4+HiqVCl9++SXc3NzQpUsXY62///477rnnHiiVSvj5+eG5557DkCFD8MQTT+Chhx6Cu7s7Dh48iE2bNuGDDz4wvvaff/6Jt99+G2PHjsWmTZuwcuVKrFu3ziLvlYjsA+ckEZFFPfPMM1AoFOjbty86d+6M3Nxci732Z599hokTJ+Lpp59GZGQkxo4di127djV7Gszb2xuffvoprr76agwYMACbN2/G2rVr0alTJwDA3LlzcfLkSXTv3h2dO3cGAAwYMABbt27FkSNHMGzYMMTExGD27NkIDg42ee2nn34au3fvRkxMDF577TXMnz8fSUlJFnuvRCQ9mRBCSF0EEZEjiYiIwIwZMy7qLE5E7QtHkoiIiIjMYEgiIiIiMoOn24iIiIjM4EgSERERkRkMSURERERmMCQRERERmcGQRERERGQGQxIRERGRGQxJRERERGYwJBERERGZwZBEREREZMb/AzQp01VVRJdrAAAAAElFTkSuQmCC\n" }, "metadata": {} } ], "source": [ "\n", "# Create a Neural LinUCB pearl agent with 1-hot action representation\n", "\n", "action_representation_module = OneHotActionTensorRepresentationModule(\n", " max_number_actions= env._action_space.n,\n", ")\n", "\n", "agent = PearlAgent(\n", " policy_learner=NeuralLinearBandit(\n", " feature_dim = env.observation_dim + env._action_space.n,\n", " hidden_dims=[64, 16],\n", " state_features_only=False,\n", " training_rounds=10,\n", " learning_rate=0.01,\n", " action_representation_module=action_representation_module,\n", " exploration_module= UCBExploration(alpha=1.0)\n", " ),\n", " replay_buffer=FIFOOffPolicyReplayBuffer(100_000),\n", " device_id=-1,\n", ")\n", "\n", "\n", "info = online_learning(\n", " agent=agent,\n", " env=env,\n", " number_of_steps=number_of_steps,\n", " print_every_x_steps=100,\n", " record_period=record_period,\n", " learn_after_episode=True,\n", ")\n", "torch.save(info[\"return\"], \"LinUCB-return.pt\")\n", "plt.plot(record_period * np.arange(len(info[\"return\"])), info[\"return\"], label=\"LinUCB\")\n", "plt.xlabel(\"time step\")\n", "plt.ylabel(\"return\")\n", "plt.legend()\n", "plt.show()" ] }, { "cell_type": "markdown", "metadata": { "id": "QNUmNO77LNGR" }, "source": [ "## Contextual Bandits learners: Linear Thompson Sampling\n", "\n", "Lastly, we describe how to use the neural version of the Linear Thompson Sampling (LinTS) algorithm with Pearl. The algorithm which uses Thompson sampling exploration with neural architectures. The LinTS sampling is closely related to the LinUCB algorithm, with a key modification that often improves its convergence in practice: sample the score function from a probability, instead of fixing it determinstically. Practically, this often reduces the over-exploring of arms, since the score may be smaller than in the LinUCB algorithm.\n", "\n", "To implement the LinTS algorithm in Pearl, use the `NeuralLinearBandit` policy learner module combined with an exploration module of type `ThompsonSamplingExplorationLinear`. This enables the agent to sample the score based on its estimated uncertainty, rather than to fix it as in LinUCB algorithm.\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "_7Cpzoi3nVAw", "colab": { "base_uri": "https://localhost:8080/", "height": 1000 }, "outputId": "2b350618-af1c-4649-bede-13ba64ad41b2" }, "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ "episode 100, step 100, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 0.09024851769208908\n", "episode 200, step 200, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 0.002633035881444812\n", "episode 300, step 300, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: -0.07467412203550339\n", "episode 400, step 400, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 1.0156103372573853\n", "episode 500, step 500, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 1.04409921169281\n", "episode 600, step 600, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 0.9837733507156372\n", "episode 700, step 700, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 0.9862121939659119\n", "episode 800, step 800, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: -0.11129053682088852\n", "episode 900, step 900, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 0.9800935983657837\n", "episode 1000, step 1000, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 0.9999375939369202\n", "episode 1100, step 1100, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 1.0631517171859741\n", "episode 1200, step 1200, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 0.9909786581993103\n", "episode 1300, step 1300, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 1.1609396934509277\n", "episode 1400, step 1400, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 0.9601272940635681\n", "episode 1500, step 1500, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 0.1314946413040161\n", "episode 1600, step 1600, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 1.0190272331237793\n", "episode 1700, step 1700, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 0.900674045085907\n", "episode 1800, step 1800, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 1.0322463512420654\n", "episode 1900, step 1900, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 0.0550333634018898\n", "episode 2000, step 2000, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 1.131584644317627\n", "episode 2100, step 2100, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 0.9966853857040405\n", "episode 2200, step 2200, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 0.8430404663085938\n", "episode 2300, step 2300, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 1.1651915311813354\n", "episode 2400, step 2400, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 0.9456443786621094\n", "episode 2500, step 2500, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 0.9995026588439941\n", "episode 2600, step 2600, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 1.1561548709869385\n", "episode 2700, step 2700, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 1.1483546495437622\n", "episode 2800, step 2800, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 1.0468939542770386\n", "episode 2900, step 2900, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 1.0259349346160889\n", "episode 3000, step 3000, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 0.853131115436554\n", "episode 3100, step 3100, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 0.9850102663040161\n", "episode 3200, step 3200, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 0.8096411228179932\n", "episode 3300, step 3300, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 1.0070034265518188\n", "episode 3400, step 3400, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 0.771452784538269\n", "episode 3500, step 3500, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 0.9490416646003723\n", "episode 3600, step 3600, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 0.8729531764984131\n", "episode 3700, step 3700, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: -0.12757402658462524\n", "episode 3800, step 3800, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 1.0074385404586792\n", "episode 3900, step 3900, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 1.033884882926941\n", "episode 4000, step 4000, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 1.046915888786316\n", "episode 4100, step 4100, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 1.0219038724899292\n", "episode 4200, step 4200, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: -0.15234242379665375\n", "episode 4300, step 4300, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 0.9609275460243225\n", "episode 4400, step 4400, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 1.034550666809082\n", "episode 4500, step 4500, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 1.2472548484802246\n", "episode 4600, step 4600, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 0.9214726686477661\n", "episode 4700, step 4700, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: -0.021003952249884605\n", "episode 4800, step 4800, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 1.0047966241836548\n", "episode 4900, step 4900, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 1.034260630607605\n", "episode 5000, step 5000, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 0.7625596523284912\n", "episode 5100, step 5100, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 0.9447240233421326\n", "episode 5200, step 5200, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 0.7146114706993103\n", "episode 5300, step 5300, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 1.044114351272583\n", "episode 5400, step 5400, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 0.9947097897529602\n", "episode 5500, step 5500, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 1.0141396522521973\n", "episode 5600, step 5600, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 1.1748143434524536\n", "episode 5700, step 5700, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 1.0289136171340942\n", "episode 5800, step 5800, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 1.0836181640625\n", "episode 5900, step 5900, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 0.92894047498703\n", "episode 6000, step 6000, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 0.9187952876091003\n", "episode 6100, step 6100, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 0.8998274207115173\n", "episode 6200, step 6200, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: -0.02114623598754406\n", "episode 6300, step 6300, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 1.0222702026367188\n", "episode 6400, step 6400, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 1.0138862133026123\n", "episode 6500, step 6500, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 1.0265398025512695\n", "episode 6600, step 6600, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 0.8881807923316956\n", "episode 6700, step 6700, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 0.9364562630653381\n", "episode 6800, step 6800, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 1.0452048778533936\n", "episode 6900, step 6900, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 0.8812369704246521\n", "episode 7000, step 7000, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 1.0886296033859253\n", "episode 7100, step 7100, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 0.8804692029953003\n", "episode 7200, step 7200, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 1.066596269607544\n", "episode 7300, step 7300, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 1.0423859357833862\n", "episode 7400, step 7400, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 1.0080853700637817\n", "episode 7500, step 7500, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 0.999132513999939\n", "episode 7600, step 7600, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 0.9367015361785889\n", "episode 7700, step 7700, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 1.008610486984253\n", "episode 7800, step 7800, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 0.9758915305137634\n", "episode 7900, step 7900, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 1.1150867938995361\n", "episode 8000, step 8000, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 0.9644365906715393\n", "episode 8100, step 8100, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 1.0202032327651978\n", "episode 8200, step 8200, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 1.0238661766052246\n", "episode 8300, step 8300, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 0.941068708896637\n", "episode 8400, step 8400, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 1.052219033241272\n", "episode 8500, step 8500, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 0.9174469113349915\n", "episode 8600, step 8600, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 0.9747493267059326\n", "episode 8700, step 8700, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 1.0260030031204224\n", "episode 8800, step 8800, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 0.9549689292907715\n", "episode 8900, step 8900, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 0.9588802456855774\n", "episode 9000, step 9000, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 1.0665802955627441\n", "episode 9100, step 9100, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 1.0095921754837036\n", "episode 9200, step 9200, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 0.9650076627731323\n", "episode 9300, step 9300, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 0.7871606945991516\n", "episode 9400, step 9400, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 1.0182504653930664\n", "episode 9500, step 9500, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 0.9524450302124023\n", "episode 9600, step 9600, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 1.0539082288742065\n", "episode 9700, step 9700, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 1.1573524475097656\n", "episode 9800, step 9800, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 0.9590475559234619\n", "episode 9900, step 9900, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 0.943342387676239\n", "episode 10000, step 10000, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", "return: 0.8112016916275024\n" ] }, { "output_type": "display_data", "data": { "text/plain": [ "
" ], "image/png": "iVBORw0KGgoAAAANSUhEUgAAAkkAAAGwCAYAAAC99fF4AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAABPtUlEQVR4nO3deVxU9f4/8NcsMDDAsO+r4oK7iIpotlJY5s1WM1MzszL9ptmmWXpts1/dTC1L82paVpqltpldo7RMBQX3fQeRYRWGdYCZz+8PZHRyVISZOTPwej4e8xDOnDPnPQdlXn62IxNCCBARERGRGbnUBRARERE5IoYkIiIiIgsYkoiIiIgsYEgiIiIisoAhiYiIiMgChiQiIiIiCxiSiIiIiCxQSl2AvRmNRpw7dw5eXl6QyWRSl0NERESNIIRAWVkZwsLCIJfbp42n1YWkc+fOITIyUuoyiIiIqAmys7MRERFhl3O1upDk5eUFoP4iazQaiashIiKixtDpdIiMjDR9jttDqwtJDV1sGo2GIYmIiMjJ2HOoDAduExEREVnAkERERERkAUMSERERkQWtbkxSYxkMBtTW1kpdRovh4uIChUIhdRlERESNxpD0D0IIaLValJSUSF1Ki+Pj44OQkBCuT0VERE6BIekfGgJSUFAQ1Go1P9CtQAiByspK5OfnAwBCQ0MlroiIiOjaGJIuYTAYTAHJ399f6nJaFHd3dwBAfn4+goKC2PVGREQOjwO3L9EwBkmtVktcScvUcF051ouIiJwBQ5IF7GKzDV5XIiJyJgxJRERERBYwJBERERFZwJDUCshkMqxbt07qMoiIiJwKQ1IL8dhjj2Ho0KEWn8vNzcWdd97Z6NeRyWRXfMTExAAATp06hUceeQRhYWFwc3NDREQE7rnnHhw+fNhK74iIWqM6gxHl+jqpy6AmMhoF8nTVyCqqlLoUq+ASAK1ASEhIo/edN28e3nnnHdP3oaGh+OyzzzBo0CAAgEKhQG1tLW6//XZ07NgRa9asQWhoKM6ePYtffvmFi3ASUZPp6wwYNPcvnCqsQIjGDe2CPE2P9kGeaB/sBT8PV6nLbNWEEDhfWYvs4kqcPV+F7POVyC6uRPb5Kpw9X7+tps6Ige0D8MXYRKnLbTaGpGsQQqCq1iDJud1dFFaZESaTybB27VoMHToUp0+fRps2bfDdd9/hww8/RFpaGtq3b4+FCxciKSkJ3t7e8Pb2Nju+YaXsBrt378aJEyeQmpqK6OhoAEB0dDQGDBjQ7FqJqPX6/VA+ThVWAAC0umpoddXYcrzQbB8/D1fz4BTkhXZBngjWqDiD1krKqmuRXVwferLPV10IRBdCUXElKmqu/pkolwE1dUY7VWtbDEnXUFVrQOcZv0py7oOvp0Dtapsf0fTp0/Gf//wH7du3x/Tp0zF8+HAcP34cSuW1zxcYGAi5XI5vv/0WkydP5sKQRGQV32acBQCMGRCDu7uH4UR+OY7ll+F4fjmO5Zfj7PkqFFfUIP1UMdJPFZsd66VSItbU4tQQorwQ7uMOuZzh6Wpq6ox4/39HsPVEEbLPV6Kk8tpr2QV5qRDpp0akrzsifNWI9HNHpK8akX5qhHi7wUXRMkbzMCS1Ui+88AIGDx4MAJg1axa6dOmC48ePIy4u7prHhoeHY/78+XjppZcwa9Ys9O7dG7fccgtGjBiBtm3b2rp0ImqB8suqseloAQDg0X7RiA30REK0r9k+lTV1OFlQcTE45ZXjeEE5zhRVokxfh93ZJdidXWJ2jNpVgY4hXogL0SAuxOvCQwNvtYu93ppDq6414JkvM/H74Xyz7b5qlwshSI0IX3dEXAhEkX5qhPu4w82ldfznmCHpGtxdFDj4eopk57aV7t27m75uuJdafn5+o0ISAEyYMAGjRo3Cpk2bsH37dqxevRpvv/02fvjhB9x+++02qZmIWq7vd52DwSjQK8oHsYGeFvdRuyrRNdwbXcPNhwTo6ww4U1RZH5ouaX06WViByhoDdmWVYFdWidkxYd5uiAu9EJxCNegU4oU2AR5QtpAWkMaorKnDk59nYMvxQri5yPH6PV3RPcIbEb5qeKoYDwCGpGuSyWQ26/KSkovLxf9FNfTjG43X14fs5eWFIUOGYMiQIXjzzTeRkpKCN998kyGJiK6LEMLU1fZAQuR1H69SKtAh2Asdgr3MttcZjDhdVInDWh0O55bhsFaHQ7llyCmpwrnSapwrrTZrQXFVyNEuyBNxoV7oFKJBXGh9q1Ogl6p5b9ABlVXXYuyynUg/XQwPVwWWPtYHiW15z9J/anmf/iQJmUyGuLg4bN26VepSiMjJ7M/R4UheGVRKOQZ3D7Xa6yovhJ52QZ64+2LjOXTVtTiiLcPhXB0Oa8vqH7k6VNQYcDBXh4O5OgA5pv0DPF3RPsgLateLrfsXx4jL/vF9w5aL22S4+KRMBni7u+Dpm2IRE+Bhtfd6PUqrajF6aTp2Z5fAy02J5Y/3Ra8o32sf2AoxJLUgpaWl2L17t9k2f3/r/89g9+7dmDlzJkaOHInOnTvD1dUVmzdvxtKlS/Hyyy9b/XxE1LJ9m5ENAEjpEgJvd9uPFdK4uaBPjB/6xPiZthmNAjklVThkCk71rU+niipQWF6DwvIiq9bw095c/L/7u1s1FDZGcUUNRi5Jw4FzOvioXbBibOJl3Zd0EUNSC7Jp0ybEx8ebbRs7dqzVzxMREYGYmBjMmjULp0+fNi0yOWvWLDz33HNWPx8RtVz6OgO+33MOAPBAQoRkdcjlsvqByn5q3NHl4pInVTUG0xinOoOAgDA9Jy58Kf7xff02YXEfAPhhdw52nD6PCV9lIu1UNF65q5NdBkIXlOnx6H/TcCSvDAGerljxRCLiQjQ2P68zkwlx6Y+15dPpdPD29kZpaSk0GvO/HNXV1Th16hTatGkDNzc3iSpsuXh9ieifftmXi/FfZiJE44a/p94KRSuYrl9nMGLOxqP4eNMJAEDXcA0WPNIL0f62637Tllbjkf9ux8mCCgRrVPjyiX5oF2R5gLyjutrnt620nmH8RETkcBoGbN/XK7xVBCSgfqzUS4PisGxMH/iqXbA/R4e752/Bz3tzbXK+s+cr8dCibThZUIFwH3d881SS0wUkqTAkERGRJC5dG+l+CbvapHJzxyCsnzQQfWJ8Uaavw4SvMjHj+/3Q11nvLg+nCyvw0MJtyCquRLS/Gque6mfTFquWhiGJiIgk0Zi1kVq6UG93fD2uH8bfHAsA+HzbGdz/yVacKapo9msfyyvDQ4u24VxpNWIDPbDqySRE+Kqb/bqtCUOSBa1smJbd8LoSUYPmro3UkigVcrw8KA6fWbH77eA5HR7+dDvyy/SIC/HCyieTEOLNsaDXiyHpEg0LLFZWVkpcScvUcF0vXciSiFonW62N5MxuudD91ju6ed1ve8+WYPji7SiqqEHXcA2+HtevRS6IaQ9cAuASCoUCPj4+yM+vX4FVrVbzrtJWIIRAZWUl8vPz4ePjwxviEpHd10ZyFqHe7vj6yX6Ys/EoPtl0Ap9vO4PMrPONnv2WcaYYjy3dgTJ9HeKjfLBsTF9e32ZgSPqHkJD69TEaghJZj4+Pj+n6ElHr5ShrIzkqlwvdb33b+GHKqt2m7rf/90B33NXtyq1u204UYezyHaisMaBvGz8sfawP78HWTLx6/yCTyRAaGoqgoCDU1tZKXU6L4eLiwhYkIgIA/H4oHyWVtQjRuGFAuwCpy3FYDd1v//fVLuw8cx7PfJmJ0UnReGVwJ6iU5r9P/zxagHGf74S+zogb2gVg8ajecHfl79zmYki6AoVCwQ91IiIbaI1rIzXVP7vflm87g8ysEnz0SLyp++23g3l45stM1BiMuDUuCB+P6GWXFbxbAw7cJiIiu2ntayM1RUP322eP1c9+25dTirvnb8H6fbn4eW8unl6RgRqDEYO6hGDhowkMSFbEliQiIrIbro3UdLfEBeHnZwfi/77ehYwL3W8yWf394f7VIwxzHuoBpYJtH9bEq0lERHbBtZGaL8zHHSuf7Ienb6pffFII4MGECHwwrCcDkg2wJYmIiOyCayNZh4tCjql3xuHWuCBkF1fi3vhwyDm2yyYYkoiIyC64NpJ19W3jh75t/KQuo0Vj2xwREdkc10YiZ8SQRERENse1kcgZMSQREZHNcW0kckYMSUREZFNcG4mcFUMSEVErdb6ixi7n4dpI5KwYkoiIWqF//3AA8W9sxLzfjtn0PFwbiZwZlwAgaoEMRgF9nQE1dUbo64xmf0b6ucPLjdOvW7Nf9uVi2dbTAIAPfjuKII0Kw/tG2eRcXBuJnBlDEpEDO3u+Emszc3A4rwz6WuNlwafGcPk2fZ0RBqO44mtG+rnj52cHQsOg1CqdK6nC1DX7AABxIV44rC3Dq+v2I8hLhds6BVv9fA1rI93BtZHICTEkETmYypo6bNivxbcZZ7H1RFGzX08uA1RKBVQuclTVGJBdXIVZPxzE+w/1sEK15EwMRoHnVu1GaVUtukd449un+2P62n1YnXEWE7/aha+f7IeekT5WOx/XRiJnx5BE5ACMRoEdp4vxbcZZrN+Xi4oag+m5/rH+uDUuCB4qJVwVcqhc5Bf+VJh97+Yih6tCccnz9X9eej+nnaeL8eCibfgu8yxSugTjji4hUrxdksjCzSeQdqoYalcF5j0cD1elHG/f1w35ZXpsPlqAx5ftwJrx/RET4GGV8zWsjRSsUeEGro1EToghiUhC2cWV+C7zLL7LPIvs4irT9mh/NR7oFYF7e4UjwldttfP1jvHDkze2xaLNJ/HK2n1IiPaFv6fKaq9PjmtX1nnM2XgUADDrX13Q5kIQclHI8fGIXhj26Tbsz9Fh9Gfp+G58fwRY4e/FxbWRIrg2EjklhiQiO6vQ12H9vlx8l3kW208Wm7Z7qpS4u3soHkiIQEK0L2Qy23yoTLm9AzYdLsCRvDK8snYfFj6aYLNzkWMoq67FpJW7YTAK09+xS3molFj6WB/c9/FWnCmqxNhlO/D1k/2gdm36R4TZ2ki92NVGzknyJQAWLFiAmJgYuLm5ITExEenp6Vfct7a2Fq+//jpiY2Ph5uaGHj16YMOGDXaslqhpjEaBbSeK8Pw3e9Dnrd/w4rd7sf1kMWQy4IZ2AZg7rCd2TE/GO/d3R+8YP5uGFpVSgTnDesBFIcOvB/KwdleOzc5FjmHm9weQVVyJcB93vHVvN4t/v4K83LD88b7wVbtgz9lSTPxqF+oMxiafs2FtpPgoH7QL4tpI5JwkDUmrVq3ClClTMHPmTGRmZqJHjx5ISUlBfn6+xf1fffVVLFq0CB9++CEOHjyIp59+Gvfeey927dpl58qJGudMUQXmbDyKG9/7A8MXb8d3mWdRWWNAmwAPvJjSEX+/fCtWPJGIofHhcHdV2K2uLmHemHRbewDAzB8O4FxJ1TWOoKYwGgX+d0CLkUvS8Pb6Q1eddWgr63blYM2uHMhlwLyHe151hllsoCf+O7oPVEo5fj+cj9e+3w8hrr9m87WR2IpEzksmmvIvwEoSExPRp08ffPTRRwAAo9GIyMhI/N///R+mTp162f5hYWGYPn06JkyYYNp2//33w93dHStWrGjUOXU6Hby9vVFaWgqNRmOdN0J0CSEENh7Mw3+3nEL6qYvdaV4qJe7uEYYHEiLQK8pH8i6uOoMRDyzcht3ZJbihXQA+f7wv5BKMG9mwPxdzfzuG/rEBeCQxqkW0OtQajPh+9zks3HwCx/PLTdvv7h6KOQ/1hKvSPv8/zSqqxF3z/0K5vg6Tk9tjcnKHRh33vwNaPL0iA0ZR3z377IVA3Vj7zpZiyEdb4KqUY8f0ZE79J6uQ4vNbsjFJNTU1yMjIwLRp00zb5HI5kpOTsW3bNovH6PV6uLm5mW1zd3fHli1brngevV4PvV5v+l6n0zWzcqIry8w6j9nrD2HH6fMAAJkMGNg+EPf3CkdKlxC4udivtehalAo53n+oBwbP/wtbjhdiRdoZjEqKsWsNfxzJr+/WMQoc1pZh6d+nkNTWHyP6ReGOziF2CxPWUlVjwModWVj850mcK60GUB+O7+wWgrW7cvDT3lxU1hjw8YheNv+7UGcwYtKqXSjX16F3tC8m3tKu0cfe0SUEs+7pitfW7cecjUcRonHDQ30av1p2w9pIKVwbiZycZCGpsLAQBoMBwcHmi5cFBwfj8OHDFo9JSUnBnDlzcOONNyI2NhapqalYs2YNDAaDxf0BYPbs2Zg1a5ZVayf6p9OFFXj318NYv08LAHBzkePxAW0wMikaod7uEld3ZbGBnpg6KA7//vEg3l5/CAPbB5pmPdla+qlijF+RgTqjQHKnIAAy/H44D9tOFmHbySIEeKowrE8EhveNsuoMP1soqazB59vOYNnW0yi+cD+0AE8VnhjYBo8kRkHj5oI7u4Xi6S8y8PvhfIz5bAcWj+4NT5XtfgXPTz2GXVkl8HJTYu7DPc2WgmiMkf2ikVtShY83ncC0tfsQqFHhlo5B1zyOayNRSyJZd9u5c+cQHh6OrVu3IikpybT9pZdewubNm5GWlnbZMQUFBRg3bhx+/PFHyGQyxMbGIjk5GUuXLkVVleUxFZZakiIjI9ndRlZRXFGD+anH8GXaGdQaBGQy4MGECEy5vSNCvN2u/QIOwGgUeHRJGraeKEKvKB9881TSdX+gXq/9OaUY/ul2lOnrcEvHQHw6qjdcFHKcK6nCyvQsrNyRjfyy+n+3MhlwS8cgPNovCjd1CHKoqeTa0mr896+T+Co9C5UX1raK8lPjqZva4v5eEZe1FqWdLMLY5TtRrq9Dz0gfLBvTBz5qV6vXlXayCMMXb4dRAPOHx+NfPcKa9DpCCDz/zR6s2ZUDtasCK5/sh+4RPlc95pd9uRj/ZSaCNSpsnXqbQ/28yLm1qu62gIAAKBQK5OXlmW3Py8tDSIjlBe4CAwOxbt06VFdXo6ioCGFhYZg6dSratm17xfOoVCqoVFwHhqyrutaAJVtOYeGmEyjT1wEAbu4YiKl3xiEuxLnCt1wuw3sP9sCgD/5EZlYJFv15EhOuo2vmep0oKMfopeko09ehb4wfPh6RAJcLoSzMxx1T7uiI/7utPX47mIcVaWfw9/Ei/H44H78fzke4jzseSYzCg70jEOQlXQg9WVCORZtPYs2us6g11P8/s1OoBs/cHIs7u4ZcMWQmtvXHl08kYvRn6didXYKHP92OL8YmItDLer+jSitr8dyq3TCK+pacpgYkAJDJZHjn/u7IL9Njy/HCC4tNDkCU/5Vb9rg2ErUkkg/c7tu3Lz788EMA9QO3o6KiMHHiRIsDt/+ptrYWnTp1wkMPPYS33367UefkwG1qDoNRYE3mWczZeBS5F8acdAnT4JW7OmGAk68o/G3GWbyweg9cFDJ8P+EGdA6z/r+PnJIqPPjJVpwrrUbXcA2+GtfvmveQO1lQjq/SsrA64yxKq2oBAEq5DCldQzAiMQpJbf3tNgh+39lSfLL5OH7Zr0XDb86+bfzwzM2xuKlDYKPrOKItw6NL0lBQpkfbAA988UQiwn2a3y0rhMCErzKxfp8WMf5q/PTsQKt06ZVV12LYou04mKtDmwAPfDe+P/w8Lm8Byy+rRtLs32EwCvw25aYWMQifHIcUn9+ShqRVq1Zh9OjRWLRoEfr27Yu5c+fim2++weHDhxEcHIxRo0YhPDwcs2fPBgCkpaUhJycHPXv2RE5ODv7973/j1KlTyMzMhI+PT6POyZBETbX5aAFmrz+Ew9oyAEC4jzteSOmAe3qESzIrzNqEEHjyiwxsPJiHuBAvfD9xAFRK6w0uLizX46GF23CysAJtAz2w+qmk61rtu7rWgJ/35uLLtDPIzCoxbW8b6IERidF4oFcEvNXWHyQsRP0aVx9vOoEtxwtN25M7BWP8zW2REO3XpNc9XViBEf9NQ05JFcJ93PHlE4nNvh3INzuy8dJ3e6GUy/Dd+P7oYcX7sOXpqnHfx1uRU1KF+CgffPVEv8uWrVj850m8tf4Q4qN8sPaZAVY7NxHQCkMSAHz00Ud47733oNVq0bNnT8yfPx+JiYkAgJtvvhkxMTFYtmwZAGDz5s0YP348Tp48CU9PT9x111145513EBbW+OZkhiS6XgfOleKdXw7jr2P1H5BebkpMvKUdRvePcajZatZQWK7HHR/8ieKKGjxzcyxeGhRnldfVVddi+KfbceCcDuE+7lj9dBLCmtFycvCcDl+mncG6XTmm+9yplHIM6RGGoT3D4eWmhEIug1wmg0Iug0IO09eX/imXA4qG7+Wyi1/LZJDLgN8O5eOTzSewJ7sEAKCQy3BPjzA8fXMsOgR7Nfu6nCupwqP/TcPJwgoEeqmwYmwiOoY07XVPFJTj7vlbUFVrwMuD4jD+5thm1/dPx/PLcP8n21BaVYvbOwdj4aMJpi41IQQGzf0LR/LK8Na9XTEiMdrq56fWrVWGJHtjSKLGyimpwvv/O4K1u3IgBOCikGFUUgwm3tIOvha6GlqKDfvr18iRy4DVTyc1uaWkQVWNAaOXpiP9dDECPF3xzVNJaBtonW6Ycn0d1u3KwYrtZ0wtfLagUsrxcJ9IjLuxrdVn2hWU6TFqaToO5ergo3bB8jF9r7sFSF9nwH0fb8WBczr0j/XHirGJNmvd3Hm6GI/8Nw01dUY82i8Kb9zTFTKZjGsjkc0xJNkBQxJdS2lVLT7ZdAJL/z6Fmrr62zIM6RGGl1I6ItLPsaeiW8uUVbuxZlcOYvzVWD9pYJPv4VVTZ8RTX+zEH0cK4KVSYuVT/dAlzNvK1da3YmRmleDL7WeQkXUedQYBg1HAIASMF/40GC9+bTQCRlH/9dV+A2rclBjdPwaj+8dY5YavV1JaWWsazO3hqsCSx/qgX1v/Rh//9vpD+PTPk/BVu+CXSTfafGblhv31M9iEAF5M6YgJt7TDzO/3Y/m2MxjSIwwfDo+36fmpdWJIsgOGJLqSwnI9vt99Dh/9fgznK+sHCCe28cMrd3Wy6tgOZ1BaVYtBc/9Ebmk1RiVF4/V7ul73axiMApNX7caPe87BzUWOL8Ymok9M81qlbEGISwMVzAKVp5vSNPPO1sr1dRi3fCe2nSyCSinHwpEJjVqX6K9jBRi5pP6el5+OTMAdXSzPDra2ZX+fwr9/PAgAmH1fN/y/DYdRUlmL5Y/3xU0dAu1SA7UuDEl2wJBEDUoqa7D9ZDG2nyzC1hOFOJp38fYR7YI8Me3OONwaFyT57UOkcumH7xdj+2Jg+8Z/8AkhMH3dfnyVlgWlXIbFo3s36gO/tauuNWDCl5lIPZwPF4UM8x6Ox13dQq+4f1G5HoPm/YWCMj0e7ReFN4d2s2O1wOz1h7Doz5Om77k2EtlSq1onicjedNW1SD9ZXL+i84kiHNLqLutqiQvxwqikGDzUO8LmCyo6uoHtAzEqKRqfbzuDF1fvxa/P3djocSbv/noEX6VlQSYDPhjWkwGpkdxcFFg4MgHPrdqNn/bmYuJXmfh/93fHg70vvyWIEAIvfbsXBWV6tA/yxPS7Otu93pcHxSG3tBo/XFhhm2sjUUvDkEQtVoW+DjtO14ei7SeKsC+nFP+8CXu7IE8ktfVHUqw/Etv4XdeU9NZg6p1x+PNoAU4XVWLWDwcwZ1jPax6zcPMJfLLpBADg7Xu7YUgzFjNsjVwUcsx7OB4erkqs2pmNF7/di8oaA0b3jzHb74vtZ5B6OB+uSjnmD4+/bDq+PdQvRNod5fq6+gHdfaPsXgORLbG7jVqM6loDMs6cx7YT9d1ne8+Wou4fqSjGX42kWH/0a+uPpLb+CNI4x61DpJRx5jweXLgVRgEsfLQXBnW9cvfPV2lZeGXtPgD1Aevpm6w/Db21EELgjZ8OYenfpwBcHCAN1C9GOeSjLaipM2LmkM4YM6CNlKVCCAGjAFuRyKbY3UZ0nfR1Bizdchp/HMnH7qwS1BiMZs9H+LqbWoqSYv0d+mazjioh2hdP3xSLjzedwCtr9yMh2s/ibTR+3HMO09fVB6TxN8cyIDWTTCbDa3d3gpebEvNSj+G9X4+grLoOk5Pb49mvd6GmzoibOwbisX+0MElVq4L5iFogtiSR08rXVeOpFRnYdcnqyyEat/pAdCEYtZYp+7amrzPgno/+xmFtGW7vHIxPRyaYDWjfdCQf4z7fiVqDwIjEKLw5tGurHfBuCw0rWQNAmwAPnCqsQICnChsmD7Tp0gREjoQtSUSNtPdsCZ78PANaXTW83V3w/B0dMLB9IGL81fxwtgGVUoEPhvXEvz7ago0H8/BdZg4eSIgAAOw4XYynV2Sg1iAwpEcYXr+HAcnaxt3YFh4qJaav24dThRUAgP882J0BicjGGJLI6Xy/OwcvfbsX+joj2gV5YvGo3mjTzHte0bV1CtXguds74N0NRzDrhwNIivVHSWUNHl+2A9W1RtzSMRBzHurBcSk28khiFDxUCrzx00GMTorBzZwxSGRz7G4jp2EwCvznf0dMM6dujQvCvId7wusad5En6zEYBR5cuBWZWSWIj/JBVlEliipq0DfGD8sf7yvJDKvWRgjBljpqlaT4/G7dC8GQ0yirrsWTn+80BaTxN8di8ajeDEh2ppDL8P5DPeHuosCurBIUVdSga7gG/32sNwOSnTAgEdkPQxI5vNOFFbj3461IPZwPlVKOeQ/3xMuD4titI5E2AR54ZXAnAEDbQA8sH9MXGoZVImqBOCaJHNqWY4WY8FUmSqtqEaxR4dORvVvdfdQc0ch+0egV5YM2AR5NvvktEZGj4283ckhCCCzbehpv/nwIBqNAz0gffDoygYs/OpAuYd5Sl0BEZFMMSeRw9HUGzFh3AKt2ZgMA7u8Vgbfu7Qo3F455ISIi+2FIIodSUKbH+BUZ2HnmPOQy4JW7OmHsDW04WJWIiOyOIYkcxv6cUjz5+U6cK62Gl5sSHw6P51owREQkGYYkcgg/7T2HF1bvQXWtEW0DPbB4VG/EBnpKXRYREbViDEkkKaNRYM7Go/joj+MAgJs6BGL+8Hh4u3NKORERSYshiSRTrq/Dc6t2Y+PBPADAkze25fpHRETkMBiSSBJZRZV44vMdOJpXDlelHO/c1w339YqQuiwiIiIThiSym3J9HX47mIef9p7Dn0cLUWMwIshLhUUjExAf5St1eURERGYYksimqmoM+ONIPn7ccw6/H86Hvs5oeq5PjC8+HN4LId5cIJKIiBwPQxJZnb7OgD+PFuKnveew8WAeKmsMpufaBHhgSPdQ3N0jDB2CvSSskoiI6OoYksgqag1G/H28ED/tzcWvB7Qoq64zPRfu444hPcJwd/dQdAnTcGFIIiJyCgxJ1GQGo0DaqSL8uCcXG/bn4nxlrem5YI0Kg7uFYUiPUPSM9GEwIiIip8OQRNfFaBTIzDqPn/bm4ud9uSgo05ueC/B0xZ1dQ3F391D0ifGDnFP5iYjIiTEk0TWdr6hB2qkibD1RhN8O5uFcabXpOW93F9zZNQR3dw9Dv7Z+UCrkElZKRERkPQxJdJlyfR12nCrG1hOF2HqiCAdzdRDi4vOeKiXu6BKMId3DMKBdAFyVDEZERNTyMCQRqmsNyDhz3hSK9p4thcEozPbpEOyJ/rEBGNAuAAPbB8DNRSFRtURERPbBkNQK1dQZsfdsCbaeKMLWE4XIPFOCGoPRbJ9ofzX6x/ojKTYA/dr6IciLaxkREVHrwpDUChiMAgfP6UwtRTtOF5utXQTUz0YbEBuApFh/JMX6I8JXLVG1REREjoEhqYXbsD8XL3+3D6VVtWbb/TxckdS2PhD1j/VHmwAPTtMnIiK6BENSC7d0y2mUVtXCS6VEYls/JMUGoH+sPzoGe3GKPhER0VUwJLVg+joDdp8tAQCsmzgAsYGe0hZERETkRDh3uwXbe7YUNXVGBHi6om2Ah9TlEBERORWGpBYs/VQxAKBPjB/HGxEREV0nhqQWrCEk9W3jJ3ElREREzochqYUyGAUyz5wHUN+SRERERNeHIamFOpSrQ5m+Dl4qJTqFaqQuh4iIyOkwJLVQO07Xd7X1ivaFglP9iYiIrhtDUgvF8UhERETNw5DUAgkhTC1JDElERERNw5DUAp0qrEBheQ1clXJ0j/CWuhwiIiKnxJDUAjW0IvWM8IFKqZC4GiIiIufEkNQCpTUsItnGV+JKiIiInBdDUgt0cTySv8SVEBEROS/JQ9KCBQsQExMDNzc3JCYmIj09/ar7z507Fx07doS7uzsiIyPx3HPPobq62k7VOj5taTWyi6sglwG9onykLoeIiMhpSRqSVq1ahSlTpmDmzJnIzMxEjx49kJKSgvz8fIv7f/XVV5g6dSpmzpyJQ4cOYcmSJVi1ahVeeeUVO1fuuNIvtCJ1DtPAy81F4mqIiIicl6Qhac6cORg3bhzGjBmDzp07Y+HChVCr1Vi6dKnF/bdu3YoBAwbgkUceQUxMDO644w4MHz78mq1PrUn6qSIAvBUJERFRc0kWkmpqapCRkYHk5OSLxcjlSE5OxrZt2ywe079/f2RkZJhC0cmTJ7F+/XrcddddVzyPXq+HTqcze7RkO07V368tkesjERERNYtSqhMXFhbCYDAgODjYbHtwcDAOHz5s8ZhHHnkEhYWFuOGGGyCEQF1dHZ5++umrdrfNnj0bs2bNsmrtjqqksgZH8soAAL3ZkkRERNQskg/cvh6bNm3C22+/jY8//hiZmZlYs2YNfv75Z7zxxhtXPGbatGkoLS01PbKzs+1YsX3tPF3fitQ20AMBniqJqyEiInJukrUkBQQEQKFQIC8vz2x7Xl4eQkJCLB7z2muvYeTIkXjiiScAAN26dUNFRQWefPJJTJ8+HXL55ZlPpVJBpWodgaFh0HZftiIRERE1m2QtSa6urkhISEBqaqppm9FoRGpqKpKSkiweU1lZeVkQUijqV5QWQtiuWCfBm9oSERFZj2QtSQAwZcoUjB49Gr1790bfvn0xd+5cVFRUYMyYMQCAUaNGITw8HLNnzwYADBkyBHPmzEF8fDwSExNx/PhxvPbaaxgyZIgpLLVWlTV12J9TCoAz24iIiKxB0pA0bNgwFBQUYMaMGdBqtejZsyc2bNhgGsydlZVl1nL06quvQiaT4dVXX0VOTg4CAwMxZMgQvPXWW1K9BYexO6sEdUaBUG83RPi6S10OERGR05OJVtZPpdPp4O3tjdLSUmg0GqnLsZoPNh7FvNRj+FePMMwfHi91OURERFYlxee3U81uoyu7eL82drURERFZA0NSC1BrMGJXVgkAhiQiIiJrYUhqAfbnlKKq1gAftQvaBXpKXQ4REVGLwJDUAjRM/e8d7Qe5XCZxNURERC0DQ1IL0DAeifdrIyIish6GJCdnNArsuHA7kj4MSURERFbDkOTkjuWXo7SqFu4uCnQJazlLGhAREUmNIcnJpZ8qAgD0ivaBi4I/TiIiImvhp6qTS7/Q1dY3xl/iSoiIiFoWhiQnJoTAjgsz2/q08ZW4GiIiopaFIcmJnT1fBa2uGi4KGeIjGZKIiIisiSHJiaVdaEXqGu4Nd1eFxNUQERG1LAxJTqyhq61vDKf+ExERWRtDkhPjTW2JiIhshyHJSRWU6XGysAIyWf3tSIiIiMi6GJKcVEMrUsdgL3irXSSuhoiIqOVhSHJSDTe17cPxSERERDbBkOSkOB6JiIjIthiSnJCuuhaHcnUAGJKIiIhshSHJCWWcOQ+jAKL81AjWuEldDhERUYvEkOSEdnA8EhERkc0xJDmhhvFIiexqIyIishmGJCdTXWvAnuxSAEAfhiQiIiKbYUhyMnuyS1BjMCLAU4UYf7XU5RAREbVYDElO5uLUf1/IZDKJqyEiImq5GJKcTPrp8wB4U1siIiJbY0hyIgajQOaZ+pDE8UhERES2xZDkRA7l6lCur4OXSom4EI3U5RAREbVoDElOJO3C+kgJMb5QyDkeiYiIyJYYkpxIwyKSvBUJERGR7TEkOQkhxMWZbRy0TUREZHMMSU7iZGEFiipq4KqUo1uEt9TlEBERtXgMSU4i/UJXW89IH6iUComrISIiavkYkpxEw3gk3q+NiIjIPhiSnET6hfFIfTgeiYiIyC4YkpzAuZIqnD1fBbkM6BXtK3U5RERErQJDkhNomNXWJcwbniqlxNUQERG1DgxJTiCd6yMRERHZHUOSE9jB8UhERER2x5Dk4M5X1OBoXjkAoE8MxyMRERHZC0OSg2toRYoN9IC/p0riaoiIiFoPhiQHZ7oVCccjERER2RVDkoNLP30eAEMSERGRvTEkObAKfR0O5JQC4KBtIiIie2NIcmC7skpQZxQI83ZDhK9a6nKIiIhaFYYkB2a6FQm72oiIiOyuycs3l5SUID09Hfn5+TAajWbPjRo1qtmF0cWb2nI8EhERkf01KST9+OOPGDFiBMrLy6HRaCCTyUzPyWQyhiQrqKkzYlf2hUHbHI9ERERkd03qbnv++efx+OOPo7y8HCUlJTh//rzpUVxcfN2vt2DBAsTExMDNzQ2JiYlIT0+/4r4333wzZDLZZY/Bgwc35a04rH05paiuNcJX7YJ2QZ5Sl0NERNTqNCkk5eTk4Nlnn4Va3fzBxKtWrcKUKVMwc+ZMZGZmokePHkhJSUF+fr7F/desWYPc3FzTY//+/VAoFHjwwQebXYsjaVgfqXeMn1lLHREREdlHk0JSSkoKdu7caZUC5syZg3HjxmHMmDHo3LkzFi5cCLVajaVLl1rc38/PDyEhIabHxo0boVarW15IujAeKZHjkYiIiCTRpDFJgwcPxosvvoiDBw+iW7ducHFxMXv+X//6V6Nep6amBhkZGZg2bZppm1wuR3JyMrZt29ao11iyZAkefvhheHh4WHxer9dDr9ebvtfpdI16XSkZjYI3tSUiIpJYk0LSuHHjAACvv/76Zc/JZDIYDIZGvU5hYSEMBgOCg4PNtgcHB+Pw4cPXPD49PR379+/HkiVLrrjP7NmzMWvWrEbV4yiO5JVBV10HtasCXcI0UpdDRETUKjWpu81oNF7x0diAZA1LlixBt27d0Ldv3yvuM23aNJSWlpoe2dnZdquvqTKz6me19YryhVLBpayIiIikcN2fwLW1tVAqldi/f3+zTx4QEACFQoG8vDyz7Xl5eQgJCbnqsRUVFVi5ciXGjh171f1UKhU0Go3Zw9GdPV8FAIgNtNyFSERERLZ33SHJxcUFUVFRVmkxcnV1RUJCAlJTU03bjEYjUlNTkZSUdNVjV69eDb1ej0cffbTZdTiavNJqAECwt5vElRAREbVeTerLmT59Ol555ZUmrYn0T1OmTMHixYuxfPlyHDp0COPHj0dFRQXGjBkDoH717ksHdjdYsmQJhg4dCn9//2bX4Gi0uvqQFKJhSCIiIpJKkwZuf/TRRzh+/DjCwsIQHR192cyyzMzMRr/WsGHDUFBQgBkzZkCr1aJnz57YsGGDaTB3VlYW5HLzLHfkyBFs2bIF//vf/5pSvsMzhSS2JBEREUmmSSFp6NChVi1i4sSJmDhxosXnNm3adNm2jh07Qghh1RochRAC2lK2JBEREUmtSSFp5syZ1q6DLijT16Gypn68F1uSiIiIpMP55Q6mYdC2l5sSatcmZVgiIiKygiZ9Csvl8qveT8yeayW1NBy0TURE5BiaFJLWrl1r9n1tbS127dqF5cuXO93q1o7GNB6JXW1ERESSalJIuueeey7b9sADD6BLly5YtWrVNRd4pCvLu9CSFMyWJCIiIklZdUxSv379zBaGpOvX0N0WypYkIiIiSVktJFVVVWH+/PkIDw+31ku2StpSPQC2JBEREUmtSd1tvr6+ZgO3hRAoKyuDWq3GihUrrFZca6TV1d+3jQO3iYiIpNWkkPTBBx+YhSS5XI7AwEAkJibC19fXasW1Rg0tSRy4TUREJK0mhaRbb70VkZGRFpcByMrKQlRUVLMLa41qDUYUVbC7jYiIyBE0aUxSmzZtUFBQcNn2oqIitGnTptlFtVb5ZXoIAbgoZPD3cJW6HCIiolatSSHpSvdNKy8vh5sbW0CaqmGNpCAvN8jlV16sk4iIiGzvurrbpkyZAgCQyWSYMWMG1Gq16TmDwYC0tDT07NnTqgW2JhfXSFJJXAkRERFdV0jatWsXgPqWpH379sHV9WKXkKurK3r06IEXXnjBuhW2Ig0tSaHe7hJXQkRERNcVkv744w8AwJgxYzBv3jxoNBqbFNVacbVtIiIix9GkMUmfffYZNBoNjh8/jl9//RVVVfVr+1xprBI1Tq7pvm3sbiMiIpJak0JScXExbrvtNnTo0AF33XUXcnNzAQBjx47F888/b9UCWxMtW5KIiIgcRpNC0uTJk+Hi4oKsrCyzwdvDhg3Dhg0brFZca9PQ3cbVtomIiKTXpMUk//e//+HXX39FRESE2fb27dvjzJkzVimstRFCmAZuc7VtIiIi6TWpJamiosKsBalBcXExVCqOp2mK0qpa6OuMANjdRkRE5AiaFJIGDhyIzz//3PS9TCaD0WjEu+++i1tuucVqxbUmDeORfNUucHNRSFwNERERNam77b333sOtt96KnTt3oqamBi+99BIOHDiA4uJi/P3339ausVVo6GpjKxIREZFjuO6QVFtbi2effRY//vgjNm7cCC8vL5SXl+O+++7DhAkTEBoaaos6WzyORyIiInIs1x2SXFxcsHfvXvj6+mL69Om2qKlV0nJmGxERkUNp0pikRx99FEuWLLF2La0aV9smIiJyLE0ak1RXV4elS5fit99+Q0JCAjw8PMyenzNnjlWKa03Y3UZERORYmhSS9u/fj169egEAjh49avacTCZrflWtkFanB8DuNiIiIkfRpJDUcKNbsh7TattsSSIiInIITRqTRNZVXWtAcUUNALYkEREROQqGJAeQf6GrzVUph4/aReJqiIiICGBIcgiXTv/nmC4iIiLHwJDkALhGEhERkeNhSHIAeQ23JOGgbSIiIofBkOQALrYkqSSuhIiIiBowJDkAU0jydpe4EiIiImrAkOQAGrrbOCaJiIjIcTAkOYBc0y1J2N1GRETkKBiSJGY0CuSX8ea2REREjoYhSWLFlTWoNQgAQJAXQxIREZGjYEiSmPZCV1uApytclfxxEBEROQp+Kkus4ca27GojIiJyLAxJEmuY/h/KhSSJiIgcCkOSxEyrbbMliYiIyKEwJEksl2skEREROSSGJIk1dLfxvm1ERESOhSFJYnk6tiQRERE5IoYkiWlNq20zJBERETkShiQJVdUYoKuuA8CB20RERI5G8pC0YMECxMTEwM3NDYmJiUhPT7/q/iUlJZgwYQJCQ0OhUqnQoUMHrF+/3k7VWlfDeCR3FwU0bkqJqyEiIqJLSfrJvGrVKkyZMgULFy5EYmIi5s6di5SUFBw5cgRBQUGX7V9TU4Pbb78dQUFB+PbbbxEeHo4zZ87Ax8fH/sVbQUNXW6i3G2QymcTVEBER0aUkDUlz5szBuHHjMGbMGADAwoUL8fPPP2Pp0qWYOnXqZfsvXboUxcXF2Lp1K1xcXAAAMTEx9izZqrS6KgDsaiMiInJEknW31dTUICMjA8nJyReLkcuRnJyMbdu2WTzmhx9+QFJSEiZMmIDg4GB07doVb7/9NgwGwxXPo9frodPpzB6OQluqB8BB20RERI5IspBUWFgIg8GA4OBgs+3BwcHQarUWjzl58iS+/fZbGAwGrF+/Hq+99href/99vPnmm1c8z+zZs+Ht7W16REZGWvV9NAfv20ZEROS4JB+4fT2MRiOCgoLw6aefIiEhAcOGDcP06dOxcOHCKx4zbdo0lJaWmh7Z2dl2rPjqTNP/NSqJKyEiIqJ/kmxMUkBAABQKBfLy8sy25+XlISQkxOIxoaGhcHFxgUKhMG3r1KkTtFotampq4OrqetkxKpUKKpVjhpCG2W3sbiMiInI8krUkubq6IiEhAampqaZtRqMRqampSEpKsnjMgAEDcPz4cRiNRtO2o0ePIjQ01GJAcnTsbiMiInJckna3TZkyBYsXL8by5ctx6NAhjB8/HhUVFabZbqNGjcK0adNM+48fPx7FxcWYNGkSjh49ip9//hlvv/02JkyYINVbaDKDUSC/rH7gdqi3u8TVEBER0T9JugTAsGHDUFBQgBkzZkCr1aJnz57YsGGDaTB3VlYW5PKLOS4yMhK//vornnvuOXTv3h3h4eGYNGkSXn75ZaneQpMVlethMArIZUCAp/O1ghEREbV0MiGEkLoIe9LpdPD29kZpaSk0Go1kdezJLsE9C/5GsEaFtFeSr30AERFRKybF57dTzW5rSUyDtjkeiYiIyCExJEmEg7aJiIgcG0OSRExrJHH6PxERkUNiSJKIli1JREREDo0hSSIN3W2hbEkiIiJySAxJErl4SxKGJCIiIkfEkCSRhpAUzJYkIiIih8SQJIGy6lpU1BgAsCWJiIjIUTEkSaBhPJKXSgkPlaSLnhMREdEVMCRJQFtaf882drURERE5LoYkCXC1bSIiIsfHkCSBhu42LiRJRETkuBiSJMDp/0RERI6PIUkCuZz+T0RE5PAYkiSQxzFJREREDo8hSQIcuE1EROT4GJLsrNZgRGF5wxIAKomrISIioithSLKzgjI9hACUchkCPBiSiIiIHBVDkp01dLUFa9wgl8skroaIiIiuhCHJzvIaZrZp2IpERETkyBiS7Kxh+j8XkiQiInJsDEl2lndJdxsRERE5LoYkO+P0fyIiIufAkGRnWna3EREROQWGJDtjdxsREZFzYEiyIyEEu9uIiIicBEOSHemq6lBdawTA7jYiIiJHx5BkR7m6KgCAj9oFbi4KiashIiKiq2FIsiPToG12tRERETk8hiQ74qBtIiIi58GQZEfaUj0AtiQRERE5A4YkOzLd3JaDtomIiBweQ5Id5XH6PxERkdNgSLKjhoHboWxJIiIicngMSXbEgdtERETOgyHJTvR1BhRV1ADgQpJERETOgCHJTvJ19TPbXJVy+KpdJK6GiIiIroUhyU5MM9s0KshkMomrISIiomthSLITrrZNRETkXBiS7ISDtomIiJwLQ5KdcPo/ERGRc2FIshMtW5KIiIicCkOSnZjGJLEliYiIyCkwJNmJlrckISIicioMSXYghDCtk8TuNiIiIufAkGQHxRU1qDEYATAkEREROQuGJDto6Grz93CFq5KXnIiIyBnwE9sOGtZI4qBtIiIi5+EQIWnBggWIiYmBm5sbEhMTkZ6efsV9ly1bBplMZvZwc3Ps8KEtrR+PxEHbREREzkPykLRq1SpMmTIFM2fORGZmJnr06IGUlBTk5+df8RiNRoPc3FzT48yZM3as+PppS6sAAMFsSSIiInIakoekOXPmYNy4cRgzZgw6d+6MhQsXQq1WY+nSpVc8RiaTISQkxPQIDg6+4r56vR46nc7sYW+c/k9EROR8JA1JNTU1yMjIQHJysmmbXC5HcnIytm3bdsXjysvLER0djcjISNxzzz04cODAFfedPXs2vL29TY/IyEirvofG0OrY3UZERORsJA1JhYWFMBgMl7UEBQcHQ6vVWjymY8eOWLp0Kb7//nusWLECRqMR/fv3x9mzZy3uP23aNJSWlpoe2dnZVn8f15J3YbVtdrcRERE5D6XUBVyvpKQkJCUlmb7v378/OnXqhEWLFuGNN964bH+VSgWVSmXPEi/D7jYiIiLnI2lLUkBAABQKBfLy8sy25+XlISQkpFGv4eLigvj4eBw/ftwWJTZbda0BpVW1ALgEABERkTORNCS5uroiISEBqamppm1GoxGpqalmrUVXYzAYsG/fPoSGhtqqzGZpuLGtu4sCGjena7gjIiJqtST/1J4yZQpGjx6N3r17o2/fvpg7dy4qKiowZswYAMCoUaMQHh6O2bNnAwBef/119OvXD+3atUNJSQnee+89nDlzBk888YSUb+OKcksvLiQpk8kkroaIiIgaS/KQNGzYMBQUFGDGjBnQarXo2bMnNmzYYBrMnZWVBbn8YoPX+fPnMW7cOGi1Wvj6+iIhIQFbt25F586dpXoLV9Ww2nawRtpxUURERHR9ZEIIIXUR9qTT6eDt7Y3S0lJoNBqbn2/h5hN455fDGNozDHMfjrf5+YiIiFoie39+Aw6wmGRLp+X0fyIiIqfEkGRjeZz+T0RE5JQYkmyMayQRERE5J4YkG8u7ZHYbEREROQ+GJBsyGAXyyi7ct40hiYiIyKkwJNlQUbkeBqOAXAYEenIJACIiImfCkGRDDeORAjxVUCp4qYmIiJwJP7ltSMvxSERERE6LIcmGLq62zZBERETkbBiSbIjT/4mIiJwXQ5INaUs5s42IiMhZMSTZkFZXBYAtSURERM6IIcmGOHCbiIjIeTEk2VCerr67jQO3iYiInA9Dko2U6+tQrq8DwJYkIiIiZ8SQZCMNXW2eKiU8VUqJqyEiIqLrxZBkIxfXSOLtSIiIiJwRQ5KNNLQkhXq7S1wJERERNQVDko1oudo2ERGRU2NIspGL0//Z3UZEROSMGJJshLckISIicm4MSTbCm9sSERE5N4YkG+Fq20RERM6NIckG6gxGFJZfuLktW5KIiIicEkOSDRSU62EUgFIug78nB24TERE5I4YkG2joagvyUkEhl0lcDRERETUFQ5INNISkYI5HIiIicloMSTbA6f9ERETOjyHJBrjaNhERkfNjSLKBPE7/JyIicnoMSTbA7jYiIiLnx5BkA3m6C2sksSWJiIjIaTEkWZkQ4uJq22xJIiIicloMSVamq6pDVa0BAFuSiIiInBlDkpU1jEfydneBm4tC4mqIiIioqRiSrIyDtomIiFoGhiQry+Nq20RERC0CQ5KVXWxJ4o1tiYiInBlDkpWxu42IiKhlYEiysourbbtLXAkRERE1B0OSleWaQhK724iIiJwZQ5KV5fHmtkRERC0CQ5IV6esMKKqoAcAxSURERM6OIcmK8i/cs81VIYefh6vE1RAREVFzMCRZUUNXW5BGBZlMJnE1RERE1BwMSVbE6f9EREQtB0OSFWlNM9sYkoiIiJwdQ5IVmUISW5KIiIicnkOEpAULFiAmJgZubm5ITExEenp6o45buXIlZDIZhg4datsCG8nU3caWJCIiIqcneUhatWoVpkyZgpkzZyIzMxM9evRASkoK8vPzr3rc6dOn8cILL2DgwIF2qvTauEYSERFRyyF5SJozZw7GjRuHMWPGoHPnzli4cCHUajWWLl16xWMMBgNGjBiBWbNmoW3btld9fb1eD51OZ/awFbYkERERtRyShqSamhpkZGQgOTnZtE0ulyM5ORnbtm274nGvv/46goKCMHbs2GueY/bs2fD29jY9IiMjrVL7PwkhkHdhnSSOSSIiInJ+koakwsJCGAwGBAcHm20PDg6GVqu1eMyWLVuwZMkSLF68uFHnmDZtGkpLS02P7OzsZtdtyfnKWtTUGQHUr5NEREREzk0pdQHXo6ysDCNHjsTixYsREBDQqGNUKhVUKtuHloaZbf4erlApFTY/HxEREdmWpCEpICAACoUCeXl5Ztvz8vIQEhJy2f4nTpzA6dOnMWTIENM2o7G+9UapVOLIkSOIjY21bdFXUFZdC42bkoO2iYiIWghJQ5KrqysSEhKQmppqmsZvNBqRmpqKiRMnXrZ/XFwc9u3bZ7bt1VdfRVlZGebNm2ez8UaNkdjWH3v/nWLqciMiIiLnJnl325QpUzB69Gj07t0bffv2xdy5c1FRUYExY8YAAEaNGoXw8HDMnj0bbm5u6Nq1q9nxPj4+AHDZdqm4KiWfMEhERERWIHlIGjZsGAoKCjBjxgxotVr07NkTGzZsMA3mzsrKglzO4EFERET2JRNCCKmLsCedTgdvb2+UlpZCo9FIXQ4RERE1ghSf32yiISIiIrKAIYmIiIjIAoYkIiIiIgsYkoiIiIgsYEgiIiIisoAhiYiIiMgChiQiIiIiCxiSiIiIiCxgSCIiIiKygCGJiIiIyAKGJCIiIiILJL/Brb013KpOp9NJXAkRERE1VsPntj1vOdvqQlJZWRkAIDIyUuJKiIiI6HqVlZXB29vbLueSCXtGMgdgNBpx7tw5eHl5QSaTWfW1dTodIiMjkZ2dbbc7FBOvu1R43aXB6y4NXndpXHrdvby8UFZWhrCwMMjl9hkt1OpakuRyOSIiImx6Do1Gw39EEuB1lwavuzR43aXB6y6NhuturxakBhy4TURERGQBQxIRERGRBQxJVqRSqTBz5kyoVCqpS2lVeN2lwesuDV53afC6S0Pq697qBm4TERERNQZbkoiIiIgsYEgiIiIisoAhiYiIiMgChiQiIiIiCxiSrGTBggWIiYmBm5sbEhMTkZ6eLnVJTmP27Nno06cPvLy8EBQUhKFDh+LIkSNm+1RXV2PChAnw9/eHp6cn7r//fuTl5Zntk5WVhcGDB0OtViMoKAgvvvgi6urqzPbZtGkTevXqBZVKhXbt2mHZsmW2fntO45133oFMJsPkyZNN23jdbSMnJwePPvoo/P394e7ujm7dumHnzp2m54UQmDFjBkJDQ+Hu7o7k5GQcO3bM7DWKi4sxYsQIaDQa+Pj4YOzYsSgvLzfbZ+/evRg4cCDc3NwQGRmJd9991y7vzxEZDAa89tpraNOmDdzd3REbG4s33njD7D5gvO7N9+eff2LIkCEICwuDTCbDunXrzJ635zVevXo14uLi4Obmhm7dumH9+vXX/4YENdvKlSuFq6urWLp0qThw4IAYN26c8PHxEXl5eVKX5hRSUlLEZ599Jvbv3y92794t7rrrLhEVFSXKy8tN+zz99NMiMjJSpKamip07d4p+/fqJ/v37m56vq6sTXbt2FcnJyWLXrl1i/fr1IiAgQEybNs20z8mTJ4VarRZTpkwRBw8eFB9++KFQKBRiw4YNdn2/jig9PV3ExMSI7t27i0mTJpm287pbX3FxsYiOjhaPPfaYSEtLEydPnhS//vqrOH78uGmfd955R3h7e4t169aJPXv2iH/961+iTZs2oqqqyrTPoEGDRI8ePcT27dvFX3/9Jdq1ayeGDx9uer60tFQEBweLESNGiP3794uvv/5auLu7i0WLFtn1/TqKt956S/j7+4uffvpJnDp1SqxevVp4enqKefPmmfbhdW++9evXi+nTp4s1a9YIAGLt2rVmz9vrGv/9999CoVCId999Vxw8eFC8+uqrwsXFRezbt++63g9DkhX07dtXTJgwwfS9wWAQYWFhYvbs2RJW5bzy8/MFALF582YhhBAlJSXCxcVFrF692rTPoUOHBACxbds2IUT9P0y5XC60Wq1pn08++URoNBqh1+uFEEK89NJLokuXLmbnGjZsmEhJSbH1W3JoZWVlon379mLjxo3ipptuMoUkXnfbePnll8UNN9xwxeeNRqMICQkR7733nmlbSUmJUKlU4uuvvxZCCHHw4EEBQOzYscO0zy+//CJkMpnIyckRQgjx8ccfC19fX9PPoeHcHTt2tPZbcgqDBw8Wjz/+uNm2++67T4wYMUIIwetuC/8MSfa8xg899JAYPHiwWT2JiYniqaeeuq73wO62ZqqpqUFGRgaSk5NN2+RyOZKTk7Ft2zYJK3NepaWlAAA/Pz8AQEZGBmpra82ucVxcHKKiokzXeNu2bejWrRuCg4NN+6SkpECn0+HAgQOmfS59jYZ9WvvPacKECRg8ePBl14bX3TZ++OEH9O7dGw8++CCCgoIQHx+PxYsXm54/deoUtFqt2TXz9vZGYmKi2XX38fFB7969TfskJydDLpcjLS3NtM+NN94IV1dX0z4pKSk4cuQIzp8/b+u36XD69++P1NRUHD16FACwZ88ebNmyBXfeeScAXnd7sOc1ttbvHYakZiosLITBYDD7kACA4OBgaLVaiapyXkajEZMnT8aAAQPQtWtXAIBWq4Wrqyt8fHzM9r30Gmu1Wos/g4bnrraPTqdDVVWVLd6Ow1u5ciUyMzMxe/bsy57jdbeNkydP4pNPPkH79u3x66+/Yvz48Xj22WexfPlyABev29V+p2i1WgQFBZk9r1Qq4efnd10/m9Zk6tSpePjhhxEXFwcXFxfEx8dj8uTJGDFiBABed3uw5zW+0j7X+zNQXtfeRDY2YcIE7N+/H1u2bJG6lBYvOzsbkyZNwsaNG+Hm5iZ1Oa2G0WhE79698fbbbwMA4uPjsX//fixcuBCjR4+WuLqW65tvvsGXX36Jr776Cl26dMHu3bsxefJkhIWF8brTFbElqZkCAgKgUCgum/GTl5eHkJAQiapyThMnTsRPP/2EP/74AxEREabtISEhqKmpQUlJidn+l17jkJAQiz+Dhueuto9Go4G7u7u1347Dy8jIQH5+Pnr16gWlUgmlUonNmzdj/vz5UCqVCA4O5nW3gdDQUHTu3NlsW6dOnZCVlQXg4nW72u+UkJAQ5Ofnmz1fV1eH4uLi6/rZtCYvvviiqTWpW7duGDlyJJ577jlTKyqvu+3Z8xpfaZ/r/RkwJDWTq6srEhISkJqaatpmNBqRmpqKpKQkCStzHkIITJw4EWvXrsXvv/+ONm3amD2fkJAAFxcXs2t85MgRZGVlma5xUlIS9u3bZ/aPa+PGjdBoNKYPpKSkJLPXaNintf6cbrvtNuzbtw+7d+82PXr37o0RI0aYvuZ1t74BAwZctsTF0aNHER0dDQBo06YNQkJCzK6ZTqdDWlqa2XUvKSlBRkaGaZ/ff/8dRqMRiYmJpn3+/PNP1NbWmvbZuHEjOnbsCF9fX5u9P0dVWVkJudz8I0+hUMBoNALgdbcHe15jq/3eua5h3mTRypUrhUqlEsuWLRMHDx4UTz75pPDx8TGb8UNXNn78eOHt7S02bdokcnNzTY/KykrTPk8//bSIiooSv//+u9i5c6dISkoSSUlJpucbpqLfcccdYvfu3WLDhg0iMDDQ4lT0F198URw6dEgsWLCgVU9Ft+TS2W1C8LrbQnp6ulAqleKtt94Sx44dE19++aVQq9VixYoVpn3eeecd4ePjI77//nuxd+9ecc8991icJh0fHy/S0tLEli1bRPv27c2mSZeUlIjg4GAxcuRIsX//frFy5UqhVqtbzVT0fxo9erQIDw83LQGwZs0aERAQIF566SXTPrzuzVdWViZ27doldu3aJQCIOXPmiF27dokzZ84IIex3jf/++2+hVCrFf/7zH3Ho0CExc+ZMLgEgpQ8//FBERUUJV1dX0bdvX7F9+3apS3IaACw+PvvsM9M+VVVV4plnnhG+vr5CrVaLe++9V+Tm5pq9zunTp8Wdd94p3N3dRUBAgHj++edFbW2t2T5//PGH6Nmzp3B1dRVt27Y1OwddHpJ43W3jxx9/FF27dhUqlUrExcWJTz/91Ox5o9EoXnvtNREcHCxUKpW47bbbxJEjR8z2KSoqEsOHDxeenp5Co9GIMWPGiLKyMrN99uzZI2644QahUqlEeHi4eOedd2z+3hyVTqcTkyZNElFRUcLNzU20bdtWTJ8+3WwaOa978/3xxx8Wf5+PHj1aCGHfa/zNN9+IDh06CFdXV9GlSxfx888/X/f7kQlxyXKjRERERASAY5KIiIiILGJIIiIiIrKAIYmIiIjIAoYkIiIiIgsYkoiIiIgsYEgiIiIisoAhiYiIiMgChiQiIiIiCxiSiMjmNm3aBJlMdtnNcomIHBlDEhFZ1c0334zJkyebbevfvz9yc3Ph7e0tTVFXEBMTg7lz50pdBhE5KKXUBRBRy+fq6oqQkBCpyyAiui5sSSIiq3nsscewefNmzJs3DzKZDDKZDKdPn76su23ZsmXw8fHBTz/9hI4dO0KtVuOBBx5AZWUlli9fjpiYGPj6+uLZZ5+FwWAwvb5er8cLL7yA8PBweHh4IDExEZs2bbpiPUII/Pvf/0ZUVBRUKhXCwsLw7LPPAqhv8Tpz5gyee+45U60NtmzZgoEDB8Ld3R2RkZF49tlnUVFRYXo+JiYGb7zxBoYPHw4PDw+Eh4djwYIF1r2YRCQ5hiQispp58+YhKSkJ48aNQ25uLnJzcxEZGWlx38rKSsyfPx8rV67Ehg0bsGnTJtx7771Yv3491q9fjy+++AKLFi3Ct99+azpm4sSJ2LZtG1auXIm9e/fiwQcfxKBBg3Ds2DGL5/juu+/wwQcfYNGiRTh27BjWrVuHbt26AQDWrFmDiIgIvP7666ZaAeDEiRMYNGgQ7r//fuzduxerVq3Cli1bMHHiRLPXfu+999CjRw/s2rULU6dOxaRJk7Bx40ZrXEYichSCiMiKbrrpJjFp0iSzbX/88YcAIM6fPy+EEOKzzz4TAMTx48dN+zz11FNCrVaLsrIy07aUlBTx1FNPCSGEOHPmjFAoFCInJ8fstW+77TYxbdo0i7W8//77okOHDqKmpsbi89HR0eKDDz4w2zZ27Fjx5JNPmm3766+/hFwuF1VVVabjBg0aZLbPsGHDxJ133mnxPETknNiSRESSUKvViI2NNX0fHByMmJgYeHp6mm3Lz88HAOzbtw8GgwEdOnSAp6en6bF582acOHHC4jkefPBBVFVVoW3bthg3bhzWrl2Lurq6q9a1Z88eLFu2zOwcKSkpMBqNOHXqlGm/pKQks+OSkpJw6NCh674OROS4OHCbiCTh4uJi9r1MJrO4zWg0AgDKy8uhUCiQkZEBhUJhtt+lwepSkZGROHLkCH777Tds3LgRzzzzDN577z1s3rz5snM1KC8vx1NPPWUau3SpqKioRr8/InJ+DElEZFWurq5mg62tJT4+HgaDAfn5+Rg4cGCjj3N3d8eQIUMwZMgQTJgwAXFxcdi3bx969eplsdZevXrh4MGDaNeu3VVfd/v27Zd936lTp8a/ISJyeAxJRGRVMTExSEtLw+nTp+Hp6Qk/Pz+rvG6HDh0wYsQIjBo1Cu+//z7i4+NRUFCA1NRUdO/eHYMHD77smGXLlsFgMCAxMRFqtRorVqyAu7s7oqOjTbX++eefePjhh6FSqRAQEICXX34Z/fr1w8SJE/HEE0/Aw8MDBw8exMaNG/HRRx+ZXvvvv//Gu+++i6FDh2Ljxo1YvXo1fv75Z6u8VyJyDByTRERW9cILL0ChUKBz584IDAxEVlaW1V77s88+w6hRo/D888+jY8eOGDp0KHbs2HHFbjAfHx8sXrwYAwYMQPfu3fHbb7/hxx9/hL+/PwDg9ddfx+nTpxEbG4vAwEAAQPfu3bF582YcPXoUAwcORHx8PGbMmIGwsDCz137++eexc+dOxMfH480338ScOXOQkpJitfdKRNKTCSGE1EUQETmTmJgYTJ48+bKVxYmoZWFLEhEREZEFDElEREREFrC7jYiIiMgCtiQRERERWcCQRERERGQBQxIRERGRBQxJRERERBYwJBERERFZwJBEREREZAFDEhEREZEFDElEREREFvx/yD0wM/xwyskAAAAASUVORK5CYII=\n" }, "metadata": {} } ], "source": [ "# Create a Neural LinTS pearl agent with 1-hot action representation\n", "\n", "action_representation_module = OneHotActionTensorRepresentationModule(\n", " max_number_actions= env._action_space.n,\n", ")\n", "\n", "agent = PearlAgent(\n", " policy_learner=NeuralLinearBandit(\n", " feature_dim = env.observation_dim + env._action_space.n,\n", " hidden_dims=[64, 16],\n", " state_features_only=False,\n", " training_rounds=10,\n", " learning_rate=0.01,\n", " action_representation_module=action_representation_module,\n", " exploration_module= ThompsonSamplingExplorationLinear()\n", " ),\n", " replay_buffer=FIFOOffPolicyReplayBuffer(100_000),\n", " device_id=-1,\n", ")\n", "\n", "info = online_learning(\n", " agent=agent,\n", " env=env,\n", " number_of_steps=number_of_steps,\n", " print_every_x_steps=100,\n", " record_period=record_period,\n", " learn_after_episode=True,\n", ")\n", "torch.save(info[\"return\"], \"LinTS-return.pt\")\n", "plt.plot(record_period * np.arange(len(info[\"return\"])), info[\"return\"], label=\"LinTS\")\n", "plt.xlabel(\"time step\")\n", "plt.ylabel(\"return\")\n", "plt.legend()\n", "plt.show()" ] }, { "cell_type": "markdown", "metadata": { "id": "pm18iQ_2LNGS" }, "source": [ "## Summary\n", "In this example, we showed how to use popular contextual bandits algorithms in Pearl." ] } ], "metadata": { "accelerator": "GPU", "colab": { "gpuType": "T4", "provenance": [] }, "custom": { "cells": [], "metadata": { "custom": { "cells": [], "metadata": { "accelerator": "GPU", "colab": { "gpuType": "T4", "include_colab_link": true, "provenance": [] }, "fileHeader": "", "fileUid": "4316417e-7688-45f2-a94f-24148bfc425e", "isAdHoc": false, "kernelspec": { "display_name": "pearl (local)", "language": "python", "name": "pearl_local" }, "language_info": { "name": "python" } }, "nbformat": 4, "nbformat_minor": 2 }, "fileHeader": "", "fileUid": "1158a851-91bb-437e-a391-aba92448f600", "indentAmount": 2, "isAdHoc": false, "language_info": { "name": "plaintext" } }, "nbformat": 4, "nbformat_minor": 2 }, "indentAmount": 2, "kernelspec": { "display_name": "Python 3", "name": "python3" }, "language_info": { "name": "python" } }, "nbformat": 4, "nbformat_minor": 0 }