{ "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "## Configurations for Colab" ] }, { "cell_type": "code", "execution_count": 1, "metadata": {}, "outputs": [], "source": [ "import sys\n", "IN_COLAB = \"google.colab\" in sys.modules\n", "\n", "if IN_COLAB:\n", " !apt install python-opengl\n", " !apt install ffmpeg\n", " !apt install xvfb\n", " !pip install PyVirtualDisplay==3.0\n", " !pip install gymnasium==0.28.1\n", " from pyvirtualdisplay import Display\n", " \n", " # Start virtual display\n", " dis = Display(visible=0, size=(400, 400))\n", " dis.start()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# 08. Rainbow\n", "\n", "[M. Hessel et al., \"Rainbow: Combining Improvements in Deep Reinforcement Learning.\" arXiv preprint arXiv:1710.02298, 2017.](https://arxiv.org/pdf/1710.02298.pdf)\n", "\n", "We will integrate all the following seven components into a single integrated agent, which is called Rainbow!\n", "\n", "1. DQN\n", "2. Double DQN\n", "3. Prioritized Experience Replay\n", "4. Dueling Network\n", "5. Noisy Network\n", "6. Categorical DQN\n", "7. N-step Learning\n", "\n", "This method shows an impressive performance on the Atari 2600 benchmark, both in terms of data efficiency and final performance. \n", "\n", "![rainbow](https://user-images.githubusercontent.com/14961526/60591412-61748100-9dd9-11e9-84fb-076c7a61fbab.png)\n", "\n", "However, the integration is not so simple because some of components are not independent each other, so we will look into a number of points that people especailly feel confused.\n", "\n", "1. Noisy Network <-> Dueling Network\n", "2. Dueling Network <-> Categorical DQN\n", "3. Categorical DQN <-> Double DQN" ] }, { "cell_type": "code", "execution_count": 2, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "/Users/jinwoo.park/miniforge3/envs/rainbow-is-all-you-need/lib/python3.8/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", " from .autonotebook import tqdm as notebook_tqdm\n" ] } ], "source": [ "import math\n", "import os\n", "import random\n", "from collections import deque\n", "from typing import Deque, Dict, List, Tuple\n", "\n", "import gymnasium as gym\n", "import matplotlib.pyplot as plt\n", "import numpy as np\n", "import torch\n", "import torch.nn as nn\n", "import torch.nn.functional as F\n", "import torch.optim as optim\n", "from IPython.display import clear_output\n", "from torch.nn.utils import clip_grad_norm_\n", "\n", "# download segment tree module\n", "if IN_COLAB:\n", " !wget https://raw.githubusercontent.com/curt-park/rainbow-is-all-you-need/master/segment_tree.py\n", "\n", "from segment_tree import MinSegmentTree, SumSegmentTree" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Replay buffer\n", "\n", "Same as the basic N-step buffer. \n", "\n", "(Please see *01.dqn.ipynb*, *07.n_step_learning.ipynb* for detailed description about the basic (n-step) replay buffer.)" ] }, { "cell_type": "code", "execution_count": 3, "metadata": {}, "outputs": [], "source": [ "class ReplayBuffer:\n", " \"\"\"A simple numpy replay buffer.\"\"\"\n", "\n", " def __init__(\n", " self, \n", " obs_dim: int, \n", " size: int, \n", " batch_size: int = 32, \n", " n_step: int = 1, \n", " gamma: float = 0.99\n", " ):\n", " self.obs_buf = np.zeros([size, obs_dim], dtype=np.float32)\n", " self.next_obs_buf = np.zeros([size, obs_dim], dtype=np.float32)\n", " self.acts_buf = np.zeros([size], dtype=np.float32)\n", " self.rews_buf = np.zeros([size], dtype=np.float32)\n", " self.done_buf = np.zeros(size, dtype=np.float32)\n", " self.max_size, self.batch_size = size, batch_size\n", " self.ptr, self.size, = 0, 0\n", " \n", " # for N-step Learning\n", " self.n_step_buffer = deque(maxlen=n_step)\n", " self.n_step = n_step\n", " self.gamma = gamma\n", "\n", " def store(\n", " self, \n", " obs: np.ndarray, \n", " act: np.ndarray, \n", " rew: float, \n", " next_obs: np.ndarray, \n", " done: bool,\n", " ) -> Tuple[np.ndarray, np.ndarray, float, np.ndarray, bool]:\n", " transition = (obs, act, rew, next_obs, done)\n", " self.n_step_buffer.append(transition)\n", "\n", " # single step transition is not ready\n", " if len(self.n_step_buffer) < self.n_step:\n", " return ()\n", " \n", " # make a n-step transition\n", " rew, next_obs, done = self._get_n_step_info(\n", " self.n_step_buffer, self.gamma\n", " )\n", " obs, act = self.n_step_buffer[0][:2]\n", " \n", " self.obs_buf[self.ptr] = obs\n", " self.next_obs_buf[self.ptr] = next_obs\n", " self.acts_buf[self.ptr] = act\n", " self.rews_buf[self.ptr] = rew\n", " self.done_buf[self.ptr] = done\n", " self.ptr = (self.ptr + 1) % self.max_size\n", " self.size = min(self.size + 1, self.max_size)\n", " \n", " return self.n_step_buffer[0]\n", "\n", " def sample_batch(self) -> Dict[str, np.ndarray]:\n", " idxs = np.random.choice(self.size, size=self.batch_size, replace=False)\n", "\n", " return dict(\n", " obs=self.obs_buf[idxs],\n", " next_obs=self.next_obs_buf[idxs],\n", " acts=self.acts_buf[idxs],\n", " rews=self.rews_buf[idxs],\n", " done=self.done_buf[idxs],\n", " # for N-step Learning\n", " indices=idxs,\n", " )\n", " \n", " def sample_batch_from_idxs(\n", " self, idxs: np.ndarray\n", " ) -> Dict[str, np.ndarray]:\n", " # for N-step Learning\n", " return dict(\n", " obs=self.obs_buf[idxs],\n", " next_obs=self.next_obs_buf[idxs],\n", " acts=self.acts_buf[idxs],\n", " rews=self.rews_buf[idxs],\n", " done=self.done_buf[idxs],\n", " )\n", " \n", " def _get_n_step_info(\n", " self, n_step_buffer: Deque, gamma: float\n", " ) -> Tuple[np.int64, np.ndarray, bool]:\n", " \"\"\"Return n step rew, next_obs, and done.\"\"\"\n", " # info of the last transition\n", " rew, next_obs, done = n_step_buffer[-1][-3:]\n", "\n", " for transition in reversed(list(n_step_buffer)[:-1]):\n", " r, n_o, d = transition[-3:]\n", "\n", " rew = r + gamma * rew * (1 - d)\n", " next_obs, done = (n_o, d) if d else (next_obs, done)\n", "\n", " return rew, next_obs, done\n", "\n", " def __len__(self) -> int:\n", " return self.size" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Prioritized replay Buffer\n", "\n", "`store` method returns boolean in order to inform if a N-step transition has been generated.\n", "\n", "(Please see *02.per.ipynb* for detailed description about PER.)" ] }, { "cell_type": "code", "execution_count": 4, "metadata": {}, "outputs": [], "source": [ "class PrioritizedReplayBuffer(ReplayBuffer):\n", " \"\"\"Prioritized Replay buffer.\n", " \n", " Attributes:\n", " max_priority (float): max priority\n", " tree_ptr (int): next index of tree\n", " alpha (float): alpha parameter for prioritized replay buffer\n", " sum_tree (SumSegmentTree): sum tree for prior\n", " min_tree (MinSegmentTree): min tree for min prior to get max weight\n", " \n", " \"\"\"\n", " \n", " def __init__(\n", " self, \n", " obs_dim: int, \n", " size: int, \n", " batch_size: int = 32, \n", " alpha: float = 0.6,\n", " n_step: int = 1, \n", " gamma: float = 0.99,\n", " ):\n", " \"\"\"Initialization.\"\"\"\n", " assert alpha >= 0\n", " \n", " super(PrioritizedReplayBuffer, self).__init__(\n", " obs_dim, size, batch_size, n_step, gamma\n", " )\n", " self.max_priority, self.tree_ptr = 1.0, 0\n", " self.alpha = alpha\n", " \n", " # capacity must be positive and a power of 2.\n", " tree_capacity = 1\n", " while tree_capacity < self.max_size:\n", " tree_capacity *= 2\n", "\n", " self.sum_tree = SumSegmentTree(tree_capacity)\n", " self.min_tree = MinSegmentTree(tree_capacity)\n", " \n", " def store(\n", " self, \n", " obs: np.ndarray, \n", " act: int, \n", " rew: float, \n", " next_obs: np.ndarray, \n", " done: bool,\n", " ) -> Tuple[np.ndarray, np.ndarray, float, np.ndarray, bool]:\n", " \"\"\"Store experience and priority.\"\"\"\n", " transition = super().store(obs, act, rew, next_obs, done)\n", " \n", " if transition:\n", " self.sum_tree[self.tree_ptr] = self.max_priority ** self.alpha\n", " self.min_tree[self.tree_ptr] = self.max_priority ** self.alpha\n", " self.tree_ptr = (self.tree_ptr + 1) % self.max_size\n", " \n", " return transition\n", "\n", " def sample_batch(self, beta: float = 0.4) -> Dict[str, np.ndarray]:\n", " \"\"\"Sample a batch of experiences.\"\"\"\n", " assert len(self) >= self.batch_size\n", " assert beta > 0\n", " \n", " indices = self._sample_proportional()\n", " \n", " obs = self.obs_buf[indices]\n", " next_obs = self.next_obs_buf[indices]\n", " acts = self.acts_buf[indices]\n", " rews = self.rews_buf[indices]\n", " done = self.done_buf[indices]\n", " weights = np.array([self._calculate_weight(i, beta) for i in indices])\n", " \n", " return dict(\n", " obs=obs,\n", " next_obs=next_obs,\n", " acts=acts,\n", " rews=rews,\n", " done=done,\n", " weights=weights,\n", " indices=indices,\n", " )\n", " \n", " def update_priorities(self, indices: List[int], priorities: np.ndarray):\n", " \"\"\"Update priorities of sampled transitions.\"\"\"\n", " assert len(indices) == len(priorities)\n", "\n", " for idx, priority in zip(indices, priorities):\n", " assert priority > 0\n", " assert 0 <= idx < len(self)\n", "\n", " self.sum_tree[idx] = priority ** self.alpha\n", " self.min_tree[idx] = priority ** self.alpha\n", "\n", " self.max_priority = max(self.max_priority, priority)\n", " \n", " def _sample_proportional(self) -> List[int]:\n", " \"\"\"Sample indices based on proportions.\"\"\"\n", " indices = []\n", " p_total = self.sum_tree.sum(0, len(self) - 1)\n", " segment = p_total / self.batch_size\n", " \n", " for i in range(self.batch_size):\n", " a = segment * i\n", " b = segment * (i + 1)\n", " upperbound = random.uniform(a, b)\n", " idx = self.sum_tree.retrieve(upperbound)\n", " indices.append(idx)\n", " \n", " return indices\n", " \n", " def _calculate_weight(self, idx: int, beta: float):\n", " \"\"\"Calculate the weight of the experience at idx.\"\"\"\n", " # get max weight\n", " p_min = self.min_tree.min() / self.sum_tree.sum()\n", " max_weight = (p_min * len(self)) ** (-beta)\n", " \n", " # calculate weights\n", " p_sample = self.sum_tree[idx] / self.sum_tree.sum()\n", " weight = (p_sample * len(self)) ** (-beta)\n", " weight = weight / max_weight\n", " \n", " return weight" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Noisy Layer\n", "\n", "Please see *05.noisy_net.ipynb* for detailed description.\n", "\n", "**References:**\n", "\n", "- https://github.com/higgsfield/RL-Adventure/blob/master/5.noisy%20dqn.ipynb\n", "- https://github.com/Kaixhin/Rainbow/blob/master/model.py" ] }, { "cell_type": "code", "execution_count": 5, "metadata": {}, "outputs": [], "source": [ "class NoisyLinear(nn.Module):\n", " \"\"\"Noisy linear module for NoisyNet.\n", " \n", " \n", " \n", " Attributes:\n", " in_features (int): input size of linear module\n", " out_features (int): output size of linear module\n", " std_init (float): initial std value\n", " weight_mu (nn.Parameter): mean value weight parameter\n", " weight_sigma (nn.Parameter): std value weight parameter\n", " bias_mu (nn.Parameter): mean value bias parameter\n", " bias_sigma (nn.Parameter): std value bias parameter\n", " \n", " \"\"\"\n", "\n", " def __init__(\n", " self, \n", " in_features: int, \n", " out_features: int, \n", " std_init: float = 0.5,\n", " ):\n", " \"\"\"Initialization.\"\"\"\n", " super(NoisyLinear, self).__init__()\n", " \n", " self.in_features = in_features\n", " self.out_features = out_features\n", " self.std_init = std_init\n", "\n", " self.weight_mu = nn.Parameter(torch.Tensor(out_features, in_features))\n", " self.weight_sigma = nn.Parameter(\n", " torch.Tensor(out_features, in_features)\n", " )\n", " self.register_buffer(\n", " \"weight_epsilon\", torch.Tensor(out_features, in_features)\n", " )\n", "\n", " self.bias_mu = nn.Parameter(torch.Tensor(out_features))\n", " self.bias_sigma = nn.Parameter(torch.Tensor(out_features))\n", " self.register_buffer(\"bias_epsilon\", torch.Tensor(out_features))\n", "\n", " self.reset_parameters()\n", " self.reset_noise()\n", "\n", " def reset_parameters(self):\n", " \"\"\"Reset trainable network parameters (factorized gaussian noise).\"\"\"\n", " mu_range = 1 / math.sqrt(self.in_features)\n", " self.weight_mu.data.uniform_(-mu_range, mu_range)\n", " self.weight_sigma.data.fill_(\n", " self.std_init / math.sqrt(self.in_features)\n", " )\n", " self.bias_mu.data.uniform_(-mu_range, mu_range)\n", " self.bias_sigma.data.fill_(\n", " self.std_init / math.sqrt(self.out_features)\n", " )\n", "\n", " def reset_noise(self):\n", " \"\"\"Make new noise.\"\"\"\n", " epsilon_in = self.scale_noise(self.in_features)\n", " epsilon_out = self.scale_noise(self.out_features)\n", "\n", " # outer product\n", " self.weight_epsilon.copy_(epsilon_out.ger(epsilon_in))\n", " self.bias_epsilon.copy_(epsilon_out)\n", "\n", " def forward(self, x: torch.Tensor) -> torch.Tensor:\n", " \"\"\"Forward method implementation.\n", " \n", " We don't use separate statements on train / eval mode.\n", " It doesn't show remarkable difference of performance.\n", " \"\"\"\n", " return F.linear(\n", " x,\n", " self.weight_mu + self.weight_sigma * self.weight_epsilon,\n", " self.bias_mu + self.bias_sigma * self.bias_epsilon,\n", " )\n", " \n", " @staticmethod\n", " def scale_noise(size: int) -> torch.Tensor:\n", " \"\"\"Set scale to make noise (factorized gaussian noise).\"\"\"\n", " x = torch.randn(size)\n", "\n", " return x.sign().mul(x.abs().sqrt())" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## NoisyNet + DuelingNet + Categorical DQN\n", "\n", "#### NoisyNet + DuelingNet\n", "\n", "NoisyLinear is employed for the last two layers of advantage and value layers. The noise should be reset at evey update step.\n", "\n", "#### DuelingNet + Categorical DQN\n", "\n", "The dueling network architecture is adapted for use with return distributions. The network has a shared representation, which is then fed into a value stream with atom_size outputs, and into an advantage stream with atom_size × out_dim outputs. For each atom, the value and advantage streams are aggregated, as in dueling DQN, and then passed through a softmax layer to obtain the normalized parametric distributions used to estimate the returns’ distributions.\n", "\n", "```\n", " advantage = self.advantage_layer(adv_hid).view(-1, self.out_dim, self.atom_size)\n", " value = self.value_layer(val_hid).view(-1, 1, self.atom_size)\n", " q_atoms = value + advantage - advantage.mean(dim=1, keepdim=True)\n", " \n", " dist = F.softmax(q_atoms, dim=-1)\n", "```\n", "\n", "(Please see *04.dueling.ipynb*, *05.noisy_net.ipynb*, *06.categorical_dqn.ipynb* for detailed description of each component's network architecture.)" ] }, { "cell_type": "code", "execution_count": 6, "metadata": {}, "outputs": [], "source": [ "class Network(nn.Module):\n", " def __init__(\n", " self, \n", " in_dim: int, \n", " out_dim: int, \n", " atom_size: int, \n", " support: torch.Tensor\n", " ):\n", " \"\"\"Initialization.\"\"\"\n", " super(Network, self).__init__()\n", " \n", " self.support = support\n", " self.out_dim = out_dim\n", " self.atom_size = atom_size\n", "\n", " # set common feature layer\n", " self.feature_layer = nn.Sequential(\n", " nn.Linear(in_dim, 128), \n", " nn.ReLU(),\n", " )\n", " \n", " # set advantage layer\n", " self.advantage_hidden_layer = NoisyLinear(128, 128)\n", " self.advantage_layer = NoisyLinear(128, out_dim * atom_size)\n", "\n", " # set value layer\n", " self.value_hidden_layer = NoisyLinear(128, 128)\n", " self.value_layer = NoisyLinear(128, atom_size)\n", "\n", " def forward(self, x: torch.Tensor) -> torch.Tensor:\n", " \"\"\"Forward method implementation.\"\"\"\n", " dist = self.dist(x)\n", " q = torch.sum(dist * self.support, dim=2)\n", " \n", " return q\n", " \n", " def dist(self, x: torch.Tensor) -> torch.Tensor:\n", " \"\"\"Get distribution for atoms.\"\"\"\n", " feature = self.feature_layer(x)\n", " adv_hid = F.relu(self.advantage_hidden_layer(feature))\n", " val_hid = F.relu(self.value_hidden_layer(feature))\n", " \n", " advantage = self.advantage_layer(adv_hid).view(\n", " -1, self.out_dim, self.atom_size\n", " )\n", " value = self.value_layer(val_hid).view(-1, 1, self.atom_size)\n", " q_atoms = value + advantage - advantage.mean(dim=1, keepdim=True)\n", " \n", " dist = F.softmax(q_atoms, dim=-1)\n", " dist = dist.clamp(min=1e-3) # for avoiding nans\n", " \n", " return dist\n", " \n", " def reset_noise(self):\n", " \"\"\"Reset all noisy layers.\"\"\"\n", " self.advantage_hidden_layer.reset_noise()\n", " self.advantage_layer.reset_noise()\n", " self.value_hidden_layer.reset_noise()\n", " self.value_layer.reset_noise()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Rainbow Agent\n", "\n", "Here is a summary of DQNAgent class.\n", "\n", "| Method | Note |\n", "| --- | --- |\n", "|select_action | select an action from the input state. |\n", "|step | take an action and return the response of the env. |\n", "|compute_dqn_loss | return dqn loss. |\n", "|update_model | update the model by gradient descent. |\n", "|target_hard_update| hard update from the local model to the target model.|\n", "|train | train the agent during num_frames. |\n", "|test | test the agent (1 episode). |\n", "|plot | plot the training progresses. |\n", "\n", "#### Categorical DQN + Double DQN\n", "\n", "The idea of Double Q-learning is to reduce overestimations by decomposing the max operation in the target into action selection and action evaluation. Here, we use `self.dqn` instead of `self.dqn_target` to obtain the target actions.\n", "\n", "```\n", " # Categorical DQN + Double DQN\n", " # target_dqn is used when we don't employ double DQN\n", " next_action = self.dqn(next_state).argmax(1)\n", " next_dist = self.dqn_target.dist(next_state)\n", " next_dist = next_dist[range(self.batch_size), next_action]\n", "```" ] }, { "cell_type": "code", "execution_count": 7, "metadata": {}, "outputs": [], "source": [ "class DQNAgent:\n", " \"\"\"DQN Agent interacting with environment.\n", " \n", " Attribute:\n", " env (gym.Env): openAI Gym environment\n", " memory (PrioritizedReplayBuffer): replay memory to store transitions\n", " batch_size (int): batch size for sampling\n", " target_update (int): period for target model's hard update\n", " gamma (float): discount factor\n", " dqn (Network): model to train and select actions\n", " dqn_target (Network): target model to update\n", " optimizer (torch.optim): optimizer for training dqn\n", " transition (list): transition information including \n", " state, action, reward, next_state, done\n", " v_min (float): min value of support\n", " v_max (float): max value of support\n", " atom_size (int): the unit number of support\n", " support (torch.Tensor): support for categorical dqn\n", " use_n_step (bool): whether to use n_step memory\n", " n_step (int): step number to calculate n-step td error\n", " memory_n (ReplayBuffer): n-step replay buffer\n", " \"\"\"\n", "\n", " def __init__(\n", " self, \n", " env: gym.Env,\n", " memory_size: int,\n", " batch_size: int,\n", " target_update: int,\n", " seed: int,\n", " gamma: float = 0.99,\n", " # PER parameters\n", " alpha: float = 0.2,\n", " beta: float = 0.6,\n", " prior_eps: float = 1e-6,\n", " # Categorical DQN parameters\n", " v_min: float = 0.0,\n", " v_max: float = 200.0,\n", " atom_size: int = 51,\n", " # N-step Learning\n", " n_step: int = 3,\n", " ):\n", " \"\"\"Initialization.\n", " \n", " Args:\n", " env (gym.Env): openAI Gym environment\n", " memory_size (int): length of memory\n", " batch_size (int): batch size for sampling\n", " target_update (int): period for target model's hard update\n", " lr (float): learning rate\n", " gamma (float): discount factor\n", " alpha (float): determines how much prioritization is used\n", " beta (float): determines how much importance sampling is used\n", " prior_eps (float): guarantees every transition can be sampled\n", " v_min (float): min value of support\n", " v_max (float): max value of support\n", " atom_size (int): the unit number of support\n", " n_step (int): step number to calculate n-step td error\n", " \"\"\"\n", " obs_dim = env.observation_space.shape[0]\n", " action_dim = env.action_space.n\n", " \n", " self.env = env\n", " self.batch_size = batch_size\n", " self.target_update = target_update\n", " self.seed = seed\n", " self.gamma = gamma\n", " # NoisyNet: All attributes related to epsilon are removed\n", " \n", " # device: cpu / gpu\n", " self.device = torch.device(\n", " \"cuda\" if torch.cuda.is_available() else \"cpu\"\n", " )\n", " print(self.device)\n", " \n", " # PER\n", " # memory for 1-step Learning\n", " self.beta = beta\n", " self.prior_eps = prior_eps\n", " self.memory = PrioritizedReplayBuffer(\n", " obs_dim, memory_size, batch_size, alpha=alpha, gamma=gamma\n", " )\n", " \n", " # memory for N-step Learning\n", " self.use_n_step = True if n_step > 1 else False\n", " if self.use_n_step:\n", " self.n_step = n_step\n", " self.memory_n = ReplayBuffer(\n", " obs_dim, memory_size, batch_size, n_step=n_step, gamma=gamma\n", " )\n", " \n", " # Categorical DQN parameters\n", " self.v_min = v_min\n", " self.v_max = v_max\n", " self.atom_size = atom_size\n", " self.support = torch.linspace(\n", " self.v_min, self.v_max, self.atom_size\n", " ).to(self.device)\n", "\n", " # networks: dqn, dqn_target\n", " self.dqn = Network(\n", " obs_dim, action_dim, self.atom_size, self.support\n", " ).to(self.device)\n", " self.dqn_target = Network(\n", " obs_dim, action_dim, self.atom_size, self.support\n", " ).to(self.device)\n", " self.dqn_target.load_state_dict(self.dqn.state_dict())\n", " self.dqn_target.eval()\n", " \n", " # optimizer\n", " self.optimizer = optim.Adam(self.dqn.parameters())\n", "\n", " # transition to store in memory\n", " self.transition = list()\n", " \n", " # mode: train / test\n", " self.is_test = False\n", "\n", " def select_action(self, state: np.ndarray) -> np.ndarray:\n", " \"\"\"Select an action from the input state.\"\"\"\n", " # NoisyNet: no epsilon greedy action selection\n", " selected_action = self.dqn(\n", " torch.FloatTensor(state).to(self.device)\n", " ).argmax()\n", " selected_action = selected_action.detach().cpu().numpy()\n", " \n", " if not self.is_test:\n", " self.transition = [state, selected_action]\n", " \n", " return selected_action\n", "\n", " def step(self, action: np.ndarray) -> Tuple[np.ndarray, np.float64, bool]:\n", " \"\"\"Take an action and return the response of the env.\"\"\"\n", " next_state, reward, terminated, truncated, _ = self.env.step(action)\n", " done = terminated or truncated\n", " \n", " if not self.is_test:\n", " self.transition += [reward, next_state, done]\n", " \n", " # N-step transition\n", " if self.use_n_step:\n", " one_step_transition = self.memory_n.store(*self.transition)\n", " # 1-step transition\n", " else:\n", " one_step_transition = self.transition\n", "\n", " # add a single step transition\n", " if one_step_transition:\n", " self.memory.store(*one_step_transition)\n", " \n", " return next_state, reward, done\n", "\n", " def update_model(self) -> torch.Tensor:\n", " \"\"\"Update the model by gradient descent.\"\"\"\n", " # PER needs beta to calculate weights\n", " samples = self.memory.sample_batch(self.beta)\n", " weights = torch.FloatTensor(\n", " samples[\"weights\"].reshape(-1, 1)\n", " ).to(self.device)\n", " indices = samples[\"indices\"]\n", " \n", " # 1-step Learning loss\n", " elementwise_loss = self._compute_dqn_loss(samples, self.gamma)\n", " \n", " # PER: importance sampling before average\n", " loss = torch.mean(elementwise_loss * weights)\n", " \n", " # N-step Learning loss\n", " # we are gonna combine 1-step loss and n-step loss so as to\n", " # prevent high-variance. The original rainbow employs n-step loss only.\n", " if self.use_n_step:\n", " gamma = self.gamma ** self.n_step\n", " samples = self.memory_n.sample_batch_from_idxs(indices)\n", " elementwise_loss_n_loss = self._compute_dqn_loss(samples, gamma)\n", " elementwise_loss += elementwise_loss_n_loss\n", " \n", " # PER: importance sampling before average\n", " loss = torch.mean(elementwise_loss * weights)\n", "\n", " self.optimizer.zero_grad()\n", " loss.backward()\n", " clip_grad_norm_(self.dqn.parameters(), 10.0)\n", " self.optimizer.step()\n", " \n", " # PER: update priorities\n", " loss_for_prior = elementwise_loss.detach().cpu().numpy()\n", " new_priorities = loss_for_prior + self.prior_eps\n", " self.memory.update_priorities(indices, new_priorities)\n", " \n", " # NoisyNet: reset noise\n", " self.dqn.reset_noise()\n", " self.dqn_target.reset_noise()\n", "\n", " return loss.item()\n", " \n", " def train(self, num_frames: int, plotting_interval: int = 200):\n", " \"\"\"Train the agent.\"\"\"\n", " self.is_test = False\n", " \n", " state, _ = self.env.reset(seed=self.seed)\n", " update_cnt = 0\n", " losses = []\n", " scores = []\n", " score = 0\n", "\n", " for frame_idx in range(1, num_frames + 1):\n", " action = self.select_action(state)\n", " next_state, reward, done = self.step(action)\n", "\n", " state = next_state\n", " score += reward\n", " \n", " # NoisyNet: removed decrease of epsilon\n", " \n", " # PER: increase beta\n", " fraction = min(frame_idx / num_frames, 1.0)\n", " self.beta = self.beta + fraction * (1.0 - self.beta)\n", "\n", " # if episode ends\n", " if done:\n", " state, _ = self.env.reset(seed=self.seed)\n", " scores.append(score)\n", " score = 0\n", "\n", " # if training is ready\n", " if len(self.memory) >= self.batch_size:\n", " loss = self.update_model()\n", " losses.append(loss)\n", " update_cnt += 1\n", " \n", " # if hard update is needed\n", " if update_cnt % self.target_update == 0:\n", " self._target_hard_update()\n", "\n", " # plotting\n", " if frame_idx % plotting_interval == 0:\n", " self._plot(frame_idx, scores, losses)\n", " \n", " self.env.close()\n", " \n", " def test(self, video_folder: str) -> None:\n", " \"\"\"Test the agent.\"\"\"\n", " self.is_test = True\n", " \n", " # for recording a video\n", " naive_env = self.env\n", " self.env = gym.wrappers.RecordVideo(self.env, video_folder=video_folder)\n", " \n", " state, _ = self.env.reset(seed=self.seed)\n", " done = False\n", " score = 0\n", " \n", " while not done:\n", " action = self.select_action(state)\n", " next_state, reward, done = self.step(action)\n", "\n", " state = next_state\n", " score += reward\n", " \n", " print(\"score: \", score)\n", " self.env.close()\n", " \n", " # reset\n", " self.env = naive_env\n", "\n", " def _compute_dqn_loss(self, samples: Dict[str, np.ndarray], gamma: float) -> torch.Tensor:\n", " \"\"\"Return categorical dqn loss.\"\"\"\n", " device = self.device # for shortening the following lines\n", " state = torch.FloatTensor(samples[\"obs\"]).to(device)\n", " next_state = torch.FloatTensor(samples[\"next_obs\"]).to(device)\n", " action = torch.LongTensor(samples[\"acts\"]).to(device)\n", " reward = torch.FloatTensor(samples[\"rews\"].reshape(-1, 1)).to(device)\n", " done = torch.FloatTensor(samples[\"done\"].reshape(-1, 1)).to(device)\n", " \n", " # Categorical DQN algorithm\n", " delta_z = float(self.v_max - self.v_min) / (self.atom_size - 1)\n", "\n", " with torch.no_grad():\n", " # Double DQN\n", " next_action = self.dqn(next_state).argmax(1)\n", " next_dist = self.dqn_target.dist(next_state)\n", " next_dist = next_dist[range(self.batch_size), next_action]\n", "\n", " t_z = reward + (1 - done) * gamma * self.support\n", " t_z = t_z.clamp(min=self.v_min, max=self.v_max)\n", " b = (t_z - self.v_min) / delta_z\n", " l = b.floor().long()\n", " u = b.ceil().long()\n", "\n", " offset = (\n", " torch.linspace(\n", " 0, (self.batch_size - 1) * self.atom_size, self.batch_size\n", " ).long()\n", " .unsqueeze(1)\n", " .expand(self.batch_size, self.atom_size)\n", " .to(self.device)\n", " )\n", "\n", " proj_dist = torch.zeros(next_dist.size(), device=self.device)\n", " proj_dist.view(-1).index_add_(\n", " 0, (l + offset).view(-1), (next_dist * (u.float() - b)).view(-1)\n", " )\n", " proj_dist.view(-1).index_add_(\n", " 0, (u + offset).view(-1), (next_dist * (b - l.float())).view(-1)\n", " )\n", "\n", " dist = self.dqn.dist(state)\n", " log_p = torch.log(dist[range(self.batch_size), action])\n", " elementwise_loss = -(proj_dist * log_p).sum(1)\n", "\n", " return elementwise_loss\n", "\n", " def _target_hard_update(self):\n", " \"\"\"Hard update: target <- local.\"\"\"\n", " self.dqn_target.load_state_dict(self.dqn.state_dict())\n", " \n", " def _plot(\n", " self, \n", " frame_idx: int, \n", " scores: List[float], \n", " losses: List[float],\n", " ):\n", " \"\"\"Plot the training progresses.\"\"\"\n", " clear_output(True)\n", " plt.figure(figsize=(20, 5))\n", " plt.subplot(131)\n", " plt.title('frame %s. score: %s' % (frame_idx, np.mean(scores[-10:])))\n", " plt.plot(scores)\n", " plt.subplot(132)\n", " plt.title('loss')\n", " plt.plot(losses)\n", " plt.show()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Environment\n", "\n", "You can see the [code](https://github.com/Farama-Foundation/Gymnasium/blob/main/gymnasium/envs/classic_control/cartpole.py) and [configurations](https://github.com/Farama-Foundation/Gymnasium/blob/main/gymnasium/envs/classic_control/cartpole.py#L91) of CartPole-v1 from Farama Gymnasium's repository." ] }, { "cell_type": "code", "execution_count": 8, "metadata": {}, "outputs": [], "source": [ "# environment\n", "env = gym.make(\"CartPole-v1\", max_episode_steps=200, render_mode=\"rgb_array\")" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Set random seed" ] }, { "cell_type": "code", "execution_count": 9, "metadata": {}, "outputs": [], "source": [ "seed = 777\n", "\n", "def seed_torch(seed):\n", " torch.manual_seed(seed)\n", " if torch.backends.cudnn.enabled:\n", " torch.cuda.manual_seed(seed)\n", " torch.backends.cudnn.benchmark = False\n", " torch.backends.cudnn.deterministic = True\n", "\n", "np.random.seed(seed)\n", "random.seed(seed)\n", "seed_torch(seed)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Initialize" ] }, { "cell_type": "code", "execution_count": 10, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "cpu\n" ] } ], "source": [ "# parameters\n", "num_frames = 10000\n", "memory_size = 10000\n", "batch_size = 128\n", "target_update = 100\n", "\n", "# train\n", "agent = DQNAgent(env, memory_size, batch_size, target_update, seed)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Train" ] }, { "cell_type": "code", "execution_count": 11, "metadata": {}, "outputs": [ { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAAv8AAAE/CAYAAADR+6wLAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAABZv0lEQVR4nO3deXxU1f3/8dcn+woIJOwQkIiCCiqK+75rtYv+qm1duqmtbbXrF6tWq7W1+2artWqtS7Uudd+1dRcRFRBZZAsCYQkgkD2ZmfP7494ZJskkJCSzZOb9fDzyyMy9d+49dwhzP3Pu53yOOecQEREREZH0l5XsBoiIiIiISGIo+BcRERERyRAK/kVEREREMoSCfxERERGRDKHgX0REREQkQyj4FxERERHJEAr+05iZTTKz982s1sy+k+z2iIiIpDIzqzKz45PdDpF4UvCf3n4EvOycK3XO/SnZjWnPzG41syVmFjKzC2Os/66ZrTezbWZ2h5nlR60bbGaPmFm9ma0ysy+0e+1xZrbYzBrM7H9mNi5qnZnZL81ss//zKzOzuJ5sgpjZwWb2gpltMbMaM3vQzEZEre/y3M2swn+/Gvz3r9OLYDq/jyIiIulKwX96Gwd82NlKM8tOYFtimQd8E3iv/QozOwmYCRwHVAATgJ9GbfIXoAUYBnwRuNnMpvivHQr8B7gaGAzMAf4d9dqLgE8DU4F9gdOBi/vsrHrBzHJ6uYvdgFvx3rNxQC3wj6j1Ozv3+4D3gSHAlcBDZlbWybFS9n0UERGR2BT8pykz+y9wDHCTmdWZ2R5mdqeZ3WxmT5tZPXCMmZ3mpwZtN7PVZnZt1D4qzMyZ2Zf9dZ+Y2SVmdqCZzTezrWZ2U7vjfsXMFvnbPhfd496ec+4vzrmXgKYYqy8AbnfOfeic+wS4HrjQP0Yx8DngaudcnXPudeBx4Dz/tZ8FPnTOPeicawKuBaaa2Z5R+/6tc26Nc24t8Nvwvrvxvk40s1f8uxGbzOzfUeumRPW6bzCzH/vL883sD2ZW7f/8IXwXw8yONrM1ZvZ/ZrYe+IeZZZnZTDNb7veoP2Bmg7vTPufcM/55b3fONQA3AYe1e19jnruZ7QHsD1zjnGt0zj0MfOC/17Hs8vsoIpLKdvK5PdTMnvSvgVvM7DUzy/LX/Z+ZrTUv3XaJmR2X3DMR6UjBf5pyzh0LvAZ8yzlX4pz7yF/1BeAGoBR4HagHzgcGAacB3zCzT7fb3QygEvg88Ae8HuHjgSnA/zOzowD81/0YL/gu849/3y6ewhS8OwNh84BhZjYE2AMIRp1TeP2UWK91ztUDyztb3+61O3M98DxeD/to4M8AZlYKvAg8C4wEJgIv+a+5EjgYmIbXS34QcFXUPofj3aEYh9eb/h28HvWj/H19gnenA/9Y861dmlMXjqTt3Z+uzn0KsMI5V9vJ+vZ68z6KiKSyrj63vw+swbvODcO77jkzmwR8CzjQOVcKnARUJbTVIt2g4D/zPOace8M5F3LONTnnXnbOfeA/n48XrB/V7jXX+9s+j/dl4T7n3Ea/t/c1YD9/u4uBXzjnFjnnAsDPgWld9f53oQTYFvU8/Lg0xrrw+tJOXruz9duAkm7mq7fiBekj/ffkdX/56cB659xv/eW1zrm3/XVfBK7z37MavPSl86L2GcLrbW92zjXivY9X+j3qzXh3Ls4KpwQ55/Z1zv1rZw01s32BnwA/jFrc1bnv7H1rrzfvo4hIKuvqc7sVGAGMc861Oudec845IAjkA5PNLNc5V+WcW56U1ot0QcF/5lkd/cTMZvgDPGvMbBtwCTC03Ws2RD1ujPG8xH88Dvijfyt0K7AFMGDULrSzDhgQ9Tz8uDbGuvD6cI91T9cPAOr8D++d+RHeOc02sw/N7Cv+8jF4dxdiGQmsinq+yl8WVuOnJ4WNAx6Jeh8X4V1UhnWjfYCXngQ8A1zmnHstalVX576z96293ryPIiKprKvP7V8Dy4DnzWyFmc0EcM4tAy7H67DZaGb3m1n0Z71ISlDwn3naB2b/wsuXH+OcGwjcghfc7orVwMXOuUFRP4XOuTd3YV8f4t1qDZsKbHDObQY+AnLMrLLd+g9jvdYfI7B7Z+vbvbZLzrn1zrmvO+dG4vXQ/9UPtFf7x4ilGi+gDxvrL4vstt32q4FT2r2PBf6dlp3y77S8iHfH5u52q7s69w+BCX4KU6z17e3y+ygikuI6/dz27+x+3zk3AfgU8L1wbr9z7l/OucP91zrgl4lttsjOKfiXUmCLc67JzA7CGxOwq24BroiqujPQzM7ubGMzyzOzArwvG7lmVhAeNAXcBXzVzCab2W54uZZ3QiSH/z/AdWZWbGaHAWcC4UD3EWBvM/ucv/+fAPOdc4uj9v09Mxvl98p8P7zvnTGzs81stP/0E7wP9yDwJDDczC73B4qVmtkMf7v7gKvMrMy8SkQ/Ae7p4jC3ADeE06X8153ZzfaNAv4L/MU5d0uMTTo9d38MxVzgGv/f4jN4VXwe7uRwu/w+ioikuE4/t83sdL/4gwHb8a4BQfPm1jnWHxjchHdnPJik9ot0SsG/fBMviK7F+3B7YFd35Jx7BK+X434z2w4sAE7p4iXP4304HopXnrIRb4AqzrlngV8B/8O73boKuKZduwuBjXgf0t9wzn3ov7YGr0LNDXgB+gzgnKjX/g14Aq+SzQLgKX8ZAH46zxc7afOBwNtmVod3x+Qy59xKf5DsCXi9QOuBpXjVlgB+hldudL5/zPf8ZZ35o7/v5/1/l1n+OXSnfV/DK4t6jXlVnur8tnbr3PHep+l479uNwFn++4mZHdHDfYmI9FddfW5X4t1drQPeAv7qnHsZL9//RmAT3nWgHG8wsEhKMaXnioiIiIhkBvX8i4iIiIhkCAX/IiIiIiIZQsG/iIiIiEiGUPAvIiIiIpIhFPyLiIiIiGSInGQ3AGDo0KGuoqIi2c0QEUlJ77777ibnXFmy25FMuk6IiMTW02tESgT/FRUVzJkzJ9nNEBFJSWa2KtltSDZdJ0REYuvpNUJpPyIiIiIiGULBv4iIiIhIhlDwLyIiIiKSIRT8i4iIiIhkCAX/IiIiIiIZQsG/iIiIiEiGUPAvIiIiIpIhdhr8m9kYM/ufmS0ysw/N7DJ/+WAze8HMlvq/d4t6zRVmtszMlpjZSfE8ARERERER6Z7u9PwHgO875/YCDgYuNbPJwEzgJedcJfCS/xx/3TnAFOBk4K9mlh2PxouIiIiISPftdIZf59w6YJ3/uNbMFgGjgDOBo/3N/gm8DPyfv/x+51wzsNLMlgEHAW/1deNFZOcaW4I8/cE6WoKhPt+3AcfsWc6wAQWdbtMaDPH0B+toaAn2+fH7kwMrBjOxvCTZzcg4by3fzMbaJs6cNirZTRERSQk7Df6jmVkFsB/wNjDM/2KAc26dmZX7m40CZkW9bI2/rP2+LgIuAhg7dmyPGy4i3fPch+v5/oPz4rb/8w8Zx3Vn7t3p+rdXbOGy++fG7fj9xY2f3UfBfxI89O4aZq3YrOBfRMTX7eDfzEqAh4HLnXPbzazTTWMscx0WOHcrcCvA9OnTO6wXkb6xrbEVgGcuO4LdivL6dN+fu/lNapsCXW5T2+Qd/+6vHkRleWmfHr8/GVDYo74WERGRuOjW1cjMcvEC/3udc//xF28wsxF+r/8IYKO/fA0wJurlo4HqvmqwiPRMON1m3JAiivL6NgAtysumqbXrdJ6mgLd+1KBChg/sPD1IRERE4q871X4MuB1Y5Jz7XdSqx4EL/McXAI9FLT/HzPLNbDxQCczuuyaLSE80tng98wU5fT/uviB358F/c2sosq2IiIgkV3e6AQ8DzgM+MLO5/rIfAzcCD5jZV4GPgbMBnHMfmtkDwEK8SkGXOucye6SfSBI1tgYpzM0mK6vTVL1dVpCbRVNr1wOJw18OFPyLiIgkX3eq/bxO7Dx+gOM6ec0NwA29aJeI9JGGliBFefEJvAtys6lr7jrnvyngfTnIz9GcgiIiIsmmq7FImmtsCVIYp+A/Pyd7pz3/4bQfBf8iIiLJp6uxSJqLZ89/YTcH/OZkGTnZ+riRxOu8MJ2ISGbS1VgkzTW0Bins4yo/YQU5WTsP/luDyvcXERFJEQr+RdJcY0uAwtz4/FfvVrWfQIiCOB1fREREekZXZJE056X9xKnnv5vVfvLjUGZUREREek7Bv0iai+eA34LcbJoCQZzrfJLu5kCIfPX8i4iIpARdkUXSXGNrkKI45dwX5GbjHLQEO+/9b24NxmWCMZHu6urLqYhIplHwL5Lm4l3nH+gy9aepVT3/mcTMJpnZ3Kif7WZ2ebttjjazbVHb/CRu7YnXjkVE+qn4JAKLSMrw0n7il/MPXl7/wMLcmNs0B9Tzn0mcc0uAaQBmlg2sBR6JselrzrnTE9g0ERFBPf8iaS0QDNESDMWv5z8n3PPfecWfplZV+8lgxwHLnXOrkt0QERHx6IosksYa/KA8uWk/qvaTwc4B7utk3SFmNs/MnjGzKYlslIhIJlPwL5LGGlu84D9ek2xFp/10RnX+M5OZ5QFnAA/GWP0eMM45NxX4M/BoJ/u4yMzmmNmcmpqauLVVRCST6IosksYaWhLV899V2o9m+M1QpwDvOec2tF/hnNvunKvzHz8N5JrZ0Bjb3eqcm+6cm15WVhb/FouIZAAF/yJprKElACQg+A/sLO1HHzUZ6Fw6Sfkxs+FmZv7jg/CuRZvj1RAV+hQR2UHVfkTSWDjtJxHVfjrjpf2o5z+TmFkRcAJwcdSySwCcc7cAZwHfMLMA0Aic4+JUjN9U61NEpA0F/yJprDFhA35jB//OOX+GXwX/mcQ51wAMabfslqjHNwE3JbpdIiKitB+RtBbO+S+M4wy/0Hnw3+ynAyntR0REJDXoiiySxhrjPeA3J5z2Ezvnv9lfrrQfERGR1KDgXySN7aj2E6+c/657/psC4VKj+qgRERFJBboii6SxcLWfwiRN8hX+UqBJviSZ4jOUWESkf1LwL5LGGuOc85+dZeRlZ0V6+NsL5/yr51+SxVC5HxGRaLoii6SxhtYgOVlGXhwH3ObnZnWe9uMvL1DPv4iISEpQ8C+SxhpbgnFL+QkryM3uIvj3q/2o519ERCQl7PSKbGZ3mNlGM1sQtezfZjbX/6kys7n+8goza4xad0unOxaRuGtsCcat0k9YQW5W59V+IgN+1fMvIiKSCrpTAuROvMlY7govcM59PvzYzH4LbIvafrlzbloftU9EeqGhNRi3Sj9hBTnd6PlXnX8REZGUsNOowDn3qplVxFpnZgb8P+DYPm6XiPSBxpZA3Ab7hnWd9qOefxERkVTS2+64I4ANzrmlUcvGm9n7ZvaKmR3Ry/2LSC80JCDtpzA3u4u0H7/ajwb8ShI5VOtTRCSst/kA5wL3RT1fB4x1zm02swOAR81sinNue/sXmtlFwEUAY8eO7WUzRCSWhpYgpQXxTfvJz82irjkQc12kzr8G/EqSmCp9ioi0sctXZDPLAT4L/Du8zDnX7Jzb7D9+F1gO7BHr9c65W51z051z08vKyna1GSLShcaWYELSfsLzCbSnUp8iIiKppTfdcccDi51za8ILzKzMzLL9xxOASmBF75ooIruqoTWQgGo/2ZH0nvbCy9XzLyIikhq6U+rzPuAtYJKZrTGzr/qrzqFtyg/AkcB8M5sHPARc4pzb0pcNFpHu8+r8x7vaT+eTfDWH035U7UdERCQldKfaz7mdLL8wxrKHgYd73ywR6QuJGPDbVbWf5kCI/JwsTInXIiIiKSG+XYIikjTOORpbkzvJV1NrUGU+Januf2c1AK3BELnZugMlIqJPQpE01RwI4RwUJqLUZyCIcx3LKTa1hpTyIymhs3EpIiKZRldlkTTV4FfgKYpzz3t+bjbOQUuwY3DVHFDPv6SGWF9ORUQykYJ/kTTV0OLV3i+K94BfP7hvaukY/De1hihQpR9JAQr9RUQ8uiqLpKlw7f14p/2Eg/umQMdBv02BIPmq8S8iIpIyFPyLpKlw2k/cJ/nyg/tYFX+a1fMvKUJZPyIiHl2VRdJUJOc/AaU+gZgVf5qU859xzGySmc2N+tluZpe328bM7E9mtszM5pvZ/nFvmIJ/ERFApT5F0lZjq5fzH/dqP3l+2k+Mnv+m1hBDitXHkEmcc0uAaQD+jO9rgUfabXYK3gzwlcAM4Gb/t4iIxJmuyiJpqtEfgBv3Ab9dpf0EguSr5z+THQcsd86tarf8TOAu55kFDDKzEfFsiFPXv4gIoOBfJG3tqPYT/1KfAE0x6qg3t4YiXw4kI50D3Bdj+ShgddTzNf6yNszsIjObY2ZzampqetUQ5fyLiHgU/IukqcbWxFb7CVcXitbUGiRfA34zkpnlAWcAD8ZaHWNZh/DcOXerc266c256WVlZXzdRRCQj6aoskqYSPeC3OUapz+aAev4z2CnAe865DTHWrQHGRD0fDVTHszHq+BcR8Sj4F0lT4eA/3sH3jmo/6vmXNs4ldsoPwOPA+X7Vn4OBbc65dYlrmohI5lK1H5E01dgSoCA3i6ysWBkWfaewk1KfgWCIQMip5z8DmVkRcAJwcdSySwCcc7cATwOnAsuABuDL8W6TU9K/iAig4F8kbTW0BONe6QeiZvht1/Pf7A8A1iRfmcc51wAMabfslqjHDrg0oW1K5MFERFKYrsoiaaqxJRj32X0hutRn257/8JeB/Bx9zIiIiKQKXZVF0pTX8x//4D8ry8jLzqKp3YDfpkjPv9J+REREUoWCf5E01diamOAfID83q0Opz2a/51/Bv6QCpfyLiHgU/IukqcaWYNxr/IcV5GZ3KPUZTgNS2o+IiEjq0FVZJE01tAYSMuAXvEG9HXL+A+r5l9ThNORXRARQ8C+SthoS2PNfmJvdsdpPuOdf1X4kFSj2FxEBFPyLpK3GliBFCep1L4gR/Id7/vNV519ERCRlKPgXSVOJ7PkvyMnukPYT7vlXnX9JBer4FxHx6KoskqYSOeA3P7djqc9m5fxLClG1HxERz06DfzO7w8w2mtmCqGXXmtlaM5vr/5wate4KM1tmZkvM7KR4NVykK9c9sZBrH/8w2c1ImkAwREswRFFuogb8Znco9alJvkRERFJPd67KdwInx1j+e+fcNP/naQAzmwycA0zxX/NXM1O3nyTci4s28PyH65PdjKRp8APvRNX590p9tkv70SRfkkJU7UdExLPT4N859yqwpZv7OxO43znX7JxbCSwDDupF+0R6rCUQYs0nDVRva6K2qTXZzUmKJr8XPnHVfrI6DvjVJF+SQpT2IyLi6c39+G+Z2Xw/LWg3f9koYHXUNmv8ZR2Y2UVmNsfM5tTU1PSiGSJtrf6kgZB/oV9eU5/cxiRJQ0vie/47Bv+a5EtERCTV7OpV+WZgd2AasA74rb/cYmwbs7/FOXerc266c256WVnZLjZDpKOqTTsC/qUbapPYkuRJTvDfPu0nSHaWkZut4F+STx3/IiKeXboqO+c2OOeCzrkQ8Hd2pPasAcZEbToaqO5dE0V6ZqUf/GdnGcs21iW5NcnR2BoAoDBRM/zmeNV+XFRuRVNriAL1+ouIiKSUXboym9mIqKefAcKVgB4HzjGzfDMbD1QCs3vXRJGeWbmpnoGFuVSWl7A0g4L/uau3smDtNmBHz39hgvLt83OzcY42g36bWoPkK99fUoRT0r+ICAA77RY0s/uAo4GhZrYGuAY42sym4d1JrQIuBnDOfWhmDwALgQBwqXMuGGO3InFTtbmeiqHFjB1cxPsff5Ls5iTMDx+cR11zgJd/eHRS0n7Am9gr8jignn9JHYr9RUQ8Ow3+nXPnxlh8exfb3wDc0JtGifRG1aYGDqzYjQllJTwxr5qGlgBFCUp/SZam1iArNtUTDDkenLOGknzvfBM2w68/i29TIMhAciNtUs+/iIhIalG3nKSVptYg1dsaqRhaTGV5CQDLN6Z/xZ9lG+sIhhz5OVnc/PJytvslThPV8x9OL4qu+NPUGlKlHxERkRSjK7OklY+3NOAcjB9aTOUwL/hfujH9K/4sWe+d4w9PmsTarY3cO+tjgITO8Au0qfjTHAiqxr+IiEiKUfAvaSVc6adiSDHjhhSTm20ZMeh3yYZa8nKyuPDQCqaOHsgSv8RpwtN+onr+m9XzLyIiknJ0ZZa0Egn+hxaTm53F+KHFLN2Q/sH/4vW1TCwrISc7i8uOrwQgJ8vIS1DwXZATI+1HPf8Zy8wGmdlDZrbYzBaZ2SHt1h9tZtvMbK7/85N4tymkEb8iIkA3BvyK9CdVm+oZUpzHwEJv0GlleSkfVm9Lcqvib8n67Ry2+1AAjplUzt6jBrB6S2PCjh8e2NvYruc/fEdAMs4fgWedc2eZWR5QFGOb15xzpyeqQYr9RUQ8Cv4lrazc5JX5DJtYXsIzC9bR1Jq+vdBbG1rYsL2ZScNLATAz/vD5aQmd4GxH2k9Unf9AkPyc9HzPpXNmNgA4ErgQwDnXArQks02gnn8RkTB1y0laqdpcT8WQHcF/5bASQg5W1KRvxZ/F/mDfcPAPMLG8lJP3HtHZS/pcYaS2f3S1n6B6/jPTBKAG+IeZvW9mt5lZcYztDjGzeWb2jJlNiXejQor9RUQABf+SRhpaAmzY3sz4oTsyDCrLvYA4nSv+LF63HYA9hw9IWhsKYpT6bA6E0vZui3QpB9gfuNk5tx9QD8xst817wDjn3FTgz8CjsXZkZheZ2Rwzm1NTU9OrRmmGXxERj4J/SRtVmxoA2qT9VAwtIjvLEpoCk2hLNtQysDCXYQPyk9aGWKU+m1qDqvaTmdYAa5xzb/vPH8L7MhDhnNvunKvzHz8N5JrZ0PY7cs7d6pyb7pybXlZW1qtGqedfRMSjK7OkjXCln/FRwX9+TjbjhhSldcWfxetr2XN4KWaWtDa0L/XpnKOpVT3/mcg5tx5YbWaT/EXHAQujtzGz4eb/wZrZQXjXos3xbJdy/kVEPAr+pV9paAnw5PzqmOuqNu+o8R+tsrwkbdN+QiHHR37wn0w7Sn16Pf8tQe+3gv+M9W3gXjObD0wDfm5ml5jZJf76s4AFZjYP+BNwjotzXo6CfxERj6r9SL9y76yPueHpRewxrJQ9hrUNeFduqqe8NJ/i/LZ/1pXlpby4aCMtgVDC6t4nytqtjdS3BJmUxHx/gKwsIy87K1LqM/wlQGk/mck5NxeY3m7xLVHrbwJuSmybEnk0EZHUpSuz9CuzVniZAR9t6NiTX9WuzGfY7uXFBEOOj7c0xL19iRar0k+yFORmRdJ+mv3f+er5lxShnn8REY+Cf+k3giHH7KotADFz+Ks21zN+SMfgf8LQEgCW16Rf3v+S9V6ln9QI/rMjpT6bA37aj3r+JUVowK+IiEdXZuk3Fq3bTm1TAKBD9Z6a2mY21bVQOaykw+smlHlfCNKx1v/i9bWM3q2QkvzkZ/AV5GZH0n2a1PMvKUY9/yIiHgX/0m+8vdLr9d971IAOA3gX+rXup4wc2OF1pQW5lJfmp2nPf/IH+4a1SftRz7+kGNX5FxHx6Mos/cbbKzYzdnARR1SWsXJTPa3BHTXlP6zeBsDkEbEHvk4oK2ZFmgX/zYEgKzbVp0TKD4R7/sMDfoORZSKpQLG/iIhHwb/0CyE/33/G+MFUlpfQGnSs2rwjjWdh9XZG71bIwKLcmK/fvayE5TX1adX7t3xjPcGQS3qln7CCnOi0H1X7kdSinH8REY+uzNIvfLSxlq0NrcyYMITKcq+nO3rQ78Lq7Z32+gNMKCthW2MrW+pb4t7WRFnsD/ZNlbSf/NwsGiJpP+r5l9SinH8REY+Cf+kXZi33SnzOGD+YieUlmMFSf9BvfXOAlZvrY+b7h+3uD/pdnkaDfpesryUvO6vNjMbJNHnEAD5Ys5UP1myL9Pwr+JdUoeBfRMSj4F/6hbdXbmHUoELGDC6iMC+b0bsVRoL/xeu34xxMGdl5z//uZV4VoHTK+1+8vpbdy0vIzU6N/8aXHjuRISX5XPnoB9S3eFWZlPYjqUKxv4iIR1dmSXnOOWav3MKMCYMjyyrLS1nqT/S1sNpLf5ncRfA/clAh+TlZaVXxJ5Uq/QAMKMjl6tMnM3/NNv75ZhWgnn9JHer5FxHxKPiXlLdsYx2b61s4ePyQyLLK8hJWbKonEAzxYfV2divKZcTAgk73kZ1ljB9anDa1/rc1tLJ+e1PKVPoJ+9S+Iziicigf+l/I1PMvqSKoEb8iIkA3gn8zu8PMNprZgqhlvzazxWY238weMbNB/vIKM2s0s7n+zy1xbLtkiFl+ff/onv+J5SW0BEKs/qSRD6u3M3nkAMysy/14FX/So+d/cQrN7BvNzLjuzL3J84N+9fyLiIiklu50y90JnNxu2QvA3s65fYGPgCui1i13zk3zfy7pm2ZKJntuwXpG71bI2MFFkWWVw7ygd9G67SzZUNvlYN+wCWXFrP6kMVKJpj9b4qc8pVLaT9j4ocX84MQ9GDu4SD3/kjLU7y8i4tnpldk59yqwpd2y551zAf/pLGB0HNomwspN9by+bBOfnz6mTc/+xHJvAO8zC9bTEgh1WeYzbPeyEoIhx8ebG+LW3kRZvL6WAQU5DB/QeapTMl105O688sOjycrq+m6MiIiIJFZfdMt9BXgm6vl4M3vfzF4xsyP6YP+Swe6dtYqcLOPzB41ps7wkP4eRAwt4YeF6oOtKP2ET2pX73FTXzAV3zOadqi0dtv39Cx/xi2cW9bb5ALy0aAPn3f42LYHQzjfupsXrtrPn8J2nOiVTKrdNMpC6/kVEgF4G/2Z2JRAA7vUXrQPGOuf2A74H/MvMYkZlZnaRmc0xszk1NTW9aYakqabWIA++u4aT9h5OeWnHHu7KYaU0tYbIz+lerfvwNis2eXn/P396Ea98VMMPHpxHU+uOVKB3V33CH19ayj9er6KuORBzX921rbGVmf/5gNeWbmLWis292leYc46PNtSx54jUS/kRSVVO0b+ICNCL4N/MLgBOB77onFdDzTnX7Jzb7D9+F1gO7BHr9c65W51z051z08vKyna1GZLGnphXzbbGVr40Y1zM9ZV+6s+eIwaQ041a96UFuQwbkM/yjfW8uXwT/3lvLUfuUcaqzQ389eXlAASCIa585AMKc7NpCYZ49aOOX0zXfNL9tKHfPr+EzXXN5OVk8cLCDd1+XVfWfNJIXXMg5Qb7ioiISOrbpeDfzE4G/g84wznXELW8zMyy/ccTgEpgRV80VDLPPW9/zMTyEg6OqvITrXKYF/x3J+UnbMLQEhav387Vjy5g7OAibj3vAM6cNpJbXl7O8po67nyzisXra/nN2VMZVJTbIWB/cn41h//yf9z22s7/rOet3srds1Zx/iEVHL1HGS8u2oDrg1rjS9an7mBfkVSlMv8iIp7ulPq8D3gLmGRma8zsq8BNQCnwQruSnkcC881sHvAQcIlzrmNCtchOfLBmG/NWb+WLM8Z2mjs+sdwLfrsz2Dds9/JiPqzezvKaeq47cwoFudlcedpe5Odm8f0H5vG7Fz7iuD3LOXWf4Ry7Zzn/XbyR1uCOXP0736gCdqQMhX28uYHP/+0tzr7lTf7z3hoaWgJc+egHlJXk8/0T9+CEycNYt60pUv++N8KVfvYYpuBfpLsU/IuIeLpT7edc59wI51yuc260c+5259xE59yY9iU9nXMPO+emOOemOuf2d849Ef9TkHR079urKMzN5rP7d15Iar8xg7juzCl8er9R3d7vhKHe3YLT9hnB0ZPKASgvLeBHJ+/J3NVbCTnHtWdMwcw4cfIwtjW2MqfqE8CrrT9n1Sd89/g92GNYKd/613ssr6nj9aWbOOMvr7N4fS2b6lr43gPz2P/6F1iwdjvXfGoKpQW5HLtnOVkGz+9C6k/Vpnp+8fQiPqlv8dtRy6hBhZQW5PZ4XyIiIpLZcpLdAJH2mlqDPDl/HaftO4KBhZ0HuFlZxvmHVPRo30dNKuPwxUP5yacmt1n+hYPGMm/1Vg6ZMIQx/nwCR1SWRXL1D9l9CPfMWkV+ThbnHzKOz+4/ijP/8gbn3jqLTXXNTCwv4e/nT2fs4CLeWr6Ze2d/zKDCXE7dZzgAQ0ryOWDcbrywcAPfO2HHMJj3P/6EymGllOR3/l/x+icX8tLijTyzYD1/P386S9ZvV8qPiIiI7BLNwCMp56VFG6lrDvCZHvTod9fuZSXc87UZDGtXHz87y/jN2VP53AE77jQU5+dw+MShvLBoPbVNrTzy3lpO33ckuxXnMWZwEbd86QC2NrZy/F7D+M83D2PckGLMjEMnDuUvX9ifGz6zT5uUpRMmD2PRuu2s3uINk3ls7lo+89c3+cqd73RaBnTB2m28tHgjn9lvFE2tQT7z1zdYXlOvwb4iPaSsHxERj4J/STmPzV1LeWk+B08YkuymcMLkYaze0sivnl1CfUuQLx08NrLuoPGDefeq4/nbeQd02XO/Y1/eXYAXF21g3uqt/Oih+VQMKWL2yi1c+8SHMQcD/+mlpQwoyOGnZ07hiW8fzh7DSgmGHJN7MMhZJNHMbJCZPWRmi81skZkd0m69mdmfzGyZmc03s/2T1VYRkUyjtB9JKdsaWnl5SQ3nHTKO7BSYHfa4vcoxg7tnrWLKyAFMGzOozfqe5N2PH1rMxPISHn5vDbe8spyy0nwe/sah3Pb6Sm5+eTl7Di9tk8a0sHo7zy/cwHeP34MBBbkMKMjl/osO5uUlGzl+r2F9dIYicfFH4Fnn3FlmlgcUtVt/Cl41uEpgBnCz/ztu+qLSlohIOlDPv6SUZxasoyUY4sxpI5PdFMAbDBwO+M87eFyvZ609YfIwFqzdTm1TgL+fP50hJfn88MRJHL9XOT99YiGPz6smFPKClD//dyml+TlceFhF5PUFudmcvPeIbs1rIJIM/sSORwK3AzjnWpxzW9ttdiZwl/PMAgaZ2YjEtlREJDMpgpCU8tjcasYPLWafUQOT3ZSIsw4YzahBhZzRB19IPj1tFOWl+fz+89PYyy9RmpVl/P7z06gsL+E7973Pkb/+H794ehHPLFjPlw+r6HLQs0gKmgDUAP8ws/fN7DYzaz8F9yhgddTzNf6yuFG/v4iIR8G/pIz125qYtXIzZ04b2ese9r70xRnjeGPmsRTl9T5LbtLwUt7+8XGcNGV4m+WlBbk8/q3DuekL+zFmtyL+9uoKSvJz+Mrh43t9TJEEywH2B252zu0H1AMz220T6z94h/jczC4yszlmNqempuNs2z2hrB8REY9y/iVlPDm/GufgjKmpkfITL519scnLyeL0fUdy+r4jWVFTRyDkGFSUl+DWifTaGmCNc+5t//lDdAz+1wBjop6PBqrb78g5dytwK8D06dMVvouI9AH1/EvKeGxuNfuOHsiEspJkNyXpJpSVaAZf6Zecc+uB1WY2yV90HLCw3WaPA+f7VX8OBrY559bFuWXx3b2ISD+h4F9SQkNLgA/WblMVG5H08G3gXjObD0wDfm5ml5jZJf76p4EVwDLg78A3492g215bGe9DiIj0C0r7kZRQvbUJgHFD2lcEFJH+xjk3F5jebvEtUesdcGki2zRn1SeJPJyISMpSz7+khOqtjQCMHFSY5JaIiIiIpC8F/5ISFPyLiIiIxJ+Cf0kJa7c2kmUwrDQ/2U0RERERSVsK/iUlrN3ayPABBZq5VkRERCSOFGlJSqje2qiUHxEREZE4U/AvKaF6a5OCfxEREZE4U/AvSRcKOdZtU8+/iIiISLwp+Jek21TXTGvQMWpQQbKbIiIiIpLWFPxL0q1VmU8RSYAFa7cluwkiIkmn4F+SLjy7r4J/EYmnVz6qSXYTRESSTsG/JJ0m+BIRERFJDAX/knRrtzZSkp/DgIKcZDdFRNLY8o11yW6CiEjSKfiXpKve2sioQYWYWbKbIiJpbHN9S7KbICKSdDsN/s3sDjPbaGYLopYNNrMXzGyp/3u3qHVXmNkyM1tiZifFq+GSPqq3NTJSlX5EJM4mlpckuwkiIknXnZ7/O4GT2y2bCbzknKsEXvKfY2aTgXOAKf5r/mpm2X3WWklLmuBLRBIhP0c3u0VEdvpJ6Jx7FdjSbvGZwD/9x/8EPh21/H7nXLNzbiWwDDiob5oq6aixJciW+hYF/yISd82BULKbICKSdLvaDTLMObcOwP9d7i8fBayO2m6Nv6wDM7vIzOaY2ZyaGpVfy1TV27xKP6MU/ItInN3++koO/cVLyW6GiEhS9fU90FgjNl2sDZ1ztzrnpjvnppeVlfVxM6S/WPuJynyKSOJUb2tKdhNERJJqV4P/DWY2AsD/vdFfvgYYE7XdaKB615sn6W5HjX8N+BURERGJt10N/h8HLvAfXwA8FrX8HDPLN7PxQCUwu3dNlHRWvbWRLINhAxT8i4iIiMTbTmdVMrP7gKOBoWa2BrgGuBF4wMy+CnwMnA3gnPvQzB4AFgIB4FLnXDBObZc0sHZrE8MGFJCbrSocIiIiIvG20+DfOXduJ6uO62T7G4AbetMoyRzVWxuV7y8iIiKSIOpulaTyJvhS8C+STsysysw+MLO5ZjYnxvqjzWybv36umf0kGe0UEclEO+35F4mXUMixbmsTJ++tfH+RNHSMc25TF+tfc86dnrDWiIgIoJ5/SaJN9c20BEOMVs+/iIiISEIo+Jek2bCtGVClH5E05IDnzexdM7uok20OMbN5ZvaMmU1JZONERDKZ0n4kaRpbvUJQxfn6MxRJM4c556rNrBx4wcwWO+dejVr/HjDOOVdnZqcCj+KVhm7D/+JwEcDYsWP7rHHOOcxizUkpIpL+1PMvSdMSCAGQl6M/Q5F04pyr9n9vBB4BDmq3frtzrs5//DSQa2ZDY+wnLjPBu5jzzouIZAZFXZI0zQGv5z9fwb9I2jCzYjMrDT8GTgQWtNtmuPld72Z2EN61aHOi2hhU9C8iGUz5FpI0zX7Pf35OdpJbIiJ9aBjwiB/b5wD/cs49a2aXADjnbgHOAr5hZgGgETjHufhG5AW5WTS1ep85IQX/IpLBFPxL0oR7/pX2I5I+nHMrgKkxlt8S9fgm4KZEtuudK49nn2ufByAUSuSRRURSi6IuSZqWSM+//gxFJL5KC3Ijj5X2IyKZTFGXJE2zgn8RSQKl/YhIJlPUJUnT3KpqPyKSeKGQgn8RyVyKuiRpWoIa8CsiidcSCBEIhojzGGMRkZSk4F+Sprk1iBnkZmuyHRFJnF8/t4SJVz7D7a+vTHZTREQSTsG/JE1zIER+TpZm2hSRhBgxsACAxetrAXjo3TXJbI6ISFIo+JekaQ6EyMvWn6CIJEa4m+GDtdsAzfQrIplJkZckTXMgRH6u8v1FJDHa32VcsqE2SS0REUkeBf+SNM2BoMp8ikjCrN3amOwmiIgknSIvSZrmQEhlPkVEREQSSJGXJE1LIKQynyKSMFNGDkh2E0REkk7BvyRNuNqPiEgiXHZcZbKbICKSdIq8JGmaW4NK+xGRuDpl7+GRx3sMK01iS0REUkNOshsgmaslGKIkX3+CIhI/fz53PxpbgwBoShERkV70/JvZJDObG/Wz3cwuN7NrzWxt1PJT+7LBkj6aW5XzLyLxlZOdRWlBLgBGx+j/O/e9z8btTYlulohI0uxy8O+cW+Kcm+acmwYcADQAj/irfx9e55x7ug/aKWmoORAkP1dpPyKSGLF6/h+fV81BP38p8Y0REUmSvoq8jgOWO+dW9dH+JAM0B0Lka4ZfEUkQzegrItJ3wf85wH1Rz79lZvPN7A4z262PjiFppiUQUs+/iCRMqIvof1tjawJbIiKSPL2OvMwsDzgDeNBfdDOwOzANWAf8tpPXXWRmc8xsTk1NTW+bIf1Qs+r8i0gChQf+xnLVowsS2BIRkeTpi27XU4D3nHMbAJxzG5xzQedcCPg7cFCsFznnbnXOTXfOTS8rK+uDZkh/0xxQqU8RSZyRgwo7Xbd43fYEtkREJHn6IvI6l6iUHzMbEbXuM4C6U6QD55w/w6+CfxFJjIGFuSy87qSY65ZurEtwa0REkqNXRdbNrAg4Abg4avGvzGwa4ICqdutEAAiEHCGHgn+RNGRmVUAtEAQCzrnp7dYb8EfgVLxKcRc6595LSNtilPsUEckkvQr+nXMNwJB2y87rVYskIzQHQgDK+RdJX8c45zZ1su4UoNL/mYE3VmxGIhrlUMkfEcls6naVpGjxg3/l/ItkpDOBu5xnFjCoXcpo3BTmZnP2AaMTcSgRkZSkyEuSojngVd1Q2o9IWnLA82b2rpldFGP9KGB11PM1/rK4MzN+ffZUHv7GoYk4nIhIylHklSI21zVz0A0v8sGabcluSkI0t/ppP6rzL5KODnPO7Y+X3nOpmR3Zbn2sxPsO+TjxLAl9wDhNQSMimUmRV4pY80kjG2ubWbKhNtlNSYhwzn9etnL+RdKNc67a/70ReISOJZ/XAGOino8GqmPsRyWhRUT6mIL/FNHQ4qXB1DcHktySxGiJDPjVn6BIOjGzYjMrDT8GTqRjyefHgfPNczCwzTm3LsFNFRHJSIq8UkRjqxf017dkRvAfyflX2o9IuhkGvG5m84DZwFPOuWfN7BIzu8Tf5mlgBbAMbzLIbyajoRcfOSEZhxURSapelfqUvpNpPf870n4U/IukE+fcCmBqjOW3RD12wKWJbFcsV5y6F397dUWbZTW1zWyqa2avEQOS1CoRkfhS5JUidgT/wSS3JDEiaT+5yvkXkdRx3G9f5pQ/vpbsZoiIxI2C/xTRmHE9/yr1KSKpZ3tTZnwGi0jmUuSVIsK5/pmT868BvyIiIiKJpsgrRYR7/usyJO2nWTP8ikgK6MvZfh99fy1f+PusPtufiEg8KPJKEeGc/4aMSfsJ9/wr519EkueaM6YAMLAwt8vt3v/4E96p2tJmWWswxK+fW0xtUysAl/97Lm8u3xyfhoqI9BEF/ymiIdLznyHBf6tKfYpI8pXk53DewePIzuo46fCdb6xk1govmP/MX9/k7FvearP+iXnV/OV/y/nls4sT0lYRkb6gUp8potHP9Q9/CUh3KvUpIqkiyyDkXIfl1z6xEIBnLz8isqxi5lMcv9cwvnH0BJ5ZsB6AD9Zuz5hiDSLS/yn4TxGZVudfM/yKSKowM0KhjsF/2Ml/aFv688VFG3hx0YbI83mrtzLlmufi1j4Rkb6kyCtFNLZmWNpPIEReThZmHW+1i4gkUpYZ7Tv+K2Y+1at9rt3ayHsffwLAqx/V8LdXlvdqfyIifUU9/yki3PPfHAgRCIbISfN0mOZAkPw0P0cR6R+yDILOUb21sU/255zjsBv/C8DsHx/H+XfMBuDio3bvk/2LiPSGoq8UEZ3rX58Bef8tgZAG+4pISsjOMhpaghzqB+y9tbymLvJ42ca6LrYUEUk8RV8pojFqcq9MyPtvDoRU5lNEUkJfp1se/7tXI4+/cNvbfbpvEZHeUvCfIhpaghTnZfuPMyX415+fiCRfINj5YN++9O6qTxJyHBGRrijnP0U0tgQpK82nfnNDRszy29wa1Oy+IpISshL0UfS5m98EoCA3i0XXncz4K55mzOBCXvvRsfzjjZVs2N7MzFP2TExjRCRjKfpKAc456lsClJXmA5kxy29LUD3/IpIqElt1rKk1xPgrngZg9ZZGQiHHT59YyC2qCCQiCaDoKwU0B0KEHJHgPxPKfTa3KudfRFJDVzX+E+H4378SeexiTDYmItKXFPyngEa/uk9ZiRf812dEzr/SfkQkNfx7zuqkHn9FTX3k8ZG//h+vL93EY3PXRpZNu+557nh9ZTKaJiJpqFfRl5lVmdkHZjbXzOb4ywab2QtmttT/vVvfNDV9NfgTfA0NB/8ZkPOvtB8RkY5Wb2nkS7e/zWX3z2VLfQsAWxtaue7JhUlumYiki76Ivo5xzk1zzk33n88EXnLOVQIv+c+lC+Eyn+G0n4wo9dmqOv8iIl0559a3Osw0XLWpnteW1gBeilAwySlLItL/xCP6OhP4p//4n8Cn43CMtBKe4GtISQYF/4EQeZrhV0SkUx9t6DhB2NG/eZnzbvdmDP7Bg/PZ/cdPJ7pZItLP9Tb6csDzZvaumV3kLxvmnFsH4P8uj/VCM7vIzOaY2ZyamppeNqN/Cwf/xXnZFOdlZ84MvxrwKyLSbbe9tiLy+O5Zq3j4vTUAvLtqS5tZhUVEutLbOv+HOeeqzawceMHMFnf3hc65W4FbAaZPn57R9y3DA34L87Ipys/JkJ7/oNJ+RER64GdPLYo8vvrRBZHHn7v5LQCqbjwt4W0Skf6nV9GXc67a/70ReAQ4CNhgZiMA/N8be9vIdBfu+S/Ky6EkPycjev41w69IejOzbDN738yejLHuaDPb5heLmGtmP0lGG9PNd/89N9lNEJF+YJejLzMrNrPS8GPgRGAB8Dhwgb/ZBcBjvW1kumvwB/wW5WVTnJ+dIT3/IZX6FElvlwGLulj/ml8sYppz7rpENSqdPfL+WgLBULKbISIprjfR1zDgdTObB8wGnnLOPQvcCJxgZkuBE/zn0oXG1qi0n7yctJ/kKxAMEQw55fyLpCkzGw2cBtyW7LZkmolXPsO1j39IIBiiqTXInKotyW6SiKSYXc75d86tAKbGWL4ZOK43jco0O9J+sinJz2FjbVOSWxRfLX7PlNJ+RNLWH4AfAaVdbHOI33lUDfzAOfdhIhrWG6/+8Bi2NrYwYmAhB97wYof1y244hYlXPpOElrV155tV3PlmVeT5c5cfiRkMLs6LzCcjIpmrtwN+pQ80+D39BTnZFOfnUL8pvXP+m1u94F9pPyLpx8xOBzY65941s6M72ew9YJxzrs7MTgUeBSpj7Osi4CKAsWPHxqW9PTF2SBFjKQLgli8dwH8Xb+CBOWsi63NStHzxSX94FYDSghw+uPakJLdGRJItNT+pMkxDS5CivGyysozivOy0T/vZ0fOvtB+RNHQYcIaZVQH3A8ea2T3RGzjntjvn6vzHTwO5Zja0/Y6cc7c656Y756aXlZUloOltnXXAaGZfGftG9sl7D+eaT01h6uiBAOxeVgzAxPISJpaXJKyNPVHblN7XFhHpHgX/KaCh1Qv+AYrzcyJ3AtJVuOdfaT8i6cc5d4VzbrRzrgI4B/ivc+5L0duY2XAzM//xQXjXos0Jb2wXxg4u4jdnT6W8tKDTbYrzc/jd56cBkJ1lALz4vaN48XtHRWZsB+8LwWs/Oiau7RUR6S5FXymgsSVIYTj49yf5CqXxlO3NAS+tSWk/IpnDzC4xs0v8p2cBC/yc/z8B5zjnUupD72tHjO/WdhOGFnPxURO49bzpbZZX+r3/V522F09953DGDC7iqtP2Ysb4wQCcf8i4vm1wN1XMfIrT//xa5PnVjy7gv4s3JKUtIpIcyvlPAQ0tAYpyvX+K4nzvd2NrMPI43TQH1PMvkgmccy8DL/uPb4lafhNwU3Ja1T3DBuzo8c8y6Kw/xsy44pS9Oiy/+UsHMG/1Vo7cY0e60teOmIBz8PbKLeS1Gx/wzGVHkJudxfG/e6VvTqALC9ZuZ/rPXuSVHx7N3bNWcfesVaz8xal8sHYb+44eFPfji0hyKfpKAQ3RPf9+wJ/Otf4jwX+ucv5FJDVZ1OP5157E/GtP7NHrBxbmtgn8wxwdv0WYwV4jBnQYK3DHhdM7bNtXNtU1M+Wa5yLP7561ijNueoNnF6yP2zFFJDWkZ9dyP9PYEp3z7/2uaw5QnsxGxVE47Uc9/yKSar57/B78/sWPGDmoMLKsJE53YZ/89uEsr6njxMnDY66fNma3uBw3lp885lVavfftVeRkGeu2NfKlg8fhD80QkTSi4D8FNLQEGVSUC0BxXk5kWboK9/wr519EUkFBbhZNfiGCbx87kVP2Gc4ew7qaomDXFfmf8UX5Oew9aiB7jxrYZn3VjacRDDmybEdltPaGDchnw/bmyPMDK3bjnapP+qR9ry3dxGtLNwGwz+hBTBszCIBZKzazqa6Z0/cd2SfHEZHkUfSVAhpbgxT6F4RwD1M6l/tsUc6/iKSQQYV5kcdZWRa3wB/gnAPH8H8n78k3j969022yswwzIz8nmw+uPZFRgwq57XwvBeiurxzEAxcf0mb7rx0xIS5tvXfWKn793GJCIcc5t87iW/96Py7HEZHEUs9/CvAG/HrpPkWZlPOvOv8ikmFysrP4RheBf3ulBbm8MfNYwLsrEHb9mVO42k/VKYjT+KkH3/UmMDt2z3RNQhXJTOp6TQHRA35L/Jz/+nRO+2lVzr+IpI7bLvB61f/yhf2T3JLuO2nvHeMEDp/YYX60PvW5m9+K6/5FJLHU859kzrnIDL+QGdV+dszwq+BfRJJv71ED2/Sq9wflpQU8e/kRLFq3PTLBWNgXZ4zl3rc/jtuxtzW28kl9C8MHFkTuOry+dBPlA/LjmjIlIn1DwX+StQRDBEMuEvyHB4Olc/C/Y4Zfpf2IiOyqPYcPYM/hA9osq7rxNN5cviluwf/LSzZy4T/eaXM8gC/d/nab5yKSuhT8J1mjn94TDvqL/S8B9c1pnPajaj8iIn3qzZnHRoopzBg/JG7HiQ78AZZuqGWgX61ORPoHRV9J1hAJ/r2gPyc7i4LcLOpb0rjn36/zr+BfRKRvjBxUSMXQYsCrFnTlqXslZKDuCb9/lRN//2rcjyP91/KaOipmPsU7VVuS3RTxKfpKsnDwHx7wC16t/3RO+2kJhMjNtg55qiIi0je+fuQEzj5gdEKOtbWhNSHHkf7pjWXevBGPz61OckskTGk/SdY+7Qe8Qb/pHPw3B0LK9xcRibOT9x7Ob8+eSsXQIkYOKuSQX/w32U2SDBTu5nO4pLZDdlDPf5I1+Ok9RdE9//k51KV1zn9QKT8iInFmZnzugNEcMG4wIwYWMvOUPQE4ojJ+pUEPuP4Fqrc28uyCdXE7hvSN+uYAf35pKYFOZpLuM+aF/66L2H9TXTO/eHoRwZC+ICSCIrAka2iNlfaTHflSkI5aAiGV+RQRSbCzDhhNSX4OPzppT5b87GRu+sJ+fPSzU/jPNw/ts2Nsrm/h0Bv/yyX3vIfzo713qrawvUmpQXe/VcUNTy3s8euaA0EqZj7FH19cyuL126mY+RTvrup9/vyX//EOv33hIx5LUDpOZ2F9bVMrX7rtbf726gpeXVqTkLZkOkVgSdbYbsAvZEraj/70REQSaWhJPgt+ehL7jB5Ifk42p+87krycLKaM9MqFHrp731YJOuWPr/HRhlrOvuUtvvbPOX2671SyrbG1Wz3WVz/2IX9/bWWX21xwx2zueH3HNqGQY+0njQD8/sWPuH/2agCe/mB9t9rWEgjxSX1LzHWz/QG4LcEQlVc+zZ1vdGzbxu1NMQfrVm2q79Y5B0Mu0uXfvuf/reWbqW8OcPYtb7F4fa2/Tcd9Ouc6PQfZNYrAkixS7Sd3R85/SX5Oms/wG1Laj4hIisjPyWblL07l3q/N6NP9Ll5fG6kENHvlFipmPgXAgrXbImVJU51zjrrmAKf/+TXO8+cyiNbYEmTqT5/n+ie9Hv2Vm+p3us+KmU9x3ROx7wC88lEN1z25kN88t4TmQJBfPLOIY3/7SmT9nW9WAXD76yup2lRPcyDIso11PPDOaqb/7AUqZj7FZfe/z/2zvXkevvWv99jv+hcAaA2GOPJX/+PZBW2/ODgHrUHHtTHa9NeXl0eOu72plbVbG7n55eUc/ZuX+c3zXhtrapsj79PfXlnOx5sbuHvWKn793GJ2//HTXP3Yhx32u2F7E+f+fRbf/ffcSOAfLRAM8evnFrOtsZX731nNfte/wEcbOm4H8El9C7W6s9QjGvCbZI1+ek902k9RXnZa9/y3BDXgV0QklZiflz1sQD4btjfH7TjhLwDQPyYEu2/2an78yAedrm/0U3fvfLOKmrpmnpq/jr+fP50TJg8DvC8PVz66gMkj2k7GdscbKynJz+Z7J04CYN7qrdwza1Vk/U3/W8bg4rwu7xTMrtrCjx6a32H5Y3OreWxuNUdPKuf5hRsA+MXTi/j3nNVsbWjlknvebfPev//xJ5HHLYEdnXMfVm+LfNl4av465n68lbVbGyPb3vzycm72vxwATB0ziHmrt/KLZxbHbG/0cZr896194F/fHGRh9XaW1dTxl/8t5y//W86ew71Zo5dtrIs5g/R+179AQW4Wi68/JeZxpSN1vyZZ+zr/EB7wm77Bf3MgqLQfEZEU9PaPj092E+KqJRDi+w/MiwSxS9bXdjke4YWFbXvJK2Y+xf2zP2bD9iZWb2kgumL1U/O9Qc6vfrQjb72uOcC/3v6Yqx5d0GHff/rvMipmPkXFzKc48y9v8OC7a9qs/90LH3V5LrEC/2gH/+KlyOO/vbqiTUnWJ+fvyPOPPu4eVz1D9dZGFqzdRk1t2y+B0YF/LPNWb+1y/eL1tVxwx2ycc5hfAyjULs3n2/e9z6l/eo3v3Pd+m9cBfLylgW/c827ki0O0ptb+cScplmDI8eCc1Qkd7LzLPf9mNga4CxgOhIBbnXN/NLNrga8D4b/+Hzvnnu5tQ9NVOL2nMDc6+M+moSXo/Qex9KuFr7QfEZHU9edz92PUboV89q9vJvzYSzfU8o83q/jZmXuT1QdzwcxdvZWm1iD7jR3E7a+vZPeyEh5+bw1b6pv5x5cP4qQ/vMrg4jweuPhg7pn1MVecuif5Odm8u2oLIwcVxrwGz/zPjjsBN31hvw7r7561irv9XvyvHT5+l9sez07Ab/3r/U7XfeHvs6ja3BCX477yUQ3jr3iafUcPBGDNJ11/oYh2o39H4dP71bB7WQktgRCTRw7Yyat2WLW5noffW8t3j6+kanMDn/7LGzz57cMZM7ioZyfRx+56q4qfPrGQptYg5x1SkZBj9ibtJwB83zn3npmVAu+a2Qv+ut87537T++alv8aWAAW5WW0+5IrzcwiGHM2BEAW56Zce0xwIUVqgjDMRkVT0qakjk3bsr981h6rNDXzt8PFMKCvp9f4+/Zc3APjBiXvwm+c/Yrw/C3J0YL2lvoXjf+eNTQinuYQNLcnrcv9dBdEAt73e9QDfVBSvwD/a/DXbdvm1F9/9buTxr8/aN/K4tqmV0oJc3qnawr7+oPaH313D9x+cx+PfOowzbvL+Fv7f9NE8MGc12xpbeXxeNZceM5EVNXWMG1KclMlHw4OZt9QnbtzCLne/OufWOefe8x/XAouAUX3VsEzR0BJsM8EXeAN+Ib7f+pOpRZN8iYikvN9/fiqn7D28Te/2hKHFDCnuOiDurlteWc4PHpxHxcynuPXV5dQ2tRLwUx+y+uCud2tU/frwXfbwgNx3qj7ZaRoLwKY6VZlJZT+MSn165aOaSHWpM/7sBfrff3AeQCTwB2+Ac64f5AeCjg+rt3Hsb1/h9ztJs4qb8DwICZwErU9yL8ysAtgPCA+F/5aZzTezO8xst744RrpqbAm2yfeHHbP9NqTpRF/NgSD5uUr7ERFJZZ/ZbzQ3f+kATt93ZKQS0LABBZwxrW/uDNz4zGIe8vPNf/70Yva59vlIGsjO8tl3JhAMcdFdXZcXPexGzXicTr71r/d53J+zYMmG2jZf/trLzvJikBcXbeC0P70OwK2vreiw3fpt3tgO8FLSNm5virm/+uYAFTOf4t/vfLzL7e9qErS+1uvcCzMrAR4GLnfObTezm4Hr8eZzuB74LfCVGK+7CLgIYOzYsb1tRr/VECP4L8n3nqdrz39zIERetoJ/kXRmZtnAHGCtc+70dusM+CNwKtAAXBi+kyypKdwR79gxWDOeZlf1bhKryT95jpZ4z1wrKeem/y2LPJ760+djbnPEr/4XefzB2h3pRy2BEKs217N6SyOvfLSxTaWlqhtP44Tfv4oZ/Opz+3LwhCFtxgps8L8U3PLKCj5/YM9i2mSM7OxV8G9muXiB/73Ouf8AOOc2RK3/O/BkrNc6524FbgWYPn16xs7n3NAapLBd2k+xn/aTrrP8tgRC6vkXSX+X4aWDxhqRdwpQ6f/MAG72f0uKCqfhJLAgCXXNAZ6aX81B44dEcvVj+cljCzhg3G6MG1KMc44F1ds7BP7RJSklMzTswnxJR/365ZjLw/MyOLcj1Si6XGr47805xxk3vc5JU4Zz6TETe3TsRAbCvan2Y8DtwCLn3O+ilo9wzq3zn34G6FjfSiIaWwIU5cZO+0nnnn/l/IukLzMbDZwG3AB8L8YmZwJ3OW86z1lmNqjdtUNSzKhBhQAcPamMPcpLuSPGbLB9be9rnos8jg60lm6o5YTfv8o/LjyQx+dV88j7a7nrrVWxdiHSJ2L9vf9v8UYK87JpaAnwlTu9FLPwYOn5a7Z1O/hPRlHH3vT8HwacB3xgZnP9ZT8GzjWzaXhfYqqAi3txjLTX0BJk+IDcNsvCA37r0zjnX6U+RdLaH4AfAR1n5PGMAlZHPV/jL1Pwn6LGDC5izlXHM7goj6wsY8FPT+KJedVc8Z8POPegMdw3e/XOd9ILe139LIuuPxmAL9/5TpvfIsmws7+/L/x9Fm8u3xx5/sOTJuGc44jKMvJzsxhSnE9ZaT5/eHGpt0ECk/53Ofh3zr1O7FQl1fTvgcaWYJvZfcGr8w9Qn4ZpP6GQozXoNMmXSJoys9OBjc65d83s6M42i7Gsw5VPY8NSy9CS/MjjkvwczjlwDKfuM4KBhbn84rP7ep/voRCTrnoW8Hrrf/bkwj4pdxmeSbexJdij2vAiyRId+AP8+rklAPzm+dhVhf7032V8er9RfVLidmcUgSVZrAG/xXnhnv/0C/7DeXFK+xFJW4cBZ5hZFXA/cKyZ3dNumzXAmKjno4HqdtvgnLvVOTfdOTe9rKwsXu2VXWRmDCzccec6K8s6fLYX5ffdnC4VM5/ia3f1/97+kQMLmOpPctUbV522V7e2O+fAMbx7VWJmbj6zk0pQV58+mRnjByekDf3Z3J3MktxXFPwnWUNLoEOd/+L89A3+m/0puJX2I5KenHNXOOdGO+cqgHOA/zrnvtRus8eB881zMLBN+f7p48uHVUQef/Po3ftsXgCAN5Zt3vlGKeyBiw/hzSuO45FvHtat7TubZGzUoEKGDSiIPD95ynCqbjyNK0/1vhCEZxb+xtG7c+Pn9mVIST4f/eyUyPZXnz6Zbxy9OwCXHVfJ7/7f1F06n/Z+e/ZU7vnqDM4/ZByv/vCYyPKvHFbB3847gB+fuifzfnIiAHd/9SAuO66SBy4+hEuP2b1Pjt/fJSr/X9OsJllja8e0n7ycLPJysqipbU5Sq+KnOeDdulXaj0hmMbNLAJxzt+Clh54KLMMr9fnlJDZN+tg1n5rCNZ+aAkBBbjbvXn0CzjleWLiBi6JmZ01XQ0vyYk4OdsbUkRzk935nZRmv/98xLFpXy9c7mY/gmEll3HHhgZgZc6q2cNYtbwHw/RP24PSpIxk1qJDzDh7HZcdXRlKyvn7kBL5+5AQArjp9cpv95eVkccbUkTw+r5qhJXlsrvNijKK8bD67/2gmjxzAnW9UccNn9mFO1RbWb2/CzLjmsQV88+iJlA/I57L75wLwjwsP5OhJZbz38VbKS/P59zurOXnv4eRkZ3F45VAOrxwKwIOXHEJLIISZMagoj4uO9IL88ADuIyq9O3oHjR/MD0/ak6bWIMs21vGv2R/zr7e9mvn3fHUGg4vzOPVPr+3Cv0b/kqjqtAr+k6glEKI16DpU+wE4dlI5/3lvLd87cVKb26r9XXMgnPaj4F8k3TnnXgZe9h/fErXcAZcmp1WSDGbGiVOGJ7sZu2xIcR6b672AfuTAAl743lFMiapGFLbHsBKe+s4RVF75TJvlL37vSCaWtx3/Pnq3IkbvVkTVjadRMfMpAN664lj+/upK7nhjJb/83L6Y3xU8vWJHysy3j6uMPL7+03v36DzOnOYF/1NHD+LYPcupbQpw/iEVAOw5fAA3fm5fAGZMGBJ5zRlTd6TyhIP/oyeVYWYcMM6bx/UHJ02KebwDK3qW6lOQm83eowZyw6f35oiJQzlh8jBy/HmBot8nr72lLF5f26P9p7pElXhXBJZEjX4N2vY9/wDfPm4itc0B7nyjKsGtiq9I8B/jC4+IiKS3e77a/6Zz+Orh43lj5rGR52ZGcX4OR+3h9VpffNSEyLp9Rw8iNzuLpTfsSLGpuvG0DoF/Z0YMLOTq0/di8fUnUx6V1tNXjttrmBdEDy2mtCCX6z+9d8wYpDMXHlrB7RdMj3wpiRcz45R9RkQC/1i+fsQEqm48jae/c0Sbv6sHLj4krm2Lp/c/3pqQ46jnP4kaWr1veO1z/gGmjBzICZOHcfvrK/jK4RWUFqRH73847Ucz/IqIZJ5wOkh/cv4h4yjIzWb5z0/l2/e9x9eP8IL9P3x+Gk8vWMcXZ4zjU/uO5PQ/v87Rk7wvBLnZWUwZOYCKIZ1PThbtgYsPiRT/MDMKYnSQXXzkhIQNCO3MtWdMSerxwxZed1Ikdpo80ptH8L2rT6CxNRiZkyLsRydPYq8RA/jyP97hiMqhvLZ0EwAvfu8ojv/dK4lt+E7UNrUm5DgK/pMoPPtc+2o/Yd85tpJPLdzAP9+s4lvHVsbcpr9pifT8K/gXEclED3/jED53s5e/vteIASxatz3JLYIZ4wczoayE+2Z7eeZnHTCaYQPyufz4Pcj1O6uys4y/fvGAyGt2K87jizPGAbD3qIEs+OlJkXl6AJ76zhHdPv5B3aiEc8Wp3avuk86u/dRkXlq8MWan6eBOBpZ/82hvsq3wOINrHlvAP99axbghRZFtXvzekRz/u1fJz8mKZCgkR2JG/CoCS6LGnQT/+4weyHF7lnPb6yvTZrZf5fyLiGS2A8btCHR/8dl9ktiSHf598SFcd+YUCv0e9+8cW8kPT9ozEvh3R0kfljWV2C48bDx3dyN1LDwW4clvH95h3U/P3JuqG09r8287sbyUqhtPY8nPTuGtK47t8JpECSVooi9FYAnw9orNLFi7rcPyHT3/nX9gfOe4SrY2tHJHH0ySkgoU/IuIyMPfOITHLj2MaWMGUXXjaZ2WtOwrT0f1wlfdeBrvXnU85aX53P3Vg3jnSq8Gfm52Fo9/6zAuPLSC0bsVdrYr6Qce/sahVN14GnuP6no+hdvOn86Dl7QdIzBiYCETy3dMtPWbs6fy2f1GRe4sTChrm8r188/03RfYRAX/+poaZ6u3NHDBP2ZTmJvNf79/NLtF3ZYKj+ruarDN1DGDOHnKcH73wkcEQo7Lj6skK8u7LRQIhqje2sSYwYVxH3zTV5pbw6U+NeBXRCRTRff+A532sFeWl7B0Y90uHeOsA0Zz/F7DGDu4KJIXHjakJJ/ZV3ac+KpyWGnK5LVL/B0/eVjM5c9dfiS/em4xXz1sPOUDCjjrgNE453DOK9P6xrJNfPG2t7n8+Eq+MGMsKzfV8ffXOnbS/vGcaVz16AJqm7qXvTFucNHON+oDCv7jyDnHTx5bgGHUNgW48ZnF/PKsfSPrd5b2E/bHc6dx9aML+NNLS1m0bjszT9mTJ+et4/53Pmbdtib2GFbCF2eM4zP7j2JAig8M3jHDr3r+RUTE01mH5/dPnMQl9+za3AC/ObtvJq6SzJOdZVxxStsxFmYWmYTrsIlDI2MIAK48bXIk+P/CjLFccuTuPDG/mjOmjuTMaaMi2wWCIe59+2OuefzDmMf97gl79PGZxKbgP46e+3A9/1tSw1Wn7UVNXTN/e2UFZ00fzYEVg3HOMXfNVgCKu0j7Aa+X/Jef25cpIwdy3ZMLeWHhBgCOqBzKhYdW8PQH67jm8Q+54elFnQb/44YUcc6BYzh935E9KuvV18Iz/KrnX0RE2nvs0sOo2lzP4vW1nDB5GPuP3a3L7ScNK+X6T+/Nex9/wtaGVm55ZXmCWioSWzgN6NJjJnZYl5OdxQWHVsQM/h+79DAGFcU3/S3SjoQcJQPVNQe49vGF7DViABceWkFLMMST89Zx5SMf8Mg3D+O6Jxby7zmrOXHyMMYM3nluoZlxwaEVTB45gHeqtnDaPiMY55cQu/io3flgzTYen7eWev9uQjTn4J2qLfzwoflc/+RCPrv/aL44YyyVw7pXd7ivtAZDLNngTciRp55/ERFpp3xAPlPHDOLMqGWXHLV7p0H9c989EthRLeflJRs7nfjprq8cRFNrx2ukSF947UfHdHoHa2ei7yIkQr8O/pdtrGPlpvpkNyOmZxesZ0NtE3/90v7kZGeRk53FT8+YwtfumsNRv36ZTXXNfPvYiXz3+D16lK9/YMXgmDPm7TN6IPuM7nxgi3OO2Su3cO/bH3Pv26u4880qDho/mLP2H91mHEI8OOeYt2YrD8xZQ01tMxOGFrNbcWqnJ4mISOI4Oo+aZp6yZyT4f2PmsRx243873fbZy49k/bYmqjZ3jA2O9CflEomHMT3I1z9+r3JeXLQxjq3pWr8O/h+fV82fXlqa7GZ06ryDx7W5ZXn85GGcsvdwXl5Sw1++sD+n7TsiYW0xM2ZMGMKMCUPYVDeZh95dw32zP+ZHD89PyPGzDI7ds5wvzhjHkXuUkZ3VPwYoi4hI/P31iwfw91dXUF4ae1bbV394DPUtAUYNKqSsNJ+a2mae93v92xs+sIDhA/t+dlyRvnLbBQcC8IMH5yUlE8JcgsoKdWX69Oluzpw5PX7dhu1N1NQ2x6FFvZedZew5vLRDr35rMER9cyBheV1dCYUcSzfW0RqM/4QW5aX5cZmqXCQTmNm7zrnpyW5HMu3qdULST1NrkGDIUay6+iJAz68R/fp/zrABBQzrZwFlbnZWSgT+4JWrmjQ8sXn/IiIivVGQq4IRIr2hUZciIiIiIhlCwb+IiIiISIZQ8C8iIiIikiEU/IuIiIiIZAgF/yIiIiIiGULBv4iIiIhIhlDwLyIiIiKSIRT8i4iIiIhkCAX/IiIiIiIZQsG/iIiIiEiGMOdcstuAmdUAq3bx5UOBTX3YnP4m088f9B7o/NP//Mc558qS3Yhk0nWixzLtnDPtfCHzzlnn27keXSNSIvjvDTOb45ybnux2JEumnz/oPdD5Z/b5y85l4t9Ipp1zpp0vZN4563z7jtJ+REREREQyhIJ/EREREZEMkQ7B/63JbkCSZfr5g94Dnb9I1zLxbyTTzjnTzhcy75x1vn2k3+f8i4iIiIhI96RDz7+IiIiIiHRDvw7+zexkM1tiZsvMbGay2xNvZjbGzP5nZovM7EMzu8xfPtjMXjCzpf7v3ZLd1ngys2wze9/MnvSfZ8z5m9kgM3vIzBb7fweHZNj5f9f/219gZveZWUEmnb/0XLpcJ3bl89/MrvDPe4mZnRS1/AAz+8Bf9yczs2ScU3f05PM+Tc63R5/x/f2ce/qZ3h/P18zuMLONZrYgalmfnaOZ5ZvZv/3lb5tZxc7a1G+DfzPLBv4CnAJMBs41s8nJbVXcBYDvO+f2Ag4GLvXPeSbwknOuEnjJf57OLgMWRT3PpPP/I/Csc25PYCre+5AR529mo4DvANOdc3sD2cA5ZMj5S8+l2XWiR5///rpzgCnAycBf/fcD4GbgIqDS/zk5kSfSQ936vE+j8+32Z3x/P+eefqb34/O9k47t6ctz/CrwiXNuIvB74Jc7a1C/Df6Bg4BlzrkVzrkW4H7gzCS3Ka6cc+ucc+/5j2vxPhRG4Z33P/3N/gl8OikNTAAzGw2cBtwWtTgjzt/MBgBHArcDOOdanHNbyZDz9+UAhWaWAxQB1WTW+UvPpM11Yhc+/88E7nfONTvnVgLLgIPMbAQwwDn3lvMG/d1Fiv6f6eHnfTqcb08/4/v9OdOzz/R+eb7OuVeBLe0W9+U5Ru/rIeC4nd356M/B/yhgddTzNf6yjODf1tkPeBsY5pxbB94FAihPYtPi7Q/Aj4BQ1LJMOf8JQA3wD/82+G1mVkyGnL9zbi3wG+BjYB2wzTn3PBly/rJL0vI60c3P/87OfZT/uP3yVPQHuv95nw7n29PP+H59zrvwmd6vz7edvjzHyGuccwFgGzCkq4P35+A/1reajChdZGYlwMPA5c657cluT6KY2enARufcu8luS5LkAPsDNzvn9gPqyaAUFz8n8kxgPDASKDazLyW3VZLi0u460YPP/87OvV+8J7vwed+vz9fX08/4fn3Ou/CZ3q/Pt5t25Rx7fP79OfhfA4yJej4a73ZRWjOzXLwP/nudc//xF2/wbwnh/96YrPbF2WHAGWZWhXf7/lgzu4fMOf81wBrn3Nv+84fwLhSZcv7HAyudczXOuVbgP8ChZM75S8+l1XWih5//nZ37Gv9x++Wppqef9/39fKHnn/H9/Zx7+pne3883Wl+eY+Q1fvrUQDqmGbXRn4P/d4BKMxtvZnl4AyQeT3Kb4srP4bodWOSc+13UqseBC/zHFwCPJbptieCcu8I5N9o5V4H37/1f59yXyJzzXw+sNrNJ/qLjgIVkyPnj3Ro+2MyK/P8Lx+HlPWfK+UvPpc11Yhc+/x8HzvErgYzHGyA4208xqDWzg/19nk8K/p/Zhc/7fn2+sEuf8f39nHv6md7fzzdaX55j9L7Owvu/0vWdD+dcv/0BTgU+ApYDVya7PQk438PxbuXMB+b6P6fi5Xa9BCz1fw9OdlsT8F4cDTzpP86Y8wemAXP8v4FHgd0y7Px/CiwGFgB3A/mZdP766flPulwnduXzH7jSP+8lwClRy6f7/4eWAzfhT/iZqj/d/bxPh/Pt6Wd8fz/nnn6m98fzBe7DG9PQitdL/9W+PEegAHgQb3DwbGDCztqkGX5FRERERDJEf077ERERERGRHlDwLyIiIiKSIRT8i4iIiIhkCAX/IiIiIiIZQsG/iIiIiEiGUPAvIiIiIpIhFPyLiIiIiGQIBf8iIiIiIhni/wN9htA1xoQY3QAAAABJRU5ErkJggg==", "text/plain": [ "
" ] }, "metadata": { "needs_background": "light" }, "output_type": "display_data" } ], "source": [ "agent.train(num_frames)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Test\n", "\n", "Run the trained agent (1 episode)." ] }, { "cell_type": "code", "execution_count": 12, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Moviepy - Building video /Users/jinwoo.park/Repositories/rainbow-is-all-you-need/videos/rainbow/rl-video-episode-0.mp4.\n", "Moviepy - Writing video /Users/jinwoo.park/Repositories/rainbow-is-all-you-need/videos/rainbow/rl-video-episode-0.mp4\n", "\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ " " ] }, { "name": "stdout", "output_type": "stream", "text": [ "Moviepy - Done !\n", "Moviepy - video ready /Users/jinwoo.park/Repositories/rainbow-is-all-you-need/videos/rainbow/rl-video-episode-0.mp4\n", "score: 200.0\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r" ] } ], "source": [ "video_folder=\"videos/rainbow\"\n", "agent.test(video_folder=video_folder)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Render" ] }, { "cell_type": "code", "execution_count": 13, "metadata": {}, "outputs": [ { "data": { "text/html": [ "\n", " \n", " " ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "Played: videos/rainbow/rl-video-episode-0.mp4\n" ] } ], "source": [ "import base64\n", "import glob\n", "import io\n", "import os\n", "\n", "from IPython.display import HTML, display\n", "\n", "\n", "def ipython_show_video(path: str) -> None:\n", " \"\"\"Show a video at `path` within IPython Notebook.\"\"\"\n", " if not os.path.isfile(path):\n", " raise NameError(\"Cannot access: {}\".format(path))\n", "\n", " video = io.open(path, \"r+b\").read()\n", " encoded = base64.b64encode(video)\n", "\n", " display(HTML(\n", " data=\"\"\"\n", " \n", " \"\"\".format(encoded.decode(\"ascii\"))\n", " ))\n", "\n", "\n", "def show_latest_video(video_folder: str) -> str:\n", " \"\"\"Show the most recently recorded video from video folder.\"\"\"\n", " list_of_files = glob.glob(os.path.join(video_folder, \"*.mp4\"))\n", " latest_file = max(list_of_files, key=os.path.getctime)\n", " ipython_show_video(latest_file)\n", " return latest_file\n", "\n", "\n", "latest_file = show_latest_video(video_folder=video_folder)\n", "print(\"Played:\", latest_file)" ] } ], "metadata": { "kernelspec": { "display_name": "rainbow-is-all-you-need", "language": "python", "name": "rainbow-is-all-you-need" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.8.12" } }, "nbformat": 4, "nbformat_minor": 4 }