{ "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "# RL with DQN for cart-pole task\n", "https://pytorch.org/tutorials/intermediate/reinforcement_q_learning.html\n", "\n", "There is a leaderboard at https://gym.openai.com/envs/CartPole-v0/" ] }, { "cell_type": "code", "execution_count": 12, "metadata": {}, "outputs": [], "source": [ "import gym\n", "import random\n", "import numpy as np\n", "\n", "import matplotlib\n", "import matplotlib.pyplot as plt\n", "import matplotlib.animation as animation\n", "from IPython.display import display, clear_output, HTML\n", "\n", "from collections import namedtuple, deque\n", "from itertools import count\n", "from PIL import Image\n", "\n", "import torch\n", "import torch.nn as nn\n", "import torch.optim as optim\n", "import torch.nn.functional as F\n", "import torchvision.transforms as T" ] }, { "cell_type": "code", "execution_count": 2, "metadata": {}, "outputs": [], "source": [ "env = gym.make('CartPole-v1').unwrapped\n", "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "`Transition` represents a single transition in the environment, mapping (state, action) pairs to (next_state, reward) result," ] }, { "cell_type": "code", "execution_count": 3, "metadata": {}, "outputs": [], "source": [ "Transition = namedtuple('Transition', ('state', 'action', 'next_state', 'reward'))" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "`ReplayMemory` is a fixed-size buffer holding the most recent `Transition`s. It also has a `sample()` method for getting a batch of transitions for training." ] }, { "cell_type": "code", "execution_count": 4, "metadata": {}, "outputs": [], "source": [ "class ReplayMemory:\n", " def __init__(self, capacity):\n", " self.memory = deque([], maxlen=capacity)\n", " def push(self, *args):\n", " self.memory.append(Transition(*args))\n", " def sample(self, batch_size):\n", " return random.sample(self.memory, batch_size)\n", " def __len__(self):\n", " return len(self.memory)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "define the network" ] }, { "cell_type": "code", "execution_count": 5, "metadata": {}, "outputs": [], "source": [ "class DQN(nn.Module):\n", " def __init__(self, h, w, outputs):\n", " super(DQN, self).__init__()\n", " self.conv1 = nn.Conv2d(3, 16, kernel_size=5, stride=2)\n", " self.bn1 = nn.BatchNorm2d(16)\n", " self.conv2 = nn.Conv2d(16, 32, kernel_size=5, stride=2)\n", " self.bn2 = nn.BatchNorm2d(32)\n", " self.conv3 = nn.Conv2d(32, 32, kernel_size=5, stride=2)\n", " self.bn3 = nn.BatchNorm2d(32)\n", "\n", " # the dimension of the fully connected layer depends on the kernel size and input size\n", " # so compute it with this function\n", " def conv2d_size_out(size, kernel_size = 5, stride = 2):\n", " return (size - (kernel_size - 1) - 1) // stride + 1\n", " convw = conv2d_size_out(conv2d_size_out(conv2d_size_out(w)))\n", " convh = conv2d_size_out(conv2d_size_out(conv2d_size_out(h)))\n", " linear_input_size = convw * convh * 32\n", " self.head = nn.Linear(linear_input_size, outputs)\n", " def forward(self, x):\n", " x = x.to(device)\n", " x = F.relu(self.bn1(self.conv1(x)))\n", " x = F.relu(self.bn2(self.conv2(x)))\n", " x = F.relu(self.bn3(self.conv3(x)))\n", " return self.head(x.view(x.size(0), -1))" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "functions for extracting and processing renderings from the environment" ] }, { "cell_type": "code", "execution_count": 6, "metadata": {}, "outputs": [ { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXAAAADECAYAAACGNXroAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAAUoElEQVR4nO3dfZRcdX3H8fdnH0IeCCQh2xiTQEB5kFoNmgIerSIPGm0Ra6mKLYKicE6xQA8HRO0RaKXCqYr0WK2cAqZgeVAeTVGIEXzA8rBAeAwYQDCJCdlAQkiJZHfz7R/3t8nMZGd3sjs7dy7zeZ0zZ+/v3jv3fu/Mne/+5nsfRhGBmZkVT1veAZiZ2cg4gZuZFZQTuJlZQTmBm5kVlBO4mVlBOYGbmRWUE7g1nKQTJf0q7ziaiV8TGwkn8NcYSc9K2ixpU8njW3nHlTdJ50m6agyXf6ekz4zV8s0G05F3ADYmjo6In+YdRJFIEqCI2Jp3LGNBUkdE9OUdh9WXe+AtRNJ3JF1f0r5I0hJlpkpaJKlH0vo0PLtk3jslfUXSr1Ov/keS9pD0fUkbJd0naW7J/CHpNEnPSFon6V8lDbq/STpA0mJJL0p6UtJHh9iG3SVdJmm1pFUppnZJ4yQtlfT3ab52SXdJ+rKkBcAXgY+l2B8q2aYLJN0FvALsI+lTkpZJejnFfkrF+o9J69ko6WlJCyRdAPwZ8K3SbzxDbVd67W5Jy7kXeMMQ2zxe0lWSXpC0Ib3WM9K0aZKukPT79L7dlMYfJmmlpM9LWgNcIalN0jkp7hckXSdpWsl6Dk3v7wZJD0k6rOL9/+f0mr4s6XZJ06vFbA0SEX68hh7As8CRVaZNBH4DnEiWcNYBs9O0PYC/SvNMBn4A3FTy3DuBp8gSze7A42lZR5J9k/sv4IqS+QO4A5gG7Jnm/UyadiLwqzQ8CVgBfCot56AU14FVtuFG4LvpeX8E3Auckqa9GVgPvAn4EnA30J6mnQdcVbGsO4HfAX+c1t0J/HnaRgHvIUvsb0vzHwy8BBxF1vmZBRxQsqzPlCx7yO0CrgGuS/O9GVg18JoMss2nAD9K70078HZgtzTtf4Brgakp/vek8YcBfcBFwC7ABOD09JrMTuO+C1yd5p8FvAB8MG3bUandVbJ9TwP7pWXdCVyY9/7e6o/cA/Cjzm9olsA3ARtKHp8tmX4I8CLwHHDcEMuZB6wvad8JfKmk/XXgxyXto4GlJe0AFpS0/w5YkoZPZHsC/xjwy4p1fxc4d5CYZgCvAhNKxh0H3FHSPhN4kiyR71sy/jwGT+D/NMzreRNweklcF1eZ707KE3jV7UpJuJeU/NO0f6F6Av808GvgLRXjZwJbgamDPOcwYAswvmTcMuCIiuf3kv2D+TxwZcUybgNOKNm+f6x4P3+S9/7e6g/XwF+bPhxVauARcY+kZ8h6r9cNjJc0EbgYWEDWmwOYLKk9IvpT+/mSRW0epL1rxepWlAw/B7x+kJD2Ag6RtKFkXAdwZZV5O4HVWckayHqLpetZCFwAXB8RywdZRqXS5yLpA2RJdr+07InAI2nyHODWGpY5EGu17epKw5WvTzVXpnVfI2kKcBXZN4w5wIsRsb7K83oi4g8VMd0oqbTO30/2j3Ev4K8lHV0yrZPsW9SANSXDr7Dj+20N5gTeYiSdSvb1+ffA2cBX06Qzgf2BQyJijaR5wINkpYSRmgM8lob3TOustAL4eUQcVcPyVpD1wKdH9QNy3wYWAe+X9K6IGDg1r9ptN7eNl7QLcD3wSeDmiOhNNeWB12AF1WvVlcuvul2S2snKG3OAJ9LoPassl4joBc4Hzk/HGW4l+5ZxKzBN0pSI2FBjTJ+OiLsGiWkFWQ/8s9XisObjg5gtRNJ+wFeAvwWOB85OiRqyuvdmYEM6sHVuHVZ5Vjo4Ooes/nrtIPMsAvaTdLykzvT4U0lvqpwxIlYDtwNfl7RbOij3BknvSdt3PFl9+ETgNGChpIFe4vPA3GoHUpNxZP/ceoC+1Bt/X8n0y4BPSToirXuWpANKlr9PLduVvtHcAJwnaaKkA4ETqgUl6b2S/iQl/o1kZY+t6fX4MfDt9Dp3Snr3ENv3H8AFkvZKy+2SdEyadhVwtKT3KzsAPD4dCJ1ddWmWOyfw16Yfqfw88BsldZB9SC+KiIdSeeGLwJWp5/lNsoNT68gOdP2kDnHcDNwPLCU72HZZ5QwR8TJZkvw4WQ99DdsPvA3mk2SJ9nGyOvcPgZmS9kzb8MmI2BQR/w10k5WFIDsoC/CCpAcGW3CK5TSy0tJ64BPALSXT7yU7KHkx2cHMn5OVHgAuAY5NZ4L8Ww3b9TmyEsQa4HvAFVW2F+B1aTs3ktWxf872EtPxZAn9CWAtcMYQy7kkbc/tkl4me58PSdu2AjiGbJ/oIeutn4VzRFNTOiBhVleSguwg4lN5x2L2WuX/rmZmBeUEbmZWUC6hmJkV1Kh64Oky4iclPSXpnHoFZWZmwxtxDzyd0vQbsktuVwL3kV3Z93j9wjMzs2pGcyHPwcBTEfEMgKRryE5DqprAp0+fHnPnzh3FKs3MWs/999+/LiK6KsePJoHPovxS4JWkc0qrmTt3Lt3d3aNYpZlZ65E06K0WxvwsFEknS+qW1N3T0zPWqzMzaxmjSeCryO7lMGB2GlcmIi6NiPkRMb+ra4dvAGZmNkKjSeD3AftK2lvSOLJLhm8Z5jlmZlYnI66BR0SfpM+R3TO4Hbg8Ih4b5mlmZlYno7qdbETcSu33RzYzszry/cDNgP4tm8vasbW/fAaV3xa9Y9zEqtPMGsX3QjEzKygncDOzgnICNzMrKNfAzYBnfnZ5WXvjqmVl7XGTppa13/SX2+/d1rHLpLELzGwI7oGbmRWUE7iZWUE5gZuZFZRr4NbCtt8Lv/eVDWVTejdvLGurreKj4l+ysibgHriZWUE5gZuZFZQTuJlZQbkGbi1ra1/vtuH+3lfLpknlfZu2znFDTjfLg/dCM7OCcgI3Mysol1CsZW3t27JtuO8Pm4act3PCbmXttnHjxyQms53hHriZWUE5gZuZFZQTuJlZQbkGblaDtg6fRmjNx3uhmVlBOYGbmRWUE7iZWUG5Bm4tK0pvCbvD7WHL223t/qhY83EP3MysoJzAzcwKygnczKygXNizlhX9228nG7G1YqoaG4zZCAzbA5d0uaS1kh4tGTdN0mJJy9PfqWMbppmZVaqlhPI9YEHFuHOAJRGxL7Aktc3MrIGGTeAR8QvgxYrRxwAL0/BC4MP1Dcts7PX3vbrtEf19ZY8dSOUPsyYw0oOYMyJidRpeA8yoUzxmZlajUZ+FEtnVEJVXQWwj6WRJ3ZK6e3p6Rrs6MzNLRprAn5c0EyD9XVttxoi4NCLmR8T8rq6uEa7OzMwqjfQ0wluAE4AL09+b6xaRWYP0vrJx23Dlr9JX2mXy9LEOx2yn1XIa4dXA/wL7S1op6SSyxH2UpOXAkaltZmYNNGwPPCKOqzLpiDrHYmZmO8GX0puZFZQvpbfWVXYL2aonUgHQPm7C2MZiNgLugZuZFZQTuJlZQbmEYjaYil/oqfxVerNm4B64mVlBOYGbmRWUE7iZWUG5Bm4ta2tf6eXzQ59GKN9C1pqQe+BmZgXlBG5mVlBO4GZmBeUauLWs0lvIxtAlcIarkZvlwT1wM7OCcgI3MysoJ3Azs4JyDdxa1pZNL25vRH/5RJX3bTonTm1ARGY7xz1wM7OCcgI3Mysol1CsZUX/lpJG+TS1lV863zFhcgMiMts57oGbmRWUE7iZWUE5gZuZFZRr4NbChrhFbMXtY/2TataM3AM3MysoJ3Azs4JyAjczKyjXwK1lbe3r3d4Y5hfTJPd1rPkMu1dKmiPpDkmPS3pM0ulp/DRJiyUtT399swgzswaqpVvRB5wZEQcChwKnSjoQOAdYEhH7AktS28zMGmTYBB4RqyPigTT8MrAMmAUcAyxMsy0EPjxGMZqNif7eV7c9shpK6cOs+e1UYU/SXOAg4B5gRkSsTpPWADPqG5qZmQ2l5gQuaVfgeuCMiNhYOi0igio/GijpZEndkrp7enpGFayZmW1XUwKX1EmWvL8fETek0c9LmpmmzwTWDvbciLg0IuZHxPyurq56xGxmZtRwGqEkAZcByyLiGyWTbgFOAC5Mf28ekwjNxsiWl9dVnVZ56XzH+F3HOhyznVbLeeDvBI4HHpG0NI37Ilnivk7SScBzwEfHJEIzMxvUsAk8In5F9cPyR9Q3HDMzq5UvLzMzKyhfSm8ta2vpT6pVaGvvLGu3j5sw1uGY7TT3wM3MCsoJ3MysoJzAzcwKyjVwa2GlJ1eVX0istvaydltHeU3crBm4B25mVlBO4GZmBeUSirWs2NpffWLFr9L7F3msGXmvNDMrKCdwM7OCcgI3Myso18CtdUT5qYJb+0t+lX6H+7UN+vskZk3FPXAzs4JyAjczKygncDOzgnIN3FpGf9+rZe2+zZuqztveOb6srXZ/VKz5uAduZlZQTuBmZgXlBG5mVlAu7FnLiP6+snZ/7+aq83aMn1zWbu/YZUxiMhsN98DNzArKCdzMrKBcQrEWVnn5/HZtHRUfDVWf1ywv7oGbmRWUE7iZWUE5gZuZFZRr4NYy2tqGumVs+e1j29r9K/TW/NwDNzMrqGETuKTxku6V9JCkxySdn8bvLekeSU9JulbSuLEP18zMBtTSA38VODwi3grMAxZIOhS4CLg4It4IrAdOGrMozcxsB8PWwCMigIH7bnamRwCHA59I4xcC5wHfqX+I1sp6e3vL2i+99NKIl9X3yoaydvRv2TbcrvIaeG/vlrL2unXrRrzeiRMnDtk2G6maauCS2iUtBdYCi4GngQ0RMXBziZXArCrPPVlSt6Tunp6eOoRsZmZQYwKPiP6ImAfMBg4GDqh1BRFxaUTMj4j5XV1dI4vSzMx2sFOnEUbEBkl3AO8ApkjqSL3w2cCqsQjQWtvdd99d1v7IRz4y4mXt87ryOwx+9ZRjtg2PG7dn2bR7f/HLsva5n/3aiNd79tlnl7XPOuusES/LrFQtZ6F0SZqShicARwHLgDuAY9NsJwA3j1GMZmY2iFp64DOBhZLayRL+dRGxSNLjwDWSvgI8CFw2hnGamVmFWs5CeRg4aJDxz5DVw83MLAe+lN6a2pYt9Tudb4/d9ihrP/jKsduG+zfvWjbtkWefq9t6N23aNPxMZiPgS+nNzArKCdzMrKCcwM3MCso1cGtqHZU/bTYa7eWXsKtzyvb1tI0vm7apd0LdVlvXbTAr4R64mVlBOYGbmRWUE7iZWUE1tDi3efNmHn744Uau0gpu+fLldVvWpvXly/rlbeduG+5jUtm0Vb/9Wd3Wu3r16rK2PwNWL+6Bm5kVlBO4mVlBNbSE0tHRge8JbjtjypQpdVvWqnXll7Svuu36ui17KJMmlZdn/BmwenEP3MysoJzAzcwKygnczKygGloD7+zsZObMmY1cpRXc9OnT8w5h1CZPLv8pN38GrF7cAzczKygncDOzgnICNzMrKN/n0ppaX19f3iGMWm9vb94h2GuUe+BmZgXlBG5mVlBO4GZmBeUauDW1yvPAjzzyyJwiGbn99tsv7xDsNco9cDOzgnICNzMrKJdQrKnNmzevrL148eJ8AjFrQu6Bm5kVlBO4mVlBOYGbmRWUIqJxK5N6gOeA6cC6hq24No6pNo6pds0Yl2OqTbPFtFdE7PBbfA1N4NtWKnVHxPyGr3gIjqk2jql2zRiXY6pNM8Y0GJdQzMwKygnczKyg8krgl+a03qE4pto4pto1Y1yOqTbNGNMOcqmBm5nZ6LmEYmZWUA1N4JIWSHpS0lOSzmnkuiviuFzSWkmPloybJmmxpOXp79QGxzRH0h2SHpf0mKTT845L0nhJ90p6KMV0fhq/t6R70vt4raRxjYqpJLZ2SQ9KWtQMMUl6VtIjkpZK6k7j8t6npkj6oaQnJC2T9I4miGn/9BoNPDZKOqMJ4vqHtI8/KunqtO/nvp8Pp2EJXFI78O/AB4ADgeMkHdio9Vf4HrCgYtw5wJKI2BdYktqN1AecGREHAocCp6bXJ8+4XgUOj4i3AvOABZIOBS4CLo6INwLrgZMaGNOA04FlJe1miOm9ETGv5PSzvPepS4CfRMQBwFvJXq9cY4qIJ9NrNA94O/AKcGOecUmaBZwGzI+INwPtwMdpjn1qaBHRkAfwDuC2kvYXgC80av2DxDMXeLSk/SQwMw3PBJ7MK7YUw83AUc0SFzAReAA4hOwCh47B3tcGxTKb7EN+OLAIUBPE9CwwvWJcbu8dsDvwW9JxrmaIaZAY3wfclXdcwCxgBTCN7AZ/i4D3571P1fJoZAll4EUasDKNaxYzImJ1Gl4DzMgrEElzgYOAe/KOK5UqlgJrgcXA08CGiBj4teE83sdvAmcDW1N7jyaIKYDbJd0v6eQ0Ls/3bm+gB7gilZr+U9KknGOq9HHg6jScW1wRsQr4GvA7YDXwEnA/+e9Tw/JBzEFE9i83l9NzJO0KXA+cEREb844rIvoj+7o7GzgYOKCR668k6S+AtRFxf55xDOJdEfE2shLhqZLeXToxh/euA3gb8J2IOAj4PyrKEjnv5+OADwE/qJzW6LhSvf0Ysn96rwcmsWOJtSk1MoGvAuaUtGencc3ieUkzAdLftY0OQFInWfL+fkTc0CxxAUTEBuAOsq+SUyQN3Eu+0e/jO4EPSXoWuIasjHJJzjEN9OKIiLVkNd2Dyfe9WwmsjIh7UvuHZAm9KfYnsn90D0TE86mdZ1xHAr+NiJ6I6AVuINvPct2natHIBH4fsG86sjuO7OvTLQ1c/3BuAU5IwyeQ1aAbRpKAy4BlEfGNZohLUpekKWl4AllNfhlZIj82j5gi4gsRMTsi5pLtQz+LiL/JMyZJkyRNHhgmq+0+So7vXUSsAVZI2j+NOgJ4PM+YKhzH9vIJ5BvX74BDJU1Mn8OB1yq3fapmjSy4Ax8EfkNWR/1SXoV/sh1nNdBL1lM5iayOugRYDvwUmNbgmN5F9rXxYWBpenwwz7iAtwAPppgeBb6cxu8D3As8RfYVeJec3sfDgEV5x5TW/VB6PDawbzfBPjUP6E7v303A1LxjSnFNAl4Adi8Zl/drdT7wRNrPrwR2aZb9fKiHr8Q0MysoH8Q0MysoJ3Azs4JyAjczKygncDOzgnICNzMrKCdwM7OCcgI3MysoJ3Azs4L6f7uR+BXil646AAAAAElFTkSuQmCC", "text/plain": [ "
" ] }, "metadata": { "needs_background": "light" }, "output_type": "display_data" } ], "source": [ "resize = T.Compose([T.ToPILImage(),\n", " T.Resize(40, interpolation=T.InterpolationMode.BICUBIC),\n", " T.ToTensor()])\n", "\n", "def get_cart_location(screen_width):\n", " world_width = env.x_threshold * 2\n", " scale = screen_width / world_width\n", " return int(env.state[0] * scale + screen_width / 2.0) # MIDDLE OF CART\n", "\n", "def get_screen():\n", " # Returned screen requested by gym is 400x600x3, but is sometimes larger\n", " # such as 800x1200x3. Transpose it into torch order (CHW).\n", " screen = env.render(mode='rgb_array').transpose((2, 0, 1))\n", " # Cart is in the lower half, so strip off the top and bottom of the screen\n", " _, screen_height, screen_width = screen.shape\n", " screen = screen[:, int(screen_height*0.4):int(screen_height * 0.8)]\n", " view_width = int(screen_width * 0.6)\n", " cart_location = get_cart_location(screen_width)\n", " if cart_location < view_width // 2:\n", " slice_range = slice(view_width)\n", " elif cart_location > (screen_width - view_width // 2):\n", " slice_range = slice(-view_width, None)\n", " else:\n", " slice_range = slice(cart_location - view_width // 2,\n", " cart_location + view_width // 2)\n", " # Strip off the edges, so that we have a square image centered on a cart\n", " screen = screen[:, :, slice_range]\n", " # Convert to float, rescale, convert to torch tensor\n", " # (this doesn't require a copy)\n", " screen = np.ascontiguousarray(screen, dtype=np.float32) / 255\n", " screen = torch.from_numpy(screen)\n", " # Resize, and add a batch dimension (BCHW)\n", " return resize(screen).unsqueeze(0).to(device)\n", "\n", "env.reset()\n", "plt.figure()\n", "plt.imshow(get_screen().cpu().squeeze(0).permute(1, 2, 0).numpy(),\n", " interpolation='none')\n", "plt.title('Example extracted screen')\n", "plt.show()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "instantiate the model & optimizer, and define utility functions" ] }, { "cell_type": "code", "execution_count": 7, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "number of parameters: 40866\n" ] } ], "source": [ "BATCH_SIZE = 128\n", "GAMMA = 0.999\n", "EPS_START = 0.9\n", "EPS_END = 0.05\n", "EPS_DECAY = 200\n", "TARGET_UPDATE = 10\n", "\n", "# Get screen size so that we can initialize layers correctly based on shape\n", "# returned from AI gym. Typical dimensions at this point are close to 3x40x90\n", "# which is the result of a clamped and down-scaled render buffer in get_screen()\n", "init_screen = get_screen()\n", "_, _, screen_height, screen_width = init_screen.shape\n", "\n", "# Get number of actions from gym action space\n", "n_actions = env.action_space.n\n", "\n", "policy_net = DQN(screen_height, screen_width, n_actions).to(device)\n", "target_net = DQN(screen_height, screen_width, n_actions).to(device)\n", "target_net.load_state_dict(policy_net.state_dict())\n", "target_net.eval()\n", "\n", "print(f\"number of parameters: {sum(p.numel() for p in policy_net.parameters())}\")\n", "\n", "optimizer = optim.RMSprop(policy_net.parameters())\n", "memory = ReplayMemory(10000)\n", "\n", "steps_done = 0\n", "\n", "def select_action(state):\n", " global steps_done\n", " sample = random.random()\n", " # change slowly from exploration to exploitation\n", " # uses the epsilon greedy policy\n", " eps_threshold = EPS_END + (EPS_START - EPS_END) * \\\n", " np.exp(-1. * steps_done / EPS_DECAY)\n", " steps_done += 1\n", " if sample > eps_threshold:\n", " # action with highest Q value\n", " with torch.no_grad():\n", " return policy_net(state).max(1)[1].view(1, 1)\n", " else:\n", " # random action\n", " return torch.tensor([[random.randrange(n_actions)]], device=device, dtype=torch.long)\n", "\n", "episode_durations = []\n", "def plot_durations(ax, fig):\n", " durations_t = torch.tensor(episode_durations, dtype=torch.float)\n", " ax.cla()\n", " ax.set_title('Training...')\n", " ax.set_xlabel('Episode')\n", " ax.set_ylabel('Duration')\n", " ax.plot(durations_t.numpy())\n", " # Take 100 episode averages and plot them too\n", " if len(durations_t) >= 100:\n", " means = durations_t.unfold(0, 100, 1).mean(1).view(-1)\n", " means = torch.cat((torch.zeros(99), means))\n", " ax.plot(means.numpy())\n", " display(fig)\n", " clear_output(wait=True)\n", " plt.pause(0.01) # pause a bit so that plots are updated\n", " " ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# training loop\n", "\n", "Huber loss is used; acts like mean squared error when error is small, and like mean absolute error when error is large, which for some reason makes it more robust to outliers when the values of the Q function is noisy.\n", "![](https://upload.wikimedia.org/wikipedia/commons/thumb/c/cc/Huber_loss.svg/450px-Huber_loss.svg.png)\n" ] }, { "cell_type": "code", "execution_count": 8, "metadata": {}, "outputs": [], "source": [ "def optimize_model():\n", " if len(memory) < BATCH_SIZE:\n", " return\n", " transitions = memory.sample(BATCH_SIZE)\n", " # Transpose the batch (see https://stackoverflow.com/a/19343/3343043 for\n", " # detailed explanation). This converts batch-array of Transitions\n", " # to Transition of batch-arrays.\n", " batch = Transition(*zip(*transitions))\n", "\n", " # Compute a mask of non-final states and concatenate the batch elements\n", " # (a final state would've been the one after which simulation ended)\n", " non_final_mask = torch.tensor(tuple(map(lambda s: s is not None,\n", " batch.next_state)), device=device, dtype=torch.bool)\n", " non_final_next_states = torch.cat([s for s in batch.next_state\n", " if s is not None])\n", " state_batch = torch.cat(batch.state)\n", " action_batch = torch.cat(batch.action)\n", " reward_batch = torch.cat(batch.reward)\n", "\n", " # Compute Q(s_t, a) - the model computes Q(s_t), then we select the\n", " # columns of actions taken. These are the actions which would've been taken\n", " # for each batch state according to policy_net\n", " state_action_values = policy_net(state_batch).gather(1, action_batch)\n", "\n", " # Compute V(s_{t+1}) for all next states.\n", " # Expected values of actions for non_final_next_states are computed based\n", " # on the \"older\" target_net; selecting their best reward with max(1)[0].\n", " # This is merged based on the mask, such that we'll have either the expected\n", " # state value or 0 in case the state was final.\n", " next_state_values = torch.zeros(BATCH_SIZE, device=device)\n", " next_state_values[non_final_mask] = target_net(non_final_next_states).max(1)[0].detach()\n", " # Compute the expected Q values\n", " expected_state_action_values = (next_state_values * GAMMA) + reward_batch\n", "\n", " # Compute Huber loss\n", " loss = F.smooth_l1_loss(state_action_values, expected_state_action_values.unsqueeze(1))\n", "\n", " # Optimize the model\n", " optimizer.zero_grad()\n", " loss.backward()\n", " for param in policy_net.parameters():\n", " param.grad.data.clamp_(-1, 1)\n", " optimizer.step()" ] }, { "cell_type": "code", "execution_count": 9, "metadata": {}, "outputs": [ { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEWCAYAAACJ0YulAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAABoCklEQVR4nO2dd5xU1dmAn3dmtsOydJCOoKCigqgYe4kl1i/22JIYNV8SY5otmma6KZquJPqpibFrYom9G0UEAUVBQDrS2/YyM+f745Y59869s3fLzO6y5/n9YGfu3HLuzLnnPW89opTCYDAYDAaAWFc3wGAwGAzdByMUDAaDweBihILBYDAYXIxQMBgMBoOLEQoGg8FgcDFCwWAwGAwuRigYDG1ARJ4WkUs6e1+DobsgJk/BsKsjIrXa23KgCUjZ769QSt1b+FYZDN0TIxQMvQoRWQl8SSn1QsBnCaVUsvCtMhi6D8Z8ZOi1iMhRIrJWRK4VkQ3A/4lIfxF5UkQ2i8h2+/VI7ZhXRORL9uvPi8gbIvJre98VInJSO/cdJyKviUiNiLwgIn8SkX8U8OswGAAjFAyGYcAAYAxwOdYz8X/2+9FAA/DHHMcfDHwEDAJuBu4QEWnHvv8EZgMDgR8CF7X7jgyGDmCEgqG3kwZ+oJRqUko1KKW2KqUeUUrVK6VqgJ8CR+Y4fpVS6q9KqRRwNzAcGNqWfUVkNHAg8H2lVLNS6g3g8c66QYOhLRihYOjtbFZKNTpvRKRcRG4XkVUiUg28BlSJSDzk+A3OC6VUvf2yTxv33Q3Ypm0DWNPG+zAYOgUjFAy9HX+kxbeBPYGDlVKVwBH29jCTUGewHhggIuXatlF5vJ7BEIoRCgaDl75YfoQdIjIA+EG+L6iUWgXMAX4oIsUicghwar6vazAEYYSCweDlVqAM2ALMAp4p0HUvAA4BtgI/AR7AyqcArFwLETncfn24nnshIt8VkacL1E7DLo7JUzAYuiEi8gCwWCmVd03FYNAxmoLB0A0QkQNFZHcRiYnIicDpwL+6uFmGXkiiqxtgMBgAK1/iUaw8hbXA/yql5nVtkwy9EWM+MhgMBoOLMR8ZDAaDwaVHm48GDRqkxo4d29XNMBgMhh7F3LlztyilBgd91qOFwtixY5kzZ05XN8NgMBh6FCKyKuwzYz4yGAwGg4sRCgaDwWBwMULBYDAYDC5GKBgMBoPBJW9CQUTuFJFNIrIw4LNvi4gSkUH2exGR34vIMhF5T0Sm5atdBoPBYAgnn5rCXcCJ/o0iMgo4HlitbT4JmGj/uxz4Sx7bZTAYDIYQ8iYUlFKvAdsCProFuAZvHfvTgXuUxSysRU2G56ttBoPBYAimoD4FETkdWKeUWuD7aATelabW2tuCznG5iMwRkTmbN2/OU0sNBsOuwo76Zp5875OubkaPoWDJa/aqUt/FMh21G6XUTGAmwPTp003hJoPBkJOv/XMebyzbwv6jqhjZv7z1A3o5hcxo3h0YBywQEYCRwLsichCwDu/ygyPtbQaDwdAh1m63lr5uTqa7uCU9g4KZj5RS7yulhiilxiqlxmKZiKYppTYAjwMX21FIM4CdSqn1hWqbwWDYNZm1fCsrt1pCwZ6MGlohnyGp9wFvAXuKyFoRuTTH7v8BlgPLgL8CX8lXuwwGQ+/hvJmz3NdGJEQjb+YjpdT5rXw+VnutgK/mqy0Gg8FgFIVomIxmg8HQKxCjK0TCCAWDwdArMJpCNIxQMBgMBoOLEQoGg6FXYDSFaBihYDAYDAYXIxQMBkOvQJn6B5EwQsFgMPQKjFCIhhEKBoOhV6AwUiEKRigYDIZegdEUomGEgsFg6BWkjVSIhBEKBoOhV2BEQjSMUDB0K95fu5M12+q7uhmGXRCjKESjkOspGAytcuof3wBg5S9O7uKWGHY9jFSIgtEUDAZDryBtZEIkjFAwGAy9AmM+ioYRCgaDoVdg8hSiYYSCwWDoFRhNIRpGKBgMhl6ByVOIhhEKBoOhV2BkQjTyJhRE5E4R2SQiC7VtvxKRxSLynog8JiJV2mfXi8gyEflIRE7IV7sMBoPBEE4+NYW7gBN9254H9lFK7QssAa4HEJG9gPOAve1j/iwi8Ty2zWAw9DKMphCNvAkFpdRrwDbftueUUkn77SxgpP36dOB+pVSTUmoFsAw4KF9tMxgMvQ/jU4hGV/oUvgg8bb8eAazRPltrb8tCRC4XkTkiMmfz5s15bqLBYNhVMCIhGl0iFETkBiAJ3NvWY5VSM5VS05VS0wcPHtz5jTMYDLskymgKkSh47SMR+TxwCnCsyvxK64BR2m4j7W0Gg8HQKRiREI2CagoiciJwDXCaUkovhfk4cJ6IlIjIOGAiMLuQbTPkl8aWFDvrW7q6GYZejNEUopHPkNT7gLeAPUVkrYhcCvwR6As8LyLzReQ2AKXUB8CDwIfAM8BXlVKpfLXNUHh++/wSLrhjVlc3w9CLMTIhGnkzHymlzg/YfEeO/X8K/DRf7TF0LVtqm9hS09zVzTD0YoxMiIbJaDYUBgVJU7vY0IUYTSEaRigYCoLCxIkbuhbT/6JhhIKhICilSKbSXd0MQy/GyIRoGKFgKAiWptDVrTD0Zsx6CtEwQsFQEJSCZNpoCoYuxMiESBihYCgICkgZVcHQhZjuFw0jFAwFQSllhIIhkEffXcurS/Jfx8yYj6JR8DIXht6JUtZMTSmFiHR1cwzdiG89uACAlb84Oa/XMY7maBhNwVAQnFma0RYMXYXpedEwQsEQmfU7G/jM715nU3Vjm491Zmkmgc3QVZg8hWgYoWCIzN/fWsWH66t5cM6a1nf24TyP5sE0tIXXl26msaWTyqCZrhcJIxQMbaY9PgHHfGQ0BUNUNtU0ctEds3l64fpOOZ9xNEfDCAVDZDrySLmaghEKhog0tVh5LbM+3sbY657i4821HTqfUVKjYYSCoSA4z6PRFAxt5bF51npbr3zUsbBV0/WiYYSCITIdmWkZTcHQVhz/U6qTpvhmkZ1oGKFgiIxjk21fmoHxKRjaRmcHJ5ieFw0jFAxtRmiHo9l+Ik2egiEqjjDorAl+IRSFNz/ewu2vfpz/C+URk9FsiE6Eh2pzTRNppRhaWRp4qBEKhqh0dk8phPnoc399G4Arjtw979fKF0YoGCLjPFK5zEcH/vQFILtkgfNA5jIfGZuvQcffHzraP0zvioYxHxki4zyU7XEpOA9kLvuwkQkGnc7uD6Z/RSNvQkFE7hSRTSKyUNs2QESeF5Gl9t/+9nYRkd+LyDIReU9EpuWrXYauwS1zkcohFArUFkPPoLMtjSZ5LRr51BTuAk70bbsOeFEpNRF40X4PcBIw0f53OfCXPLbL0AVE0xTMQ2vI0NmDuHFnRSNvQkEp9Rqwzbf5dOBu+/XdwBna9nuUxSygSkSG56tthvbhjNntCUmN5FNoT6MMuyydsVCf3lfNpCMahfYpDFVKOYVMNgBD7dcjAL3K2lp7WxYicrmIzBGROZs3539hDkMG19Hs8yqk0oo/vLiUnfUt4cdGCEk1z6xBpzM0hZhZu6PNdFn0kVJKiUibf3Wl1ExgJsD06dPNMNINeGnxJn7z/BJWbasP3SfKegrG5mvQ8U8S2jNp0EWCmXREo9CawkbHLGT/3WRvXweM0vYbaW8zdCPCzEcNdmnjXCWOM+sphNsEzENr0OmM/qBrCqZsezQKLRQeBy6xX18C/FvbfrEdhTQD2KmZmQzdhLCZfMoe6BOxcFU9U/so+vX++tpyHpu3NvoBhl2KThnEPT6Fjp+uN5A385GI3AccBQwSkbXAD4BfAA+KyKXAKuAce/f/AJ8BlgH1wBfy1S5Dx/Gvp+CEmcZyCQW39lG4VPAPAj/9zyIA/mfqyHa1sz28u3o7+47oRyJuUni6ms4Yw/UuaWRCNPImFJRS54d8dGzAvgr4ar7aYugcwmZazmAeSVPoxslrC9ft5LN/fpP/PWp3rj1xUtc2xtApmoIeFFHI6COlVLsWo+oOmOmQoc34u7oTZhrPqSnY+3bj5LVNNdba04vXV3dxSwwQ4GhuRw+RdmgKa7bVM/76p1iysabN13Ov1dWduQMYoWDoMKkIQoFImkIPfpIMnU5n9Afd0Rz1fP95fz1pBQ/Pbb8/qyf3ZCMUDG3GrxU7s/9ELLw7RVmjuasfJCOTuhed8XN0VUhqT450MkLBEJmwmZajKeRKFDLJa4a24l+lrz39Q4W8zjc9WSiY0tmGyGQymr04yyUm4q37FHKup9DFz1EP9QvusnSkVtHHm2t57oONnm2FHKd7sEwwQsHQdvxRFdE0he6f0dyTH+RdkY70h3Nvn8WW2ibPtqiz987oBj1ZUzDmI0Nkwvq541PI9RBH0RR68HNkyAPZ0UfRqW9OAlCSyAxxhTUfFfBinYwRCobIOIO+XyFwzEe5nrpMmYvu62g25qPuRWdMEkqL4kwa1rfzThgRoykYegVu7SPfdqfMRa7HoCesp9CDn+Ndko4MrJmyKooRVWXWts5oVNTrd0LZ767CCAVD28nyKTh/Wx/woyavdbWAMHQ9HekBblVepdzyK/5opnxiNAVDr8bRFFpSrVdAjVrmoifbZA2dg7+vtGecTaWVW//IhKRGwwgFQ2TCurnjJ2hO5hAKkZLXMp/lEjD5pqfWrNnl6MC4qk9CnKi4wiavFe5anY0RCobIhPsUIgiFCMlr+iCQc788Y0xX3YPOmG1bmoItFDp8tuj05D5khIKhwziz+qYI5qPceQoZcmkUht5BZxTES6tMSfdCDtQ9ufsaoWBoA0r7P0NzMor5yCJqnkIhNIVVW+v42X8WZQ0WxnzUPehQ9JH22vUpmNpHkTBCwRAZt5/7OryjKeQ2H7UtoznXYjydxRV/n8vM15azbFNt3q9lCCbX7L1Dw6p2sLibCjdQ91yRYISCoR34O3wUoeCQyyyULrCm0JXObINFrgl1Z5l7usTR3IPtR0YoGCKjJwRd98h7/PGlpYAmFDockqppCjnyGToLt8CfMRd1GTkTHv0+hTZ0CV0rcPMUTEG8SJiCeIbIKM2ncP87awD42jETaU61ISQ1V/JaoaOPnGgqIxO6jFzaQEe6gH7aTJ5C207YkW7Rk30KkYSCiAwGLgPG6scopb7YnouKyDeBL2E9lu8DXwCGA/cDA4G5wEVKqeb2nN+QX/z9vSUZxadg/Y36sBQi+sjVFPJ+JUMYuUuj9GDzUQ8WClHNR/8G+gEvAE9p/9qMiIwAvg5MV0rtA8SB84BfArcopSYA24FL23N+Q/4J9SnkMh/Zf3M5kAutKTgPrnTBoGGwyDV4dlYX6ArzYA92KUQWCuVKqWuVUg8qpR5x/nXgugmgTEQSQDmwHjgGeNj+/G7gjA6cf5dn2abaDq0h2x7cYqi+B9mpkhot+ijH+QscfRSWjGcoHO11NG+uaeJvry8P3UffGrdHuUI6f3tD8tqTIvKZzrigUmod8GtgNZYw2IllLtqhlErau60FRgQdLyKXi8gcEZmzefPmzmhSj+T4W17lOw8tKOg1QyJS3VlRU6Q8he6jKfhLgRvfQvci17j69fvm8ZOnFrFkY+vhxF2R0dwbNIWrsARDo4jU2P+q23NBEekPnA6MA3YDKoATox6vlJqplJqulJo+ePDg9jRhl6ArOl1m2YTgsJCcIZ5uRnOruwAF8il0ILrF0Dnk1BRyDOM7GlqAcI1Sn6kbn0LbiORoVkr17cRrHgesUEptBhCRR4FDgSoRSdjawkhgXSde09CJ+Mdr530uoZB2zUetm5is/QonFHryrK6nk2vg93eVtphk9D2lndFHHWGXFwoAInIacIT99hWl1JPtvOZqYIaIlAMNwLHAHOBl4CysCKRLsJzbhm6EG5KaJRTscNMIdY1ypR94NIVC5Cko5flrzEeFJ5dAzlU6uy0CwtEUTJ5CNCKZj0TkF1gmpA/tf1eJyM/bc0Gl1NtYDuV3scJRY8BM4FrgWyKyDCss9Y72nN+QR0LMR87DloxUEK/7+BQcnEv15Ae5p9KWMhdBXULaEiZgluOMRFRN4TPA/kpZi8yJyN3APOD69lxUKfUD4Ae+zcuBg9pzvt6MUqrgIXfZtnhrQ+6q2K0nr+nDQEshoo+cvz34Ae7p5OwNfk2hDeYf/VDB0gKjHt2W7rBqax2n/P4NnrjyMMYOqnC392STZFvKXFRpr/t1cjsM7aQrxrOshzVCG9q68lqqAOYjpy2ucMj7FQ1+coeket8HagoR5kMilgkpH8/KI++uo6YpyWPzvC7QnjzRiKop/ByYJyIvYwneI4Dr8tYqQ2QK2fXCQ1Jbb4WzSxS/Q2v7dRZ+QeUms+X9ygaHtpS50Pdty5grYhmZ8mHScXIf4jFvr+nJmkLU6KP7ROQV4EB707VKqQ15a5UhMmmliBdoGFO+mbW7vQ3n6E7rKfiFXCbk1lAo2hKS6nE0+3JMctFW81FbcASNXyj0ZE0hp/lIRCbZf6dh1SZaa//bzd5m6GK6wqHVPk3BCUnNpSlkPitkRnOm/T33Qe6p5PrGs0Of2/n7iOWQzsej4mTz+4XTrqwpfAu4HPhNwGcKqzSFoQsppExwLpUrVLC1Y7uVpuCGpFrvnUsa81HhyDmj9n2mdwm/dpcLQWxNIY/mI/Gbj3quVMgpFJRSl9svT1JKNeqfiUhp3lpliExBhUKIeSXoAfBHRWVCUqMJhUJWSW3LAGPoXNqiKeiDepRJhoOIbT7Kh6ZgK7TZPoWe25miRh+9GXGboRO4f/ZqfvfC0kj7FrLzpUNGz6AmhD3QqRzt1e+lkJqC39FsKBxRF12y3gft0/o1BMd8lAdNwT5nTPw+hU6/VMHIqSmIyDCswnRlIjKVjGZdiVXd1JAHrnv0fQCuOm5iq/sWciALKwsRpinohpgwTWFnfQtrttezz4h+nu3JkKiOzsTvSejBz3HPJceXniv6KLNP12oKGaHQ9nZ1V1rzKZwAfB6rFtFvte01wHfz1CZDGyhk18vE9UeYwYW89wuFC+6YxcJ11az8xcm+PAVLL8+nfd/vaO7JESM9lVzfeLaZMnufSEIBsfIU2tQyInW+sOijXdbRrJS6G7hbRM7s4PoJhjyhCrj2fGbw9LUhwgwuTFNYuM4qtptMpT3CxtnNr5Z3Jn5Hs5EJhSe3n9nvaFZZn0UdfPOVp+D4FGK7kE8hap7CIyJyMrA3UKptvylfDTNEoyvMR1FmcNnNsjaEOZCbU2nPMYW4r4yj2asB9dzHuecRNcPd/97/2+VCLKdCZKHfliilsOijnqx1Ri2IdxtwLnAlltA9GxiTx3YZIlJY85HzN3wGF4ZrqvEJhYQ9w2pOpj334p4yr/Yju01u2/J4LUMguaOP/NpmkEba+jVsmZAXwhzNPbkvRY0++pRS6mJgu1LqR8AhwB75a5YhKoXVFILNR1FsvWHltRNxTShoxxSi5ITfl1AIOWTw0u4qqSETlEBEiMXyE30UFk3Xk81HUYWCk6NQLyK7AS1YGc6GLqYrQlKzw0WDoo+C9/AfW2QvoNvk0xRyha52Fs4VMqWze+6D3FXsbGjhh49/QGNLKud+C9ft5PZXP87anusrz9IUAvpZNEez41NodddW25R1/bTj2/C3tecSVSg8ISJVwK+w1kFYCfwzT20ytIFCjmPOQ+Uf2AN9Cv73Icd6hILuU7D3y2dV8IyPJFgDMrTOrS8s4a43V/Lw3LU59zvlD2/w86cXt+ncuTRSV6BHMNNYIakS2VcQNDlYv7OBhuZswecU840SPttTaFUoiEgMeFEptcOOQBoDTFJKfT/vrTO0SmGFgmMCSgdu18lOPLK1DN923aegi5Jcazl3Fn5hUMjlGncVnPUx2quxti15rb2aglUlNWoTgyY5h/z8JS6+8+3Q6/v7dU8OSW1VKNgL6/xJe9+klNqZ11YZItMV0Uf+ATtaRrNzbLCm4I8+cguN5dHCn52nkLdL7fK097vLHZLa+r7Rk9ei5ymEnfKdlduzr5/2+qXa0q7uSlTz0YsicqYUeokvQ6t0hU/BH0EU2IYQt0O2ULC6VFNLynOI/xr5wLlCWKa2If/kjj7yv89sCAt6CMItnR3xWWnLMxXmZ+vJfSmqULgCeAhoEpFqEakRkeo8tssQka4xH/nV+ux9s7Ke7b/+hyfRiqaQV/yagjEf5Z0oJiGHXNV4Vcg+QYjQJvORynoRjqM19yqfAoBSqq9SKqaUKlZKVdrvK9t7URGpEpGHRWSxiCwSkUNEZICIPC8iS+2//dt7/t5EVziao+QphD0kfn+EJ09BD0lN53+g9i/H2ZNnd11FW20H2bP/zOtnFq5n7HVPsb2uGYha5iJKG8VT++iahxdw3sy3QvcPW0wqiDZpzz2EqMlrRwT968B1fwc8o5SaBOwHLMJa3vNFpdRE4EXMcp+RKFTnW7u9ntkrtgHZs/1ATSEkRM8fLVKcCAlJdUP92t3kVsmYjxxPc899kHsK2f018/6vr68AYNnmWuuTHFqF3x+kM3fVtqxtMS366ME5a5m1fBurttYFttGfaJlr1u/00yxHcx4CJbbUNvH3t1Z2/ol9RF2j+WrtdSlwEDCXdiyyIyL9sNZ4/jyAUqoZaBaR04Gj7N3uBl4Brm3r+XsbhRIKF90x232dHWmR3YasR9/ekFtTyGx3r5FPoeDO8vJ+KYNNLpOQUz4oTEYH/T5BA/aZf8nWAvQ8hfLiOPXNKeav2cGYgRWhbYyiQYaVXc/Hc/n1++bx5sdbOWT3gUwY0rfTz+8QtfbRqfp7ERkF3NrOa44DNgP/JyL7YQmXq4ChSqn19j4bgKHtPH+volAD2ZbaJvd1KhX+YDuEJR6lFZ4FeFyfQtJXEC8kKagzUb6/hXBu93ZyDfROpJl/nQsHj6NZ60+t4UYf+YRCU0vwdN7vr8jVB8PMR/notttss1pLKr/9NKqj2c9aYHI7j00A04C/KKWmAnX4TEXK6hWBdy4il4vIHBGZs3nz5nY2oWcQxVlVKIdWQqsC6dcUAu3+WT6FzGvd/FTsJq+lPMekfLO1fJAVkprHaxksIpkefX8dgpbjjJqnYJ3P2jcR0/pcAP4Q5VwLPoWZOfMxmXFOme8Y0Eiagoj8gcxvFAP2x8psbg9rgbVKKScT5GEsobBRRIYrpdaLyHBgU9DBSqmZwEyA6dOn79LPcSqt3NpAYRRqcuvM6CHIqZa9f7jlGDbWNDGiqsw+rx2SmuVTcM7dvhusaWxhW11zoHkgq63uAGP9NYHX+SOKmSVswA+aAEVdjjMWw+2EztoHTckQTcE1X3mFQxCO6bEQIamZemD57aBRNYU5WGaeucBbwLVKqQvbc0Gl1AZgjYjsaW86FvgQeBy4xN52CfDv9px/VyJKWGahfKO6puAPSQ2OPsq2EzinOPQXL2nnzYSkepfjtJ629t7fmX95kyN/9UqkfTMPf+uDgKFjZEelZe/jap5R9o1iPsIaSJ3+pQc3BOGPPso1MUm6/TT/PgWHbqEpKKXuFpHB9uvOsNlcCdwrIsXAcuALWALqQRG5FFgFnNMJ1+nRROlXhXI06xpLLmdh2DaFIhGL0exLh84kr/kczdpuug8iKks21kbe18iAwpEdlZbtJwirgOpNXgveJwh3OU77vTM5aQop4ucPvc41OUuGRB/lo08Vqp+2tkazAD8AvoY1aIuIJIE/dGSBHaXUfGB6wEfHtvecuyJRVONCCYWiWEap1NsV5tMIsgencsTpNae85iP/AJDP2ZHfoWjMR/kjl6aQNZGIoClEW0/BW/vIOaZ185H9N0d4aYtb+8l/jnz4FKxz5nHZcuv8rXz+TeBQ4ECl1AClVH/gYOBQEflmfptm6E7mI30NWl0ohD2UUTNXnYHYn7zmETxtbm3bSPsHAaM65I1ICzK5+7Z+bFRNQV+j2TkmTChkJgmtX6PFVmmz/Gx5cCoUql+2JhQuAs5XSq1wNiillgMXAhfns2GGaB2rYD6FeLCmEPbA5Ao9rCiOa8dbf5uS3tpHukDMtzbU1vV+De0nWja8Y9P3fvbchxsZe91TnhLWkWfkkl2nKEwoZIhgPnKEgm+fHz7xIWff9ma0trWRfPfT1oRCkVJqi3+j7Vcoyk+TDA5O592ws5HbXv243aWDO4MwR3NkoaDg0sPGATB5eKW2XXtQtWPSHhNVu5sdCVdDwJiP8k0uk5DzvbemsW2sbmyzIBfI+CpcoZA7JDUdIQIu5fbf7M+Cqqp2BL+mky9aEwrN7fzM0Ak4He6r/3yXXzy9mI83ZztPu9rRHOYmCMpdqChJcPjEQT4twPrbklKeY7zmozxrCmaRnU5le12zO4P2E2WFMmdbmBZQ05hsY0E88SyyE9mnEKFfpDq4nkRb8Gfg54vWhMJ+dlVU/78aYEp+m2Zwfvy6piQQnMlYqHFMX5g8qbXDX7bCwZtoZM/A7fOkAzSNVFp5Hr6gSJP2EMUE55a5MFKhw7Sk0kz98fPc8NjCwM+zk9fCf+ewn666saXVfXSsfqclozk+hZCMZr9PIWfymiqgULD/5vtaOaOPlFLxXJ8b8ou70Iw9IEdZ4SxftGgzP28+gfU6JsGCwHpt/RWxHNZBmsJj89bxyY6GrPPqx7eHtFLEWkn2MYvsdB7N9uz78QWf8Muz9s36PMrA31op8+oGTShETF7T8xRaMx9lNBVve4JIhSSv5YO2hOF2hPaWuTAUgLQ24EL7Q/I6g2ZN1dYfAOd1ScI7f1ABrwWxNQXtM+2m3l6RqW6pK0VppVizrZ51mtCISqQIroA2G9qH+1uHyOFcVVLdmXArFXJ36EIhivkIPKWzW4s+ypXMGJak1tCSYsGaHa22JQo1jS18+En2cjVtqffUEYxQ6MY4HS6WQ1MoVBE3PeksFaApOFmiDl5NIePAjYn3PsIeao+jGTj85pc9mdBRiTKpCivAZmg7mVIMYZ973wd95U6fCvs5dtS3ZOUc6BT5SsOI2JORiNFH/qq5+iTIn83vfPbou+s4/U//9XzWr6x9sTgX3zmbz/z+9aztRlMweEwzEK3GUL5oSaU5db/dGDuwPPAhyRYK2mv7r2Cbj3SfQojTLErYaxSiqPUmP6HzcByvYRnoURzNad9M3c+OhmbNCZy9T2VpEWcfMNJ9LwIlRTFXCLiO5pCM5syMPHuy0OwTJLkmZQP7FId+lot5q3cEntsVCnmeCBqh0I3JZNhaD1jQAFeo2W1zMk1FcTxrUM+Yj3xCQX+t+RRiWT6F4PanPJpG+9sdxXyUSV4LnqF+tKGGtz7e2v5G9CJa+76zkxqz90m2oinsrM9tPkqmFeVaLowApYk4jbYQ0BMmg/AnM+qPXYsvqirX/cY7GNscdm5jPurFOP0voylk94ZCzW5bUoqieAzxRQ+FCgWPppARbnHf8WHt9wjADtxjrhIFDtmLqngveMKtr3H+X2e1vxG9COe3jWo+CurTmXLUwT/8nFXbtWzj4DbEtbIsiFBaFKPBFgqtmY/8ZhqPpuATCn5zkqcdHXw4wyK1jPmoF+N0CqfERDIoJLVAQqE5maYoHiMm3hmME5LqdzQHhZQ60Ud6Xw/VFDrLfNQWR7Pyvje0Hff7juhoDg6ecAa/4HMs21TL5pqmwPOBNVDreTUClBXHaWzxZh+3p0qqPyw8lynH+WhbXTNjr3uqzUtphpnajPmoF+M3HwXlBBTMfJRKU5yIIUibfQoOgrWAunP88x9uZM6q7YHX866y1X4i1drxzcCMb6H9OBOXUE3B14WDwk5T6fDP/IQ5qvW8GhG/+cja3npIqspqc0syuvnI6efrtltRcw/MWRO6b67j3Xbl0I46EyMUujF+R3OQplCoTMrmZJriuHdQ19uYJRT0UENdU9CiQC67Z07oNTtLUwibVQXlUYSZjwytE9W0EbTORtY5WvEpeM4X8PumlPKUZQEoKcpoCq1GH/kmB15NIXOMUiq0jVXlRZFWmdtY3RhebiNEgBrzUS/GmYU4sx6/kwsKM6v1awORHM0BJqKg6KMwwpLX2pqsF3apIBOWEQbtx5/9GzX6KMh15C7FGuH3yFrHQClSaeWp6itYPoWmlpRHiLSk0oHXyOVTCArH9nP1CXvy6clDW+1PSikO/tmLXPnPeRHvzduufBFpkR1D16B8QiHIqZXvukCQidKwfArB5qPcjmYLJ/ooSqf2+iQyr5tT6Sz/RS7C1PtAoaN87w2RiSpYs/IUAs1H4T6FkkTMM8P37+P3w4FtPiqK05hMue0rTsRoTqZJplVWXkOuNZqd1/fPXs2KLXVhtxlp8uN8/NyHGwM/z9I0fO3LF0YodGMcxcCZdL24aBMn7D3M0+HzXRwLMhpKcSKGiK9KamhGc/agLljRR23WFLTtjS1tEwph5iPdP9Na9JGhdTIDqPU3akZzmE8Agn1JfUoSNCUztTj9M31nEuDVFCyfQktKuY7iUlsotKSsAApPm9y2ev/qbbvu0feDbg+wJnH65CfsuwirG+a/ltsupz1dXBDPUGCCFppxNIVH3l3LzNeWe/YvxADmhOEl4pZQ0K8ZpikEmQWCoo/C8NZRyrxuDEk4Cj9P8MX0GPXMwxbdlm3w4gzGzkAX5mjOXo4zg3NMrpDUihLvPNa/j3NsIktTsPpnXbNVXLK0yJpYtCSDzEde1TFs8acwHN9Za5nZrQ3u2Vqu8Sn0SoIcrLrPbM32es/+hRi/nDYVxSTLfBStzEVmu0i0kLowR3NbhULYQ+wRCs7D73tviI5/DYJwn4L3fc41QlT20pNFcfH0tbSCvb//DH96eRkQYj5CXCFQ32T1H+d9Uyq7P2XWUYBPdjRw1m1vZbWtqjy8hIXfd+YISv+tthYuHfacGKHQywhSVfXwOn9URSGqpDpRT7GYtdat3sbQkNSAN07yWpTcgVzmo7YQJn+C7NJB8fEvLgq29xq8uOajNg50Qb4nXVNI+Ew78ZhQVqSv3Keoa07xq2c/8hyrCwUE95j6FkdTsM4bXI4+oyG86ctkd56FPiXhlveYXWspyCeh05rW0etCUkUkLiLzRORJ+/04EXlbRJaJyAMi0r7CIT2coEgHfdYVj/mdYvlvk0cl980AMz6FHJqCk9GM1f5kSoWG4bnn1Y7XB5LOMh95Qwu9f3UpdOnd4WGzhgzOb1Rvr/0RuSBekKPZ/iHqm1NZk6CYeIWC/+dN6n3VRrBqHwHU+TQFf6mLxpaU6xdJK7IWC3LXT8jx4FlmUt2k1k6hkBWpteubj64CFmnvfwncopSaAGwHLu2SVnUxnlBOd9aT2eZ/SArhU9Cdd/6HPeNT8DmaA3wCTvRRcyrNnjc+k/uaARFO0Ik+hZD1IcCYj9pDWsHsFdv43N/eBsKdq7lqH+m+nScWfMKrSzZT3+z9veMxoUyra+QfWDPPTOahEdHMR45Pwe6v+uSgrinJpO89wwu2dqgIr4qaq7wF2DW+7H3aqylkaVXO9l1RUxCRkcDJwN/s9wIcAzxs73I3cEZXtK2r0Qeonz+9GKW82ZmxLPNRtPP++ZVlzA3JHm6NjKYQy7LxptKZyCRPuwJeC9GLhIUlDDW2uti6v30RfAq+GVjYQ2dWZgsnrRRvL9dNLcG/c1jsvWefNLy8eFPg8fFYZoCH7Jl8UptIOVq1gCYULCHjaA56P9iprdNgtU1lawqtDPSQMR+lNU2hhGZGpNbCnDvh4S/CP84i3VxPEcnQ84QWxMuzVOiqkNRbgWuAvvb7gcAOpZTzDa0FRgQdKCKXA5cDjB49Or+t7AL0wXDZplqqG5NZPgWn7ot//1zc/Ixlc135i5Pb3CbHjhqPSZYD0XlmcpbO1sxgfqESek2t4+sPbls1hbCvJyj6KPM3fGaXiEe8gV6C/p2FOZd1/ONZUDmTlFIebUDHMh9lruMvUJfSNAXH2WuVuXDMR9YQ45igPOuEBNjwwzQF/3V1+tWv4vCPfsOZ8WXwxhJ2X7GIBSWPUFrdAk9CsnQgicatDLpzBu+W7OSd9J7w+LNQ0hf2/Ix7Hn8Fg0IlrxVcUxCRU4BNSqm57TleKTVTKTVdKTV98ODBndy6rsf/0DQn055JV1yEA3/6gvu+EJNX3XmXbT5yCuL5Q1KzH3bHfNSWa4JPU8hD9JGziz8KKetcRlPIIky7ipynoH/mzsLTntLXOn5NoTkZPGgnYuLRSv2aQpBPwd+30kplOaJbK5Hx6dgcTp91LqOq59FP6uCFHzB8xWO8kd6HW8u/TtOXZzNhx+9ZUjQZaa6lgkaOic+H9x+Gd+6Auz7DTxN3uNf3t8f6G3jpTqMrNIVDgdNE5DNAKVAJ/A6oEpGErS2MBNZ1Qdu6HP8stdGXmp9lPopg/+6ouukM/ImYZD3soctxBvkUaIP5yCMUgrWGSOcJGcibdEezb5nDsK+rEImCPQ09ckvXFCLnKWhvM6aZ7P7kEI+JJ9jCnwDmzOxjMXH9byK4Gp7TfzLRR7pQ8J5LBTia00qRTitPPxTSxFAcFlvITUV3UV0xhkcn3cotr3/Ce1/sx0s1I7nsoeVMLqnkC5XjgWUcX3MjL3zzcI675Q1KaWLx906FdBKe/S4XzLmTzfSD+klAVdZ3sMtVSVVKXa+UGqmUGgucB7yklLoAeBk4y97tEuDfhW5bd8A/s21oSeVcxD7KQNXSwdHMGViDzEfhIakq+7Wd6RkFfVauP7i51PYgoiSvuZqCs2uY+choClmotmoKWT9f5kA9FDUsOi0uwutLt7jvsxa90TSFmOtTEBK249npP27ymsdfla0p+M1HybTy9MHDYu8zq+RrfFx6EXcX/5Iqapk9+XoaywZTRxlNY4/mX4sbtOOdY4Wdjdb1GimBeBEUlcGnb+KN1N58I/EoY179ZuBXtcuZj3JwLfAtEVmG5WO4o4vb0yX4H66G5hRhTldr/9Y7SFAsdltwbJuJAPNReEiq/sb6E5PskNow9MFDfwjbrikEb/dXu/T8DTlXqoPf466IvnSmLgiye4p3fwePpqAyTlx/1JFDPCZ87egJlBfHGdSnJNR8pGsUovW7jKbQuvlIka2JpJPNNG9cwlC28cvETO4q+iUNqoT56fH8qOUipjXdxuYB01yBdOsLS3nq/fXu8bqQCTRBlfTlwpYb+FXLOfRb9yo8fS3UbHDbA7um+chFKfUK8Ir9ejlwUFe2pzvgV6/9moJ/5hIkEw6/+SUmD6tk5sXTrWPaOLv24/Ep+J71tq3RHN3R7NEUku0XCtGij7x/w9YI7s6awrcfXED/8iJuPGWvgl7Xn/gXdf+g9xmfgqIhRCjEYsJ3TtiTbx+/B4f8/KVQTSEuXjNTwi8UEo7mYO2/7w+fpTjLBKrcCdG58Ze5PP4kY5/cRjzdxNul0KQSPJI6gh8nL6SW8kwbJWMmXbPNW4GgVaFgc2fqRL4woZ5Bb98GS56B0//UK/IUDAH4H5qbn1nsLiMIUNvkDWEL8ims2dbgqbzYVpOLn6QmFGJZ0UcZLSKsXZ48hYg+Ba+juQM+hUiOZkdDsEIHi9ON0LCddDp4wOmOvLt6Ox+ury74dXXnp64dRM5T0PpJUjMfOZrCI/97iGd/J/jLiWTL9ilY7+PxjKNZRDI+Bcd8VOzVFKobk2ypbfKcSylQzXVcl/gnvyz6K32kgdUjTmL7p27g2dR0Ptv8I65NXk5TvMJznJARSFmlLbS+3JQjO7+BUj46/Hdw6fOW2nzXydwqv+HY2FxKGjZC7ebQYzuKqZLazfDPAt5dvcPzfkd9s+d9lHGqo+YjT/JaiKPZP9h7ZoBOSCrRzUc6HfMpBG9v9jiaLQY0rmFWydfov70WfgmxAbuzh1zOEjUKsARy39KEJ/qlu7Ctrpn+OerxgLXSWDKlsorKdQRdu/Kaj4LJlaeg5wDUt6TYb2Q/poyo8uzvLYktWQXtnL6W0M1HkPEpuJpCxqcQNHGYEfuQc2sWc+S2/zIgsZ7XUlO4rOXb/GjKAUwb058rXtrb3TcRi9Gi1VCKaZMf/6RNF2KtZfUn0wpGHQRffRve+C0HvnobxxfPhld/A68Ch38Hjv1eznO0ByMUuhmtqYbb67wJNpF8Cm2cXftJuT6FWJatOLOoivcYb5kL3H3aIhRErEGjuQNCIczk05xMs7us4xuJR5i0OAHr4Yp1S+gvtTxccgZnHboPzP4b/y7+Hn8pvZQ7dx7A0b9+hZH9y3jj2mPa1IZ8k0yl2dnQQkMrdaFO/+N/Wbyhpl25KmHoaw/ov2zkgnja65THfJSkrDgeYG7UEzmz+4ObUyMhPoVUtk+httmrfZ8Qm83txbfSUp9gZfk+fKXuUmalLbNcSqks30NRXNDz3vTr+fN1dG2zNa3XFVbF5XDMjRz+8n5MT73HFVNiHDI8ZgmMPGCEQjejtTF+u09TiGJebK1ue+vHt+5TiInw6tVHsXpbPRfdMdub0exqCtnmp1zERUgq5dUUkmmSqTQ3/mshlx8xnvGD++Q8R5jQTDfV8pui25gg66htHg1qEJ+U7s4N285jedWnOeuII6jf/WRW334O32r6C98qhXfSe/DNHV+F5npIlECse2gMO+wRqamVHI7FG2o6/drekNTgfR58J7M2cXZIqhZ95HM0D60szZpE6PvHRMJ9Ch5NQbJ9ClpIarX9/Q2gmu8kHuSM+H9Zkh7BjUP+wMCqKmZt25C537TK8gX4/Wm678zf+3StPZdPQb8X91iV4JX0/hw2cjKHHD4+57EdwQiFboZeLjvI9JEtFFqXCv4IjbbidTT7NQW7fr7AmIEVbK9vcds1b/V2po7unxFcbdQUPh9/mlJpYNgnGzk7voijY/P5uOZ/WfjJCB5+ZwUr1m3ggSs/HW7ARpttNe60ojg2L4b3H+biJS9RJHV8W32dwXufxw0n78XfHpjPf7asY0/7kGT/iXyh+Rp+UfkIS+oruCj+PG+UXAU/uwoqhsAXn4GBu0e+n85kwZod7DmsL6VFcbbXWX2irYl9nUEU5+d976zO2t9Bf+s6mpXlaC4rjiMirsboPz5QKDjmo7hPU7B9Ck2+6KOmZJrmde/x56Jb+XTMyqd9JHU4f06dTn9Ksr7TZDpbU/DnVOhJmlnBChEdzfq9OGSij/Lr2zJCoZvhdJpfnrkvVz/8XtbnzqDr4BccQc7QoLWd24LrvAsISXVO7WgAzuf/fHs1LyzaxB2XTGePoX3dz1qTCWfE3mBCbB1D2c7Z8dcgDix4iGNtc3l6yTukV/ZlWWk1bAN+nICyATBkEoyYDmMPhcoRnBV/lSmynN3nvwjvLIeVr1vJQQB9hrFkwFHc9MlBLIrvyXlu9JF3gEum02xgIH/ofy1zq7fzWOowDo+9R4U0cVXDE/Cf78CFj+YUSvmgprGF0//0Xw4aO4AHv3wI2xyh0EEzYXvQ8xT0sUr/Spw1DCA7T0G3uesVSOubU5TbA3dMK7eud++YZPvLdM1VD34osn0Kjim1gnq+GH+ac+b8gMF1SxkaK+HO1Ik8nDqSpWokAP1UdkJbKq2yHMT+MtpOiXjr+/Heb1t8Clm+Dlcw5jyswxih0M1wzYg+lXTG+AFsrW1m6aZa3/7eHhJkp+yoUNCdd9mO5oymoP9dtN4yVazZVs/EIbZQEMkKd6ukjqsSj7Jf7GOWpEfyucRLANSoMl5QB/GL5rO4YsYwZs7aQJw0Nw59kz2H9+fv79XRv18lX5hWBXWbYdWbsOJ1eOO3APzaESJLi2DQRJjxFRi+H/QdBqMO5vGnl/LhxtXEtbr3mXIXzr1Zr4rtMrWL1BgWpcYAcNWph8DTV8NjX4bjfgCVu7Xnq20XW2stITB75TZaUmlXe+wSTcHuWsqX6KX3Ez1iLmeegqspWKHYTqmLuAipgFXHYiJZA6sz6Cfs2kfgDXBoTqWpooZpL13AkUWLWFAzntVTvsulc8ewwy3FZrcNlZXQ9pOnFnHxIWM828pLfJoCmqaAF4+moAmXfX/4LP/40sHsO7Iqs29IpJbRFHoZzszLb2YpK4oH1oPx94+g2YfjXGvvhNZTEM//meNotj9xNAY3KikmnvUUnIeljEZOjb/F5fGnGCsbWKGG87nES6SVcFzzr1iudqNfWRE7VQvrKiayVFld9d5BB3HZIeP5/bw32a+iii8cd6h28zWw7l2o28zl9y3k/fR4fnTuoRw/NWPi2VrbxJzFW2lOpSlOxEillRaSauH8Bs5D6RfQABx4KexcDW/9CbYutUIH2+Fj2FzTxLzV2zl+72GRj9mheTXrm1Jss4MPGltSkQvTdZSF63ZaJR/c2X12ATe3jZoj199fg4RC2jbROCaeWAywu3Vr5qNlm61J08j+ZR7zUSImVFLL2LoNfLv4d1TsXM/nm6/mlfRUjqwZzA6yQzzTaWgK8Mfd89Yqz/uKYu8wGosB6Wzz0eINNSzS/Dq6k7y6McnM15bzu/OmZn0fDu7kZRetkmoIwfm9/Q7ZeEyyFhi39m9dU3BXTmvnYOHmIsQD8hQ0H4jnmm5UUmYmLgJ9GtdzVfwRvph4mn5Sz2bVjx+WXcs/dkxhANUMlh0sV95Zt9/RnHFc+yjpC+OPBOC5e8us/eNlnl0uvXsO89fs4OR9h1Mcj9GkAvIVHJ9CKlwoKIkhx/8Ehu4Dj10BC+6HqRdk7dcaF93xNos31LD4xydGDnXVw5KbNU0hrSxzSnEi/0LhlD+8AcBxk4fa11au1gjenIU6LREtrMgbZPpMMp2mKZl2s+T1PpfyaSN+89H81TsYO7Cc/hXFmeNUmtgrP2Neya+Ib1Y0SDGLj76dje8OhvXVvOVbXc1BEU3L9k/WrDyFzDl0vvevhe5rvxkqHhOPeSmseGBawZbaJvqU5Cc82iSvReDTv32Vo3/9SkGuFbQus/VeAss2Z2sK4eajdqQIAFr0kWSbj5zIjZg2KwOtXpKI25lHrXuSY148lW8WPcLs9CTObvo+Bzb9mfrxJ/LklYexjUo+Uply6M45nAe/rChOcyrt3mMUGeefba3aWue2uzgRsx36XrtRlvkoQCi43/OUs2HgRPj3V+CWKfCXw+D2I+Dec+DRK6AueMBxWL65zr7H3IPPKX94nU/9/EUAdmh+pZZU2vUpAJ5Ex0Kg+xSCzEctqXRgnSn3eO21MwN2QmtLijLmI8eE5/UpZGsKC9ftZL9RVUAmoXLMx/+A127maXUIP6+8kYOb/kj1iCN5+qrDmTq6KjTMWeWowaTjz/vQHc25JvX+c8d9659nl87OmI8O++VL3PL8klbb1h6MUGiFe95aydJNtazYUleQ690324rUEBGevPIwRg2wZrphmoI/OSaoE7e45qP2SYW0x6RlnaM4HmNw3xIemrMWyAgcZ3aW1ASRUooDZTHT517Hzv57c0LTL7is5Tu8oyZZ51PBUUnOWO0MKhUlcZqTmUEmyt0oBYvWV/PnV5Z5FizaUttMRXEC0RZDcb7LnQ0t/OrZxe7AXxLwvbtCIRaHi/8Fx34fRkyzipqVVELtRnjvfnjnrznbl/IJvjAWrqvmk52NgFdTaEml3egjaD0sFTrX/KCXBPHM4u2//hpGfjv5ba98nPVZo7MQjqMpxISieLY5JhbLFqY7GloY3KfEPQ5gxIpHYcQBXK2u4vZNe1FNH7e/DawoCb03paJl0Ff4fQqao7k5h1DxnzsWE08/8JSfV8oVMC2pNI0tacqL82PoMeajVvj+vz8o6PUce2VMhH1G9GPPoZWs2dZgPxhB5iPvez1aYkttk1U0TEvoaQ9JLXnNGburyos4fu+h/GOWE27o1xRw70MBVxc9QEPZcN49bCYf3b/Yc35FdpkM6xzOgGndU1mxpSm0pdRFKq248V8LmbtqOwePG+gKxs01jYwZWMHWuibNVmv93VbXzJ9ezgxWwZpCCrC92f1GwuHfzr743z8L7/wNZvwvlPYLbJ9zj2H3tH5nA8MqSz3btvs1BU1I+KNlgkimFcXtVRt96HkKQUtU1vnLsmgD3Y76ZpZrky3no/oW6xhHKFjlVWLUNac8/T0Ri3nuVynlcVAnYsKhsfep3LkYDvsN8bVa4pvdD5x8hSA21TSG+kl0sjQFMkvo5vo9/Fq9X1PQ5Z1+37WNSfu6+cmTMZpCDvLt0Ml1PeeZ7VOSUaGLAsxHfruj3tGm/8RajCfZQfORm6cQz5iP+pQmGNwnM1g55xYFY2QDn1XP8+eiWznx1dMYe8c+HBT7iOXjL0AVVfhPT1qpQE3BubcN9gy5X1kRzcm0O5uKttKXYkBFMQBPLPjEbeeW2mY7lFCyFtlxcB7oQKEQYfDl2O9B3RZ45DJINgXuorTZn5+PNtRwyM9f4u43V3q268tGNieVR1PwR8sE0Zk1nPTaR3rhRee3qfdlC4etk6HjhLA68f8xyfwGetv9lXkbW9IoBWX2DDoeE65M/IvG8uEw9SKfk9o5h3dgHVGV8UFtr2+hpil4uUy9T/gdzXqNr1y5CFnmo7jXp6BrVfp9O23Kl6ZghEIOOlpIrq3UaQ+Q06mcDh4TSASZj3zPlX/GqbSM4PY6mj0+BVsj6FuSYEBFptaOc+5hr17NqyXf4qbYXzkw9hF1ZcOpmXAav0+ewarx54aaiRKxcC3oxcWbmDa6ilH9yy3zUSr3wKfPRtNaWYKaxqTHD9GnJGEPDl4Hs4PzIAZpaGG25gffWcPvX1xqvdltKpz8G1j6LNyyNzx3I2xbHnhc0ODhmCzf9DlC/Y7mbfXN9C21+kmUsNSOrq+hoyeVBWsK/jUKMq/DvkNn0HPWUY5JRkvWf9sSn5PVEUDlxXGo3cTVm29gRmwRayZcCIkSz8Dq9EO/pjBmYDlRKEnE3L7kdzTHtBIbucx5/mc1LuLRTHQBqgt7oynkkZ31Lby7envo5/4H9cNPqtlS28Q7K7eFlvZtC8s313pK61Y3ZoSC0+HK3LA8oShwQM3tU6huTLrmo/ZGKqb05DVNU+hvz8Ddc29ZSuXiB3ghNZXjmm5mRtMf+c2Qn7Pq4Jv4bfIcVFFFoGBKK0WATPDc2xF7DKY4EfOYj8JuR3/43/x4qxvX35RMea5vCQVxzUb+Mc0RpkGaQphZ4JpH3uO3ugPwwEvh3Hth1MEw6y/whwPgnjPgk3mB1/KitP8z6NE8lk+hxZ3hzl6xLbBdOp25LoQesaV/7872Or+moP2mYSYzJ3jB0QSuPHYi5x9kBSDos+dS3+/y8kdWWGlZcRye+AaTG+dzX/Jo1o4/L+vaTj/wawpRhUJpUdw9R5D5KIqm4LTXYfGGatbbWjF4v89G7TevcYSC8Sl0Pj94fCH/mv8Jr119NKMDOoO/037m96+7r/9n6ghuOXf/Dl3/mN+8CuAWKKvWzALOTKOs2Laririp+jr+gczfCbfWNrkJPVFXPfPjjFcJrXR2RXGCAeUZoVBStx6e/iqqqILrGi9nC5YN/eG5a5m7yhK8gmSZaMDxKeT2l/QpSVAcj3kdzSG3ox/37/mfuK+bkmmvUChNIHr0ka9t/uQ1nShRKS6TT7H+Va+HV34G794D9y6Eix93d2mLn0T/jWsbk9Q2JdmtqozFG2r4yVOL+FIrdXE6U1PQy1wkA4RCvU9T0OcwYQOms90RxhfNGMM8e/KmN92vKXznoQUATFlxB3z0FE8M+BLXf3IMtyWs+lipdLZQ8GsKuRzPOiWJmDspyY4+yqz61paEwndWbufMv7zpvteFmB5VVtNkjRP+pLnOondrCvYgrNdm0cn14H/4Scdq11c3tmRv04SCaz6yO35aQVHAjDWXTwFga11zh81HqXQ6E2YXoCkMoJrxj58Bmxax+eibXYHg4JhBRLLLHIClEejhtr89Z7+se6soSVCciNGih6RGXN3LoSmZ9giSCltTUO5x3v3DFhACS1P440tLOW/mW4HXCqRyOJz2B/jaHKvkxm2H8fPEX7k8/gQtgX1NtP8zNCdTrn9pzXZL0xw9IDOpaa0eVhTnaVSc7yyVVh6fgvM7B2kKD89dy34/ei7L3+BHn8UXuSGp9gUbqzmw/jX2kDUcEvuA02L/5Svxf3F/8Y+Z/MEtMOVsXuh/rts2va2QmXT5NYVh/bxO/TA8moI/T0EyQR2t1TfKhS5kPWuqGE0hf8Tt2enSjbWBn+eavbVnXQCd99bszNpWE2A+0teSHR7QYZWyIlRmr9jG6fuPyLJhbq1t8tSDaQ/JtMosgm5vs3wKllC4NPEfEnUb4fKXqS/eA3sxvSyE4AFbKRiomaKce9Z3LS+OU5yIWQXMnMFHu51UWnHv26s478DR4UKhxWs+6mvP8NJK8Z/312etkuUMJn6HJli/1a+fs8xE6bRqmxY2aCJ8ZRY8cz3nf/AoAAs2nQPjrJLcyzbVsnpbHWHmo+ZkmoqSBDvqW9w2HzNpCM9/uJF1Oxqob07lXDOhMx3NYXkKrvkowKdw0xMfUN2YZN2ORnKhf+9uKep0Cj54DN64hYvXL+Bi38R+RXooq6Zdx5iTryb2gFU7LKhKsKP8OX6LycMr+dxBozj/oNGMG1TBBX97u/W2uT6FAPORff42aZQ+9N+pIch8lCdNoVcLha11VkTIxurgzpnL0RyPCWm7REKQA7g1nl5ordvqDPSptPJUQM04mjN134cGCgXFRXfMZtmmWo7fa1hWm7fUNrd5tTI/qXQmvn9wX+spbE6lqUpt5Uvxp7gi/iQ1E06lcrepSI58DstUk709bZdm+MkZ+3Dv26sDy3lUFCcoScQtoRBwP/e/s5rv//sD6ppSXDhjdNbnkK0p9ClNEItZwucr976btb8zyAU5mt9ft8N9XdOUpF+Zd4Ebf7mJrL7Sdxic/X9MmfsZ3in5Xya9cgUkr4T9L+C4384C4E+fm5Z1XaWsheMrii2hsNoWCrtVlfKVo3fnhscWUteUzCkUOloLS8cZuPx5Cs7rrOgjpehbWkR1YzJLCPvRZ/HFjVs4UBZzXd2/4aEF0GcYT4z9Li8v2UZc0sxPT2CdGkQ9pfx76qGMiSdcQRI0SXB+G8cvURwXLjpkLACHThjU6n1bmoL12j849y0tcj/TI6xK7ElNVNIhmoLjiO/MxZJ0Cm4+EpFRIvKyiHwoIh+IyFX29gEi8ryILLX/9s93WxwH5IYQoZAr7DAWE87/6ywm3PB0m6+bTisem7cOyAw4R9z8sqcqqjOcOOaj5lQ6K14drEF2/Y4GwDKH+ePCN9c0uYNAewtp6ZrCOdNHAYpz1v2Ckt/tzY1F9/KJGsiWQ2602p1zwiyBpg1n04UzxvD0VYdnqfRg2U9LiyyfgvNg6Q/NpmpLwDc0J8NXW0umPQNXn5IEgoSaMZzEoyBN4RNtluuY/fR784dbfu2+dwP7Sg3lXNXyNVqKKuH578EDFyBY9+dEWelfqVOl04k2WrPN+u2HVpa61TrDwij1c3QWzVrf8moK1t8sTSGt3HY62eVhOLN4Vs9i/N8P4qGSm9g7tQhO/R1860MWDz+dR9NH8FDqKJaqkdRjPR9uIT27zwbJQMeE5vdLRKUkEXMnSv5JzD4jKgMtCYtuOrFN19Cd6rpvwpkU7UohqUng20qpvYAZwFdFZC/gOuBFpdRE4EX7fV7Zaq/JuqW2KXD2lFNTEHjbjvR4aM4aXly0MXRfP/UtKTfT0/mx19kDu4Mz8JVpK0QFmY/SSrm+hu31zXzg83Vsqml07629g0EqbecRNNcz+aM/8974vzB161Mw/Ysc2/Qrjmj+Hem+Vr2iXCYqkUxHP36voVx62Dgg2zwSlI+h13n5i50Fq/9mzvdYWhwPzC/pU5KgKZnyRA05Ian+da8dnN8oyKewbnvm93J8U/os0J8v8J/3rYVakqk0Nz3xoZt7AfBs+kBePfYJOOlmWP0W30w8zDRZQqMtrPS7aUnZmoI9sH64vpry4jh9i4RByQ2Acm3OOlHyA6IStHpYOiT6KFtTyMysV25tTVOIwY418PjXSSfKuLblMj5f9gc44PMQiwdOHiBjfnTs+qkA85Hzm+dKXmutbeK+DtAUNKGw59C+vPKdo9oc6OH8Zve+vcrtPzpBGnVnUHChoJRar5R6135dAywCRgCnA3fbu90NnJHPdqzdXk9dc4pRA8pQCjbVZCcX5dIU9JnA1Q+/x6V3z4l87Xq7QxbFJTQ6oUEb5MA2HwVoCopMdMyO+hbmr9nBoRMGctzkoQzuW8KGnY1srW2miCSSznZuB7FuR4Nrw/xkRwPVjS2W2ePt2+CVn1PZ+AkccQ185td8rEbYR7Xe4S2fgvW6OBHj4HEDrHvwaQ9B5pry4kRWCGKzNrg531dZUTxQI+pXVsSmmibP4uwThvQhJkJtU/Bv4Pw2Qe3RhbijKehaWmNIyPLbK7Zx539XcOO/3vfeSzoN0y+FwZP5euJfPFryQ055+USOiC2gOFlDMS0kSLohuSPiOzgm9i7Hxubym773wS17ceiTx/BC8dWMevoSau6/jPSCB2DzEkglPRFHH2/O+NC21jZ5aidFQRfGrtamvCvkpZWirinpuZaz3RGyq1sTCiTh3rNg23K2H/4jHkgdzScy1P08bEBvTVMYVlnKlBFWMESYYLntwmzTnY7uaK4sLeKzU0dwwt5D+dn/TLGurU2ORg8sZ+yg7KTN1nAmUDc8tpCH5671fFaciAX2y86gS30KIjIWmAq8DQxVSq23P9oADA055nLgcoDRo4Ntx1G49hHLVDN9zADWbFvHxupGTzYjkDNJqr1OW8jEmQ+sKAl9IBu1QQ6gKZV2nbH6qmxppdzOsXZ7Pau21nPO9FF89egJfOnud1i3o5GdDS08U3wto9gMt+0Fo2fAyAOtE8SLobgPlPSB4gooruC0m2cxZfdR3HXZ4XzqFy8BcGifT+CNW2H3Y+CixwK+D+tvLnu1VTE14/R2Fzf3jeFBhf8qSuJZFSH1azlCTF+QRadfWZFnIK8qL2LUgHIQqA2IBIPgjOYHLp/BuTNn8cnObE1B1zjC8hgyGmLaa25KKogn4EvPc84Pb2e4bOWm4se4p/iXsBoohe2qD+X37sPMpi3s/cnHxIptJ29jCUw4lq1Fw9ny3pv0q95Aet279F38oHXyRBmJYfvy00QfFqtRfP2+NAeNG8DQylIOsLPe27Jus0co2PeplNc8mUorzv/rLN5b6w2oUEq534Fjtr3l3P345gMLPPsJafq9c4u1Ut75D1A06lh46jm3KiuED+jlWkaz1Rbvb/G3S6a7v2mQaRDgxH2GM3Zgeag2U5KIcfK+w7n/nTWUFMX4rS88XZ80BoU056IobiWJBj1KRXGrPpI/4qkz6TKhICJ9gEeAbyilqnWnnFJKiUigjquUmgnMBJg+fXq79eANOxuZMKQPnzt4NI/NW+cJB3XI5aANGrii4swoB1QUs6G6MdCs4zxgrlBoSRGLCXNuPI43P97K1++zkp9mvrbcHVSdNXhH9reE29DKUuau2o6IMFK2sIyR7FXWH+b9A2bPDG3f3FJoWptA3TWDW4oUMRTHJOdD+QDLnqvhLJXoDPC5ar0ImYelvDieVVHVwXmIBlQUu0KzvDi7TLD++ziaQlMyFbhudVV5xhF85rSR/PiMvQGr3TUhQsExfehCoY9ty9ev4QiFbz4w390WVm5iuy8b2aHJeV3Sl9lqMijoN+F0Ns5/hiMG7qRm+yZGyhY+TYwmVcQLQ7/A7WtGEUPx669fzJjhg6ndWsd5c17h7DEjeXjuai4asZGbDq+ADe+h1rzDmfHXKJUWxspGtq8axtARgxjKtqwQ4tbwrjNs3affp6AUWQLB2i/bXFdqD+5V1PCVxOOU0MzesVWUzVpiZYXvcQL9RJh9w7Ge3JgwTcHZnhEKKvBz63X44JrL3FOSiPOTM/bh28fvGXgOfdLY1rGioiRBY0sq0OxVXpxgZ0MLlb7Ahs6kS4SCiBRhCYR7lVKP2ps3ishwpdR6ERkObMpnGzZWN3H29JFU2V9uTYAdNlekQDwoBTcizkxpYJ9i+zqZAWRYZSlnHjCCU/a1bPRu9JE9aAzqU+IKCvAOTne8sQKA4f3K7L+lbvG0REmSF1P7s9cld0OqBbYug1gCUs3QXGctUNNcR3X1dn71xLtMkHVclNzJdFlBGmF+bG8O//xMqPJqZ1VlRWyvb3EfgsnD+/KN4yZy6wtLs+5bBE7cexhfO3oClx0xnnftpDb/GO5E6FSUxNlm+yIriuNZg4AzY3196WY22+a/ppZ0oJDVhcKmmkZ3NikQaIOHjIDTq6SWFsXpX27d8/hBFSzfUucKhXdX79CODRYKGzVfgi5AWwL62qamIp5NH8iG0ioWJK1zv/o/R/G5377GhaPGMHe19XuX96kEMstCPjZvHYoYi4v2hv0PAc5ne00TB/70eX6Z+CuXJp6GRyyn99ul8FpqCnBaYHuD8K9vAc56CorSohh9S4tCv9OPN9e65cId9p97PbNKXqGMJvpJPc0qzno1kNSJNxOf+jk3emFIX6/51K8p9Csr4luf3sONLHKFgq87FMczx4VpCpC7gOTI/mUk7ErBgcfqS4G2oin8+YJpnui38qI4yZQilc6uv1ZRHLeEQukuJBTE+sXuABYppX6rffQ4cAnwC/vvv/PVhprGFmqbkgyrLHUlblAyWS6hkOxAWJ+uKYB3cBjct4SrT5jkvndmUfqseO/dKnOe34lSGmL/jZEmLooWlbBCJeNFMGRy4LGbNtXy95QVknfyecdxuG1eGFZayqwB2ZmyVeXFbK9vcW3WIsI3jtsjVCgk4jG+c8Ke9gbrj38MdyKd9IlSIh4LNB9tr2vmojtmu9uakukQn0JmhnnafplFfGIinrIROo72oWsKxfEYYwdVsH31DsYPrmDVtnp2NrRk+UXCNKb1WqTbR9oqXEFmtx0NllZRr82snTyNPloYpBOJ5Ggxzozdu4BNGhBuSH6RZ9PT+eqnhnHAiHL++chDfC7xMtw6BYbsBVVjYPCeMO0Sy5wVgN4XmzRHczKV5uBxA5k0vC93/XdlwJGKFxaupYQ0TRRRSR0HxxYzfOVjrGUQ76fH8Q85lWeapwDCyhm5TVr+ScJB4wZwyafGuu/DHM3FAfkPQWFzzkRn35H9srSe/UdX5WybrmQEBU7ofGbKcM/7eFzcdT78GqcTVOL85vmgKzSFQ4GLgPdFZL697btYwuBBEbkUWAWck68GOHkJw/qVuhK3uiF7ZpPLfOSEs/r3D1y60YeT5emk1E/78fPuZ36V1QnL09uyW1UZK39xMmOveyrw/EMqrfM6deWLsK7XQoJU2soevvK+eSzfXMtTXz/cc6wuHHXHaVjY7l7DK1mxpS5SZJM/AznjU/Ae66jb/qSjIPORk2vi0NiS4vy/zsq6tqMpnH/QaM6ePirTphzPa0NA9FFRPMYg+3vdd2QV767ewc6GFlcIHDNpCC8t3tSqptCcTPPAO2soL45T35wK7GvOYjq6uaXOLRaXPdv1z5zTfp8FkCTBS+lpHNF/Lw6YOo4fPVBOkgQXj6yE9Qusta6bqmHuXVYxv2FTrDUiNHQzkfPaWaPZKYWilw7ZTz7mksRznBH7LzHbKpxUMRJi3XNL6UBO2/ETtlFp+c2aozm+/ffbv9w7e3bKwvjlbaB2EDCRcJ7Fn/3PFPYZ0c/zvOlrKQeh95kw30cY1sptVhlt/3oUunM7XxRcKCil3iA8VOXYQrRhw05rIBlWWUppUYyiuLRZU9hcmx2tVNPYwsA+rddOcerBOOYjHf8X49jXW6vY2rc04ZrAnMHTOX9GKMStBzdulZEOQvetBJnU/Nx81r6csM8w9hjqXfT8vstmZA/OEvzW/zzqUSOvXX00a+1SDrpQOGjcAN5fu5MtPuF8z6xVgQOs8xD5B4Rc5bcbAqKPiuIZH8SkYX2tdaQbWtx6NENsc0LYCmiOcN3Z0ALb69lnRD9mr9hGSyrNvW+v4sg9Brv7ukKhURcK2bkT+j3opjtn7F67vZ6731rpacfWumbSaUUTxXw/+QUuPkublS98FJ6+Bu74NBSVw96fhX3PhhHToaSPT6tRTJI1xNMVxFIpjqx9iTPe/SdfTeygJVFEgiR9xXLKP5M6kIXpsSiEMmlim6pkY3wYl555MdvusKKx2rK8pP+3rCr3Pk+JEEdzlIkbZLKe/ZrnnZ+fnpWs6Gf0gHJ+fPrebK9v8WimUZg2uoo3lm0hpVRW4U3np64s27U0hS6nrjlJVXkRw/uVISL0LS1yH/TmZJpk2lrVaEvAwO8QFDVU3ZiMJBScSoh6aQcH/xhVWVbEuEEVXO2YXDSO32soz31o5UdMG92f6WP6s1aLn3fakrBXPW8h0WoCmy4IwuL3dSpKEoGd/pDdB2Zt8w+/7hK6Pq+CrvaPHljuFivUzQUHju3P7BXbsrLRw7Q7575LfCaH4ghOQH3w6VtaxLUnTuLGfy3kkN0HUllmZec6g7WjRTiawpbaJo9d2GnvjoZmFIpxgyooTsTYUN3I719axjgtdNExH+nJaLW28Akb2M6cNtIVCo4Gduldc/hoY41nvy21zeH9e5/PwphDYcWrsORZWPQ4zP8HIDBwAkOq9uaniRr6Sw0VNHJk/D2PB3B76SgeSh1NnyJFfYtiYXoca0r3YHbjUJxesP+oKuav2UH/kiK+WNzHPdbxobVmcoHsxDP/QH32AaO4581VnLbfCM92/bvbY2hfhlWWcu2Jk/Dz3ZMm8+2HFjBhSB/P9mMmBQZGehDJZEi3haGVJVxz4iQW3jmbTdWNrqYweXgl4wdVuL/jLqUpdAdO2HsYJ+w9zH1fWZpwzUcX3vE2s1ds47lvHuEmSUUlLIpFZ8GaHdzyglUzZ0CQUPC9j8eEl79zVOC5Zl48nTeXbeFzf3ubxpYUVx470fO5I3QcTSFpawo6/nIMYeajzsA/K3fyLvYZ4Y1+cQaGA8cO8Gwv1dTwPiXWQ/GtB72hjGG4S2v6VPkoURxOe/Yd2Y/iRIypo/u7ZrfKUisaxJnND3KCB1rSNDSnmPGzFz3fuaPZbK9rIZWGaaOLKY7H2GhnZDtaEQT7JWoDNAUd3aHuRAkFabWbqhs56GcvavumvQ7RvkNh33Osf001lllp/QL4ZD7la2dzTnwjq9UQ+koDL6amsr7vFKSlltoh02kcezS3vLiCoeUlbGxo4qCxA3jwy4e45pcbT55MTWOS+Wt22EEFmX7hBFH0L89+Nvz4k7eqfOajsYMqeP9HJ2Qdp6/yV1GSYNZ3gw0Un5owiLeuL4jxwuWlbx9FRUmCfUf247UlW9wIuKtP2INjJg3l2N+8AliTk3zRK4WCH2u2Zw2GTj36jzd5k24uP2I82+uaeciXRKKzMyCsFawM3EnD+jK8qpSL7sgU2gr6Ydu6jrIzYAUtcOKo4kW2ptBMgrc+3spb2qItdc0pN2oFvL4VR1O477IZ7lrR7aGiOE5dcypLyOwxtC9PXnkYk4Z5TU/lxQme/cYRnsqf+v0AXHTIGH77/EfuwPfE1w7jtD+94TFFHbHHYF5bYtWsbwopWeGfcTlx4P72PHnlYVkzRrBmp2u3N7jmo0G2+ei2Vz/mw/XVgb9LUVxoTqXZUttEv/IiihMx1tt5D61lGzvfYZimoP+Wzu8XtNTpgrU7PO9/9p9FHDNpCIdPHOzZXtuU5Af/Xs4NJx/NgD2sAXbBym2ce9t/SWu5r/uW92PNtnpOHjycITHrO61vSnHohIHcduEBnnMO0rTpZMpbj8rRBoMmTH7G+RLCqspaPwbav1b52989tt2lYqLi9PH9R1Xx6LvrWGqPQ2VF1u/qaKT5NB/16tLZDpWlRdQ0Jj21WJb5hMKM8QM4YIy3HJP/YXNKRNc0trBsUw3v2xELv3xmMV+46x0uu2eOZyGdoDjrtnbXKSP6cf5Bo7n5rH1D9ykSW1NQca74+1zu0pZ31H0Ia7fXezJQHZv27oMrGNk/2uIjQbx6zdGcsf9uzBifbVLaZ0S/wIKCew7r6wo8B/376lOS4PT9LbPAgIpipozs5xEIlaUJbj4z8504SVZ+oeCP4ggqnZyIWetlB9m7HZ+C87CO6l/OGfvvxubaJs/3rOO0G6yBrCgunsVVcuEsCqWHVeroA97OBivDPSgk0u+L+b//rnSjuJRSzF+zA6UUj767lkfeXcsfX1rm7tuSTHsEAli1hHY2tNC/vNj1CdU0JZkyoipr8tO/otgVXuXFCTcgAjLmvyiagv57nH3ASI7ba0irx3SEoZWlbrh3vnC+u72GWxGGC9bsADJakSPo82k+MkIB6FdexIadjRz5q1fcbf6Z1NiBFZ4F0oGs1PX5dpz6FX+fy3G/fY1T//iGZ1buNwcEdfy2TmIS8Rg//+wUdh+cPYsFa0AbUmGX3w5QDHVz0WG/fNmTTu84RTtajXFQnxJuPW9qpNlfLvyDsjNTDHL63fn5Az0DvCOQpvkEu998NH5Q9veYq0y662i2v8fKsiJuPW8qx0yyBqgRVWVZ9637YPqXF1FRkojk1IfMokGOcBs/OLx8ws6GFs7403+z6mrlQinFS4s3ccaf/svfZ61yt+uO86Cgh+rGFtLK+j70PqybeH50mpUwOKp/mavpTBzah5H9y1xT5wg78fK0/aM5Z2eMt0yMvzp7v1ajfJx9u5Kw3+u4yV6B5owtH663apk536OjrQ6virbuQ3sw5iOsSIGn3lvv2fbCok0kYsL8HxxPKqXoV17kzmIqSxNUNyYZO7Dc1Sj6lRXx6Lx1fLy5lgVaTLPzowJughXABz86IXCwDVs4pr28/8MTiG1aCH8LFgo1jUnueWslf3454z9JxIRkWrF6Wz0i+Su81Vb8s3zHfq9H7Dj4tYwT9xnGgu8fTz+f3bnSpylMGt6XV5d4l0lsTSik0sr1bTjF3sYOtB7q/UdVMbJ/Gbe/ttw95mBtcKoqL2Kv4ZVZCV1lRfHQCCawzEeLbjoxcBnTjrBuR4M7Gfjb6yu4+JAxgDcvJ8jE5VSprSovdgtNgrfvXPKpsZy6324MqCjmreXWZGnfkVWICPuPquLFxZuYMLgP87//6axIojD+funBkctR//3SgztcRr6jPH3V4dryr5nv8S8XHuC5j4EVxfQtSbDIHj+c/vyPLx3M5pomxkZcNrQ9GE0BPF+wsxYsWHb6PiUJdyC57PDxXH3Cnuw3qgrw1l3f3962wJfk8qtnFwde0xEIT155mPeDzpUJlBXHKREn+ih7cK9uaOHlxZvcgWBQn2IeuOIQAJ5ZuIGxAyvabYPtbPztOGXf3fjmcXtwzYneyKxvHDfRVb//+aWDueeLBwFkCQTIaArTRldx/UmT+OzUkZ7Pf/o/++TMSPUPMn1tB7ijxew/qopvfnoPvnLU7u4+JYlMdnZ5ccLtOzpOwbZjJw3he6fsxblabgVYA0pZcXCl0Hu/dLC7ep3DHkP78NeLp4feh8Os5du4b7a1EuHqbfX8n52EtqG6kcfmrUX5Ct85ONpDVVmR53O/78PRms6cNpJva9/LV4+ZwIUzRnPa/rtFFghghQv3iajJFsVjeVuDIColiThlxda/ipKE2x7/fYgIYwaVu9YFJwO/T0mCcYPy+0waoUBmVpeIWQu9DLWTv5xZkkNFSYKvHj3BXlMAzjxgpDuLnDTc6yx1yFULCCyb+v99/kD3/RcPHduue8hJyjJtOJrCxCF9OHyiJdA21TR57Nm3nLs/B4zpzxF7DCaVVoEDVldSnIhx2eFWye2KkgRXHTcxq678VcdOdB+aT00YxBEBmoSD41MY0reUK47c3aOWTxrWlwsOHhN2KACH2+c+aZ9hbt4LwNTRVQzqU8KRew6mtCjOt4/3Cq4/fW4aIpbv5PCJgz0ml+JEjK8fO5Gyojg3nbEPlx42jhtPsTLQPzvV8kdMHBrc38CarHx22khGDyjnvANHMaKqjOtPmuwJE47HxDMBcvjOQwtYuC6j3Tqmp9eXbuGbDyzgtaVbPJoAwHkHZgRW/4oij0YclOQJlinwymMnuibBaaP785MzpjBhSPh99TZmjLN+r+H9SvOawezHmI/I2O8OnTCIeEx4+7vH5dz/1P1241TbLlxRHKe6McnkYd7SEzeftS/XaIvm5OLoSUPaVKWyzdgls5O2pvDcN49gR30LU3/8PNc/6i3hPGaA9V18/lNjeG3JZqa1ks5faJb85KTQzxyzV1tmUY65zi0fUOIIiRKe+cYRrR6//6iqwN9uzMAK5tyY6UfO5MFJbjt28lBW/Nw6bmhlKUt/chK/eu4jbn91OcMqSzls4iAW/TizKEvf0iL3Ov6KnGG8ds3RnvdO7sTU0VU89pVDAThn+kj+589vMqKqjB31zaElPxwuudNyRleWJhjZv5wP11dz+RHjuf+dNYBVTkQvQ5+vJSN7AzeeshfXnDjJyhLv4PK/bcEIBayH8s7PT2f62LY7oipKLP+CHh530+l7c8q+w12h8Kuz9qWiJEFZcZzieCyrRHfeSVmztWZlF4EToX9FMX/83FR+98JSlm6q5ctH7s4BY/q7iWJH7zmEv1wwjaP2bH9ExyvfOSpSAlxn8crVR3lmqVFwzB5OspSI8PCXD3Ednp3JfZfNYPchwY7GRDzmOsz95Ro6i9KiOP+49GD2GZGZwDh27KGVJfQpSXiS3G678AC+/I+5AHz/lL34cH21G4jQmExz1xcO5IP11YzXghycdSsAvnjoOL5w6Li83EtvIWr2dWdihIJNlCzFIG45d39+/exHTB5eyUn7DOOsA0Zy7GTvuc46YGTX2uVT1sB85oHjOKxvJsHtlH1347Ulm1m6qZZBfYr59F6ZdosIJ/kKdbWV9iws0hFG9i9vc+jspycPZZ8RlXzt6AnutvZMDqIQlOWtc+DYAYwfVOFqofngsIne9YenjOjH1NFV3HT6Pry+dAsPzVlDcSLGlw4fz4n7DOPznxrL7kP6cNGMMWyuaWLlljq21jXzneP3ZEhlqVt08Z4vHsQ9b61kQEUxN5w8mZue+JBrTtwzbwvBGPKHBK2Z21OYPn26mjMn+opnhWbCd/9DMq3yaxqKwqIn4YEL4IrXYbg3n+HJ9z7ha/+cx+0XHeDJ8jYYDLsuIjJXKRUYeWA0hTwy+4bjsip9dgnOMpzxbLPEKfvuxoQhfdgzh+PSYDD0HoxQyCMdTdbqNFKOUAhuzySfk9xgMPRejMGvN+AIhZiZAxgMhtwYodAbsKOPwjQFg8FgcDBCoTeQtsNCjVAwGAytYIRCb8DVFIz5yGAw5MYIhd5AK45mg8FgcDBCoTfgOprzV4PdYDDsGnQ7oSAiJ4rIRyKyTESu6+r27BKkmgGBmKlDYzAYctOthIKIxIE/AScBewHni8heXduqXYB0i5W41k1KYBsMhu5Ld/M8HgQsU0otBxCR+4HTgQ879SrLXoBnb+jUU3Zrajca05HBYIhEdxMKI4A12vu1wMH6DiJyOXA5wOjR2fXgI1FSCYP3bH2/XYXBe8Lw/bu6FQaDoQfQ3YRCqyilZgIzwSqI166TjDoIRt3Tmc0yGAyGXYJu5VMA1gH6uoMj7W0Gg8FgKADdTSi8A0wUkXEiUgycBzzexW0yGAyGXkO3Mh8ppZIi8jXgWSAO3KmU+qCLm2UwGAy9hm4lFACUUv8B/tPV7TAYDIbeSHczHxkMBoOhCzFCwWAwGAwuRigYDAaDwcUIBYPBYDC4iFLty//qDojIZmBVOw8fBGzpxOb0BMw99w7MPfcOOnLPY5RSg4M+6NFCoSOIyByl1PSubkchMffcOzD33DvI1z0b85HBYDAYXIxQMBgMBoNLbxYKM7u6AV2Auefegbnn3kFe7rnX+hQMBoPBkE1v1hQMBoPB4MMIBYPBYDC49EqhICInishHIrJMRK7r6vZ0FiJyp4hsEpGF2rYBIvK8iCy1//a3t4uI/N7+Dt4TkWld1/L2IyKjRORlEflQRD4Qkavs7bvsfYtIqYjMFpEF9j3/yN4+TkTetu/tAbv8PCJSYr9fZn8+tktvoJ2ISFxE5onIk/b7Xfp+AURkpYi8LyLzRWSOvS2vfbvXCQURiQN/Ak4C9gLOF5G9urZVncZdwIm+bdcBLyqlJgIv2u/Buv+J9r/Lgb8UqI2dTRL4tlJqL2AG8FX799yV77sJOEYptR+wP3CiiMwAfgncopSaAGwHLrX3vxTYbm+/xd6vJ3IVsEh7v6vfr8PRSqn9tZyE/PZtpVSv+gccAjyrvb8euL6r29WJ9zcWWKi9/wgYbr8eDnxkv74dOD9ov578D/g38Onect9AOfAu1lrmW4CEvd3t51jrkxxiv07Y+0lXt72N9znSHgCPAZ4EZFe+X+2+VwKDfNvy2rd7naYAjADWaO/X2tt2VYYqpdbbrzcAQ+3Xu9z3YJsJpgJvs4vft21KmQ9sAp4HPgZ2KKWS9i76fbn3bH++ExhY0AZ3nFuBa4C0/X4gu/b9OijgORGZKyKX29vy2re73SI7hvyhlFIiskvGIItIH+AR4BtKqWoRcT/bFe9bKZUC9heRKuAxYFLXtih/iMgpwCal1FwROaqLm1NoDlNKrRORIcDzIrJY/zAffbs3agrrgFHa+5H2tl2VjSIyHMD+u8nevst8DyJShCUQ7lVKPWpv3uXvG0AptQN4Gct8UiUizkRPvy/3nu3P+wFbC9vSDnEocJqIrATuxzIh/Y5d935dlFLr7L+bsIT/QeS5b/dGofAOMNGOXCgGzgMe7+I25ZPHgUvs15dg2dyd7RfbEQszgJ2aStpjEEsluANYpJT6rfbRLnvfIjLY1hAQkTIsH8oiLOFwlr2b/56d7+Is4CVlG517Akqp65VSI5VSY7Ge15eUUhewi96vg4hUiEhf5zVwPLCQfPftrnakdJHz5jPAEiw77A1d3Z5OvK/7gPVAC5Y98VIsW+qLwFLgBWCAva9gRWF9DLwPTO/q9rfzng/Dsru+B8y3/31mV75vYF9gnn3PC4Hv29vHA7OBZcBDQIm9vdR+v8z+fHxX30MH7v0o4MnecL/2/S2w/33gjFX57tumzIXBYDAYXHqj+chgMBgMIRihYDAYDAYXIxQMBoPB4GKEgsFgMBhcjFAwGAwGg4sRCgaDhoik7IqUzr+cVXRF5MsicnEnXHeliAzq6HkMho5iQlINBg0RqVVK9emC667EiivfUuhrGww6RlMwGCJgz+RvtmvbzxaRCfb2H4rId+zXXxdrXYf3ROR+e9sAEfmXvW2WiOxrbx8oIs+JtR7C37ASj5xrXWhfY76I3G6XezcYCoIRCgaDlzKf+ehc7bOdSqkpwB+xqnb6uQ6YqpTaF/iyve1HwDx723eBe+ztPwDeUErtjVXTZjSAiEwGzgUOVUrtD6SACzrzBg2GXJgqqQaDlwZ7MA7iPu3vLQGfvwfcKyL/Av5lbzsMOBNAKfWSrSFUAkcAn7W3PyUi2+39jwUOAN6xK72WkSl4ZjDkHSMUDIboqJDXDidjDfanAjeIyJR2XEOAu5VS17fjWIOhwxjzkcEQnXO1v2/pH4hIDBillHoZuBarXHMf4HVs84+9FsAWpVQ18BrwOXv7SUB/+1QvAmfZ9fMdn8SY/N2SweDFaAoGg5cye0Uzh2eUUk5Yan8ReQ9rjeTzfcfFgX+ISD+s2f7vlVI7ROSHwJ32cfVkSh7/CLhPRD4A3gRWAyilPhSRG7FW24phVbz9KrCqk+/TYAjEhKQaDBEwIaOG3oIxHxkMBoPBxWgKBoPBYHAxmoLBYDAYXIxQMBgMBoOLEQoGg8FgcDFCwWAwGAwuRigYDAaDweX/AblEbfbXIj3GAAAAAElFTkSuQmCC", "text/plain": [ "
" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "num_episodes = 500\n", "fig = plt.figure()\n", "ax = fig.add_subplot(1, 1, 1)\n", "for i_episode in range(num_episodes):\n", " # Initialize the environment and state\n", " env.reset()\n", " last_screen = get_screen()\n", " current_screen = get_screen()\n", " state = current_screen - last_screen\n", " for t in count():\n", " # Select and perform an action\n", " action = select_action(state)\n", " _, reward, done, _ = env.step(action.item())\n", " reward = torch.tensor([reward], device=device)\n", "\n", " # Observe new state\n", " last_screen = current_screen\n", " current_screen = get_screen()\n", " if not done:\n", " next_state = current_screen - last_screen\n", " else:\n", " next_state = None\n", "\n", " # Store the transition in memory\n", " memory.push(state, action, next_state, reward)\n", "\n", " # Move to the next state\n", " state = next_state\n", "\n", " # Perform one step of the optimization (on the target network)\n", " optimize_model()\n", " if done:\n", " episode_durations.append(t + 1)\n", " plot_durations(ax, fig)\n", " break\n", " # Update the target network once in a while, copying all weights and biases in DQN\n", " if i_episode % TARGET_UPDATE == 0:\n", " target_net.load_state_dict(policy_net.state_dict())\n", " torch.save(policy_net.state_dict(), f\"dqn_model_episode{i_episode}.pth\")" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "try out the policy from the learned model" ] }, { "cell_type": "code", "execution_count": 16, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "frames: 46\n" ] }, { "data": { "text/html": [ "\n", "\n", "\n", "\n", "\n", "\n", "
\n", " \n", "
\n", " \n", "
\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
\n", "
\n", " \n", " \n", " \n", " \n", " \n", " \n", "
\n", "
\n", "
\n", "\n", "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "# load model\n", "policy_net.load_state_dict(torch.load('dqn_model_episode470.pth'))\n", "\n", "env.reset()\n", "last_screen = get_screen()\n", "current_screen = get_screen()\n", "state = current_screen - last_screen\n", "# %matplotlib ipympl\n", "\n", "fig = plt.figure()\n", "ims = []\n", "while True:\n", " # select action\n", " with torch.no_grad():\n", " action = policy_net(state).max(1)[1].view(1, 1)\n", " _, _, done, _ = env.step(action.item())\n", " \n", " last_screen = current_screen\n", " current_screen = get_screen()\n", " state = current_screen - last_screen\n", "\n", " # visualization\n", " ims.append([plt.imshow(env.render(mode='rgb_array'))])\n", " if done:\n", " break\n", "print(f\"frames: {len(ims)}\")\n", "ani = animation.ArtistAnimation(fig, ims, interval=100)\n", "html = HTML(ani.to_jshtml())\n", "display(html)\n", "plt.close()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "works fairly well!" ] } ], "metadata": { "interpreter": { "hash": "916dbcbb3f70747c44a77c7bcd40155683ae19c65e1c03b4aa3499c5328201f1" }, "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.10.4" } }, "nbformat": 4, "nbformat_minor": 2 }