{ "nbformat": 4, "nbformat_minor": 0, "metadata": { "colab": { "provenance": [], "collapsed_sections": [ "UxK08xN_wco-", "qTS3Lm1nWiMK" ], "toc_visible": true }, "kernelspec": { "name": "python3", "display_name": "Python 3" }, "language_info": { "name": "python" }, "widgets": { "application/vnd.jupyter.widget-state+json": { "e77a04ad372b47e89944d1524a8777c1": { "model_module": "@jupyter-widgets/controls", "model_name": "HBoxModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HBoxView", "box_style": "", "children": [ "IPY_MODEL_21e16d6d8e4d4e08bc3d26b1ffcc10bd", "IPY_MODEL_b35662ce35744a2e80c495da554b22fb", "IPY_MODEL_c190a3bf3649425aa55c17e1e820b3d6" ], "layout": "IPY_MODEL_f359ca695a7d4f1e9b7f2422ed1b0e8f" } }, "21e16d6d8e4d4e08bc3d26b1ffcc10bd": { "model_module": "@jupyter-widgets/controls", "model_name": "HTMLModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_225658a1cf92482b95c6a55fc6ed30d8", "placeholder": "​", "style": "IPY_MODEL_6cbf426e997c40c5982949b9ebf4cd3f", "value": "Action = -0.28 | Reward = -3.93: 100%" } }, "b35662ce35744a2e80c495da554b22fb": { "model_module": "@jupyter-widgets/controls", "model_name": "FloatProgressModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "ProgressView", "bar_style": "success", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_eee1d763560e4a44bf8cf30f1f38718b", "max": 500, "min": 0, "orientation": "horizontal", "style": "IPY_MODEL_93f84f7a1c9645e489f282085995692d", "value": 500 } }, "c190a3bf3649425aa55c17e1e820b3d6": { "model_module": "@jupyter-widgets/controls", "model_name": "HTMLModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_3ae83ec6a80a43a7858ccb1b6b6da579", "placeholder": "​", "style": "IPY_MODEL_726ead1902e44663a75600233e7930ea", "value": " 500/500 [00:15<00:00, 91.08it/s]" } }, "f359ca695a7d4f1e9b7f2422ed1b0e8f": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "225658a1cf92482b95c6a55fc6ed30d8": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "6cbf426e997c40c5982949b9ebf4cd3f": { "model_module": "@jupyter-widgets/controls", "model_name": "DescriptionStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "eee1d763560e4a44bf8cf30f1f38718b": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "93f84f7a1c9645e489f282085995692d": { "model_module": "@jupyter-widgets/controls", "model_name": "ProgressStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "bar_color": null, "description_width": "" } }, "3ae83ec6a80a43a7858ccb1b6b6da579": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "726ead1902e44663a75600233e7930ea": { "model_module": "@jupyter-widgets/controls", "model_name": "DescriptionStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "d95ca03a9a8f412a9cd1a7715e428dcc": { "model_module": "@jupyter-widgets/controls", "model_name": "HBoxModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HBoxView", "box_style": "", "children": [ "IPY_MODEL_103ce5a1781d482dbb319b3ee8348f1d", "IPY_MODEL_37872f88dac04d2496cfc5aeff0b5eeb", "IPY_MODEL_4dd2a5f7f7bf478cb0d7765b51dde923" ], "layout": "IPY_MODEL_8a9ea81f0a914869a47bbfe19c8b138e" } }, "103ce5a1781d482dbb319b3ee8348f1d": { "model_module": "@jupyter-widgets/controls", "model_name": "HTMLModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_619b046f60be4cc897643d33869aaf7e", "placeholder": "​", "style": "IPY_MODEL_12fcfe38f0dc4aba897c00b9a103fd91", "value": "Action = -0.00 | Reward = -0.00: 100%" } }, "37872f88dac04d2496cfc5aeff0b5eeb": { "model_module": "@jupyter-widgets/controls", "model_name": "FloatProgressModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "ProgressView", "bar_style": "success", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_73013360efe04199b565e43956b8248b", "max": 500, "min": 0, "orientation": "horizontal", "style": "IPY_MODEL_25bb2652a8454e72900b940b74efd18a", "value": 500 } }, "4dd2a5f7f7bf478cb0d7765b51dde923": { "model_module": "@jupyter-widgets/controls", "model_name": "HTMLModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_85fd867ff2d740ebb4923345b22d3c01", "placeholder": "​", "style": "IPY_MODEL_a8d68c64556742d49a8984bf565009d2", "value": " 500/500 [00:40<00:00, 17.75it/s]" } }, "8a9ea81f0a914869a47bbfe19c8b138e": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "619b046f60be4cc897643d33869aaf7e": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "12fcfe38f0dc4aba897c00b9a103fd91": { "model_module": "@jupyter-widgets/controls", "model_name": "DescriptionStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "73013360efe04199b565e43956b8248b": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "25bb2652a8454e72900b940b74efd18a": { "model_module": "@jupyter-widgets/controls", "model_name": "ProgressStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "bar_color": null, "description_width": "" } }, "85fd867ff2d740ebb4923345b22d3c01": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "a8d68c64556742d49a8984bf565009d2": { "model_module": "@jupyter-widgets/controls", "model_name": "DescriptionStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } } } } }, "cells": [ { "cell_type": "markdown", "source": [ "# HW5: Model Predictive Control\n", "\n", "> **Solution**\n", "\n", "\n", "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/DeepRLCourse/Homework-5-Questions/blob/main/RL_HW5_MPC.ipynb)\n", "[![Open In kaggle](https://kaggle.com/static/images/open-in-kaggle.svg)](https://kaggle.com/kernels/welcome?src=https://raw.githubusercontent.com/DeepRLCourse/Homework-5-Questions/main/RL_HW5_MPC.ipynb)\n", "\n", "## Overview\n", "Here the goal is to use **MPC** for [gymnasium environments](https://gymnasium.farama.org/).\n", "More specificly we focus on the [Pendulum](https://gymnasium.farama.org/environments/classic_control/pendulum/) environment and try to solve it using [mpc.pytorch](https://locuslab.github.io/mpc.pytorch/).\n", "\n", "\n" ], "metadata": { "id": "UxK08xN_wco-" } }, { "cell_type": "code", "source": [ "# @title Imports\n", "\n", "# Stuff you (might) need\n", "import random\n", "import numpy as np\n", "import gymnasium as gym\n", "\n", "import torch\n", "from torch import nn\n", "import torch.autograd\n", "from tqdm.notebook import trange\n", "import math\n", "\n", "# Stuff used for visualization\n", "from matplotlib import pyplot as plt\n", "from gymnasium.wrappers import RecordVideo\n", "import base64\n", "import imageio\n", "from IPython.display import HTML\n", "import warnings\n", "warnings.filterwarnings(\"ignore\")" ], "metadata": { "id": "SBD13rNwmXpx", "cellView": "form" }, "execution_count": null, "outputs": [] }, { "cell_type": "code", "source": [ "# @title Visualization Functions\n", "\n", "def embed_mp4(filename):\n", " video = open(filename,'rb').read()\n", " b64 = base64.b64encode(video)\n", " tag = '''\n", " '''.format(b64.decode())\n", "\n", " return HTML(tag)\n", "\n", "\n", "def plot_results(rewards, actions):\n", " plt.plot(rewards, label='Rewards')\n", " plt.plot(actions, label='Actions')\n", " plt.legend()\n", " plt.title(f\"Total reward: {sum(rewards):.2f}\")\n", " plt.show()" ], "metadata": { "cellView": "form", "id": "Avcbl8VMmgm1" }, "execution_count": null, "outputs": [] }, { "cell_type": "markdown", "source": [ "# Explore the Environment (25 points)" ], "metadata": { "id": "qTS3Lm1nWiMK" } }, { "cell_type": "markdown", "source": [ "To better understand the environment, let's first see what a random agent does." ], "metadata": { "id": "yyCz0lxFWsFv" } }, { "cell_type": "code", "source": [ "# Initialize the pendulum environment with video recording enabled\n", "env = gym.make('Pendulum-v1', render_mode='rgb_array')\n", "\n", "# Create a directory to save the video\n", "video_directory = \"random_videos\"\n", "env = RecordVideo(env, video_directory)\n", "\n", "# Set the number of steps to record\n", "num_steps = 500\n", "\n", "# TODO: Reset the environment to get the initial state\n", "state, info = env.reset()\n", "\n", "for _ in (pbar := trange(num_steps)):\n", " # TODO: Sample a random action\n", " action = env.action_space.sample()\n", "\n", " # TODO: Step the environment\n", " state, reward, terminated, truncated, info = env.step(action)\n", "\n", " # TODO: Render the environment\n", " env.render()\n", "\n", " # TODO: If done reset and get new state\n", " if terminated or truncated:\n", " state, info = env.reset()\n", "\n", " pbar.set_description(f'Action = {action[0]:.2f} | Reward = {reward:.2f}')\n", "\n", "# Close the environment to finalize the video\n", "env.close()\n", "\n", "# Show the video\n", "embed_mp4(f'{video_directory}/rl-video-episode-0.mp4')" ], "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 533, "referenced_widgets": [ "e77a04ad372b47e89944d1524a8777c1", "21e16d6d8e4d4e08bc3d26b1ffcc10bd", "b35662ce35744a2e80c495da554b22fb", "c190a3bf3649425aa55c17e1e820b3d6", "f359ca695a7d4f1e9b7f2422ed1b0e8f", "225658a1cf92482b95c6a55fc6ed30d8", "6cbf426e997c40c5982949b9ebf4cd3f", "eee1d763560e4a44bf8cf30f1f38718b", "93f84f7a1c9645e489f282085995692d", "3ae83ec6a80a43a7858ccb1b6b6da579", "726ead1902e44663a75600233e7930ea" ] }, "id": "xZI6WO4bmgjS", "outputId": "79f3ff19-78b3-46ff-b6f5-2a8d12d1483a" }, "execution_count": null, "outputs": [ { "output_type": "display_data", "data": { "text/plain": [ " 0%| | 0/500 [00:00" ], "text/html": [ "\n", " " ] }, "metadata": {}, "execution_count": 3 } ] }, { "cell_type": "markdown", "source": [ "The goal of the [Pendulum](https://gymnasium.farama.org/environments/classic_control/pendulum/) environment in [Gymnasium](https://gymnasium.farama.org/) is to swing a pendulum to an upright position and keep it balanced there.\n", "In this environment, you control a torque that can be applied to the pendulum.\n", "The objective is to apply the right amount of torque to swing the pendulum up and maintain its upright position." ], "metadata": { "id": "-8SUlhUHWvbC" } }, { "cell_type": "markdown", "source": [ "## Simulation Tools" ], "metadata": { "id": "wIGVVtRU2kY4" } }, { "cell_type": "markdown", "source": [ "Both the `angle_normalize` function and the `PendulumDynamics` class are fundamental components for accurately simulating, analyzing, and controlling the pendulum system.\n", "They ensure consistency in angle representation and provide a realistic model of the pendulum's behavior, enabling effective control strategies.\n", "\n", "We use the `angle_normalize` function for:\n", "\n", "* **Consistency**: When dealing with angles, it's important to keep them within a standard range to ensure consistency in calculations.\n", "* **Handling Wrapping**: Angles can wrap around when they exceed $2\\pi$ or drop below $-2\\pi$. Normalizing angles helps avoid confusion and errors that can arise from angle wrapping.\n", "\n", "\n", "And we use the `PendulumDynamics` class for:\n", "\n", "* **Modeling Physical Behavior**: The `PendulumDynamics` class models the physical behavior of the pendulum.\n", "* **Simulation and Control**: This class allows us to simulate the pendulum's response to different actions, which is crucial for designing and testing control algorithms.\n", "* **Optimization**: Understanding the dynamics of the pendulum helps in optimizing the control inputs. The class encapsulates the physics involved, enabling us to apply control techniques like Model Predictive Control (MPC) to achieve the desired behavior.\n", "\n", "\n" ], "metadata": { "id": "s7oVXleZ2rY8" } }, { "cell_type": "code", "source": [ "class PendulumDynamics(nn.Module):\n", " def forward(self, state, action):\n", " th = state[:, 0].view(-1, 1)\n", " thdot = state[:, 1].view(-1, 1)\n", "\n", " g = 10 # default value of the environment (not 9.81)\n", " m = 1\n", " l = 1\n", " dt = 0.05\n", "\n", " u = action\n", " u = torch.clamp(u, -2, 2)\n", "\n", " newthdot = thdot + (-3 * g / (2 * l) * torch.sin(th + np.pi) + 3. / (m * l ** 2) * u) * dt\n", " newth = th + newthdot * dt\n", " newthdot = torch.clamp(newthdot, -8, 8)\n", "\n", " state = torch.cat((angle_normalize(newth), newthdot), dim=1)\n", " return state\n", "\n", "\n", "def angle_normalize(x):\n", " return (((x + math.pi) % (2 * math.pi)) - math.pi)" ], "metadata": { "id": "N9y8iJ0u2Lgp" }, "execution_count": null, "outputs": [] }, { "cell_type": "markdown", "source": [ "# Model Predictive Control (50 points)" ], "metadata": { "id": "eSb9nKlw4Gra" } }, { "cell_type": "markdown", "source": [ "[mpc.pytorch](https://locuslab.github.io/mpc.pytorch/) is a library that provides a fast and differentiable [Model Predictive Control](https://en.wikipedia.org/wiki/Model_predictive_control) (MPC) solver for PyTorch. It was developed by researchers at [LocusLab](https://locuslab.github.io/) and is designed to integrate seamlessly with PyTorch, allowing for efficient and flexible control of dynamic systems.\n", "\n", "If you are interested to learn more, check out [OptNet](https://arxiv.org/abs/1703.00443) and [Differentiable MPC](https://arxiv.org/abs/1810.13400)." ], "metadata": { "id": "yc8yKVti4tMp" } }, { "cell_type": "markdown", "source": [ "## Quick Setup" ], "metadata": { "id": "PfDVMqgi5kax" } }, { "cell_type": "markdown", "source": [ "In order to install this library you can use `pip`:" ], "metadata": { "id": "qU9dpT0C5nFg" } }, { "cell_type": "code", "source": [ "! pip install mpc" ], "metadata": { "id": "HEyJ0SPi4W28", "colab": { "base_uri": "https://localhost:8080/" }, "outputId": "4d6a3b79-6d90-4cf4-eabf-93c576aa0521" }, "execution_count": null, "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ "Requirement already satisfied: mpc in /usr/local/lib/python3.11/dist-packages (0.0.6)\n", "Requirement already satisfied: numpy<2,>=1 in /usr/local/lib/python3.11/dist-packages (from mpc) (1.26.4)\n", "Requirement already satisfied: torch in /usr/local/lib/python3.11/dist-packages (from mpc) (2.5.1+cu124)\n", "Requirement already satisfied: filelock in /usr/local/lib/python3.11/dist-packages (from torch->mpc) (3.17.0)\n", "Requirement already satisfied: typing-extensions>=4.8.0 in /usr/local/lib/python3.11/dist-packages (from torch->mpc) (4.12.2)\n", "Requirement already satisfied: networkx in /usr/local/lib/python3.11/dist-packages (from torch->mpc) (3.4.2)\n", "Requirement already satisfied: jinja2 in /usr/local/lib/python3.11/dist-packages (from torch->mpc) (3.1.5)\n", "Requirement already satisfied: fsspec in /usr/local/lib/python3.11/dist-packages (from torch->mpc) (2024.10.0)\n", "Requirement already satisfied: nvidia-cuda-nvrtc-cu12==12.4.127 in /usr/local/lib/python3.11/dist-packages (from torch->mpc) (12.4.127)\n", "Requirement already satisfied: nvidia-cuda-runtime-cu12==12.4.127 in /usr/local/lib/python3.11/dist-packages (from torch->mpc) (12.4.127)\n", "Requirement already satisfied: nvidia-cuda-cupti-cu12==12.4.127 in /usr/local/lib/python3.11/dist-packages (from torch->mpc) (12.4.127)\n", "Requirement already satisfied: nvidia-cudnn-cu12==9.1.0.70 in /usr/local/lib/python3.11/dist-packages (from torch->mpc) (9.1.0.70)\n", "Requirement already satisfied: nvidia-cublas-cu12==12.4.5.8 in /usr/local/lib/python3.11/dist-packages (from torch->mpc) (12.4.5.8)\n", "Requirement already satisfied: nvidia-cufft-cu12==11.2.1.3 in /usr/local/lib/python3.11/dist-packages (from torch->mpc) (11.2.1.3)\n", "Requirement already satisfied: nvidia-curand-cu12==10.3.5.147 in /usr/local/lib/python3.11/dist-packages (from torch->mpc) (10.3.5.147)\n", "Requirement already satisfied: nvidia-cusolver-cu12==11.6.1.9 in /usr/local/lib/python3.11/dist-packages (from torch->mpc) (11.6.1.9)\n", "Requirement already satisfied: nvidia-cusparse-cu12==12.3.1.170 in /usr/local/lib/python3.11/dist-packages (from torch->mpc) (12.3.1.170)\n", "Requirement already satisfied: nvidia-nccl-cu12==2.21.5 in /usr/local/lib/python3.11/dist-packages (from torch->mpc) (2.21.5)\n", "Requirement already satisfied: nvidia-nvtx-cu12==12.4.127 in /usr/local/lib/python3.11/dist-packages (from torch->mpc) (12.4.127)\n", "Requirement already satisfied: nvidia-nvjitlink-cu12==12.4.127 in /usr/local/lib/python3.11/dist-packages (from torch->mpc) (12.4.127)\n", "Requirement already satisfied: triton==3.1.0 in /usr/local/lib/python3.11/dist-packages (from torch->mpc) (3.1.0)\n", "Requirement already satisfied: sympy==1.13.1 in /usr/local/lib/python3.11/dist-packages (from torch->mpc) (1.13.1)\n", "Requirement already satisfied: mpmath<1.4,>=1.1.0 in /usr/local/lib/python3.11/dist-packages (from sympy==1.13.1->torch->mpc) (1.3.0)\n", "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.11/dist-packages (from jinja2->torch->mpc) (3.0.2)\n" ] } ] }, { "cell_type": "markdown", "source": [ "While `mpc` offers a lot, in this notebook we are going to focus only on the core features.\n", "To learn more checkout the [GitHub repository](https://github.com/locuslab/mpc.pytorch) of this project." ], "metadata": { "id": "J9Ev1gLC5uGd" } }, { "cell_type": "code", "source": [ "from mpc import mpc" ], "metadata": { "id": "euuwSTJ14WO-" }, "execution_count": null, "outputs": [] }, { "cell_type": "markdown", "source": [ "⚠️ If you're running this noteook in colab, please **restart the session**.\n", "Go to _Runtime_ and select the _Restart session_ option.\n" ], "metadata": { "id": "xI9d0IuQEjPT" } }, { "cell_type": "markdown", "source": [ "## The Cost Function" ], "metadata": { "id": "tf0ALVu7Ggsh" } }, { "cell_type": "markdown", "source": [ "The `define_swingup_goal` function creates a cost function that the MPC algorithm uses to determine the optimal control actions to achieve the desired pendulum swing-up task.\n", "It considers both the desired state (upright and stationary) and penalizes large control inputs to ensure smooth control actions." ], "metadata": { "id": "l9pgPno8GjFV" } }, { "cell_type": "code", "source": [ "def define_swingup_goal():\n", " goal_weights = torch.tensor((1., 0.1)) # Weights for theta and theta_dot\n", " goal_state = torch.tensor((0., 0.)) # Desired state (theta=0, theta_dot=0)\n", " ctrl_penalty = 0.001\n", " q = torch.cat((goal_weights, ctrl_penalty * torch.ones(1))) # Combined weights\n", " px = -torch.sqrt(goal_weights) * goal_state\n", " p = torch.cat((px, torch.zeros(1)))\n", " Q = torch.diag(q).repeat(TIMESTEPS, N_BATCH, 1, 1) # Cost matrix\n", " p = p.repeat(TIMESTEPS, N_BATCH, 1)\n", " return mpc.QuadCost(Q, p) # Quadratic cost" ], "metadata": { "id": "13hsSR8cGLzA" }, "execution_count": null, "outputs": [] }, { "cell_type": "markdown", "source": [ "## Running MPC" ], "metadata": { "id": "4ILZGtcdLeFb" } }, { "cell_type": "markdown", "source": [ "To run the MPC, in each iteration:\n", "\n", "1. First you obtain the current state of the environment and convert it to a tensor.\n", "2. Then you recreate the MPC controller using the updated `u_init` and calculate the optimal control actions based on the current state, dynamics, and cost function.\n", "3. Next you take the first planned action and update `u_init` with the remaining actions.\n", "4. Finally you take a step in the environment and store the rewards and actions.\n", "\n", "Remember that `u_init` serves as the initial guess for the control inputs." ], "metadata": { "id": "JskzIJuLMpoe" } }, { "cell_type": "code", "source": [ "# Select the hyperparameters\n", "RUN_ITER = 500\n", "TIMESTEPS = 10\n", "N_BATCH = 1\n", "LQR_ITER = 5\n", "\n", "\n", "# Setup the environmnet\n", "env = gym.make('Pendulum-v1', render_mode='rgb_array')\n", "video_directory = \"mpc_videos\"\n", "env = RecordVideo(env, video_directory)\n", "env.reset() # Reset the underlying environment\n", "env.unwrapped.state = [np.pi, 1] # Environment must start in downward position\n", "\n", "\n", "# Define the cost function and initialize u\n", "cost = define_swingup_goal()\n", "u_init = None\n", "\n", "\n", "# Run MPC\n", "rewards, actions = [], []\n", "for _ in (pbar := trange(RUN_ITER)):\n", " state = env.unwrapped.state.copy()\n", " state = torch.tensor(state).view(1, -1)\n", " # recreate controller using updated u_init (kind of wasteful right?)\n", " ctrl = mpc.MPC(2, 1, TIMESTEPS, u_lower=-2.0, u_upper=+2.0,\n", " lqr_iter=LQR_ITER, exit_unconverged=False, eps=1e-2,\n", " n_batch=N_BATCH, backprop=False, verbose=0, u_init=u_init,\n", " grad_method=mpc.GradMethods.AUTO_DIFF)\n", "\n", " # compute action based on current state, dynamics, and cost\n", " nominal_states, nominal_actions, nominal_objs = ctrl(state, cost, PendulumDynamics())\n", " # TODO: Take first planned action\n", " action = nominal_actions[0]\n", " u_init = torch.cat((nominal_actions[1:], torch.zeros(1, N_BATCH, 1)), dim=0)\n", "\n", " # TODO: Take a step in the environment\n", " s, r, _, _, _ = env.step(action.detach().numpy())\n", "\n", " # TODO: Store the latest action and reward\n", " rewards.append(r.item())\n", " actions.append(action.detach().numpy().item())\n", "\n", " pbar.set_description(f\"Action = {actions[-1]:.2f} | Reward = {rewards[-1]:.2f}\")\n", " env.render()\n", "\n", "env.close()\n", "\n", "# Plot the results\n", "plot_results(rewards, actions)\n", "\n", "# Show the policy learned\n", "embed_mp4(f'{video_directory}/rl-video-episode-0.mp4')" ], "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 1000, "referenced_widgets": [ "d95ca03a9a8f412a9cd1a7715e428dcc", "103ce5a1781d482dbb319b3ee8348f1d", "37872f88dac04d2496cfc5aeff0b5eeb", "4dd2a5f7f7bf478cb0d7765b51dde923", "8a9ea81f0a914869a47bbfe19c8b138e", "619b046f60be4cc897643d33869aaf7e", "12fcfe38f0dc4aba897c00b9a103fd91", "73013360efe04199b565e43956b8248b", "25bb2652a8454e72900b940b74efd18a", "85fd867ff2d740ebb4923345b22d3c01", "a8d68c64556742d49a8984bf565009d2" ] }, "id": "tE45H5NI05Q_", "outputId": "d877a84a-5170-4842-f121-6c33b7d163ae" }, "execution_count": null, "outputs": [ { "output_type": "display_data", "data": { "text/plain": [ " 0%| | 0/500 [00:00" ], "image/png": "iVBORw0KGgoAAAANSUhEUgAAAioAAAGzCAYAAAABsTylAAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjAsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvlHJYcgAAAAlwSFlzAAAPYQAAD2EBqD+naQAAWE5JREFUeJzt3Xl8U1X6P/DPzdp9oy1QKYUCsimLoFhkBwVFEUcRGJ0BRfip4FcGBoe6goroyLiA4zpYVHAZF3QEVEAWQVEUAQVkU6AIlBYo3Zs0yfn9kdzbpEuaPbn183698mpzc3NzesPMeXzOc86RhBACRERERBFIE+4GEBERETWGgQoRERFFLAYqREREFLEYqBAREVHEYqBCREREEYuBChEREUUsBipEREQUsRioEBERUcRioEJEREQRi4EKUTOyadMmSJKETZs2hbspIbds2TJIkoSjR4+GuylEFEAMVIj8JEmSRw9PgocnnngCH3/8cdDbTJ574okncPnllyMtLQ1RUVHo1KkTZs6ciaKiIpfz5s2b5/b7//rrr5VzX3vtNQwePBgtW7aE0WhE+/btcdttt3kcZA0ZMqTBzxg1alSD5//4448YM2YMUlJSEBMTg4suugiLFy/2+Z4QhZIu3A0gUru33nrL5fmbb76JdevW1TvetWvXJq/1xBNP4KabbsLYsWMD2UTyw44dO9CrVy9MmDAB8fHx+OWXX/Daa69h9erV2LVrF2JjYwEAf/rTn9CxY8d677///vtRXl6OSy+9VDm2c+dOtG/fHmPGjEFycjKOHDmC1157DatWrcLu3buRkZHRZLvatGmDhQsXuhxr6H1r167Fddddh969e+Ohhx5CXFwcfv31V/z+++/e3gqisGCgQuSnW2+91eX5t99+i3Xr1tU7HukqKiqUTjcS2Ww2mM1mREVFhfRzP/zww3rHcnJycNNNN+HTTz/FhAkTAAA9evRAjx49XM47fvw4fv/9d9xxxx0wGAzK8RdffLHeNceOHYu+ffvizTffxNy5c5tsV2JiYpP/xkpLS/HXv/4Vo0ePxgcffACNhkl0Uh/+qyUKgYqKCsyePRuZmZkwGo3o3LkzFi1aBOfNyyVJQkVFBd544w0llT958mQAwLFjx3D33Xejc+fOiI6ORosWLTBu3Dif6zHkYYp9+/bhz3/+M5KTkzFgwADl9eXLl6NPnz6Ijo5GSkoKJkyYgOPHjyuvL168GFqtFufPn1eO/etf/4IkSZg1a5ZyzGq1Ij4+Hv/4xz+UY4sWLUL//v3RokULREdHo0+fPvjggw/qtVGSJMyYMQMrVqxA9+7dYTQa8fnnnwMA9u7di2HDhiE6Ohpt2rTB448/DpvNVu8aJSUl2L9/P0pKSny6T41p164dALj8/Q155513IITALbfcErBrOrNYLCgvL2/09bfffhunT5/GggULoNFoUFFR0eB9IopkzKgQBZkQAmPGjMHGjRsxZcoU9OrVC1988QXmzJmDEydO4NlnnwVgH0K64447cNlll2HatGkAgA4dOgAAvv/+e3zzzTeYMGEC2rRpg6NHj+Kll17CkCFDsG/fPsTExPjUtnHjxqFTp0544oknlKBpwYIFeOihh3DzzTfjjjvuQFFREZYsWYJBgwZh586dSEpKwsCBA2Gz2bB161Zce+21AIAtW7ZAo9Fgy5YtyvV37tyJ8vJyDBo0SDn2/PPPY8yYMbjllltgNpvx7rvvYty4cVi1ahVGjx7t0r4NGzbgv//9L2bMmIHU1FS0a9cOBQUFGDp0KCwWC+bOnYvY2Fi8+uqriI6Orvf3rVy5Erfddhvy8vKUoM8XQgicPXsWFosFhw4dwty5c6HVajFkyBC371uxYgUyMzNd/n5nZ8+ehdVqRX5+Ph599FEAwPDhwz1q08GDBxEbGwuz2YyWLVti6tSpePjhh6HX65Vz1q9fj4SEBJw4cQJjx45V3vOXv/wFzz77bMizU0Q+EUQUUNOnTxfO/9P6+OOPBQDx+OOPu5x30003CUmSxOHDh5VjsbGxYtKkSfWuWVlZWe/Ytm3bBADx5ptvKsc2btwoAIiNGze6beMjjzwiAIiJEye6HD969KjQarViwYIFLsd//vlnodPplONWq1UkJCSI++67TwghhM1mEy1atBDjxo0TWq1WlJWVCSGEeOaZZ4RGoxHFxcWN/i1ms1lcdNFFYtiwYS7HAQiNRiP27t3rcnzmzJkCgPjuu++UY4WFhSIxMVEAEEeOHFGO5+XlCQAiLy/P7f1oyqlTpwQA5dGmTRvx3nvvuX3Pnj17BADlHjXEaDQq12zRooVYvHixR+25/fbbxbx588SHH34o3nzzTTFmzBgBQNx8880u5/Xo0UPExMSImJgYcc8994gPP/xQ3HPPPQKAmDBhgkefRRRuDFSIAqxuoDJt2jSh1WpFaWmpy3lyoLFkyRLlWGOBijOz2SzOnDkjioqKRFJSkpg5c6bymreByubNm12OP/PMM0KSJHHo0CFRVFTk8ujatasYMWKEcu6oUaPE5ZdfLoQQYu/evQKA2LFjh9BoNGLt2rVCCCFuuOEG0aNHj0bbce7cOVFUVCTuuusukZSU5PIaADF06NB677nwwguVz3V299131wtUAsVkMol169aJTz/9VDz66KOiV69eYunSpW7fk5ubKwCI3bt3N3rOhg0bxJo1a8S//vUv0bt3b7Fw4UKf2zh16lQBQGzbtk05lp2dLQCIO++80+Xc//f//p8AIA4ePOjz5xGFCod+iILs2LFjyMjIQHx8vMtxeRbQsWPHmrxGVVUVFi5ciLy8PJw4ccKltsWf+ov27du7PD906BCEEOjUqVOD5zsPKwwcOBDz5s1DVVUVtmzZgtatW+OSSy5Bz549sWXLFlx55ZXYunUrbr75ZpdrrFq1Co8//jh27doFk8mkHJckqcn2Afb71a9fv3rHO3fu7P6PdaOkpARVVVXKc4PBgJSUFJfnI0aMAABce+21GD58OK644gqkp6crQ1/OhBB4++23cdFFF9UrsHU2dOhQAMDVV1+N66+/HhdddBHi4uIwY8YMr/+G2bNn47XXXsP69etx+eWXA4AyHDZx4kSXc//85z/jlVdewbZt2xr9rokiBQMVIhW45557kJeXh5kzZyInJweJiYmQJAkTJkzwqziybl2HzWaDJEn47LPPoNVq650fFxen/D5gwADU1NRg27Zt2LJlCwYOHAjAHsBs2bIF+/fvR1FRkXIcsNexjBkzBoMGDcKLL76I1q1bQ6/XIy8vD2+//XaT7QuWe++9F2+88YbyfPDgwW7Xvenfvz9at26NFStWNBiofP311zh27Fi96cPudOjQAb1798aKFSt8ClQyMzMBAOfOnVOOZWRkYO/evWjZsqXLuenp6QCA4uJirz+HKNQYqBAFWVZWFtavX4+ysjKXrMr+/fuV12UNZRUA4IMPPsCkSZPwr3/9SzlWXV3t1QwRT3To0AFCCLRv3x4XXnih23Mvu+wyGAwGbNmyBVu2bMGcOXMAAIMGDcJrr72GL7/8Unku+/DDDxEVFYUvvvgCRqNROZ6Xl+dxG7OysnDo0KF6xw8cOODxNeq67777XKb6JicnN/me6urqRrNZK1asgCRJ+POf/+xVO6qqqlyyTN747bffAABpaWnKsT59+mDdunU4ceKES8bp5MmT9c4lilScnkwUZNdccw2sViteeOEFl+PPPvssJEnC1VdfrRyLjY1tMPjQarUuwz0AsGTJElit1oC29U9/+hO0Wi3mz59f7/OEY+aLLCoqCpdeeineeecd5Ofnu2RUqqqqsHjxYnTo0AGtW7d2+TskSXJp99GjR71ajfeaa67Bt99+i+3btyvHioqKsGLFinrnejo9uVu3bhgxYoTy6NOnDwD7tPLKysp653/44YcoLi5G3759671WU1OD999/HwMGDEDbtm3rvW6xWBrMZGzfvh0///xzvWvu378f+fn5yvPS0tJ6wYwQAo8//jgAYOTIkcpxedht6dKlLuf/5z//gU6na3LWElEkYEaFKMiuu+46DB06FA888ACOHj2Knj17Yu3atfjkk08wc+ZMZQoyYP8v4PXr1+OZZ55BRkYG2rdvj379+uHaa6/FW2+9hcTERHTr1g3btm3D+vXr0aJFi4C2tUOHDnj88ceRm5uLo0ePYuzYsYiPj8eRI0ewcuVKTJs2DX//+9+V8wcOHIgnn3wSiYmJuPjiiwHYhxU6d+6MAwcO1JsSPHr0aDzzzDMYNWoU/vznP6OwsBD//ve/0bFjR/z0008etfG+++7DW2+9hVGjRuHee+9VpidnZWXVu4a/05MPHTqEESNGYPz48ejSpQs0Gg1++OEHLF++HO3atcO9995b7z1ffPEFzp492+jaKeXl5cjMzMT48ePRvXt3xMbG4ueff0ZeXh4SExPx0EMPuZzftWtXl6GoH3/8ERMnTsTEiRPRsWNHVFVVYeXKlfj6668xbdo0XHLJJcp7e/fujdtvvx2vv/46LBaLcp33338fubm5Hq2ASxR24avjJWqe6s76EUKIsrIy8be//U1kZGQIvV4vOnXqJJ5++mlhs9lcztu/f78YNGiQiI6OFgCUGUDFxcXitttuE6mpqSIuLk6MHDlS7N+/X2RlZbnMEvJ21k9RUVGDr3/44YdiwIABIjY2VsTGxoouXbqI6dOniwMHDrict3r1agFAXH311S7H77jjDgGgwZkxS5cuFZ06dRJGo1F06dJF5OXlKe1xBkBMnz69wfb99NNPYvDgwSIqKkpccMEF4rHHHhNLly4N+PTkoqIiMW3aNNGlSxcRGxsrDAaD6NSpk5g5c2aj927ChAlCr9eLs2fPNvi6yWQS9957r+jRo4dISEgQer1eZGVliSlTpjQ4YwmAGDx4sPL8t99+E+PGjRPt2rUTUVFRIiYmRvTp00e8/PLL9f49CWGfJTZv3jyRlZUl9Hq96Nixo3j22Wd9uR1EYSEJUSe/S0RERBQhWKNCREREEYuBChEREUUsBipEREQUsRioEBERUcRioEJEREQRi4EKERERRSzVL/hms9lw8uRJxMfHN7r8OBEREUUWIQTKysqQkZEBjabxvInqA5WTJ08qm3ERERGRuhw/fhxt2rRp9HXVByryJm/Hjx9HQkJCmFtDREREnigtLUVmZqbLZq0NUX2gIg/3JCQkMFAhIiJSmabKNlhMS0RERBGLgQoRERFFLAYqREREFLFUX6NCRER/LEIIWCwWWK3WcDeF3NBqtdDpdH4vHcJAhYiIVMNsNuPUqVOorKwMd1PIAzExMWjdujUMBoPP12CgQkREqmCz2XDkyBFotVpkZGTAYDBwoc8IJYSA2WxGUVERjhw5gk6dOrld1M0dBipERKQKZrMZNpsNmZmZiImJCXdzqAnR0dHQ6/U4duwYzGYzoqKifLoOi2mJiEhVfP0vcwq9QHxX/LaJiIgoYjFQISIioogV9kBl4cKFuPTSSxEfH4/09HSMHTsWBw4cCHeziIiICMCQIUMwc+bMsH1+2AOVzZs3Y/r06fj222+xbt061NTU4KqrrkJFRUW4m0ZERBQQkydPhiRJkCQJer0e7du3x3333Yfq6upwNy3ihX3Wz+eff+7yfNmyZUhPT8eOHTswaNCgeuebTCaYTCbleWlpaXAaduBz4LdNgbmWzghcOgVIahuY6zXGYgK+fREoO+16PCkT6HcXwAI0IqKwGTVqFPLy8lBTU4MdO3Zg0qRJkCQJTz31VLibBiEErFYrdLqwhwX1RFyLSkpKAAApKSkNvr5w4ULMnz8/+A05/h3w3UuBu56pFLj22cBdryG/bgDWz2v4tQv6Am37BffziYhCTAiBqprQr1Abrdd6vYaL0WhEq1atAACZmZkYMWIE1q1bh6eeego2mw1PPfUUXn31VRQUFODCCy/EQw89hJtuugkA0LdvX0yYMAF///vfAQBjx47F6tWrUVxcjLi4OPz+++/IzMzEoUOH0LFjR7z11lt4/vnnceDAAcTGxmLYsGF47rnnkJ6eDgDYtGkThg4dijVr1uDBBx/Ezz//jLVr1+LSSy/FXXfdhY8++gjx8fHK5zl78cUX8eyzz+L48eNITEzEwIED8cEHH/hzO92KqEDFZrNh5syZuOKKK3DRRRc1eE5ubi5mzZqlPC8tLUVmZmbgG9NuABCIhYR+/wE4shkwlft/raaYyuw/k9oCF4+z/77rHaDsJFB5JvifT0QUYlU1VnR7+IuQf+6+R0cixuB7F7pnzx588803yMrKAmD/j/Dly5fj5ZdfRqdOnfDVV1/h1ltvRVpaGgYPHozBgwdj06ZN+Pvf/w4hBLZs2YKkpCRs3boVo0aNwubNm3HBBRegY8eOAICamho89thj6Ny5MwoLCzFr1ixMnjwZa9ascWnH3LlzsWjRImRnZyM5ORlz5szB5s2b8cknnyA9PR33338/fvzxR/Tq1QsA8MMPP+D//u//8NZbb6F///44d+4ctmzZ4vN98EREBSrTp0/Hnj17sHXr1kbPMRqNMBqNwW9Mx+H2h7++fckeqIgQRPw2x2e06AQMf9j+++8/2AMVM2t+iIjCadWqVYiLi4PFYoHJZIJGo8ELL7wAk8mEJ554AuvXr0dOTg4AIDs7G1u3bsUrr7yCwYMHY8iQIVi6dCmsViv27NkDg8GA8ePHY9OmTRg1ahQ2bdqEwYMHK591++23K79nZ2dj8eLFuPTSS1FeXo64uDjltUcffRRXXnklAKC8vBxLly7F8uXLMXy4vf9744030KZNG+X8/Px8xMbG4tprr0V8fDyysrLQu3fvoN63iAlUZsyYgVWrVuGrr75yuSmqJznqQoQt+J8lf4bkVItijLf/NIcgo0NEFGLRei32PToyLJ/rraFDh+Kll15CRUUFnn32Weh0Otx4443Yu3cvKisrlYBBZjablSBg4MCBKCsrw86dO/HNN98owcuTTz4JwD4xZc6cOcp7d+zYgXnz5mH37t0oLi6GzWbvH/Lz89GtWzflvL59+yq///rrrzCbzejXr7ZMICUlBZ07d1aeX3nllcjKykJ2djZGjRqFUaNG4YYbbgjqSsFhD1SEELjnnnuwcuVKbNq0Ce3btw93kwJLDhpsIcioyFkbjdP/gAyx9p+hGHoiIgoxSZL8GoIJpdjYWGVo5vXXX0fPnj2xdOlSpdRh9erVuOCCC1zeI48gJCUloWfPnti0aRO2bduGK6+8EoMGDcL48eNx8OBBHDp0SMmoVFRUYOTIkRg5ciRWrFiBtLQ05OfnY+TIkTCbzfXa5I34+Hj8+OOP2LRpE9auXYuHH34Y8+bNw/fff4+kpCRfbkuTwj4NZPr06Vi+fDnefvttxMfHo6CgAAUFBaiqqgp30wIj3BkVOVDh0A8RUcTQaDS4//778eCDD6Jbt24wGo3Iz89Hx44dXR7ONZiDBw/Gxo0b8dVXX2HIkCFISUlB165dsWDBArRu3RoXXnghAGD//v04e/YsnnzySQwcOBBdunRBYWFhk23q0KED9Ho9vvvuO+VYcXExDh486HKeTqfDiBEj8M9//hM//fQTjh49ig0bNgToztQX9jD0pZfsM2uGDBnicjwvLw+TJ08OfYMCTc5uhCJQkbM2LoGKYyySQz9ERBFl3LhxmDNnDl555RX8/e9/x9/+9jfYbDYMGDAAJSUl+Prrr5GQkIBJkyYBsPeTS5YsQVpaGrp06aIce+GFFzBu3Djlum3btoXBYMCSJUtw5513Ys+ePXjssceabE9cXBymTJmCOXPmoEWLFkhPT8cDDzzgsl/PqlWr8Ntvv2HQoEFITk7GmjVrYLPZXIaHAi3sgYoQItxNCK6wZ1TkQIUZFSKiSKLT6TBjxgz885//xJEjR5CWloaFCxfit99+Q1JSEi655BLcf//9yvkDBw6EzWZzKZodMmQInn/+eZf/2E9LS8OyZctw//33Y/HixbjkkkuwaNEijBkzpsk2Pf300ygvL8d1112H+Ph4zJ49W1k2BLAPQX300UeYN28eqqur0alTJ7zzzjvo3r17YG5KAySh8kihtLQUiYmJKCkpQUJCQribU9/OFcAndwMdrwRuDd48cwDAd68An90HdL8BGLfMfuybF4C1D9inK9/4n9pzK84AMS0CMwWbiCgEqqurceTIEbRv3x5RUVHhbg55wN135mn/HfYalWYv3BkVYwMZle+XAk93BN68vnbtFSIiogjEQCXYlBqVEK6jIjnP+qlTo1J0AFgzB4Cwr++y9qHgt4uIiMhHDFSCLdwZlbrTkw9+4Ro07XobKG+6GpyIiCgcGKgEm7KOSigClYbWUakz9HPcMe1sxHygzaWA1QTseCP4bSMiIvIBA5VgC0tGxalA1nkdFSFqA5W2lwN9Jtt/3/dx8NtGRETkAwYqwRbSGhU5UGmkRqX4KFBRBGgNQOteQOdrAI0OOL0HOHM4+O0jIiLyEgOVYAt3jYrRKVA5c8j+e+qFgD4KiEkB2jvm4zOrQkREEYiBSrDJ2Y1w7/VjswBnHVmTpLa1r3cfa//JQIWIiCIQA5VgC3dGRe+04VThPvvPxNq9I9B5tD2YKvgZOPtr8NtIRETkBQYqwSbvkRCudVS0OkDnWA2waL/9p3NGJbYF0H6g/fdfPg1+G4mIKCCWLVsWtB2LIwkDlWCTg4ZQ7FQgZ1Sch34AwOhYmvi0I6OSlOn6etfr7D8ZqBARBdW2bdug1WoxevRor97Xrl07PPfccy7Hxo8fX29n4+aIgUqwKeuohLBGpe7+PbGp9p81jrVUnDMqANDlWgAScOIHoPRkUJtIRPRHtnTpUtxzzz346quvcPKkf/9/Gx0djfT09AC1LHIxUAk2ZXpyKGpUHFkbqU5GJTbN9XlinUAlvpV98TcA2L86OG0jIgoGIezrRIX64UOWvLy8HO+99x7uuusujB49GsuWLXN5/dNPP8Wll16KqKgopKam4oYbbgBg3yH52LFj+Nvf/gZJkiA5/mO0oaGfl156CR06dIDBYEDnzp3x1ltvubwuSRL+85//4IYbbkBMTAw6deqE//3vf8rrxcXFuOWWW5CWlobo6Gh06tQJeXl5Xv+tgaQL66f/EUjhqFGpE386Byr6GPu05Lq6Xgf8vt0+/HPZ1OC1kYgokGoqgScyQv+595+snVXpof/+97/o0qULOnfujFtvvRUzZ85Ebm4uJEnC6tWrccMNN+CBBx7Am2++CbPZjDVr1gAAPvroI/Ts2RPTpk3D1KmN///zypUrce+99+K5557DiBEjsGrVKtx2221o06YNhg4dqpw3f/58/POf/8TTTz+NJUuW4JZbbsGxY8eQkpKChx56CPv27cNnn32G1NRUHD58GFVVVb7dowBhoBJsUigzKo3UqDgHKklt6w8NAUDXa4F1DwFHtwIVZ2qHi4iIKCCWLl2KW2+9FQAwatQolJSUYPPmzRgyZAgWLFiACRMmYP78+cr5PXv2BACkpKRAq9UiPj4erVq1avT6ixYtwuTJk3H33XcDAGbNmoVvv/0WixYtcglUJk+ejIkTJwIAnnjiCSxevBjbt2/HqFGjkJ+fj969e6Nv374A7LUx4cZAJdjCUqNSN6PiFHQk1imklaVkA617Aqd2A3s+AvpNC04biYgCSR9jz26E43O9cODAAWzfvh0rV64EAOh0OowfPx5Lly7FkCFDsGvXLrfZEk/88ssvmDbN9f+7r7jiCjz//PMux3r06KH8Hhsbi4SEBBQW2jenveuuu3DjjTfixx9/xFVXXYWxY8eif//+frXLXwxUgk2ZnhzCWT91a1TinIqt6s74cdZjgj1Q+eldBipEpA6S5PUQTDgsXboUFosFGRm1w1RCCBiNRrzwwguIjo4OWVv0er3Lc0mSYHNswXL11Vfj2LFjWLNmDdatW4fhw4dj+vTpWLRoUcjaVxeLaYMt0mpU6s74cXbxTfYg58SO2uX2iYjILxaLBW+++Sb+9a9/YdeuXcpj9+7dyMjIwDvvvIMePXrgyy+/bPQaBoMBVqv7fqRr1674+uuvXY59/fXX6Natm1ftTUtLw6RJk7B8+XI899xzePXVV716f6AxoxJsYVlC302g0tjQD2DPvHQcDhxaC/z0HjDswcC3kYjoD2bVqlUoLi7GlClTkJiY6PLajTfeiKVLl+Lpp5/G8OHD0aFDB0yYMAEWiwVr1qzBP/7xDwD2WpGvvvoKEyZMgNFoRGpq/TrCOXPm4Oabb0bv3r0xYsQIfPrpp/joo4+wfv16j9v68MMPo0+fPujevTtMJhNWrVqFrl27+ncD/MSMSrCFdAl9eXqyu4xKlvtr9Bhv/7n7XcBqCVzbiIj+oJYuXYoRI0bUC1IAe6Dyww8/ICUlBe+//z7+97//oVevXhg2bBi2b9+unPfoo4/i6NGj6NChA9LS0updBwDGjh2L559/HosWLUL37t3xyiuvIC8vD0OGDPG4rQaDAbm5uejRowcGDRoErVaLd9991+u/OZAkIUJRPBE8paWlSExMRElJCRISEsLdnPoKfwFevByIaQHc91twP+uj/2evL7nyMeCK/6s9bpan7wlg9kEgvmXj16ipAp7tDlSeBca9UbtpIRFRmFVXV+PIkSNo3749oqKiwt0c8oC778zT/psZlWAL96aEAGCIAUY8Agye6z5IAQB9NNB3iv33bf8OfBuJiIi8wEAl2JQalVAEKnKNirb+awP+BgzN9ew6l94BaA32BeCOfx+49hEREXmJgUqwyYurhTOj4q34lsDFN9t/3/qsf9ciIiLyAwOVYFP2+gnj9GRfXPF/9mzQgdXA0a+bPp+IiCgIGKgEWyTUqPgirTPQZ7L99y/uD83QFRGRB1Q+B+QPJRDfFQOVYAvpOiqN7PXjqyG5gDEBOLUL2PlWk6cTEQWTvKJqZWVlmFtCnpK/q7qr4XqDC74Fm1ozKgAQlwYMvg9Y+yDweS6QdQWQ2jEw1yYi8pJWq0VSUpKyL01MTAykhjZZpbATQqCyshKFhYVISkqCVuv7f0AzUAm2sNSoBCijAgCX3w0c/AI4ugX44DZgyjpAz/ULiCg85N2D5WCFIltSUpLbHZ89wUAl2JyzG0LUzgIKhkBnVAB7oPWnV4GXrgAKfgI+nGJfCE7LfzpEFHqSJKF169ZIT09HTU1NuJtDbuj1er8yKTL2NsHmHDTYrMHt4N2to+KPhAxg3DJgxU3A/lXAx3cBY18EtL6PORIR+UOr1QakE6TIx2LaYHPJqAS5TiUYGRVZ9mB7JkXSAj//F3hzLFBxJvCfQ0RE5ISBSrA5ZzeCXadiC2KgAgBdrgEmvA0Y4oFjW4EXc4Cf3q/dDJGIiCjAGKgEW3PJqMg6jwKmfgmkXghUFAIf3QH8Zziw73/cbZmIiAIuIgKVf//732jXrh2ioqLQr18/l62tVc95Bk6w11IJVo1KXWmdgTu3AsMeBHTRwIkdwH//AjzTFVgzBzjwGVB1PrhtICKiP4SwF9O+9957mDVrFl5++WX069cPzz33HEaOHIkDBw4gPT093M3zX3PLqMh0RmDQHOCSycB3LwM/vG7PsGx/1f6ABKR1AdK7AGldgaS2QEJrIL41EJduX0gu2AEVERGpXtgDlWeeeQZTp07FbbfdBgB4+eWXsXr1arz++uuYO3duvfNNJhNMJpPyvLS0NGRt9YlLjUqQA5VgrKNShxACZyvMOHa2AmfLzSirtqA8+i+o7HsTLjj3Ldqf2YQLSn5ESnU+UPSL/YGVDV6rRjLCpI2BSRsLsyYGFo0eNkkHq6Sz/4TO6bkWNuXvkiAAQJIgILkeczwXkuQ4VjsdXECq8x4iIvJEzMXX4uJBN4Tls8MaqJjNZuzYsQO5ubnKMY1GgxEjRmDbtm0NvmfhwoWYP39+qJrov2aQUTldWo2Pd57A5oNF+On3EpSbGqtFSQdwM4CbkY5idNMcxYXS7+gknUAr6RxaSsVoKRUjUXIsqSxM0FtMiLMUB7S9REQUWNuOtAL+iIHKmTNnYLVa0bJlS5fjLVu2xP79+xt8T25uLmbNmqU8Ly0tRWZmZlDb6RfHf9kDIoQ1KoEJVIorzHhm3UG89/1xmK21QZYkARmJ0UiLNyI+Sof4KB1iDDrotRI0kgSdRoJG0w5a6RJIGgm/aSQcdUpiaG1mGKyVjkeF8lMraqARFmiEBVphhcZmcXpugQQbIOR8SJ1ciRCAklNx/Gzi2B8b7wEReS7xwgFh++ywD/14y2g0wmg0hrsZ3tFoAZslBBkVR+cTgKGffSdLcfuy71FQWg0A6JOVjGt7tEZOhxZo1yIWUXrWlxARUfCFNVBJTU2FVqvF6dOnXY6fPn3a770BIoqyMWGw11GRa1T8y6gcLizHxNe+RUlVDbJTY/H4DRchJ7sFN/8iIqKQC+v0ZIPBgD59+uDLL79UjtlsNnz55ZfIyckJY8sCTM5whKpGxY/ZNNU1Vty5fAdKqmrQKzMJK6dfgf4dUhmkEBFRWIR96GfWrFmYNGkS+vbti8suuwzPPfccKioqlFlAzYKc4QhVjYofGZWXNv2Kw4XlSI834rW/9kViNPfzISKi8Al7oDJ+/HgUFRXh4YcfRkFBAXr16oXPP/+8XoGtqmlCnFHxsUalsKwaL2/+FQAwb0x3pMWrrBaIiIianbAHKgAwY8YMzJgxI9zNCB552CRk66j4llF5a9sxmCw29MpMwtUXNaMaISIiUq2IWEK/2QtZjYpj1o8P05PNFhtWfJcPAJg2KJs1KUREFBEYqISCCmpUvj58BucqzEiNM+Kqbs1o2I2IiFSNgUooqKBG5dOfTgIArrm4FXRa/rMgIqLIwB4pFCJ8HRWbTeDLXwoBAKMvbh3oVhEREfmMgUooRPg6KvtOlaKkqgZxRh36ZCUHoWFERES+YaASCkqNSrADFd8yKtt+PQsAuKx9Cod9iIgoorBXCgV5Fk6E1qhs+80eqPTv0CLQLSIiIvILA5VQCFmNihyoeD61WAiB3cfPAwD6tksJQqOIiIh8x0AlFCK4RuV0qQlnK8zQaiR0aRUfpIYRERH5hoFKKETwOip7TpQAADqlxyFK7/tmhkRERMHAQCUUIngdlT0n7YFK94zEYLSIiIjILwxUQiGC11H55VQpAKB7RkIwWkREROQXBiqhIIV41o8XNSq/FVUAADq1jAtGi4iIiPzCQCUUQrGOihAAhOvnNcFqEzh2thIA0K5FbJAaRkRE5DsGKqEQihoV52t7GKicPF8Fs9UGg1aDjKToIDWMiIjIdwxUQiEUNSrOM4o8DFSOnLEP+2S1iIFW4/naK0RERKHCQCUUQrGOivO1PaxRkQOVdqkc9iEiosjEQCUUQrGOivA9o9KegQoREUUoBiqhEPIaFc8yKifPVwEAMpNZn0JERJGJgUooyHvvBDNQ8aFG5VRJNQCgdSIDFSIiikwMVEIhQmtU5EClVWJUMFpERETkNwYqoRCSGhXvpiebLFacKTcBAFozUCEiogjFQCUUQlqjItUONblRWGoPUgw6DVJiDcFrFxERkR8YqISCMvQTgnVUPBz2kQtpWydGQfIgsCEiIgoHBiqhEIq9fpSdkz37SgtK5UJaDvsQEVHkYqASChr/a1RsNuH+BDlb4+HUZM74ISIiNWCgEgp+ZlTW7zuNPo+vw/+9sxPWxgIWLzMqRWX2GpX0eKNPbSIiIgoFBiqh4Mf05PyzlbhrxQ4UV9bgf7tP4t8bDzd8opc1KmcdM35axLGQloiIIhcDlVDwI6Py6U8nUWOtzaK89/1xCNFAVkU+5mFh7NkKMwAgJZYZFSIiilwMVEJBznL4UKPy+Z4CAMBD13aDUafBifNVOHi6vP6JXtaonHMEKsyoEBFRJGOgEgo+ZlQKy6rx84kSSBJwfa8MXNExFQCw/pfT9U/2skblbLkjUOEaKkREFMEYqISCj+uo7D1ZCgDITo1FapwRAxyBys788/VP9qJGRQihZFS42BsREUUyBiqh4OOmhL+csgcq3TISAQBdWscDAA6eLqt/shcZlTKTBWar/fwWrFEhIqIIxkAlFJQaFe8ClX2OjEq31gkAgM4t7YFK/rlKVJgsrid7UaNyzjHsE2PQItrgWU0LERFRODBQCQUfa1TkjEpXRyalRZwRqXH2DMihwjoFtV5kVM5y2IeIiFQibIHK0aNHMWXKFLRv3x7R0dHo0KEDHnnkEZjN5nA1KXh8qFExW2w4cqYCANDVkVEBgC6t7EHLgYJS1zfI2RqNB4GKsoYKh32IiCiy6cL1wfv374fNZsMrr7yCjh07Ys+ePZg6dSoqKiqwaNGicDUrOHzIqJwqqYJNAFF6jcvqsR3T47D18Bn8WlTh+gYvMirK1GRmVIiIKMKFLVAZNWoURo0apTzPzs7GgQMH8NJLL7kNVEwmE0wmk/K8tLS00XMjhlyjcmgdUF3i0Vv0JVWYpzuNpCg9pM82KMfHFZagna4Y7Q7GAkirfUPZKftPD2pUSqpqAABJ0XqP2kJERBQuYQtUGlJSUoKUlBS35yxcuBDz588PUYsCJCrJ/vPULvvDAxkAJusA1ADYXnu8O4DuOgDnXY/XflZik9curbYHKgkMVIiIKMJFTKBy+PBhLFmypMlhn9zcXMyaNUt5XlpaiszMzGA3zz+XTQV0RsBc0fS5Dtt+PYvtx86hxwWJGNo5XTl+uqwa735/HLEGLe4YkO36JkkDdBvT5LVLq+wzhhioEBFRpAt4oDJ37lw89dRTbs/55Zdf0KVLF+X5iRMnMGrUKIwbNw5Tp051+16j0QijUWVFoDEpwICZXr3lvcKd+NhyEnO7dcHQwR2U49pyE57dth6wAH8ZNApGnffTi5WMSlTExKlEREQNCnhPNXv2bEyePNntOdnZtZmAkydPYujQoejfvz9effXVQDdHtX4vrgIAZCbHuBxvEWuAUaeByWJDQUk1slrEen1tuUaFGRUiIop0AQ9U0tLSkJaW1vSJsGdShg4dij59+iAvLw8aD6bW/lGcOG8PVC5IjnY5LkkSLkiKxm9nKnDifJVPgUqpHKhEMVAhIqLIFrbc/4kTJzBkyBBkZWVh0aJFKCoqUl5r1apVuJoVEWw2gaIy+8ymlgn1h7kuSHYEKo6si7dKq+01KonMqBARUYQLW6Cybt06HD58GIcPH0abNm1cXhNChKlVkeF8VQ0sNvs9aGgvnvT4KABAUbmp3mueUDIq0axRISKiyBa2sZbJkydDCNHg44/ujCMASY7Rw6Cr/xWlxtsXajtT5tsqviUc+iEiIpVgUUgEkod9UhtZ4j7VkWU5W+F9RqW6xgqTxb6KLYtpiYgo0jFQiUByRiUtvuFApUWcPaNyttz7jEqZoz5FkoB4I4d+iIgosjFQiUBNZlQcx8/4UKMir6ESb9RBo5F8bCEREVFoMFCJQEUeZlTO+JBRKeUaKkREpCIMVCKQnFFpLFCRMyrnKkyw2rwrPpYLaTk1mYiI1ICBSgRqaugnJdaeUbEJ4Hyld1mVcpO9RiWW9SlERKQCDFQikFwkm+oY4qlLr9UgKcaeETlb4V2gUmmyAgDiGKgQEZEKMFCJQPLwTHJMw4EK4FRQW+ZdQW2F2Z5RiTF4v5khERFRqDFQiUDFjuEcOWvSEHn4x9uMSoVj6IcZFSIiUgMGKhHGZLGi0mwfnklyk1GRi2Hl7IunKhzXjjEwUCEiosjHQCXCyIGHpokF2XwNVCqVYloO/RARUeRjoBJhSiprpw+7W5BNDlRKmVEhIqJmjIFKhDnvCDzcDfsAQJKvQz9KjQozKkREFPkYqESYYkdxbFMLsiXGsEaFiIiaPwYqEaY2o9JEoMIaFSIi+gNgoBJh5BoVd2uoALV79TCjQkREzRkDlQhzvsrDoR8/a1S4hD4REakBA5UIc77Sy6GfSi+Hfswc+iEiIvVgoBJhlECliYyK/HqZyeLVDsoVjr1+Yjn0Q0REKsBAJcLIQzmJTWRUEpwCGU/XUrHaBKpq5BoVZlSIiCjyMVCJMGWOGpJ4o/tARa/VINYRbHhapyIP+wCsUSEiInVgoBJhyqrtQUdcVNOBhLcFtfIeQlqNBKOOXz0REUU+9lYRprzakVHxIFCJj7IHKuUmSxNn2skzfmIMWkhS48vzExERRQoGKhGmrNqzoR+gNusiv6cpLKQlIiK1YaASQSxWm1Ls6klGRa4z8TSjwkJaIiJSGwYqEcQ54PCkRiXeEahUeBioVDsCFaOegQoREakDA5UIIg/hROk10Gub/mrifMyoROn5tRMRkTqwx4ogcqAS50F9ClA79ONpjYqcUYlmRoWIiFSCgUoEkTMjCR4M+wC1w0OeDv2YamwAgCgGKkREpBIMVCKIvIaKJ4W0QG2NCod+iIiouWKPFUHkgMOTQlrA96EfZlSIiEgtGKhEkFIv1lABvB/6qebQDxERqQwDlQgir0rraUbF56EfHQMVIiJSBwYqEcTbGhVvF3xTZv0Y+LUTEZE6sMeKIOXKzskezvrxskbFZGFGhYiI1CUiAhWTyYRevXpBkiTs2rUr3M0JG3noJ9bDQCXeyxqVKjOLaYmISF0iIlC57777kJGREe5mhF2F2btART6vqsYKi9XW5PlKMS33+iEiIpUIe6Dy2WefYe3atVi0aJFH55tMJpSWlro8motKR8Yj1uhZIOF8nrwzsjvVytBP2L92IiIij4S1xzp9+jSmTp2Kt956CzExMR69Z+HChUhMTFQemZmZQW5l6MhDODEGzzIqRp0WBkfQUWaqafJ8Dv0QEZHahC1QEUJg8uTJuPPOO9G3b1+P35ebm4uSkhLlcfz48SC2MrTkjEqch0M/ABDjGMaRgxB3qi32oR/u9UNERGoR8EBl7ty5kCTJ7WP//v1YsmQJysrKkJub69X1jUYjEhISXB7NhVyjEuNFDUmsI/tS6UGgYuLKtEREpDKe/6e7h2bPno3Jkye7PSc7OxsbNmzAtm3bYDQaXV7r27cvbrnlFrzxxhuBblrEqzTJNSqefy3RjqBGDnLc4V4/RESkNgEPVNLS0pCWltbkeYsXL8bjjz+uPD958iRGjhyJ9957D/369Qt0s1Sh3OR9RsWroR9mVIiISGUCHqh4qm3bti7P4+LiAAAdOnRAmzZtwtGksLJYbTA5akhiPSymBWrrTTwZ+uFeP0REpDYcA4gQlTW1gUaMh9OTAae1VDwIVDj0Q0REahO2jEpd7dq1gxAi3M0IG7k+RaeRYNB6Hkh4WqNiswmYOeuHiIhUhv9pHSGcZ/xIkuTx+2I8HPqRh5UADv0QEZF6MFCJEL7M+AE8L6atchpaYqBCRERqwUAlQviyhgoAxBg9W0dFnvGj10rQajzP2BAREYUTA5UIIS+f73VGRRn6cV+jItenGHXMphARkXowUIkQFfKGhF5MTQZqi2mbyqiYHbsr67XMphARkXowUIkQlUpGxcuhHw+X0JczKgbunExERCrCXitCyBkVT3dOlsmBTVVNE0M/VgYqRESkPuy1IoSvGRV5TZQKk4cZFS/WaCEiIgo39loBdPxcJbYeOuPTwnVyRiVa7+30ZM9Wpq1RalT4lRMRkXpEzMq0aldYVo0xL2xFcWUNJlyaiYV/utirhdvk6cPeT092FNM2NfSjzPphoEJEROrBXitAFqz+BcWVNQCAd78/jt2/l3j1fnl6cbS3gYqHC76xmJaIiNSIvVYAmC02rNt3GgBwQVI0AOCzPae8ukZVjW/78MQ4hoqarFHh0A8REakQe60A+DG/GJVmK1LjDMi9pgsA4PM9BV7VqsgZEW8zKvL5VTVW2GyNfx4zKkREpEbstQLgq4NFAIABHVMxtHM6tBoJx85WorDM5PE15OnFXteoOJ1fbWk8q6JMT2ZGhYiIVIS9VgD8mF8MAOjfIRWxRh06pMUCAPae9LxORc6oeLthoPP51TW2Rs+TMyp6ZlSIiEhF2Gv5SQiBAwVlAIBuGQn2n63tP/edLPX4OpXK9GTvAhWtRlKWxa+uaTyjIk9PNjKjQkREKsJey09F5SYUV9ZAkoCO6XEAagOWfac8D1R8nZ4MAFGOjQbdBSqsUSEiIjVir+WngwXlAIB2LWKVYZjuGYkAgL0+ZFS8HfoBAKNeDlQ8GPphRoWIiFSEvZaf9hfYg5HOLeOVY50cmZXj5yqVAKEpVf5kVPT2r9F9Ma19RhAzKkREpCbstfz0a5E9o9KpZZxyLC3eiGi9FjYBnDxf5dF1fJ2eDNRmYTj0Q0REzQ17LT/ln6sEAGS1iFWOSZKEtikxAIBjjtfdqbHaYHGsgRLj5V4/QG1GxeRu6MdqD2I4PZmIiNSEvZafjp+zZ0wyk6Ndjmc6ApX8sxVNXqPKKRMSZfD+K4n2IKNSY+HQDxERqQ97LT9YbUIZ2pEDE1lWC0eg4kFGRR720WoknzIeytAPF3wjIqJmhr2WH06VVMFiE9BrJbRMiHJ5TQ5Ujp31PFCJ1mu92nFZZtR5PuuHGRUiIlIT9lp+kId9LkiKhlbjGmAoQz8eZFT8mZpsf5/9a3S3gzI3JSQiIjVir+WH48X2IKTusA8AZCTaa1YKSqubvI4/U5MBD4d+mFEhIiIVYq/lhxPF9oxKmzqFtADQKtE+FHS+ssZtkSvgOvTjC2UdFQ79EBFRM8Neyw8FJfZsSauE+oFKQpROCTzk8xojZ1R8WUMFqF1C3+RuHRWlmNb7GhgiIqJwYaDiB3lYp1Wisd5rkiQpWZWmhn8qzRYA/mRUPJiebGVGhYiI1Ie9lh9OOwKQujN+ZK0cx083Eaj4syEh4OXQj9a3zyAiIgoHBip+OK1kVBoJVBzHTzU19OP3rB/Pi2n1HPohIiIVYaDio+oaK4orawDUZk7qkjMtTdWoVDuCCL8DFU9qVDj0Q0REKsJey0eFpSYAgFGnQWK0vsFzWid6GKjUyBkV376O2kCFs36IiKh5Ya/lowKnYZ/GVpNNj7cX2RaVm9xeSw4w/F3wzZOMipGBChERqQh7LR8VNFFICwAt4uyBytkmAxV7gOFrECFPT5aHkBpSY+HKtEREpD5h77VWr16Nfv36ITo6GsnJyRg7dmy4m+SRQkegImdNGtIizgAAOFthdnstkyVAxbQeLKHPoR8iIlITXTg//MMPP8TUqVPxxBNPYNiwYbBYLNizZ084m+SxM+X24CPNTaCSGmt/razaApPFqmweWFft0I+vNSqOoZ9GZv3YbAI1VgGAuycTEZG6hC1QsVgsuPfee/H0009jypQpyvFu3bq5fZ/JZILJVDuUUlpaGrQ2ulNUZm9DalzjgUpCtA46jQSLTeBchRmtE+uvYAs4F9MGZ9ZPja12SEjPjAoREalI2HqtH3/8ESdOnIBGo0Hv3r3RunVrXH311U1mVBYuXIjExETlkZmZGaIWuzrjqDtJcxOoSJKElFjH8E9548M/SqDSSMalKU0t+GZ2ql1hRoWIiNQkbL3Wb7/9BgCYN28eHnzwQaxatQrJyckYMmQIzp071+j7cnNzUVJSojyOHz8eqia7UAIVN0M/QG1B7Rk3BbVygGH0cehHHlJqLKPCQIWIiNQq4L3W3LlzIUmS28f+/fthcwxHPPDAA7jxxhvRp08f5OXlQZIkvP/++41e32g0IiEhweURDp4M/dhf9yCjEqBiWpPFBiFEvdflQlqdRoJGw5VpiYhIPQJeozJ79mxMnjzZ7TnZ2dk4deoUANeaFKPRiOzsbOTn5we6WQFlswllJk9qvMHtuS3koZ+KpjMqvgYqzrsumyy2etepsTgKaVmfQkREKhPwQCUtLQ1paWlNntenTx8YjUYcOHAAAwYMAADU1NTg6NGjyMrKCnSzAup8VQ2sNnvn3yLWs6EfdxkVk1Kj4us6KrXvq66x1gtUzFb79RmoEBGR2oRt1k9CQgLuvPNOPPLII8jMzERWVhaefvppAMC4cePC1SyPyMM+STH6Jjt/T9ZS8XfWj06rUWYXNVRQa+Jib0REpFJhXUfl6aefhk6nw1/+8hdUVVWhX79+2LBhA5KTk8PZrCbJhbFN1acAtWupuFud1t9NCeX3lpssDRbUcg0VIiJSq7AGKnq9HosWLcKiRYvC2QyvydkReeqxO8r0ZI8yKr4HElF6DcpNDS/6Js/64T4/RESkNuy5fFDsCDpaeBCotGhi1o8Qwu+hH6B2inJVA8voc+dkIiJSK/ZcPjjnCFSSPQhUUp3WUWlo6nCNVcBRl+vzgm+A+0XfaqysUSEiInViz+WD4krH0E+M5xkVk8WGigayHc5DNb4u+AY4LaPfwNCPiRkVIiJSKfZcPvAmoxJj0CHaEUQ0VFArD/tIkn81JMqibw0U0yo7JzOjQkREKsOeywdKRiVW79H5clblTAN1Kqaa2kJXSfJ91Vh3Qz9yjQo3JCQiIrVhz+WDcxU1AIBkD4Z+gNqi23MNzPwx+bl8vizKzX4/NcyoEBGRSrHn8kGxF9OTAefVaRsa+nGsoeJHIS0ARBkaD1Q4PZmIiNSKPZeXhBA45xj68Taj0tBaKoFYQwVwyqhY3Az9aLkhIRERqQsDFS9Vmq1Kx+9pRkUuui1uMFDxf1Va+/vlGhU3xbTMqBARkcqw5/KSXGdi0GkQY/AsuJBn/VQ1EETIgYXR70BFHvppPKPCQIWIiNSGPZeXnNdQ8XSWjhzQNLRqrLzuia87J8s8yahwwTciIlIb9lxe8mYNFVm0wV1GJUBDP+5m/TCjQkREKsWey0vnK+1Tkz1dQwWoHfqpbCijEqhiWr2bWT+OjIqRGRUiIlIZ9lxeUjIqHs74Aeyr0wKNDP3INSr+Tk/2YME3ZlSIiEht2HN5qXZVWm8ClcaHfuR9ePzNqBjd7PXDGhUiIlIr9lxe8iWjEqUM/VjqvVY79BOoWT+NL/jGjAoREakNey4v+ZVRcVuj4l+gEs3pyURE1Ayx5/KSL7N+3A391C6hz+nJREREdbHn8lKxY0PCFJ+GfoK/4JupgSX05U0JudcPERGpDXsuLyn7/HgxPVnOqJgsNlhtwuW1aktg1lGRgxCzm71+uHsyERGpDXsuLwghvN45GaidngzUH5oJ1Doq8vRmU0Ozfiwc+iEiInViz+WFMpMFFkdGxJtZP85DLnWHf5RAxc91VOTPMDVUTGu1t5nFtEREpDbsubwgZ1NiDFqvhmo0Gql2Y8I6gYopQEvoGx0ZmYZqVMyOLAsDFSIiUhv2XF7wZQ0VWWMzf5RNCQM09GO22mCrUwfDWT9ERKRW7Lm84MsaKrLGFn0L1DoqzsNLdbMqNRZR7xwiIiI1YM/lhXOOqcnerKEia2zRt9rdk/3NqDgHKq6fIWdUOPRDRERqw57LC8qMnxjPpybLGh36CdCmhDqtBlqNBKB+RoXTk4mISK3Yc3mhdg0V7zMq0YaGF30L1NAP0PjMH6VGhRkVIiJSGfZcXqjNqPgQqDQy66c6QLsnA06BitPQjxCCGRUiIlIt9lxe8GWfH5m86JtzMa1zEBGYjEr9ZfRrrLUzgFijQkREasOeywv+zPppaJ0T598DEqjo62dU5GEfgBkVIiJSH/ZcXvBnHRU5EKl2qh9xXk7f392TgYZrVGqcgiFmVIiISG3Yc3mhuNKxc7IvGZUG6kfkoEWnkaALQLajoaEfOaOi1UjKrCAiIiK1YKDiIatN4LwPOyfLGgoiAjnjx/4ZDQz9KBsSMkghIiL1YaDiodKqGsgr0ydF+zL0Y7/VzsM9gVo+X9ZQHYyy2BvrU4iISIXC2nsdPHgQ119/PVJTU5GQkIABAwZg48aN4WxSo+Q1VOKNOp9qPRrOqNhcXvOX8hlONSrK1OQAfQYREVEohTVQufbaa2GxWLBhwwbs2LEDPXv2xLXXXouCgoJwNqtBxX5MTQach2UaGvoJUEbFzdCPgUM/RESkQmELVM6cOYNDhw5h7ty56NGjBzp16oQnn3wSlZWV2LNnT6PvM5lMKC0tdXmEgj9rqADOs36ci2mDVaPivI4K9/khIiL1Clvv1aJFC3Tu3BlvvvkmKioqYLFY8MorryA9PR19+vRp9H0LFy5EYmKi8sjMzAxJe5U1VHzY5wdoLKNic3nNXw3O+rEwUCEiIvUKW+8lSRLWr1+PnTt3Ij4+HlFRUXjmmWfw+eefIzk5udH35ebmoqSkRHkcP348JO31Z+dkwKnQ1SmjYrIEOKPS0Gcwo0JERCoW8N5r7ty5kCTJ7WP//v0QQmD69OlIT0/Hli1bsH37dowdOxbXXXcdTp061ej1jUYjEhISXB6hUJtR8XHox5HtqA7J9OT6C77pOeuHiIhUSBfoC86ePRuTJ092e052djY2bNiAVatWobi4WAk2XnzxRaxbtw5vvPEG5s6dG+im+cXfGpWGsh3y0E/gimkbX/CN05OJiEiNAh6opKWlIS0trcnzKisrAQAajWsHqtFoYLPZGnpLWCk7J/s868ceRJgbyqgEbHqym1k/HPohIiIVClvvlZOTg+TkZEyaNAm7d+/GwYMHMWfOHBw5cgSjR48OV7MaJa+j4ss+P0AjC77JxbQBr1FpoJiWGRUiIlKhsPVeqamp+Pzzz1FeXo5hw4ahb9++2Lp1Kz755BP07NkzXM1qVKAyKi6zfgK9Mm0Dn8HpyUREpGYBH/rxRt++ffHFF1+EswkeO6cEKoGcnhz8vX5MHPohIiIVY+/lAbPFhtJqCwAgJdbo0zUaXvDNUUwbqBoVZXipfjEtZ/0QEZEasffygDw1WSMBSdH+ZVQsNgGLI3gwBXgJfTkYkYMTAKix2HdSZEaFiIjUiL2XB86W19anaDS+7ZljdApG5ECiOsALvukcbbPK2zwDMFvtn8FiWiIiUiP2Xh44W2ECALTwcdgHcN0hWR6aCfQ6KjrHVG+Ltf6sn0At009ERBRK7L08cM7PGT8AoNVI0Dt2MJaLXQNdTKtzXL/GWptRkX9njQoREakRey8PKEM/cb4HKoDT9GElo2J1Oe4vORixOC2Yx1k/RESkZuy9PCAP/aT6kVEBnBZ9UzIqgR360TpqVCzONSoMVIiISMXYe3mgdujH9xoVoIGMSoCLaeWhJYvVuZiW05OJiEi92Ht5IHBDP66LvpmUjEqgZv3UL6atYUaFiIhUjL2XB846Mir+Dv3Ie/rIxbSmAC+hrxTT2upnVIzMqBARkQqx9/JAIGb9ALUZlXrTkwNUTCtnVKwN1Kjodb6t/0JERBRODFQ8cLbcsY5KgIZ+5Nk+wZueXH8JfYM2MJ9BREQUSgxUmuC8z48/C74BtXUiZosNFqtNmZ0TsCX0lRoVzvohIqLmgb1XE+R9frQaCYk+7vMjkzMqZqsN1U67KAc6o9Lg0I+WQz9ERKQ+DFSaIM/4SY7xfZ8fmXNGxXkX5UAtby/v9VPjtOCbPAzEjAoREakRe68m1O7z4199ClC7MaBzoGLUaSBJgcl26BzXF6I2q6LM+mGgQkREKsTeqwmBmvEDOGVUrDanVWkDV+SqcxrekTMpSo0Ki2mJiEiFGKg0QR768XfGD1AbqJicMiqBKqQFaotpgdqMihywcHoyERGpEQOVJgR26Mee1TBbbE6LvQUu06F1qqGRZ/4omxJywTciIlIh9l5NCNQ+PwBg1NfWqFSZA7vYG+A6s0cuqOX0ZCIiUjP2Xk0I6NCPXExrtQZl6EeSpNodlK0CQginBd/4VRMRkfqw92pCkWNV2tQ4/zMqLtOTHUM/xgAO/QC1U5QtNhusNgEhXD+biIhITdh7NaGw1B6opCcEYOjHZR2VwM/6AZwCFWttNgVgoEJEROrE3ssNIQSKyhyBSnwAMyrW2lk/0QEc+gFq11Kx2GxKfQoA6Dn0Q0REKsTey43zlTVKViItEIFKAwu+BTqjolc2JqzNqEhSbaaFiIhITRiouFHoyKYkx+hhDMDsHOd1VORpw4Gc9QMAOsdaKlabcFrsLXCr3xIREYUSAxU3CsuqAQDp8VEBuV5De/0EctYPULuWSo3V5hKoEBERqRF7MDcCWUgLOE9PtqHKHNyhH4tNoMax6BsLaYmISK3Yg7lx2pFRCUR9ChCi6cmOYMglo8JAhYiIVIo9mBtKRiUoQz/y9OQAz/pxDP1YbQJmq9Xlc4mIiNSGPZgbp0vtGZWWARr6MTYwPTngxbRap3VULPahH05NJiIitdKFuwGR7OT5KgDABUnRAbme86aEwVvwrXboR+PIrrCYloiI1Io9mBsnztszKhmBClSchn7k3ZOjDYH9CuRiWpfpyRz6ISIilWJGpRHVNVaccezzE7CMSkPTk4O0jkqNTcAmOD2ZiIjUjYFKIwpK7NmUaL0WSTH6gFxTWfDNGsShH6VGxQabY5E3ZlSIiEitgtaDLViwAP3790dMTAySkpIaPCc/Px+jR49GTEwM0tPTMWfOHFgslmA1yStyfUpGUlTAVnV1XkK/qkaenhycWT/2YloO/RARkboFLaNiNpsxbtw45OTkYOnSpfVet1qtGD16NFq1aoVvvvkGp06dwl//+lfo9Xo88cQTwWqWx04ogUpghn0A14ChvNoekAU+oyJvSihgFfKsHy6fT0RE6hS0QGX+/PkAgGXLljX4+tq1a7Fv3z6sX78eLVu2RK9evfDYY4/hH//4B+bNmweDwdDg+0wmE0wmk/K8tLQ04G0HgJOOQto2yYELVIxOgUppdQ2AwNeo1K5Ma4NFWZk2sJ9BREQUKmEbE9i2bRsuvvhitGzZUjk2cuRIlJaWYu/evY2+b+HChUhMTFQemZmZQWmfTiuhdWIU2iTHBOyazkWtleZg7fUjT0+u3T2ZxbRERKRWYevBCgoKXIIUAMrzgoKCRt+Xm5uLkpIS5XH8+PGgtG/60I7Yljsc04d2DNg1NRpJqSGRBXyvH01tMW1tjQqHfoiISJ28ClTmzp0LSZLcPvbv3x+stgIAjEYjEhISXB5qUrewNTpYs35sAjXMqBARkcp5VaMye/ZsTJ482e052dnZHl2rVatW2L59u8ux06dPK681VwadRhn2AYJYTMtZP0RE1Ax4FaikpaUhLS0tIB+ck5ODBQsWoLCwEOnp6QCAdevWISEhAd26dQvIZ0SiutkNY4CDCGV6ss0GEwMVIiJSuaDN+snPz8e5c+eQn58Pq9WKXbt2AQA6duyIuLg4XHXVVejWrRv+8pe/4J///CcKCgrw4IMPYvr06TAaA7MJYCRyDhoMOo2yH0+g6JyKaeWhH25KSEREahW0QOXhhx/GG2+8oTzv3bs3AGDjxo0YMmQItFotVq1ahbvuugs5OTmIjY3FpEmT8OijjwarSRHBOYMSFYRMR+1ePzYO/RARkeoFLVBZtmxZo2uoyLKysrBmzZpgNSEiRRtqa1ICXZ8C1BbTcnoyERE1B+zBQsx5lk8wAhV5HRULMypERNQMsAcLsWhDbRIr0Iu9Ac7rqHB6MhERqR97sBCLdgpOgjP0U7vXD2f9EBGR2rEHC7EYl4xK4AMVZa8fp5VpOeuHiIjUij1YiEUFvUbFUUzrvDItMypERKRS7MFCLMZ51k8QAgh56MfqPOuHgQoREakUe7AQC/asH73TyrTKrB8O/RARkUqxBwsx13VUgpdRsa9MKwAwo0JEROrFHizEgp1R0TGjQkREzQh7sBCLCdHKtBYrpycTEZH6sQcLsehgF9NqatdRMVusADg9mYiI1Is9WIi5DP0YgruOilyjYmRGhYiIVIo9WIi5ZlSCuI4KpycTEVEzwB4sxIJdoyIP85itNlhtwuUYERGR2rAHCzHXlWmDUaNiz6hUma3KMWZUiIhIrdiDhViw9/qRZ/1Umi3KMU5PJiIitWIPFmLRQc+o2K9Z4ZRRkQtsiYiI1IaBSog5F9NqpMAHEHJGxXmxNykIn0NERBQKDFRCzLmYVgTh+nULZ1mfQkREasZeLMScAwkhAh+qyNOTaz+P2RQiIlIvBiph1DoxOuDX1GuYUSEiouZD1/QpFGgr7uiHE+er0LV1QsCvrauTQWGgQkREasZAJQyu6JgatGvXC1Q4NZmIiFSMvVgzo6sz9MNVaYmISM3YizUzdTMq3JCQiIjUjL1YM8NiWiIiak7YizUzdTMqHPohIiI1Yy/WzOg0nPVDRETNB3uxZkaSJJdF3zjrh4iI1Iy9WDPknFXRM6NCREQqxl6sGXKuSzEyo0JERCrGXqwZchn6YUaFiIhUjL1YM+S8ESFn/RARkZqxF2uGnFenZUaFiIjUjL1YM+S8lgoDFSIiUrOg9WILFixA//79ERMTg6SkpHqv7969GxMnTkRmZiaio6PRtWtXPP/888Fqzh+KjtOTiYiomQja7slmsxnjxo1DTk4Oli5dWu/1HTt2ID09HcuXL0dmZia++eYbTJs2DVqtFjNmzAhWs/4QdFoO/RARUfMQtEBl/vz5AIBly5Y1+Prtt9/u8jw7Oxvbtm3DRx995DZQMZlMMJlMyvPS0lL/G9vMOGdUuCkhERGpWUT1YiUlJUhJSXF7zsKFC5GYmKg8MjMzQ9Q69XCe6RNnDFosSkREFHQRE6h88803eO+99zBt2jS35+Xm5qKkpER5HD9+PEQtVA/ndVTio/RhbAkREZF/vApU5s6dC0mS3D7279/vdSP27NmD66+/Ho888giuuuoqt+cajUYkJCS4PMiV8zoqCdHMqBARkXp51YvNnj0bkydPdntOdna2Vw3Yt28fhg8fjmnTpuHBBx/06r3UMOd1VJhRISIiNfMqUElLS0NaWlrAPnzv3r0YNmwYJk2ahAULFgTsun90zuuoJEQxo0JEROoVtF4sPz8f586dQ35+PqxWK3bt2gUA6NixI+Li4rBnzx4MGzYMI0eOxKxZs1BQUAAA0Gq1AQ2G/og0EmtUiIioeQhaoPLwww/jjTfeUJ737t0bALBx40YMGTIEH3zwAYqKirB8+XIsX75cOS8rKwtHjx4NVrP+EEwWq/J7PDMqRESkYpIQQoS7Ef4oLS1FYmIiSkpKWFjrcP0LW7H79xIAwNEnR4e5NURERPV52n9HzPRkCpxKs7Xpk4iIiFSAgUozVFXDQIWIiJoHBirNUBUzKkRE1EwwUGmGmFEhIqLmgoFKM8QaFSIiai4YqBAREVHEYqDSDA3tbF8wr1979ztRExERRTquBtYMPXNzL/xv90mM6ZkR7qYQERH5hYFKM5Qca8Ck/u3C3QwiIiK/ceiHiIiIIhYDFSIiIopYDFSIiIgoYjFQISIioojFQIWIiIgiFgMVIiIiilgMVIiIiChiMVAhIiKiiMVAhYiIiCIWAxUiIiKKWAxUiIiIKGIxUCEiIqKIxUCFiIiIIpbqd08WQgAASktLw9wSIiIi8pTcb8v9eGNUH6iUlZUBADIzM8PcEiIiIvJWWVkZEhMTG31dEk2FMhHOZrPh5MmTiI+PhyRJAb12aWkpMjMzcfz4cSQkJAT02lSL9zk0eJ9Dh/c6NHifQyNY91kIgbKyMmRkZECjabwSRfUZFY1GgzZt2gT1MxISEvg/ghDgfQ4N3ufQ4b0ODd7n0AjGfXaXSZGxmJaIiIgiFgMVIiIiilgMVNwwGo145JFHYDQaw92UZo33OTR4n0OH9zo0eJ9DI9z3WfXFtERERNR8MaNCREREEYuBChEREUUsBipEREQUsRioEBERUcRioEJEREQRi4FKI/7973+jXbt2iIqKQr9+/bB9+/ZwN0lVvvrqK1x33XXIyMiAJEn4+OOPXV4XQuDhhx9G69atER0djREjRuDQoUMu55w7dw633HILEhISkJSUhClTpqC8vDyEf0XkW7hwIS699FLEx8cjPT0dY8eOxYEDB1zOqa6uxvTp09GiRQvExcXhxhtvxOnTp13Oyc/Px+jRoxETE4P09HTMmTMHFosllH9KxHvppZfQo0cPZXXOnJwcfPbZZ8rrvM/B8eSTT0KSJMycOVM5xnvtv3nz5kGSJJdHly5dlNcj6h4Lqufdd98VBoNBvP7662Lv3r1i6tSpIikpSZw+fTrcTVONNWvWiAceeEB89NFHAoBYuXKly+tPPvmkSExMFB9//LHYvXu3GDNmjGjfvr2oqqpSzhk1apTo2bOn+Pbbb8WWLVtEx44dxcSJE0P8l0S2kSNHiry8PLFnzx6xa9cucc0114i2bduK8vJy5Zw777xTZGZmii+//FL88MMP4vLLLxf9+/dXXrdYLOKiiy4SI0aMEDt37hRr1qwRqampIjc3Nxx/UsT63//+J1avXi0OHjwoDhw4IO6//36h1+vFnj17hBC8z8Gwfft20a5dO9GjRw9x7733Ksd5r/33yCOPiO7du4tTp04pj6KiIuX1SLrHDFQacNlll4np06crz61Wq8jIyBALFy4MY6vUq26gYrPZRKtWrcTTTz+tHDt//rwwGo3inXfeEUIIsW/fPgFAfP/998o5n332mZAkSZw4cSJkbVebwsJCAUBs3rxZCGG/r3q9Xrz//vvKOb/88osAILZt2yaEsAeVGo1GFBQUKOe89NJLIiEhQZhMptD+ASqTnJws/vOf//A+B0FZWZno1KmTWLdunRg8eLASqPBeB8Yjjzwievbs2eBrkXaPOfRTh9lsxo4dOzBixAjlmEajwYgRI7Bt27Ywtqz5OHLkCAoKClzucWJiIvr166fc423btiEpKQl9+/ZVzhkxYgQ0Gg2+++67kLdZLUpKSgAAKSkpAIAdO3agpqbG5V536dIFbdu2dbnXF198MVq2bKmcM3LkSJSWlmLv3r0hbL16WK1WvPvuu6ioqEBOTg7vcxBMnz4do0ePdrmnAP9NB9KhQ4eQkZGB7Oxs3HLLLcjPzwcQefdY9bsnB9qZM2dgtVpdbj4AtGzZEvv37w9Tq5qXgoICAGjwHsuvFRQUID093eV1nU6HlJQU5RxyZbPZMHPmTFxxxRW46KKLANjvo8FgQFJSksu5de91Q9+F/BrV+vnnn5GTk4Pq6mrExcVh5cqV6NatG3bt2sX7HEDvvvsufvzxR3z//ff1XuO/6cDo168fli1bhs6dO+PUqVOYP38+Bg4ciD179kTcPWagQtRMTJ8+HXv27MHWrVvD3ZRmq3Pnzti1axdKSkrwwQcfYNKkSdi8eXO4m9WsHD9+HPfeey/WrVuHqKiocDen2br66quV33v06IF+/fohKysL//3vfxEdHR3GltXHoZ86UlNTodVq61U3nz59Gq1atQpTq5oX+T66u8etWrVCYWGhy+sWiwXnzp3j99CAGTNmYNWqVdi4cSPatGmjHG/VqhXMZjPOnz/vcn7de93QdyG/RrUMBgM6duyIPn36YOHChejZsyeef/553ucA2rFjBwoLC3HJJZdAp9NBp9Nh8+bNWLx4MXQ6HVq2bMl7HQRJSUm48MILcfjw4Yj798xApQ6DwYA+ffrgyy+/VI7ZbDZ8+eWXyMnJCWPLmo/27dujVatWLve4tLQU3333nXKPc3JycP78eezYsUM5Z8OGDbDZbOjXr1/I2xyphBCYMWMGVq5ciQ0bNqB9+/Yur/fp0wd6vd7lXh84cAD5+fku9/rnn392CQzXrVuHhIQEdOvWLTR/iErZbDaYTCbe5wAaPnw4fv75Z+zatUt59O3bF7fccovyO+914JWXl+PXX39F69atI+/fc0BLc5uJd999VxiNRrFs2TKxb98+MW3aNJGUlORS3UzulZWViZ07d4qdO3cKAOKZZ54RO3fuFMeOHRNC2KcnJyUliU8++UT89NNP4vrrr29wenLv3r3Fd999J7Zu3So6derE6cl13HXXXSIxMVFs2rTJZZphZWWlcs6dd94p2rZtKzZs2CB++OEHkZOTI3JycpTX5WmGV111ldi1a5f4/PPPRVpaGqdy1jF37lyxefNmceTIEfHTTz+JuXPnCkmSxNq1a4UQvM/B5DzrRwje60CYPXu22LRpkzhy5Ij4+uuvxYgRI0RqaqooLCwUQkTWPWag0oglS5aItm3bCoPBIC677DLx7bffhrtJqrJx40YBoN5j0qRJQgj7FOWHHnpItGzZUhiNRjF8+HBx4MABl2ucPXtWTJw4UcTFxYmEhARx2223ibKysjD8NZGroXsMQOTl5SnnVFVVibvvvlskJyeLmJgYccMNN4hTp065XOfo0aPi6quvFtHR0SI1NVXMnj1b1NTUhPiviWy33367yMrKEgaDQaSlpYnhw4crQYoQvM/BVDdQ4b323/jx40Xr1q2FwWAQF1xwgRg/frw4fPiw8nok3WNJCCECm6MhIiIiCgzWqBAREVHEYqBCREREEYuBChEREUUsBipEREQUsRioEBERUcRioEJEREQRi4EKERERRSwGKkRERBSxGKgQERFRxGKgQkRERBGLgQoRERFFrP8PqcGIA6PcweoAAAAASUVORK5CYII=\n" }, "metadata": {} }, { "output_type": "execute_result", "data": { "text/plain": [ "" ], "text/html": [ "\n", " " ] }, "metadata": {}, "execution_count": 8 } ] }, { "cell_type": "code", "source": [ "# Show the policy learned\n", "embed_mp4(f'{video_directory}/rl-video-episode-0.mp4')" ], "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 501 }, "id": "4FX9ANmENtUO", "outputId": "27d04b94-f443-4bf8-c017-f2c14a6f1e51" }, "execution_count": null, "outputs": [ { "output_type": "execute_result", "data": { "text/plain": [ "" ], "text/html": [ "\n", " " ] }, "metadata": {}, "execution_count": 9 } ] }, { "cell_type": "markdown", "source": [ "# Questions (25 points)" ], "metadata": { "id": "jskFvBFHLido" } }, { "cell_type": "markdown", "source": [ "Based on your experiments, answer the following questions:\n", "\n", "\n", "\n", "* How does the number of LQR iterations effect the MPC?\n", "* What if we didn't had access to the model dynamics? Could we still use MPC?\n", "* Do `TIMESTEPS` or `N_BATCH` matter here? Explain.\n", "* Why do you think we chose to set the initial state of the environment to the downward position?\n", "* As time progresses (later iterations) what happens to the actions and rewards? Why?\n", "\n", "`Your Answers:`\n" ], "metadata": { "id": "9Pk2jwFVNzTZ" } } ] }