{"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"name":"2022-01-21-gridworld-td.ipynb","provenance":[{"file_id":"https://github.com/recohut/nbs/blob/main/raw/T122762%20%7C%20Training%20RL%20Agent%20in%20Gridworld%20with%20Temporal%20Difference%20learning%20method.ipynb","timestamp":1644659399764}],"collapsed_sections":[],"authorship_tag":"ABX9TyObH3bb/z+IhSDM8ZLwKd4r"},"kernelspec":{"name":"python3","display_name":"Python 3"},"language_info":{"name":"python"}},"cells":[{"cell_type":"markdown","metadata":{"id":"bmIy05Cqwt3X"},"source":["# Training RL Agent in Gridworld with Temporal Difference learning method"]},{"cell_type":"markdown","metadata":{"id":"CeesDf_Xupst"},"source":["## Imports"]},{"cell_type":"code","metadata":{"id":"qCl7x_aFuhxe"},"source":["import gym\n","import numpy as np\n","import matplotlib.pyplot as plt\n","import seaborn as sns"],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"tR76MDYoujcE"},"source":["## Gridworld"]},{"cell_type":"code","metadata":{"id":"PTl1sTu6ujZe"},"source":["class GridworldV2Env(gym.Env):\n","    def __init__(self, step_cost=-0.2, max_ep_length=500, explore_start=False):\n","        self.index_to_coordinate_map = {\n","            \"0\": [0, 0],\n","            \"1\": [0, 1],\n","            \"2\": [0, 2],\n","            \"3\": [0, 3],\n","            \"4\": [1, 0],\n","            \"5\": [1, 1],\n","            \"6\": [1, 2],\n","            \"7\": [1, 3],\n","            \"8\": [2, 0],\n","            \"9\": [2, 1],\n","            \"10\": [2, 2],\n","            \"11\": [2, 3],\n","        }\n","        self.coordinate_to_index_map = {\n","            str(val): int(key) for key, val in self.index_to_coordinate_map.items()\n","        }\n","        self.map = np.zeros((3, 4))\n","        self.observation_space = gym.spaces.Discrete(1)\n","        self.distinct_states = [str(i) for i in range(12)]\n","        self.goal_coordinate = [0, 3]\n","        self.bomb_coordinate = [1, 3]\n","        self.wall_coordinate = [1, 1]\n","        self.goal_state = self.coordinate_to_index_map[str(self.goal_coordinate)]  # 3\n","        self.bomb_state = self.coordinate_to_index_map[str(self.bomb_coordinate)]  # 7\n","        self.map[self.goal_coordinate[0]][self.goal_coordinate[1]] = 1\n","        self.map[self.bomb_coordinate[0]][self.bomb_coordinate[1]] = -1\n","        self.map[self.wall_coordinate[0]][self.wall_coordinate[1]] = 2\n","\n","        self.exploring_starts = explore_start\n","        self.state = 8\n","        self.done = False\n","        self.max_ep_length = max_ep_length\n","        self.steps = 0\n","        self.step_cost = step_cost\n","        self.action_space = gym.spaces.Discrete(4)\n","        self.action_map = {\"UP\": 0, \"RIGHT\": 1, \"DOWN\": 2, \"LEFT\": 3}\n","        self.possible_actions = list(self.action_map.values())\n","\n","    def reset(self):\n","        self.done = False\n","        self.steps = 0\n","        self.map = np.zeros((3, 4))\n","        self.map[self.goal_coordinate[0]][self.goal_coordinate[1]] = 1\n","        self.map[self.bomb_coordinate[0]][self.bomb_coordinate[1]] = -1\n","        self.map[self.wall_coordinate[0]][self.wall_coordinate[1]] = 2\n","\n","        if self.exploring_starts:\n","            self.state = np.random.choice([0, 1, 2, 4, 6, 8, 9, 10, 11])\n","        else:\n","            self.state = 8\n","        return self.state\n","\n","    def get_next_state(self, current_position, action):\n","\n","        next_state = self.index_to_coordinate_map[str(current_position)].copy()\n","\n","        if action == 0 and next_state[0] != 0 and next_state != [2, 1]:\n","            # Move up\n","            next_state[0] -= 1\n","        elif action == 1 and next_state[1] != 3 and next_state != [1, 0]:\n","            # Move right\n","            next_state[1] += 1\n","        elif action == 2 and next_state[0] != 2 and next_state != [0, 1]:\n","            # Move down\n","            next_state[0] += 1\n","        elif action == 3 and next_state[1] != 0 and next_state != [1, 2]:\n","            # Move left\n","            next_state[1] -= 1\n","        else:\n","            pass\n","        return self.coordinate_to_index_map[str(next_state)]\n","\n","    def step(self, action):\n","        assert action in self.possible_actions, f\"Invalid action:{action}\"\n","\n","        current_position = self.state\n","        next_state = self.get_next_state(current_position, action)\n","\n","        self.steps += 1\n","\n","        if next_state == self.goal_state:\n","            reward = 1\n","            self.done = True\n","\n","        elif next_state == self.bomb_state:\n","            reward = -1\n","            self.done = True\n","        else:\n","            reward = self.step_cost\n","\n","        if self.steps == self.max_ep_length:\n","            self.done = True\n","\n","        self.state = next_state\n","        return next_state, reward, self.done"],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"PMeLTRHPujWT"},"source":["## Visualization function"]},{"cell_type":"code","metadata":{"id":"4wfzk8Kiu28Q"},"source":["def visualize_grid_state_values(grid_state_values):\n","    \"\"\"Visualizes the state value function for the grid\"\"\"\n","    plt.figure(figsize=(10, 5))\n","    p = sns.heatmap(\n","        grid_state_values,\n","        cmap=\"Greens\",\n","        annot=True,\n","        fmt=\".1f\",\n","        annot_kws={\"size\": 16},\n","        square=True,\n","    )\n","    p.set_ylim(len(grid_state_values) + 0.01, -0.01)\n","    plt.show()"],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"JrwdkXQwu3tM"},"source":["## Temporal Difference learning method"]},{"cell_type":"code","metadata":{"id":"2dlmJ4BAu76v"},"source":["def temporal_difference_learning(env, max_episodes):\n","    grid_state_values = np.zeros((len(env.distinct_states), 1))\n","    grid_state_values[env.goal_state] = 1\n","    grid_state_values[env.bomb_state] = -1\n","    # v: state-value function\n","    v = grid_state_values\n","    gamma = 0.99  # Discount factor\n","    alpha = 0.01  # learning rate\n","\n","    for episode in range(max_episodes):\n","        state = env.reset()\n","        done = False\n","        while not done:\n","            action = env.action_space.sample()  # random policy\n","            next_state, reward, done = env.step(action)\n","\n","            # State-value function updates using TD(0)\n","            v[state] += alpha * (reward + gamma * v[next_state] - v[state])\n","            state = next_state\n","    visualize_grid_state_values(grid_state_values.reshape((3, 4)))"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/","height":324},"id":"6ecXS55wu-r6","executionInfo":{"status":"ok","timestamp":1638441777337,"user_tz":-330,"elapsed":1941,"user":{"displayName":"Sparsh Agarwal","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"13037694610922482904"}},"outputId":"02df2eda-3cce-4977-888d-e0c681f7638f"},"source":["if __name__ == \"__main__\":\n","    max_episodes = 4000\n","    env = GridworldV2Env(step_cost=-0.1, max_ep_length=30)\n","    temporal_difference_learning(env, max_episodes)"],"execution_count":null,"outputs":[{"output_type":"display_data","data":{"image/png":"iVBORw0KGgoAAAANSUhEUgAAAdAAAAEzCAYAAACBjVIsAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3dd3QV1d7G8e8vAQIJEMDQQ1NAQRALYH2vXmzYUO9VBBXEBqLYK2JXULAXLNh71ysKSrNRRAgoRTpISSgJJUCAkJDs948cMCEhZZIzJzl5PqxZnNnT9syCPNl79swx5xwiIiJSMhGhroCIiEhFpAAVERHxQAEqIiLigQJURETEAwWoiIiIBwpQERERDxSgIiJSoZnZW2aWbGbzD7DczOwFM1tmZnPN7OiyOK4CVEREKrp3gO6FLD8LaBOY+gOvlMVBFaAiIlKhOed+BTYXssr5wHsux3Sgjpk1Lu1xFaAiIhLumgJrcs0nBspKpUppd1CUv7cv0bsCfTArJSHUVagUXp8zIdRVqBTGv6zr7Ac3IdH8PJ6dHu8tDyYmDSCn63WvUc65UWVSqVIIeoCKiIiURiAsSxOYSUCzXPPxgbJSUReuiIj4w8zbVHqjgb6B0bjHAVudc+tKu1O1QEVExB9BarKZ2cfAKUCcmSUCDwJVAZxzrwJjgbOBZcBO4MqyOK4CVERE/FE2rcl8nHO9i1jugBvK+rgKUBER8YevQ5aCTwEqIiL+CFILNFQUoCIi4o8wG7aqABUREX+oBSoiIuJBeOWnAlRERHwSEV4JqgAVERF/hFd+KkBFRMQnugcqIiLiQXjlpwJURER8onugIiIiHoRXfipARUTEJ7oHKiIi4kGYdeGG2YuVRERE/KEWqIiI+CO8GqAKUBER8YnugYqIiHgQXvmpABUREZ+E2SAiBaiIiPgjvPJTASoiIj7RPVAREREPwuzBSQWoiIj4Qy1QERERD8IrPxWgIiLiE7VARUREPNA9UBEREQ/UAhUREfEgvPIz3BrUIiJSbkWYt6kIZtbdzBab2TIzu6eA5f3MLMXM/gxM15TF6agFKiIi/ghCF66ZRQIjgdOBRGCmmY12zi3Yb9VPnXODyvLYaoGKiIg/zONUuK7AMufcCudcBvAJcH4Qap+PAlRERCqypsCaXPOJgbL9/dfM5prZF2bWrCwOrAAVERFfmJnXqb+ZJeSa+pfw0N8CLZ1zRwATgHfL4nx0D1RERHxhHu+BZjs3Chh1gMVJQO4WZXygbB/n3KZcs28AIzxVZD9qgYqIiC/MvE1FmAm0MbNWZlYN6AWMzntca5xrtgewsCzOp1K1QLOysvj6o2+YOXUWq1asZnf6bpo0a8x5F5/DGT1OIyKi8N8nJk+cys/jfmHpwmWkbtlK/Yb1ObHb8fS68mKiY6J9OouKYeX81SSMm03S0rVsXLOR2nG1ufPdW4q17ZYNqTzV7/kCl933+d3UqFm9LKta4R1+UBuuObwnh9Rpzo7MXfyUOJ13/vqSjOzMQrc7Iu5Qnvy/fCP+ScvYyX/H3BCs6lYoTeMac/cl19O57RF0Org90dVr0PLy41i1IbHIbc2Muy+5ngHnXE6jevVZvGYFj3zwHF9NGetDzcuniCCMwnXO7TGzQcA4IBJ4yzn3l5k9AiQ450YDN5lZD2APsBnoVxbHrlQBmrE7g4/f/IxTz/k3F/Q6j+rRNZg5NYHnh77EmlWJXHvzVYVu/+UHX1O/URz9buhLXIODWL54BR+M+pi5CfN45q0RRQZwZbJ8zgpWzV9N07ZNMDN279xd4n2cfMlJHHbsoXnKompUK6sqhoVWteN5/MQ7mLVhPg/89hyNoutzTYeexFWvy7CZrxRrHyPnfMCSLX/vm89y2cGqboXTuklLep58LrOWzmPy/N85s/Mpxd720X53csdFAxjy9ghmLZ1Hr1N68Pn9r3Lu/f34fsaPwat0Oea1C7cozrmxwNj9yh7I9XkwMLisj1upArRaVDXe+eZ1asXW2ld2VNdOpG1LY/Sn39F3wGVEVY864PYPPXs/derG7ps/4piO1Kpdi6ceepa5s+ZxZJdOQa1/RfLv3idz6mWnAPDZiK9Y9dfqEu+jbqO6NG8XX8Y1Cy992l3Axl1beGzGy2S5LGAhe7L3cGfna/lsyViWbV1V5D7WbF/Hoi0rgl/ZCujXedNp1PMoAK4+q3exA7R+nYO446IBPPHpyzz9xWsA/DxnGq2btuSJqwcrQMNEpWoyRUZG5gnPvdoe3obMjEy2pW4rdPvc4Zl7W4CNyZvyLavMIorx9hApnUiLpHPDjvyaNCMQnjl+SZpBRlYmxzc+KoS1Cw/OOU/bndn5FKKqRfHBpK/ylH8w6SuOOLgdLRuVyVMUFY7XUbjlVaVqgR7IvFnzqVkrhnpx9TxtC9C8VeX8DxFM49+exOgXv6Nq9Wq06tiC06/oRqNWDUNdrXKjSUx9oiKrsXJbngGHZGbvYd2OZJrXblKs/dzduT+1o2qxI3MnszbM582/Pidl1+ZgVLnSOLxFW9Iz0lmW9Hee8r9WLgGgffM2rFy/pqBNw1o5zkJPigxQMzuMnLc67H0wNQkY7Zwrk1FMoZbw22x+nTiFvtddRmSVyBJtuzF5E++99iFHdT2Stu3bBKmGlU9k1Ui6nH0MbY4+hJjYaFLWbOSXT6fw2u1vMfC5a2jQvH6oq1gu1KpWE4C0jB35lm3P3EGtqjGFbr8jcxdfLP2BuRsXsTMzndZ1mtPr0HN5Lu4+rv/xQbZmbA9KvSuDerXqkJqWv0dr8/bUnOW16/pdpXKhPLcmvSg0QM3sbqA3Oa9GmhEojgc+NrNPnHNPBLl+njnnyM7KOxhi/4BctWI1Twx5kk6dO9LziotKtP9dO3fx8O2PERkZyW0P3lzq+lZUzjmys/N2c0VGlu7OQO16tbjgxnP3zbfs0II2nVvzwoCX+fmTyfS86z+l2n9FFWFle8dl+dbVLN/6z73peZsWM2/jEl445X4uOOR03l34VSFbi5RcpQpQ4GrgcOdcnvHwZvYM8BdQYIAG3hLRH2Do84/Q+8pLyqCqJTN31nzuvu7ePGU/JHy77/O6xPXce8P9NGrSkPufHFKi1ufu9N08eOujrEtaz5OjHqd+w7gyq3dF8/e8Vbx5d96Xegz9/sEyP06d+rG0OLw5SUvWlvm+K4KCHjm5ZmLOv++a1fK3NGtVjWFVesmv1bKtq0hMW0/buq28VVQA2JK2lTo1a+crr1erDgCbt23xu0rlgoXZ95kVFaDZQBNg/6F8jQPLCuRyvTXi7+1LvN2FL6U27Q7hhfeeKXBZyoaN3HP9EKJjohn60sPE1Cz+M5x79uzhsbufYOnCZQwb+QitWrcsoxpXTE1bN2bg89f6d8Dw+v9XbEtTVzHop4fzlK1LSyYjK5MWtfK+9rNqRBUaxzRgclJCKY4Ykv+2YeOvlYupXq06hzRpyfK1K/eVt2+Rc6tnweqlIapZaFW2FugtwCQzW8o/L+ttDrQGyvRrYcpadEx0gfclU7dsZfD19wEwbOSjxNbJP7L2QLKzsxl+31PMSZjLw88+QLuOh5VZfSuqqOgo4tsWb7BKaaQmb2XVX6tpd0LlvOa79qSzNHVlvvKEDfP4V3wX3l/0P7IDz2/+X9MuVIusym/r/yjxcdrUaUl8rcZMWVua8JUfEn4mIzODy7pdyCMfPLuv/PJT/8O8vxdVygFEUMkGETnnfjCztuR8XUzuQUQzncs1br6C2J2+myGDHmDDumRue+AmNiZvZGPyxn3Lm7dqvq81OvG7H3nm0ed54uXHOOKYjgCMHP4qkydOpddVPaleI4qF8xbt2zauQVyl7srd347UHfw9L6fjYmvKVjJ3ZzJ/cs7X8zVoXp8GLXIGAv09dyVvDX6P/9x6PkedlvMc7djXx+GyHc3bNcsZRJS4iV8/m4JFGKf0+r/QnFA59f6i//H8yfczpOv1fLtiEg2j47i2Q09+TZrJstR/Oo5Oa3YCtx19FXdPeZJ5mxYDOaNv1+/YyLKtq0jL2EnrOs25pO05bNq1hf8tnxiqUyp3/vt/5wBwTJucnwNndfk3KVs3k7J1E7/OnQ5A5g8reXf8F1zzzB0ApKRu4pkvX2dw7xvYviuN2Uvnc8kp59HtyBPp8cCVoTmRciAYbyIKpSJH4TrnsoHpPtQl6LZsTmX54pwHxoff93S+5cNfHUanzjn/SbJdNtlZ2eR+DGzmtFkAfPLWZ3zy1md5tr3s2t70GXBpkGpe8WxYncLHwz7PU7Z3vttlJ3Nqi1OAnI7C7GyX53m7Bs0bMGNMArMnziFjVwbRtWtwcKdWdLvsZOrH65eU3FZsXcPgqU9xdYeLefT4W9mRuZOJq6fx9oIv86xnZkRGROZpAazclsQp8cdy/iGnEhVZjS3p25i6dhbvL/wf2zLSfD6T8uuLB17LM//KzY8D8POc3/j3HRcDUCWySr7Bc0PeHk7arh3cfOHVNKpbn8WJK+j52EDG/D7Jn4qXQ+HWhWteHxQurlDdA61sZqWoy80Pr8+ZEOoqVArjX9Z19oObkOhroh10/wme8mDTo9PKZfJWqjcRiYiIlBW9iUhERHwRZj24ClAREfFHuN0DVYCKiIgvFKAiIiIeKEBFREQ8UICKiIh4EGb5qQAVERF/qAUqIiLigQJURETEg0r3LlwREZGyEGb5qQAVERF/qAtXRETEA0MBKiIiUmJqgYqIiHigABUREfEgzPJT3wcqIiL+MDNPUzH2293MFpvZMjO7p4DlUWb2aWD572bWsizORwEqIiIVlplFAiOBs4D2QG8za7/falcDW5xzrYFngeFlcWwFqIiI+CJILdCuwDLn3ArnXAbwCXD+fuucD7wb+PwFcKqVwQ1ZBaiIiPgiSAHaFFiTaz4xUFbgOs65PcBW4KDSno8CVEREfGHmdbL+ZpaQa+of6nMBjcIVERGfeO01dc6NAkYdYHES0CzXfHygrKB1Es2sChALbPJUmVzUAhUREV8EqQt3JtDGzFqZWTWgFzB6v3VGA1cEPl8E/Oicc6U9H7VARUTEF8F4kYJzbo+ZDQLGAZHAW865v8zsESDBOTcaeBN438yWAZvJCdlSU4CKiIgvgvUiBefcWGDsfmUP5PqcDlxc1sdVgIqIiC/0Kj8REREPFKAiIiIeKEBFREQ8CLP8VICKiIg/1AIVERHxQgEqIiJScmqBioiIeBBm+akAFRERf6gFKiIi4kG4BaheJi8iIuKBWqAiIuKLcGuBKkBFRMQXYZafwQ/Qg6rXD/YhBBjyzXuhrkKl8NLFg0JdhUrh+KEtQl0FCQK1QEVERDxQgIqIiHigABUREfFAASoiIuJBmOWnAlRERPyhFqiIiIgHClAREREPFKAiIiIehFl+KkBFRMQfaoGKiIh4oQAVEREpObVARUREPIgIr/xUgIqIiD/CrQWqL9QWEZGwZWb1zGyCmS0N/F33AOtlmdmfgWl0cfatABUREV9EmHmaSukeYJJzrg0wKTBfkF3OuSMDU49inU9payYiIlIcZuZpKqXzgXcDn98FLijtDvdSgIqIiC8iPE6l1NA5ty7weT3Q8ADrVTezBDObbmbFClkNIhIREV947Y41s/5A/1xFo5xzo3Itnwg0KmDTIblnnHPOzNwBDtPCOZdkZgcDP5rZPOfc8sLqpQAVERFfeO2ODYTlqEKWn1bIMTeYWWPn3DozawwkH2AfSYG/V5jZz8BRQKEBqi5cERHxRYgGEY0Grgh8vgL4Zv8VzKyumUUFPscBJwILijyf0tZMRESkOEI0iOgJ4HQzWwqcFpjHzDqb2RuBddoBCWY2B/gJeMI5V2SAqgtXRER8EYoWm3NuE3BqAeUJwDWBz9OAjiXdtwJURER8UQbdseWKAlRERHwRbq/yU4CKiIgv1AIVERHxILziUwEqIiI+UQtURETEAwWoiIiIBxpEJCIi4kG4tUD1JiIREREP1AIVERFfhFf7UwEqIiI+CbcuXAWoiIj4QgEqIiLigUbhioiIeKAWqIiIiAfhFZ8KUBER8YlaoBVYVlYWH733CdMmT2PFshWkp++mWYt4Lu51EeddeC4REcV7LHbqr9N49833WLRgMRERRvMWzbnx9kF0ObZzkM+gYmhUM47BpwzgxOZHY8C01X8y7JdXWbc9pchtq0VW5ZYTruC8dt2oHRXDwuQVPDXlTRKS5ge/4hXQ8nkrmDb2d1YvSWTD6mTq1I/lkY/uK9a27w//mN/HJxS4rGGz+tz/zj1lWdUKLWVxCst/Xs7mvzezbd02outF0+PZHsXePmNHBn98/AeJsxLJysgirk0cR192NHWa1QlircsfBWgFtnv3bt4a9TbnnHcWvS6/hBrRNZg6eRpDH3qclX+v4uY7bixyH1999jUjhj1Fz94Xc/WAK8l2jiWLlpC+K92HMyj/qleJ4t2LhpORlcnd454C57j5xCt476Lh9Hh/ILv27C50+2Gn38rJrboyYvIbrNm6nss6nceb/xnKJZ/cyqKUFT6dRcWx+I+lLJ/3N80PjQeD3TsLv765de9zOiedd0Kesk3rN/PO0A/oePzhZV3VCm39X+tJWZJCvVb1wGBP+p5ib+uc45dnfmFHyg6O6XsM1WKqsWD0AiYNm8RZQ88iul50EGtevmgQUQUWFRXF/374ktjY2H1lXY/rwvat2/nso88ZMOhaqlevfsDt1yat5Znhz3HT7TdyaZ9e+8qPP/G4oNa7IunZsTvNYhvR/Z1rWL11HQCLN/7NuCvf4pIjzuGd2V8dcNtD41pxXrtuDB73NF8tmADAzMS5jOk7ipuP78vA0Q/5cQoVSvfLT+fsvmcC8O6wD1k+/+9ib1u/SRz1m8TlKVs0awkAx57ZpewqGQY6XNCBjv/pCMC0V6axccnGYm+bNDuJjUs20m1wNxq2bwhAXOs4vr3tWxZ+t5Bj+h4TlDqXR+H26rtwO59CRUZG5gnPvdp3aEdGRgapW7YWuv3or7/DIoz/9rwwWFWs8LodfBxz1i/aF54Aids2MHvtX5x6SOG/aJx6yPFkZGUydsmv+8qyXDZjlvzMSS2Opmpk1aDVu6Iq7m2H4poxIYFmbeNp3LJRme63orMI7y2npNlJ1KhbY194AlSLrkaTo5qQODuxLKpXYZiZp6m8qlQBeiCzE/6gVu1axNU/qND15syeQ8tWLRj//QQu6P5fjut0IheedRGfffyFTzUt/1of1IIlG1flK1+2aRWt6zUvYtvmJG3dQPp+3bzLNq2iWpVqtKjTpEzrKnktn/83KUkbOfYM3csvS1uTthIbn/8X99imsezctJPM9MwQ1Co0Isw8TeVVperCLchvU6czcdwkrhvUnypVCr8cKSkb2ZicwgtPv8T1N19HfLN4Jo2bxJNDnyJrTxa9+1ziU63Lr9jqtdi2e3u+8q3padSuXqvIbbcWsG1qelpgec2yqaQUaMb4BCKrRNK521GhrkpYydiRQUxcTL7yqJpR+5ZXrV45elfKcxh64TlAzexK59zbZVmZsuScIysrK0/Z/gG5Yvnf3HfnAxzT9Rj6Xt2n6H1mZ7Njx06GP3Y/3U7/NwBdju3M2rXreOeNd+l1ec9y3d0gFZdzjuzs7DxlkZGRZbb/zIxMZv8yhw7HtadmbOX9RcU5h8t2ecoiItVRV1bC7edjaVqgDwMFBqiZ9Qf6Azz38jNceU2/UhzGm9kzZ3PdVTfkKZs5f/q+z4lrkhh07U00iW/Mk88PL7L1CRBbJxZWreHYE7rmKT/uhGP5bcp0NqZspH6D+mVzAhXUtvQ0akflb2nGVq/JtvT8rcv9t21aq2G+8jqBlufWQEu0Mlo6Zzkv3P5KnrKXJj1dZvufN+0vdqXtqvTdt8mLkvlx2I95ynq/37tU+6wWXY2MnRn5ynen5dyqqBZTrVT7r0giwuxVCoWmhpnNPdAiIP9PugDn3ChgFMC2zC3uQOsF02GHH8a7nxTcQN6wPpkbrhlETEw0L772PDVr5u9eKcjBh7Ri3pwDP49oZTygoyJatmkVbQ5qka/8kHotWLZ5dZHbntb6BKpXicpzH/SQei3I2JPBqtS1ZV7fiqJ523jufPmWoO3/9/EJ1IyN4fBj2wXtGBVBvZb1OOPhM8p0n7Xja7N+3vp85duSthF9UHSl6b6F8GuBFvUTvyHQFzivgGlTcKtWOjExMbTv0C7PBLBl8xZuuDbnec+XXn+ROnWL/yDzKaeeAsD0qb/nKf9tynQaNGxAXFzhg5Aqgx9XTKdT48OIj/1nFGfT2g05ukl7flw+vZAt4ccVv1Mtsird2/7fvrJIi+Dstv9iyurZZGZVnsEW+6seXZ0WhzbLM5WVbZu3s3DmYjp3O5rIKmXXLVwRVa1RlYMOPijPVFrxR8Wza8sukhcm7yvL3JVJ0p9JND26aan3L6FTVL/ld0BN59yf+y8ws5+DUqMgSk9P58YBN7MuaR33PzqE5A3JJG/45x91q0Na7WuNjvlmLI8+MJSRb7zIMV2OBuDEf51A567H8PjDT5C6JZWm8U2ZOH4S06f9zgOPFe/tL+Hus3nfc9mRPXi5x4M8P/VdHHDzCX1Zn5bCp/PG7luvSa0GTLjqbV6e/iEjf/8IgIUpyxmz+GfuPXkAVSKqkLh1Pb07nUN8bCPu+GFEiM6ofNuemsayOcsB2JycSsbuTP74ZQ4AjVo03Pc4ytI5y3nxjle57M5L8nXTzpw0m+zs7ErffVuY9G3pJC/K+Vmxc9NO9uzew+oZOT0qsU1jiW2aM8o2eWEyPz7xI8deeyytTmoFQNOjmxLXOo7fXv2NI3sdmfMihW8XgIN251SuFn+lGkTknLu6kGWXln11gmvzps0sXpjzoPj9dz+Yb/mrb43kmK45DzVnu+ycQUjunx5oM+PJF0Yw8rmXGTXydbZt207LVi14dPjDdD/nTH9OopzbtWc3V3xxN4NPHsCI7ndiZvy2+k+G/fIaOzP/eVuTmVElIhKzvJ0gg8c9w60n9uOWE/pSO6omi1JWcM3X97EgeZnfp1IhrFu5njcfeS9P2d75s/qewTmBAN07CMntNxAJYMb4mTRu1YhmbeODX+EKamvSVqa+ODVP2d75Dhf+85IFR84gpNwDkSzC+Nft/+KPj/8g4d0EsjKziGsdR7fB3Yg5qHi3j8KFheAeqJldDDwEtAO6OucKfH+lmXUHngcigTecc08UuW/ngnuLMlT3QCubLi+VbqCDFM9LFw8KdRUqhalrC35Hr5Sth7o+5GuiDZl+n6c8GHrcY57raWbtgGzgNeCOggLUzCKBJcDpQCIwE+jtnFtQ2L4r/XOgIiLij1B04TrnFkKRA5i6AsuccysC634CnA8oQEVEJPSs/L78rimwJtd8InBsURspQEVExBdeW6C53y0QMCrwuOTe5ROBgl7gPMQ5942ngxaDAlRERHzh9TnQ3O8WOMDy07zWKSAJyP1sWHygrFDltj0tIiLhxTz+8cFMoI2ZtTKzakAvYHRRGylARUTEF6H4NhYzu9DMEoHjgTFmNi5Q3sTMxgI45/YAg4BxwELgM+fcX0XtW124IiLii1C8ys859zXwdQHla4Gzc82PBcbuv15hFKAiIuKLiDDr9FSAioiIL8LtZfIKUBER8YUCVERExINK9X2gIiIiZUUtUBEREQ/C7evMwmtIlIiIiE/UAhUREV+E4vtAg0kBKiIivoiw8Or0VICKiIgvNIhIRETEA3XhioiIeBBuo3AVoCIi4gu1QEVERDxQC1RERMQD0yhcERGRklMXroiIiAfqwhUREfFAz4GKiIh4oK8zExER8UAtUBEREQ80CldERMQDdeGKiIh4oC5cERERD8LtOdDw6pAWERHxiVqgIiLiC3XhioiIeBBug4jUhSsiIr4wi/A0le6YdrGZ/WVm2WbWuZD1VprZPDP708wSirNvtUBFRMQXIRpENB/4D/BaMdb9t3NuY3F3rAAVERFfhOIeqHNuYbCOrS5cERHxhXn84xMHjDezWWbWvzgbBL0FWi0iKtiHEGDWoE9DXYVKYVtmaqirUCkcVrd9qKsgQeC1FRgItNyhNso5NyrX8olAowI2HeKc+6aYhznJOZdkZg2ACWa2yDn3a2EbqAtXRER84XUUbiAsRxWy/DSvdcq1j6TA38lm9jXQFSg0QNWFKyIivjAzT5MP9Yoxs1p7PwNnkDP4qFAKUBER8YUR4Wkq1THNLjSzROB4YIyZjQuUNzGzsYHVGgJTzGwOMAMY45z7oah9qwtXRER8EaJRuF8DXxdQvhY4O/B5BdCppPtWgIqIiC/C7WXyClAREfFFhN6FKyIiUnJqgYqIiHigb2MRERHxoLQjasub8DobERERn6gFKiIivlAXroiIiAfh9oXaClAREfGFWqAiIiIe6DEWERERD9QCFRER8SDcHmNRgIqIiC/0Kj8REREPdA9URETEA90DFRER8UAtUBEREQ/UAhUREfEgQqNwRURESk4tUBEREQ90D1RERMQDtUBFREQ8CLcWaHjd0RUREfGJWqAiIuKLcGuBKkBFRMQfugcqIiJScmqBioiIeBBuo3A1iEhERHxhHv+U6phmT5rZIjOba2Zfm1mdA6zX3cwWm9kyM7unOPtWgIqIiC9CEaDABKCDc+4IYAkwOF+9zCKBkcBZQHugt5m1L2rHClAREfGFmXmaSsM5N945tycwOx2IL2C1rsAy59wK51wG8AlwflH7VoCKiIgvQtQCze0q4PsCypsCa3LNJwbKCqVBRCIi4guvYWhm/YH+uYpGOedG5Vo+EWhUwKZDnHPfBNYZAuwBPvRUiQIoQEVExBdeu2MDYTmqkOWnFXHcfsC5wKnOOVfAKklAs1zz8YGyQilARUTEF6F4DtTMugN3ASc753YeYLWZQBsza0VOcPYCLi1q35UqQLOysvjg3Q+ZPHkKy5ctJ31XOs1bNOeSS3tywYXnExFR9C3hP2b/yasvv8biRYvZnb6b5i2a0+vSS7jwvxf4cAYVQ1ZWFh++9zFTJ09lxbK/SU9Pp1mLZvTsdRE9LjyvyOvcv991zEqYXeCy4088jpdeeyEY1a5wsrKy+OLDr/h9ygxWLl9FegGQSawAAA79SURBVPpu4ps35fye53HW+WcWeZ1nTEvgk3c+ZeWK1aRtSyO2biyHd2pPvwF9aHlIC5/OovzLysriyw+/ZsaUBFauWMXu9N00bd6E83uex5k9Ti/yOv86cTI//vALSxYsJXVLKg0a1eekbidy6VWXEB0T7dNZlA8heg70JSAKmBA4/nTn3HVm1gR4wzl3tnNuj5kNAsYBkcBbzrm/itqxFdyaLTvpWTuDe4AS2LljJ2d0O5Nze5zLscd1JTo6msm/TuGD9z6kb78+3HbnrYVuv2TxEi7v1ZeOnTpyeZ9LqV6jBhPGTeTLz79kyAOD6dmrp09nkt+e7MyQHXt/O3fu5KxTz+Wc886m63FdiI6uwZTJ0/jovY+5/IrLuOWOmwrdfsXyFexI25GnbO6ceTwz4jnuvu8ueva6KJjVL9S2zNSQHXt/u3bu4uIzL+WMc0/j6K5HUSO6Br9PmcEXH35Fzz7/5bpb+xe6/aTvf2LpoqW063AYsXXrkLw+mY/e/pSUDSm8+dlrNGrS0KczyS/LZYXs2PvbtXMXvbr34fRzT+WorkdSo0Z1ZkxN4MsPv+aiy//DgFuvKXT7QX1voUHjBpx48nHENYxj2aLlvPfahzRrGc8L7zxTrF/cg6VZzMG+JtrC1Dme8qBdnU7l8g0MlSpAs7KySNueRmyd2DzlDwx5iO/HfM/k6b9QvXr1A27/wrMv8u7b7zH5t1/y/ObYp3dfAN7/+L3gVLwYylOAZmVlkZaWRmxs3uv88H2P8sPYcfw0bWKh17kgj9z/KGO/+4FxP4/Nt18/lacAzcrKYkfaDmrH1s5TPvyhp5n0/Y98+8tXRFWPKtE+V69cwxUXXs3AW/vTs2/oflEpTwGac513Uju2Vp7yJx96hh9/+Jn//fx5odc5dUsqdermfXZ//HcTGfHA0zz56uMc1fXIoNS7OPwO0EWpcz3lwWF1jiiXAVqpHmOJjIzMF54AHTq2JyMjg9Qthf9wzMzMpErVKvn+s9SsVYvs7HLze0LIRUZGFhhy7Tu0C1znrSXa365d6Uwc/yP/OuX/Qhqe5U1kZGS+8AQ4rH1bMjMy2ZpasusMEBvYX0SVyFLXL1zkXOda+coPPXzvdd5W6Pb7hyfAoe3bArAxeVPZVLKCCMVzoMFUqQL0QBJmzqJW7VrE1Y8rdL0eF/YAYPiwESQnJ7Nt23a+/PwrZkyfweVXXOZHVSu02Ql/BK7zQSXa7qdJP7Njxw7OPf+cINUsvMyZNZeatWpyUFzxrnNWVhaZmZkkrkri6ceep15cPU7tfkpwKxkG5s6aF7jO9Uq+7ex5ADQ/uFkRa4aXcvAcaJkqchCRmR1GzgOlvzvn0nKVd3fO/RDMyvlh6pRpjP9hAjfceD1VqhR+Odq0ac2b77zBrTfdxqcffwZAlSpVGPLgvZx1dnc/qlthTZv6GxPGTWTgoAFFXuf9jRk9hnr16nHCSccHqXbhY8a0BH6e8CtXXX8FkcVsRV7f5yaWLFwKQNNmTXjmtRHUrVc3mNWs8GZOm8UvEybTb2CfYl/nvTYmb+TdV97n6GOP2tcSlYqp0J9kZnYTcAOwEHjTzG7e+1AqMAwotwHqnCMrK+99lP1/cC9ftpx77hhMl65duPKafkXuc9XKVdx+8x0c0voQ7ntwCNWjovjpx58Z+vAwoqpFcc55Z5flKVQIxbnOK5avYMid99O56zFccXXfEu0/JTmFGdNn0vvyS0ocvOHEOUd2Vnaesv1/cK9cvorHBj/OkV060bvfJcXe972P3c2OHTtYl7ieT9//nDsG3sOLbz9DoyYFPZce3opznVetWMWwe5/gyM5H0KtfyQYO7tq5iwdufYTIyEjufKjwQYvhqDy3Jr0o6ifStcAxzrk0M2sJfGFmLZ1zz8OBr0Tut0a89MqLXH3tVWVU3eJLmDmLa/pdm6dszoI/9n1OXJPIgGsG0jS+Kc+++HSxfji/+NxLVKlahRdffp6qVasCcOzxx7I1dSsjHn+Ss87pHtIRdaEwa+ZsBlw1MG/Z/Bn7PieuSeL6a2+kSXwTnnp+RIlDcOx335Odnc25PSp39+2cWXO59do785T99Mf4fZ/XJq7jzoH30KhpQx59+sEStYpaHNwcgPYd23HsSV3odXYfPnrrU2677+ayqXwFMmfWPO7of3eesomz/3nz29rEddw1cAiNmjTioafvL9F13p2+m/tueYh1Set4+vUR1G9Yv8zqXVGU5/uZXhT10yxib7etc26lmZ1CToi2oJAAzf3WiFCNwm1/eDs++uyDApdtWL+Ba68aQM2aNXnl9ZHUrFmzWPtcunQZbQ9tuy889+pwRAfGjvmezZs2F3kfNdy0O/ww3v/knQKXbVi/gYHXXE9MTAwvvfZCsa9zbt99M4a2h7ah7WGVu6urbbs2vPrBSwUuS9mQwu0D7iK6ZjRPjnycmJoxno9Ts1ZNmjZrQtKatZ73UZG1bdeakR88X+CylA0p3HXdYGJionli5GMlus57Mvfw8F1DWbJgKcNfHsrBbVqVVZUrmMoVoBvM7Ejn3J8AgZboucBbQMeg164UYmJiOLzD4fnKN2/eTP+rrwPgtTdeoW7d4t/riYs7iMWLFpOZkUnVav+E6Ly584iKiqqUI0RjYmJo3yH/t/5s2byFgdcOAuDl11+kbgEjEYuyYP4CViz/m9vuuqXU9azoomOiOfTw/L9EpG5O5fbrclpMT73yBLF1S/dvcPOmLaxeuYbTzu5Wqv1UVNEx0QXel0zdkspdA+8FYPgrQ0t0nbOzsxl23wj+nDmHx557iPZHtCuz+lY04dYCLaq/sS+wPneBc26Pc64v8K+g1SpI0tPTGXjtDaxNWsuNNw9iw4YNzJ0zd9+UlrZvjBTffvMtR3fsTMLMhH1lvS69hKTEJG684WZ+mvQT06b+xrDHnuD7MT9wca+L8oRqZZaens4NA25iXdI6rr9pIMkbkpk3Z96+Kfd1/u6bMXTtdDyzZuZ/89B3o8cSWSWSs87RAK2C7E7fzV033Mv6tRu4etCVpGzYyIK5C/dNuV9GMe7bCZzauTt/JszdV3b/bQ/x3qgPmPLTNP6Y+Sejv/iOW665ncjISHr2Cd0zoOXN7vTd3HPDfaxfu4Erb7ii0Os8/ruJnNHlHObM+uc6v/DESH6dMJmLLv8P1WtUz7NtyoaUUJxSyFSqUbjOucRClk0t++oE16ZNm1m0cBEAg++6N9/yN955nS5dOwOQnZ0zOCb3eyZOP/N0Rr76Im+/+Q4PP/AIu3dn0KxZPPfeP5iLev7Xl3OoCDZv2szihYsBuO/uB/Itf+2tV+jc9RgAst3e65y3pz8zcw/jvh/PCSceT72DSv6YQGWwZfMWli5aBsDQe5/It/zZ15/kyM6dgNyDY/65zu07tuOnCb/w2ftfsmfPHho0rE+nzkdw2VW9KuUAogPZsjmVZYuWA/D4kBH5lj81ajhHdj4CAJedc51z/3OeOTXnl/CP3vyEj978JM+2ffpfxhXXXR6kmpc/5TkMvahUbyIKZ+XpTUThrDy9iSiclac3EYUzv99EtDJtqac8aFmzTblM3sr7XICIiPgq3FqgClAREfGFAlRERMSDcBuFqwAVERFfqAUqIiLigVqgIiIiHqgFKiIi4okCVEREpMTCKz4VoCIi4pNwuwdaub57S0REpIyoBSoiIj4JrxaoAlRERHwRXvGpABUREd+EV4QqQEVExBcaRCQiIiJqgYqIiD/0JiIREREPFKAiIiIVhJk9CZwHZADLgSudc6kFrLcS2A5kAXucc52L2rfugYqIiC/MzNNUShOADs65I4AlwOBC1v23c+7I4oQnKEBFRCSMOefGO+f2BGanA/FltW8FqIiI+MK8/jHrb2YJuab+HqtwFfD9AZY5YLyZzSru/nUPVEREfOKtO9Y5NwoYdcC9mk0EGhWwaIhz7pvAOkOAPcCHB9jNSc65JDNrAEwws0XOuV8Lq5cCVEREfBGsMbjOudMKPa5ZP+Bc4FTnnDvAPpICfyeb2ddAV6DQAFUXroiI+CIUg4jMrDtwF9DDObfzAOvEmFmtvZ+BM4D5Re1bASoiIj4xj1OpvATUIqdb9k8zexXAzJqY2djAOg2BKWY2B5gBjHHO/VDUjtWFKyIivgjFaxScc60PUL4WODvweQXQqaT7VoCKiIhPwutNROrCFRER8UAtUBER8YW+zkxERETUAhUREX/o21hEREQ8UYCKiIiUWHjFpwJURER8Em6DiBSgIiLiEwWoiIhIiYVXfCpARUTEN+EVoQpQERHxRbjdA9WLFERERDywA3y3aFkK+gFERMQTX5uE6Vk7PeVB9cjoctl09SNAKxwz6++cGxXqeoQ7XWd/6Dr7Q9e58lEXbsH6h7oClYSusz90nf2h61zJKEBFREQ8UICKiIh4oAAtmO5j+EPX2R+6zv7Qda5kNIhIRETEA7VARUREPFCA7sfMupvZYjNbZmb3hLo+4cjM3jKzZDObH+q6hDMza2ZmP5nZAjP7y8xuDnWdwpGZVTezGWY2J3CdHw51ncQf6sLNxcwigSXA6UAiMBPo7ZxbENKKhRkz+xeQBrznnOsQ6vqEKzNrDDR2zs02s1rALOAC/XsuW5bzfroY51yamVUFpgA3O+emh7hqEmRqgebVFVjmnFvhnMsAPgHOD3Gdwo5z7ldgc6jrEe6cc+ucc7MDn7cDC4Gmoa1V+HE50gKzVQOTWiaVgAI0r6bAmlzziegHjoQBM2sJHAX8HtqahCczizSzP4FkYIJzTte5ElCAioQ5M6sJfAnc4pzbFur6hCPnXJZz7kggHuhqZro1UQkoQPNKAprlmo8PlIlUSIF7cl8CHzrnvgp1fcKdcy4V+AnoHuq6SPApQPOaCbQxs1ZmVg3oBYwOcZ1EPAkMbnkTWOiceybU9QlXZlbfzOoEPtcgZxDiotDWSvygAM3FObcHGASMI2fAxWfOub9CW6vwY2YfA78Bh5pZopldHeo6hakTgT5ANzP7MzCdHepKhaHGwE9mNpecX8InOOe+C3GdxAd6jEVERMQDtUBFREQ8UICKiIh4oAAVERHxQAEqIiLigQJURETEAwWoiIiIBwpQERERDxSgIiIiHvw/VekxSPRDZTUAAAAASUVORK5CYII=\n","text/plain":["<Figure size 720x360 with 2 Axes>"]},"metadata":{"needs_background":"light"}}]},{"cell_type":"markdown","metadata":{"id":"6F1CvviNvf0U"},"source":["---"]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"eY5_ri7ovC5d","executionInfo":{"status":"ok","timestamp":1638441822532,"user_tz":-330,"elapsed":3730,"user":{"displayName":"Sparsh Agarwal","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"13037694610922482904"}},"outputId":"f3416f07-e18a-4b51-d436-2948f1f20495"},"source":["!pip install -q watermark\n","%reload_ext watermark\n","%watermark -a \"Sparsh A.\" -m -iv -u -t -d"],"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["Author: Sparsh A.\n","\n","Last updated: 2021-12-02 10:43:44\n","\n","Compiler    : GCC 7.5.0\n","OS          : Linux\n","Release     : 5.4.104+\n","Machine     : x86_64\n","Processor   : x86_64\n","CPU cores   : 2\n","Architecture: 64bit\n","\n","gym       : 0.17.3\n","matplotlib: 3.2.2\n","numpy     : 1.19.5\n","seaborn   : 0.11.2\n","IPython   : 5.5.0\n","\n"]}]},{"cell_type":"markdown","metadata":{"id":"eQZmkNqovgsp"},"source":["---"]},{"cell_type":"markdown","metadata":{"id":"Jpw59lINviPG"},"source":["**END**"]}]}