VizDoom-Basic-Copy1.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "ename": "ModuleNotFoundError",
     "evalue": "No module named 'tensorflow'",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mModuleNotFoundError\u001b[0m                       Traceback (most recent call last)",
      "Cell \u001b[1;32mIn[1], line 1\u001b[0m\n\u001b[1;32m----> 1\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mtensorflow\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m \u001b[38;5;21;01mtf\u001b[39;00m\n\u001b[0;32m      2\u001b[0m physical_devices \u001b[38;5;241m=\u001b[39m tf\u001b[38;5;241m.\u001b[39mconfig\u001b[38;5;241m.\u001b[39mlist_physical_devices(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mGPU\u001b[39m\u001b[38;5;124m'\u001b[39m)\n\u001b[0;32m      3\u001b[0m tf\u001b[38;5;241m.\u001b[39mconfig\u001b[38;5;241m.\u001b[39mexperimental\u001b[38;5;241m.\u001b[39mset_memory_growth(physical_devices[\u001b[38;5;241m0\u001b[39m], \u001b[38;5;28;01mTrue\u001b[39;00m)\n",
      "\u001b[1;31mModuleNotFoundError\u001b[0m: No module named 'tensorflow'"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "physical_devices = tf.config.list_physical_devices('GPU')\n",
    "tf.config.experimental.set_memory_growth(physical_devices[0], True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "ename": "OSError",
     "evalue": "[WinError 127] The specified procedure could not be found. Error loading \"C:\\Users\\91701\\anaconda3\\envs\\please\\lib\\site-packages\\torch\\lib\\nvfuser_codegen.dll\" or one of its dependencies.",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mOSError\u001b[0m                                   Traceback (most recent call last)",
      "Cell \u001b[1;32mIn[1], line 1\u001b[0m\n\u001b[1;32m----> 1\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mtorch\u001b[39;00m\n\u001b[0;32m      2\u001b[0m torch\u001b[38;5;241m.\u001b[39mcuda\u001b[38;5;241m.\u001b[39mset_device(\u001b[38;5;241m0\u001b[39m)\n",
      "File \u001b[1;32m~\\anaconda3\\envs\\please\\lib\\site-packages\\torch\\__init__.py:122\u001b[0m\n\u001b[0;32m    120\u001b[0m     err \u001b[38;5;241m=\u001b[39m ctypes\u001b[38;5;241m.\u001b[39mWinError(last_error)\n\u001b[0;32m    121\u001b[0m     err\u001b[38;5;241m.\u001b[39mstrerror \u001b[38;5;241m+\u001b[39m\u001b[38;5;241m=\u001b[39m \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m Error loading \u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mdll\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m or one of its dependencies.\u001b[39m\u001b[38;5;124m'\u001b[39m\n\u001b[1;32m--> 122\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m err\n\u001b[0;32m    123\u001b[0m \u001b[38;5;28;01melif\u001b[39;00m res \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[0;32m    124\u001b[0m     is_loaded \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mTrue\u001b[39;00m\n",
      "\u001b[1;31mOSError\u001b[0m: [WinError 127] The specified procedure could not be found. Error loading \"C:\\Users\\91701\\anaconda3\\envs\\please\\lib\\site-packages\\torch\\lib\\nvfuser_codegen.dll\" or one of its dependencies."
     ]
    }
   ],
   "source": [
    "import torch\n",
    "torch.cuda.set_device(0)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 1. Getting VizDoom Up and Running"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: vizdoom in c:\\users\\91701\\anaconda3\\envs\\please\\lib\\site-packages (1.2.0)\n",
      "Requirement already satisfied: numpy in c:\\users\\91701\\anaconda3\\envs\\please\\lib\\site-packages (from vizdoom) (1.24.3)\n",
      "Requirement already satisfied: pygame>=2.1.3 in c:\\users\\91701\\anaconda3\\envs\\please\\lib\\site-packages (from vizdoom) (2.4.0)\n",
      "Requirement already satisfied: gymnasium>=0.28.0 in c:\\users\\91701\\anaconda3\\envs\\please\\lib\\site-packages (from vizdoom) (0.28.1)\n",
      "Requirement already satisfied: jax-jumpy>=1.0.0 in c:\\users\\91701\\anaconda3\\envs\\please\\lib\\site-packages (from gymnasium>=0.28.0->vizdoom) (1.0.0)\n",
      "Requirement already satisfied: farama-notifications>=0.0.1 in c:\\users\\91701\\anaconda3\\envs\\please\\lib\\site-packages (from gymnasium>=0.28.0->vizdoom) (0.0.4)\n",
      "Requirement already satisfied: cloudpickle>=1.2.0 in c:\\users\\91701\\anaconda3\\envs\\please\\lib\\site-packages (from gymnasium>=0.28.0->vizdoom) (2.2.1)\n",
      "Requirement already satisfied: typing-extensions>=4.3.0 in c:\\users\\91701\\anaconda3\\envs\\please\\lib\\site-packages (from gymnasium>=0.28.0->vizdoom) (4.5.0)\n"
     ]
    }
   ],
   "source": [
    "!pip install vizdoom"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Cloning into 'ViZDoom'...\n"
     ]
    }
   ],
   "source": [
    "!cd github & git clone https://github.com/mwydmuch/ViZDoom"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "from vizdoom import * \n",
    "\n",
    "import random\n",
    "\n",
    "import time \n",
    "\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Setup game\n",
    "game = DoomGame()\n",
    "game.load_config('github/VizDoom/scenarios/basic.cfg')\n",
    "game.init()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "actions = np.identity(3, dtype=np.uint8)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "state = game.get_state()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([50.])"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "state.game_variables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "reward: -4.0\n",
      "reward: -4.0\n",
      "reward: -4.0\n",
      "reward: 99.0\n",
      "Result: 87.0\n",
      "reward: -4.0\n",
      "reward: -9.0\n",
      "reward: -4.0\n",
      "reward: -4.0\n",
      "reward: -4.0\n",
      "reward: -9.0\n",
      "reward: -4.0\n",
      "reward: -4.0\n",
      "reward: -4.0\n",
      "reward: -4.0\n",
      "reward: -4.0\n",
      "reward: -9.0\n",
      "reward: -4.0\n",
      "reward: -4.0\n",
      "reward: -9.0\n",
      "reward: -4.0\n",
      "reward: -4.0\n",
      "reward: -4.0\n",
      "reward: -9.0\n",
      "reward: -4.0\n",
      "reward: -4.0\n",
      "reward: -4.0\n",
      "reward: -4.0\n",
      "reward: -4.0\n",
      "reward: -4.0\n",
      "reward: -4.0\n",
      "reward: -9.0\n",
      "reward: -4.0\n",
      "reward: -4.0\n",
      "reward: -9.0\n",
      "reward: -4.0\n",
      "reward: -4.0\n",
      "reward: -4.0\n",
      "reward: -4.0\n",
      "reward: -9.0\n",
      "reward: -4.0\n",
      "reward: -4.0\n",
      "reward: -4.0\n",
      "reward: -4.0\n",
      "reward: -4.0\n",
      "reward: -4.0\n",
      "reward: -4.0\n",
      "reward: -4.0\n",
      "reward: 99.0\n",
      "Result: -113.0\n",
      "reward: -4.0\n",
      "reward: -4.0\n",
      "reward: -9.0\n",
      "reward: -4.0\n",
      "reward: -4.0\n",
      "reward: -4.0\n",
      "reward: -4.0\n",
      "reward: 99.0\n",
      "Result: 66.0\n",
      "reward: -4.0\n",
      "reward: -4.0\n",
      "reward: -9.0\n",
      "reward: -4.0\n",
      "reward: -4.0\n",
      "reward: -4.0\n",
      "reward: -9.0\n",
      "reward: -4.0\n",
      "reward: -4.0\n",
      "reward: -4.0\n",
      "reward: -4.0\n",
      "reward: -9.0\n",
      "reward: -4.0\n",
      "reward: -4.0\n",
      "reward: -4.0\n",
      "reward: -4.0\n",
      "reward: -9.0\n",
      "reward: -4.0\n",
      "reward: -4.0\n",
      "reward: -9.0\n",
      "reward: -4.0\n",
      "reward: -4.0\n",
      "reward: -4.0\n",
      "reward: -9.0\n",
      "reward: -4.0\n",
      "reward: -4.0\n",
      "reward: -4.0\n",
      "reward: -4.0\n",
      "reward: -9.0\n",
      "reward: -4.0\n",
      "reward: -4.0\n",
      "reward: -9.0\n",
      "reward: -4.0\n",
      "reward: -4.0\n",
      "reward: -4.0\n",
      "reward: -9.0\n",
      "reward: -4.0\n",
      "reward: -4.0\n",
      "reward: -9.0\n",
      "reward: -4.0\n",
      "reward: -4.0\n",
      "reward: -4.0\n",
      "reward: -4.0\n",
      "reward: -9.0\n",
      "reward: -4.0\n",
      "reward: -4.0\n",
      "reward: -4.0\n",
      "reward: -4.0\n",
      "reward: -9.0\n",
      "reward: -4.0\n",
      "reward: -4.0\n",
      "reward: -4.0\n",
      "reward: -4.0\n",
      "reward: -9.0\n",
      "reward: -4.0\n",
      "reward: -4.0\n",
      "reward: -4.0\n",
      "reward: -4.0\n",
      "reward: -9.0\n",
      "reward: -4.0\n",
      "reward: -4.0\n",
      "reward: -4.0\n",
      "reward: -9.0\n",
      "reward: -4.0\n",
      "reward: -4.0\n",
      "reward: -4.0\n",
      "reward: -9.0\n",
      "reward: -4.0\n",
      "reward: -4.0\n",
      "reward: -9.0\n",
      "reward: -4.0\n",
      "reward: -4.0\n",
      "reward: -4.0\n",
      "reward: -9.0\n",
      "reward: -4.0\n",
      "Result: -390.0\n",
      "reward: -4.0\n",
      "reward: -4.0\n",
      "reward: 99.0\n",
      "Result: 91.0\n",
      "reward: -4.0\n",
      "reward: 99.0\n",
      "Result: 95.0\n",
      "reward: -4.0\n",
      "reward: -4.0\n",
      "reward: -9.0\n",
      "reward: -4.0\n",
      "reward: -4.0\n",
      "reward: -4.0\n",
      "reward: -4.0\n",
      "reward: -4.0\n",
      "reward: -4.0\n",
      "reward: -4.0\n",
      "reward: 99.0\n",
      "Result: 54.0\n",
      "reward: -4.0\n",
      "reward: -4.0\n",
      "reward: -4.0\n",
      "reward: -9.0\n",
      "reward: -4.0\n",
      "reward: -4.0\n",
      "reward: -9.0\n",
      "reward: -4.0\n",
      "reward: -4.0\n",
      "reward: -4.0\n",
      "reward: -4.0\n",
      "reward: -4.0\n",
      "reward: -4.0\n",
      "reward: -4.0\n",
      "reward: -4.0\n",
      "reward: -4.0\n",
      "reward: -4.0\n",
      "reward: -4.0\n",
      "reward: -9.0\n",
      "reward: -4.0\n",
      "reward: -4.0\n",
      "reward: -4.0\n",
      "reward: -4.0\n",
      "reward: -4.0\n",
      "reward: -4.0\n",
      "reward: -4.0\n",
      "reward: -4.0\n",
      "reward: 99.0\n",
      "Result: -24.0\n",
      "reward: -4.0\n",
      "reward: -4.0\n",
      "reward: -4.0\n",
      "reward: -4.0\n",
      "reward: -9.0\n",
      "reward: -4.0\n",
      "reward: -4.0\n",
      "reward: -4.0\n",
      "reward: -4.0\n",
      "reward: -4.0\n",
      "reward: -4.0\n",
      "reward: -9.0\n",
      "reward: -4.0\n",
      "reward: -4.0\n",
      "reward: -9.0\n",
      "reward: -4.0\n",
      "reward: -4.0\n",
      "reward: -4.0\n",
      "reward: -4.0\n",
      "reward: -4.0\n",
      "reward: -4.0\n",
      "reward: -9.0\n",
      "reward: -4.0\n",
      "reward: -4.0\n",
      "reward: -4.0\n",
      "reward: -9.0\n",
      "reward: -4.0\n",
      "reward: -4.0\n",
      "reward: -4.0\n",
      "reward: 98.0\n",
      "Result: -43.0\n",
      "reward: -4.0\n",
      "reward: -9.0\n",
      "reward: -4.0\n",
      "reward: -4.0\n",
      "reward: -9.0\n",
      "reward: -4.0\n",
      "reward: -4.0\n",
      "reward: -4.0\n",
      "reward: -9.0\n",
      "reward: -4.0\n",
      "reward: -4.0\n",
      "reward: -4.0\n",
      "reward: -9.0\n",
      "reward: -4.0\n",
      "reward: -4.0\n",
      "reward: -4.0\n",
      "reward: -4.0\n",
      "reward: -4.0\n",
      "reward: -4.0\n",
      "reward: -9.0\n",
      "reward: -4.0\n",
      "reward: -4.0\n",
      "reward: -4.0\n",
      "reward: -9.0\n",
      "reward: -4.0\n",
      "reward: -4.0\n",
      "reward: -4.0\n",
      "reward: -9.0\n",
      "reward: -4.0\n",
      "reward: -4.0\n",
      "reward: -9.0\n",
      "reward: -4.0\n",
      "reward: -4.0\n",
      "reward: -4.0\n",
      "reward: -4.0\n",
      "reward: -9.0\n",
      "reward: -4.0\n",
      "reward: -4.0\n",
      "reward: -4.0\n",
      "reward: -9.0\n",
      "reward: -4.0\n",
      "reward: -4.0\n",
      "reward: -4.0\n",
      "reward: -9.0\n",
      "reward: -4.0\n",
      "reward: -4.0\n",
      "reward: -4.0\n",
      "reward: -4.0\n",
      "reward: -4.0\n",
      "reward: -4.0\n",
      "reward: -4.0\n",
      "reward: -4.0\n",
      "reward: -9.0\n",
      "reward: -4.0\n",
      "reward: -4.0\n",
      "reward: -4.0\n",
      "reward: 97.0\n",
      "reward: -1.0\n",
      "Result: -188.0\n"
     ]
    }
   ],
   "source": [
    " \n",
    "episodes = 10 \n",
    "for episode in range(episodes): \n",
    "    \n",
    "    game.new_episode()\n",
    "     \n",
    "    while not game.is_episode_finished(): \n",
    "         \n",
    "        state = game.get_state()\n",
    "         \n",
    "        img = state.screen_buffer\n",
    "        \n",
    "        info = state.game_variables\n",
    "        \n",
    "        reward = game.make_action(random.choice(actions),4)\n",
    "         \n",
    "        print('reward:', reward) \n",
    "        time.sleep(0.02)\n",
    "    print('Result:', game.get_total_reward())\n",
    "    time.sleep(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "game.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "tags": []
   },
   "source": [
    "# 2. Converting it to a Gym Environment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: gym in c:\\users\\91701\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (0.26.2)\n",
      "Requirement already satisfied: numpy>=1.18.0 in c:\\users\\91701\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from gym) (1.24.3)\n",
      "Requirement already satisfied: cloudpickle>=1.2.0 in c:\\users\\91701\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from gym) (2.2.1)\n",
      "Requirement already satisfied: gym-notices>=0.0.4 in c:\\users\\91701\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from gym) (0.0.8)\n"
     ]
    }
   ],
   "source": [
    "!pip install gym"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: opencv-python in c:\\users\\91701\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (4.7.0.72)\n",
      "Requirement already satisfied: numpy>=1.21.2 in c:\\users\\91701\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from opencv-python) (1.24.3)\n"
     ]
    }
   ],
   "source": [
    "!pip install opencv-python"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "from gym import Env\n",
    " \n",
    "from gym.spaces import Discrete, Box\n",
    " \n",
    "import cv2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "ename": "AttributeError",
     "evalue": "'NoneType' object has no attribute 'screen_buffer'",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
      "Cell \u001b[1;32mIn[13], line 1\u001b[0m\n\u001b[1;32m----> 1\u001b[0m \u001b[43mgame\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mget_state\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mscreen_buffer\u001b[49m\u001b[38;5;241m.\u001b[39mshape\n",
      "\u001b[1;31mAttributeError\u001b[0m: 'NoneType' object has no attribute 'screen_buffer'"
     ]
    }
   ],
   "source": [
    "game.get_state().screen_buffer.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "class VizDoomGym(Env): \n",
    "    \n",
    "    def __init__(self, render=False): \n",
    "        \n",
    "        super().__init__()\n",
    "         \n",
    "        self.game = DoomGame()\n",
    "        self.game.load_config('github/VizDoom/scenarios/basic.cfg')\n",
    "        \n",
    "        \n",
    "        if render == False: \n",
    "            self.game.set_window_visible(False)\n",
    "        else:\n",
    "            self.game.set_window_visible(True)\n",
    "        \n",
    "        \n",
    "        self.game.init()\n",
    "        \n",
    "        \n",
    "        self.observation_space = Box(low=0, high=255, shape=(100,160,1), dtype=np.uint8) \n",
    "        self.action_space = Discrete(3)\n",
    "        \n",
    "    \n",
    "    def step(self, action):\n",
    "        \n",
    "        actions = np.identity(3)\n",
    "        reward = self.game.make_action(actions[action], 4) \n",
    "        \n",
    "        \n",
    "        if self.game.get_state(): \n",
    "            state = self.game.get_state().screen_buffer\n",
    "            state = self.grayscale(state)\n",
    "            ammo = self.game.get_state().game_variables[0]\n",
    "            info = ammo\n",
    "        else: \n",
    "            state = np.zeros(self.observation_space.shape)\n",
    "            info = 0 \n",
    "        \n",
    "        info = {\"info\":info}\n",
    "        done = self.game.is_episode_finished()\n",
    "        \n",
    "        return state, reward, done, info \n",
    "    \n",
    "    \n",
    "    def render(): \n",
    "        pass\n",
    "    \n",
    "     \n",
    "    def reset(self): \n",
    "        self.game.new_episode()\n",
    "        state = self.game.get_state().screen_buffer\n",
    "        return self.grayscale(state)\n",
    "    \n",
    "    \n",
    "    def grayscale(self, observation):\n",
    "        gray = cv2.cvtColor(np.moveaxis(observation, 0, -1), cv2.COLOR_BGR2GRAY)\n",
    "        resize = cv2.resize(gray, (160,100), interpolation=cv2.INTER_CUBIC)\n",
    "        state = np.reshape(resize, (100,160,1))\n",
    "        return state\n",
    "    \n",
    "    \n",
    "    def close(self): \n",
    "        self.game.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "env = VizDoomGym(render=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "state = env.reset()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "tags": []
   },
   "source": [
    "# 3. View Game State"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[[55],\n",
       "        [50],\n",
       "        [59],\n",
       "        ...,\n",
       "        [57],\n",
       "        [57],\n",
       "        [66]],\n",
       "\n",
       "       [[68],\n",
       "        [65],\n",
       "        [65],\n",
       "        ...,\n",
       "        [56],\n",
       "        [67],\n",
       "        [72]],\n",
       "\n",
       "       [[49],\n",
       "        [79],\n",
       "        [66],\n",
       "        ...,\n",
       "        [79],\n",
       "        [51],\n",
       "        [29]],\n",
       "\n",
       "       ...,\n",
       "\n",
       "       [[75],\n",
       "        [63],\n",
       "        [62],\n",
       "        ...,\n",
       "        [44],\n",
       "        [71],\n",
       "        [60]],\n",
       "\n",
       "       [[15],\n",
       "        [48],\n",
       "        [47],\n",
       "        ...,\n",
       "        [49],\n",
       "        [69],\n",
       "        [47]],\n",
       "\n",
       "       [[22],\n",
       "        [14],\n",
       "        [26],\n",
       "        ...,\n",
       "        [57],\n",
       "        [37],\n",
       "        [39]]], dtype=uint8)"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "env.reset()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "env.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from stable_baselines3.common import env_checker"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "env_checker.check_env(env)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 3. View State"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: matplotlib in c:\\users\\91701\\anaconda3\\envs\\please\\lib\\site-packages (3.7.1)\n",
      "Requirement already satisfied: fonttools>=4.22.0 in c:\\users\\91701\\anaconda3\\envs\\please\\lib\\site-packages (from matplotlib) (4.39.4)\n",
      "Requirement already satisfied: pyparsing>=2.3.1 in c:\\users\\91701\\anaconda3\\envs\\please\\lib\\site-packages (from matplotlib) (3.0.9)\n",
      "Requirement already satisfied: python-dateutil>=2.7 in c:\\users\\91701\\anaconda3\\envs\\please\\lib\\site-packages (from matplotlib) (2.8.2)\n",
      "Requirement already satisfied: cycler>=0.10 in c:\\users\\91701\\anaconda3\\envs\\please\\lib\\site-packages (from matplotlib) (0.11.0)\n",
      "Requirement already satisfied: numpy>=1.20 in c:\\users\\91701\\anaconda3\\envs\\please\\lib\\site-packages (from matplotlib) (1.24.3)\n",
      "Requirement already satisfied: packaging>=20.0 in c:\\users\\91701\\anaconda3\\envs\\please\\lib\\site-packages (from matplotlib) (23.1)\n",
      "Requirement already satisfied: contourpy>=1.0.1 in c:\\users\\91701\\anaconda3\\envs\\please\\lib\\site-packages (from matplotlib) (1.0.7)\n",
      "Requirement already satisfied: pillow>=6.2.0 in c:\\users\\91701\\anaconda3\\envs\\please\\lib\\site-packages (from matplotlib) (9.5.0)\n",
      "Requirement already satisfied: kiwisolver>=1.0.1 in c:\\users\\91701\\anaconda3\\envs\\please\\lib\\site-packages (from matplotlib) (1.4.4)\n",
      "Requirement already satisfied: six>=1.5 in c:\\users\\91701\\anaconda3\\envs\\please\\lib\\site-packages (from python-dateutil>=2.7->matplotlib) (1.16.0)\n"
     ]
    }
   ],
   "source": [
    "!pip install matplotlib"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "from matplotlib import pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.image.AxesImage at 0x2658bc94dc0>"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAh8AAAFlCAYAAABLDIrrAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAACRb0lEQVR4nO29e7RddZXlv3LvTcI7yCOPCwkECCYk4ZXwCDgsSzJESktpKFSasmisHrYaSgFLBXtADVGM0KOUAhHErtLuLimVbimRLrTpAMEgCZDwCo8kQAiBkPCSXARJyL3n94e/s/uzV87c7JvHySHMOQZjfHPu3t/H+j7OYc7vWmtIo9FohGEYhmEYRpvQta07YBiGYRjGOwv+8WEYhmEYRlvhHx+GYRiGYbQV/vFhGIZhGEZb4R8fhmEYhmG0Ff7xYRiGYRhGW+EfH4ZhGIZhtBX+8WEYhmEYRlvhHx+GYRiGYbQV/vFhGIZhGEZbsdV+fFx11VWx//77xw477BDHHHNM3H333VurKcMwDMMw3kYYsjVyu/z0pz+Nv/qrv4prrrkmjjnmmLj88svj+uuvjyVLlsTIkSMr3x0YGIhVq1bFrrvuGkOGDNnSXTMMwzAMYyug0WjEq6++Gr29vdHV9RbcRmMr4Oijj27MmjWr+Hd/f3+jt7e3MXv27Ld8d+XKlY2I8H/+z//5P//n//zf2/C/lStXvuV3fU9sYaxfvz4WLlwYF1xwQfFZV1dXzJw5M+66666Nnl+3bl2sW7eu+Hfj/ydiDj/88Oju7o4NGzYUf9thhx2Kcn9/f1EeGBgoykOHDi3Vz78NHz68KPf29hblFStWFOU333yzKCvmpbu7e6P+5j5FRPT09LR8bo899ijKu+66a1F+5plnijJtMmzYsKJMe1T9suT7tBvxrne9qyjvsssuRfnZZ59tWY9qL39Ou7G/tBuf2XPPPYvyTjvtVJRXrVpVlH//+98XZdqDNo4ozwH7pezGPtEenK+1a9cW5fXr17ccQ+4L1x3LrJdtE/vtt19R5rhfeeWVlv2IKI+b65zt1ekHxzR27Nii/NprrxXlF198sdR2Q5CntLNaE+oZlmmP5557rtTGH/7wh5bjoD1YV501wfLOO+9clLlOIyJWr17dsh8KaqxvvPFGUebaVv3IfWE/uF7U+qozL2pNRETstttuRZlr/vXXXy/KXC9cH3m/tmqDZ07Veb7PPvsUZZ7br776alGmbfmuGp/6Xsl935x+8BnuVYJzEaHXJ/u77777FmWeD7/73e9aPs9y1XyrdUHsuOOOxbO//e1vS99rClv8x8eLL74Y/f39MWrUqNLno0aNiscee2yj52fPnh1f+9rXNvq8u7s7enp65MJVCyYvbi4S/o2LWi0qTrLqhzp4q55j26ofnHBVj1oIVe8Tg+1HncMsojw3deavTj/YNj/PY1NfNMpuLLNt1W/1pZaf4zriO3UO4jr2yD90VT/Uj+MtNS8RGx/STagDrc4PH9WP3LZaI4Q6uFU/+Iw6M/LflM3VeuTnam2rfuS+KBsoe9SZF7Um8t9UP9ReqPPjo+rsqvofzbfqh/qeUOuu6n8mN6cfdeyRv1fU+iTqnF/qf6irnqlzZlWdwwpb/MfHYHHBBRfEeeedV/y7r68vxo4dG7vuumv09PRUHrKDBQ3yxBNPFGVO7N57712U+QtcHbCbgr6+vqLM/4vcfffdizL/74L/V8X/M6qCYjtog5deeqko89cx7+Xw//qrfmgpqF/27Mfzzz9flJu/oCP0/2FtCmgP9YX8wgsvFGWyILS5Omg2F5xjMmB77bVXUa76IlRQX4QEP+f/va5Zs6Yoc15Y3hSoueCXDvvEtZnHQKZMtcG9y/fV2uRc8P9YuRdyXaofhJoL9oP/V1z1P1Uvv/xyy7q4Z+q0TbAfai4iyl9+/L/7OvZgP9QPQe63zCixj2RXeFazH2qtKnuwXMUucl/yHFV1DXb/5LVJO3At8PuK3yV1/ieiLtgX9oNsXLPet7znAWzxHx977bVXdHd3lw6uiD8eZKNHj97o+eHDh8tDwDAMwzCM7Q9b3NV22LBhMW3atJgzZ07x2cDAQMyZMydmzJixpZszDMMwDONthq0iu5x33nlx5plnxvTp0+Poo4+Oyy+/PF577bU466yzatfx8ssvR3d3d4wbN674jBTfU089VZTrSjN1KD9e2OK9FdJYpDxJdVXpdIS6V0LanxdiebmM4yZFznJV28oGfJ7U2oEHHliUOW5e0s2Xo9RFJiVd0ba8qET7jxgxoiivXLmyKNNmEfXu5qj+kZJUF2IpAfCiX0SZplX6vpp79oOUNeUfMoSUZiLKkgDbUzZQUgTtT3uQCs+XPtWFtsHagGWypPycMkhEWcLcHBtwrLQz139ev5QLeTapPaYuu7KvvPjNMtd/RFk2Ju2vLsyr8071lRLwhAkTSm1z/nkmcC7UnRZ1p4LPsO18cZH/pv3zXmzVHsen1j8lkSwdHXDAAUWZ5zDnPp/DTSgbcB1w3eULxvwe5Dnz+OOPt+xHnXtH6hmuu4iyxMQzmWdQcy4Gcz1hq/z4+PjHPx4vvPBCXHTRRbF69eo4/PDD41e/+tVGl1ANwzAMw3jnYatdOD377LPj7LPP3lrVG4ZhGIbxNsU293ZReOGFF6Krq6tEpdLff9KkSUWZ8Q/yRVfSYFVuVE1QRiGNOGbMmKJMSYS36inZ5LrUDXP2g5QpY1ywH4y9wLFl+UHdwK7TNum4pUuXtmybVGyWANj2YL2ESOOSCqed6Y2Tb6RzDvLfWoH94/OkPUn1kxLmvFTVpUD783l6LNAepN4zLctxk8qt0w/lcUKo/uU+qvgmCrQB51vJYfm2voonwXIdG5COJs3MfZEpdfZdySh1bMAxkfJmPxh7J7fH9VlnrxOUpMhMMxYRZbWIss159iq3XWUDSnlcz+xHXo88F3nW13HtVK7mlFUZsyNL15S6Btu2irHDtlnO0gelZp479PKpA9qT66sqng3394MPPliUW+2rwZz3TixnGIZhGEZb4R8fhmEYhmG0FR0ruzQajWg0GiVaiRFSSUGzPHHixFI9pAh5Q1kFpVF07fLly4syqUbSk7yRHFGm7UiV1QnFTCqVdC/7xPYoiURo7wQ1VtU2+8q2KYHRBhHlW9ukSeu0rbyKSBdS9qI3SH6f8g9Rp22+S1qc8ltuW3ku1YEaN+lXFcAoonxLX3l9qPZYVmuT/WD/qlDHzqRpVWA8rv8sHyipRUG1zTWs6uRZEqFtVSegF9umlMd+8IzK0gc9GwbbNkHqneuZY8trWQU4G2zbpPd5frH+3LYKOKfa5ufcI7Q526a8kee3KthdK7Btes7wDKEN+B2Trw8oeavO/uZ5fNBBBxVlSm6U9R5++OFSXUrWa2XnwQSiNPNhGIZhGEZb4R8fhmEYhmG0FUMam5KwYyuir68vRowYEVOnTo3u7m55G15lgsy0LKlE0roqQ6gKAKPMVJXIiXIQ+86AOHxHJXHjjXuVPCi3TWqPwXhUIKzBtq1o44iynSkBkEZWAYbqtMf5pudLRJkqpbcR21AZf9m2CgqkvG4iyhQxKXo1PhWISa01UuTZ60MFWVL5QWgnld6AbbDtTPnz32pNsS7aQAXhop0pRXAtR5TXs9oPdQKf8V16PNCLKI9beQmpsarAZ7Q/zytKtVVtK8lBrSmWmRuE/VBnRoTeixwroRLnMQsrZU7aPO8F5VVUJ/kjpUPamXuHY83fJSrHCW2gZD168PD84feYOh8jymuKfVTnlPJionSkJPV8HnBe3yqpan9/fyxevDjWrl0r8ww1YebDMAzDMIy2wj8+DMMwDMNoK/zjwzAMwzCMtqJj73y8733vi56eHhkxsY5rVf4bNS5qV7yzQA1ORcRT7k25bfU+kyNRa1PjI+q6ttE9ke5c1O8Ga1tCJWrLdSn3vS3VXtaZOVbOpYq8V2cdqTVE172IcpRA2r+OPq/6pCJTZj11S41VRe3kvYsqV/HBjpXgGKhxc7/ktjfn+FLRXHkPIru4bilwrLStSky2uVB3fOgCyjsHm/u1oO71cC55L6rO+sj9Uu8o23JtDzYyaBXUfTmeD7T/YN108zvqbg3vGA7WtlV25b+5PjmXzXFv2LAh5syZ4zsfhmEYhmF0HvzjwzAMwzCMtqJjI5yuWrUquru7S1E8STfRrZHlKsqIIM1Eunz//fdv+S7d3lTUzip6i+2RjqK7GV3BGJWU7RF122Pf2cbo0aPfsr26dCih3C3pmso2OH9KZqjTVgYlH7oqMppfnaRjqj3OXUSZZiX1y4iNKjKlAmlcjoeUZ0SZDqVUMNj2uG7oolclddFtvU5CwzrtcU/S5tnVVkXbrNMeJTruSa5ZtpclmMHuDc4Z5QDOMd3D6aY+2H2R2+NY2TaTQtLldLAJISPK7tGUrnjOPP3000WZ+35T2iPYhopYyrWppIsqcO0wlAG/o2hD2rZOVGvVVkQ5qjNtS7fkJUuWFGWVULUOslzC70TKoWyvOX9OLGcYhmEYRsfCPz4MwzAMw2grOlZ2efXVV6OrqysWL15cfEbKZ/z48UWZFBgp9YjyrV8VbZCU2KOPPtqyPVJPpDBJs5EyjdDRFEmVsT3SaZQoSLeTBmZ7VXITqXdKMLQHqUP2m7fCq9ojSBmSemRdbE9Ff1XeCKqtiLJ8pBJJkcKm/FOH2mZ7WQ5jpENSl6QiKcfUGR/fpbyRo8rypjvb4NwPln5l/9gPekhElNdzXRnyrdrj3JGep403tz2Ce5rt8WyokvgU+A73NNcH9xVtSZvXHRuf43lCGYtJy9j2YL2T8nM8hzk+Si1su87c5f3Ns5Dj43rZUu3lPcbvAEpibI/nV532VKToAw88sPQc9/7SpUuLsvLaUWuV/aBUxTK/I+q214RlF8MwDMMwOhb+8WEYhmEYRlvRsUHGpkyZslGyNHZV0TvZC0B5dFCOqZNsSiU04s3qTHUtX768KOdEYK3qUonNSJmSuiU9lhOLqWRHKqAU+84kYrwZz/GoBHVVY1KJ4vbbb7+iTMmAEhrfraIzVdIl2pBeKk888UTLvqo1USXNsD2uC6410rV1EkTRHgwORa+liPKcsQ2uizprQu0rroO8xzhPKuASodYdQdqZXgrZ20WNiWXameuINDftyfZId9ObI6LeXlLeUVxrlAaU/arWOW2g2uOYKLVQrlAB4vKaYBv0UOLnXHcqiSSh7JdlNnp9cC0oOVMl1+OY2B6lo2xzSph1bKjGxACXlKc4Vq6JPKa3Su4WUR4f26BsRU8xjifLqoQaX3Nv9Pf3x3333ecgY4ZhGIZhdB7848MwDMMwjLaiY71d9t9//xg6dGiJDiJ1qGgsUtwREY8//nhRJm09YcKEoszb7XXaIB1Hb44MJV/wdjRvD6s2OCbKBLyhP23atFLbK1asKMoMjsR6SUGTQlMU7cEHH1yUSQtWBV8ilcg2+AzlHNqJc6S8ZjJtr9qjFwHr4g122kzd6lY2iyhTkk899VTLcZA+ZRuKmlayAseT2+a64HOkplmXkpgI9pXUbUR53SoPGdWGCkinJAC2ldug3WhPtSZIC1PCZIAmtpfnW8krfI6UPPtHCZjjVjJUbpvrkBIO1xppdY6P79aRIjIokyopglAygfLm4TlN+TKifHZyPauzWuXXUlI25z5LHwTnVUmxbI/efTzPKR3RBlmeUvNBG1IW5LlGm/GMY520R1WuLj5HCYzruS7MfBiGYRiG0Vb4x4dhGIZhGG1Fx8ou99xzT3R1dZUoZFJJDP5Thy7P4A39rdUGaUi2QSqRbSg5RoG0KssR5ZvupPAG24aiv1X9m9IGQXqYdB8pPsommRqtExyJ3iDqpjvXhwpUVAXahO9TsuBYFf2qkOUHemuoPCyDzbuibuXnvBib0wbXFOlvenOoPEURgw9ExzZoJ5bp4VKn/qo2VPAyzvdgg9tFlL1MeGaxXuY1GWwbrJ8yS0R5jgfbBqHkFXpbMMBVbrvO+qrTBmXfwda/uW3UDVzHNijhsA1+R82bN29Qbaj6q9qYP3/+Rm04yJhhGIZhGB0L//gwDMMwDKOt6FjZZdiwYdHV1VW6Tf3AAw8UZVJBpORzgBTS56RTeQOebdx3331FmVQ2b5Hzeeb3yJQTaWu+w3w1pOEPOuigokwvBd4kVh4jmY5mG/QgOfzww4vyk08+WZRJ16qAaLQf28s5EA499NCirPIsqBvwLJPSpf1IC/LGe0TZbopuJJVOCpQ32JkrReXDqaJlVRu8kc655/jqeH3ktUbZjfOtUtOr4EsE22D92dulThuE8kShbbg+uDbzuFUgri3VRlWgL5WLhvZXeYSqPBuaqPLyoZzDfuR8Q4Npg2Og1MJzIkJ7zymvJ7ZND0DuY8oSbC/bvMrbrAlKRmyD9ao2qgJ48TnmFqPd6OWm0trnuWzVHr8LIsrrlh6PlO1pZ+WJxQCBtA37SmklQge/rNobdWDmwzAMwzCMtsI/PgzDMAzDaCs6NrfLSSedFEOHDi1JDqTcSBOR9s3DITWn8gLwGXVbV1GKpNwyHU3JR72vJA5SXfQsUf3OtCoD2dA+9CKgrESZgdStCtxE5DGwDd7+puyiAhpxXmlble8kByHielGBnwjOC21O+Y79oKdMphpJgaq+0/uH80rPKOWFQeT5rrMmaX/apk6/afPs/aDWJN9XbdDmSrrgnFaNm+C80uZcL5TDVO4m1e8M2pPeLtxLKqCa6jf3EfdwBp9Tfed65hnJeSUNT5ktezfV2UvsEz0p6FFGrzV1BuRxq77T44fzqmyjck7x+ZzLh/PK3DUq3wlB23BeuJcoa2cvQfZd2Yf24FrdnH5HlPecsnlzvWzYsCHuvPNO53YxDMMwDKPz4B8fhmEYhmG0FR0ru4wZMya6urpKN5f33nvvokz6m7RspgjVDXhSQqQbSf/x5jK9EeoGhhk9enRR5i1v3uzOuWhagRQtaWPaIwcZq/LCaYJTT3uwDb5Le5A6rLrpzL+xXlKBjz76aFGuY2f2m/R8RJkKpA2q8jS0qletD84F7RFRL6Aa7cF6KXvxtjnXeRU4T1xrXINcI0oOIGgP9o83/SPKc0bvpjoBukhHk4Imbcz9nfOHqLWngn7RHrQBA4BtSuCnSZMmFWXOGddInaOWFDc97HKfKOnS+6TOXFJ+5hnCethv5SmT66W0c8ABBxRlrg9KLXUCUtHGEREHHnhgUVZnNb3T6uRE4ZlBzz2uiQgtGxPqDKHEyrNTrY+8rpXn08SJE1v2ifuQ9lBebvw8S9nc+7QBy826BgYGYvXq1ZZdDMMwDMPoPPjHh2EYhmEYbUXHyi6TJ0+O7u5umaKbFB9vJecgY7z5zPcpz9AErIveICzzxrCqM4P0oaqXXgMct6qXt7TzuEmVkT4cbL0s77PPPkWZ9GmuR9Htql7lhcRb3XW8A6rqpQ0odZHeZBusV+V/IU2d61VeH6Q32Qbpb9ZDCl/VWVUvvV0oC5EKV4HTVNAirq3cL1WvCqzEermGGQxJpZ/flHq533g2kApX6epzW/Q0YNAvyjn0DqhTL+UmrgnKLLleBlFTXhw8fyhlkzonPc/n8/5W+5h7g/Uy8J+yAT+nPdjXiLLsNth6WaZcy3OG0mT24qvTX64DrinuadbDvcN6stRFKZVzP9h6+TklSL5Lu+ZxvFVAtv7+/njkkUcsuxiGYRiG0Xnwjw/DMAzDMNoK//gwDMMwDKOt6NjEcjvttFP09PSU9EwmGqMWXuWiRO1s8uTJRZn6Ju9wMMqcug5D/Y33ILJWRpda3g1hH9mGcpGk5k1Nsq5rFu8+sEx3LOrfyn2R9zHojsW+RpST2jFioAL1Ybq9UffnHRPeXahy82W9U6dOLcq8D0AtXdXLuaNmyrnPf+NareMOyrsMyj2Wc1TlDsp+8I4D7cm62Iaqlxp31nJ5B4Sug1VRE1vVS72c+5N7supelXINVvd0eA9I1cu5z+uc9waY7KuOPZUrMG3w+OOPt+x3Vb3sL91/eZfkoYceKsrKPZz94xkcUXa1ZntM/Fk1T03wXgkTqakknFX1sr+8J8LzjvPCvtKWVWeqqpfzpKK2sl5+Thdcrid+d0REPPjggy3fJ2hPuvFz7pcsWVKUH3nkkZb9y+PmPUiV1LN5z6mO+3QTZj4MwzAMw2gr/OPDMAzDMIy2omNdbffdd9/o6uoq0dGkf0hR0ZUuu6Qp2p/0EKlORrijixHpNFLFSk6JKFNz7DspeeXmyHGTLif1x+dJp0WUqVKVQIg0Mu1JuYJUpUoylxMwUZLhOFR0Q/ZPRS9Vbr45siv7yDY4Zyqi7ZNPPtmyHwTHk92bSR2TJldUrALXDSlTtW5yf5Xswv5RcqgT7ZTjpv0iyjIM+0gXRtU/zhelMUottF/VcaWivLJMiZXSjtp7XDe0X36OsouS2dg/utFyvkm3V0X75fzThuwjbaXkRWVPuhHniLZcL5wn1T/2nRFKCdW/DNqW+08laKuT2I/2owSWoydzHaoEk2r/0IYcH88vtV9yvewXy5TFuXbUdxTLjEib5VLlNk27NesaGBiIZ5991q62hmEYhmF0HvzjwzAMwzCMtqJjZZf3v//90dPTU6K3lPTBIeTbtqS7GJGNFDvfUTeUGSmScgU9RrK3i6qLdBUTA5F6VDfmFcWXkwHxJjLpOEXBKUqYlDVtVtfrgJIWKU16RZBuVLQgx8qEW9kLgHKCWtqkFbk+1K3wuluEFKjyLMlrpAlF1dNjgfXQAypCJ/+i3Ujvcy9wDSs5jP3LEU7pTULZpU6SQMpClNa4ZrkXqry6aAOOlRQ7663jjaP2fURZIuF+U/tK7QWea8pmeQ1yrJQy+L4aq5KY6GlGypyJH/OY1Fi5F7heOJdKuiDy5zwvKZtx3anzkjaklwn3Pc+4vD5UklKuEa47lYhTgVJXljZ5rrGszkuVOI/zTTmlKgkkx8rvBu77xx57rGj3qaeesuxiGIZhGEbnwT8+DMMwDMNoKzpWdhk/fnx0dXWVKB/SOKT1mFwnez8QKoEcKS5SgaSlFH3HG9c56JQKZKZoQY714IMPLsqkdEmRq3oiyrQbbzITDI6jEnRxrKTv2N6KFStk24pOJX1HupFUpaL9SRGS+sv1MoiOggruxXoy7azA/lJOIC3OYEEKtB/roUS0bNmy0jvKq0v1iXNJ+UBJYOxTplNpK5VUTdXF/UPpkGtK1ROhx0f6mxIVZQm1p7mv3v3udxdlyiObUhf3tPIoU/JUHjdtxX5kGfKtQC8Rnld16+H+4TqvE3BPeUCxnnwucd3XCbTGPU0plLKqOvuyhM91TqlFSTVqzdNjkdcHmDSPaz7XpQKqMcgYJTfWRZuxbZ7BHGeElo8o/TW9K/v7+2Pp0qVbXnaZPXt2HHXUUbHrrrvGyJEj4+STT97IxfONN96IWbNmxZ577hm77LJLnHrqqaWNZhiGYRjGOxuD+vExd+7cmDVrVsyfPz9uueWWePPNN+MDH/hAiXk499xz45e//GVcf/31MXfu3Fi1alWccsopW7zjhmEYhmG8PbFZsssLL7wQI0eOjLlz58Z73/veWLt2bey9995x3XXXxV/8xV9ExB9vwU6aNCnuuuuuOPbYY9+yzqbscsABB0R3d3dJEiEtxZv/OfcAQQpJUVykkkhvkVYihUQPkKVLl7Z8PkIHuSJVxmfo7UIqkNQm6THKPJldIm1K+6jb9+wff0wyYA+fYZ9oj4gyJUlpR3mvUDJQQYhI3SrvpPw33oznOGhnzhnrohyg8uxkzxUlMfG2vwoKRJuzHxwP7Zo9TpTHiqKmaWclDxKtAgo1QQmHY6J92CfWRamEXge0B6WgTIVzfEpKJVWvPOO43xhoUEmCEXq+lRcax8TzS8k03Lec+4jymVUnzxVtQ/mu6aWQQdtkm9NTh1IG16CSVNgn9oNzzDVYdaZyTNyjtAfPQRVkTAVxzAHUlBTF84teNFzbSo5UdqJEF1EOTEb78B1lZwaeo8RXJ0BcRHntce5pj+b36cDAQKxcuXLre7s0J7l5oC5cuDDefPPNmDlzZvHMxIkTY9y4cXHXXXe1rGPdunXR19dX+s8wDMMwjO0Xm/zjY2BgIM4555w4/vjjY8qUKRHxx19kw4YNK/2fQ8Qff02q/6OaPXt2jBgxovgvx6swDMMwDGP7wibLLp/97Gfj5ptvjnnz5hVU03XXXRdnnXXWRrH5jz766PjTP/3TuPTSSzeqZ926daXn+/r6YuzYsTFt2rTo7u4u3eAlDckb3qT9M5WkqFFSVPyxRGqOtBLpPxVgK+d+oBeB6gfpP1LpbE/R11Xpi0mVqXEoyo/1kmYjnaly0uR/c0ycM5XfQ9GIlM9oj7zW1Pjq9ENJFBwr+5HXWh25gzQ+qX7S8PxcBf3KHky8ca/kLeWxQpspmrpqbNw/DIBEmY5QY6JtVSClDJXzQr2v1hpBOZI0dVW+DfW+8nJTQeFoG54nWR7hGlH94LzwnOEc18lTlaVQepHlvE6t6uK80ObsB8fDZ7Kkzn1MKaNOP5SMTglenR+5L3yfUiPtpvYx21AB2KqCyrFeSkYqwCLnWO0lSqd5rbFtSkY8h5t7d8OGDXHvvffWkl1a+xi9Bc4+++y46aab4o477ihpXKNHj47169fHK6+8Ulr4a9as2Sg6YBPDhw8v/cAwDMMwDGP7xqBkl0ajEWeffXbccMMNceutt26U7XDatGkxdOjQmDNnTvHZkiVL4umnn44ZM2ZsmR4bhmEYhvG2xqBkl8997nNx3XXXxS9+8YtS8J0RI0YUN9U/+9nPxr/927/Fj370o9htt93ib/7mbyIi4re//W2tNpreLkcddVT09PTI2PW8Oc5Lqsz7EVH2TuDtY1LvvDnOW9ekm3gLWqWkzrehFc1Nqowgjai8MMgokX4j5ZZBGjMH5WqCtD8pWo6P3ghksvhuhE5VrsbBenljXtmANG6mZTnfammTaeMdI7bNG+xqDJSkIsrriJKMojopF7JeRdWr4HYR5XnlOJQ8RtvSM0SlNlf7MKIc0Iu0LNekGgf3AtcUvaSIKumDMgW9o1ReDe5Pnhvsd90L8NzfXNvshwL3Mb0cSH9X5RnhfqWHF88TUuQq+JW6b8c1UQXOJSU0BmdTa4o2YD/yuUbpV3kEMnAax8e5UPIb9xHPpYiyHbiv2AZlFH7fcA0qmYd7Ou9vdUaqfcnvK56RtCfPGcouuW2eZfwbx8QgY4888siWl12uvvrqiIh43/veV/r8hz/8YfyH//AfIiLiO9/5TnR1dcWpp54a69atixNPPDG+973vDaYZwzAMwzC2Ywzqx0cdkmSHHXaIq666Kq666qpN7pRhGIZhGNsvOja3y7HHHhs9PT0laoe3dknvUoKhN0FE+Ra0ut1O6pY3jknBMYiXSnudcwSQ/iPFVed9UuGkvUjZqTj9EWXqklSlosL5Pt+t03ZeQqS/KUVRnlHUL2lnPs+y8sKIKNtcBXWqk3qa73Lc6t3cNtcaaWfVd7ZNSpdUb91xk4Im9a7kH8pHXINsu8rjhPIRKV5F17MueoZQrlCyS+4HJTiWGfxPeYPwebbNvDLqzNiU97lPOG7WQw859W5E+Zyj7FXnfcoBlLoYRK2qbb7PdUeZSHk0qXeZpqPuuPk+A8mp9PBcmzyj6rZNuZYeQKptlUuMThpcK+rdqvc5Zzw3KI9wbVLmeeKJJ4oyz92q6wNsg/1tttHf3x8PP/zw1g8yZhiGYRiGMVj4x4dhGIZhGG1Fx8ou06ZNi56enpL3CWkv3kLnDeqcA4EyCmkg5YnC53lzmWYijU66PKdBZhvKQ4aUGD/PeUNatcHcLjnYFqUCFYCHFCbpOEoUKiAO6XlKFBFljyHSmJSSOH98XuV8IU1Nu1JSiijLR7ShSmfe6sZ2RJlSZL/pTZPT2NNWKl01x53fb4I31dk/5UUUUbYJaXjKZkrC5ByRyqZkw/WVJT51y5525hzz5r5am1znpHpz2yq/EedPefzQw0Xlf2F7uW1KCNxvKqcT1xfXBM8cjpt2YlsR5T3H/nKsPC+5JpQHjqLtKQtFlM8/5W3BM1UF0+PztE2V14eS8uhpQ3vQBpQT1Bms1k1EeQ+onELsL9cE6+UzXB88X/lubo9zybVD8Oxjexw3zzLOVz5baAeeWVzDzXf6+/tjyZIlll0MwzAMw+g8+MeHYRiGYRhtxSaFV28HBgYGor+/v0RhkkIjPUkqKHsBqPwSpNAU9a4CTZFmU143EWW6UVFQpMFIGytqjjKPSmEdUabEVH4W5UGiUkwrD4Ts5cPb2JQKVHAcUsr0MCIVTnqR75J2jyiPj3NMLwDahlF6efubEgrHTaqeOR0iyvOv8v9wfLypzjGRDmX/uLazLMf3KbXQE4V0Ofuncvyo/ZbHpqQJ7h9+znnh+uLcqb2Xwb9l2bMJ2o2yksqFodoj3R1RLxAa5ULaTaWN5/nD/D0534YKMkZPNbbHvrI9lhmgjHshSx20JyVJzlmdnDZc25SClISV3+E64tnJd2hP1Z4KsJgDVnK9sG115nCv50CMTagcMdlrk2uKz3HNs+/8jqJ3JT9XQRyztxDf4T7mc81zoyrfWIaZD8MwDMMw2gr/+DAMwzAMo63wjw/DMAzDMNqKjnW1nT59evT09JS0emrQvAPAOweM2BdR1rDpykS9imVqqXQP5D0I6pl008qgaVlmf+lORw2OOhtdHqmpKXfhCB21lf1ge4zYp+4+UEOmdp7vPlD3pG15/4B2VnPJd5nIcPHixUVZ3emJKN+P4f0Kdf+AY6KGrNxMc+Iprk/OGXVn5epJTZ/1KD2fazBCJ7qi6znnkrq6crtl/x566KGW/YsorzXakPcMuJ55N4A6PO8EcQxcEzkpI59TY+L7Kiomx0QbHHLIIUX54YcfDgX2g3esqMnz/hP7xzXBOxSPPPJIy/5FlNcU7xbQ5rw3xvXBfcEIm8rFOJ8tXJPc++wj+8cxcZ1zTah98eijj5baZh9VokrOH+9a8B4Fx8C7NQ8++GDL53N7vNembK7uUilXYt4dUVF5I8q25fpiXSpaKu/X8S4PP893PuiiTDtzPTft0d/fHw8++KBdbQ3DMAzD6Dz4x4dhGIZhGG1Fx7rarl+/Pvr7+0tSCd3yJk2aVJRJKzGhVISO4kaKkPQ5KXa2raJRki6ky25ug++rz0nN8XPSk+wrqbnseqlciemaSiqWrmeUm+heyOdJt+ZERKSBOSa6orJPHBNtThqSFDTpUD6f+0KphdQ7bUDKlDbgWI844oiivHDhwqKc1wTHeuSRRxblRYsWFWXS2aTxDz/88JbPk2KlBDllypRS25SiOFZKWirKK9cRx0D6VckE+TkV8bIqQdtb1cO5yDIn54/jVu6gKuEc9wufJ+2fx8D3KYnxzGHf+Tz3CCU6SkG0OfsUUabMaQPajZIb+8FEalxfe++9d1FWEZ0jyvuVe0m9w/5RFqc9aQPOEV2Vq95R4+aYaDN+F/DMoJ3zuUbZkuct7cF+0B5cj3yGZyI/z21z3as2VARwSrSUvrnf1BkQEXHUUUcVZbobs66mO2+dfV60WftJwzAMwzCMLQD/+DAMwzAMo63oWG+XiRMnRnd3t7yJryJIMppbRJkqo2xDupE3fUlHk54kxURQPuAt99xfgreSeVtZPU8bkL7j1OUEXaTtSEMqLx/lLUGaVEUezLejeZueNDJtToqc0g5v3ysqVUW3reo7qXC2rbwA2G8+TztnzyrKIuw71w4lLdqN4yCFT6qY/ciJBOmtxL6r6Jmky5988slN7nfuO70AVJRLzt9BBx1UlEnP83nlrZLfYd9JU3NNcL9y/5CeV95hWValVwZtqM4sevaofmcZUbXN+edc0ruDfef8UX7jmqWdOUc50iflTI6Pe1Sd1ewHJQ7KuJz7fJ7T640259pmG+w7x8o54vdC1TpXa1hF1uXzXIOcY7U22Y/cX4LvUAqaOXNmUaYNuVcpzVDK5lkZUZaV2N9WHkb2djEMwzAMo2PhHx+GYRiGYbQVHSu7TJkyJbq7u0s0EakhUkmkhXLAK8oP9EDgrV1SrkzKRdOwbZV8i3Rk7hflHNLqpPEpOeyxxx5FmTKN8rrJtBzpV/U+x60CiJE6V/Rplh9U0qUcKKkJUnl8l7Sd8uAg3RpRltBIQ5JCpW35OfvKeWGdXE85oR7/zfkgvck1RYmCVKcKZFaV3I1rkjQy1x3rJfWr1jzboDRG+S2iTC8rDxcVcIlBw5QEybnIx5XyFqMNuF641tg2vQ5Yjwr8l/tLW7FermflwcZ1TttybWYKW+0HrlWeWayX4+MzXGs8P7IUxH5l+a8JZTfuBX6uvEHyOlcSFevlvmc/OFbWyzOEY8uJQrl/WC/PDc6TOte4DlQAwtw254ayGb/HaA/OnwpGmWWlJrKnDfcc36EnUvM7ZsOGDXHPPfdYdjEMwzAMo/PgHx+GYRiGYbQVHRtkbMOGDdFoNEr0Pmk6SgmkqBi8J6JMGVGqIW2mPEBYJp3JdwnSW7mPpMpIu5HKoxxA2oz0Lqku2iZTXOyLCnTEfpAWZPAegv1gv0nb57qUREJanLfQlSTCIGN8hvMSUb4BXydXDvtOGYTrhvkeKFXl3C4cB//GG/f0oKKkxfVB+U7lxsmSD+vifqB9GESNEt9hhx1WlJl7g+uDngxcmxHlG/c5KFQTKn8P7ZkDBDbBgGo5vwrniXlYaGf2jzak/ED7UR6cOHFiUX7ggQdKbSuPLa5B2pA5RDhHfJ724H5hvqWI8nqmlx3nlWcAPblIqzMYGNcgbZa9HzgOyh0cE88+FfyNcgXXjQr6FVE+w9lHnjmUL9gezwMlhfJcyzIb2+DcsE8c07Rp04oy1xdlap4TfDfLybxOwHWrgp0pbyPaQ11doG0iymcIJUXW25wLBxkzDMMwDKNj4R8fhmEYhmG0FR3r7TJ+/Pjo6uoqUXnKU4NUWQ7GQwqPtDopI9KsSlpQt7FV6vqIMnXGm8iUS5Q3D2+nMzgR5RV6MjAdeYS+5U0qkeNmXQweo2QolSI6omwfUnakX1VqbJW3QD2foZazomJVLpI69eRn1JpSEhifYV0qD5CqJ2JjmrYJUsXKa0TdvieqnlG34VVgJLXmOaY6z+R/8zm+zzLXo3qmTp/q1lXnGeXxozxoIsrnHG3O5+o8U6e9HERLzbFaB8qbhJIPpVSeXdnmfF95OXKdK+8Mfq/wXc59/i457rjjWr7DPlKSpMRHeZdnON9V8lRE2Vb8HuN5RDurIGiU6OgpowJtRujvDH7e/J4dGBiIZ5991t4uhmEYhmF0HvzjwzAMwzCMtqJjvV1Gjx4dPT09JTmBNBYpJlJamZYldUYvApWWnZIDbyiTllL0d5WCRYqK8ozKu8J6SRGyTEqdt+0jytQl+8t+kDrk+Pg5vYdUWvVMT1IW4Tzl4DVN0OacY3o2KC+KnAOBtCLtSRtw7tWtd46B9uBN/+zlQ3qTlDDHRFqWFKa6iU95S+W6iSjfeieVq4JqcW3TE4L1cJ1y7rOcQjuzj/ycN+u5VmlneiHRzkqGiiivPXoL0ObcJ5QzueYp4y5evLgoH3744UU5yw9sm/PEtUoKm3uBz5P+pm2q8spw/nkm8NzgXue4OVaVT4RzzP0ZUZ5LJRVwrFxH3DNsg3PMvcD1FFGeM+XtwjIlCu4retGooHe5bXocZW+zJmh/tsd1rtLX02Y5yBi/o5TNud84Z+w39wj7Sk+4vL+5jthHjmPq1KlFncxHVQUzH4ZhGIZhtBX+8WEYhmEYRlvRsbLLG2+8Ed3d3SWqk3Qh6SZS3DkwDKk5BtEh/cd05KTYSU3To4P9IGWaQXqfdJeizdSNe4Lv8vlMERKks5lXhhS2ut1O27INSig57bXKJUNKn+3RBrS/yrNAujAHGSONSYmD64AUJm3AvpK2JI3LcWeZjXajrUi9q5w7vBnOPqkAcTkQkLIV6+U7rJc36Wkbtsd1kL1dFL2scqTwc5XPRXmEZc8q5cHDNcmxKu8mjoHj43xlqp12JlXN57julCzL9pRXEWnxiHKgKhX0kGuV88JzTXmlKMkmojxu5QnBetk/9kPZjIHSssx26KGHFmWVJ4kyD23As4JrnucPvxey1KvmT9mQZxz7wTnmOlV7J6K8Tzgf/F6ZNGlSUeaYWK/K5aOuG0SU50Z56zX75CBjhmEYhmF0LPzjwzAMwzCMtqJjZZfhw4dHT09PibZUt/irAiApzwZFi5OiIu3GW+h8l5RiviXM9++///6WbZCmUkGBCNLifDe3raQTRS8riYmSA/tNyi1T4fvss09RVumcSWertPEqxTe9M3JAHPZReWGQwqZ3DN9lvaSW2b9Md3OsKo0754neV/vtt19RJpXK4EtcH9nm7C+D0vHmOcdKWYNUKqUx9pttZ4mPNLyieFU+CtbL+aanE/OrMA9KRHlN1cnvQSqc/aA9Sa+zvZxnhOPmWlB7RgUwZD9Uuvs83/RgoH2UbMl+0IONkizHoMoRZds+/vjjRXny5MlFWXlVcN0x5xHXLyWlfA7yHKCdVaBJ5pDivHIM7BPP1ywnqyBeau55BvBzyhjc3zw3c9s8K/hOlXdOq/b4DL83lVQbUV4jKt9Tc00NJmapmQ/DMAzDMNoK//gwDMMwDKOt6FjZ5dVXX43u7m6ZD4RUKsukt/I7BGmmOre0WS/bo4xBqjH/jV4m9JDhLWPKNKRiSYOx36QLs5cP6UmVUlkFe+KNb1LN/Jw3x7PnBeli2lZRe3yGtDPpYSVPUa6IKM8T6VsV3Ijj5pjYHmnOqqBpnD/OB+eJVCXpTK4D3twnjc76s8ym5Ad6crEfbIPrMecQaYK24dqMKMs57C8DfWUauQnKCdx7tB/LzWBGTVAupA25BilRqLxDKg8T+0Sb5b7n4IZN8AxRuUy4D2knnl1Z+jjooIOKMiUASpKciyzbNMH1zzOKayW3rfLSqECAHBPX6WOPPVaUuS+qpGwlp/FM4DscN/eb8hrk5/lMpTTEtc3+8ozjmGhPyot8/rnnnivKOaCdmj/udeUZx2e4d1XgwBy4kd6d/I5rFWTP3i6GYRiGYXQs/OPDMAzDMIy2wj8+DMMwDMNoKzr2zsdrr70WXV1dMhootUblUhmh9VtqvNRWqZtRK6Oup9wD830Taq4qmh/1NZWwjqD2R407R19U7nTU56nPqURLy5YtK8rUyKl/5iiE7AvvgyjdkvbnmHhPg89Qi811UpOkbbleqLNyfagxsFyVNIl1cV1wHIxOSDdA5SrI9cyx5kRjvPvAOVbRNlkv+8H+Kddc6skROiEW78So5Hpc81x3vH+g3HQjyuuCz/Fz9ok2VIn2aDPex8hu3by/oKKzqgSP7BPXPNcv91W2eb4L0QqcF46bdzB434HnFd0+mdwzomwf5brMZ7gX1BpUSQjzuHnGco45Fzz71B032lm542ZwnjjfKrko7/jwDhLvgnDdqeivEWXb0m7KjZZ2rkrU1+pz3keKKI9V3bX0nQ/DMAzDMDoe/vFhGIZhGEZb0bGyy7Bhw6K7u7tE0TIyJWkp0lg50RvpPLpeqiRgKtomKSZSXaTKMl2lXMFYL6lVJTGxf3xXUbQRZWmC9CupY/aXdmNfSZeTOlQ0aa6X/eBzyk2YdHROGtcEqT26p0WUaWQVDZZUsXIDJNXMdcOxZVdb5QbNNkgJU5KiPdX4uNYyHU03R9Ks3D/sH91GSf1yrHyX1DSluNwe15pyS1b24NxxDbIe7ouIMh3N9cn1rNrgXmKZa57zQto9omw37lH2ie9wnXP+aBvanEnUsszG9cVzg2cC55tQsnEdKSGiLOHwHY6P65Y2ZP+4v9kG91geA9c5bUD7K5lZuaIqmSyPm+tWzSXnQkWK5nzzedojr3M+R5vUiS7McavnOW7aLz/Xyr024v/Nd+53Fcx8GIZhGIbRVvjHh2EYhmEYbUXHyi6jR4+Onp4eSbPVkUciytS2ujVPmo9lShmMxqduGOeodOyL8rRh8iFS4SrhE+2hokNmcBzsI+k49om2VVFe+W6OBLho0aKizCRnfIfSB9smTa2SnJHuzlEnczTGJhRlStuQ9mT/KPFRtqJnSEQ5uiTrVbf62Q+uU35OKYh9ytF0KRPx/UmTJhVlrn8lO9LjgSDdevDBB5f+prySmJTtgAMOKMqkhFXSQxWxMktxjL7IuacNuHf5Pj28VqxY0bIe9iPbXHmQMKos1wvnnu0xIRslZK7/LG1yXah9zGdUckOuG3qDKG+7iLJ0RVvRPpQ8VXJQnn3sa1V0VSU/sD3uPbbBNcG9xL2gkn5GlOeb9lSyC+vl/CkPNO6XLKuqM4TPUWLiOapkUdpGnX0R5fXCcVMibLbd39+/0bmoYObDMAzDMIy2wj8+DMMwDMNoKzpWdlm7dm10d3eXqC+VVIhyQKareHNdSTik70hRkfacMGFCy/bUzfYM5XmhEhmRwidlRwqMFHmmytSNalJ2pLzpHaC8bljmu1l24fsqUBvnkn1Vif1oP9qDlH9EmS4mra7oaJWAjxSykkHyfJM2pYcF7cF3lEeB8u4gnZmDqzEoFJ8jrc66SLOSpua7HAPlgHvvvbfUtkpsxjWi6G8VDEkl/soeCNy76oa/StSnAl4RVUnOuEa4brmmVKAvtkdpRgW5yvOtAlJxLumVRC8mvkuJiGuF7eUzVXnoqXNDBZ5ivXyeNstyMs8K5bnEM5n7gv3gemTbPFty25QOKXEw8KDyZuO+5xlCCVkFHYwon89ckzz/+J1GG3BPc09yrXEtcx9FaM8zjrt5PrctyNi3vvWtGDJkSJxzzjnFZ2+88UbMmjUr9txzz9hll13i1FNP3SgyoGEYhmEY71xs8o+Pe+65J77//e+XfNEjIs4999z45S9/Gddff33MnTs3Vq1aFaeccspmd9QwDMMwjO0DmyS7/P73v48zzjgjfvCDH8Q3vvGN4vO1a9fGP/7jP8Z1110X73//+yMi4oc//GFMmjQp5s+fH8cee2ztNl599dXo7u7eSE5ogpSbChQVUabp1I1h0k+k7/g5aSwVWCdTpqTEFD3JPqkAQSoIF2Wk7GlDeow2VEG4CNZF+pX94Bhyvg22x3doc3rzkM5kv0kvkkYnlZqDEFEe4N9IY9LTgKwc6UbS1MpTKduPXgvqNjwpzMcff7woU66gbeoEVYoo06mcM3pvKXmStCzXLKXG+++/v+XnERH33XdfUebeINWs1gTb496lhMXxUEaNKFPVzKWhZE62Qdvyc5WbiBR5hM6TxLGSLuccEVwTrLMqqBzng/VyL9HTjGsq27AJzhf3Qs4bxb5QYlISGueCa5B7kjZj248++mipbfZdBY/j3uV+5Vxy7tkn7hc+n//Ns1pJpjxHKfcpiYn9yOe5yl/GMvvB70Ql5eV5bYLnTER5vyqvqWY/lHzZCpvEfMyaNSs+9KEPxcyZM0ufL1y4MN58883S5xMnToxx48bFXXfd1bKudevWRV9fX+k/wzAMwzC2Xwya+fjJT34SixYtinvuuWejv61evTqGDRu20UW8UaNGbfR/DU3Mnj07vva1rw22G4ZhGIZhvE0xqB8fK1eujC984Qtxyy23bOThsKm44IIL4rzzziv+3dfXF2PHjo3e3t7o6ekp0VXqlnXOa0KQ8lMBnvLt3iZII5JqJI1VFcte5UogNUXajX2ilMF+sx4VUCu3oW6eqxwPKrgRn1eeLxE6EBBpOo5VeRgpKpw2y3kISMOzLq4RSjNk2jjHpEBpS44hU+EcK+lb2oprh1KLokaVVJJvlfNvXC8qT8+SJUta9u9973tfUZ43b15R/upXv1qUKRdFRMydO7co056UZ+ghQw8LlSuH8gHnUQVBi9DBm7heWBfzj1DK47uUfPI5QcmOa0r1kWtHBd7iOmB7XAcREQcddFBRpmxGGYTv07uJ7algW1xflIIidH4WngPsr/Iq4llG23AdZC8f9pHrXI2P9lRnsDpfWU9E+QyhfWhn5TnJ8dE2So7P5wHXMMfK9Zxt1YQaq5K1s3Si+ss+Nu2x1bxdFi5cGM8//3wceeSR0dPTEz09PTF37ty44ooroqenJ0aNGhXr16/fyP1xzZo1JR2OGD58eOy2226l/wzDMAzD2H4xKObjhBNOiIceeqj02VlnnRUTJ06Mr3zlKzF27NgYOnRozJkzJ0499dSI+OP/YT399NMxY8aMLddrwzAMwzDethjUj49dd901pkyZUvps5513jj333LP4/K//+q/jvPPOiz322CN22223+Ju/+ZuYMWPGoDxdIv5Ir/X09JRoQZV2nLRQpqtISynpROUYUCmOSS+S5cmeOUpOULfNSX2xPZX6mze2c8ptRdHzc1J5vOHM/qmU7nw3U22kTUkPk8ZU1Cjr5eccn5JyInQQtZwboxU4R0peUV5WEeVx8yY+1wjr5d0oekUomUx5SUWU54/riFQx5RJ+TpmAtD3bpl3vuOOOUtscB2UwSmCkprlPaHPOMeuklJNlTu4Btb4Uva/sT9tWSR8E7ak84NgGc9LwPpySofIe47ifeOKJoqxkEM4fy0peJIVfJXVxv3I/cD1S+lY5YzivSpLNfeFY1XeAOp/5PMdKCSt7fXBuuFbZHuvlHDOAG+uh/Yh8trANFZSO64Vj5fiUDEXZkN+HuS71HdX8fDCyyxaPcPqd73wnurq64tRTT41169bFiSeeGN/73ve2dDOGYRiGYbxNsdk/Pm6//fbSv3fYYYe46qqr4qqrrtrcqg3DMAzD2A4xpKGu2W8j9PX1xYgRI+K9731v9PT0SDqZn5OOzhdWSU8yCBFpJtLqKpU96dpWt3xbtc3nSIHytjjpNdKTKhATqWLSb5m+o63YL0U3km5nam2CtDHpctKLEeW5IcVIOk9R7KS5SbfTfoqijSiPdeXKlUWZkXjZDzXf7DflCgYSy/Qk5SDahHQ27c81qNJQUz7gOme+koiIp556qihzXmlD5vog9c56jzrqqKJMO9M75rTTTiu1/bOf/axlXWyb8zp9+vSizP3DNUjJhtIfbRZR3huknelRwz3Gva4km4cffrgov/e97y3KeY6UzMDL9XyHMoHysKDkybXCPEUR5b1IKYJzz2fYV0rgDObGM4T7LZ8tKnAg17mS+DiXrIdzoaSZiPK5ozxFVA4dFRyN5x3XUA4iqM5bJclzbXN8nAvOMfdLdtDg/uG5Q0mE55c612hzrhvaIEubXLe0AaXl5jsbNmyI22+/PdauXfuWziPOamsYhmEYRlvhHx+GYRiGYbQVW/zC6ZbCyy+/HN3d3TKwGOkq0luZMlIZdZcvX16USdPVSQVNipVyTI5vwrpapR/OZZUHghQfb00TOQAS6V4VLE3JWKQ2KRHRHizzmdweaT7SjRwTqVHS38o7QHk95bpIN1I24DMcN2lCziXbI22cb8OrYHf0UOK6pZcC16DKrUOqOOd+YL2UF5V0SIlo2rRpRfmxxx4rypQ4vvjFLxblb33rW6W2SRFz7ai8Exw3g4lxfOw3382eFyrvBOeJn6vASmxD5QHKnheKPqdtVZA+tsdzjeuOY6MUlN/hnmHbKjgUKXlS/Zwvyj/Z84L24d+4DmhbJaNw73Iv8WzIgb5UkCzKJWoNqmBgtAHHloPpKdmMfVeBJTkXKjga93TOWcUzlt8ZlM2UZFcnNwvnMXvScV+ybe6xrR5kzDAMwzAMY3PhHx+GYRiGYbQVHSu77LjjjtHT01Oi3VSQK9JYOSsuaWsVl57vk5Yija7yc7Bc1bbKvaECgKmcHqTyVLCzDPZdpXqv8wxpQXXjPaJM57Eu5WlDmo/SB+eY/VCeR7kvio7eb7/9Wo5JBfRSayh72nCs7Af7S9mMHgysV+W5IOVJujaifAOeN9dVvhP2lWnLSe/yNjvzvGQanjQ3ccIJJxRlyo6U0Eid02Yq31IO5KeCCNKDiuuOa0LlfFF5hLKky/2nJDe+w3WkAhDSBgw+lqULzg3BcVMOoA2UDMXxcF9kGp4S5rvf/e6izPXFNcx6eUaquVdSYYT2NOScKZmnKgdYE1zbeb7ZNu3D55R8wXfZD55L6rzL4Pj4nFrDtBP7rbwJs9TFung+tAo0adnFMAzDMIyOhX98GIZhGIbRVvjHh2EYhmEYbUXH3vloakcqIiS1K2qN2R2OGhV18vxcE9TjqH3xLgLrpCaZ9cmsnTWhkjxRg1PRTtmGihIaUdb/lJZIG7If1OfV3ROlIUeU7cz+Uqfm+KiH0maTJk0qykuXLi3K48ePL8rZ3ZV9ZJRS2paa6dixY4sy71fQ5tTqiay7894N7yzQZZjzxPnj/Q/2lbZhnblP/Bv7fsMNNxRlarcHH3xwUeYcHXbYYUWZCeROPvnkopwju9LF7+abby7KdKml+y91bkae5dyxTtqJY4goR0LlOufcM/rrPvvs07IftJ+KTJz3M+eAf+MZQldiujEffvjhb9k291F2Md5///2LMl0heedA3U9S7pLsK23O+iOilFyUc3zEEUcUZXWfhpE+lXsz70Xl+2QEz3DeBeI+5PpQrr3qPM93RFTkZoZzyHexmuBYVdTVqhAC6k6MAsfBtcl1wL3AseUzVbkYt7pnk78LqmDmwzAMwzCMtsI/PgzDMAzDaCs6NrHckUceGd3d3SXKiBQoI3iSJiJdG1GmqEh3kQqkpEIaUkURVFE0c6RPUoF8h3Qv6yXtSSqPFB/pUNZJOjOiTOUyUh/pWvaD46CLH6UuSi1cNplq43zwfcoPpENJBbJeShSkYmnngw46qNQ232cUWyZiIzVNSpEUKG3DNUU6lInrIsoyFuUESg6UdrjuSIHyGeV2mN0f+Q7tTIqddDTXHalUJs6j/VRiv4iy3SgZqQSApOfZV46BchjXU3ZvZt+VKz7XDiWjY445pihzjgjaOSe1Uy7GSiblmqK0ybVJqp9yUU40xnXBfclxU17kWcQ+0W2W++L9739/Uaa0EqFdtkeOHNmybSXdUs6h+3Wm/Qmea9xvdRJSsm0+z33BdZcjnHItcI5ZF9eRkqn5vUL70a45mR/f4ZnF847nFKPY8l0+o6Lp5v2t+tsq6WV/f38sW7bMieUMwzAMw+g8+MeHYRiGYRhtRcd6u7z22mvR3d1doipJc6uIozkhDykjFc2ScgeTbJFuJF3FNkgv5pvOpDoVxci+q0iHpMQ4HtJ62QNBecU8+OCDRXnChAlFmRR7VfTSVs9QAoso06+kQDNl3qqvtCftzPmmp0Zum9SoStpHGYtUP6lt0rXsh/I+iSivEc4NPRVoG8pHpDBZD9eESoqW+3XIIYcUZa5h0tn0JCIoByjvgKzUckxc8/TgmT9/flHmWFWCSM4r5557J/9NJXzk3mBETq5t2pPvUv7kOPM7pJiVxxznm/S+ipZJ+2VvF7bBMvvBtaO8cQhKmOxrlpMZQZfSBCMHsz16BakIxGpNZOpeRabm2lFyGPck9z3nm3IFz4D8nJKV+A7PIq5bjlV5cOY9xjOc3zOsi2ft9OnTi/KiRYuKMqU8QkVkjiiPlXPfSraxt4thGIZhGB0L//gwDMMwDKOt6Fhvl2nTpkVPT0/pBraifHjDuIoaJSXMz1kmta2S5PAmMp9ftmxZ6TlShirRFetasWJFy8/pycK+qmA1EWWqk7f9KX2wXtJ3pB5JC5Ku5RiyNKMki4kTJxZlUuGUGVSAJyVbVYHUKsfHOaM0wzm67777ijJlGnoLZW8X0qEqIRzXs7oxT9tSLqIslGUX5YHFtm+88caiTKmR8tvcuXOLsgqMlOVF7jnlmcI+kR6mVxDHSnmRNPqBBx5Yapt2oM1JYROklLlHeIZwLjjuTClz3bLvPGfUONgP2olnCPdLTubHf7MfHBPtoc4K5b3AszbbknPMfcyzgmuEHoQ81yglcDxKaowo24pzz7OJc0HpSgUR5H5TifkitA1ZF7+XOA7OkTpH+Xz+HqP8V+f8U8EkOQaWecZRgo8oe6dxfK2S/G3YsCEWLlxobxfDMAzDMDoP/vFhGIZhGEZb0bHeLgMDA9Hf31+i7UmNkrolxcfPI8rSiQqMRWpP3aDmM6RSVf0R2ruGdCHbI0Wu8lmQKmtFezVB6pH9IEVIeYW5JkitkeLju6QBs9eHoifpecEgY3ye9lDSRZXN+W9S0LQhKW+uA97qVwGT2Kc8boLUu8rXoRRP2ow0cF06mtQ21wGDavEGPGUXzgtpeNLrDIoVUfZMoc1pW0p8pHjpxcF+c45UAKkI7XlG+7BPKncH21CSSM4rQ6mA61Pte7ZBSZHUO23LMyB7IHB9cp7UuUY7KRmXY2XbWXbhuuVc0j4cE/tKSl9JY7RZlvjYd5ULi3tG7V0lDRPZ5irInwqixnFzH3KOlbRJr7P8N7bBeeI65+fqnOF+U1cBIiLuvPPOosy8WrR/cy+pqwqtYObDMAzDMIy2wj8+DMMwDMNoKzpadhkyZEiJGiVNxzJvJeccCJQQSEOyrIICEeqWtYp7H6HTtbMNUmKk/VkvqWne3q5qm/QkKUaWKXEw+Axtw36TAiU9z3JGnaAztIEak8pDk+WHnPOkiZyC/q2e53yT5lQeCxFlypF2riObEXy+br9ZV1UgtCZIl1Mioj15Q5+ySw5QRgmHVCy9H2hPZVvlRUZbZpvlfCvquSaU9wppe85rVSpz2pCokuNaQQUMU2sz94tgH+ltwPERtLlam3mdEzwL6wTmowcPoc6AbPPFixcXZRWkj2V+H7DMdcd+c07zdwnH9PDDD7fsI+3Bc5HzSkmEZcpFOaickhdVMEp1fYBlfocqySyifD5QNuNzTa81tS5bwcyHYRiGYRhthX98GIZhGIbRVnSs7NJoNKLRaEgZhEFweDOYHhwROrgXJQTSr6QqKUWQdlZBZRi4KULnGCB485n0FsdNOpRUGW/bV6U5J03HG9gM3MVU6sozh1CSSETZPpTE6DmjJCaOVbWt1kQVlCxB2pmUIfvEZzhHVZIS+851oG6Yk7qtY4McwId2JsVO2YbzdOyxxxZlplVnPyjdHXfccUWZQaNyvVOmTGnZJ0XxktLnXiUdTTvl/ECsl8+pYErKU4pl1qkC8eW+cKxKLlGBn9gn2oZzXxULks9xTXKt1skNovLb5LZVPh4+p1LIU8blXqBUxXqyzVVOInW208uE+5t2Yp+4V+ldmdvg/mN/ufe4dlavXt3yXfZDyfkR5fmgtMl1x3coH/E7SkljXB85YCWf4/cVx7QpsUrNfBiGYRiG0Vb4x4dhGIZhGG1Fx+Z2mTRpUnR3d5eoMpUPhAGvMuVJelPd2ma+CNZ7//33F2Wm4qZcQQotp3cnhU0KlLQ1+0uKkePmrWvWSZknB+whnccp5q1r0p7qRjTbY/9I3+UcCKRQ1ZhIMZJuJ8WnKF0VUCu3ocZB+pR95TgoV/B5lQujahzskwqCpjws1BiyHKO8wtiGmgsV6E7dsM97jGuhzjgGKxNkCppQ3ijq9r5KHc536+yL3EdS1aT9uabyWm2C9mA9VR4ItJvyDlHj4LscA8G5yPt7c8ah1iCfUZJn7jvf55pXXkx1AjeqvRpRti3zZVF+4JqitwvlHJ45fJf1V3lMqXW7zz77FGXO92233VaU61xDoFQVUZZwlGdj0+b9/f3x8MMPO7eLYRiGYRidB//4MAzDMAyjrehY2WX06NHR1dVVutlLDxd6ejA3RQ5Kw7TnKp8CPUBIu5ESo7QzZsyYokwJINPwpPzYHum/6dOnF2UVzIqUOilFUo1VwbYUjcm62D/Sp6RcaT8if67i+3OpKe8OQlHq/DzTk5wD2pPv0LYqNbbqN22ZA+ooqlndaCc4Lyp3EN8lLR6hc6RQOuF6VAGlVJlUb74Nr3KQcNxqDStZiWtbpU6PKM83549tsE98vo68wrHmcSuvHeXJorzWWK+yed4jfEf1V+V+UnZimftKSTMR5fNBzT37pORPtT7yuaY8zPLcNME5UoG+uCYoP+RAX9w/HDftqc4pnpGPPfZYUVaeKzm4oPqaZn8pj3DPMKcWPXBom6pAkZRh2EfasDm+/v7+ePLJJy27GIZhGIbRefCPD8MwDMMw2oqOlV2mTJkS3d3dJbqQVJTyGMlUj6LhCb7DgCwMWMbgSaRr6R1TdTta0fWKGiXq5Kmokl1UkCZ1Y14FxarjQZP7ogL7cF74vJojJdlkkN5kIDOOg20oOylU3cRXkooKFqUCYSlJROUjiiivW+4T1qU8L+rIDIoWz/9WAahoNzXHrIf9I53PgFARZfmJtqLcpPIZVe2fVv3I86tybPCdOpKWynXDPZI9fmirut4araC8TDh3WV5UHkNKwlGyi/L0oz2qxkDJQknCBMengm0R+TxQXkV19o8KHMjPuceqArvxfOZ+4FwwQBrrUhLa/vvvX5Tz95CSpVrl9dmwYUPce++9ll0MwzAMw+g8+MeHYRiGYRhthX98GIZhGIbRVnRsYrnhw4dHT09PKbmbikqnkiZFlHUwRoBjpDdqV+PGjSvKhx12WFFW0RqVRhuhI2nyOaWlK/1b3RHJuqy6H6A0VBW9VGm3RHYDpFsy78QQKkof54KfK00+6/B0wc73IpqgDZR+TShX2WwPFb1RRRmt4+qp+pS12zpujkoXV67E7AfXcl5DfIf2ZyJBpXOrCLoqum3e36pf2VWxCdpJ3bFS94DyHlP7ku+ouefaVBFf1X2r/G91HqnIsHXuiKi7SRF67/L+gZr7KjfaVs9U9V2dnSoCK8F7E6of+fygrZSrc50zX63BqrOF79Dll/uB4xg7dmxRVndVfvGLXxRlnif5TOUaZr9a3SGrmrsMMx+GYRiGYbQV/vFhGIZhGEZb0bGyyy677BI9PT0l+ogR3CihkNLKdLRy81L0PulGRV/zGbad6UmVpEvRhcpFUkVDJL2YqUPaQUWRVG5vatwq0VVOakephf0gDaloeI5JRWlVUSMzKDOwj3WoZhUllPRnFS1LqGib7BP7WsftNkdjVdFBCeXCqGQo1sn+kbKOKI+P0YK5RpSrIdcX62FfFd0dUbanso+Sc/gM15dKUpb3GNtu5XYYUR43KXK6iapIqRx3XmtqD6horuxTHfdfnkVZZiPdruaeUO7idaKxZumIc8O/0eWa61PNEe3JepT7dER1QssmVJgBvqu+e6rmm2EDOJfKFVz1j3N30kknFWXOY5Y2CUZ9bSUFqXADrWDmwzAMwzCMtsI/PgzDMAzDaCs6NsLpKaecEkOHDpXeAYo6r6J9lOxCmklF/CMoJSgpIkInNlNyh3qG9Q7WiyVCe1Ko2/7KA4FJiZQ0kPuiZCk1f2q+CeUJkf9WJ2mckn/q2Cnf7CYdyvbU7XaunZdeeqkoKw8J1b8IHXFRyWnKo0ZFcKX0VOXlo/rENUxPM9alPHv4DL1pIvTterUf1J4k1HxnDxq1VtWaqpJom2C/GSGyyuOE4DypfVxHYlXeOxHlc0dFKq5zNikPQiUFRehke2yvKiFfE5Qf1JrI79IOdbymiM05lzLqRJLlvFCSqpLqW9UTUT6bVLTg5jy9+eabceONNzrCqWEYhmEYnQf/+DAMwzAMo63oWG+X/v7+6OrqkrfklYdKpiMVLasCdFFS4c1q0nrKi6KKpiOdrTxqSInx3ToBbaoSyyn7KK8bUnb0UCH9VuV5wdvmtJWSsQjWq8atPJgiynPJMmlWQlG0KimgkjcidDAyBl9S3hO0oQqUVkVH8x16WSnKW0mYnHtS+MqjItdFKC8mJbmpfahknYgypaz2q5Jl+TwlRRX4KaNuMLJWz6sEZFxrXLPZ5kru4Dt1JF0VhI7P533L/U27qYBvbEN5QNUNcKYCtXEu1blLqESaRFXgRnorMaAa21ZnKr02uc45nirvJpWsU9mN+1h9X6nAcRE6KWerfmxVb5dnn302/vIv/zL23HPP2HHHHWPq1Klx7733Fn9vNBpx0UUXxZgxY2LHHXeMmTNnxrJlywbbjGEYhmEY2ykG9ePjd7/7XRx//PExdOjQuPnmm+ORRx6Jv//7v493vetdxTOXXXZZXHHFFXHNNdfEggULYuedd44TTzxRxh4wDMMwDOOdhUF5u5x//vlx5513xm9+85uWf280GtHb2xtf/OIX42//9m8j4o/BUUaNGhU/+tGP4hOf+MRbttH0djnrrLNi2LBhJRpLUTp1g9LUCQBDWontUX5QVF6m79RNa0UdqnwPdYJq5T6pvBOsN99ib0LlcFGB0jI9qQIacRxKjlF0YV1PFNbF29YqqJO6qU771Xk+okyhquBZ/JzPK0peBQarGwCJ9XJ9Klpc5aCgvEFbRpT3lZI5lUcNQduqfZvXP9/helFSEO3EvtKeysskt0070IZqDdOrjp4zlMzUOsh2pT2V50tVwCxVbxOt8nY0oSRX5bXG/zlV0gBtWbWeeGbxDFJ7Xc2fsjPrzHud/67jucTnlTxCKBk810VvMSVXqbNFjYFt5yBjSpYimm2sX78+fvrTn255b5cbb7wxpk+fHqeddlqMHDkyjjjiiPjBD35Q/H358uWxevXqmDlzZvHZiBEj4phjjom77rqrZZ3r1q2Lvr6+0n+GYRiGYWy/GNSPjyeffDKuvvrqmDBhQvz617+Oz372s/H5z38+/tt/+28REbF69eqIiBg1alTpvVGjRhV/y5g9e3aMGDGi+I/h1A3DMAzD2P4wKG+XgYGBmD59enzzm9+MiIgjjjgiFi9eHNdcc02ceeaZm9SBCy64IM4777zi3319fTF27Njo6+uLoUOHypvxdWjciPLN/zoyg6Kj1Z0VdaM894ttq5T1iiqr8nJo4uWXXy79m9QZ2yPVVufmOd9lnaRiq+7zkPpVeSfUvKg+qZvfuQ1Fv1YFKWvVpxdffLEoU37L79I+9HCpI5Wp9Okcj7pJH1G2s5In+Y6yLcH1z5wO2bupTqAv9olzzzY4X8q7jDJGrpf2VB4kKrcLwTwaSr7MbRC0B/uuvEHUXFRR9bThYDwMIso2pOeF8rDLe6xOf7kXKCuxLuVhUeXtomQe1Qbnm1KXkm6VPFXVL7at2qhzXlLqygHtlESlznblGaokJqJKZlGB9Zpj2mreLmPGjIlDDjmk9NmkSZPi6aefjoiI0aNHR0TEmjVrSs+sWbOm+FvG8OHDY7fddiv9ZxiGYRjG9otB/fg4/vjjY8mSJaXPli5dGvvtt19ERIwfPz5Gjx4dc+bMKf7e19cXCxYsiBkzZmyB7hqGYRiG8XbHoGSXc889N4477rj45je/GR/72Mfi7rvvjmuvvTauvfbaiPgjNXTOOefEN77xjZgwYUKMHz8+Lrzwwujt7Y2TTz55UB3r6uqKrq6uEs2m6L6qnBcqQJG6Da+CQJFOUjJIprFU7gH2iUyPCopFelKlHc8UsqK5Seep8amY/xwr+5dlLyWPqdvmKucIoeg+yiC5bUW/8rY42ybdyPmipwdtk+2k8gWpIFB1qGYltWQPhzp7Q8kEKs8L6WSu0yyzqOBqyiNDeUWwDdbDfuf5Vrk4lKcBy4OVYatQx0ulTsBE9lvR5fl9tqFynLBeSi1KrmPgrCxFKNlMnTlKylb2qBOMMEJ7QSnvMiW3qnVTZXPaluuW7/MMULKxkj7yea68PnlmKS83QvVDnfkR2vuO66I5DtVuKwzqx8dRRx0VN9xwQ1xwwQVx8cUXx/jx4+Pyyy+PM844o3jmy1/+crz22mvx6U9/Ol555ZV4z3veE7/61a+kvmoYhmEYxjsLgw6v/uEPfzg+/OEPy78PGTIkLr744rj44os3q2OGYRiGYWyf6NjcLt3d3dHT01MrgEvV7WhFh/JzShG81c+bxKTZVP4EeqvkPqpcIeqWNgPzkMJUN/czDU86j/1VNL6i3Vgv6T5SfFWBtzg3Kr+N8kygPZWXSfZ+qLNe6BmkbqfTTkoOyJQ8/61u+yuvFgWVAySPU8kPdSQYzj3tTDtV5bRREgdtqCQOtSaUnet6Xig6m54GSh6pmzdKSUNKlmW9SoZSUkKWj9U64p5R55QKysjyK6+80rJ/EWUPQvZLeaJwL6jAgXWlFiUdqvOkTj3q+SzxrVixoigznARtwLmg15Q6l1QwvbzHlMzNs0nNhTrb67atzil1laAunNXWMAzDMIy2wj8+DMMwDMNoKzpWdvnDH/4QGzZsKNFKipYlFVuVhyDLIk2QYiQdSlpKBTUjvZXTtrMvbIPeE+pGNG8xs426OS9UWmkl4ShwTCqIUKYtVY4T5X2iJLQqiaOJfJGZ9iE9TOmDfWIAMY6J9ZIqps0yHanytqhb8nVu+ytviSxjqMBWSsZiXVxHDI5Ge7zwwgtFOQdAUtIa7X/nnXe2fH/ChAlFmeuA6452zW3XoY6VtKD2t5qvKsmnTpC3TOM3wT2p9nTe3+w7bcU2lJTEZ5SHV5Yzieeee64o03NG7W/2T3njEFVnVJ3za7C5TAg+zzM7ImLkyJEtn6sjO3LdKnmQ5RyIjO+zXp5rnEt6K3Eu1Tmj9nBEef+9Vb6tKgk+w8yHYRiGYRhthX98GIZhGIbRVnSs7DJ06NAYOnRoiSpWuRuqUhHz3yrAFm8lk7JW3irqdnkGaSl6r6ib6mxPUezqZnamhJXcodLX1wmSpGSFDNWG6p+69U7pg3NclfmYc1wn7TWlARUwiX1SHg4RZTurHBaEklRUoCM+U5VfhW2rW+jKo0N5ZVUFV1PvK5lhn332Kcoqv4cK7JbXOf+m8icpDxdlA0VH57NFSYFsW50bKoeIOgPyWuMZos4TNQ7liUIoD7QI7dVSJ0Aax62Cv6nzJ0J7CRGK+lfBGpUUmudIBXZTa0oFslRzrKTC3AbPOOX9qM58VacK/Behcwy1kvXqePAV7dd+0jAMwzAMYwvAPz4MwzAMw2gr/OPDMAzDMIy2omPvfDQjnCodXWnTVYmgqG8qnZT6cJVrYx2oOwe8s0BXYo5DjVu5CtLlLf+N7ygtnDZQ7lQs893stlblntiqT8rtk3PJfnMe87xk97gmlBapIoDyc2rcjIDLezwROmqiShKoXOPoPkd9lzbId26U9q4in6oEd7Stmrt8f4bvr169uuX7dM9cvnx5Ueba+bM/+7OiTDupJGX532p8tBtdhjl/ShdX+yiiPAf77rtvUeZYee9F7T11z6AqamSdOxysa7CJ27ju8n0T3sfhOFQbym23TtI+2iyi7Bqvzk6VyFElOuQzKjlkRMgkp8qtnvavujvUBNdKvrei3HbZHt9XZ+qee+5ZlDl3nKOqCMbqO7E5PuVm3gpmPgzDMAzDaCv848MwDMMwjLZiSKMO99VG9PX1xYgRI+L000+PYcOGySh9iuLOdLSinVkvaW7lzsjPlQtVpidJV/Ed9km5OymX2sHSpxGatq7jbqmSJnE8VYmclLyi3Mpoc5WYrCqKHu2jpKE6br7Kha0qIZVynVVQdLuaF+WWnVEnaZZyYVeujEoCiygn3KIbLV3Y2caqVatavsu5P+SQQ1r2KVPWXIeqv3ymTqJErjW113MbtDPXneofoeh5tQ4iyuPg+VXH1VNJVcrtOa81zqWSo9VaVWED6iSDy33P7uZvBSXPK7k1y45qbtQz6pxXUraK7pz/xrOszte3kqTYJ85R/g5V+0/JLjfddFOsXbu2JIu1gpkPwzAMwzDaCv/4MAzDMAyjrehYb5cmSCvlZDtNVFF2pIZ4S5s3nOlpoG6YkwbjM6SgMwWmEoQpCk4lZqpDi2coyk9FiWVCMUb95Pg4blK6WWJQlKaiWWlbeqtwDLx5Tlow0/BKXlHyj5pvJUtUUeFVicBagW2rKKhqreS2aStFy6r2OJecC0VrZ8qZczB16tSi/NRTTxXlZ599tijTnmPGjCnKKmHW6NGjW/Yvj4NQ3hZKdqGd6u4xNR9KYlX1qjEQVVS48j7hGUcKXI1bjadKXhyslMHzRHlvcQyZuudcch1yTGp/qwi4daKVVo1JJRgcrOxVFeFUrRGel/TeUvu+zpWBHJlYycmtItfWOfeK92s/aRiGYRiGsQXgHx+GYRiGYbQVHSu77LTTTjFs2DApS5DeIp2WaR9SSKSPlFyiAjypW+FKPsjvqEBhiioj1Firkvgomo7v0zb0UlBSF+l8UoRVt+HVLW3allC35Gk/zl3VjXQlP7B/pKYVRa4SZlUlWOM42AaD/FBKUp5Yij7N46Y9FY2skjEqeYVzz36sWbOm9NzBBx/cso+k0mkrBsRTY2Iwt/Hjxxfl7Gmj1gvLHJ9KKEl6X3kS5baVzMb9yvJee+3Vsg1Fw9P+eT/X8VZSsqMKtqUCG2ZU9asJjonzrWxG26pAXfk5QskBKskfg+Fx7yiPn1yv8nZRCevUeac8XLLMppLzcd2ybe4rtq3OxFaeK2/Vx1ZrajDOs2Y+DMMwDMNoK/zjwzAMwzCMtqJjZZcNGzZEV1eXDHJVN/BTrrMVVP4FFUBH0ev5JrHKC8HnSP2y7xw3qS7lfVLlecF62V/elGZd6ta1kqHyuNXtaNqT7bFMGapK0mrVVoYKwsbx0VOAVGrOV9NEleylbnrXudWvwHnlu5nGVfKF2g+Ugig10jYqWNYzzzxTqotBw9gvvkMbcN2pAIEs0xOLnmm5vyqgoFr/o0aNKsoq+FKVzFYn7xTHzb2upDU1X1XSJudJnV/0ElH24Bj4eVXAK+UJxn5w/pS8qLznsvzAelUAN+UNx/LIkSNbfq7WQW5DnQ9Kaldzr4IZVkkfSlZVZyTtwX2v1kE+x1QeolYyp3O7GIZhGIbRsfCPD8MwDMMw2oqOlV3eeOONGBgYkDQRoWjA/G9FoSp6WUk7pDCVl0GuS9GYioZUeUbU8zkYj5I1lGdDDizTqr2qAFsK6iY/qX62zZTZpNhpZ6aiz9KY8jogOA4VCI71Ku+ALJsoTwXlbaEoYZVDQt2Yz+NQ6cIpMZESVum6KYHNmzevKI8bN67U9kEHHVSUH3jggaLMMdHTQ+Ui6evrK8q0wcqVK1uOJyLigAMOKMq0M6WdOkG1CLU+8vmjpMcq2aAJJW3WySmUofa08gwhVCC/qgB4SnJQGGwgsiopm889+eSTRZkSGtcz1x33pMoVVTeoHN9X3yucMxXIT0mkOfgh50B5HarvCSXrqbFWBY1UEnITll0MwzAMw+hY+MeHYRiGYRhtRcfKLuvWrYuBgQGZ1p50U5XHg/JwIW2mAqfwGVLQiv7OlJPyziFtpm4ZK1lIUXYvvfRSqW3mZyGU3VQQmzrBdKpu4nNMpMLZP9Ltu+++e8vn69C1ETrIkqIelQRGkF7kWKvoSVLYSl5RbVNWolSibqdH6PXCPr3wwgtFWXkxce75/OOPP96yHBFx6623FmXSsqofCpTfKG0y4NjkyZNL7zBom/JuUqnDlZShqPCqIILKI0B566m+qv2mAp9FlM8pni3cCyq/hwqwqALuRZTnRuXLUnuJNuC+Yr9ZztIm7aPyAimJlvPHeurI4/lv6ruI31fskwripdZKnm++o75LlHSlPIGUVFuVL0v1sWkryy6GYRiGYXQs/OPDMAzDMIy2omNll2HDhsn0vxHVwX8IUkvq9jdpf9Jg6laxAinIt+pXq/ZIj5G+I5XFm/+KOowoB7VRlB8pa0XZqXwB9K7J41bBl0iHks7ef//9izLHShqedKHKCZHfr4M66czVGsrUMm1eZ70o6p3zotZslYxBu3Edcf4YKIy25bxw3XCO2L+IsocLgzcxB4wKqEY5k+uZ/Z42bVpRzt4utCHbVutFfU6oNVTl7aLmQ0mQlOUI5YWh1krV31Q+HUL1m59nmVkFgFNB3gjuET6j5O4sm6s2uI+5/pV8pOaYsm+WH1RAPIU6ZwtRNceqH3xH7XWV50VBzV1E2T6tkL10qmDmwzAMwzCMtsI/PgzDMAzDaCs6VnZpNBoxMDAgvRdUwKtMd/MdlT6dUEGqlKcBpZlcJ9+v4/nC53nDn9Q05Y6qwEOk61UgLUVbqkBHKo9GpiCV5wbboExETx3WpW6Fq6BMGSpQmFpHKlAUx1p1I12tEbahvI3qeE4QmS5XHkqKxlfeBWzv5ZdfLsq0DYOKRZTngPYh1a88SJSXAmWJBQsWFOUsuzDgmcpVxH2lAkqxf6SvWc7rXEkF7LvyYOPzynOiam2z7yrluZJMCSVt890qqYvnjJKNOSYGAyNoG3Uu5bbrBPdSZ6SSRCjB5/mm3Eg7KylPzQtlao5HnZUZKhCmksvVd5TKf5TPNfadslkr2NvFMAzDMIyOhX98GIZhGIbRVnSs7DJ06NAYNmyY9JxQgcWqbqSr9NF1gqgQpPhIuWXZhZQa/0bajNS28iBReTjY7zxu9p1BgZT8oFKy83nakjfKMxVHmjbnnGlCpb1me3W8hTKdrIIsKfpVSQBEnXWQoTws1E11Si2sl3NX1Se1zkl/qxxGyjasc+rUqUWZMkiuS9HwpJTZp6effroo9/b2FmXm+Jk0aVJRnj9/fqlt2kHZUHnRcL4VnV8lg6jgUip4HNeqWjt1820oylxJxSqnEOelTt6P/DeVh4XrdrC5jeoE/osoz7eSE+rkxVK5fLKHEIOw0QY8C9V64VpTZy33Tt6TygNIyW91vh/V+Z/XuZKA+E6zf5ZdDMMwDMPoWPjHh2EYhmEYbYV/fBiGYRiG0VZ07J2PP/zhD7FhwwbpBqhcaKtATUy5B/IZtk2wbfaJ2maui26AKtoj3bzqgP2u0oQJda+BYL9VdFTeVcltVbknNlHHpVBpt0SOAFpnLSgXyzptKxfc/G+lmSstlp+rexMs53Hzfd4XogZLzZrPq/W49957F2VGsszumbzXQzdTPrfvvvsWZa5btsG5Y594p+iII44otc09w/aUTl3n/hRtVnUHif9Wd9OIOlFQaRveJahKJMh7G8ptV91b4jpi2yphWf63cntnvVw7dKtX56hqK6PqPkgTysW47v2WOuC+ot3UvR4VzqEqQao6E+pE760zPpUQMkLvpVbu0XXmpAkzH4ZhGIZhtBX+8WEYhmEYRlvRsbLLiBEjYtiwYSUKSLnxqMiZGayrThQ80mB13JgorURoFzNFN5IaHWzEviq3T+WGpsakJA7S4qTpcpIs5b6nynWiPRLKNhFlqpp1DZbW5Vpj5E3lnlkXXJ8qKqmygaJrI8rjoBRByYL7ZLAJ9Zi0jdR5RNnmdJflGiElzHrpasu55F6lxJcTY9EFmLZVrpu0uZLT6kRBzahKxNZEHZlBSb35edalEjYqibVO21WJ05RbMcvqrFVuzKyzKnKzmj+C64Dzotx8CeWCm/ur5pu2VdF7VbRr5XocUS/ZIedPjU+52Kt6InSIi1ZSrxPLGYZhGIbRsfCPD8MwDMMw2oqOlV1WrlwZPT09tW7tktIiRRsxeBqfIHXINpQHQgYpqDqST52IngQptDFjxpT+xvZUQjhF4/NdJrWjZ4KKLhhRtpW6gV2nT0Sd6LYR5fmgtKDGR6holKtWrSrK2aOpTj8I2oZRILkO6B2gEpNlO3FMlDtUcjFlZ9bzwgsvFOXdd9+9KD/66KOld5gsjAkR+c5ee+1VlBm9dM2aNUVZJYZTUkREeayTJ08uyg888EBRVpSySuanaPuqBIoE51VFzyRUojC1dyLKY6LMptYBodamitqZPciU15SSOJSdVbTZKk8UVRdRx+YKPOOqpEkVpZT9pZ1oQ56p3CNKro7QEppKsKqkbCXfqAi/+X3apJXU6winhmEYhmF0LPzjwzAMwzCMtqJjZZdddtklhg4dWivRWFUwKhWYrE5gqzr0PKnR3D/2nYGfFF1LGpKUFik71kkKOfdV9Z20P2lu9omeDSpgWNXNbBWESFFy6iY4x6pox6qb8Sq5laKjVcC4ESNGFOWqwE0qOZOSalRCKuVVUSU/EJsjs3Hux44dW5R//vOfF+XsUUapk/XSzpRXKOfUkUL333//ovzss8+WnlOy7Ac+8IGifNNNNxVlRelzfXHdsX+ZhldniAo+xvXBNcFn2B7tXCUfUCog6sic3G9q/WdvNkIFV6Nt60iVddZ/rpf95flaB7SnSp5JCTGinGhOyXccB6VG9puyKtcE12ZOyMm/qfWiZDPubxU8rkrqUlcGiOZa3WpBxvr7++PCCy+M8ePHx4477hgHHnhgfP3rX9/oDsRFF10UY8aMiR133DFmzpwZy5YtG0wzhmEYhmFsxxjUj49LL700rr766vjud78bjz76aFx66aVx2WWXxZVXXlk8c9lll8UVV1wR11xzTSxYsCB23nnnOPHEE+X/iRuGYRiG8c7CkMYgAtt/+MMfjlGjRsU//uM/Fp+deuqpseOOO8Y///M/R6PRiN7e3vjiF78Yf/u3fxsRfwxyNGrUqPjRj34Un/jEJ96yjb6+vhgxYkScdtppMXToUHkLl/SOog4jyhSSoqgU6tCWqq26ILWq8qiowFlElQeIknPq3HpXoASTaXjSmIr2V7fkVdvK/lVB5RSqaN0mlJSnKM/8DqEoaHVzXEkRpDwHc6u8FWj/Ouv5ueeeK8rz5s0rPcc5UJ4b9IghuI4U9cuxUhaKiPjgBz9YlPfcc8+izFwyTz31VFFeuXLlW7ZBcI7yHqvjSUcoqVIFmxus91uE9nbh+LieVcCrKtQJqEa7sU91gulVnWUKKriXQp0zLstFlHHZRyXL1pEg1Pfb5tqgjleLOnfrehO2+nz9+vXxs5/9LNauXbuRdJQxKObjuOOOizlz5sTSpUsj4o/ubPPmzYuTTjopIiKWL18eq1evjpkzZxbvjBgxIo455pi46667Wta5bt266OvrK/1nGIZhGMb2i0FdOD3//POjr68vJk6cGN3d3dHf3x+XXHJJnHHGGRERsXr16ojY+P9yRo0aVfwtY/bs2fG1r31tU/puGIZhGMbbEIP68fGzn/0sfvzjH8d1110XkydPjvvvvz/OOeec6O3tjTPPPHOTOnDBBRfEeeedV/y7r68vxo4dG+vXr49Go1GLfiKVlG/j8t9V8kwTKrCPCvakbkpHaAqONCupPNJUVd4kreqsCoBE2nMwt5Ez2A/SrZmuVSm+FdRNfkUDV813nfbq0NlsW1HWuR6VB4TP8XY7514FH1MeMdlmddYL+8QxKQ8CeqXcd999RXn06NGlevk/Fsrjh94udeQ3fn7UUUcV5exxQplHeSrQS4Hr5aGHHmr5uVpDmcquI98RKn29osurgmUp+pvrhVB5P6q8eRTYXzLVbJsBtlQOHRUQsCqXVR2b1/nOULJ2lQSsgvdxrOxvHS8fjof1bMo5Pdh3uHe41nJQuTpoznFd6S5ikD8+vvSlL8X5559f3N2YOnVqrFixImbPnh1nnnlmcSitWbOmFHFzzZo1cfjhh7esc/jw4bVdCA3DMAzDePtjUHc+Xn/99Y1+iXZ3dxe/AsePHx+jR4+OOXPmFH/v6+uLBQsWxIwZM7ZAdw3DMAzDeLtjUMzHn//5n8cll1wS48aNi8mTJ8d9990X3/72t+NTn/pURPyR9jnnnHPiG9/4RkyYMCHGjx8fF154YfT29sbJJ588qI4NHz58kzwZMuXGf6sAKaSrSK0p2URJOVWSj8qFQqmFNF+d3BZ8JksAioYfrIzF50kjVt3qpoSgUiyruaVt2J6iFKuoxjoeJHXqUvkTMi2rAuIpuU95u6g8QkRdyp9rkNIH80uo9niLn2UGSYrQabq5butQ+kqCefrpp4vyRz/60dI7zI2hpBPS2SzTBhyTkrey9KHGpPYlx6TWfx2Pvvw3os4ZoqSdut56ao1wTSovmDp1Dva8yu+rc0PlbqqT8yhCy8OUbVQuJXV2st/87qmaC87fYFLY53q5XyiV57lTa2RT5okY1I+PK6+8Mi688ML43Oc+F88//3z09vbGf/pP/ykuuuii4pkvf/nL8dprr8WnP/3peOWVV+I973lP/OpXv6qdkMswDMMwjO0bg/rxseuuu8bll18el19+uXxmyJAhcfHFF8fFF1+8uX0zDMMwDGM7RMfmdnkrKDptU27qKupLBT1S3gG5bUotBx98cFEmRcVnVFpvVScpt/w826C0o97h56QF1Q3vulDUNKlDJSHwc+VlsimB3RSUtwqhbu5HaGlHeVANVmqpCxVwiZ4J9FhRnlhc/5QoXnrppVJ7io5WAY0Itfe41pi75IADDii9zz1HaYHrmXVRXuFeonxDb6MqbxAlPdaRmFSujro0ep19qXLM1PH2qjrX6siqdc5nJXHUlR/qoM45o5DXtZrvOl6UlPt4HivZKttVndtcR9zr6pqA8nBRn0f8MVBoE9wnbK/5zmC+I5zV1jAMwzCMtsI/PgzDMAzDaCs6XnZRFDIpscEENmkFlR5ZldkPlnOqdtKTTAVOeo1tq+BZdXK+ZI8Y0psqZL1KtVzHU6MKdXIMECrYkKKvlewVMXhqVeWhqZMHKEtddSQVok5Omzr1R+gcOpxXBgMbP358USb1qyhvSh85gvHjjz/esr+M9ZOD4LUC1znztHzoQx8qynm+eZGd71M64TrnrX6Om9SyCsCW6Wh1PgwWpNSZvr4qWJ+i1dU64vOsl6noKQ3wc/Ypomxbgm3znTp7YXOh9gb3KPcI17lam/ksUbblmNgPts11yvZefvnlosz1mOdb9ZH1sh8qaBj3CM8G9jV/X7Av3Cf8bmjaczAymZkPwzAMwzDaCv/4MAzDMAyjrehY2aVVXhcVJIYUU1WodkWTquBGBNvgbeWqm+Mq6BHLpDCVtKPeJS2YKcI6QdE4VlLT6nnlWZDlERUoiVByDilvPsM6lfyW+8IxKSpW5SKhdwfHTTmmKp9OnbTeKocFbaCeyTZXHieK8ma+E1LsKm8RaepMw++zzz5F+cUXXyzKTGWv9hvnYvLkyS3LVd4Pao1wXl955ZWWz3ANK08GtlflSafmr07AMtpA0eJZWlbnBvvBsXKt0p4qpwolmHym8h2OQ0kLSsYi1H6pyt1UJycOoexJ+/PzbHM1f+pMVeco7aS8TPK4KXWq/UM5TZ1ZSm5q5bnSRJ0zvPl+HZm9CTMfhmEYhmG0Ff7xYRiGYRhGW+EfH4ZhGIZhtBUde+dj/fr1G+mDysWySqdTGiO1rzrumSr63Ka42FHPow6vokuqcaskP1X9Up9Tx6WezD7xboCKspdRJ/oitUcV6VMlSJs0aVKp3meeeaYoK3c/BaV5K0095ytS9w/U/LFMG6iEc0TWVutorYceemjLz5WbrtKg8x0A2oHa9IQJE4oy3QiVzs2xUlOnbfL+VnuGzylXSD7P9upGA1X95fmg+qcSWNI26u5Pfo76PudCueUre/KZqjsA6i4b61W2Ifi5skd+V7mn836XSnim7svVdXmvumPX6n11Vqt7WHRxzS7JjMxbJxoy21Z51TjfVeER1PlANNdE1XfBRu/UftIwDMMwDGMLwD8+DMMwDMNoKzpWdnnmmWeiu7s7nnjiieIzFYmUNHVOyKMizpGmIz2vJBwVeVO5zea/sa46kgrdAxnhcdWqVUX50UcfLcp0/83tEcoVmTQr31VSQpXbJ22rKLvBykpqvrk+IspzyX4odzO6pzF63/HHH1+UGamTiZVyhFOVLOzuu+8uyu9+97uL8siRI6MVlHvzgw8+WJT333//0juMQKrcADnHCxYsKMq0wSc/+cmW/eYzmepVEXRpZ1K8KvqoisxaFUmU9f7Jn/xJUb755ptb9pfuv1wrlAxUUscsAXC+ee5wfSoZRUmNSg6okn/YnqpXJYDju8pFOFPpbIMJM2lPrjsl0wzWFT6iHCmazyl598gjjyzKt99+e1Guk/SwKrGcii5dJ7wC2+O7/DwnbyQYFbXOGmS9HIOSBPN5rpKO8v3mXAwmwrSZD8MwDMMw2gr/+DAMwzAMo60Y0hhMJpg2oK+vL0aMGBHHHHNM9PT0SCqQID2W6UlSzaSE+M6SJUuKco7e+FYghXzIIYeU/kZacOHChUWZN/9JZ5OS57jZp5UrVxZl0vZ5Gjk+Sjik81R0Qyb1InXO6JWk2jMty/5SDqCEQNso+UjZhmOlV0pEeay0Ie1Bupe0MesaPXp0UeZtcdb/4Q9/uNQ227vpppuKMmUbPsO6OD568PyP//E/ivKBBx5YlDMdTcmI9pkyZUpR/u1vf1uU991336Ks5KkXXnihKPf29hbl++67r9Q2+3vUUUcVZa559ok2YF1cEyoiavbY4dpjH2mPJ598sigzcq1KaEiwr9kDQUVZVh48PH9Yr5JmuGbzOqf3A23A/cMotmz7scceK8pKplSeEBERe+21V1HmGt5vv/2KMs9FZQ/aXK21jGXLlrX8nGuENuR6pM2UpKUi3UaU54lriucD6+KcUa5VX7lqDWYoLyglu3BeeZ7zPFbyWwbX0QEHHLBR2xs2bIh777031q5du9FVgAwzH4ZhGIZhtBX+8WEYhmEYRlvRsd4uQ4cOjZ6eHnlLWyX5yc+rYDfK24K0Eik41Q/lERNRpp1VUDNSdo888khRZmItPqNuR+dAMgzERbpRUYy0B+lF3i5XN/HzDWdSrqTwmICMnhvsex3bTJ06tSjTeyFC30Kvk+yLtDqpZVL4lMwWLVpUaptBtdi2StrHup5++umiTLqSz5AmzfIin2Ogr/Hjxxfle+65pygrel/VyTnNdOpzzz1XlOllQs8syp8qYZ1K8kebcR1ERBxxxBFFmWNSkqKittme2tPZo0J5WCgoTzN1Rq1evbpln/I73DOk+ilbsl6OSY2Vc5G9f1RANkqsXPMqKaSS2bgmuPciymuS65AyKeVh2pznCSUD7j2VIDBCB1mkdMWz9vLLLy/KX//614vyxz72saJMOebHP/5xUc6SD/tyyimnFOV/+qd/KsqUFJW3ywc/+MGiTHmd85j3CM/Ua6+9tihT1mvK+YMJumnmwzAMwzCMtsI/PgzDMAzDaCs6VnbZeeedY+jQoZI6JK2nbotHlCk/Uomk9lQwK0VVso2DDjqoKK9YsWKjMbRqg9QoKTs+TyqbcgXpb9JsmQImpUaqkwGCVOAb2oYBbeoELcr10iuGn6v8BHVsQykoexg99NBDRZk0KeePn9NuHBMpRfaJtCztH1GmiLnuVEA7UsKkkB9//PGiXCdQXW6P/V26dGlRps05btqG/eB4nn/++ZbliLLcRK+dE044oSiT7v3+979flOm9QFmJbdBO2eOE64veFmouOMek4VUAPOVBkN9n3wlFf/NznnGKwq/ytCE4DvaJMi4/V0HJuGYp40XoQFq0h/K+Ir3P8fGMonyTx8nzluvigQceKMoqrw/PbZ4hKnAWJZ6I8hnENcXPuR75/qmnnlqUP/OZz7T8nKA9Isp76V/+5V+KMs8g2oNnwK9+9auifOmllxblT33qU0WZgSwpnUZEfPzjHy/KtBUlo+Y6cpAxwzAMwzA6Fv7xYRiGYRhGW9Gxsssuu+wSQ4cOLdGTytuCz+Qb5aQVVQptUtB8nlD04sMPP1yUSRdGlCmqOinWVRA1UrGKKq7qr6pXpZsmLatu9CvJpqoNYnNswzozHT1u3LiiTDqb75DuZZAxzj29XRiUiWso55UhOCaVX4XUtoLKX5HpTZU3hGPl2uH65/pSKbGVNBNRlr74PiUYzuUnPvGJovzrX/+6Zb8XL15clEmR5yCAlAg5JlLCtLmSKlWOGa7HbJs66ehZl/L4oQzINtS+reojoVLLjx07tigzWJaqJ3uc0IbsBz1IuA8pFXPdUlJhsDJ+ntcaZWCe9dzHDBrJM5n2p6zKtcL6ue4iyvmUlLxAierLX/5yUVb7jXIH5ZTsgcYAgTzzmLuGwRo5PnrHcI9ddNFFRZkBE+lBk/925ZVXFmXOTfM7oypAWYaZD8MwDMMw2gr/+DAMwzAMo63oWNmlu7s7enp6ZEAo0jtVdDTp7zpphlXgG1LWij7Nt8LpscKbz/RE4Th4Q5mUNWk2RbfmoFN1ggeputSYKGPkoGYEx8Sxku6dP39+USaNW8c2VTlO6CVBu9E+nAsGgiN1y3dpD+bGIbUZUQ7oNW/evKLMuaQNSfFyHLzRT88X1p/XOcfNnD8MvnTnnXcW5RkzZrSsqyrYkGqb7XEuSdfTC4BzMXHixKJMj4XDDz+8KNNmlJFy25RgVDA+nhV8hvue7al6IrR0yDaUF5nKt6G8TChdROj9p/Ye50wF4VKefnncy5cvL8qUiWi3OXPmtHyfbbBPXL//8T/+x6LMXFYZtBs9ArmXOD7uV7bHYHicU+Zkiiivba4XdQ6zf/wuoP251yndZfmC8hHz4DDonjrPVe6ZK664oijzTOS5FKGDJ/KMbK6XwaSKM/NhGIZhGEZb4R8fhmEYhmG0FR0ruwwfPjyGDRsmU3GrGPz5lrC6vU8obwuCtDFvR5Puy94LDOzD4Fekq3j7mM8TSgpSeVoitCcLbcXPFf1K6pZt8HZ69rrhbW7S/pSrmOqaOUfq2EZR3BHl+eP7KqgW6UYV7Il1kv7k7f6I8g1/0qFcF6yLtDoDZFFeIQXK8WRJRN3qnz59elFm+npS3ir4G5/5wAc+UJTpTRBRprC5T7huDzvssKL8f/7P/ynKzEFE2YvBx+hlkL3RaFvOMSls0t/KQ4zvcn2pvDcR5bNFBd5SeVS43zgmPkOaP9PwlCHZHtct15fKK8N6aUs+T5kgImLSpElFmRIAba5yZ9GefF4Ff8vS5qGHHlqUFyxYUJSfeeaZosw1qCQt2o/SDqUW5fmY+1vnrKX3HCXZX/ziF0WZc5fXHc8B5T1E7xPaX50bc+fObVknv98iIv7t3/4tWoH7pDlPVcEnM8x8GIZhGIbRVvjHh2EYhmEYbUXHyi5dXV3R1dVVoowULVVFR6ugJ6SrSOWpm/EsK4qVNGJEOVjX0Ucf3bJt1ssx0auC0gApUNJjeZykTSkNkZ5XFKGiSXt7e4syPVeqAl6xH5QDWCYlXyclc1UgJnpJqMBkKvAZU2ur+Vap1yN0Pgvl2aA8qFgPbZlTmxPsF+nQ3/zmN0WZcg7rZZn9owTz4IMPFuVM4TPI2PHHH1+UKa/Qo4D2YM4KylZc//RwOfbYY0tt33rrrUVZrS+Cfef8Kc8VJU/l52hDJQuq3FKqTpa59/LflOceyzwj6SHENcXnq4KYcd1SNuN5p2yo5oVtcP1n76abbrqpKD/11FNF+d//+39flCkLUh6+//77W46Ba41eKZSUIspnnrKPCgxHWeOjH/1oUaas9MMf/rAoZ/mC/2bQr//6X/9rUVZriqA9L7744qJ8ySWXFOX3v//9pXco/333u98tylxTzbbrXHNowsyHYRiGYRhthX98GIZhGIbRVnSs7DJmzJjYYYcdSgFnSPGRYlKBsyI0fa68HxQFrdJCk+7OVJmioEjNqXHw5jJvSn/7298uyqQXc9AhdctePUNPD/aJNCnbqEo1rmQl1Q/OmcrnkoOotXo+omxz3oBnoBxSoKQOKUswrTSfWb16dVEmXZtBu3Eu6RnCtUNZgrlkVAC83Had4E3sEz0hVGp4Ps/5Zi6SiLKHAG/lMxgVKWzmbaHnBMfHNU/b3HbbbaW2lXTF/pLGp1SpJAfKTXz3xBNPLLXNM4GSMOeC65F9VbIX91JVfiaOj3Kvyn/F/cP1r9pQgbryc8wbwjk+/fTToxVUoEi2R8/ALOnSe45rmGuN+42eL5T1KMcwP4ryHokon5HKK1JdAaCdP/OZzxTlv/iLvyjKtCv3YUTERz7ykaL8P//n/yzK3HvK+4oyJ8fHvcB5ZP8iIj7+8Y8XZe4Z7vWm95BzuxiGYRiG0bHwjw/DMAzDMNoK//gwDMMwDKOtGNIYTCaYNqCvry9GjBgRf/ZnfxZDhw6VrlnU0KpcbdX9jKqoqK3eJVTE0ao7H8q9s06CIup/dHmkVpnHre6osI/KJVDdAVCJj/J9DDVW5VZcx41ZJRLMdz5Ugjy6F1KbVu2peqrc6pTeqe4UEbSHWrNVW1WNg/NNzZpzyXsNvLvAz6vc+Jgci5Ej6YI7ZcqUosz9ygiPHB/dyxkBN+vw6s4O7+b86Z/+aVH+yU9+0rIu1kMdnXuMd1UiyrZVkWE5f7xrwT3GcStX/3x/TK0L1bbaS+oZtT8z+A7XF11WecapvcQQAnSnzneb6A7Pc4dRcDkvDDPA9cwIurz7wLOB70aU74wwCaKae46V32Nnnnlmy+f/9V//tShXnWu8x8IEkYsWLSrKnAu6vP+v//W/Wtapvusiynd+/umf/qkoM2Fmc/42bNgQd955Z6xdu3ajvZph5sMwDMMwjLbCPz4MwzAMw2grOtbVdsOGDTFkyJASHUqaiBQmaVXSffnfdBFkYjO6rPLzn/70p0X5pJNOKsqk7EjLXnPNNaW26aJEynbfffctynTpzEmUmiAFSvcyRZ1HlKlV0tGkN0877bSi/N//+38vyhzT3XffXZRpJ7qIZZuT8uaYOI5nn322KFPaYSRHUv3z588vyl/96leL8nXXXVdqm1Qp54nUcR13RNKWpE9Jf5PyjCjbgeNjcj7S7YzGSsqVa402Z5K4q6++utT25z73uaLMRFBcFyrKK8vcVwTtlOVFjlUl3OL657o98MADizIT39FFmOsg08Ok9Nl3yke//e1vizLlN9ZL6p1jveOOO4ryzJkzS21zvrnm6SbM+WYk3xtvvLEof/CDHyzK6mz53ve+V2qbrqzqbHniiSeKMilw9oP1cp3Szjmyror4yzOHspmSfJQbMiWKLCfT1ZZzrKITUzJQLqcqcjDPgIiyey6hxkRZlnVR2mEIgCr3Zp6LtAlleBU9+dxzzy3KlEV5njDydY7sSlmV4JiaY1XhFFrBzIdhGIZhGG2Ff3wYhmEYhtFWdKzsMn/+/BgyZEhJMiAdSpqU9Btp3Pzv//2//3dRJhVIapURF0kr3XvvvS2f541fUsUREbfffntRPu6444oy5RnSwKQwGeWSIMVHeix7nFCe4XN8f8WKFUWZt7ff8573FGXePCftWNfm//AP/1CUZ82aVZQPOOCAokx5i9KYSshGKSJ7gHCsylNKReskTU36lPUwCSFtEFEeN8tcB0zSRJmAZbXWuDZzRFtKAJxjrknKHcrTI0cvbfVMBttmvxhhljQ3PV94Q59RKrkvVBK1CD1PpN65bklnk4JWdVI+qJpvUtO33HJLUSadTXlxsGdL9vpQZ8u1115blCmv0G7sB+eFa5BzSu+fiI2lkFZtsF7uXUoc3Ie0Lb05GEUzonyucR2xT5xXniG0Odc/k61xj1QluWR7qsz3+X3FNc/vJCX9RZTnX3nM8R3uBdqQc79w4cKiTDmesmFExJw5c4oyx8c91rS5ZRfDMAzDMDoW/vFhGIZhGEZb0bFBxvbaa6/o6uoq0Wak5kgPV91cVsGiSF2RiiVtpCgk1qkCU2WoW9D8nAm01M1lUtBsm3aKKFOopBIp7TA4jhoT6XY+XzfQF+eDVGAdu/EZFXQtQwU0Yt/ZD3pqcEykmkl5sp58I71O4Do1ZypoFD9XMlJEeY0obx7SywzKRE8Nygxcm6wzyw+kz/kO7aYCadEbSlHvtHkG9xLngxQ7ZRcl4ajgUHXOg80F+8F9z72n1tPm9ov2YLkqQRj7wvnmvqQ8ot7lXlAymfJEyVDB2Vjm2qZtlbySzyWuc+4f7quq95vgd5c6d3NQP65trlWVlFPZgN5vSqbJ4BxzPijVNJ/p7++Pxx9/3EHGDMMwDMPoPHTchdPmr7HmrzX1i5a/9qvStvN9xT7UKas66zIfKix3ndDKimVQv3Tr1qX6oepVNq9iPpQ9B8t8EFXhnhWrxP9bUOOoM9aqtabGR9SxbZ25yFB23pz9U2ce67an1jnfVXNU9X/2dd5Xa7XOHlHlLYnB9iNjc/pV9zyp017V3hhMe3Xqyaiz7uqcRarOXK9aa1Xvt3pX7fUcz0adZXXsqRi+qlQRhHq/1b5v9b2t0HGyyzPPPBNjx47d1t0wDMMwDGMTsHLlylLAu1bouB8fAwMDsWrVqmg0GjFu3LhYuXLlW2pH2xP6+vpi7NixHvc7BB63x/1OgMf9zhh3o9GIV199NXp7ezdKhpjRcbJLV1dX7LvvvsWFs9122+0dMWkZHvc7Cx73Owse9zsL76Rx82JvFXzh1DAMwzCMtsI/PgzDMAzDaCs69sfH8OHD4+/+7u82iqWwvcPj9rjfCfC4Pe53At6p466DjrtwahiGYRjG9o2OZT4MwzAMw9g+4R8fhmEYhmG0Ff7xYRiGYRhGW+EfH4ZhGIZhtBX+8WEYhmEYRlvRkT8+rrrqqth///1jhx12iGOOOSbuvvvubd2lLYrZs2fHUUcdFbvuumuMHDkyTj755FiyZEnpmTfeeCNmzZoVe+65Z+yyyy5x6qmnxpo1a7ZRj7cOvvWtb8WQIUPinHPOKT7bXsf97LPPxl/+5V/GnnvuGTvuuGNMnTo17r333uLvjUYjLrroohgzZkzsuOOOMXPmzFi2bNk27PHmo7+/Py688MIYP3587LjjjnHggQfG17/+9Y2SzL3dx33HHXfEn//5n0dvb28MGTIk/vVf/7X09zpjfPnll+OMM86I3XbbLXbffff467/+6/j973/fxlEMHlXjfvPNN+MrX/lKTJ06NXbeeefo7e2Nv/qrv4pVq1aV6tjexp3xmc98JoYMGRKXX3556fO347i3NDrux8dPf/rTOO+88+Lv/u7vYtGiRXHYYYfFiSeeGM8///y27toWw9y5c2PWrFkxf/78uOWWW+LNN9+MD3zgA/Haa68Vz5x77rnxy1/+Mq6//vqYO3durFq1Kk455ZRt2Osti3vuuSe+//3vx6GHHlr6fHsc9+9+97s4/vjjY+jQoXHzzTfHI488En//938f73rXu4pnLrvssrjiiivimmuuiQULFsTOO+8cJ554YrzxxhvbsOebh0svvTSuvvrq+O53vxuPPvpoXHrppXHZZZfFlVdeWTyzPYz7tddei8MOOyyuuuqqln+vM8YzzjgjHn744bjlllvipptuijvuuCM+/elPt2sIm4Sqcb/++uuxaNGiuPDCC2PRokXx85//PJYsWRIf+chHSs9tb+Mmbrjhhpg/f3709vZu9Le347i3OBodhqOPProxa9as4t/9/f2N3t7exuzZs7dhr7Yunn/++UZENObOndtoNBqNV155pTF06NDG9ddfXzzz6KOPNiKicdddd22rbm4xvPrqq40JEyY0brnllsaf/MmfNL7whS80Go3td9xf+cpXGu95z3vk3wcGBhqjR49u/Jf/8l+Kz1555ZXG8OHDG//yL//Sji5uFXzoQx9qfOpTnyp9dsoppzTOOOOMRqOxfY47Iho33HBD8e86Y3zkkUcaEdG45557imduvvnmxpAhQxrPPvts2/q+OcjjboW77767ERGNFStWNBqN7XvczzzzTGOfffZpLF68uLHffvs1vvOd7xR/2x7GvSXQUczH+vXrY+HChTFz5szis66urpg5c2bcdddd27BnWxdr166NiIg99tgjIiIWLlwYb775ZskOEydOjHHjxm0Xdpg1a1Z86EMfKo0vYvsd94033hjTp0+P0047LUaOHBlHHHFE/OAHPyj+vnz58li9enVp3CNGjIhjjjnmbT3u4447LubMmRNLly6NiIgHHngg5s2bFyeddFJEbL/jJuqM8a677ordd989pk+fXjwzc+bM6OrqigULFrS9z1sLa9eujSFDhsTuu+8eEdvvuAcGBuKTn/xkfOlLX4rJkydv9PftddyDRUdltX3xxRejv78/Ro0aVfp81KhR8dhjj22jXm1dDAwMxDnnnBPHH398TJkyJSIiVq9eHcOGDSs2aROjRo2K1atXb4Nebjn85Cc/iUWLFsU999yz0d+213E/+eSTcfXVV8d5550XX/3qV+Oee+6Jz3/+8zFs2LA488wzi7G1Wvdv53Gff/750dfXFxMnTozu7u7o7++PSy65JM4444yIiO123ESdMa5evTpGjhxZ+ntPT0/sscce240d3njjjfjKV74Sp59+epHddXsd96WXXho9PT3x+c9/vuXft9dxDxYd9ePjnYhZs2bF4sWLY968edu6K1sdK1eujC984Qtxyy23xA477LCtu9M2DAwMxPTp0+Ob3/xmREQcccQRsXjx4rjmmmvizDPP3Ma923r42c9+Fj/+8Y/juuuui8mTJ8f9998f55xzTvT29m7X4zbKePPNN+NjH/tYNBqNuPrqq7d1d7YqFi5cGP/wD/8QixYtiiFDhmzr7nQ0Okp22WuvvaK7u3sj74Y1a9bE6NGjt1Gvth7OPvvsuOmmm+K2226Lfffdt/h89OjRsX79+njllVdKz7/d7bBw4cJ4/vnn48gjj4yenp7o6emJuXPnxhVXXBE9PT0xatSo7XLcY8aMiUMOOaT02aRJk+Lpp5+OiCjGtr2t+y996Utx/vnnxyc+8YmYOnVqfPKTn4xzzz03Zs+eHRHb77iJOmMcPXr0RhfqN2zYEC+//PLb3g7NHx4rVqyIW265pWA9IrbPcf/mN7+J559/PsaNG1eccStWrIgvfvGLsf/++0fE9jnuTUFH/fgYNmxYTJs2LebMmVN8NjAwEHPmzIkZM2Zsw55tWTQajTj77LPjhhtuiFtvvTXGjx9f+vu0adNi6NChJTssWbIknn766be1HU444YR46KGH4v777y/+mz59epxxxhlFeXsc9/HHH7+RK/XSpUtjv/32i4iI8ePHx+jRo0vj7uvriwULFrytx/36669HV1f5iOnu7o6BgYGI2H7HTdQZ44wZM+KVV16JhQsXFs/ceuutMTAwEMccc0zb+7yl0PzhsWzZsvi///f/xp577ln6+/Y47k9+8pPx4IMPls643t7e+NKXvhS//vWvI2L7HPcmYVvfeM34yU9+0hg+fHjjRz/6UeORRx5pfPrTn27svvvujdWrV2/rrm0xfPazn22MGDGicfvttzeee+654r/XX3+9eOYzn/lMY9y4cY1bb721ce+99zZmzJjRmDFjxjbs9dYBvV0aje1z3HfffXejp6encckllzSWLVvW+PGPf9zYaaedGv/8z/9cPPOtb32rsfvuuzd+8YtfNB588MHGRz/60cb48eMbf/jDH7ZhzzcPZ555ZmOfffZp3HTTTY3ly5c3fv7znzf22muvxpe//OXime1h3K+++mrjvvvua9x3332NiGh8+9vfbtx3332FV0edMX7wgx9sHHHEEY0FCxY05s2b15gwYULj9NNP31ZDqoWqca9fv77xkY98pLHvvvs27r///tI5t27duqKO7W3crZC9XRqNt+e4tzQ67sdHo9FoXHnllY1x48Y1hg0b1jj66KMb8+fP39Zd2qKIiJb//fCHPyye+cMf/tD43Oc+13jXu97V2GmnnRr/7t/9u8Zzzz237Tq9lZB/fGyv4/7lL3/ZmDJlSmP48OGNiRMnNq699trS3wcGBhoXXnhhY9SoUY3hw4c3TjjhhMaSJUu2UW+3DPr6+hpf+MIXGuPGjWvssMMOjQMOOKDxn//zfy59+WwP477tttta7uczzzyz0WjUG+NLL73UOP300xu77LJLY7fddmucddZZjVdffXUbjKY+qsa9fPlyec7ddtttRR3b27hbodWPj7fjuLc0hjQaCDdoGIZhGIaxldFRdz4MwzAMw9j+4R8fhmEYhmG0Ff7xYRiGYRhGW+EfH4ZhGIZhtBX+8WEYhmEYRlvhHx+GYRiGYbQV/vFhGIZhGEZb4R8fhmEYhmG0Ff7xYRiGYRhGW+EfH4ZhGIZhtBX+8WEYhmEYRlvx/wE2nXNj7J1TIgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.imshow(cv2.cvtColor(state, cv2.COLOR_BGR2RGB))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 4. Setup Callback"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Looking in links: https://download.pytorch.org/whl/cu113/torch_stable.html\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "ERROR: Could not find a version that satisfies the requirement torch==1.10.1+cu113 (from versions: 2.0.0, 2.0.1)\n",
      "ERROR: No matching distribution found for torch==1.10.1+cu113\n"
     ]
    }
   ],
   "source": [
    "!pip install torch==1.10.1+cu113 torchvision==0.11.2+cu113 torchaudio===0.10.1+cu113 -f https://download.pytorch.org/whl/cu113/torch_stable.html"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Looking in indexes: https://download.pytorch.org/whl/cu118\n",
      "Requirement already satisfied: torch in c:\\users\\91701\\anaconda3\\lib\\site-packages (1.12.1)\n",
      "Collecting torchvision\n",
      "  Downloading https://download.pytorch.org/whl/cu118/torchvision-0.15.2%2Bcu118-cp310-cp310-win_amd64.whl (4.9 MB)\n",
      "     ---------------------------------------- 4.9/4.9 MB 3.8 MB/s eta 0:00:00\n",
      "Collecting torchaudio\n",
      "  Downloading https://download.pytorch.org/whl/cu118/torchaudio-2.0.2%2Bcu118-cp310-cp310-win_amd64.whl (2.5 MB)\n",
      "     ---------------------------------------- 2.5/2.5 MB 4.2 MB/s eta 0:00:00\n",
      "Requirement already satisfied: typing_extensions in c:\\users\\91701\\anaconda3\\lib\\site-packages (from torch) (4.4.0)\n",
      "Collecting torch\n",
      "  Downloading https://download.pytorch.org/whl/cu118/torch-2.0.1%2Bcu118-cp310-cp310-win_amd64.whl (2619.1 MB)\n",
      "     ---------------------------------------- 2.6/2.6 GB ? eta 0:00:00\n",
      "Requirement already satisfied: requests in c:\\users\\91701\\anaconda3\\lib\\site-packages (from torchvision) (2.28.1)\n",
      "Requirement already satisfied: numpy in c:\\users\\91701\\anaconda3\\lib\\site-packages (from torchvision) (1.23.5)\n",
      "Requirement already satisfied: pillow!=8.3.*,>=5.3.0 in c:\\users\\91701\\anaconda3\\lib\\site-packages (from torchvision) (9.4.0)\n",
      "Requirement already satisfied: jinja2 in c:\\users\\91701\\anaconda3\\lib\\site-packages (from torch) (3.1.2)\n",
      "Requirement already satisfied: filelock in c:\\users\\91701\\anaconda3\\lib\\site-packages (from torch) (3.9.0)\n",
      "Requirement already satisfied: sympy in c:\\users\\91701\\anaconda3\\lib\\site-packages (from torch) (1.11.1)\n",
      "Requirement already satisfied: networkx in c:\\users\\91701\\anaconda3\\lib\\site-packages (from torch) (2.8.4)\n",
      "Requirement already satisfied: MarkupSafe>=2.0 in c:\\users\\91701\\anaconda3\\lib\\site-packages (from jinja2->torch) (2.1.1)\n",
      "Requirement already satisfied: idna<4,>=2.5 in c:\\users\\91701\\anaconda3\\lib\\site-packages (from requests->torchvision) (3.4)\n",
      "Requirement already satisfied: charset-normalizer<3,>=2 in c:\\users\\91701\\anaconda3\\lib\\site-packages (from requests->torchvision) (2.0.4)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in c:\\users\\91701\\anaconda3\\lib\\site-packages (from requests->torchvision) (2022.12.7)\n",
      "Requirement already satisfied: urllib3<1.27,>=1.21.1 in c:\\users\\91701\\anaconda3\\lib\\site-packages (from requests->torchvision) (1.26.14)\n",
      "Requirement already satisfied: mpmath>=0.19 in c:\\users\\91701\\anaconda3\\lib\\site-packages (from sympy->torch) (1.2.1)\n",
      "Installing collected packages: torch, torchvision, torchaudio\n",
      "  Attempting uninstall: torch\n",
      "    Found existing installation: torch 1.12.1\n",
      "    Uninstalling torch-1.12.1:\n",
      "      Successfully uninstalled torch-1.12.1\n",
      "Successfully installed torch-2.0.1+cu118 torchaudio-2.0.2+cu118 torchvision-0.15.2+cu118\n"
     ]
    }
   ],
   "source": [
    "!pip3 install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cu118"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting git+https://github.com/DLR-RM/stable-baselines3\n",
      "  Cloning https://github.com/DLR-RM/stable-baselines3 to c:\\users\\91701\\appdata\\local\\temp\\pip-req-build-car7sce_\n",
      "  Resolved https://github.com/DLR-RM/stable-baselines3 to commit 9c338f917a822c8be13b1aa9f7b2319770481b62\n",
      "  Installing build dependencies: started\n",
      "  Installing build dependencies: finished with status 'done'\n",
      "  Getting requirements to build wheel: started\n",
      "  Getting requirements to build wheel: finished with status 'done'\n",
      "  Preparing metadata (pyproject.toml): started\n",
      "  Preparing metadata (pyproject.toml): finished with status 'done'\n",
      "Requirement already satisfied: gymnasium==0.28.1 in c:\\users\\91701\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from stable-baselines3==2.0.0a9) (0.28.1)\n",
      "Requirement already satisfied: numpy>=1.20 in c:\\users\\91701\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from stable-baselines3==2.0.0a9) (1.24.3)\n",
      "Requirement already satisfied: torch>=1.11 in c:\\users\\91701\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from stable-baselines3==2.0.0a9) (2.0.1)\n",
      "Requirement already satisfied: cloudpickle in c:\\users\\91701\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from stable-baselines3==2.0.0a9) (2.2.1)\n",
      "Requirement already satisfied: pandas in c:\\users\\91701\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from stable-baselines3==2.0.0a9) (2.0.1)\n",
      "Requirement already satisfied: matplotlib in c:\\users\\91701\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from stable-baselines3==2.0.0a9) (3.7.1)\n",
      "Requirement already satisfied: jax-jumpy>=1.0.0 in c:\\users\\91701\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from gymnasium==0.28.1->stable-baselines3==2.0.0a9) (1.0.0)\n",
      "Requirement already satisfied: typing-extensions>=4.3.0 in c:\\users\\91701\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from gymnasium==0.28.1->stable-baselines3==2.0.0a9) (4.5.0)\n",
      "Requirement already satisfied: farama-notifications>=0.0.1 in c:\\users\\91701\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from gymnasium==0.28.1->stable-baselines3==2.0.0a9) (0.0.4)\n",
      "Requirement already satisfied: filelock in c:\\users\\91701\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from torch>=1.11->stable-baselines3==2.0.0a9) (3.12.0)\n",
      "Requirement already satisfied: sympy in c:\\users\\91701\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from torch>=1.11->stable-baselines3==2.0.0a9) (1.12)\n",
      "Requirement already satisfied: networkx in c:\\users\\91701\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from torch>=1.11->stable-baselines3==2.0.0a9) (3.1)\n",
      "Requirement already satisfied: jinja2 in c:\\users\\91701\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from torch>=1.11->stable-baselines3==2.0.0a9) (3.1.2)\n",
      "Requirement already satisfied: contourpy>=1.0.1 in c:\\users\\91701\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from matplotlib->stable-baselines3==2.0.0a9) (1.0.7)\n",
      "Requirement already satisfied: cycler>=0.10 in c:\\users\\91701\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from matplotlib->stable-baselines3==2.0.0a9) (0.11.0)\n",
      "Requirement already satisfied: fonttools>=4.22.0 in c:\\users\\91701\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from matplotlib->stable-baselines3==2.0.0a9) (4.39.4)\n",
      "Requirement already satisfied: kiwisolver>=1.0.1 in c:\\users\\91701\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from matplotlib->stable-baselines3==2.0.0a9) (1.4.4)\n",
      "Requirement already satisfied: packaging>=20.0 in c:\\users\\91701\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from matplotlib->stable-baselines3==2.0.0a9) (23.1)\n",
      "Requirement already satisfied: pillow>=6.2.0 in c:\\users\\91701\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from matplotlib->stable-baselines3==2.0.0a9) (9.5.0)\n",
      "Requirement already satisfied: pyparsing>=2.3.1 in c:\\users\\91701\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from matplotlib->stable-baselines3==2.0.0a9) (3.0.9)\n",
      "Requirement already satisfied: python-dateutil>=2.7 in c:\\users\\91701\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from matplotlib->stable-baselines3==2.0.0a9) (2.8.2)\n",
      "Requirement already satisfied: pytz>=2020.1 in c:\\users\\91701\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from pandas->stable-baselines3==2.0.0a9) (2023.3)\n",
      "Requirement already satisfied: tzdata>=2022.1 in c:\\users\\91701\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from pandas->stable-baselines3==2.0.0a9) (2023.3)\n",
      "Requirement already satisfied: six>=1.5 in c:\\users\\91701\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from python-dateutil>=2.7->matplotlib->stable-baselines3==2.0.0a9) (1.16.0)\n",
      "Requirement already satisfied: MarkupSafe>=2.0 in c:\\users\\91701\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from jinja2->torch>=1.11->stable-baselines3==2.0.0a9) (2.1.2)\n",
      "Requirement already satisfied: mpmath>=0.19 in c:\\users\\91701\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from sympy->torch>=1.11->stable-baselines3==2.0.0a9) (1.3.0)\n",
      "Note: you may need to restart the kernel to use updated packages.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  Running command git clone --filter=blob:none --quiet https://github.com/DLR-RM/stable-baselines3 'C:\\Users\\91701\\AppData\\Local\\Temp\\pip-req-build-car7sce_'\n"
     ]
    }
   ],
   "source": [
    "pip install git+https://github.com/DLR-RM/stable-baselines3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: stable-baselines3[extra] in c:\\users\\91701\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (2.0.0a9)\n",
      "Requirement already satisfied: gymnasium==0.28.1 in c:\\users\\91701\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from stable-baselines3[extra]) (0.28.1)\n",
      "Requirement already satisfied: numpy>=1.20 in c:\\users\\91701\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from stable-baselines3[extra]) (1.24.3)\n",
      "Requirement already satisfied: torch>=1.11 in c:\\users\\91701\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from stable-baselines3[extra]) (2.0.1)\n",
      "Requirement already satisfied: cloudpickle in c:\\users\\91701\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from stable-baselines3[extra]) (2.2.1)\n",
      "Requirement already satisfied: pandas in c:\\users\\91701\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from stable-baselines3[extra]) (2.0.1)\n",
      "Requirement already satisfied: matplotlib in c:\\users\\91701\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from stable-baselines3[extra]) (3.7.1)\n",
      "Requirement already satisfied: opencv-python in c:\\users\\91701\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from stable-baselines3[extra]) (4.7.0.72)\n",
      "Requirement already satisfied: tensorboard>=2.9.1 in c:\\users\\91701\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from stable-baselines3[extra]) (2.13.0)\n",
      "Requirement already satisfied: psutil in c:\\users\\91701\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from stable-baselines3[extra]) (5.9.5)\n",
      "Requirement already satisfied: tqdm in c:\\users\\91701\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from stable-baselines3[extra]) (4.65.0)\n",
      "Requirement already satisfied: rich in c:\\users\\91701\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from stable-baselines3[extra]) (13.3.5)\n",
      "Requirement already satisfied: shimmy[atari]~=0.2.1 in c:\\users\\91701\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from stable-baselines3[extra]) (0.2.1)\n",
      "Requirement already satisfied: pillow in c:\\users\\91701\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from stable-baselines3[extra]) (9.5.0)\n",
      "Requirement already satisfied: autorom[accept-rom-license]~=0.6.0 in c:\\users\\91701\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from stable-baselines3[extra]) (0.6.1)\n",
      "Requirement already satisfied: pygame in c:\\users\\91701\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from stable-baselines3[extra]) (2.4.0)\n",
      "Requirement already satisfied: jax-jumpy>=1.0.0 in c:\\users\\91701\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from gymnasium==0.28.1->stable-baselines3[extra]) (1.0.0)\n",
      "Requirement already satisfied: typing-extensions>=4.3.0 in c:\\users\\91701\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from gymnasium==0.28.1->stable-baselines3[extra]) (4.5.0)\n",
      "Requirement already satisfied: farama-notifications>=0.0.1 in c:\\users\\91701\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from gymnasium==0.28.1->stable-baselines3[extra]) (0.0.4)\n",
      "Requirement already satisfied: click in c:\\users\\91701\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from autorom[accept-rom-license]~=0.6.0->stable-baselines3[extra]) (8.1.3)\n",
      "Requirement already satisfied: requests in c:\\users\\91701\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from autorom[accept-rom-license]~=0.6.0->stable-baselines3[extra]) (2.30.0)\n",
      "Requirement already satisfied: AutoROM.accept-rom-license in c:\\users\\91701\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from autorom[accept-rom-license]~=0.6.0->stable-baselines3[extra]) (0.6.1)\n",
      "Requirement already satisfied: ale-py~=0.8.1 in c:\\users\\91701\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from shimmy[atari]~=0.2.1->stable-baselines3[extra]) (0.8.1)\n",
      "Requirement already satisfied: absl-py>=0.4 in c:\\users\\91701\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from tensorboard>=2.9.1->stable-baselines3[extra]) (1.4.0)\n",
      "Requirement already satisfied: grpcio>=1.48.2 in c:\\users\\91701\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from tensorboard>=2.9.1->stable-baselines3[extra]) (1.54.2)\n",
      "Requirement already satisfied: google-auth<3,>=1.6.3 in c:\\users\\91701\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from tensorboard>=2.9.1->stable-baselines3[extra]) (2.18.1)\n",
      "Requirement already satisfied: google-auth-oauthlib<1.1,>=0.5 in c:\\users\\91701\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from tensorboard>=2.9.1->stable-baselines3[extra]) (1.0.0)\n",
      "Requirement already satisfied: markdown>=2.6.8 in c:\\users\\91701\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from tensorboard>=2.9.1->stable-baselines3[extra]) (3.4.3)\n",
      "Requirement already satisfied: protobuf>=3.19.6 in c:\\users\\91701\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from tensorboard>=2.9.1->stable-baselines3[extra]) (4.23.1)\n",
      "Requirement already satisfied: setuptools>=41.0.0 in c:\\users\\91701\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from tensorboard>=2.9.1->stable-baselines3[extra]) (67.8.0)\n",
      "Requirement already satisfied: tensorboard-data-server<0.8.0,>=0.7.0 in c:\\users\\91701\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from tensorboard>=2.9.1->stable-baselines3[extra]) (0.7.0)\n",
      "Requirement already satisfied: werkzeug>=1.0.1 in c:\\users\\91701\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from tensorboard>=2.9.1->stable-baselines3[extra]) (2.3.4)\n",
      "Requirement already satisfied: wheel>=0.26 in c:\\users\\91701\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from tensorboard>=2.9.1->stable-baselines3[extra]) (0.40.0)\n",
      "Requirement already satisfied: filelock in c:\\users\\91701\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from torch>=1.11->stable-baselines3[extra]) (3.12.0)\n",
      "Requirement already satisfied: sympy in c:\\users\\91701\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from torch>=1.11->stable-baselines3[extra]) (1.12)\n",
      "Requirement already satisfied: networkx in c:\\users\\91701\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from torch>=1.11->stable-baselines3[extra]) (3.1)\n",
      "Requirement already satisfied: jinja2 in c:\\users\\91701\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from torch>=1.11->stable-baselines3[extra]) (3.1.2)\n",
      "Requirement already satisfied: contourpy>=1.0.1 in c:\\users\\91701\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from matplotlib->stable-baselines3[extra]) (1.0.7)\n",
      "Requirement already satisfied: cycler>=0.10 in c:\\users\\91701\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from matplotlib->stable-baselines3[extra]) (0.11.0)\n",
      "Requirement already satisfied: fonttools>=4.22.0 in c:\\users\\91701\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from matplotlib->stable-baselines3[extra]) (4.39.4)\n",
      "Requirement already satisfied: kiwisolver>=1.0.1 in c:\\users\\91701\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from matplotlib->stable-baselines3[extra]) (1.4.4)\n",
      "Requirement already satisfied: packaging>=20.0 in c:\\users\\91701\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from matplotlib->stable-baselines3[extra]) (23.1)\n",
      "Requirement already satisfied: pyparsing>=2.3.1 in c:\\users\\91701\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from matplotlib->stable-baselines3[extra]) (3.0.9)\n",
      "Requirement already satisfied: python-dateutil>=2.7 in c:\\users\\91701\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from matplotlib->stable-baselines3[extra]) (2.8.2)\n",
      "Requirement already satisfied: pytz>=2020.1 in c:\\users\\91701\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from pandas->stable-baselines3[extra]) (2023.3)\n",
      "Requirement already satisfied: tzdata>=2022.1 in c:\\users\\91701\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from pandas->stable-baselines3[extra]) (2023.3)\n",
      "Requirement already satisfied: markdown-it-py<3.0.0,>=2.2.0 in c:\\users\\91701\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from rich->stable-baselines3[extra]) (2.2.0)\n",
      "Requirement already satisfied: pygments<3.0.0,>=2.13.0 in c:\\users\\91701\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from rich->stable-baselines3[extra]) (2.15.1)\n",
      "Requirement already satisfied: colorama in c:\\users\\91701\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from tqdm->stable-baselines3[extra]) (0.4.6)\n",
      "Requirement already satisfied: importlib-resources in c:\\users\\91701\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from ale-py~=0.8.1->shimmy[atari]~=0.2.1->stable-baselines3[extra]) (5.12.0)\n",
      "Requirement already satisfied: cachetools<6.0,>=2.0.0 in c:\\users\\91701\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from google-auth<3,>=1.6.3->tensorboard>=2.9.1->stable-baselines3[extra]) (4.2.4)\n",
      "Requirement already satisfied: pyasn1-modules>=0.2.1 in c:\\users\\91701\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from google-auth<3,>=1.6.3->tensorboard>=2.9.1->stable-baselines3[extra]) (0.3.0)\n",
      "Requirement already satisfied: six>=1.9.0 in c:\\users\\91701\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from google-auth<3,>=1.6.3->tensorboard>=2.9.1->stable-baselines3[extra]) (1.16.0)\n",
      "Requirement already satisfied: urllib3<2.0 in c:\\users\\91701\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from google-auth<3,>=1.6.3->tensorboard>=2.9.1->stable-baselines3[extra]) (1.26.15)\n",
      "Requirement already satisfied: rsa<5,>=3.1.4 in c:\\users\\91701\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from google-auth<3,>=1.6.3->tensorboard>=2.9.1->stable-baselines3[extra]) (4.9)\n",
      "Requirement already satisfied: requests-oauthlib>=0.7.0 in c:\\users\\91701\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from google-auth-oauthlib<1.1,>=0.5->tensorboard>=2.9.1->stable-baselines3[extra]) (1.3.1)\n",
      "Requirement already satisfied: mdurl~=0.1 in c:\\users\\91701\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from markdown-it-py<3.0.0,>=2.2.0->rich->stable-baselines3[extra]) (0.1.2)\n",
      "Requirement already satisfied: charset-normalizer<4,>=2 in c:\\users\\91701\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from requests->autorom[accept-rom-license]~=0.6.0->stable-baselines3[extra]) (3.1.0)\n",
      "Requirement already satisfied: idna<4,>=2.5 in c:\\users\\91701\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from requests->autorom[accept-rom-license]~=0.6.0->stable-baselines3[extra]) (3.4)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in c:\\users\\91701\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from requests->autorom[accept-rom-license]~=0.6.0->stable-baselines3[extra]) (2023.5.7)\n",
      "Requirement already satisfied: MarkupSafe>=2.1.1 in c:\\users\\91701\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from werkzeug>=1.0.1->tensorboard>=2.9.1->stable-baselines3[extra]) (2.1.2)\n",
      "Requirement already satisfied: mpmath>=0.19 in c:\\users\\91701\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from sympy->torch>=1.11->stable-baselines3[extra]) (1.3.0)\n",
      "Requirement already satisfied: pyasn1<0.6.0,>=0.4.6 in c:\\users\\91701\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from pyasn1-modules>=0.2.1->google-auth<3,>=1.6.3->tensorboard>=2.9.1->stable-baselines3[extra]) (0.5.0)\n",
      "Requirement already satisfied: oauthlib>=3.0.0 in c:\\users\\91701\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from requests-oauthlib>=0.7.0->google-auth-oauthlib<1.1,>=0.5->tensorboard>=2.9.1->stable-baselines3[extra]) (3.2.2)\n"
     ]
    }
   ],
   "source": [
    "!pip install stable-baselines3[extra]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: stable_baselines3 in c:\\users\\91701\\anaconda3\\envs\\please\\lib\\site-packages (2.0.0a9)\n",
      "Requirement already satisfied: torch>=1.11 in c:\\users\\91701\\anaconda3\\envs\\please\\lib\\site-packages (from stable_baselines3) (2.0.1)\n",
      "Requirement already satisfied: pandas in c:\\users\\91701\\anaconda3\\envs\\please\\lib\\site-packages (from stable_baselines3) (2.0.1)\n",
      "Requirement already satisfied: cloudpickle in c:\\users\\91701\\anaconda3\\envs\\please\\lib\\site-packages (from stable_baselines3) (2.2.1)\n",
      "Requirement already satisfied: gymnasium==0.28.1 in c:\\users\\91701\\anaconda3\\envs\\please\\lib\\site-packages (from stable_baselines3) (0.28.1)\n",
      "Requirement already satisfied: matplotlib in c:\\users\\91701\\anaconda3\\envs\\please\\lib\\site-packages (from stable_baselines3) (3.7.1)\n",
      "Requirement already satisfied: numpy>=1.20 in c:\\users\\91701\\anaconda3\\envs\\please\\lib\\site-packages (from stable_baselines3) (1.24.3)\n",
      "Requirement already satisfied: jax-jumpy>=1.0.0 in c:\\users\\91701\\anaconda3\\envs\\please\\lib\\site-packages (from gymnasium==0.28.1->stable_baselines3) (1.0.0)\n",
      "Requirement already satisfied: typing-extensions>=4.3.0 in c:\\users\\91701\\anaconda3\\envs\\please\\lib\\site-packages (from gymnasium==0.28.1->stable_baselines3) (4.5.0)\n",
      "Requirement already satisfied: farama-notifications>=0.0.1 in c:\\users\\91701\\anaconda3\\envs\\please\\lib\\site-packages (from gymnasium==0.28.1->stable_baselines3) (0.0.4)\n",
      "Requirement already satisfied: filelock in c:\\users\\91701\\anaconda3\\envs\\please\\lib\\site-packages (from torch>=1.11->stable_baselines3) (3.12.0)\n",
      "Requirement already satisfied: sympy in c:\\users\\91701\\anaconda3\\envs\\please\\lib\\site-packages (from torch>=1.11->stable_baselines3) (1.12)\n",
      "Requirement already satisfied: networkx in c:\\users\\91701\\anaconda3\\envs\\please\\lib\\site-packages (from torch>=1.11->stable_baselines3) (3.1)\n",
      "Requirement already satisfied: jinja2 in c:\\users\\91701\\anaconda3\\envs\\please\\lib\\site-packages (from torch>=1.11->stable_baselines3) (3.1.2)\n",
      "Requirement already satisfied: pillow>=6.2.0 in c:\\users\\91701\\anaconda3\\envs\\please\\lib\\site-packages (from matplotlib->stable_baselines3) (9.5.0)\n",
      "Requirement already satisfied: packaging>=20.0 in c:\\users\\91701\\anaconda3\\envs\\please\\lib\\site-packages (from matplotlib->stable_baselines3) (23.1)\n",
      "Requirement already satisfied: python-dateutil>=2.7 in c:\\users\\91701\\anaconda3\\envs\\please\\lib\\site-packages (from matplotlib->stable_baselines3) (2.8.2)\n",
      "Requirement already satisfied: contourpy>=1.0.1 in c:\\users\\91701\\anaconda3\\envs\\please\\lib\\site-packages (from matplotlib->stable_baselines3) (1.0.7)\n",
      "Requirement already satisfied: fonttools>=4.22.0 in c:\\users\\91701\\anaconda3\\envs\\please\\lib\\site-packages (from matplotlib->stable_baselines3) (4.39.4)\n",
      "Requirement already satisfied: kiwisolver>=1.0.1 in c:\\users\\91701\\anaconda3\\envs\\please\\lib\\site-packages (from matplotlib->stable_baselines3) (1.4.4)\n",
      "Requirement already satisfied: cycler>=0.10 in c:\\users\\91701\\anaconda3\\envs\\please\\lib\\site-packages (from matplotlib->stable_baselines3) (0.11.0)\n",
      "Requirement already satisfied: pyparsing>=2.3.1 in c:\\users\\91701\\anaconda3\\envs\\please\\lib\\site-packages (from matplotlib->stable_baselines3) (3.0.9)\n",
      "Requirement already satisfied: pytz>=2020.1 in c:\\users\\91701\\anaconda3\\envs\\please\\lib\\site-packages (from pandas->stable_baselines3) (2023.3)\n",
      "Requirement already satisfied: tzdata>=2022.1 in c:\\users\\91701\\anaconda3\\envs\\please\\lib\\site-packages (from pandas->stable_baselines3) (2023.3)\n",
      "Requirement already satisfied: six>=1.5 in c:\\users\\91701\\anaconda3\\envs\\please\\lib\\site-packages (from python-dateutil>=2.7->matplotlib->stable_baselines3) (1.16.0)\n",
      "Requirement already satisfied: MarkupSafe>=2.0 in c:\\users\\91701\\anaconda3\\envs\\please\\lib\\site-packages (from jinja2->torch>=1.11->stable_baselines3) (2.1.2)\n",
      "Requirement already satisfied: mpmath>=0.19 in c:\\users\\91701\\anaconda3\\envs\\please\\lib\\site-packages (from sympy->torch>=1.11->stable_baselines3) (1.3.0)\n",
      "Note: you may need to restart the kernel to use updated packages.\n"
     ]
    }
   ],
   "source": [
    "pip install --upgrade stable_baselines3\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "import os \n",
    "\n",
    "from stable_baselines3.common.callbacks import BaseCallback"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "class TrainAndLoggingCallback(BaseCallback):\n",
    "\n",
    "    def __init__(self, check_freq, save_path, verbose=1):\n",
    "        super(TrainAndLoggingCallback, self).__init__(verbose)\n",
    "        self.check_freq = check_freq\n",
    "        self.save_path = save_path\n",
    "\n",
    "    def _init_callback(self):\n",
    "        if self.save_path is not None:\n",
    "            os.makedirs(self.save_path, exist_ok=True)\n",
    "\n",
    "    def _on_step(self):\n",
    "        if self.n_calls % self.check_freq == 0:\n",
    "            model_path = os.path.join(self.save_path, 'best_model_{}'.format(self.n_calls))\n",
    "            self.model.save(model_path)\n",
    "\n",
    "        return True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "CHECKPOINT_DIR = './train/train_basic'\n",
    "LOG_DIR = './logs/log_basic'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "callback = TrainAndLoggingCallback(check_freq=10000, save_path=CHECKPOINT_DIR)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 5. Train Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "from stable_baselines3 import PPO"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "env = VizDoomGym()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "ename": "ModuleNotFoundError",
     "evalue": "No module named 'tensorflow'",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mModuleNotFoundError\u001b[0m                       Traceback (most recent call last)",
      "Cell \u001b[1;32mIn[15], line 1\u001b[0m\n\u001b[1;32m----> 1\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mtensorflow\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m \u001b[38;5;21;01mtf\u001b[39;00m\n\u001b[0;32m      2\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mstable_baselines3\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m PPO\n\u001b[0;32m      3\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mgym\u001b[39;00m\n",
      "\u001b[1;31mModuleNotFoundError\u001b[0m: No module named 'tensorflow'"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "from stable_baselines3 import PPO\n",
    "import gym\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "Cell \u001b[1;32mIn[31], line 18\u001b[0m\n\u001b[0;32m     16\u001b[0m \u001b[38;5;66;03m# Create and train the PPO model\u001b[39;00m\n\u001b[0;32m     17\u001b[0m model \u001b[38;5;241m=\u001b[39m PPO(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mCnnPolicy\u001b[39m\u001b[38;5;124m'\u001b[39m, env, device\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mauto\u001b[39m\u001b[38;5;124m'\u001b[39m)\n\u001b[1;32m---> 18\u001b[0m \u001b[43mmodel\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mlearn\u001b[49m\u001b[43m(\u001b[49m\u001b[43mtotal_timesteps\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;241;43m10000\u001b[39;49m\u001b[43m)\u001b[49m\n",
      "File \u001b[1;32m~\\anaconda3\\envs\\please\\lib\\site-packages\\stable_baselines3\\ppo\\ppo.py:308\u001b[0m, in \u001b[0;36mPPO.learn\u001b[1;34m(self, total_timesteps, callback, log_interval, tb_log_name, reset_num_timesteps, progress_bar)\u001b[0m\n\u001b[0;32m    299\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mlearn\u001b[39m(\n\u001b[0;32m    300\u001b[0m     \u001b[38;5;28mself\u001b[39m: SelfPPO,\n\u001b[0;32m    301\u001b[0m     total_timesteps: \u001b[38;5;28mint\u001b[39m,\n\u001b[1;32m   (...)\u001b[0m\n\u001b[0;32m    306\u001b[0m     progress_bar: \u001b[38;5;28mbool\u001b[39m \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mFalse\u001b[39;00m,\n\u001b[0;32m    307\u001b[0m ) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m SelfPPO:\n\u001b[1;32m--> 308\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43msuper\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mlearn\u001b[49m\u001b[43m(\u001b[49m\n\u001b[0;32m    309\u001b[0m \u001b[43m        \u001b[49m\u001b[43mtotal_timesteps\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mtotal_timesteps\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    310\u001b[0m \u001b[43m        \u001b[49m\u001b[43mcallback\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcallback\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    311\u001b[0m \u001b[43m        \u001b[49m\u001b[43mlog_interval\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mlog_interval\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    312\u001b[0m \u001b[43m        \u001b[49m\u001b[43mtb_log_name\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mtb_log_name\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    313\u001b[0m \u001b[43m        \u001b[49m\u001b[43mreset_num_timesteps\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mreset_num_timesteps\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    314\u001b[0m \u001b[43m        \u001b[49m\u001b[43mprogress_bar\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mprogress_bar\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    315\u001b[0m \u001b[43m    \u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[1;32m~\\anaconda3\\envs\\please\\lib\\site-packages\\stable_baselines3\\common\\on_policy_algorithm.py:259\u001b[0m, in \u001b[0;36mOnPolicyAlgorithm.learn\u001b[1;34m(self, total_timesteps, callback, log_interval, tb_log_name, reset_num_timesteps, progress_bar)\u001b[0m\n\u001b[0;32m    256\u001b[0m \u001b[38;5;28;01massert\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39menv \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[0;32m    258\u001b[0m \u001b[38;5;28;01mwhile\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mnum_timesteps \u001b[38;5;241m<\u001b[39m total_timesteps:\n\u001b[1;32m--> 259\u001b[0m     continue_training \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mcollect_rollouts\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43menv\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcallback\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mrollout_buffer\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mn_rollout_steps\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mn_steps\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m    261\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m continue_training \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mFalse\u001b[39;00m:\n\u001b[0;32m    262\u001b[0m         \u001b[38;5;28;01mbreak\u001b[39;00m\n",
      "File \u001b[1;32m~\\anaconda3\\envs\\please\\lib\\site-packages\\stable_baselines3\\common\\on_policy_algorithm.py:178\u001b[0m, in \u001b[0;36mOnPolicyAlgorithm.collect_rollouts\u001b[1;34m(self, env, callback, rollout_buffer, n_rollout_steps)\u001b[0m\n\u001b[0;32m    175\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39maction_space, spaces\u001b[38;5;241m.\u001b[39mBox):\n\u001b[0;32m    176\u001b[0m     clipped_actions \u001b[38;5;241m=\u001b[39m np\u001b[38;5;241m.\u001b[39mclip(actions, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39maction_space\u001b[38;5;241m.\u001b[39mlow, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39maction_space\u001b[38;5;241m.\u001b[39mhigh)\n\u001b[1;32m--> 178\u001b[0m new_obs, rewards, dones, infos \u001b[38;5;241m=\u001b[39m \u001b[43menv\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mstep\u001b[49m\u001b[43m(\u001b[49m\u001b[43mclipped_actions\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m    180\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mnum_timesteps \u001b[38;5;241m+\u001b[39m\u001b[38;5;241m=\u001b[39m env\u001b[38;5;241m.\u001b[39mnum_envs\n\u001b[0;32m    182\u001b[0m \u001b[38;5;66;03m# Give access to local variables\u001b[39;00m\n",
      "File \u001b[1;32m~\\anaconda3\\envs\\please\\lib\\site-packages\\stable_baselines3\\common\\vec_env\\base_vec_env.py:180\u001b[0m, in \u001b[0;36mVecEnv.step\u001b[1;34m(self, actions)\u001b[0m\n\u001b[0;32m    173\u001b[0m \u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[0;32m    174\u001b[0m \u001b[38;5;124;03mStep the environments with the given action\u001b[39;00m\n\u001b[0;32m    175\u001b[0m \n\u001b[0;32m    176\u001b[0m \u001b[38;5;124;03m:param actions: the action\u001b[39;00m\n\u001b[0;32m    177\u001b[0m \u001b[38;5;124;03m:return: observation, reward, done, information\u001b[39;00m\n\u001b[0;32m    178\u001b[0m \u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[0;32m    179\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mstep_async(actions)\n\u001b[1;32m--> 180\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mstep_wait\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[1;32m~\\anaconda3\\envs\\please\\lib\\site-packages\\stable_baselines3\\common\\vec_env\\vec_transpose.py:95\u001b[0m, in \u001b[0;36mVecTransposeImage.step_wait\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m     94\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mstep_wait\u001b[39m(\u001b[38;5;28mself\u001b[39m) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m VecEnvStepReturn:\n\u001b[1;32m---> 95\u001b[0m     observations, rewards, dones, infos \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mvenv\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mstep_wait\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m     97\u001b[0m     \u001b[38;5;66;03m# Transpose the terminal observations\u001b[39;00m\n\u001b[0;32m     98\u001b[0m     \u001b[38;5;28;01mfor\u001b[39;00m idx, done \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28menumerate\u001b[39m(dones):\n",
      "File \u001b[1;32m~\\anaconda3\\envs\\please\\lib\\site-packages\\stable_baselines3\\common\\vec_env\\dummy_vec_env.py:58\u001b[0m, in \u001b[0;36mDummyVecEnv.step_wait\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m     55\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mstep_wait\u001b[39m(\u001b[38;5;28mself\u001b[39m) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m VecEnvStepReturn:\n\u001b[0;32m     56\u001b[0m     \u001b[38;5;66;03m# Avoid circular imports\u001b[39;00m\n\u001b[0;32m     57\u001b[0m     \u001b[38;5;28;01mfor\u001b[39;00m env_idx \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mrange\u001b[39m(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mnum_envs):\n\u001b[1;32m---> 58\u001b[0m         obs, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mbuf_rews[env_idx], terminated, truncated, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mbuf_infos[env_idx] \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43menvs\u001b[49m\u001b[43m[\u001b[49m\u001b[43menv_idx\u001b[49m\u001b[43m]\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mstep\u001b[49m\u001b[43m(\u001b[49m\n\u001b[0;32m     59\u001b[0m \u001b[43m            \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mactions\u001b[49m\u001b[43m[\u001b[49m\u001b[43menv_idx\u001b[49m\u001b[43m]\u001b[49m\n\u001b[0;32m     60\u001b[0m \u001b[43m        \u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m     61\u001b[0m         \u001b[38;5;66;03m# convert to SB3 VecEnv api\u001b[39;00m\n\u001b[0;32m     62\u001b[0m         \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mbuf_dones[env_idx] \u001b[38;5;241m=\u001b[39m terminated \u001b[38;5;129;01mor\u001b[39;00m truncated\n",
      "File \u001b[1;32m~\\anaconda3\\envs\\please\\lib\\site-packages\\stable_baselines3\\common\\monitor.py:94\u001b[0m, in \u001b[0;36mMonitor.step\u001b[1;34m(self, action)\u001b[0m\n\u001b[0;32m     92\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mneeds_reset:\n\u001b[0;32m     93\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mRuntimeError\u001b[39;00m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mTried to step environment that needs reset\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m---> 94\u001b[0m observation, reward, terminated, truncated, info \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43menv\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mstep\u001b[49m\u001b[43m(\u001b[49m\u001b[43maction\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m     95\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mrewards\u001b[38;5;241m.\u001b[39mappend(\u001b[38;5;28mfloat\u001b[39m(reward))\n\u001b[0;32m     96\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m terminated \u001b[38;5;129;01mor\u001b[39;00m truncated:\n",
      "File \u001b[1;32m~\\anaconda3\\envs\\please\\lib\\site-packages\\shimmy\\openai_gym_compatibility.py:255\u001b[0m, in \u001b[0;36mGymV21CompatibilityV0.step\u001b[1;34m(self, action)\u001b[0m\n\u001b[0;32m    246\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mstep\u001b[39m(\u001b[38;5;28mself\u001b[39m, action: ActType) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m \u001b[38;5;28mtuple\u001b[39m[Any, \u001b[38;5;28mfloat\u001b[39m, \u001b[38;5;28mbool\u001b[39m, \u001b[38;5;28mbool\u001b[39m, \u001b[38;5;28mdict\u001b[39m]:\n\u001b[0;32m    247\u001b[0m     \u001b[38;5;124;03m\"\"\"Steps through the environment.\u001b[39;00m\n\u001b[0;32m    248\u001b[0m \n\u001b[0;32m    249\u001b[0m \u001b[38;5;124;03m    Args:\u001b[39;00m\n\u001b[1;32m   (...)\u001b[0m\n\u001b[0;32m    253\u001b[0m \u001b[38;5;124;03m        (observation, reward, terminated, truncated, info)\u001b[39;00m\n\u001b[0;32m    254\u001b[0m \u001b[38;5;124;03m    \"\"\"\u001b[39;00m\n\u001b[1;32m--> 255\u001b[0m     obs, reward, done, info \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mgym_env\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mstep\u001b[49m\u001b[43m(\u001b[49m\u001b[43maction\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m    257\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mrender_mode \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[0;32m    258\u001b[0m         \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mrender()\n",
      "Cell \u001b[1;32mIn[13], line 28\u001b[0m, in \u001b[0;36mVizDoomGym.step\u001b[1;34m(self, action)\u001b[0m\n\u001b[0;32m     25\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mstep\u001b[39m(\u001b[38;5;28mself\u001b[39m, action):\n\u001b[0;32m     26\u001b[0m     \u001b[38;5;66;03m# Specify action and take step \u001b[39;00m\n\u001b[0;32m     27\u001b[0m     actions \u001b[38;5;241m=\u001b[39m np\u001b[38;5;241m.\u001b[39midentity(\u001b[38;5;241m3\u001b[39m)\n\u001b[1;32m---> 28\u001b[0m     reward \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mgame\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mmake_action\u001b[49m\u001b[43m(\u001b[49m\u001b[43mactions\u001b[49m\u001b[43m[\u001b[49m\u001b[43maction\u001b[49m\u001b[43m]\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m4\u001b[39;49m\u001b[43m)\u001b[49m \n\u001b[0;32m     30\u001b[0m     \u001b[38;5;66;03m# Get all the other stuff we need to retun \u001b[39;00m\n\u001b[0;32m     31\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mgame\u001b[38;5;241m.\u001b[39mget_state(): \n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "from stable_baselines3 import PPO\n",
    "import gym\n",
    "\n",
    "\n",
    "gpus = tf.config.list_physical_devices('GPU')\n",
    "if gpus:\n",
    "    try:\n",
    "        tf.config.experimental.set_memory_growth(gpus[0], True)\n",
    "    except RuntimeError as e:\n",
    "        print(e)\n",
    "\n",
    "\n",
    "env = VizDoomGym()\n",
    "\n",
    "\n",
    "model = PPO('CnnPolicy', env, device='auto')\n",
    "model.learn(total_timesteps=10000)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "ename": "ImportError",
     "evalue": "cannot import name 'VizDoomEnv' from 'stable_baselines3.common.envs' (C:\\Users\\91701\\anaconda3\\envs\\please\\lib\\site-packages\\stable_baselines3\\common\\envs\\__init__.py)",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mImportError\u001b[0m                               Traceback (most recent call last)",
      "Cell \u001b[1;32mIn[45], line 3\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mgym\u001b[39;00m\n\u001b[0;32m      2\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mstable_baselines3\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m PPO\n\u001b[1;32m----> 3\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mstable_baselines3\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mcommon\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01menvs\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m VizDoomEnv\n\u001b[0;32m      4\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mstable_baselines3\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mcommon\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mvec_env\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m DummyVecEnv\n\u001b[0;32m      5\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mtensorflow\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m \u001b[38;5;21;01mtf\u001b[39;00m\n",
      "\u001b[1;31mImportError\u001b[0m: cannot import name 'VizDoomEnv' from 'stable_baselines3.common.envs' (C:\\Users\\91701\\anaconda3\\envs\\please\\lib\\site-packages\\stable_baselines3\\common\\envs\\__init__.py)"
     ]
    }
   ],
   "source": [
    "import gym\n",
    "from stable_baselines3 import PPO\n",
    "from stable_baselines3.common.envs import VizDoomEnv\n",
    "from stable_baselines3.common.vec_env import DummyVecEnv\n",
    "import tensorflow as tf\n",
    "\n",
    "# Set the TensorFlow device to CPU\n",
    "tf.config.set_visible_devices([], 'GPU')\n",
    "\n",
    "# Create the Gym environment\n",
    "env = DummyVecEnv([lambda: VizDoomGym()])\n",
    "\n",
    "# Create and train the PPO model\n",
    "model = PPO('CnnPolicy', env, verbose=1, learning_rate=0.0001, n_steps=2048)\n",
    "# model.learn(total_timesteps=10000)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'PPO' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "Cell \u001b[1;32mIn[1], line 1\u001b[0m\n\u001b[1;32m----> 1\u001b[0m model \u001b[38;5;241m=\u001b[39m \u001b[43mPPO\u001b[49m(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mCnnPolicy\u001b[39m\u001b[38;5;124m'\u001b[39m, env, tensorboard_log\u001b[38;5;241m=\u001b[39mLOG_DIR, verbose\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m1\u001b[39m, learning_rate\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m0.0001\u001b[39m, n_steps\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m2048\u001b[39m, device\u001b[38;5;241m=\u001b[39mdevice)\n",
      "\u001b[1;31mNameError\u001b[0m: name 'PPO' is not defined"
     ]
    }
   ],
   "source": [
    "model = PPO('CnnPolicy', env, tensorboard_log=LOG_DIR, verbose=1, learning_rate=0.0001, n_steps=2048, device=device)\n",
    "# model.learn(total_timesteps=10000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Using cpu device\n",
      "Wrapping the env with a `Monitor` wrapper\n",
      "Wrapping the env in a DummyVecEnv.\n",
      "Wrapping the env in a VecTransposeImage.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\91701\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\stable_baselines3\\common\\vec_env\\patch_gym.py:49: UserWarning: You provided an OpenAI Gym environment. We strongly recommend transitioning to Gymnasium environments. Stable-Baselines3 is automatically wrapping your environments in a compatibility layer, which could potentially cause issues.\n",
      "  warnings.warn(\n"
     ]
    }
   ],
   "source": [
    "model = PPO('CnnPolicy', env, tensorboard_log=LOG_DIR, verbose=1, learning_rate=0.0001, n_steps=2048)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Logging to ./logs/log_basic\\PPO_4\n",
      "---------------------------------\n",
      "| rollout/           |          |\n",
      "|    ep_len_mean     | 26.2     |\n",
      "|    ep_rew_mean     | -41.6    |\n",
      "| time/              |          |\n",
      "|    fps             | 16       |\n",
      "|    iterations      | 1        |\n",
      "|    time_elapsed    | 121      |\n",
      "|    total_timesteps | 2048     |\n",
      "---------------------------------\n",
      "------------------------------------------\n",
      "| rollout/                |              |\n",
      "|    ep_len_mean          | 27.8         |\n",
      "|    ep_rew_mean          | -52.9        |\n",
      "| time/                   |              |\n",
      "|    fps                  | 14           |\n",
      "|    iterations           | 2            |\n",
      "|    time_elapsed         | 282          |\n",
      "|    total_timesteps      | 4096         |\n",
      "| train/                  |              |\n",
      "|    approx_kl            | 0.0038523844 |\n",
      "|    clip_fraction        | 0.0679       |\n",
      "|    clip_range           | 0.2          |\n",
      "|    entropy_loss         | -1.09        |\n",
      "|    explained_variance   | 6.99e-05     |\n",
      "|    learning_rate        | 0.0001       |\n",
      "|    loss                 | 1.04e+03     |\n",
      "|    n_updates            | 10           |\n",
      "|    policy_gradient_loss | 0.00148      |\n",
      "|    value_loss           | 2.89e+03     |\n",
      "------------------------------------------\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 30.2        |\n",
      "|    ep_rew_mean          | -75.9       |\n",
      "| time/                   |             |\n",
      "|    fps                  | 14          |\n",
      "|    iterations           | 3           |\n",
      "|    time_elapsed         | 434         |\n",
      "|    total_timesteps      | 6144        |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.014724802 |\n",
      "|    clip_fraction        | 0.141       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -1.08       |\n",
      "|    explained_variance   | 0.127       |\n",
      "|    learning_rate        | 0.0001      |\n",
      "|    loss                 | 1.07e+03    |\n",
      "|    n_updates            | 20          |\n",
      "|    policy_gradient_loss | 0.00152     |\n",
      "|    value_loss           | 2.92e+03    |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 30          |\n",
      "|    ep_rew_mean          | -71.3       |\n",
      "| time/                   |             |\n",
      "|    fps                  | 14          |\n",
      "|    iterations           | 4           |\n",
      "|    time_elapsed         | 584         |\n",
      "|    total_timesteps      | 8192        |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.014776904 |\n",
      "|    clip_fraction        | 0.182       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -1.07       |\n",
      "|    explained_variance   | 0.32        |\n",
      "|    learning_rate        | 0.0001      |\n",
      "|    loss                 | 1.11e+03    |\n",
      "|    n_updates            | 30          |\n",
      "|    policy_gradient_loss | -0.00199    |\n",
      "|    value_loss           | 2.38e+03    |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 26.9        |\n",
      "|    ep_rew_mean          | -47.7       |\n",
      "| time/                   |             |\n",
      "|    fps                  | 13          |\n",
      "|    iterations           | 5           |\n",
      "|    time_elapsed         | 739         |\n",
      "|    total_timesteps      | 10240       |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.023205355 |\n",
      "|    clip_fraction        | 0.232       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -1.06       |\n",
      "|    explained_variance   | 0.471       |\n",
      "|    learning_rate        | 0.0001      |\n",
      "|    loss                 | 1.64e+03    |\n",
      "|    n_updates            | 40          |\n",
      "|    policy_gradient_loss | -0.00342    |\n",
      "|    value_loss           | 2.97e+03    |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 26.3        |\n",
      "|    ep_rew_mean          | -42.6       |\n",
      "| time/                   |             |\n",
      "|    fps                  | 13          |\n",
      "|    iterations           | 6           |\n",
      "|    time_elapsed         | 905         |\n",
      "|    total_timesteps      | 12288       |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.008148033 |\n",
      "|    clip_fraction        | 0.207       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -0.974      |\n",
      "|    explained_variance   | 0.524       |\n",
      "|    learning_rate        | 0.0001      |\n",
      "|    loss                 | 1.35e+03    |\n",
      "|    n_updates            | 50          |\n",
      "|    policy_gradient_loss | -0.00131    |\n",
      "|    value_loss           | 2.95e+03    |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 19.5        |\n",
      "|    ep_rew_mean          | -6.49       |\n",
      "| time/                   |             |\n",
      "|    fps                  | 13          |\n",
      "|    iterations           | 7           |\n",
      "|    time_elapsed         | 1067        |\n",
      "|    total_timesteps      | 14336       |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.014340086 |\n",
      "|    clip_fraction        | 0.221       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -0.97       |\n",
      "|    explained_variance   | 0.622       |\n",
      "|    learning_rate        | 0.0001      |\n",
      "|    loss                 | 1.74e+03    |\n",
      "|    n_updates            | 60          |\n",
      "|    policy_gradient_loss | 0.00198     |\n",
      "|    value_loss           | 2.69e+03    |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 22.8        |\n",
      "|    ep_rew_mean          | -22.6       |\n",
      "| time/                   |             |\n",
      "|    fps                  | 13          |\n",
      "|    iterations           | 8           |\n",
      "|    time_elapsed         | 1228        |\n",
      "|    total_timesteps      | 16384       |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.018170951 |\n",
      "|    clip_fraction        | 0.306       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -0.888      |\n",
      "|    explained_variance   | 0.673       |\n",
      "|    learning_rate        | 0.0001      |\n",
      "|    loss                 | 1.86e+03    |\n",
      "|    n_updates            | 70          |\n",
      "|    policy_gradient_loss | 0.0106      |\n",
      "|    value_loss           | 2.78e+03    |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 19          |\n",
      "|    ep_rew_mean          | 5.3         |\n",
      "| time/                   |             |\n",
      "|    fps                  | 13          |\n",
      "|    iterations           | 9           |\n",
      "|    time_elapsed         | 1388        |\n",
      "|    total_timesteps      | 18432       |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.012238877 |\n",
      "|    clip_fraction        | 0.163       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -0.874      |\n",
      "|    explained_variance   | 0.668       |\n",
      "|    learning_rate        | 0.0001      |\n",
      "|    loss                 | 2.33e+03    |\n",
      "|    n_updates            | 80          |\n",
      "|    policy_gradient_loss | 0.00683     |\n",
      "|    value_loss           | 3e+03       |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 17.7        |\n",
      "|    ep_rew_mean          | 9.48        |\n",
      "| time/                   |             |\n",
      "|    fps                  | 13          |\n",
      "|    iterations           | 10          |\n",
      "|    time_elapsed         | 1547        |\n",
      "|    total_timesteps      | 20480       |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.019757453 |\n",
      "|    clip_fraction        | 0.296       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -0.919      |\n",
      "|    explained_variance   | 0.6         |\n",
      "|    learning_rate        | 0.0001      |\n",
      "|    loss                 | 2.09e+03    |\n",
      "|    n_updates            | 90          |\n",
      "|    policy_gradient_loss | 0.0176      |\n",
      "|    value_loss           | 3.77e+03    |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 11.5        |\n",
      "|    ep_rew_mean          | 47          |\n",
      "| time/                   |             |\n",
      "|    fps                  | 13          |\n",
      "|    iterations           | 11          |\n",
      "|    time_elapsed         | 1710        |\n",
      "|    total_timesteps      | 22528       |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.070085004 |\n",
      "|    clip_fraction        | 0.431       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -0.903      |\n",
      "|    explained_variance   | 0.418       |\n",
      "|    learning_rate        | 0.0001      |\n",
      "|    loss                 | 1.32e+03    |\n",
      "|    n_updates            | 100         |\n",
      "|    policy_gradient_loss | 0.00993     |\n",
      "|    value_loss           | 3.41e+03    |\n",
      "-----------------------------------------\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 20.8       |\n",
      "|    ep_rew_mean          | -7.79      |\n",
      "| time/                   |            |\n",
      "|    fps                  | 13         |\n",
      "|    iterations           | 12         |\n",
      "|    time_elapsed         | 1872       |\n",
      "|    total_timesteps      | 24576      |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.04211025 |\n",
      "|    clip_fraction        | 0.35       |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.813     |\n",
      "|    explained_variance   | 0.461      |\n",
      "|    learning_rate        | 0.0001     |\n",
      "|    loss                 | 1.72e+03   |\n",
      "|    n_updates            | 110        |\n",
      "|    policy_gradient_loss | 0.0305     |\n",
      "|    value_loss           | 3.22e+03   |\n",
      "----------------------------------------\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 14.6        |\n",
      "|    ep_rew_mean          | 24.4        |\n",
      "| time/                   |             |\n",
      "|    fps                  | 13          |\n",
      "|    iterations           | 13          |\n",
      "|    time_elapsed         | 2033        |\n",
      "|    total_timesteps      | 26624       |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.024705831 |\n",
      "|    clip_fraction        | 0.317       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -0.793      |\n",
      "|    explained_variance   | 0.545       |\n",
      "|    learning_rate        | 0.0001      |\n",
      "|    loss                 | 1.02e+03    |\n",
      "|    n_updates            | 120         |\n",
      "|    policy_gradient_loss | 0.022       |\n",
      "|    value_loss           | 2.42e+03    |\n",
      "-----------------------------------------\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 10.4       |\n",
      "|    ep_rew_mean          | 49.3       |\n",
      "| time/                   |            |\n",
      "|    fps                  | 13         |\n",
      "|    iterations           | 14         |\n",
      "|    time_elapsed         | 2195       |\n",
      "|    total_timesteps      | 28672      |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.03988925 |\n",
      "|    clip_fraction        | 0.271      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.78      |\n",
      "|    explained_variance   | 0.631      |\n",
      "|    learning_rate        | 0.0001     |\n",
      "|    loss                 | 1.14e+03   |\n",
      "|    n_updates            | 130        |\n",
      "|    policy_gradient_loss | 0.00653    |\n",
      "|    value_loss           | 2.4e+03    |\n",
      "----------------------------------------\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 10.7        |\n",
      "|    ep_rew_mean          | 51          |\n",
      "| time/                   |             |\n",
      "|    fps                  | 13          |\n",
      "|    iterations           | 15          |\n",
      "|    time_elapsed         | 2359        |\n",
      "|    total_timesteps      | 30720       |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.028933875 |\n",
      "|    clip_fraction        | 0.235       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -0.726      |\n",
      "|    explained_variance   | 0.742       |\n",
      "|    learning_rate        | 0.0001      |\n",
      "|    loss                 | 982         |\n",
      "|    n_updates            | 140         |\n",
      "|    policy_gradient_loss | 0.000455    |\n",
      "|    value_loss           | 2.12e+03    |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 11.5        |\n",
      "|    ep_rew_mean          | 47.3        |\n",
      "| time/                   |             |\n",
      "|    fps                  | 12          |\n",
      "|    iterations           | 16          |\n",
      "|    time_elapsed         | 2524        |\n",
      "|    total_timesteps      | 32768       |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.036457054 |\n",
      "|    clip_fraction        | 0.33        |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -0.7        |\n",
      "|    explained_variance   | 0.505       |\n",
      "|    learning_rate        | 0.0001      |\n",
      "|    loss                 | 708         |\n",
      "|    n_updates            | 150         |\n",
      "|    policy_gradient_loss | 0.0256      |\n",
      "|    value_loss           | 1.86e+03    |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 8.31        |\n",
      "|    ep_rew_mean          | 63.9        |\n",
      "| time/                   |             |\n",
      "|    fps                  | 12          |\n",
      "|    iterations           | 17          |\n",
      "|    time_elapsed         | 2690        |\n",
      "|    total_timesteps      | 34816       |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.028051306 |\n",
      "|    clip_fraction        | 0.312       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -0.661      |\n",
      "|    explained_variance   | 0.679       |\n",
      "|    learning_rate        | 0.0001      |\n",
      "|    loss                 | 835         |\n",
      "|    n_updates            | 160         |\n",
      "|    policy_gradient_loss | 0.0123      |\n",
      "|    value_loss           | 1.82e+03    |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 8           |\n",
      "|    ep_rew_mean          | 64.5        |\n",
      "| time/                   |             |\n",
      "|    fps                  | 12          |\n",
      "|    iterations           | 18          |\n",
      "|    time_elapsed         | 2856        |\n",
      "|    total_timesteps      | 36864       |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.051675197 |\n",
      "|    clip_fraction        | 0.334       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -0.597      |\n",
      "|    explained_variance   | 0.453       |\n",
      "|    learning_rate        | 0.0001      |\n",
      "|    loss                 | 565         |\n",
      "|    n_updates            | 170         |\n",
      "|    policy_gradient_loss | 0.0252      |\n",
      "|    value_loss           | 1.24e+03    |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 7.39        |\n",
      "|    ep_rew_mean          | 67.9        |\n",
      "| time/                   |             |\n",
      "|    fps                  | 12          |\n",
      "|    iterations           | 19          |\n",
      "|    time_elapsed         | 3027        |\n",
      "|    total_timesteps      | 38912       |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.024714755 |\n",
      "|    clip_fraction        | 0.227       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -0.51       |\n",
      "|    explained_variance   | 0.524       |\n",
      "|    learning_rate        | 0.0001      |\n",
      "|    loss                 | 184         |\n",
      "|    n_updates            | 180         |\n",
      "|    policy_gradient_loss | 0.0163      |\n",
      "|    value_loss           | 600         |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 5.84        |\n",
      "|    ep_rew_mean          | 76.4        |\n",
      "| time/                   |             |\n",
      "|    fps                  | 12          |\n",
      "|    iterations           | 20          |\n",
      "|    time_elapsed         | 3195        |\n",
      "|    total_timesteps      | 40960       |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.039291855 |\n",
      "|    clip_fraction        | 0.282       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -0.504      |\n",
      "|    explained_variance   | 0.523       |\n",
      "|    learning_rate        | 0.0001      |\n",
      "|    loss                 | 190         |\n",
      "|    n_updates            | 190         |\n",
      "|    policy_gradient_loss | 0.0143      |\n",
      "|    value_loss           | 475         |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 5.05        |\n",
      "|    ep_rew_mean          | 80.7        |\n",
      "| time/                   |             |\n",
      "|    fps                  | 12          |\n",
      "|    iterations           | 21          |\n",
      "|    time_elapsed         | 3363        |\n",
      "|    total_timesteps      | 43008       |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.059733562 |\n",
      "|    clip_fraction        | 0.252       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -0.494      |\n",
      "|    explained_variance   | 0.496       |\n",
      "|    learning_rate        | 0.0001      |\n",
      "|    loss                 | 124         |\n",
      "|    n_updates            | 200         |\n",
      "|    policy_gradient_loss | 0.0358      |\n",
      "|    value_loss           | 566         |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 5.09        |\n",
      "|    ep_rew_mean          | 80.9        |\n",
      "| time/                   |             |\n",
      "|    fps                  | 12          |\n",
      "|    iterations           | 22          |\n",
      "|    time_elapsed         | 3530        |\n",
      "|    total_timesteps      | 45056       |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.036866806 |\n",
      "|    clip_fraction        | 0.194       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -0.412      |\n",
      "|    explained_variance   | 0.511       |\n",
      "|    learning_rate        | 0.0001      |\n",
      "|    loss                 | 180         |\n",
      "|    n_updates            | 210         |\n",
      "|    policy_gradient_loss | 0.0141      |\n",
      "|    value_loss           | 288         |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 4.86        |\n",
      "|    ep_rew_mean          | 82.6        |\n",
      "| time/                   |             |\n",
      "|    fps                  | 12          |\n",
      "|    iterations           | 23          |\n",
      "|    time_elapsed         | 3696        |\n",
      "|    total_timesteps      | 47104       |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.054542426 |\n",
      "|    clip_fraction        | 0.23        |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -0.413      |\n",
      "|    explained_variance   | 0.586       |\n",
      "|    learning_rate        | 0.0001      |\n",
      "|    loss                 | 54.6        |\n",
      "|    n_updates            | 220         |\n",
      "|    policy_gradient_loss | 0.00726     |\n",
      "|    value_loss           | 185         |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 3.91        |\n",
      "|    ep_rew_mean          | 86.7        |\n",
      "| time/                   |             |\n",
      "|    fps                  | 12          |\n",
      "|    iterations           | 24          |\n",
      "|    time_elapsed         | 3863        |\n",
      "|    total_timesteps      | 49152       |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.045228794 |\n",
      "|    clip_fraction        | 0.182       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -0.387      |\n",
      "|    explained_variance   | 0.422       |\n",
      "|    learning_rate        | 0.0001      |\n",
      "|    loss                 | 151         |\n",
      "|    n_updates            | 230         |\n",
      "|    policy_gradient_loss | 0.0104      |\n",
      "|    value_loss           | 211         |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 3.92        |\n",
      "|    ep_rew_mean          | 87          |\n",
      "| time/                   |             |\n",
      "|    fps                  | 12          |\n",
      "|    iterations           | 25          |\n",
      "|    time_elapsed         | 4028        |\n",
      "|    total_timesteps      | 51200       |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.025672887 |\n",
      "|    clip_fraction        | 0.136       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -0.312      |\n",
      "|    explained_variance   | 0.561       |\n",
      "|    learning_rate        | 0.0001      |\n",
      "|    loss                 | 52          |\n",
      "|    n_updates            | 240         |\n",
      "|    policy_gradient_loss | 0.0246      |\n",
      "|    value_loss           | 139         |\n",
      "-----------------------------------------\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 5          |\n",
      "|    ep_rew_mean          | 81.4       |\n",
      "| time/                   |            |\n",
      "|    fps                  | 12         |\n",
      "|    iterations           | 26         |\n",
      "|    time_elapsed         | 4187       |\n",
      "|    total_timesteps      | 53248      |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.02455904 |\n",
      "|    clip_fraction        | 0.11       |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.273     |\n",
      "|    explained_variance   | 0.766      |\n",
      "|    learning_rate        | 0.0001     |\n",
      "|    loss                 | 20.7       |\n",
      "|    n_updates            | 250        |\n",
      "|    policy_gradient_loss | 0.0443     |\n",
      "|    value_loss           | 38.2       |\n",
      "----------------------------------------\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 4.07       |\n",
      "|    ep_rew_mean          | 86.4       |\n",
      "| time/                   |            |\n",
      "|    fps                  | 12         |\n",
      "|    iterations           | 27         |\n",
      "|    time_elapsed         | 4345       |\n",
      "|    total_timesteps      | 55296      |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.04684023 |\n",
      "|    clip_fraction        | 0.132      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.203     |\n",
      "|    explained_variance   | 0.567      |\n",
      "|    learning_rate        | 0.0001     |\n",
      "|    loss                 | 64         |\n",
      "|    n_updates            | 260        |\n",
      "|    policy_gradient_loss | 0.00218    |\n",
      "|    value_loss           | 96.6       |\n",
      "----------------------------------------\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 4.4         |\n",
      "|    ep_rew_mean          | 85.3        |\n",
      "| time/                   |             |\n",
      "|    fps                  | 12          |\n",
      "|    iterations           | 28          |\n",
      "|    time_elapsed         | 4507        |\n",
      "|    total_timesteps      | 57344       |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.020744763 |\n",
      "|    clip_fraction        | 0.0671      |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -0.208      |\n",
      "|    explained_variance   | 0.694       |\n",
      "|    learning_rate        | 0.0001      |\n",
      "|    loss                 | 64.7        |\n",
      "|    n_updates            | 270         |\n",
      "|    policy_gradient_loss | 0.0184      |\n",
      "|    value_loss           | 50.9        |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 3.72        |\n",
      "|    ep_rew_mean          | 87.9        |\n",
      "| time/                   |             |\n",
      "|    fps                  | 12          |\n",
      "|    iterations           | 29          |\n",
      "|    time_elapsed         | 4679        |\n",
      "|    total_timesteps      | 59392       |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.021920828 |\n",
      "|    clip_fraction        | 0.0835      |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -0.177      |\n",
      "|    explained_variance   | 0.687       |\n",
      "|    learning_rate        | 0.0001      |\n",
      "|    loss                 | 41.9        |\n",
      "|    n_updates            | 280         |\n",
      "|    policy_gradient_loss | 0.0118      |\n",
      "|    value_loss           | 64.1        |\n",
      "-----------------------------------------\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 3.76       |\n",
      "|    ep_rew_mean          | 87.7       |\n",
      "| time/                   |            |\n",
      "|    fps                  | 12         |\n",
      "|    iterations           | 30         |\n",
      "|    time_elapsed         | 4837       |\n",
      "|    total_timesteps      | 61440      |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.02073367 |\n",
      "|    clip_fraction        | 0.0794     |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.166     |\n",
      "|    explained_variance   | 0.69       |\n",
      "|    learning_rate        | 0.0001     |\n",
      "|    loss                 | 16.5       |\n",
      "|    n_updates            | 290        |\n",
      "|    policy_gradient_loss | 0.0147     |\n",
      "|    value_loss           | 54         |\n",
      "----------------------------------------\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 4.09        |\n",
      "|    ep_rew_mean          | 86.6        |\n",
      "| time/                   |             |\n",
      "|    fps                  | 12          |\n",
      "|    iterations           | 31          |\n",
      "|    time_elapsed         | 5002        |\n",
      "|    total_timesteps      | 63488       |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.016004087 |\n",
      "|    clip_fraction        | 0.0713      |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -0.138      |\n",
      "|    explained_variance   | 0.867       |\n",
      "|    learning_rate        | 0.0001      |\n",
      "|    loss                 | 12.8        |\n",
      "|    n_updates            | 300         |\n",
      "|    policy_gradient_loss | 0.00606     |\n",
      "|    value_loss           | 21.2        |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 3.69        |\n",
      "|    ep_rew_mean          | 87.8        |\n",
      "| time/                   |             |\n",
      "|    fps                  | 12          |\n",
      "|    iterations           | 32          |\n",
      "|    time_elapsed         | 5161        |\n",
      "|    total_timesteps      | 65536       |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.014132462 |\n",
      "|    clip_fraction        | 0.0471      |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -0.118      |\n",
      "|    explained_variance   | 0.792       |\n",
      "|    learning_rate        | 0.0001      |\n",
      "|    loss                 | 16.9        |\n",
      "|    n_updates            | 310         |\n",
      "|    policy_gradient_loss | 0.00879     |\n",
      "|    value_loss           | 47.4        |\n",
      "-----------------------------------------\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 3.61       |\n",
      "|    ep_rew_mean          | 88.5       |\n",
      "| time/                   |            |\n",
      "|    fps                  | 12         |\n",
      "|    iterations           | 33         |\n",
      "|    time_elapsed         | 5327       |\n",
      "|    total_timesteps      | 67584      |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.20815942 |\n",
      "|    clip_fraction        | 0.0813     |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.103     |\n",
      "|    explained_variance   | 0.641      |\n",
      "|    learning_rate        | 0.0001     |\n",
      "|    loss                 | 12.5       |\n",
      "|    n_updates            | 320        |\n",
      "|    policy_gradient_loss | 0.00171    |\n",
      "|    value_loss           | 48.6       |\n",
      "----------------------------------------\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 3.99        |\n",
      "|    ep_rew_mean          | 86.8        |\n",
      "| time/                   |             |\n",
      "|    fps                  | 12          |\n",
      "|    iterations           | 34          |\n",
      "|    time_elapsed         | 5491        |\n",
      "|    total_timesteps      | 69632       |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.022235364 |\n",
      "|    clip_fraction        | 0.0739      |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -0.154      |\n",
      "|    explained_variance   | 0.841       |\n",
      "|    learning_rate        | 0.0001      |\n",
      "|    loss                 | 3.57        |\n",
      "|    n_updates            | 330         |\n",
      "|    policy_gradient_loss | 0.0227      |\n",
      "|    value_loss           | 18.2        |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 3.91        |\n",
      "|    ep_rew_mean          | 86.9        |\n",
      "| time/                   |             |\n",
      "|    fps                  | 12          |\n",
      "|    iterations           | 35          |\n",
      "|    time_elapsed         | 5656        |\n",
      "|    total_timesteps      | 71680       |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.027310528 |\n",
      "|    clip_fraction        | 0.0829      |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -0.128      |\n",
      "|    explained_variance   | 0.729       |\n",
      "|    learning_rate        | 0.0001      |\n",
      "|    loss                 | 13.6        |\n",
      "|    n_updates            | 340         |\n",
      "|    policy_gradient_loss | 0.00838     |\n",
      "|    value_loss           | 54.9        |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 3.84        |\n",
      "|    ep_rew_mean          | 87.6        |\n",
      "| time/                   |             |\n",
      "|    fps                  | 12          |\n",
      "|    iterations           | 36          |\n",
      "|    time_elapsed         | 5816        |\n",
      "|    total_timesteps      | 73728       |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.033747293 |\n",
      "|    clip_fraction        | 0.128       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -0.142      |\n",
      "|    explained_variance   | 0.763       |\n",
      "|    learning_rate        | 0.0001      |\n",
      "|    loss                 | 13          |\n",
      "|    n_updates            | 350         |\n",
      "|    policy_gradient_loss | 0.0316      |\n",
      "|    value_loss           | 36.3        |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 3.72        |\n",
      "|    ep_rew_mean          | 88.1        |\n",
      "| time/                   |             |\n",
      "|    fps                  | 12          |\n",
      "|    iterations           | 37          |\n",
      "|    time_elapsed         | 5982        |\n",
      "|    total_timesteps      | 75776       |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.015444106 |\n",
      "|    clip_fraction        | 0.0683      |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -0.0895     |\n",
      "|    explained_variance   | 0.929       |\n",
      "|    learning_rate        | 0.0001      |\n",
      "|    loss                 | 3.01        |\n",
      "|    n_updates            | 360         |\n",
      "|    policy_gradient_loss | 0.00595     |\n",
      "|    value_loss           | 8.52        |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 3.9         |\n",
      "|    ep_rew_mean          | 87          |\n",
      "| time/                   |             |\n",
      "|    fps                  | 12          |\n",
      "|    iterations           | 38          |\n",
      "|    time_elapsed         | 6152        |\n",
      "|    total_timesteps      | 77824       |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.025757907 |\n",
      "|    clip_fraction        | 0.0462      |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -0.0825     |\n",
      "|    explained_variance   | 0.845       |\n",
      "|    learning_rate        | 0.0001      |\n",
      "|    loss                 | 4.21        |\n",
      "|    n_updates            | 370         |\n",
      "|    policy_gradient_loss | 0.00851     |\n",
      "|    value_loss           | 22.5        |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 4.07        |\n",
      "|    ep_rew_mean          | 86.3        |\n",
      "| time/                   |             |\n",
      "|    fps                  | 12          |\n",
      "|    iterations           | 39          |\n",
      "|    time_elapsed         | 6317        |\n",
      "|    total_timesteps      | 79872       |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.030794248 |\n",
      "|    clip_fraction        | 0.0494      |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -0.0764     |\n",
      "|    explained_variance   | 0.763       |\n",
      "|    learning_rate        | 0.0001      |\n",
      "|    loss                 | 13.9        |\n",
      "|    n_updates            | 380         |\n",
      "|    policy_gradient_loss | 0.00723     |\n",
      "|    value_loss           | 40.3        |\n",
      "-----------------------------------------\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 3.94       |\n",
      "|    ep_rew_mean          | 86.7       |\n",
      "| time/                   |            |\n",
      "|    fps                  | 12         |\n",
      "|    iterations           | 40         |\n",
      "|    time_elapsed         | 6491       |\n",
      "|    total_timesteps      | 81920      |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.04789641 |\n",
      "|    clip_fraction        | 0.0681     |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.0893    |\n",
      "|    explained_variance   | 0.673      |\n",
      "|    learning_rate        | 0.0001     |\n",
      "|    loss                 | 59.4       |\n",
      "|    n_updates            | 390        |\n",
      "|    policy_gradient_loss | 0.0146     |\n",
      "|    value_loss           | 51.9       |\n",
      "----------------------------------------\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 4.39        |\n",
      "|    ep_rew_mean          | 85.1        |\n",
      "| time/                   |             |\n",
      "|    fps                  | 12          |\n",
      "|    iterations           | 41          |\n",
      "|    time_elapsed         | 6666        |\n",
      "|    total_timesteps      | 83968       |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.030988967 |\n",
      "|    clip_fraction        | 0.0593      |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -0.0783     |\n",
      "|    explained_variance   | 0.694       |\n",
      "|    learning_rate        | 0.0001      |\n",
      "|    loss                 | 27.8        |\n",
      "|    n_updates            | 400         |\n",
      "|    policy_gradient_loss | 0.031       |\n",
      "|    value_loss           | 50          |\n",
      "-----------------------------------------\n",
      "------------------------------------------\n",
      "| rollout/                |              |\n",
      "|    ep_len_mean          | 4.02         |\n",
      "|    ep_rew_mean          | 86.8         |\n",
      "| time/                   |              |\n",
      "|    fps                  | 12           |\n",
      "|    iterations           | 42           |\n",
      "|    time_elapsed         | 6836         |\n",
      "|    total_timesteps      | 86016        |\n",
      "| train/                  |              |\n",
      "|    approx_kl            | 0.0122012645 |\n",
      "|    clip_fraction        | 0.0624       |\n",
      "|    clip_range           | 0.2          |\n",
      "|    entropy_loss         | -0.0837      |\n",
      "|    explained_variance   | 0.724        |\n",
      "|    learning_rate        | 0.0001       |\n",
      "|    loss                 | 36.2         |\n",
      "|    n_updates            | 410          |\n",
      "|    policy_gradient_loss | -0.000233    |\n",
      "|    value_loss           | 47.7         |\n",
      "------------------------------------------\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 3.86        |\n",
      "|    ep_rew_mean          | 87.1        |\n",
      "| time/                   |             |\n",
      "|    fps                  | 12          |\n",
      "|    iterations           | 43          |\n",
      "|    time_elapsed         | 7004        |\n",
      "|    total_timesteps      | 88064       |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.023504965 |\n",
      "|    clip_fraction        | 0.0414      |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -0.0746     |\n",
      "|    explained_variance   | 0.893       |\n",
      "|    learning_rate        | 0.0001      |\n",
      "|    loss                 | 6.91        |\n",
      "|    n_updates            | 420         |\n",
      "|    policy_gradient_loss | 0.0222      |\n",
      "|    value_loss           | 14.1        |\n",
      "-----------------------------------------\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 3.81       |\n",
      "|    ep_rew_mean          | 87.4       |\n",
      "| time/                   |            |\n",
      "|    fps                  | 12         |\n",
      "|    iterations           | 44         |\n",
      "|    time_elapsed         | 7173       |\n",
      "|    total_timesteps      | 90112      |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.02306491 |\n",
      "|    clip_fraction        | 0.0635     |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.0762    |\n",
      "|    explained_variance   | 0.825      |\n",
      "|    learning_rate        | 0.0001     |\n",
      "|    loss                 | 14.6       |\n",
      "|    n_updates            | 430        |\n",
      "|    policy_gradient_loss | 0.00996    |\n",
      "|    value_loss           | 25.9       |\n",
      "----------------------------------------\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 4.04        |\n",
      "|    ep_rew_mean          | 86.6        |\n",
      "| time/                   |             |\n",
      "|    fps                  | 12          |\n",
      "|    iterations           | 45          |\n",
      "|    time_elapsed         | 7342        |\n",
      "|    total_timesteps      | 92160       |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.034339607 |\n",
      "|    clip_fraction        | 0.0548      |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -0.0675     |\n",
      "|    explained_variance   | 0.818       |\n",
      "|    learning_rate        | 0.0001      |\n",
      "|    loss                 | 6.97        |\n",
      "|    n_updates            | 440         |\n",
      "|    policy_gradient_loss | 0.0114      |\n",
      "|    value_loss           | 27.1        |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 4.36        |\n",
      "|    ep_rew_mean          | 85.2        |\n",
      "| time/                   |             |\n",
      "|    fps                  | 12          |\n",
      "|    iterations           | 46          |\n",
      "|    time_elapsed         | 7507        |\n",
      "|    total_timesteps      | 94208       |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.029521972 |\n",
      "|    clip_fraction        | 0.0424      |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -0.0616     |\n",
      "|    explained_variance   | 0.786       |\n",
      "|    learning_rate        | 0.0001      |\n",
      "|    loss                 | 26          |\n",
      "|    n_updates            | 450         |\n",
      "|    policy_gradient_loss | -0.00385    |\n",
      "|    value_loss           | 27.4        |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 3.67        |\n",
      "|    ep_rew_mean          | 88.3        |\n",
      "| time/                   |             |\n",
      "|    fps                  | 12          |\n",
      "|    iterations           | 47          |\n",
      "|    time_elapsed         | 7681        |\n",
      "|    total_timesteps      | 96256       |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.024536945 |\n",
      "|    clip_fraction        | 0.0486      |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -0.0675     |\n",
      "|    explained_variance   | 0.813       |\n",
      "|    learning_rate        | 0.0001      |\n",
      "|    loss                 | 8.68        |\n",
      "|    n_updates            | 460         |\n",
      "|    policy_gradient_loss | 0.0121      |\n",
      "|    value_loss           | 20.6        |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 3.94        |\n",
      "|    ep_rew_mean          | 87.2        |\n",
      "| time/                   |             |\n",
      "|    fps                  | 12          |\n",
      "|    iterations           | 48          |\n",
      "|    time_elapsed         | 7845        |\n",
      "|    total_timesteps      | 98304       |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.029442247 |\n",
      "|    clip_fraction        | 0.0464      |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -0.0725     |\n",
      "|    explained_variance   | 0.812       |\n",
      "|    learning_rate        | 0.0001      |\n",
      "|    loss                 | 4.71        |\n",
      "|    n_updates            | 470         |\n",
      "|    policy_gradient_loss | 0.014       |\n",
      "|    value_loss           | 27.2        |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 4.36        |\n",
      "|    ep_rew_mean          | 85.5        |\n",
      "| time/                   |             |\n",
      "|    fps                  | 12          |\n",
      "|    iterations           | 49          |\n",
      "|    time_elapsed         | 8017        |\n",
      "|    total_timesteps      | 100352      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.026748152 |\n",
      "|    clip_fraction        | 0.0584      |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -0.0722     |\n",
      "|    explained_variance   | 0.818       |\n",
      "|    learning_rate        | 0.0001      |\n",
      "|    loss                 | 13.6        |\n",
      "|    n_updates            | 480         |\n",
      "|    policy_gradient_loss | 0.00494     |\n",
      "|    value_loss           | 24.6        |\n",
      "-----------------------------------------\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<stable_baselines3.ppo.ppo.PPO at 0x121bebfaf10>"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.learn(total_timesteps=100000, callback=callback)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 6. Test the Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "from stable_baselines3.common.evaluation import evaluate_policy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "model = PPO.load('./train/train_basic/best_model_60000')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "env = VizDoomGym(render=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\91701\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\stable_baselines3\\common\\vec_env\\patch_gym.py:49: UserWarning: You provided an OpenAI Gym environment. We strongly recommend transitioning to Gymnasium environments. Stable-Baselines3 is automatically wrapping your environments in a compatibility layer, which could potentially cause issues.\n",
      "  warnings.warn(\n",
      "C:\\Users\\91701\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\stable_baselines3\\common\\evaluation.py:67: UserWarning: Evaluation environment is not wrapped with a ``Monitor`` wrapper. This may result in reporting modified episode lengths and rewards, if other wrappers happen to modify these. Consider wrapping environment first with ``Monitor`` wrapper.\n",
      "  warnings.warn(\n"
     ]
    }
   ],
   "source": [
    "\n",
    "mean_reward, _ = evaluate_policy(model, env, n_eval_episodes=100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "89.13"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mean_reward"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'obs' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "Cell \u001b[1;32mIn[32], line 1\u001b[0m\n\u001b[1;32m----> 1\u001b[0m model\u001b[38;5;241m.\u001b[39mpredict(\u001b[43mobs\u001b[49m)\n",
      "\u001b[1;31mNameError\u001b[0m: name 'obs' is not defined"
     ]
    }
   ],
   "source": [
    "model.predict(obs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total Reward for episode 0 is 91.0\n",
      "Total Reward for episode 1 is 91.0\n",
      "Total Reward for episode 2 is 91.0\n",
      "Total Reward for episode 3 is 95.0\n",
      "Total Reward for episode 4 is 95.0\n",
      "Total Reward for episode 5 is 67.0\n",
      "Total Reward for episode 6 is 95.0\n",
      "Total Reward for episode 7 is 91.0\n",
      "Total Reward for episode 8 is 56.0\n",
      "Total Reward for episode 9 is 95.0\n"
     ]
    }
   ],
   "source": [
    "for episode in range(10): \n",
    "    obs = env.reset()\n",
    "    done = False\n",
    "    total_reward = 0\n",
    "    while not done: \n",
    "        action, _ = model.predict(obs)\n",
    "        obs, reward, done, info = env.step(action)\n",
    "        time.sleep(0.20)\n",
    "        total_reward += reward\n",
    "    print('Total Reward for episode {} is {}'.format(episode,total_reward))\n",
    "    time.sleep(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}