notes/05_policy_gradient/cartpole_reinforce.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Policy Gradient\n",
    "\n",
    "In this notebook, we will implement REINFORCE algorithm to play Catpole.\n",
    "\n",
    "## Step 1: Import the libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import time\n",
    "import random\n",
    "import gym\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from collections import namedtuple, deque"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "import torch.nn.functional as F\n",
    "import torchvision.transforms as T\n",
    "from torch.distributions import Categorical"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 2: Create our environment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\users\\deepanshu\\.conda\\envs\\ai\\lib\\site-packages\\gym\\logger.py:30: UserWarning: \u001b[33mWARN: Box bound precision lowered by casting to float32\u001b[0m\n",
      "  warnings.warn(colorize('%s: %s'%('WARN', msg % args), 'yellow'))\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[1]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# if gpu is to be used\n",
    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "\n",
    "env = gym.make('CartPole-v1')\n",
    "env.seed(1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 3: Viewing our Enviroment\n",
    "Execute the code cell below to play Cartpole with a random policy."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The size of state is:  (4,)\n",
      "No. of Actions:  2\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAW4AAAEICAYAAAB/Dx7IAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAVEklEQVR4nO3df5Bd5X3f8ffHQgbH4CCihRGSiFQs2ghPI+Id1Q1pS4NjCK0rnKkzchtHmSEjpoXUbt3WyJ460KmmTsbY/WkPcmCiCbYV+QdF8TixMbXHZYYgrzC/BMgoRoAsFQkIARJXscS3f9wjcy1W2ru/WD2779fMnXvuc55z7verkT57ePZcbqoKSVI7XjfTBUiSxsfglqTGGNyS1BiDW5IaY3BLUmMMbklqjMGtk1aSDyX5vameO8C5Ksmbp+Jc0nSI93HrtZDkN4APAOcDLwC3ARuq6vmZrGs0SQpYUVW7R9n3TeBtwOG+4V+qqrtfo/Ikr7g1/ZJ8APgd4N8BP0kv+H4auCPJ649zzCmvXYXjdm1Vnd73eFVon+T1q3EGt6ZVkjcBNwC/VVV/UlU/rKo9wK/SC+9f6+Zdn+QLSW5N8gLwG93YrX3n+vUkTyR5Nsl/SLInydv7jr+1217WLXesS/JkkmeSfLjvPKuT3J3k+ST7k/yP4/0AGWevleSaJI8Bj3Vj/zXJU0leSLIjyd/rm399ks93Pb+Y5MEkFyTZkORAd9w7+ub/ZJKbu5q/n+Q/JZk32brVHoNb0+3ngdOAL/UPVtVLwB8Dv9Q3vAb4AnAm8Jn++UlWAp8E/jmwiN6V++Ix3vsXgL8JXAp8JMnPdONHgH8NLAT+brf/X46zr+O5Evg7wMru9beBVcBZwGeBzyc5rW/+O4E/ABYA3wG+Su/f5WLgPwI39c3dTG+J5s3ARcA7gN+corrVEINb020h8ExVHR5l3/5u/1F3V9X/qqqXq+oHx8z9p8AfVdVdVfXXwEeAsX5Bc0NV/aCq7gfuB34WoKp2VNWfVtXh7ur/JuAfjKOn/9ZdrT+f5N5j9v3nqnruaP1VdWtVPdu9143AqfR+mBz1f6rqq92fz+eBIeCjVfVDYAuwLMmZSc4Bfhl4f1X9ZVUdAD4BrB1H3ZolXIfTdHsGWJjklFHCe1G3/6inTnCec/v3V9VfJXl2jPf+v33bfwWcDpDkAuDjwDDwE/T+HewY41z9/lVVHe8Olh/roVvf/82u/gLexI//sHq6b/sH9H7IHel7TVf3ucB8YH+So/Nfd+z7aW7wilvT7W7gEPAr/YNJ3kjvCvLOvuETXUHvB5b0Hf8G4KcmWNOngEfp3TnyJuBDQE58yMB+1EO3nv1Beuv5C6rqTOAvJvheT9H7c1xYVWd2jzdV1YVTUbTaYnBrWlXVX9D75eR/T3J5kvlJltFbFthLb313EF8A3pnk57tfJN7AxMP2DHq3JL6U5G8B/2KC5xnkfQ4DB4FTknyE3hX3uFXVfuBrwI1J3pTkdUnOTzKeJR7NEga3pl1V/S69q9qP0QvMe+hdQV5aVYcGPMdO4LforfvuB14EDtC7Ch2vfwv8s+4cnwb+cALnGMRX6f0C9rvAE8D/Y3JLG78OvB54GPhzej/MFk2yRjXID+CoSUlOB56nt9zx+EzXI72WvOJWM5K8M8lPdOvjHwMeBPbMbFXSa2/agrtbz9yVZHeS66brfTSnrAH2dY8VwNryPxk1B03LUkn3aa7v0vtwxV56H0J4T1U9POVvJklzzHRdca8GdlfV97oPS2yhd7UkSZqk6foAzmJ+/Lfne+l9DHhUCxcurGXLlk1TKZLUnj179vDMM8+MesvrdAX3aG/2Y2sySdYD6wHOO+88RkZGpqkUSWrP8PDwcfdN11LJXmBp3+sl9H6h9CNVtamqhqtqeGhoaJrKkKTZZ7qC+9vAiiTLu0+5rQW2TdN7SdKcMi1LJVV1OMm19D45Ng+4pfvkmyRpkqbt/w5YVV8BvjJd55ekucpPTkpSYwxuSWqMwS1JjTG4JakxBrckNcbglqTGGNyS1BiDW5IaY3BLUmMMbklqjMEtSY0xuCWpMQa3JDXG4JakxhjcktQYg1uSGmNwS1JjDG5JaozBLUmNmdR3TibZA7wIHAEOV9VwkrOAPwSWAXuAX62qP59cmZKko6biivsfVtWqqhruXl8H3FlVK4A7u9eSpCkyHUsla4DN3fZm4MppeA9JmrMmG9wFfC3JjiTru7Fzqmo/QPd89mgHJlmfZCTJyMGDBydZhiTNHZNa4wYurqp9Sc4G7kjy6KAHVtUmYBPA8PBwTbIOSZozJnXFXVX7uucDwG3AauDpJIsAuucDky1SkvSKCQd3kjcmOePoNvAO4CFgG7Cum7YOuH2yRUqSXjGZpZJzgNuSHD3PZ6vqT5J8G9ia5CrgSeDdky9TknTUhIO7qr4H/Owo488Cl06mKEnS8fnJSUlqjMEtSY0xuCWpMQa3JDXG4JakxhjcktQYg1uSGmNwS1JjDG5JaozBLUmNMbglqTEGtyQ1xuCWpMYY3JLUGINbkhpjcEtSYwxuSWqMwS1JjRkzuJPckuRAkof6xs5KckeSx7rnBX37NiTZnWRXksumq3BJmqsGueL+feDyY8auA+6sqhXAnd1rkqwE1gIXdsd8Msm8KatWkjR2cFfVt4DnjhleA2zutjcDV/aNb6mqQ1X1OLAbWD1FtUqSmPga9zlVtR+gez67G18MPNU3b2839ipJ1icZSTJy8ODBCZYhSXPPVP9yMqOM1WgTq2pTVQ1X1fDQ0NAUlyFJs9dEg/vpJIsAuucD3fheYGnfvCXAvomXJ0k61kSDexuwrtteB9zeN742yalJlgMrgO2TK1GS1O+UsSYk+RxwCbAwyV7gt4GPAluTXAU8CbwboKp2JtkKPAwcBq6pqiPTVLskzUljBndVvec4uy49zvyNwMbJFCVJOj4/OSlJjTG4JakxBrckNcbglqTGGNyS1BiDW5IaY3BLUmMMbklqjMEtSY0xuCWpMQa3JDXG4JakxhjcktQYg1uSGmNwS1JjDG5JaozBLUmNMbglqTFjBneSW5IcSPJQ39j1Sb6f5L7ucUXfvg1JdifZleSy6SpckuaqQa64fx+4fJTxT1TVqu7xFYAkK4G1wIXdMZ9MMm+qipUkDRDcVfUt4LkBz7cG2FJVh6rqcWA3sHoS9UmSjjGZNe5rkzzQLaUs6MYWA0/1zdnbjb1KkvVJRpKMHDx4cBJlSNLcMtHg/hRwPrAK2A/c2I1nlLk12gmqalNVDVfV8NDQ0ATLkKS5Z0LBXVVPV9WRqnoZ+DSvLIfsBZb2TV0C7JtciZKkfhMK7iSL+l6+Czh6x8k2YG2SU5MsB1YA2ydXoiSp3yljTUjyOeASYGGSvcBvA5ckWUVvGWQPcDVAVe1MshV4GDgMXFNVR6andEmam8YM7qp6zyjDN59g/kZg42SKkiQdn5+clKTGGNyS1BiDW5IaY3BLUmMMbklqjMEtSY0Z83ZAaS7YsenqV429df1NM1CJNDavuCWpMQa3JDXG4JakxhjcktQYg1uSGmNwS8cx2p0m0snA4JakxhjcktQYg1uSGmNwS1JjDG5JasyYwZ1kaZJvJHkkyc4k7+vGz0pyR5LHuucFfcdsSLI7ya4kl01nA5I01wxyxX0Y+EBV/QzwNuCaJCuB64A7q2oFcGf3mm7fWuBC4HLgk0nmTUfxkjQXjRncVbW/qu7ttl8EHgEWA2uAzd20zcCV3fYaYEtVHaqqx4HdwOqpLlyS5qpxrXEnWQZcBNwDnFNV+6EX7sDZ3bTFwFN9h+3txo491/okI0lGDh48OP7KJWmOGji4k5wOfBF4f1W9cKKpo4zVqwaqNlXVcFUNDw0NDVqGJM15AwV3kvn0QvszVfWlbvjpJIu6/YuAA934XmBp3+FLgH1TU64kaZC7SgLcDDxSVR/v27UNWNdtrwNu7xtfm+TUJMuBFcD2qStZkua2Qb667GLgvcCDSe7rxj4EfBTYmuQq4Eng3QBVtTPJVuBhenekXFNVR6a8ckmao8YM7qq6i9HXrQEuPc4xG4GNk6hLknQcfnJSkhpjcEtSYwxuSWqMwS0Bb11/00yXIA3M4JakxhjcktQYg1uSGmNwS1JjDG5JaozBLUmNMbglqTEGtyQ1xuCWpMYY3JLUGINbkhpjcEtSYwxuSWqMwS1JjRnky4KXJvlGkkeS7Ezyvm78+iTfT3Jf97ii75gNSXYn2ZXksulsQJLmmkG+LPgw8IGqujfJGcCOJHd0+z5RVR/rn5xkJbAWuBA4F/h6kgv8wmBJmhpjXnFX1f6qurfbfhF4BFh8gkPWAFuq6lBVPQ7sBlZPRbGSpHGucSdZBlwE3NMNXZvkgSS3JFnQjS0Gnuo7bC8nDnrppLVj09UzXYL0KgMHd5LTgS8C76+qF4BPAecDq4D9wI1Hp45yeI1yvvVJRpKMHDx4cNyFS9JcNVBwJ5lPL7Q/U1VfAqiqp6vqSFW9DHyaV5ZD9gJL+w5fAuw79pxVtamqhqtqeGhoaDI9SNKcMshdJQFuBh6pqo/3jS/qm/Yu4KFuexuwNsmpSZYDK4DtU1eyJM1tg9xVcjHwXuDBJPd1Yx8C3pNkFb1lkD3A1QBVtTPJVuBhenekXOMdJZI0dcYM7qq6i9HXrb9ygmM2AhsnUZck6Tj85KQkNcbglqTGGNyS1BiDW5IaY3BLUmMMbklqjMEtSY0xuCWpMQa3JDXG4JakxhjcktQYg1vqvHX9TTNdgjQQg1uSGmNwa9ZLMvBjOo6XpprBLUmNGeSLFKQ55Y/2rf/R9jvP3TSDlUij84pb6tMf2qO9lk4GBrckNcbglqTGDPIt76cl2Z7k/iQ7k9zQjZ+V5I4kj3XPC/qO2ZBkd5JdSS6bzgakqXTsmrZr3DoZpapOPKF3j9Mbq+qlJPOBu4D3Ab8CPFdVH01yHbCgqj6YZCXwOWA1cC7wdeCCE33T+/DwcI2MjExNR9IxXsvb9Mb69yQNanh4mJGRkVH/8g7yLe8FvNS9nN89ClgDXNKNbwa+CXywG99SVYeAx5Psphfidx/vPXbs2OE9sJoV/Hus18JAtwMmmQfsAN4M/M+quifJOVW1H6Cq9ic5u5u+GPjTvsP3dmPHnnM9sB7gvPPO44knnph4F9IJeMWtFg0PDx9330C/nKyqI1W1ClgCrE7ylhNMH+1fyav+NlfVpqoarqrhoaGhQcqQJDHOu0qq6nl6SyKXA08nWQTQPR/opu0FlvYdtgTYN+lKJUnAYHeVDCU5s9t+A/B24FFgG7Cum7YOuL3b3gasTXJqkuXACmD7VBcuSXPVIGvci4DN3Tr364CtVfXlJHcDW5NcBTwJvBugqnYm2Qo8DBwGrjnRHSWSpPEZ5K6SB4CLRhl/Frj0OMdsBDZOujpJ0qv4yUlJaozBLUmN8X/rqlnPe6s123jFLUmNMbglqTEGtyQ1xuCWpMYY3JLUGINbkhpjcEtSYwxuSWqMwS1JjTG4JakxBrckNcbglqTGGNyS1BiDW5IaY3BLUmMG+bLg05JsT3J/kp1JbujGr0/y/ST3dY8r+o7ZkGR3kl1JLpvOBiRprhnkixQOAb9YVS8lmQ/cleSPu32fqKqP9U9OshJYC1wInAt8PckFfmGwJE2NMa+4q+el7uX87nGirxRZA2ypqkNV9TiwG1g96UolScCAa9xJ5iW5DzgA3FFV93S7rk3yQJJbkizoxhYDT/UdvrcbO/ac65OMJBk5ePDgJFqQpLlloOCuqiNVtQpYAqxO8hbgU8D5wCpgP3BjNz2jnWKUc26qquGqGh4aGppQ8ZI0F43rrpKqeh74JnB5VT3dBfrLwKd5ZTlkL7C077AlwL4pqFWSxGB3lQwlObPbfgPwduDRJIv6pr0LeKjb3gasTXJqkuXACmD71JYtSXPXIHeVLAI2J5lHL+i3VtWXk/xBklX0lkH2AFcDVNXOJFuBh4HDwDXeUSJJU2fM4K6qB4CLRhl/7wmO2QhsnFxpkqTR+MlJSWqMwS1JjTG4JakxBrckNcbglqTGGNyS1BiDW5IaY3BLUmMMbklqjMEtSY0xuCWpMQa3JDXG4JakxhjcktQYg1uSGmNwS1JjDG5JaozBLUmNMbglqTEGtyQ1xuCWpMakqma6BpIcBP4SeGama5kGC7Gv1szW3uyrLT9dVUOj7TgpghsgyUhVDc90HVPNvtozW3uzr9nDpRJJaozBLUmNOZmCe9NMFzBN7Ks9s7U3+5olTpo1bknSYE6mK25J0gAMbklqzIwHd5LLk+xKsjvJdTNdz3gluSXJgSQP9Y2dleSOJI91zwv69m3oet2V5LKZqXpsSZYm+UaSR5LsTPK+brzp3pKclmR7kvu7vm7oxpvu66gk85J8J8mXu9ezpa89SR5Mcl+SkW5sVvQ2IVU1Yw9gHvBnwN8AXg/cD6ycyZom0MPfB34OeKhv7HeB67rt64Df6bZXdj2eCizvep830z0cp69FwM9122cA3+3qb7o3IMDp3fZ84B7gba331dffvwE+C3x5tvxd7OrdAyw8ZmxW9DaRx0xfca8GdlfV96rqr4EtwJoZrmlcqupbwHPHDK8BNnfbm4Er+8a3VNWhqnoc2E3vz+CkU1X7q+rebvtF4BFgMY33Vj0vdS/nd4+i8b4AkiwB/hHwe33Dzfd1ArO5txOa6eBeDDzV93pvN9a6c6pqP/QCEDi7G2+y3yTLgIvoXZ0231u3nHAfcAC4o6pmRV/AfwH+PfBy39hs6At6P1y/lmRHkvXd2GzpbdxOmeH3zyhjs/n+xOb6TXI68EXg/VX1QjJaC72po4ydlL1V1RFgVZIzgduSvOUE05voK8k/Bg5U1Y4klwxyyChjJ11ffS6uqn1JzgbuSPLoCea21tu4zfQV915gad/rJcC+GaplKj2dZBFA93ygG2+q3yTz6YX2Z6rqS93wrOgNoKqeB74JXE77fV0M/JMke+gtOf5ikltpvy8Aqmpf93wAuI3e0ses6G0iZjq4vw2sSLI8yeuBtcC2Ga5pKmwD1nXb64Db+8bXJjk1yXJgBbB9BuobU3qX1jcDj1TVx/t2Nd1bkqHuSpskbwDeDjxK431V1YaqWlJVy+j9O/rfVfVrNN4XQJI3Jjnj6DbwDuAhZkFvEzbTvx0FrqB3x8KfAR+e6XomUP/ngP3AD+n9pL8K+CngTuCx7vmsvvkf7nrdBfzyTNd/gr5+gd5/Xj4A3Nc9rmi9N+BvA9/p+noI+Eg33nRfx/R4Ca/cVdJ8X/TuOru/e+w8mhOzobeJPvzIuyQ1ZqaXSiRJ42RwS1JjDG5JaozBLUmNMbglqTEGtyQ1xuCWpMb8f4iVaWTpYabeAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "print(\"The size of state is: \", env.observation_space.shape)\n",
    "print(\"No. of Actions: \", env.action_space.n)\n",
    "env.reset()\n",
    "plt.figure()\n",
    "plt.imshow(env.render(mode='rgb_array'))\n",
    "env.close()\n",
    "plt.title('Original Frame')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def random_play():\n",
    "    env.reset()\n",
    "    while True:\n",
    "        env.render(mode='rgb_array')\n",
    "        action = env.action_space.sample()\n",
    "        state, reward, done, _ = env.step(action)\n",
    "        if done:\n",
    "            env.close()\n",
    "            break\n",
    "random_play()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 4: Creating Deep Learning Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Policy(nn.Module):\n",
    "    def __init__(self, state_size = 4, action_size=2):\n",
    "        super(Policy, self).__init__()\n",
    "        self.fc1 = nn.Linear(state_size, 16)\n",
    "        self.fc2 = nn.Linear(16, action_size)\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = F.relu(self.fc1(x))\n",
    "        x = self.fc2(x)\n",
    "        return F.softmax(x, dim=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 6: Creating out agent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "GAMMA = 1.0            # discount factor\n",
    "LR = 1e-2              # learning rate \n",
    "\n",
    "class Agent():\n",
    "    def __init__(self, state_size, action_size):\n",
    "        self.action_size = action_size\n",
    "        self.state_size = state_size\n",
    "        \n",
    "        # Q-Network\n",
    "        self.policy_net = Policy(state_size, action_size).to(device)\n",
    "        self.optimizer = optim.Adam(self.policy_net.parameters(), lr=LR)\n",
    "        \n",
    "        self.saved_log_probs = []\n",
    "        self.rewards = []\n",
    "        \n",
    "    \n",
    "    def step(self, reward, log_probs):\n",
    "        # Save rewards and log_probs\n",
    "        self.rewards.append(reward)\n",
    "        self.saved_log_probs.append(log_probs)\n",
    "        \n",
    "                \n",
    "    def act(self, state):\n",
    "        \"\"\"Returns actions for given state as per current policy.\"\"\"\n",
    "        \n",
    "        state = torch.from_numpy(state).float().unsqueeze(0).to(device)\n",
    "        probs = self.policy_net(state).cpu()\n",
    "        m = Categorical(probs)\n",
    "        action = m.sample()\n",
    "        return action.item(), m.log_prob(action)\n",
    "        \n",
    "    def reset(self):\n",
    "        self.saved_log_probs = []\n",
    "        self.rewards = []\n",
    "        \n",
    "    def learn(self, gamma):\n",
    "        discounts = [gamma**i for i in range(len(self.rewards)+1)]\n",
    "        R = sum([a*b for a,b in zip(discounts, self.rewards)])\n",
    "        \n",
    "        policy_loss = []\n",
    "        for log_prob in self.saved_log_probs:\n",
    "            policy_loss.append(-log_prob * R)\n",
    "        policy_loss = torch.cat(policy_loss).sum()\n",
    "        \n",
    "        self.optimizer.zero_grad()\n",
    "        policy_loss.backward()\n",
    "        self.optimizer.step()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 7: Watching untrained agent play"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "agent = Agent(state_size=4, action_size= env.action_space.n)\n",
    "\n",
    "# watch an untrained agent\n",
    "state = env.reset()\n",
    "agent.reset()\n",
    "for j in range(200):\n",
    "    action, _ = agent.act(state)\n",
    "    env.render()\n",
    "    next_state, reward, done, _ = env.step(action)\n",
    "    state = next_state\n",
    "    time.sleep(0.05)\n",
    "    if done:\n",
    "        break \n",
    "        \n",
    "env.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 8: Loading Agent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "start_epoch = 0\n",
    "scores = []\n",
    "scores_window = deque(maxlen=100)\n",
    "\n",
    "# To Load checkpoint uncomment code\n",
    "# checkpoint = torch.load('cartpole_solved.pth')\n",
    "# agent.policy_net.load_state_dict(checkpoint['state_dict'])\n",
    "# agent.target_net.load_state_dict(checkpoint['state_dict'])\n",
    "# agent.optimizer.load_state_dict(checkpoint['optimizer'])\n",
    "# start_epoch = checkpoint['epoch']\n",
    "# scores = checkpoint['scores']\n",
    "\n",
    "# index = 1\n",
    "# for i in reversed(scores):\n",
    "#     scores_window.append(i)\n",
    "#     if index == 100:\n",
    "#         break\n",
    "#     index += 1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 9: Train the Agent with DQN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train(n_episodes=2000):\n",
    "    for i_episode in range(start_epoch+1, n_episodes+1):\n",
    "        state = env.reset()\n",
    "        agent.reset()\n",
    "        step = 0\n",
    "        while True:\n",
    "            action, log_probs = agent.act(state.copy())\n",
    "            state, reward, done, _ = env.step(action)\n",
    "            step += 1\n",
    "            reward = -100 if done and step < 450 else reward  \n",
    "            agent.step(reward, log_probs)\n",
    "            if done:\n",
    "                break\n",
    "        agent.learn(GAMMA)\n",
    "        scores_window.append(step)       # save most recent score\n",
    "        scores.append(step)              # save most recent score\n",
    "        \n",
    "        print('\\rEpisode {}\\tAverage Score: {:.2f}'.format(i_episode, np.mean(scores_window)), end=\"\")\n",
    "        \n",
    "        if i_episode % 100 == 0:\n",
    "            print('\\rEpisode {}\\tAverage Score: {:.2f}'.format(i_episode, np.mean(scores_window)))\n",
    "        if np.mean(scores_window)>=450:\n",
    "            print('Environment solved in {:d} episodes!\\tAverage Score: {:.2f}'.format(i_episode-100, np.mean(scores_window)))\n",
    "            state = {'epoch': i_episode,'state_dict': agent.policy_net.state_dict(),'optimizer': agent.optimizer.state_dict(),\n",
    "                     'scores': scores }\n",
    "            torch.save(state, \"cartpole_solved.pth\")\n",
    "            break\n",
    "    \n",
    "    return scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 100\tAverage Score: 23.70\n",
      "Episode 200\tAverage Score: 25.49\n",
      "Episode 300\tAverage Score: 30.43\n",
      "Episode 400\tAverage Score: 51.99\n",
      "Episode 500\tAverage Score: 179.75\n",
      "Episode 600\tAverage Score: 178.88\n",
      "Episode 700\tAverage Score: 260.52\n",
      "Episode 800\tAverage Score: 239.01\n",
      "Episode 900\tAverage Score: 333.37\n",
      "Episode 1000\tAverage Score: 426.80\n",
      "Episode 1100\tAverage Score: 282.35\n",
      "Episode 1194\tAverage Score: 451.46Environment solved in 1094 episodes!\tAverage Score: 451.46\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEGCAYAAACKB4k+AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO2deZgU1dX/v2d6FvZlYNiXAQQUUFBGBbeoCKIY0V80ajSu0ZiYGBPfvIK7byQxMS4xicmrxmiMYkjceMUNcMEdERdkXwVkG/aB2WfO74+u6qmurn3pruo5n+eZp7urbt26VVN1zj3n3HsuMTMEQRAEAQAKct0AQRAEITqIUhAEQRBSiFIQBEEQUohSEARBEFKIUhAEQRBSFOa6AX7o3r07l5eX57oZgiAIseLTTz/dycxlRvtirRTKy8uxaNGiXDdDEAQhVhDR12b7xH0kCIIgpBClIAiCIKQQpSAIgiCkEKUgCIIgpBClIAiCIKQIVSkQ0QYiWkJEnxPRImVbKRHNJaLVymdXTfnpRLSGiFYS0elhtk0QBEHIJBuWwinMPIaZK5Tf0wDMZ+ahAOYrv0FEIwBcCGAkgMkAHiaiRBbaJwiCICjkYp7CVAAnK9+fBPA2gJuU7c8ycx2A9US0BsAxAD7MQRuFPGDrvhp8sWkvJo/qneumeObDtbswc+FGlHdr17KRCOce2ReDurcP5Zw7qmox8+NNKEwQLh0/EB3bFAEANu6qxvtrd6KsQwl2V9fjvKP6oaCAXNe/80AdfvfaCrQvKUTHkkKUFCXAzKhvbAYA1DQ0YffBBrQrToDB+HpXNY4dVAoAaWWICN07FONAXROWbN6Lw/t2xvJtVRhY2g7rdh5EAQFNzYzD+3bGkm/2oX9pO+ytbsDRg0pRWVWHjiWF+GDtTvQvbYfdB+uxevsBjBtcirbFhShOEDbsqkZp+2L0L22H4gShvolx+XHlSDi45r+8vRY7qmpx9ug+WLBqJ5Zt3YfhPTtiyTf7cHjfzq7vmRHDenXEWUf0CaQuLWErBQbwBhExgP9l5kcA9GTmrQDAzFuJqIdSti+AjzTHbla2pUFE1wC4BgAGDBgQZtuFmDP+N28CADbcMyXHLfHORY+2vBKkyCJmYH9NA+48e2Qo55z9+RY8MG8VAGBIWfuUUj3p3rfSyiWI8J2x/VzXX3H3PNfHvLt6p22Zt1ZWOto++4stpnWs3F5leY7xg7thRJ9OlmU27a7Gb19bAQD4+/sbUttfX7o91R5yr0szOOuIPrFUCscz8xZF8M8lohUWZY1uU8YKQIpieQQAKioqZIUgoVXw/I+Pw1EDkuG38b+Zj+r6xtDO1dDU8lo1NZuX21vT4PtcpwwvSwntD6adikcWrMMTH2wwLf+3yyowtEfHDAWlr8svc64/AVMeei9je7ODRcnqGpsMt6vtu/y48tAUehCEGlNg5i3K5w4ALyDpDtpORL0BQPncoRTfDKC/5vB+AMxVuiC0Igo0Xcu2RQnUNFhIa584EXxeycVKj4UeXFwFJl351rBQZWhKgYjaE1FH9TuASQC+AjAbwGVKscsAvKR8nw3gQiIqIaJBAIYCWBhW+4TWQz4sOasVUW2KEqipN+6NBkFzc8v94kxjPcWvXl7mvm5ddeTSj0IEU9eLmSAvTLhXCk7iBm4xa1/UCNN91BPAC8o/vRDAM8z8GhF9AmAWEV0FYCOA8wGAmZcS0SwAywA0AriOmcN78oVWA7O5IIkLaZZCcQI1DeG5j5pCVKJNOq2g/bdYCXwnmB1bVFCAWrizrEwtBQslqWmJ8daYPIOhKQVmXgdgtMH2XQAmmBwzA8CMsNoktE6amVFg8qLGBa1AaVuUCDWm0KzvztuUdTMCKUMpuPy3kOX/MdqWgln7oobMaBbynvg7j9KFZ5uQYwpaS8HOaGh0oUCS5fXtJs03shH61rh1K1mRCCGmEBdLQZSCkPfkQUghTbAlCsKNk1iNOMos664ddpaCreC0jCkkP4sTBejSrii13cudCkOAh2J8hIAoBSHvCXM0TbbIZpBSe7/s7lxmz98avVLQCkoieweLtfMo2lI36u1TEaUgCDFArxPC1HNuev++LQUPgtJsxFKQetOsLkdhZpNjxX0kCBEhDwyF9B51yD1ON4LefUzBvDzBXnBaDWGNutCNevtURCkIeU8+uI/0wtDZ0EhvpLmPbO6dX0uhQCeBnMxbMCvhds6D5TlMA83e73uQ7QsTUQpC3hNXlfDKkq2p7/rx/GESpqVg6T7yHVMwpkvbIpM92SUeKkGUgtAKiOuM5h8/vTj1PVeBZjuamoJzHznF3GdvvGPMgC4obV/s+7yAvw6GWAqCEBECkEM5J7KBZpcNyahbHyuxjSlY7LM4buJhPW3b5rQur8RDJYhSEFoDeaAUtJZC2B1ON0q0yeeQVP2l2AXRrSa4mVoQoMDumUxeE4Q8IMygbLbIsBRCPJebNBd+YwpBKrsgZa6vHEzBNSMniFIQ8p78cB+lp4MIE1dpLlzHFNItC7czmr1kSQ2WPHiYbBClIOQ9cQ00a8lWioSGpmZs31/ruKfsdkiqVRCb4LOXbTrhjCPhuolAExyRizWaBSGrxF8lZPaCw1J0v35lOT5at9txeTfuo8qqOnznL+lLruuvy9ZSgMU8hQDFrlld/mIK8VALYikIeU8eGArpIipE2fKebi1ku3iMG0vhqY++ztiWvThAPARyFBClIOQ9+eA+ypzRHA49OpW4Ku8mId6jC9ZlbkxLiOcgdbaFqWDmYvOWX8n1IZpj462ARCkIeU/8VYI+91F4JPR5J2xwYynUNGQupKgX2PbuI4vcR1b7ghqS6uPYuKgKUQpC3pMHhkLWep96qyroRXb0kAdl53aeghdCudsx0QqiFIS8Jx8S4mW4RkK6JLe3ym2aCz2Zk9dsynsQrAwOTB7nwaNkiygFIe/Jh/c4bZ5CiFaDWwUaqKXgJHe2RZGou/JlkR1BiAj5EWhO/x3WFelvld2tcztPQU9GTMG2vLe9rhWG2ZwHB8+SeWpvl23IEaIUhLwnD3RCejqIEM/jNiWI2+U49aTHFJxdWUxka2wRpSDkPfmgFLIlCF3HFAJ0Hxn9zizvYXipRRI9q2OMyINHyRZRCkJeMnfZ9tT3fEiIl60ZzRnuI5vyfpWCLne2fZZUys4azWEQ8ealEKUg5CUfr9uV+p4fCfGMvweNPtB8sK7Rsny2LQUvBJn7SFJnC0JMKdCM4ZRAs3P09d4xeykqq+pMyzsZfbTrQB1uePYzw33ay3IiNH0nzXOIvxnNwbUjF0hCPCEv0bpb4q8SshNoPvzO11FVm2kZbN9fi7KOxukvnFgKf3xzDV78fIvhvgxLwb6ZplgdG3M5nVXEUhDykoTmyc4HSyEzphBs/U3NbKgQ7HBiKRQXmosZbQzByTQFq/UUsjGj2U98SuYpCEIOSWgthfjrhNDFidehpU6W4yxOWCiFLMjJ5HKcuRfIEWiCI0QpCHkJ5Zn7KD3QHLx0cbuCWuo435aC5js5Ed4WazRnoyeeDw+TDaIUhLwkoQk050Puo8zU2cFek9d0FU5yHxVZWgou5w9YFDfbx2DXLsQoWBa5InSlQEQJIvqMiF5WfpcS0VwiWq18dtWUnU5Ea4hoJRGdHnbbhPxFm0AuD3RCGvVNzdi0uwY79tcGVqfXoaVOlElRIl3A9jAJWjtMfeTJlxbUI+CkHrPnLS56JhuWws8ALNf8ngZgPjMPBTBf+Q0iGgHgQgAjAUwG8DARJbLQPiHC1DU2eQoUU57FFLS8vWIHAOD2l5YGVmdjk9eYgv3NbV+SPshx1g/Hp767FZTZkqvhnCceWiFUpUBE/QBMAfCYZvNUAE8q358EcI5m+7PMXMfM6wGsAXBMmO0Tos2+mgYMv/U1/PmtNa6P1bqP8mFGs5a2xUkhe7De/WghM6x6/FaC24mloFccndoWtdSdkRDPe5ZUK4LqGDipJ+5PW9iWwoMA/huAthvSk5m3AoDy2UPZ3hfAJk25zcq2NIjoGiJaRESLKisrw2m1EAn2HKwHAMxatNn1sfnsPmpfkjSg7WYbu8Gr+8jJ6CN93dqRYfqZ2n5yHwXZD4+LqycMQlMKRHQWgB3M/KnTQwy2ZTypzPwIM1cwc0VZWZmvNgrRplDxRXsRWAV55D664vjytN9ti5JKobo+c3lLrzR4dB85sRT0ZUgjddwusmNVxjQnEigwa9HXPAWleVGfNxOmpXA8gLOJaAOAZwGcSkT/BLCdiHoDgPK5Qym/GUB/zfH9ABhPgxRaBYXKesFeBJZWKWzdVxNYm3KB3qWiXluQ7iMrxWvl0nGisJt1ZQrMLAUnriPbEplk231oJvTjYnyEphSYeToz92PmciQDyG8y8yUAZgO4TCl2GYCXlO+zAVxIRCVENAjAUAALw2qfEH3UNeS9DJfUxhSuecqpsRpNzFwZNfX+1jLQ0hDiPAV9mYI0l5EupuBoRnP44jXMOQ9RH+6ai9xH9wCYRURXAdgI4HwAYOalRDQLwDIAjQCuY+bg7GMhtnizFEJoSI4wv5TgesBeYwp6K8C47vT/n1keJ3KQOjvXSKA5IJj5bWY+S/m+i5knMPNQ5XO3ptwMZh7CzMOZ+dVstE2IMMrb5UlgRbw35gazSwkyJbg+zcU/rzrW4XFeLIX0NRTc4H65nCSu3fguT7J5TzV+/cpySyUpMQVB8In66nhxH+WPSjB3NwQ5U1t/j4f0aJ/6/n9fmof2nCjspmZOs9wKLOIIUU9ZbXa1P3nmMzyyYB2WbtlvPnktJk+lKAUhsqgvl5eJVXlkKJiKEieuG6dYuej+8vZaLN2yz3CfU0tBHTQApFsKbt18lllSs+JoM8ZNQsGoxxREKQiRRR014kX2lRTm0WR4s5w+AUq6usZ0oaYXsDUmw1+dzlNIFBA6tUmGMLULIGXOU4iGwDTNo2Rz0/NhoqQssiNEFj9CT59vJ85kYxH55z51P0EQsM+uure6Ho8sWIfiwgK8+pMT8ZFmmVSveHHDZNeNbzIkNSYxBVEKQmTRvjo7D9ShewfjRGqGx2oOPvfIjInxscJsKc4gYwovf7k19f30kT0dH2cXU7j39ZUAgPrGZpR3b4/y7u3T9qePRLIX916ypHrBfJGd4OuMGuI+EiKLtkf15oodFiUNjlVe3wLyPls3KpjGFELocb7x85Pwx4uOcixg7WIKdpZExoxmB+c1jylYEW7vXKvO7P4tUXGRmSFKQYgsQci8wkSB5wVkok6QQ1JVenduY7kojh47S6HAhYRJzlOwKZPr/nYA9zzq7iNRCkJeor53xYkCz0tNRgXTjmUIsqVNkbsAvd291a8trSdbvWa3cthru6zOE3ULQUWUghAPXL7U6stZlCDPKRyiglnvOAz3kbpKmlPxZWet2CsFzXfYC85sxRT8EO+nTQLNQoTxI/PUQ4vy2FIIUikMKG2HowZ0cX2cnSvEbh6Cl8lrpjEFi4NdWwpm9QQg8qNuMYilIEQWPy+gKqyKEgV5YCkYE2iai6ZmFFqspWyGraVgoxXS5ymEs8BOxcBS9we5RNsuOwUkMQVB8EgQlkKigCL/EtqShZ5lctax+5xE9paCu7Y7CTS7CTa//NMTcNEx/V13MMwnr7mqxlGdUUOUghBZOO2726BC8iNRQKGM0skm2ZAljc2cWtTIDfYxBZf7fUhOoyP7dmkLIorEQks5HznlEFEKQmTx08NXlUiigDynhY4K2ehhNjY1p+UncirA/FoKWpcRwYGlYJH7SHvwtd8aAgBoVxJsuhNnqbNt5mZE3GSQQLMQWfyIcvXlTVD83Ud6AR3G9WS4jxxi15Iw4jlOWnntt4akFAPg/lny2qu3Ok9c0lyIpSBEliDenYJ8cB+F3LFcumUfquubLGd+ex0Bta+mwXJ/ZkI8y+I5d8CYXa22XRGX+baIUhDykpZAs/dVxaJC2ILwT2+uAQAs3ri35Zz6fEsmt9ButO/OA3WW+730yM3cL5aps11PXnNXPt5PWDqiFIQI42dIavIzQRTKJK9sEraloA4b3ba/1vWxdvd210EbpaAbkmqfN8jbvqDw4/pRmxf1mIIoBSGy+BuSqiTEK4jGyBM/hC1EypTss/eed0Tgddc1WJsSHuwEb8txhp4QT3Mus5XXYhJTkECzEFnShqR6THNRWEBoivhLGAU6ty3CycN7pH47T3NhfG/31TSgpLDA1pLQ6ztfy3F6PzQwoi7wnSBKQYgsQUxeKxD3kS2Nzc1IuJh5rMUsXDP6rjcwqHt7EICB3drh0UsrjOvNGFll3VYnwWhDAnoEnFQT99XXxH0kRBZfLxe3zFOIuU4IfdJTU7P7mccqVj3j9TsPookZY/p3wbCeHQ3LBLowTgTqsnL1RT2WoCJKQYgsQaW5iP3oo5BlSXMzQ5/2yKkAW1t5EP/9ny9MlUMzMxIBXoCTTKph4ievUTxUgigFIcL4UgppaS7irRQ8zClzRZMDwW11C2ct2ozq+ibDfc3N2es9ZyONxIZdB/HvRZsMTu585bWoIzEFIbIEkSU1EZG8N37w6tpxSnMz22Yz3bSn2nJ/fWMz2hssof3N3hpLpeb2yjwvfuP2PCYtu+fVFQCA8yv6e2pHHBBLQYgsWmHuVa4X5IH7KEwamprx/GffYPOemrTtepE4Y85yy3qO/NVc1DUaWwtWSs2tjI+LC8aQmDRelIKQl6RiCnkw+ihM7NJQtGAv0apqGw23W81qzpacdDtU1KtxZpn7KCZaQZSCkJekxxRy25YoY5XvyC1mIm9/rbnicesOyvWMZtNz5+7UgSNKQYgsXjv4zIz/eXkZAP+B5pXbqnDjrC9y6oIKc7RNjUmA2MspzdppvZi9+/OY1mWxz31MwR1OJlrGZESqKAUhumgDzW7k+q6D9anvfpXCT55ZjOcWb8aaHQc81xFlahqMlYIXzJcNDW6YZlRcMHp31Beb9pqUjB+iFITI4lWWa4dXFhCh2UcvX123uNEuHWhMWbmtylE5J71c0yUs/VbsFIu6gg4rWdVnNmouGurMntCUAhG1IaKFRPQFES0loruU7aVENJeIViufXTXHTCeiNUS0kohOD6ttQjzQvlpuZId2tEuhz5iCuvBMTt1HIdb9i1lfmJzTe6o6fS/a0n2k+213l8MIABufx2behtn2PIhfhWkp1AE4lZlHAxgDYDIRjQMwDcB8Zh4KYL7yG0Q0AsCFAEYCmAzgYSIKdi09IVZohYubl03bUyvw6T5ScwKFsYKYFbFMrKbIUb3+tJzlm6WYQtBYPVMSUzCBk6iO2CLljwFMBfCksv1JAOco36cCeJaZ65h5PYA1AI4Jq31C9PEqFrVCKVFgn/PfilxZCk6bHEXloXe1RWGYpushqbb1mRwXE8FvRagxBSJKENHnAHYAmMvMHwPoycxbAUD5VPP19gWgnT++WdkmtFK8yjutEkgQ2a4OZkVhIvmWf/r1HjQGOHzTjv/7ckvqu9UqaP/+dHM4DdCntHZxqP5+uxl9ZHceJ0NSs5Gl1Owc1q6yeGiMUJUCMzcx8xgA/QAcQ0SjLIob3bGMW0xE1xDRIiJaVFlZGVRThUji7eXWKgW/7qPCguQr8tvXVuCBeas81+OWd1Y6e7bnLtseckucM/H+d/Dnt9ZkrF9hJaQzYgohWT5B12odaI43WRl9xMx7AbyNZKxgOxH1BgDlc4dSbDMAbUKRfgC2QAczP8LMFcxcUVZWFmq7hejgpvenfWH9zmhWLQUAWLZlv+d6XEOGXzPwmyyvd+c2/irQsKOqDve+vjLD1WZlqel7/nWNNiu1OfDPBNEjD8MNFBfXUpijj8qIqIvyvS2A0wCsADAbwGVKscsAvKR8nw3gQiIqIaJBAIYCWBhW+4ToE4j7KJEcfeS1B1oYdopSE5wKtqCS5T14wZj08/uoVq8UrO68/t9ipxSssLxnWRqS+u7qSqxyOMw3qoSZJbU3gCeVEUQFAGYx88tE9CGAWUR0FYCNAM4HAGZeSkSzACwD0AjgOmYObmaNEDsCCTSTOkzSm6BT3UfZxqku8qsTGpsZFx3TH+ccaR2+c3OeDKVgNVJH99vWUrDaF+iUB+vKzKzPB+etNq/TV4uyR2hKgZm/BHCkwfZdACaYHDMDwIyw2iTEC8+WgkYoqUNKm5lR4OG1TGjcR9lc3EVrAYS5HkFygZ1gr+tAnXFiPCeYZVpV8T5Pwbup8MbPT8KkBxbo6stfZEazEFm8uny0h6nCVR/8dEqu3Edb99c6Kte/aztf52lsNl5gx89V62MvbmI6dQ1+3EfZw9OzGZOggigFIbJ4dx+1HKkKdc8pMzRKIZuv9IJVzkYflRR6e4U/37QXh9/xOvbVNCDhwEXmJni7ZW/62gyWI3VcxhS8BpH9DGpyNCwyj3D8RBHRCUR0hfK9TAkGC0JopC2y4+It1A9J1W9zQ1GOYgparDqYXlNf//XttahS3Dz69ZmT5/SuAme8kr4gj9Wd1/9f7NxHVmSzI+7lcerXpS0AYGC39gG3JlgcPfFEdAeAmwBMVzYVAfhnWI0SBMC7Hzh9RrO/GcnaIalRpNHjdWnvrSNLwcdtcONq+eXpwz23o02ReVYcX5ZCQKbCxBE98cwPjsXlx5V7b0wWcNoNOhfA2QAOAgAzbwHQMaxGCQIAzza6Vgip/nKvWSpyFVOwQivQ6z0O4dSnArE9pw+h6maiV+/ObfHdin6u6v/l6cMx/YxDMeWI3u4b5xEvlicRcNwh3W3Xw841Tkcf1TMzExEDABFF2/4R8oIghqQWpGIK3mpz0osOGysR4tV9lKY4Da5Rf06vlsK3R/fBD08a7KgdTjBqRtuiBK48YRB2OAzOuyfzrPkcU3CqFGYR0f8C6EJEVwO4EsCj4TVLENJx8xKm5z5KfgbhPsrZ4BGLE3ud7GU0lyMM/nhRxqh0f1hOVAj2VFZ46WTEJfeRI6XAzL8nookA9gMYDuB2Zp4basuEVk8gM5oL4u8+GtHb3FN70OOcgLQRWjmMmwSR6siJTvMzT8Go/lZtKSgzkl9n5tMAiCIQsob2RXYjtjhA91GhE4d7yIwdWGq6z+tEMaO5HFrcZi/1inv3kcVEviz2xD09TrnvXzjC9olXUk1UE1HnLLRHEFKkDUl1cVzakFSfgeZ010r23+r2xeYjajqWFHpWCunWlKcqbHnvplNsywTR43byXwl+nkL+2gpOYwq1AJYQ0VwoI5AAgJmvD6VVQqtn54E6/ObVFZ6OTe8FJz+9zmjWUllVi6raBnRsU+S7LqdYtbpDm0IcqPVvKRSFpBWczHVw+29xsp5C0BhdRx5PaHY8JHUOgNsALADwqeZPEELhlheWYPlWb6mqDS2FAMbzf7F5H0697x1P9XilQ4l5v62sYwm27a917YKpbWjCe2t2pn63M7BG9K6YKHtLHCmfgM8ZwQXvAsNpoPlJIioGMEzZtJKZG8JrltDa8ZNCudnAX+71JdYfV1lV57FV3nj88qNN9w0p64AvN+/DroP16N6hxFF95dPmoLxber6ktsVhJku2xu2/xUj8qzrBSjUELcS9uI9iYig4ntF8MoDVAP4M4GEAq4jopBDbJQjpuHirtT1ndQj+S59/k9XlNIOir5IawYgu7ZJurCqXLqQNu6p9tckpjkYFuQ00R8QH0+otBQD3AZjEzCsBgIiGAZgJYGxYDRNaL83NjI0+BJeRpXDf3FVoYsYNpw0zOcqYXL/7RiODVIHUVknrUFPvLF/Q5X83XrOqyWBptCBkr9fRQG6PU0tbK4xg/5Oe3GkRUWh2OI0pFKkKAQCYeRWS+Y8EIXAenLcK63YetC9oglFMAQCWellOM9ddQgs5osYCahqcKYW3TdZ9djJrO6zb4DrQHE4zXBPWWtJRwKlSWEREfyOik5W/RyGBZiEk3vC5GL2ZUlj89R6UT5uD5z7d7LiuXLz6ae4vCymoxgKcWgpmnDGql6/jzXDSMR7Rp1NgJ8pqTMHL6KNgmxAaTpXCjwAsBXA9gJ8huWTmtWE1SmjduPWR61Ff2Fk/HJ8mmHYdrAcA3PjvLxzVs7+2wfG6BkGiTclh5XJQLYWXv9zi+VzDe3YMbUiqE3p2auOqfFQ8MHlsKDhWCoUA/sDM/4+ZzwXwEADzWTWC4IN9Nf4GtqmWQgF5Ty0NAD+b+Rm+2LwvY/v2kBKvMTMemr8a2zUjnCwtBSWm8Ownm/DFpr2uzzekrD0ev8J8dFNa2yI82qYlpmBeJvAhqV7uR0QUmh1OlcJ8ANphEG0BzAu+OYLg3EduxutLtwFI9rKNgqhO+dok2P3zf33uuU4rPv16D+6fuwo3zmqp3yroqp3D4EWR3jLlMMvRTb4JQQhGJamcWApAG2Y+oP5QvvtbHFYQTDDKVe/mHfznRxtT9TQ0eX97i02Wuqz1qbTMUNuqTV1h1btUh6QCwPOLjeMkFz/2ESbc97bhPqORTWbnDUsIuu49G5QPY+irbX0ejomKQrPDqVI4SERHqT+IqAJAjUV5QfCMn/dXOxdhZ1UdGn0oBbP1jxMhZU5VXRLa6RRWglu70tiLnxvHFd5fswtrK41HchWGvFZEtoSgep7sJsTLX1PB6TyFGwD8m4i2IKkk+wC4ILRWCYJHtP74Yb064gNNOge3lBQah83CUgpq91MrcML0Q4e9fpCTtrs2FDzej6BFuJdQVV7EFIjoaCLqxcyfADgUwL8ANAJ4DcD6LLRPEAA4tx6KNGsDDCnr4Mt9VFJk/HqE3cO2S2ut4lfIWF1HXFwdgOY+ZLXJ+Wsp2D3d/wugXvk+HsDNSKa62APgkRDbJQiB4HbFtWVb9qN82hzM/mIL3l1tbGUEucZudX0jqmqTQWK1pdqYSkBrxhsS9kjUMGS01zqjME8hLti5jxLMvFv5fgGAR5j5OQDPEVE4QzAEwQCnvWL9YCO3Q1LnL09OnJv+3JemZRp8JOvTM/43b2JfTQP+ceUxKWXQ5NB95KY373U50rAJIvWDkyGpQRPNuxkMtkqBiAqZuRHABADXuDhWEALDac9Mv27CcUO6GZZ7dME6jB/SDet3HsS3R/dJbVdXWqu2GGE0pEd7Z41xgBRW4UgAACAASURBVDqU9NLHF+Lwvsl1rLSXEFS+HKMhq1ZB+EByH4Ugpb3WGYXU2XGJKdgJ9pkA3iGinUiONnoXAIjoEACZs3oEIcfo100Y3b+LYbkZryxPfdcqBdWlYvXSh9XpXlt5QKnf2Qn0Qqa5mU1dW2t2HMjY5sZ6CGs9hQBGpDpKnR00+bzymqVXkZlnALgRwBMATuCWYREFAH4abtMEwT1+3SROksPt2B/Omgrq22U2ac6OPdX1eGDuKgyePidj39xl2zK2+ZntHSWcuNGCHkLqZU5kXIL3TtZo/oiZX2Bm7TKcq5h5cbhNEwT3qL1sNzN1tQKj0EEQed7y7Rh266u4+YUl7hto1Q6XvU8i4Jmrj039Hnv3PPxh/mo0M/DDpxZhb3V9at+j72YOFrSySIIQX2G4S6yX48ziPIXWaikIQrbZdcC4F+60p6cKupvOONTxOes1s8WczkGob2zGMx9vdHwOJ3jpuA/u3sFw++tLt+PlL7ca7vvl6cNxxfHlOOGQ7u5PGDBGctx1cD0HHXDt43jDs585OiYuMQVRCkKk8LTmgQZVvidcvIHa1NPaeQ5O+fTrPbj5hSW+XRRuXV8EshQ0t774leH2np3a4I5vj0wF1Q3r1lXsLVV0GIFmq/PlBrPZ5HElNKVARP2J6C0iWk5ES4noZ8r2UiKaS0Srlc+ummOmE9EaIlpJRKeH1TYhulhN1nKCKljdjMHXJuBzElPQ8v2/fYzv/OUDPPPxRmzzmT01W8NGvc1PiMYCAoaWRfCnsUXWU/BGI4AbmfkwAOMAXEdEIwBMAzCfmYcimX11GgAo+y4EMBLAZAAPE5Gk525lLNyw276QBS1ps71ZCk5iClq0E9w27MzO2scqRN4ETZQCnu6X3vQ2wzvwyWsSU3APM29Vg9HMXAVgOYC+AKYCeFIp9iSAc5TvUwE8y8x1zLwewBoAx4TVPiF6vLVyBx6av9pwn9NX8NyH3wfgLj9RQxPjoJKZ1E9eo7rGcLKnWuKhuV5yEkVlXL7RvycXax97y30UHWVsRVZiCkRUDuBIAB8D6MnMW4Gk4gDQQynWF8AmzWGblW36uq4hokVEtKiyMvurYgnh8eHaXab7nAolNdeRm1QUP3r6U4y843UA3mIKKn4ysnqBkL1ev9O5E1oczVNwUOjP30slaLYUrFb3IuiefT5nSQ1dKRBRBwDPAbiBma2iiI7SvDDzI8xcwcwVZWVlQTVTiABWaw27fQXduI/WKamlm5vZdUxBS2Mzp/IYAcl1F+Yu246lW4Kd5+k3i6qXHqvaM85FZ1cbAzGcvJa1lrSg/ge27nO+gkA87ISQlQIRFSGpEJ5m5ueVzduJqLeyvzeAHcr2zQD6aw7vByC/wvqCJY0WM4Lc9szcjD5S+WZvDf7L4frNRjz+3nocfucbqTWTD73tNVz9j0WY8tB7nuvMFRmzpU1iNX06t8Hofp1N6ghKDLbUYzWENasxBaW+99eYW7dxJczRRwTgbwCWM/P9ml2zAVymfL8MwEua7RcSUQkRDQIwFMDCsNonRA8/aa71eOnwz5iz3HZZS+1qZ3rUIPmbK3Zg3rLthmXqG5tR7zCh3q1TDrMtYyYHzxvbz/UxlqiWgm7zB9Mn4OJjB3qp0RNBKRr/1SRviJtqYhJSCNVSOB7A9wGcSkSfK39nArgHwEQiWg1govIbzLwUwCwAy5Bcr+E6Zs5B5E7IFgfqGvGLWZ+nBLF21TQ9bhWGF0vBiSugW/ti2zLLtuzHD/6xyHDfiNtfw3H3zHfUnvJu9on3zITkob064uJjB5gc4+j0aaiJBt0cmy0Z6Gw5Tt0xPs/JOXSnhU1omU6Z+T2Y3/sJJsfMADAjrDYJ0eLv763H84u/Qe/ObfDL0w9Fg8WQDicje9bsqEp99zKKaMs++3kGbYvtR0mv2FaVse3Q217FA98dg8Zmxs4D9QZHZeJEDVpd5YxzD8fwXh1x+0tLdcfY3xu9slHdR8ljvSXsC7OMW4jIl0/JS4xFRh8JgkNUIWVlKdQ22LtcTrt/QUudHt6/yir7RHdtTJbotKO2oRm/eXWFq2Ps4ihE5tepCiCrrKLu2qIe7P7YsEmt0WwVU9ApMr/rJMnoI0HIAlZDOt3OAahzoES8UOhjyKrbYZF+xA5b+De8XAGbxBQA8+uK0iS5oGnRkfl3jaIUhMhg5T5a5jInUm1IE8n8CIH9NY2uyjvpjNq1JyhLodlLTMGRa8iBKyuA82TGFPwJ83yOKYhSECKDlfvo4/Xu0l/U1IdjKfgRAtqRTQ1NzZhw39sZZQoI6NGxRPnlSCsYbw5YWqXHFPRNiKFk9Os+kjQXghA+Qc4Irijval/IA0HJ2qG3vIq1lQcN9114dHK6TllKOXhvj/H+eE1es8OJAtQ/WW4uo1enNqYVxiV47AZRCkJk8JJKwYj7vzsaPY1e5Jhw/YShmPXD8Rg7sNRwv/Yu2bpWjHr2PuRY0CLQiXvLTvB6Sgro4qC2xQl8fvvEtG3NFjEWIy4bn725HH4RpSBEBjuVoF9/2Qy/6bft+PvlR+NszbrOQUJEKEwU4JhBxgrBfX0G23zVl3m0aaA5Sp3ooGMKLtxH950/GndNHeXrfNlElIKQM1IjOJT3026YX5NDSyJsYXTKoT0wok+ncE/iEC/uCz8uj8mjejk/j6P5EE7qcVaHer5SBxMM/T4jbgLN37GYXR5FRCkIkcHOEHC6CI1e6E0c0dNrkzJQhUGDw1QV7ut3u/qae/zIwxtOG+rj6HApLizA775zBP597fiMffqevdt7oFdwMiRVELKAnThsdOw+Sv/96KUV+NHJQ7w1yoR6i5FSfmhf4i7JgLcsqe6PUXHjmsuW+0grmL97dH/0L22XUaZz2/ScVX4DxP9etClrK+VlG1EKQnSwcx85HJ1k1HvzO4NVj1YpTBrRE0//4Fjfdf7hwjF45gfjXBxBse+pBuVisuPe80brzusS3QEvf7kVT36wIVpxk4AILfeRILjF3lJwllrbSAEEHXxuaEye79Yph+EHJw42LHPy8DK8vdL5QlBTx2SsKeWboCaveTnWadEhZe1Nh+c6Oo+DE3XVxxkCeBx2HqhD787xHeVmhigFITLYDUk1M9d3VNXimBktmUePHJA5RyHoDl19U3LGdHGhubGdDe+CnUA0akK2rAtHs5UJmHP9iWjw4Y7LRlzF6FKaOWIjrAJC3EdCbDCLKWzZ25Ld9IRDuqOXQe8t6ElG151yCE4c2j2td7/gl6eklamuc57W4q+XjA2sbbZETJC1KUqgYxvzdSpCy5Lqk2Tw2roedSJinBClIEQGu4E3K7YZ5z8q1PiLzNZmDlqw9O7cFk9ddWxaAHNAt/QAZ58ubXHckG62dQ0ua+9qqKcWT4FmT2dSj3URaPZxHjdko7dumAhQLAVBCAf1vdIbAm2K0h/PK58wXrimtqEl+V2hiVIIe0Kbyt3njMKVxw/CKcPLcOOkYXjmavPA8diBSTeXa9eJ5j55cQVlKzVDEHMQkmWCb28Qt6C5mQNJ1hc1JKYgmLJmxwG8sWwbfnzyIaHUr7cM9GP0CwsKANgLTO1aC9v3Gy+Uk61385JxztMZqAosyJxPTvA3ozmwZgSI+0a57SR0KClEz04l2L6/Zc2NZEzBup44LrsgloJgyoWPfIjfvbYSB134xoOkMEH40/eOtC2ntRSWukyxnQ1G9Dae/az60Z2u2WxENIV0kigni3MfaCb06tw2bVsz21sKcUSUgmBKTX1S2AaVqM4O/WkKCwpw1hF9cPWJgyyPC2vtBCO83IoendKznXbvUIzrTz0EPzo5OZTV6aQ8I8JOBufkfL4eD2f+I+vdWZrAV6RzTebr6muiFARTVBM7WxM39cqnSFnlbFTfzqltq7ZXYfnWdGvAySpruey03nbWiLTfiQLCLyYNT81edprozwhPuY8i37/NRvvcn0O/7neeTmgWpSCY4zRRXVDolYL6Ek4d0xfnK0nFJj2wAGf84d20ctpJbUHMLA6aIWUd8PJPT8B/TRoGoGW+RTJm4jzRnx6i7FsKQROEgsrWPShKpIvLZmaP61lEG1EKginq8M5s5XjRn0b7Eg4u65C2b19NA95ZlZwtrJWpPUwWpsm1f3tU3874/rhyAMlx+UBLoDnM+2ukb3zdiRzcRvsRPtlplBdLIY4eJhl9JJiiuo+89mQdQ8bCUTu8VD9z+KczP8OCVZkpJHpGOO1A53ZFuOXMw1JZWwsT/pWCJ3kY8d7rt4/ojZkLN6JTm0Lsrw1nkIOXW6C6M1vgWAp9O8RSEExRZXK2LIUMpaCxFEp0SmHN9irDOjpZzIyNAlefNBjl3dsDaLGEfAWa7VYlM9jtx2UTdDzCqH3HHdIdG+6ZgiE9OmTuVHj3v0/BmYcnJ/z1L21rWi5I9JbC8q1Vtvm64ug+EktBsCBc94Y+x33mPIWWN0qvFLYazEd45foTA2xd+JhNtBPSMVJ8/Uvb4c/fOwqb99QYpsq2o7re/Yi1Ql1M4fNNe/NyBJJYCoIp2bIUHpq/Gk3NnOGmKkyYu4+M3sWencwXuo9ij00NNLvF738jW1lSHdXn51giTwoBAA54mHtjpMTzTyWIUhAsKDDx9YfBym1V0Gd7uPjYltnBJYUJ2zpyHUx2S2GGjzp4Ag8054Aw2tvB5WJGgLESz0NDQZSCYE42Ywpb99WkDUndcM8UnKdZ27akyP5RtfLG+PGFv3Td8Z6PtSIbSsGIuCnPoHn00grMOHeU6+OM3X35pxVEKQimUIijj/7x4QZs3F2d+r11X63lzOkj+3exrdNK2PmRg6M159bHQfxQ5NF95JegZzT7IRcKauKInhnLczohYaDE89FSkECzYIoqs4JK2LZi2350KClE13bFuP2lpWn7bn3xK8vAa5d2xagY2BWLvt5jWiZuHWCzNN9BoNZsPPooXoTxf/WSNVef5gLIRztBlIJggepyCSr30eQHkzORxw0uNdxvNzTT7kXOVnrsqGB1tWEJq6B69kS57WV7uYyExBT8QUSPE9EOIvpKs62UiOYS0Wrls6tm33QiWkNEK4no9LDaJTgnrJjCR+t2B1qfip+O97lHBr8+slPGOHCNeSWqQktV4LlS454sBQP3UbaSRWaTMJ2aTwCYrNs2DcB8Zh4KYL7yG0Q0AsCFAEYqxzxMRPbDTYRQUV+ccx/+wHdd2RjPbRVMthIBw3t29ORjDoL3bjolkvmajChtXxyYEHejwENZZMdBmRnnjsJMzSJJ+slrAFBdn5u08mESmlJg5gUA9F3CqQCeVL4/CeAczfZnmbmOmdcDWAPgmLDaJjgjSG/MPz/eGFxlJoTpPTr+kOSymkHrtn5d26WypUadvl2MZw57uSWqGypnHj8H57342IEYr1lOVT95DQC27avL2BZ3sj38oSczbwUA5bOHsr0vgE2acpuVbRkQ0TVEtIiIFlVWZua+EYIjSB/9Wyt2BFaXGVbttbsUO0vmulPCWX0uF3j9twYpwHM9mdvLs200EGLb/pogmhMpojIk1fEAYGZ+hJkrmLmirKws5Ga1boJUCm7WIb75zEMdlRuk5BBSsWpuUC6IKMSy3bjigmwvBVifm2crjHvupUqjeSW1DtbyiBvZVgrbiag3ACifavdxM4D+mnL9AGzJctuyTmNTs6fp9tnC78v46pKtKJ82B/trGxwHq39+2jBcc9IQR2X1Pt7WNvrICvVOBOrusrm/Rj53M1oCzbn5n3kZDmxkKWQrWWQ2ybZSmA3gMuX7ZQBe0my/kIhKiGgQgKEAFma5bVnn+mc/w6g7Xs91MwAkZxRf9/Ti1BKcQfCXd9YCANZVHnQ818HAbWtelvRKwfmxWkSXOINgLcTPO6qf6b6MulwFmoPHk6VgMCQ19LTyOSDMIakzAXwIYDgRbSaiqwDcA2AiEa0GMFH5DWZeCmAWgGUAXgNwHTNnb+HdHPHKkm25bkKKX7+yAnOWbMUby4zbVN/Y7FphFCsSfl9NA2oanB3rpgenFyxhzWjWEhcZEEYzg1xlrGWpV2ctLe/WLvU8BYGX58HIfZSPWVJDG/bAzBeZ7JpgUn4GgBlhtUewxu4dmfzgAqzbeRAb7pniuE51vYDLHndu9Ol7/1r0KSay6S6K/rrG2SGoNB+qq8mpUph/48mBnFfFyyQ8Q0shD91H8RgLJ2QNs3d03c6DrusqKnTfs3Mj6HOUOigyZDtvEHNwi9WrBqGj+shdvMLZ+YOKKQTRmmjRyl+raBAFE1R9R7Q9Qb9Cp9hDFlAr95G+t+6m955PmUGnnXEYOpYUorfnpUeDuRffcRFDyGiB8v/I1bOvvQNO76OR+8jO0vG6ZkYuiV+L85AI6ARDMeFXdOgXxnGCGz2S67HuuWLyqF5YctfpaFNkPuk/rFujCvGu7Ypw33dHe64nlUIlRw+/aimM6tsJH0439GhnYGSt2LmPgrZwsoEohQgQpfwpQTalyENg0M1LlIvef3T+U9mHqOX6tffeyzNz65QR6FBSiNL2xfbnDSPNheq+cuH+Mer121sKohQED0QhVtVizmu3eauruZnxjw83eArCuRH07vLnpPOrqSOdHywASD4b6vPhV9R9e3QffHXX6Y5W1AsT9Ql94+cn2ZY1ch/ZDbU2WoMh6kigOQJEwVJw+ugys63gfvWrbRnrJTjFylLwM/pIX3S0x8ykcXnFw36itPczbIMtzPUUVHfYsJ4dbY8xDDSLpdA6aWxqdpWmwS251gkPzV+N5z/7JtkWzXajl7HewX3YV9PguS1WQ1L1+FtBLH4vaxQIcuW5XKJ6gty8e0YJ8ZptYwrxE7Hxa3EOOO6eN3HEnW+EVn+uLYX7565KfdeOBjESnPWN9kqh0Y2jVoe7yWveBXseDUZyja9rTz0e8b6BXhaQMur1S0yhlbKjqs7xjFwv2K045rv+pmY0OrR07Fry9spKHPk/b6Cq1twaqPORJMzqHdIrKT8xhXT3R2ZFz5iscZAf/WTvlHUswSXjBuDJK49ObQu7TxNKmovUEGznGLuPrI+R0UethMfeXYcP1uwMrL4Zc5YFVpcRFTPm4bh73nRW2OYhv+v/lmJPdQPWVppPZpvxynIXrUvHS1I1L9gdO6pf57TfcbMswmouEeHucw7HyD6d7QtHGPUx01rGf7zoSMtjDOcp2HToxFJoJdw9Zzm+99jHpvunPfclfjHrc8f1zVq0OYhmmbK3ugE7qtwvBmIkCHceqAcAtCsOZ9SIqxnNruIPLWWPKS+1DSzG71U2xigGEJdrU+N4RsLYP5mj7b49uo/lEUZDUu3cqWIptGL2VTfglheWoLahCc9+sgnPL/4m103yhNNAopPYghfcjD5ylWlTU/be84+wfVn1LiV1PP2hvexHqUSduLjAtuytBQD07my84psfCjy4j4yemZXbqyyP8TJXJ9fIkFQXNDWzqTB5cP4qPP3xRgyPudBgbrlOK7GpH42lmtFe8tRricrkNX3Nw3p2xL+vHY/R/bwNZc0VcR5ldfrInpi5cBN6dvKazsMct1laAW8C3u/7kAvip8ZyyJCbX0GtScBZFYpaH+O6ygP47Wsr0NDUjDtnL8X2/bWht/GDNTvx+HvrPR9/8wtLMOTmV5I/LIRugy7CNv6e+Tjp3rc8n1fFTWqMMN83o0s/urzUU+qOXBCGNWBXZ9A6+ldTR+HLOyeF4oJJzWh2oRSctEP/fEhMoRVgphRUtI/YlU98gr+8vRazFm3CEx9swG0vfmV57LIt+7F4457U76++2YfyaXPw4dpdYGZHycO+99jH+J+XnQeu11YeSPut6jS70UpVtQ34zSvLcfSMedhRVYvt++uweY//9Wrd5Mx3FVOw2T9ucLe033HuYdsRlysrTBSgU5uiUOomg5gCAFx+XDkev7zC8Jgim9jG0B4d8N5Np6Rti6NSEPeRAdv316IoUWCYl0WfQvrZhRtNzVu1N62me7CaADdz4UZMf34JAKTWLDjrj+8BAO6Y/RVWbT+A6Wccih9+y9lSlXWNTY5SCEy47x3D7QfrrJXf3XOWY71yL+Yu257avmaHtY/VjlxZCmcc3huLb5uIo341F0D8RhvpUZufL5PNgiY1JFV3e+482zz9iZ2lMKxnR/TomC4L9J2NOCCWgoaF63djzY4DOPbX8zH27rmGZa564pO039OeX4IrdNv0qNaFlQ9cVQhG7D6YHPHzjw+/tjzPlZp2qKOEtOhzEVnNNaiqa7C0TDbtrk591y5eftr9CyzbqHLUAGPfvJXfNrP37jHSbIKT5GytmZjrSYzo3Sn13dPKazazk/UKeMM9U9C/tJ37E+UYUQoavvu/H+K0+5M9Z2Zgxbb9GWX2mqRwMBL43+xNulN+/coKX+1SBXxDUzOWbtmH6c8vQU19E8qnzcEdL7W4pN5csSP1/UBtY0Y9t774Fe6cvRR7q+uVdpnPJ6iqbbQcYaSdcGfnUjPiCJOArTv3kfPzxV2gBYnXAH3cbY7nf3wcvrxzEgBvgWa7obG5TlcTFKIULPj06z0Z27T/+Kc/bum5a10oZvhduu9AXSOmPPQeZi7ciL+9tw4A8KSJ9WA0A3vmwo144oMN+I2ipNQhf0bsq2nAim3OXEH3vr7SUTktFeVdDbcXF4aTEM8tcXcfqRjFRoJe2CYuLqo2RYlUjMJLoNnueTtuSPxcRUZITMEHt7zQ0ktXrQIrwV/XmBTUuw7U4U9vrTEt99nGPYY96er6FkGvdQ/d/MISbNxVnVb2nD+/j2MGleKScQMz6tmwKxkLGNmnE95ZVWnYhvvecC/o3XDaYT0Nt0dlhbR8CTTHRWBnG7NAs/Ux5iy8ZQLKOpT4a1REEKVggRfBoBXcej5atxs/eWYxauqbMF/j6tFz7sMf4MXrjrc8z57qFqXwzMcbDcssXL8bC9fvzti+cnsVPt+0F53amo/s+GRDppUUJGZBO6/psF/+6QmOy+rp2CbzNYiIbvKMlayLiuLNJV4mr7W1mMWvDzDHGVEKFnh5d4ziEFpe/nIrBtgEnwoIKLEZhfPS51tct01lb3UDzvnz+56PDwKzFNlu4gRBLAgEAH+yyXkj5B8lylKmw3p2cHxMm6IEFt16GirunhdWsyJBq1UKD7+9Btv31eKuqaNMyyzbst9WyOt5Zck22zJ2KSI6ty3C+wEm3AOAIWXtLZPYZRszId63i7eUBnZuACOrb871J6BLu2L0MBhSHPe+dFDtLyDg1EN7YN5yc8s2jnRuW4SZV4/DyL6d7Atr6J4nLiIrWq1S+N1rSZ+5lVJ46qOv8dRH1sNAvbC7OnO4qJYCItw9x3umUSNK2xdHTClkiq1B3dsbLmSiok8xrg0SdiixfpSNlJBVpk9xsQCLb5uIdsUJbN5Tg3nLd2BI9/Y2R8Trno3Pk8Bw0LT60UcPzksuMONlWKVX7CyFXQetlYYXzhjVO/A6g6Jru/QRIWacNLQs7Xf3jsleW6c2hSi3EVhu3ARA3MSbO5xeW2n7YrQpSuCQHh3wxBVHY8a5h4faLiEaiFKYtxpb9takjSQSwmGUian+8MVjAdgHma875RDcNPlQAMCVxw/CsYNKAQAnDiuzOgwAMHZgaSxTDnhFVZhGeBmPdPLwHpaBViF/aPVKAQAqq+rw3OJw1zQAgBOHdg/9HGY0M+OOb49wfdy/rhmHX51j7mJzw8yrx6X9vuG0obh+wtDUpCA7F1BxYQGu/dZg/PWSsbh1ymGpYHwbB+k8AKCPi3hF3L1H3z4iF5ahDH/NB1qlUvhkQ/owzakGI3HMerUA0F7TYxrTP3M+wf87qq/hcU9dZbzEIwD07ux+SNuvpprnaTHiiuMHoV9X54Lx+lMPwbGDu+H7BnMdVGacOwq3TjnMcF+igLD4tomp3x11yc1uOG0YfjFxGPZVJ2eJq24kK4gIk0f1QkEBYeKIXvjpqYfg9rPcKzszVDdTnGMKw3p2sGz/wBimXhCyR6tUCgO72b8U2lEwF1T0T9vXRhnOdvtZI3D/d0entpcpJvthvTrhad0av+MGJ10dPznlEMPzdevgLu/Oyrsn4/vjyzO2dzERrGpMttFmUdkN90xJWTQ/nzgstf3Mw3vh9+ePzih/8bEDccXxg/Dn7x2Vtv3f147H+zedmpFP6MELxmDSiPSJa+rsa7d58xMFhBsnDUdnB8oEaFktzmr+ycyrx6XWZz7tsJ74w4VjXLUpCmjnyhRp8vX8/vzR2HDPFHQNOMeTeg4nCRiF6NMqlYI68/DqEweZltHOGP7VOaPwwbRTU0JfnXh15IAuGFzWIaVAphyeNNmbmHH8Id3x10vG4mxliT9VkXzv2AGpekf2abFGThneI+38k0f2slReRbrkXNdPGIqfTRiKzzQ9cy3HH5IU9Nd+a7BpnSqPXVaBD6efmtbbfPjisThvbD/87rwjUtv+ceUxAJL3Y4rOXXF0eSl6GVg/5xzZF49cmp6a+PSRvXDtt4bgZhOLIygevbQC/zVpGPqXmltL3TqU4DjlXj12WQWmjjG2+qLEp7eelvb76hNb/sfnHtUXPzp5CJbcOQnnje0XyvnVc9w4aRievWac/QEx54vbJ+GL2yeZ7n/wgjEZrtI4QUHnQckmFRUVvGjRIl91lE+bAwDo0bEE15w0GBcdMwDtSwqxansVJj2wAN8fNzDlU/9w7S48OG8V/uv04bjr/5biP9cehzZFCWzZW4MNuw5i14F6/HTmZ/j7FUenhPzSLfsw5aH3cMuZh+Hqk5Iv67xl23GwvhGnHNoDuw/U45f/+QIPXzwWZR1LsPtgPa596lP87rwjUiNqyqfNQXGiAN06FGPrvmS+IjW99t/eW49Nu6vTUv7OWrQJM+Ysx5mH98Lwnh1x+fGZyu9AXSPeX7MT3xpWhrdXVqJ35zbo1LYIg2yHHQKLN+5Bt/bFGNgtvax6L+dcf0LavHMamgAACF1JREFUcM+nPtyATzbswUMySSw0yqfNQceSQiy56/RcNwWbdlej8kAdjhpgnN8qn2DmWLoaiehTZjZcOCJySoGIJgP4A4AEgMeY+R6zskEohf21Dfhs414cNaBLhs97855qlHUscWUWr995MEOwbth5EAO7tfP88FRW1aGkqAB1Dc1YW3kAX32zDz840b7Hn232VteDGYG7JwR7dh6oQ3FheIvSCPlFbJQCESUArAIwEcBmAJ8AuIiZDZcSC0IpCIIgtDaslELUYgrHAFjDzOuYuR7AswCm5rhNgiAIrYaoKYW+ADZpfm9WtqUgomuIaBERLaqsNE77LAiCIHgjakrByOme5t9i5keYuYKZK8rK7GeyCoIgCM6JmlLYDEA7KaAfAO85ogVBEARXRE0pfAJgKBENIqJiABcCmJ3jNgmCILQaIpU6m5kbiegnAF5Hckjq48y8NMfNEgRBaDVESikAADO/AuCVXLdDEAShNRI195EgCIKQQyI1ec0tRFQJwM/SaN0BBLvuZe6Qa4kmci3RpLVfy0BmNhy+GWul4BciWmQ2qy9uyLVEE7mWaCLXYo64jwRBEIQUohQEQRCEFK1dKTyS6wYEiFxLNJFriSZyLSa06piCIAiCkE5rtxQEQRAEDaIUBEEQhBStUikQ0WQiWklEa4hoWq7bYwcR9Seit4hoOREtJaKfKdtLiWguEa1WPrtqjpmuXN9KIsr9Go0aiChBRJ8R0cvK71heBwAQURci+g8RrVD+P+Pjej1E9HPl+fqKiGYSUZu4XAsRPU5EO4joK802120norFEtETZ9xDlYK1Nk2u5V3nGviSiF4ioS2jXwsyt6g/JnEprAQwGUAzgCwAjct0umzb3BnCU8r0jkqvTjQDwOwDTlO3TAPxW+T5Cua4SAIOU603k+jo01/MLAM8AeFn5HcvrUNr4JIAfKN+LAXSJ4/UguW7JegBtld+zAFwel2sBcBKAowB8pdnmuu0AFgIYj2Qa/1cBnBGRa5kEoFD5/tswr6U1WgqxW92Nmbcy82LlexWA5Ui+xFORFEpQPs9Rvk8F8Cwz1zHzegBrkLzunENE/QBMAfCYZnPsrgMAiKgTki/w3wCAmeuZeS9iej1I5kJrS0SFANohmbY+FtfCzAsA7NZtdtV2IuoNoBMzf8hJqfoPzTFZw+hamPkNZm5Ufn6E5LICQAjX0hqVgu3qblGGiMoBHAngYwA9mXkrkFQcAHooxaJ8jQ8C+G8AzZptcbwOIGltVgL4u+IOe4yI2iOG18PM3wD4PYCNALYC2MfMbyCG16LBbdv7Kt/126PGlUj2/IEQrqU1KgXb1d2iChF1APAcgBuYeb9VUYNtOb9GIjoLwA5m/tTpIQbbcn4dGgqRNPP/wsxHAjiIpJvCjMhej+Jvn4qkC6IPgPZEdInVIQbbInEtDjBre+SviYhuAdAI4Gl1k0ExX9fSGpVCLFd3I6IiJBXC08z8vLJ5u2ImQvncoWyP6jUeD+BsItqApNvuVCL6J+J3HSqbAWxm5o+V3/9BUknE8XpOA7CemSuZuQHA8wCOQzyvRcVt2zejxS2j3R4JiOgyAGcBuFhxCQEhXEtrVAqxW91NGTXwNwDLmfl+za7ZAC5Tvl8G4CXN9guJqISIBgEYimTQKacw83Rm7sfM5Uje9zeZ+RLE7DpUmHkbgE1ENFzZNAHAMsTzejYCGEdE7ZTnbQKSsas4XouKq7YrLqYqIhqn3INLNcfkFCKaDOAmAGczc7VmV/DXku3IehT+AJyJ5AietQBuyXV7HLT3BCRNvy8BfK78nQmgG4D5AFYrn6WaY25Rrm8lcjCCwsE1nYyW0Udxvo4xABYp/5sXAXSN6/UAuAvACgBfAXgKyREtsbgWADORjIU0INlLvspL2wFUKNe/FsCfoGR9iMC1rEEydqC+/38N61okzYUgCIKQojW6jwRBEAQTRCkIgiAIKUQpCIIgCClEKQiCIAgpRCkIgiAIKUQpCK0SImoios81f5bZconoWiK6NIDzbiCi7h6OO52I7iSirkT0it92CIIZhblugCDkiBpmHuO0MDP/NczGOOBEAG8hmYDv/Ry3RchjRCkIggYlBce/AJyibPoeM68hojsBHGDm3xPR9QCuRTIHzTJmvpCISgE8jmSSvGoA1zDzl0TUDcnJSGVIzvglzbkuAXA9kim3PwbwY2Zu0rXnAgDTlXqnAugJYD8RHcvMZ4dxD4TWjbiPhNZKW5376ALNvv3MfAySs0AfNDh2GoAjmfkIJJUDkJwN/Jmy7WYkUxUDwB0A3uNkwrzZAAYAABEdBuACAMcrFksTgIv1J2Lmf6Elt/7hSM5QPVIUghAWYikIrRUr99FMzecDBvu/BPA0Eb2IZGoLIJmK5DsAwMxvElE3IuqMpLvn/ynb5xDRHqX8BABjAXyiLIjVFi0J2/QMRTJVAQC04+SaGoIQCqIUBCETNvmuMgVJYX82gNuIaCSsUxUb1UEAnmTm6VYNIaJFALoDKCSiZQB6E9HnAH7KzO9aX4YguEfcR4KQyQWazw+1O4ioAEB/Zn4LycWCugDoAGABFPcPEZ0MYCcn17zQbj8DyYR5QDJB23lE1EPZV0pEA/UNYeYKAHOQjCf8DskEjmNEIQhhIZaC0Fppq/S4VV5jZnVYagkRfYxkp+ki3XEJAP9UXEME4AFm3qsEov9ORF8iGWhWUzbfBWAmES0G8A6SKarBzMuI6FYAbyiKpgHAdQC+NmjrUUgGpH8M4H6D/YIQGJIlVRA0KKOPKph5Z67bIgi5QNxHgiAIQgqxFARBEIQUYikIgiAIKUQpCIIgCClEKQiCIAgpRCkIgiAIKUQpCIIgCCn+P/IoJPKGFTUKAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "scores = train(2000)\n",
    "# plot the scores\n",
    "fig = plt.figure()\n",
    "ax = fig.add_subplot(111)\n",
    "plt.plot(np.arange(len(scores)), scores)\n",
    "plt.ylabel('Score')\n",
    "plt.xlabel('Episode #')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Your score is:  500.0\n"
     ]
    }
   ],
   "source": [
    "# watch an trained agent\n",
    "state = env.reset()\n",
    "agent.reset()\n",
    "score = 0\n",
    "while True:\n",
    "    action, _ = agent.act(state)\n",
    "    env.render()\n",
    "    next_state, reward, done, _ = env.step(action)\n",
    "    score += reward\n",
    "    state = next_state\n",
    "    if done:\n",
    "        print(\"Your score is: \", score)\n",
    "        break \n",
    "        \n",
    "env.close()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}