notes/03_temporal_difference/mountain_climb_sarsa.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# SARSA\n",
    "\n",
    "In this notebook, we will implement SARSA algorithm to play Hill Climbing.\n",
    "\n",
    "## Step 1: Import the libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import time\n",
    "import random\n",
    "import gym\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from collections import namedtuple, deque"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 2: Create our environment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\users\\deepanshu\\.conda\\envs\\ai\\lib\\site-packages\\gym\\logger.py:30: UserWarning: \u001b[33mWARN: Box bound precision lowered by casting to float32\u001b[0m\n",
      "  warnings.warn(colorize('%s: %s'%('WARN', msg % args), 'yellow'))\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[1]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "env = gym.make('MountainCar-v0')\n",
    "env._max_episode_steps = 1000\n",
    "env.seed(1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 3: Viewing our Enviroment\n",
    "Execute the code cell below to play Cartpole with a random policy."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The size of state is:  (2,)\n",
      "No. of Actions:  3\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAW4AAAEICAYAAAB/Dx7IAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAZEUlEQVR4nO3df7BcZ33f8fcHWcgE27EdXXuEJCLVEW1kJsjxHZXGaetighUnVKZTMiI/UGacEdPaKbTuDwumSOpUU5LB0F9jJiL2RBMDivjhWmVIQFFhCDOOxZWRsWRZ+AYr1kW30rXBsZ1QBcnf/rHP2sdXe+/dX2f3PGc/r5mdPfvsOWef5969n/3eZ8/ZVURgZmb5eM2wO2BmZp1xcJuZZcbBbWaWGQe3mVlmHNxmZplxcJuZZcbBbZUl6YOSfr/f67axr5D0U/3Yl1kZ5OO4bRAk/SZwJ3AN8DzwALA1Ip4bZr9akRTAmoiYbHHf14C3AucKzb8QEQ8NqHtmrritfJLuBH4H+HfAj9MIvp8E9kt67RzbXDS4Hnbsjoi4pHC5ILQr3n/LnIPbSiXpMmAH8NsR8ScR8aOIOAH8Co3w/vW03nZJn5N0v6Tngd9MbfcX9vVeSX8p6VlJ/1HSCUlvL2x/f1pelaY7Nkt6WtIzkj5U2M96SQ9Jek7StKT/OdcLSIdjDUm3S3oSeDK1/TdJJyU9L+mQpH9YWH+7pM+mMb8g6TFJb5K0VdKZtN07Cuv/uKR7U5+/J+k/S1rUa78tPw5uK9vPARcDXyg2RsSLwB8Dv1Bo3gh8Drgc+FRxfUlrgXuAXwOW0ajcly/w2D8P/F3gJuDDkn46tZ8H/jWwFPgH6f5/2eG45nIr8PeBten2N4F1wJXAp4HPSrq4sP47gT8ErgC+BXyZxt/lcuA/Ab9XWHc3jSmanwKuA94B/Faf+m0ZcXBb2ZYCz0TEuRb3Taf7mx6KiP8VES9FxA9nrfvPgf8dEd+IiL8FPgws9AbNjoj4YUQ8CjwKvAUgIg5FxJ9HxLlU/f8e8I87GNN/T9X6c5IemXXff4mI7zf7HxH3R8Sz6bHuBpbQeDFp+rOI+HL6+XwWGAM+EhE/AvYAqyRdLulq4BeBD0TEX0fEGeDjwKYO+m014Xk4K9szwFJJF7UI72Xp/qaT8+znDcX7I+JvJD27wGP/38Ly3wCXAEh6E/AxYBz4MRp/B4cW2FfRv4qIuY5gedUY0vz+b6X+B3AZr36xOl1Y/iGNF7nzhdukfr8BWAxMS2qu/5rZj2ejwRW3le0h4Czwz4qNkl5Po4I8UGier4KeBlYUtn8d8BNd9ukTwBM0jhy5DPggoPk3advLY0jz2f+Bxnz+FRFxOfBXXT7WSRo/x6URcXm6XBYR1/aj05YXB7eVKiL+isabk/9D0gZJiyWtojEtMEVjfrcdnwPeKenn0huJO+g+bC+lcUjii5L+HvAvutxPO49zDpgBLpL0YRoVd8ciYhr4CnC3pMskvUbSNZI6meKxmnBwW+ki4ndpVLUfpRGYD9OoIG+KiLNt7uMo8Ns05n2ngReAMzSq0E79W+BX0z4+CfxRF/tox5dpvAH7HeAvgf9Hb1Mb7wVeCzwO/IDGi9myHvtoGfIJOJYlSZcAz9GY7nhq2P0xGyRX3JYNSe+U9GNpfvyjwGPAieH2ymzwSgvuNJ95XNKkpLvKehwbKRuBU+myBtgU/pfRRlApUyXpbK7v0Di5YorGSQjviYjH+/5gZmYjpqyKez0wGRHfTSdL7KFRLZmZWY/KOgFnOa9+93yKxmnALS1dujRWrVpVUlfMzPJz4sQJnnnmmZaHvJYV3K0e7FVzMpK2AFsA3vjGNzIxMVFSV8zM8jM+Pj7nfWVNlUwBKwu3V9B4Q+llEbErIsYjYnxsbKykbpiZ1U9Zwf1NYI2k1ekst03AvpIey8xspJQyVRIR5yTdQePMsUXAfenMNzMz61Fpnw4YEV8CvlTW/s3MRpXPnDQzy4yD28wsMw5uM7PMOLjNzPpIEocO9et7OVrzV5eZmZVgrvC+/vrePx/KwW1mNkCtAr3TMPdUiZlZZlxxm5kNkKdKzMwqqh8BPRdPlZiZ9VmZoQ0ObjOz7Di4zcwy4+A2M8uMg9vMLDMObjOzzDi4zcwy4+A2M8uMg9vMLDMObjOzzPR0yrukE8ALwHngXESMS7oS+CNgFXAC+JWI+EFv3TQzs6Z+VNz/JCLWRcR4un0XcCAi1gAH0m0zM+uTMqZKNgK70/Ju4NYSHsPMbGT1GtwBfEXSIUlbUtvVETENkK6varWhpC2SJiRNzMzM9NgNM7PR0evHut4QEackXQXsl/REuxtGxC5gF8D4+Hi5H6VlZlYjPVXcEXEqXZ8BHgDWA6clLQNI12d67aSZmb2i6+CW9HpJlzaXgXcAR4B9wOa02mbgwV47aWZmr+hlquRq4AFJzf18OiL+RNI3gb2SbgOeBt7dezfNzKyp6+COiO8Cb2nR/ixwUy+dMjOzufnMSTOzzPjLgs3M+iRNHb98vZCI7g6oc3CbmfWg3ZBuZ9t2g9zBbWbWgV6Cul/7dnCbmc1joTDtdrqjm8dqcnCbmbUwV4j2M6jn2/f4+Pic6zm4zcySVmFdZlB3y8FtZiMvl8BucnCb2Ujr9siOYXJwm9lIyjGwmxzcZjZScg7sJge3mY2EOgR2k4PbzGqvGNo5B3aTg9vMaqtugd3kTwc0s1oq89T0YXPFbWa1U9dKu8nBbWa10gztOgZ2k4PbzGqh7lV20YJz3JLuk3RG0pFC25WS9kt6Ml1fUbhvq6RJSccl3VxWx83MmkYptKG9Nyf/ANgwq+0u4EBErAEOpNtIWgtsAq5N29wjaVHfemtmViDpVVMjoxDa0EZwR8TXge/Pat4I7E7Lu4FbC+17IuJsRDwFTALr+9RXM7OXjVqVXdTt4YBXR8Q0QLq+KrUvB04W1ptKbReQtEXShKSJmZmZLrthZqNu1EIb+n8cd6sDJ1v+VCNiV0SMR8T42NhYn7thZnU2CkeOzKfb4D4taRlAuj6T2qeAlYX1VgCnuu+emdmrjXpoQ/fBvQ/YnJY3Aw8W2jdJWiJpNbAGONhbF83MLnwjcpQteBy3pM8ANwJLJU0B24CPAHsl3QY8DbwbICKOStoLPA6cA26PiPMl9d3MRsQovxHZyoLBHRHvmeOum+ZYfyews5dOmZk1ucq+kD9kyswqz6H9aj7l3cwqyZX23Fxxm1nlOLTn5+A2s0pxaC/MwW1mleHQbo+D28wqwaHdPge3mQ2dQ7szDm4zs8w4uM1sqFxtd87BbWZD49Dujk/AMbOB82eP9MYVt5kNlEO7dw5uMxsKh3b3HNxmNjCe0+4PB7eZDYRDu38c3GZWOod2fzm4zaxUDu3+c3CbWWmKR5BY/ywY3JLuk3RG0pFC23ZJ35N0OF1uKdy3VdKkpOOSbi6r42aWD1fb/dVOxf0HwIYW7R+PiHXp8iUASWuBTcC1aZt7JC3qV2fNLB+eIinPgsEdEV8Hvt/m/jYCeyLibEQ8BUwC63von5llyKFdrl5Oeb9D0nuBCeDOiPgBsBz488I6U6ntApK2AFsKt/1LNqsBh3b5un1z8hPANcA6YBq4O7W3eiei5W8vInZFxHhEjF9//fWNjf1GhlnWHNqD0VVwR8TpiDgfES8Bn+SV6ZApYGVh1RXAqd66aGZmRV0Ft6RlhZvvAppHnOwDNklaImk1sAY42M4+m6/QrrrN8uRqe3AWnOOW9BngRmCppClgG3CjpHU0pkFOAO8DiIijkvYCjwPngNsj4ny7nYkIJHm+2ywzDu3BWjC4I+I9LZrvnWf9ncDOXjplZvnwf8mDV7kzJ4tTJn5CmFVbsdJ2tT04lQtu8L9bZjnw9MjwVDK4wW9WmpnNpbLBDQ5vs6pytT1clQ5uMzO7UOWD21W3WXUUDxpwtT08lQ9ucHibVYG/nb06sghucHibVYVDe/iyCW5weJsNi6dHqiWr4DYzswyD21W32WC52q6e7IIbHN5mg+LQrqYsgxsc3mZlc2hXV7bBbWblcUFUbVkHt6tus/7z8drVl3Vwg8PbrCwO7erKPriLHN5mvfG8dh5qEdzFJ5nD26w7Du18LBjcklZK+qqkY5KOSnp/ar9S0n5JT6brKwrbbJU0Kem4pJvLHECTn2xmNiraqbjPAXdGxE8DbwVul7QWuAs4EBFrgAPpNum+TcC1wAbgHkmLyuj8bJ7vNuuOq+28LBjcETEdEY+k5ReAY8ByYCOwO622G7g1LW8E9kTE2Yh4CpgE1ve74/P0F3B4m7XLoZ2fjua4Ja0CrgMeBq6OiGlohDtwVVptOXCysNlUapu9ry2SJiRNzMzMdN5zM+uZC5w8tR3cki4BPg98ICKen2/VFm0XvJRHxK6IGI+I8bGxsXa70RZX3WadcbWdl7aCW9JiGqH9qYj4Qmo+LWlZun8ZcCa1TwErC5uvAE71p7vtc3ibzc9TJPlq56gSAfcCxyLiY4W79gGb0/Jm4MFC+yZJSyStBtYAB/vX5c45vM1ezaGdt4vaWOcG4DeAxyQdTm0fBD4C7JV0G/A08G6AiDgqaS/wOI0jUm6PiPN973kbIuLlJ6gkP0nNcGjXwYLBHRHfoPW8NcBNc2yzE9jZQ7/6phjeZmZ1UIszJxfi+W6zBlfb9TASwQ0ObzOHdn2MTHCbjTIXLPUyUsHtqttGkT9fu35GKrjB4W2jy6FdHyMX3ODwttHhee16GsngNjPL2cgGt6tuqztX2/U1ssENDm+rL4d2vY10cIPD2+rHoV1/Ix/cZnXiAmQ0OLhx1W314OO1R4eD28wsMw7upFh1u/K23BTntV1t15+Du8BPeDPLgYN7Fs93W258FMnocXC34PC2XDi0R5ODew4Ob6s6h/boaufLgldK+qqkY5KOSnp/at8u6XuSDqfLLYVttkqalHRc0s1lDsBsFLmgGG3tfFnwOeDOiHhE0qXAIUn7030fj4iPFleWtBbYBFwLvAH4U0lvGtYXBvei+X2V/qJhqyo/L0fTghV3RExHxCNp+QXgGLB8nk02Ansi4mxEPAVMAuv70dlh8JSJVY2nSKyjOW5Jq4DrgIdT0x2Svi3pPklXpLblwMnCZlPMH/TZcHjbsDm0DToIbkmXAJ8HPhARzwOfAK4B1gHTwN3NVVtsfsGzTNIWSROSJmZmZjru+CAV/0gc3jYsDm1raiu4JS2mEdqfiogvAETE6Yg4HxEvAZ/klemQKWBlYfMVwKnZ+4yIXRExHhHjY2NjvYxhIPzHYmZV0c5RJQLuBY5FxMcK7csKq70LOJKW9wGbJC2RtBpYAxzsX5eHx/PdNiyutq2onaNKbgB+A3hM0uHU9kHgPZLW0ZgGOQG8DyAijkraCzxO44iU23M8omQuPtLEBs2hbbMtGNwR8Q1az1t/aZ5tdgI7e+iXmeH/7qw1nznZBU+Z2CD487VtLg7uLjm8bVAc2jabg7sHDm8ri+e1bT4O7j5xeFu/OLRtIQ7uHvmPy8wGzcHdB54ysX5xtW3tcHD3icPbeuXQtnY5uPvI4W3dcmhbJxzcfebwtk45tK1TDm4zs8w4uEvgqtva5WrbuuHgLonD2xbi0LZuObgHwOFtszm0rRcO7hJFhCtvu4BD23rl4B4Ah7c1ObStHxzcZgPiF27rFwf3gLjqtiZX29YrB/cAObxHl6dIrJ8c3APm8B49Dm3rt3a+5f1iSQclPSrpqKQdqf1KSfslPZmuryhss1XSpKTjkm4ucwA5c3jXn0PbytBOxX0WeFtEvAVYB2yQ9FbgLuBARKwBDqTbSFoLbAKuBTYA90haVEbnc+XDBEeDQ9vKsmBwR8OL6ebidAlgI7A7te8Gbk3LG4E9EXE2Ip4CJoH1fe11TTi868uhbWVqa45b0iJJh4EzwP6IeBi4OiKmAdL1VWn15cDJwuZTqW32PrdImpA0MTMz08sYzCrFL8RWtraCOyLOR8Q6YAWwXtKb51m91bP2grIjInZFxHhEjI+NjbXX2xpy1V0vxUrb1baVpaOjSiLiOeBrNOauT0taBpCuz6TVpoCVhc1WAKd67mmNObzNrBPtHFUyJunytPw64O3AE8A+YHNabTPwYFreB2yStETSamANcLDfHa8bh3f+PK9tg3JRG+ssA3anI0NeA+yNiC9KegjYK+k24Gng3QARcVTSXuBx4Bxwe0ScL6f79RIRSEKS//gz49C2QVowuCPi28B1LdqfBW6aY5udwM6eezeCHN55Kf6H5N+XDYrPnKwgT5vkx6Ftg+TgriiHd/U1/ytyaNugObgrzOFdXf6d2DA5uCvO4V09fiPShs3BnQGHd3U4tK0KHNyZcHgPV/NIH3Bo2/A5uDPi8B4+h7ZVgYM7Mw7vwXOlbVXj4M5QMbwd4OXx9IhVlYM7U8UgGcXwboZqWS9ePiPSqqydzyqxippdedc5YBYK536O31W2VZ2Duwbq9vkmw/oPwlW25cJTJTWT+7RJ7v03GwRX3DXRrLqhv9MGZapSSHt6xHLi4K6RVkebVDGIqhjYUM2flVkrniqpoVE94qTTsTq0LVeuuGuqzkecbN++fd7bC3FgW+5ccddcFU/W6SUsW4X09u3b2w5vh7bVQTtfFnyxpIOSHpV0VNKO1L5d0vckHU6XWwrbbJU0Kem4pJvLHIAtrC5TJ51W1rM5tK0u2pkqOQu8LSJelLQY+IakP073fTwiPlpcWdJaYBNwLfAG4E8lvclfGDxcs486abZVSa9TIHOt78C2ulmw4o6GF9PNxeky37N/I7AnIs5GxFPAJLC+555az2Z/zVaVqu+5pkB6MXt6yKFtddHWHLekRZIOA2eA/RHxcLrrDknflnSfpCtS23LgZGHzqdQ2e59bJE1ImpiZmelhCNapYoCX+XkfC/Whab6A7ia8WwW2Q9vqpK3gjojzEbEOWAGsl/Rm4BPANcA6YBq4O63eKgEu+KuJiF0RMR4R42NjY1113nozO8yqVIEXtRu827Ztc2DbSOjoqJKIeA74GrAhIk6nQH8J+CSvTIdMASsLm60ATvWhr1aCZrgNuwJvx7Zt29i2bVvL+7Zv3/6quXsHttVZO0eVjEm6PC2/Dng78ISkZYXV3gUcScv7gE2SlkhaDawBDva32zYIVQzvVorTKQ5sGwXtHFWyDNgtaRGNoN8bEV+U9IeS1tGYBjkBvA8gIo5K2gs8DpwDbvcRJXlo9e06Zb6519zfjh07Otpurnlvh7aNClXhyT4+Ph4TExPD7oa10KrqLuM5M1d4N6dG5qr+q/D8NSvD+Pg4ExMTLZ/4Dm5r23xTJ/16Hu3YseOCNxnLeiyzKpsvuP1ZJda2+b6ouJugnWubVlMhDmuzVzi4rWOtQnS+4O7mTU4HtdncHNzWF/04I9NhbdYeB7f1nQPYrFz+WFczs8w4uM3MMuPgNjPLjIPbzCwzDm4zs8w4uM3MMuPgNjPLjIPbzCwzDm4zs8w4uM3MMuPgNjPLjIPbzCwzDm4zs8w4uM3MMuPgNjPLjIPbzCwzlfiyYEkzwF8Dzwy7LyVYiseVm7qOzePKy09GxFirOyoR3ACSJiJifNj96DePKz91HZvHVR+eKjEzy4yD28wsM1UK7l3D7kBJPK781HVsHldNVGaO28zM2lOlitvMzNrg4DYzy8zQg1vSBknHJU1KumvY/emUpPsknZF0pNB2paT9kp5M11cU7tuaxnpc0s3D6fXCJK2U9FVJxyQdlfT+1J712CRdLOmgpEfTuHak9qzH1SRpkaRvSfpiul2XcZ2Q9Jikw5ImUlstxtaViBjaBVgE/AXwd4DXAo8Ca4fZpy7G8I+AnwWOFNp+F7grLd8F/E5aXpvGuARYnca+aNhjmGNcy4CfTcuXAt9J/c96bICAS9LyYuBh4K25j6swvn8DfBr4Yl2ei6m/J4Cls9pqMbZuLsOuuNcDkxHx3Yj4W2APsHHIfepIRHwd+P6s5o3A7rS8G7i10L4nIs5GxFPAJI2fQeVExHREPJKWXwCOAcvJfGzR8GK6uThdgszHBSBpBfBLwO8XmrMf1zzqPLZ5DTu4lwMnC7enUlvuro6IaWgEIHBVas9yvJJWAdfRqE6zH1uaTjgMnAH2R0QtxgX8V+DfAy8V2uowLmi8uH5F0iFJW1JbXcbWsYuG/Phq0Vbn4xOzG6+kS4DPAx+IiOelVkNorNqirZJji4jzwDpJlwMPSHrzPKtnMS5JvwyciYhDkm5sZ5MWbZUbV8ENEXFK0lXAfklPzLNubmPr2LAr7ilgZeH2CuDUkPrST6clLQNI12dSe1bjlbSYRmh/KiK+kJprMTaAiHgO+BqwgfzHdQPwTyWdoDHl+DZJ95P/uACIiFPp+gzwAI2pj1qMrRvDDu5vAmskrZb0WmATsG/IfeqHfcDmtLwZeLDQvknSEkmrgTXAwSH0b0FqlNb3Asci4mOFu7Iem6SxVGkj6XXA24EnyHxcEbE1IlZExCoaf0f/JyJ+nczHBSDp9ZIubS4D7wCOUIOxdW3Y744Ct9A4YuEvgA8Nuz9d9P8zwDTwIxqv9LcBPwEcAJ5M11cW1v9QGutx4BeH3f95xvXzNP69/DZwOF1uyX1swM8A30rjOgJ8OLVnPa5ZY7yRV44qyX5cNI46ezRdjjZzog5j6/biU97NzDIz7KkSMzPrkIPbzCwzDm4zs8w4uM3MMuPgNjPLjIPbzCwzDm4zs8z8f1ztokWCW8WiAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "print(\"The size of state is: \", env.observation_space.shape)\n",
    "print(\"No. of Actions: \", env.action_space.n)\n",
    "env.reset()\n",
    "plt.figure()\n",
    "plt.imshow(env.render(mode='rgb_array'))\n",
    "env.close()\n",
    "plt.title('Original Frame')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def random_play():\n",
    "    env.reset()\n",
    "    while True:\n",
    "        env.render(mode='rgb_array')\n",
    "        action = env.action_space.sample()\n",
    "        state, reward, done, _ = env.step(action)\n",
    "        if done:\n",
    "            env.close()\n",
    "            break\n",
    "random_play()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 4: Creating State Preprocess Function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "pos_space = np.linspace(-1.2, 0.6, 12)\n",
    "vel_space = np.linspace(-0.07, 0.07, 20)\n",
    "\n",
    "def get_state(state):\n",
    "    pos, vel = state\n",
    "    \n",
    "    pos_bin = np.digitize(pos, pos_space)\n",
    "    vel_bin = np.digitize(vel, vel_space)\n",
    "    return (pos_bin, vel_bin)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 6: Creating out agent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "GAMMA = 0.99            # discount factor\n",
    "LR = .1                 # learning rate\n",
    "\n",
    "class Agent():\n",
    "    def __init__(self, p_size = 12, v_size = 20, action_size = 3):\n",
    "        self.action_space = list(range(0, action_size))\n",
    "        \n",
    "        # Intilizing state\n",
    "        self.states = []\n",
    "        for p in range(p_size + 1):\n",
    "            for v in range(v_size + 1):\n",
    "                self.states.append((p, v))\n",
    "        \n",
    "        # Iitliazing Q\n",
    "        self.Q = {}\n",
    "        for s in self.states:\n",
    "            for a in self.action_space:\n",
    "                self.Q[s, a] = 0\n",
    "        \n",
    "                \n",
    "    def act(self, state, eps=.0):\n",
    "        \"\"\"Returns actions for given state as per action value function.\"\"\"\n",
    "        state = get_state(state)\n",
    "        values = np.array([self.Q[state, a] for a in self.action_space])\n",
    "        action = np.argmax(values)\n",
    "        g_action = self.eps_action(action, eps)\n",
    "        return g_action\n",
    "    \n",
    "    def eps_action(self, action, eps):\n",
    "        g_action = np.random.choice(self.action_space) if np.random.random() < eps else action \n",
    "        return g_action\n",
    "    \n",
    "    def step(self, s, a, ns, r):\n",
    "        na = self.act(ns)\n",
    "        s = get_state(s)\n",
    "        ns = get_state(ns)\n",
    "        self.Q[s, a] = self.Q[s, a] + LR * (r + (GAMMA * self.Q[ns, na]) - self.Q[s, a])\n",
    "        return na\n",
    "        \n",
    "        "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 7: Watching untrained agent play"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "agent = Agent(p_size=12, v_size=20, action_size= 3)\n",
    "\n",
    "# watch an untrained agent\n",
    "state = env.reset()\n",
    "for j in range(200):\n",
    "    action = agent.act(state, 0.5)\n",
    "    env.render()\n",
    "    next_state, reward, done, _ = env.step(action)\n",
    "    state = next_state\n",
    "    if done:\n",
    "        break \n",
    "        \n",
    "env.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 8: Loading Agent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "start_epoch = 0\n",
    "eps_start = 1.0\n",
    "scores = []\n",
    "scores_window = deque(maxlen=100)\n",
    "\n",
    "# To Load checkpoint uncomment code\n",
    "# checkpoint = np.load('mountain_climb_sarsa.npy', allow_pickle=True)\n",
    "# agent.Q = checkpoint[()]['Q']\n",
    "# scores = checkpoint[()]['scores']\n",
    "# eps_start = checkpoint[()]['eps_start']\n",
    "# start_epoch = len(scores)\n",
    "\n",
    "# index = 1\n",
    "# for i in reversed(scores):\n",
    "#     scores_window.append(i)\n",
    "#     if index == 100:\n",
    "#         break\n",
    "#     index += 1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 9: Train the Agent with DQN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train(n_episodes=50000, eps_end=0.01):\n",
    "    global eps_start\n",
    "    eps = eps_start\n",
    "    for i_episode in range(start_epoch+1, n_episodes+1):\n",
    "        state = env.reset()\n",
    "        score = 0\n",
    "        action = agent.act(state, eps)\n",
    "        while True:\n",
    "            next_state, reward, done, _ = env.step(action)\n",
    "            action = agent.step(state, action, next_state, reward)\n",
    "            action = agent.eps_action(action, eps)\n",
    "            state = next_state\n",
    "            score += reward\n",
    "            if done:\n",
    "                break\n",
    "        eps = eps - 2/n_episodes if eps > eps_end else eps_end # decrease epsilon\n",
    "        scores_window.append(score)       # save most recent score\n",
    "        scores.append(score)              # save most recent score\n",
    "        eps_start = eps\n",
    "        print('\\rEpisode {}\\tAverage Score: {:.2f}\\tEpsilion: {:.2f}'.format(i_episode, np.mean(scores_window), eps), end=\"\")\n",
    "        if i_episode % 1000 == 0:\n",
    "            print('\\rEpisode {}\\tAverage Score: {:.2f}\\tEpsilion: {:.2f}'.format(i_episode, np.mean(scores_window), eps))\n",
    "    \n",
    "    return scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZQAAAEGCAYAAABCa2PoAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO3deXwU5f3A8c83ByEcAbmRwyCgHCogERBBxaOgWLW1Vmyt2tryU7G29rBQa6u1WlutbbEetV5Vq6K1FisegCcegKAghyDhhnCjJNwJeX5/7GzYZGc3e8zszM5+369XXtl9Znb2md3Z+c5zzPOIMQallFIqXXleZ0AppVQwaEBRSinlCA0oSimlHKEBRSmllCM0oCillHJEgdcZ8Eq7du1MaWmp19lQSqmsMn/+/O3GmPZ2y3I2oJSWljJv3jyvs6GUUllFRNbGWqZVXkoppRyhAUUppZQjNKAopZRyhAYUpZRSjtCAopRSyhEaUJRSSjlCA4pSSilHaEBRKiCmLtjIJ+u+8DobKofl7I2NSgXJ3oM1/OjZBXXP19w51sPcpGftjj3MWLqF7488OqXX79h9gEsemk2T/DzOPb4T153R2+EcZt7C9V+yYutuvjG4q9dZiUsDilIOmvSfRXy4cjtv/3xURt/3QHVtvec1h2opyM/OCohv/WMOG7/cx/8WVlBSXMiTVw1N6vX//HAt5Vt3A7B0U2VdQKn4ch8i0LlVseN5dtsF970PoAFFqVzyzNx1XmcBgCUVlQzo1trrbKRkz8EaABZu2JXS6xdt+NI2ffidbwLZXXrzu+y8hFFKxRW+os02Wyv38+Xe6qReY4zhvRXbqa1NfTpzYwy79lWza181QZgWfdnmSrZW7s/4+2pAUcol63bspWp/9Mlx1bbd/Gbq4rgnQGOMYye2Vxdt4s1lW9hatZ+tVe6eZHbuOcimXftSfv2QO96IStv4Zfzt9Zj0Cpc9ModH318NQPm23XHXN8bwwvwN9T7f8+59jwG3TmfArdPrSjKpWra5kppDtY2u95PnFjB/rbOdKN5bsZ3SidMY85dZjPjDW45uOxEShGicirKyMqOjDatUvTB/A93aNGNIjzb10ksnTgNC1Srhx6t/fy4iAoQajL/+wAes3bGX6TecyjEdW0Zte1vVAU66fWbddsJqDtWydFMl1YcMg486goG/nc6Xe6t59Moyvvd4asfypUO6cdWIHjQvKqhrW9hffYh9Bw9xRPMm7Nh9gMG/m8mfLh7ARYO7ctXjH7FscxUbv9zHS9edwgld61erRe5/Mv41Zy03vbjYdllJ0wI+vWV0zNeG33NAt9YsXG9f3WWnbfMmzL/57HrbCFtz51i2Vu1nyO2hAFd++zkx26TCr50wqicDurZm/JPz6dGuOW/97PSodcKfy5d7DzLwtzPqll83qhc/G30sNYdq+fu7q7hieCktigqiXn9clxJemjCCM/70NtWHDC9ddwptWxQB8Nj7q7n1f0vr5a1f5xJ+NvoYzujTMeHPpTEiMt8YU2a3TNtQVE7bX32Iwvw8DtUamhQ0XmC/YcoC5qzaQcWu0JX+p7d8hQ/Kd9CpVVMGRrRZRJ6gpny0nnFDurO0opJzJ8+qS8+zgkxDZ9z9dt3j2lpDXl5ovdteXso/PwyNHL7qjnPrqoZeX7wlwb2N9szc9Twzdz0APx99LHe9vtx2vZ8+v5CRvdvxxrKtdWkfrNwRFVAi7a8+xOdbqqLWufPVZRzXpYTzTjiyLi1WMAGo3F/DC/M3cJFNg/SsFdvqHicTTAB27DnI8be8zvsTz4haVjpxGi0jTuifrP+Sk0rbRK0X6b63VtY9Xr19DwCLN+6iR7vmUeuu2bG33vO/vVXOz0YfS6+bXgWo+x76H1nCtOtH1q23eGMlW6r2173+a/d/wA9G9uDknm2jggmEOiV87/F5GWs30iovlbP+PX8DfW5+jRNvm8Exv3o1ode8+MnGumACoZLK1U/N58L73o9ZRTXxP4vYe7CGHzxRvxQxa8U2fvdy9Emg6kBN3ePI6pfpSw8HjgffPXzymjJvfUJ5b0ysYBLWsDpqW9WBuOv/4oVPOf9v70fV5T/4zkque/qTpPL24DsrbdN37jmY1HYaqtpfw9ditDdFfg+V++zbdeLV8Ow9WMN5977Htx+eE7Xso9U7E9rWkorKqNLTwvWHOyus27mXm6cu4ax73o2Zj0zSgKJy1vQlmwHYFeNkMXPpFkonTmNdg6vJSHe+uqzu8cpte2KuV33IRLUF3Pq/pTz83uq4edxcub8u6NRGnHA+WZfc1bgbPli5I+7ycIlh94EajDFpnfxrbU6223cfqHdyTVW87y1s+pItPDV7Ld95ZA5vfLaF1xZvYs6qHcxeFR0YwjZ8Efq+F0SUnA7UHIq5/to4x1mk+94qT2g9O7v2VjN28ixWNtLOlCoNKConfLapMu6P2c6LCzYCsNDqhmr3+gM1hxtfz7rnnZjbilG7lZCH31tN6cRpRJ5TZyxNvZrLKSu3xj8pScROP/TuKk68bQbrdx4+aVY00tge6ZBNB4av3vteXUO826bMW8+v/ruYWSu2c9U/53H1Ux9zyUOz2fBF7CBwsCa6Yf6bD34IgCF6fxI9RhZtTD2Ivr50M0sqKvnLzBUpbyMebUNRgbdp1z7O+essxp3UjTsvOoGn56yjd8cWSW3jtpeXsnlX6j2k9lcnF8zsbG2kiinTDh6qZf3OvSzbXMXVT823PekDGOCx99cAh6/aIVSdd0qvtrxfHr+kA6E2h7mrd1JTW8vwnu0A2JTG95EJk9+IPmmH761ZtrkqalmskrKTnpodaoP738IK7r10kOPb14CiAi/8Qw1XE/3yxUW2622rOkD7lkV1zyPvPn+kkaqpxoR7C9n5fEsVx3RsiTGGDxupRvKbkX+075r6nUfm1DVM3zP9czZb7SiX/mN2vfUSCSZh3/x76Oq+T6eWtidkL+yOaGdpaHqMUmTDNpGw8//m3r1DpROn8auxffk0xZtFE6XdhlUgHaypZcCt0/n56GMxhEoYx3Zsyes3nBrzBw0w9vjO9O3ckqfnrKtrfO9U0rTuhOimyZcO4vpnkmusVipVqfb8itdtWAOK8q3K/dUs3rCL4b3a1aUZYyjfuptubZrRtDCfnXsO0qKooK7Lb82hWt5avo1pn1bw3wUVXmVdKd9zI6BolZfKiGWbK5m6oIJfjOlju/z5eesZdnRburVpVpd27VMf8175dub+8kw6lDQF4M8zV9TVTecJRFbbn92voy8aq5XKVdrLS2XEmL/M4oG3V9oONTFn1Q5+/u9PGfnHt/h43Rd1vX+WbwnVkw+54w3eWr6Vqv3V9Ro6G7YBazBRyltaQlEZddEDH/CTs4/h5J5t6de5hKdmr+X3EfdyfP3+DwB44Zrh9W6c++5jH2U8r0qp5GhAUWnZta+aXXur6d62GcYYbvz3pzw/fwOPXllWN35Qw77698z4HGbEr6K66IEPXM+7UspZWuWl0vLVe9/j1LtCXUf3V9fy/PwNADz49qq6dWKNeqpVVEoFiwYUlZZ1O+3vFLa7E1gpFWwaUJQrPlrzRcw7p5VSwaQBRbnmQM2huDcRKqWCxXcBRUTuEpFlIvKpiLwoIq0jlk0SkXIRWS4ioyPSB4vIImvZZJF0huJTSimVCt8FFGAGcJwx5gTgc2ASgIj0A8YB/YExwP0ikm+95gFgPNDb+huT6UwrpVSu811AMcZMN8aER1ybDYSnabsAeNYYc8AYsxooB4aISGegxBjzoQmNI/MEcGHGM66ifLap0ussKKUyyO/3oXwPmGI97kIowIRtsNKqrccN06OIyHhCJRm6d+/udF5z2urte6LmabjyUb0ZUalc4klAEZGZQCebRTcZY6Za69wE1AD/Cr/MZn0TJz060ZiHgIcgNDhkktlWcYyKmAc9rCrO0N5KqeDxJKAYY86Kt1xErgDOA840h4dD3gB0i1itK1BhpXe1SVcuS3YGRKVUsPmuDUVExgC/AM43xkTeNfcSME5EikSkB6HG97nGmE1AlYgMs3p3XQ5MzXjGA2rnnoN8EWMu8GN/9VqGc6OUcsKIiCkhnOTHNpS/AUXADKv372xjzNXGmCUi8hywlFBV2ARjTPgS+RrgcaAYeNX6U42o+HIfu/ZV07dzCdOXbKZF04K66VXDTrxtBpD63AlKKf9xayQL3wUUY0yvOMtuB263SZ8HHOdmvoJo+J1vAqFgMf7J+XWPG7Ny227aNGviat6UUtnHd1Veylvn3TuLh2etomp/db300onTOFgTmmP9zD+9wyCr5KKUyj5uTdSrASVH7D1YQ601ttadry7jmbnrbNdbvLGS3037jJNunxm1bOqCja7mUSmV3XxX5aWct7/6EP1+/ToAq+44lwffWVlv+aDfTrd5TS1vLqs/vPz6nXvZtGufexlVSmU1DSg5YE/E/SDVtbVRy7/YWx2VBvC9x+fVez75zXImv1nubOaUUhmnVV5KKaUc4VYvLw0oSimVY7SEomK65aUlOu+IUspzGlAC4PEP1nidBaVUFnFrxigNKDlAR8FUSkXSKi+llFKOcOsiUwNKjnHrykQplUW0hKKUUsrPNKDkmOfnb2h8JaVUoOl9KCplkdVcH67c7l1GVD1n9e3odRZ8za05O5R7NKAopXzJrato5R4NKErlgJk/OdXrLCRNO5C4R7sNB9CTs9dSOnEaO2NMsauUUzq1KubfV5/M1af19DorKsA0oHhoykehOUk2fuHMkPCJBCe96vMPEejSujhj71dW2oZfjDk2Y++XLj1W3dO8yJ2B5jWgBEz51t1RaYsrdtU91h+pv1x9emZLDOLWmBsqq5QddYQr29WAkoUO1tRGTdEbdtvLS+se7zlQwyuLNvHdxz6qS3ttyWbX86eUE7RRPvtoQMlCVzw6l+NvCc2yGJ7WN2zRxlBpZEnFLvr/5nWu/dfHGc+f8re/fWuQ11mIMrxn26g0LU27RweHVHU+XLWj7vHRv3wlann51irGTn4vk1lSGXBmnw6ObKcw318/+7+OG8j1Z/aOSv/OyUd5kBuVDn8dWTnK6aL9Wfe86+j2lHuSuVAM6gX7BQO72KYP7Naa/103IqN5adYk37Vtv/HT01zbdrLcakvTgOIhSep0Eu2Hz3ziUE5UNjABrgOKtWvHd22V0Xw89J0y17bds30L17btFxpQstj/FlZ4nQWVhmQuJ2ZPOjOwJRQ/KXaxhJILNKAo5SOnHdOe5/7v5Kj0Tq2aptVI7ffOwnbVvukWyCae0ye9DaikaUBRykfGHt+ZIT3a2C7TEor79Dad9GhA8YEAV40rpXxIuw0HkF4NqahjIM4xEeRGebviV7zdLSrQU5cf6beSJYwxVHzpzJhfSmW7J743xOssKBsaULLECx9vZPidbzJ/7U6vs6KU4+wKI/Huz0qkrJZKBYBWGqRHA4rPVe6vZkvlfuatCQWSz7dED/6ockOsKqA7v358ZjPisSnjh9mmP3XV0JjLgqQw379hTwOKz426622G3vGG19lQLmisDe3IVk3rPY91xR5rOy9eO5z8PP+efCLZBctYAbRVs0Lb9BG923FSqX0PuURlw2jM0284Le3x2NK9qToWDSg+EK/4vkMn3wq0WD/sT2/5Cm/89PR6abFOsLG2Mai7O0OUN8apGBbvdxHrs8jLE8aferQzGfCpHu2ac94JR3qdDVu+DSgi8jMRMSLSLiJtkoiUi8hyERkdkT5YRBZZyyZLNlxmoPW12ei6Ub1c3X74mChpWhh117ZTnbz81FmsT6eWgHdD1R/drrkn75uq317Q35Ht5FS3YRHpBpwNrItI6weMA/oDY4D7RST8i3sAGA/0tv7GZDTDGeCnk0Auu+4MZwOK3y99brvwOFe3P2V89KgAiXDqc2vZ1J2ZC53yq7F96z2//ORSbzKSIF8GFODPwI3UL/VeADxrjDlgjFkNlANDRKQzUGKM+dCEOuo/AVyY8RxnyAvzN3idhZzmdmCPV7j24ir+tN7tXdt2UUFeXXuIfRtKnF5eHl9gpTrj4dybzkxq/ZKm9u1FfuW7gCIi5wMbjTELGyzqAqyPeL7BSutiPW6Ybrft8SIyT0Tmbdu2zcFcu2/KvNCuz1v7hcc5UV5J9CR6Vt+OUQ36yTq7X0deuX4k3ds2S2s7vtcggCda8Ek1nnVoWf97Oatvx7jrXzS4Kyd2b53iu4V0LCmKSnOrYOxJQBGRmSKy2ObvAuAm4Nd2L7NJM3HSoxONecgYU2aMKWvf3r0rr2QF+g5o5ZiCBLuLTjynDx9MSu5KuKHCfKHfkSUJrfvyD+vPWZJKE6b9fSipSeb39O2h3es9P7ZTy7RP4E7pf2QJ+XnCY99N7ybO0raZayfyJKAYY84yxhzX8A9YBfQAForIGqAr8LGIdCJU8ugWsZmuQIWV3tUm3f8ifnh/nbmCpRWVVB+q5ZaXlrCt6oCHGVOZYNc7K96p+O6LB0SdANfcOdaV4VqS6VZa2qBh24kLpP877ei4DeaR1X//uLyMu75xQtQ6jcW1W77aj2+WdauX1rQwn/9ce0pymU3A3RcPSPo1bVuEShatirOn2stXVV7GmEXGmA7GmFJjTCmhYHGiMWYz8BIwTkSKRKQHocb3ucaYTUCViAyzenddDkz1ah9SUWsMf575ORfe/z4zlm7h8Q/WcNvLS73OlvKZzq2Kufm8fim91u+N/w1NOqdvwiWds/t15OIGgSERV57SI+nXhCUbNM87obNterxdTOQ9bhxzbFL5cJuvAko8xpglwHPAUuA1YIIx5pC1+BrgYUIN9SuBVz3JZJpqDtVysKbW62yoDEr2PO/H2tGo8S1TqfLy446ladr17k5fvObOsVx7emq9Dt26wPB1nzmrlBL5/Hbgdpv15gHu9m/MkEO1oR9WttzhnGvc7mmVzg+9RVEBew/W0KV1sXMZ8isXvobwPTFO6X9kZqcvTobeKZ9jNJwoO/EC2pjjOrHq92OTmsb2P9cOdyJbjXrtxyPjLvdD+cSuZPXYlSfZjg9WmK+nTjv6qfhAAEv7yicaq36KeVWe5BXNlcNL4y5v07wJH0w8I7mNxmBi9e1Mwr+vTuyGylF9OjD06LZR6X8ZNzC9DFji7UY2nhc0oHio4cEUefxs2629vHJRtjWeh91yfppDgiR78kzjZNu2eRPK0hxEsnMrZ6oVszBmxKUBxQciTyLhA2zWiu2Ub9Wh6v0kP08cvWpMJXhk41UruFdnbydbP6NMyqmxvFTIup17vM6CivDw5WWObzNbSyTJamw/3ejskGwQ89tXkcpnsubOsVFdlDMZXzWg+IzfDmp12BHNm7j+HsmcBBtrt4ildcR8IpksOTgp2ZPkMR1b0LIo1Kk1mU4Lbvj56MTuHcnGkpYGFKWySOQ5JtV2i2E2jcwNpRNmUnltcaG7dzA8/3/Duevi0N30mRyKxM6EUb342VeOATJzAZnJSwYNKD4QvhIxJniNdEHj5+8nmSvapoXO/fSdqLYbdnQbV6cyLirMo3lR5m67+50Dw/5rCUUlpdF65Sw8oFR9Q3u04dYkShKpnJyzodJKIO4JXUQYN6R7zOWRjPH/b+Nrg+oPeJ4rbWUaUHzsqn/O8zoLKk3tWxYxpEd6XVQjOTVESTKbefKqIVw2rPGTvd0w6ZFaFRfyr+8PTfyNM+zCQc5Nq+unAGL3Vbs1qa0GFKVcFnMueHG/UbyxAJTIeWVk7/Z0Kml8fpXwuFLx3tGpYWHcOB/+YKS/5qL3alrkdGhA8Zna2uw7iFRsmRjPKZEjJukutA4P8Oj0FbEbVV7J5nHqhFMSKrn5UaAm2FL1Rf42bnzhU8/yoZx3xfCjHN1evPOoX6pZEs2GW/m1r+Jx/n0GdGvNNwYnP2x+opwKmtrLK0f45PevXCSIp1UXbtWVx3/P2Msy+Uk0zEcmGvKLC527xyXV7Db8zn15Y6OIjBCR71qP21uTXCkHfLap0ussqAQl2yiebDDJVAAI3xRZ4MI0CXYfUVGBs9eu6QRpP5TkvO6l5unQKyLyG+AXwCQrqRB4yp0s5Z5/z9/gdRZUgrxu4XLqRDTxnD6suuNcCmIMw97wfFNUEPvKO5H2mUzeA5JN/BDcnJTot/w1YBDwMYAxpkJEnJ2NRimfS+W3LyQ3oGSmzi8iktTJ7PLhR1F1oIbJb6yIuU54aJp2LYrYXLkfgJd/OIIOLeN3J1YxJHnx8OiVzo81l6xEy6EHTaisbwBExNuxC5QKAEGypiGtqCCfn5x9TNx1zh9wJH+6eABXn3a4++1xXVrRIaLLcRCn+o0UqyouE2OmDep2RL3n3xqa+R5oiQaU50Tk70BrEfkBMBP4h3vZyi0B/40pB5U0jV2p4PVxJCJcNLgr+T6czTAvw3VLiQYQp+ZVgegCTfien1N7t4ta161PI6EqL2PM3SJyNlAJHAv82hgzw6U85QwveuAof2vskLA7Zvx2FDmdH5H0g2W+S7+1dEtczYtit02l2zuw6xHFzPnlmbRvUcTd0z9Pa1uJajSgiEg+8Lox5ixAg4hDLrjvfRau/9LrbKgM6NEu+2qI/X6tk+x53O/7E+mrA46kdXEh409N/879jjFGOPBs6BVjzCFgr4i4f8tvDtFgkjuaFxVw83n9vM5GxsQ712eiVs4u2GSiNqDhgJCpyhe47cLj6NamWUqv97KdKtFeXvuBRSIyA6ibRtAYc70ruVLKhzLxM83WCa/c5HXbUKL6dS7h1cWbPHv/hoHEiyr1RFvPpgE3A+8C8yP+lMopqZ7cYl01+il8vHL9yJRel+nzlkjiwf3no4+liQ87CWRJjExaoo3y/xSRJkC43+ByY0y1e9nKLdk4qmgQjOzdjlkrtmf0Pced1I0Tux8RGrPNT9EE6HdkiSPb8dNuTRjViwmjenmdjZi8KpGObTDvvFMSvVP+dGAFcB9wP/C5iJzqSo4Cav7anazevsd22eZdBzKcGwXw3VNKk39RmrG/WZOCuHOau3W1n+x208lGvI+oMM9/pQWneN3wn+ihef0ZvWjXwp2bTRP9dv8EfMUYc5ox5lRgNPBnV3IUUBc98CGj7n7bdtn23RpQVG7o3rYZN53b1+tsOCbRk3gywSabbydINKAUGmOWh58YYz4nNJ6XSsHMpVu8zoICBh/l3EyKKnGXDXN2SP9MmDrhFN79+Sivs+F7ifbymicijwBPWs+/jTbKp+z7T+jUvl7767iBtCr23zVR9l6bHpbI7I6pKCrI40BNLZD5rrEDurXO6PulIuGPxMUSUKIllGuAJcD1wI+ApcDVbmVKKbelUq3gxs/QieoNv3XpOKtvB1e2m+h9GdrJxTuJllAKgL8aY+6BurvndQhRpVKQDae7dAKd020AqQ69ks1tEdkq0RLKG0DkKGbFhAaIVCqnpHv12/hYXSlsM7WseC6dWis/BuVCn93v4sUd84l+Ak2NMbvDT6zHqY0LoFSOy9YA4JRkg6bTN5O65dIhmR8uPlJLayRqL0tmiQaUPSJyYviJiJQB+9zJklLBobUuzkn0o8zkzYKRMatJCtMcO3l8NHVwPvtUJfoJ/Bh4XkRmici7wLPAdW5lSkR+KCLLRWSJiPwxIn2SiJRby0ZHpA8WkUXWssmilacqa/nj0L3hrPiTaXnNreHoVXriBhQROUlEOhljPgL6AFOAGuA1YLUbGRKRUcAFwAnGmP7A3VZ6P2Ac0B8YA9xvdQ4AeAAYD/S2/sa4kTelnK5F8etpsXtbf9don9IretIoP/Pr9+y0xkoofwcOWo9PBn5JaPiVL4CHXMrTNcCdxpgDAMaYrVb6BcCzxpgDxpjVQDkwREQ6AyXGmA+taYqfAC50KW9KuWpUn/ZeZyEr5OcF9xTt9p65uf3GAkq+MWan9fgS4CFjzAvGmJsBt0ZcOwYYKSJzROQdETnJSu8CrI9Yb4OV1sV63DA9ioiMF5F5IjJv27ZtLmRdZYtMnY6SKdGMOrY9RQXe14MrlarG7kPJF5ECY0wNcCahaqVEXxuTiMwEOtksusna7hHAMOAkQvPZH439OcDESY9ONOYhrJJVWVmZH3seKp9L9aCJDCx64Lmrsa7d2TK/SjZqLCg8A7wjItsJ9eqaBSAivYBdqb6pNZ2wLRG5BviPVX01V0RqgXaESh7dIlbtClRY6V1t0pXyHYnxWDmvYd8cbcd3X9wqL2PM7cBPgceBEeZwx+484Icu5em/wBkAInIM0ATYDrwEjBORIhHpQajxfa4xZhNQJSLDrN5dlwNTXcqbUo4KwkkuCPugnJHInPKzjTEvGmMip/793BjzsUt5ehQ4WkQWE+qefIUJWQI8R2gcsdeACdZ89xBqyH+YUEP9SuBVl/KWtn+8u8rrLKgUuXHiHHtCZ9q31FGMsldq9WduVrt5Oae8v8YKAIwxB40xlxljjjPGnGiMeTNi2e3GmJ7GmGONMa9GpM+z1u9pjLnOePmJNuL2Vz7zOgsqhsmXDnJ8m40FoaKCfP5w0fGOvFcmB0V8+gdDA93TKlPc/AT9PKe8UoE3rEd2zo/ixImjf5LT/w7vmfn7QLRqzf9S7qmllAqGlXecqx0EHBKrbsRPg3e4mRUNKConefn79luNrFZdOc9H8SOjtMpLqbAETgKJDAA48yenUdRgPbv2DT9dtQaJz+J18rL4sNCAolQSWhQ1Xqjv1aFFzGUaQzJHP+rM04CiVBxXjejhdRYa5VUVWqrvm/UliAhB2hcnaEBRygVfPzE0nFwm2ycyOQ+IE++rpTV3eXGhoY3ySrngdxcez6Rz+3o+LWyQztnpnh9zpTSRDTM2KhV4Tl7h5+cJJU0LHdueOizd86UfSkaZvAm1oUTaAVOlJRSX3fX6Mu57a6XX2VApyvar2izPfvBlOLr9amxfrhhe6tr2NaC4qLbWaDBRgH1gcvpUEu+q1w9X5bnMLx//90ce7er2tcrLRaf84c3GV1K+FnmSdqJKzI367Vy5nyVHdjOraUBx0aZd+73OgnLRS9ed4nUWlMe0SrE+DShKWZK9Ai7Iy+2fj99PptlaovGq+7cTcvsXoVQasvWE5TT9HFSYBhSl4ojXy1o26xgAABH1SURBVEtPpErVpwFFqRSlWjWR7V2RlYpFA4rKaU9dNbTucV6GixxawsldQb2o0ICicpJd6aJN8yauvd8FA7vQpXUx3x7a3bX3UNknaNcUGlBUTsvUEBidWjXl/YlncFTb5hl5P5UZiZY0Mlka9XICNw0oSgVYEK6Ae7YPBeFEz5N+mxHTK5E3vJY0zcygKDr0ilLK154dfzJLN1Vyx7TPknpdEIKpU16/4VRWb9vj+vtoCUXlhGnXj3B8m75rVM/whXmmCgLtWxZx2jHtM/NmPuDGcdW5VTHDe7VzfsMNaEBROaH/ka1s0xvr+htZfeLUD93pdhuv45rX76/8QwOKCqwurYu9zkJcTp2I44UnbU1QmaQBJU1zVu1gf/Uhr7OhbPTtXJLwukN6tHExJxkSJ0Jl8/hQKjVedE7QgJKG8q27ueSh2dzy0hKvs6KSFK6+Clc/FebrCVclr12L1O5dcvNUr1MAZ6ld+6oBWL6lyuOcKDunH5t4Q65ewatUHN2+Rb3nyQYK33XsSJMGFBVY3x7anRO6tqJ1s9Tndtc2COWEXJkETe9DSYuebvxMRHjpOue7Cyt/01+ld7SE4oDcuPZQDaX6veuN3JmRK6UCP9GAohSZG9MrUlBOeEHZD5U+DSgqpzV+Y2OGMuISPdVnn2z+zjSgqJwU/tF6UTJRKqh8F1BEZKCIzBaRBSIyT0SGRCybJCLlIrJcREZHpA8WkUXWssmiZXCVpEB0G9bYmHWyvQTckO8CCvBH4FZjzEDg19ZzRKQfMA7oD4wB7heRfOs1DwDjgd7W35hMZDR8MDSMX/sOHuK5eeszkQXlsmwowQQgFCakeVHo556v14u+5cduwwYIj5nRCqiwHl8APGuMOQCsFpFyYIiIrAFKjDEfAojIE8CFwKsZzXWEU+96i21VB7x6e6UC6YHLBvPiJxvp1aFF4ytniaDFRj8GlB8Dr4vI3YRKUMOt9C7A7Ij1Nlhp1dbjhume0WDif16WO4JWzZEpHUuacvVpPb3OhorDk4AiIjOBTjaLbgLOBG4wxrwgIt8EHgHOwr5kb+Kk273veEJVY3Tv7vzc3gdrajnmV54VjFSGpXt1mYmL01yMXRqwveNJQDHGnBVrmVVl9SPr6fPAw9bjDUC3iFW7EqoO22A9bphu974PAQ8BlJWVOX7Y7TlQ4/QmlXKEk1Ur7VsW1d+2c5t2VNCqkxKlc8rXVwGcZj0+A1hhPX4JGCciRSLSg1Dj+1xjzCagSkSGWb27LgemZjrTkJtXg9nuqDah+crP7NvB45xkh1k3jmLmDac1vqLynBedXf3YhvID4K8iUgDsx6qiMsYsEZHngKVADTDBGBOeiOQa4HGgmFBjfEbqncIBJEcvhLJa+Dvr3rYZC3/zFUqa+vGn4D/d2jTzOgvKx3z3KzLGvAcMjrHsduB2m/R5wHEuZ62ep+es45cvLmqYj0xmQTmkVXHs0Yj1K1WZls1VdX6s8soKDYOJCqZsiCfZkMdIXt3bk22fU7p0xsYsl2sHrPIZmytbP1/sBmJ0glS5eLLXGRsDQqtHck1qP1zXDhM9/rJO0IKqBhQHzFv7BT969hOvs6GyjUPnkmCdktKXDcPlBJUGFIdMXVChB3LAFRXozyWbaKDNPP2FKBVHuBpzYLfW9O7YEoCbz+tHEw0udbK5V5Jylv4qVE6KdRL8ZllX2/SLI9KvGtGDz393jhvZUiqraUBJkV6VBdMfvzHA6ywolbU0oDhJm1CUUjlMA0qKtIDirnOPtxuM2l86lBQ1vpKNTN5wptc42SebuxJrQEmR3c1D+uPNLSVNC2mZxhhgmTxxZO8pKpiCeq7QgOIgvbExiBr5UvU7V5GSPB6C1harASVFATsOVAzhi4RsrobINX64sAtaoEiUBpQU2R0wemNjcGX7CcLNI9PLsaPi8mu+AkwDSor0itVd7l9lBu/7s/vIgreXys80oDjID0XtoNDPMnF6Ia78QgNKqvRHrFKksVIFlQaUFGk8UenSkoVyg5czx2pAUSoOrXpT2cqLzhIaUFJk38tLBZWfCxMa9LJPvO/MqTigUwBnEbteXrW1+st2wtcGdfE6C1nJz0Evk7LpV+jGd6ZTAGch2xJKNh3JPvajM3tHpd16fn9H36NN8yYJrZfovUVTxg9LJzvKBRpgM08DioP0xkbnNPwsrxhe6ti2H7vyJIb0aJPUaxq76Ot7ZEkaOVIqGDSgpMju/KIllOwwqk8Hx7Z1y/n9adm0gGaF+Ym/KIPHiR6SKpNSHyo1x+lowwrgosFduWiw/SyPjclklYxW/6hM0BKKg2q1iBI4QflKA7Ibyuc0oKRIq7zc5bfPMhvGbtOxvJTXNKCkyvaX6rOzoMoJese98gsNKEoplWFB7RGqASVFdheFel+jUt7zW3VpPHaly2wucWpASZFdLy9tlHdOkD/JoF6d+k02n5izlQaUFOmd8t554ZrhXmfBEXrCU0Gj96E4SANKZgw+6oiMvdf1Z/Zm7Y69jDm+U8beU6lspQElRbbdhrUqI3C6tWnGc1ef7HU2lMoKWuWVIts75TWeKKVymCcBRUQuFpElIlIrImUNlk0SkXIRWS4ioyPSB4vIImvZZLHO6CJSJCJTrPQ5IlKa2b05TAOKUiqXeVVCWQx8HXg3MlFE+gHjgP7AGOB+EQmPuvcAMB7obf2NsdKvAr4wxvQC/gz8wfXco1VeSinVkCcBxRjzmTFmuc2iC4BnjTEHjDGrgXJgiIh0BkqMMR+a0DRkTwAXRrzmn9bjfwNnioszzDz30XrOvucdduw5GLXs6ifnu/W2OSVPhKKC4NbGFuSF9q0wP/Q/33reJMV9zs8Ta3vRh31xk9D1mBulZ7v3S0R4GJtw3pwWPnYK8urnL986LbhxbB3RrBBIfJ/C332B9T8yT+FlqWpSd1xlvhuh3xrluwCzI55vsNKqrccN08OvWQ9gjKkRkV1AW2B7w42LyHhCpRy6d++eUgZbNyukd8cWFBXmsXhjJQDtWhSxffcBBnZvTcWizSltN1eUNC2gcn9N3fPSts1Ys2MvAB1aFnHZsKPo1qaY315wHN3aNGN71YG67rVPXTWUHXsOAHDvpYMoLszntmlL+WLPQf474RT+u6CCFVuqWLqpktsvPD7qvR+78iT2Vx9qNI9Pf38oD7+3mu+d0iOpfXv0yjIO1tQ2ut75A4/k861VTBjVC4ARvdpx7ek9uWpE9PtNu34EH63eWS/tletHMmf1jrrnY4/vzGebqrjmtJ5Rr3/x2uG8uWxrwsFq8qWD6k6OjXn5hyOZtWKb7bLfXXgcx3dpZbusuEk+E8/pw9n9OkYt+81X+zG0R9uE3v/p7w9la9WBqPSffuUYigry+Nqg+qNAD+/ZlgmjevLdBt/rPd8cQOdWxfXSXrjmZMq37o7atl36xHP6cFJpG2v5cN74bAtFBaHA8tqPR/J++Y6o7fzfqT3Zd/AQVw4vpSBPOPWY9hyqNZw7eRY3jukTtf793z4xZrD627cG0bLp4e/s1+f1o33LIsb0z3zPRHFr3mERmQnY7dFNxpip1jpvAz8zxsyznt8HfGiMecp6/gjwCrAO+L0x5iwrfSRwozHmqyKyBBhtjNlgLVsJDDHGRH+LEcrKysy8efMc2FOllModIjLfGFNmt8y1Ekr45J+kDUC3iOddgQorvatNeuRrNohIAdAKqH9Jp5RSynV+q6h+CRhn9dzqQajxfa4xZhNQJSLDrPaRy4GpEa+5wnr8DeBN41axSymlVEyetKGIyNeAe4H2wDQRWWCMGW2MWSIizwFLgRpggjEmXOl9DfA4UAy8av0BPAI8KSLlhEom4zK3J0oppcJca0PxO21DUUqp5MVrQ/FblZdSSqkspQFFKaWUIzSgKKWUcoQGFKWUUo7I2UZ5EdkGrE3x5e2wuRM/4HSfc4Puc25IZ5+PMsa0t1uQswElHSIyL1Yvh6DSfc4Nus+5wa191iovpZRSjtCAopRSyhEaUFLzkNcZ8IDuc27Qfc4NruyztqEopZRyhJZQlFJKOUIDilJKKUdoQEmSiIwRkeUiUi4iE73OTzJE5FER2SoiiyPS2ojIDBFZYf0/ImLZJGs/l4vI6Ij0wSKyyFo2OTzlsjXtwBQrfY6IlGZy/+yISDcReUtEPhORJSLyIys9sPstIk1FZK6ILLT2+VYrPbD7bOUpX0Q+EZGXreeB3l8AEVlj5XeBiIQnKvRuv40x+pfgH5APrASOBpoAC4F+XucrifyfCpwILI5I+yMw0Xo8EfiD9biftX9FQA9rv/OtZXOBkwEhNI3AOVb6tcCD1uNxwBQf7HNn4ETrcUvgc2vfArvfVv5aWI8LgTnAsCDvs5WPnwBPAy/nwrFt5WUN0K5Bmmf77fkHkk1/1gf+esTzScAkr/OV5D6UUj+gLAc6W487A8vt9g143dr/zsCyiPRLgb9HrmM9LiB0J654vc8N9n8qcHau7DfQDPgYGBrkfSY0i+sbwBkcDiiB3d+IPK4hOqB4tt9a5ZWcLsD6iOcbrLRs1tGEZsTE+t/BSo+1r12sxw3T673GGFMD7ALaupbzJFnF9UGErtgDvd9W9c8CYCswwxgT9H3+C3AjUBuRFuT9DTPAdBGZLyLjrTTP9tuTGRuzmNikBbXfdax9jfcZ+PbzEZEWwAvAj40xlVYVse2qNmlZt98mNNPpQBFpDbwoIsfFWT2r91lEzgO2GmPmi8jpibzEJi1r9reBU4wxFSLSAZghIsvirOv6fmsJJTkbgG4Rz7sCFR7lxSlbRKQzgPV/q5Uea183WI8bptd7jYgUAK0ITcvsKREpJBRM/mWM+Y+VHPj9BjDGfAm8DYwhuPt8CnC+iKwBngXOEJGnCO7+1jHGVFj/twIvAkPwcL81oCTnI6C3iPQQkSaEGqle8jhP6XoJuMJ6fAWhNoZw+jirl0cPoDcw1ypCV4nIMKsnyOUNXhPe1jeAN41V+eoVK4+PAJ8ZY+6JWBTY/RaR9lbJBBEpBs4ClhHQfTbGTDLGdDXGlBL6Tb5pjLmMgO5vmIg0F5GW4cfAV4DFeLnfXjcqZdsfcC6hnkIrgZu8zk+SeX8G2ARUE7ryuIpQfegbwArrf5uI9W+y9nM5Vq8PK73MOnBXAn/j8IgLTYHngXJCvUaO9sE+jyBURP8UWGD9nRvk/QZOAD6x9nkx8GsrPbD7HJHf0zncKB/o/SXU23Sh9bckfD7ycr916BWllFKO0CovpZRSjtCAopRSyhEaUJRSSjlCA4pSSilHaEBRSinlCA0oSiVJRA5Zo7uG/+KOOi0iV4vI5Q687xoRaZfC60aLyC0icoSIvJJuPpSKRYdeUSp5+4wxAxNd2RjzoJuZScBI4C1Co02/73FeVIBpQFHKIdbQH1OAUVbSt4wx5SJyC7DbGHO3iFwPXA3UAEuNMeNEpA3wKKEb1fYC440xn4pIW0I3o7YndFOZRLzXZcD1hKZRmANca0Ljd0Xm5xJCI8weDVwAdAQqRWSoMeZ8Nz4Dldu0ykup5BU3qPK6JGJZpTFmCKG7jf9i89qJwCBjzAmEAgvArcAnVtovgSes9N8A7xljBhEaAqM7gIj0BS4hNDDgQOAQ8O2Gb2SMmcLh+W+OJ3Qn9CANJsotWkJRKnnxqryeifj/Z5vlnwL/EpH/Av+10kYAFwEYY94UkbYi0opQFdXXrfRpIvKFtf6ZwGDgI2vU5GIODwDYUG9Cw2kANDPGVCWwf0qlRAOKUs4yMR6HjSUUKM4HbhaR/sQfItxuGwL80xgzKV5GrClh2wEFIrIU6GzNkfJDY8ys+LuhVPK0ykspZ10S8f/DyAUikgd0M8a8RWgyqNZAC+BdrCoraz6P7caYygbp5wDhucHfAL5hzYERnkP8qIYZMcaUAdMItZ/8kdDggQM1mCi3aAlFqeQVW1f6Ya8ZY8Jdh4tEZA6hi7VLG7wuH3jKqs4S4M/GmC+tRvvHRORTQo3y4eHCbwWeEZGPgXeAdQDGmKUi8itCM/XlERo9egKw1iavJxJqvL8WuMdmuVKO0dGGlXKI1curzBiz3eu8KOUFrfJSSinlCC2hKKWUcoSWUJRSSjlCA4pSSilHaEBRSinlCA0oSimlHKEBRSmllCP+H6RD9jWC3tg/AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "scores = train()\n",
    "# plot the scores\n",
    "fig = plt.figure()\n",
    "ax = fig.add_subplot(111)\n",
    "plt.plot(np.arange(len(scores)), scores)\n",
    "plt.ylabel('Score')\n",
    "plt.xlabel('Episode #')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Your total score is:  -197.0\n"
     ]
    }
   ],
   "source": [
    "# watch an trained agent\n",
    "state = env.reset()\n",
    "score = 0\n",
    "for j in range(200):\n",
    "    env.render()\n",
    "    action = agent.act(state)\n",
    "    state, reward, done, _ = env.step(action)\n",
    "    score += reward\n",
    "    if done:\n",
    "        print(\"Your total score is: \", score)\n",
    "        break \n",
    "        \n",
    "env.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = {\n",
    "    \"Q\": agent.Q,\n",
    "    \"scores\": scores,\n",
    "    \"eps_start\": eps_start\n",
    "}\n",
    "np.save(\"mountain_climb_sarsa.npy\", data)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}