SK_agent.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# %load SK_agent.py\n",
    "import numpy as np\n",
    "import random\n",
    "from collections import defaultdict\n",
    "from gym.wrappers import Monitor\n",
    "\n",
    "class Agent:\n",
    "\n",
    "    def __init__(self, algo,nA=6,alpha=0.001,gamma=0.9):\n",
    "        \"\"\" Initialize agent.\n",
    "\n",
    "        Params\n",
    "        ======\n",
    "        - nA: number of actions available to the agent\n",
    "        \"\"\"\n",
    "        self.nA = nA\n",
    "        self.Q = defaultdict(lambda: np.zeros(self.nA))\n",
    "        self.alpha=alpha\n",
    "        self.gamma=gamma\n",
    "        self.policy=defaultdict(lambda:np.zeros(self.nA))\n",
    "        self.algo=algo\n",
    "        self.epsilon=1\n",
    "\n",
    "    def select_action(self, state,env):\n",
    "        \"\"\" Given the state, select an action.\n",
    "\n",
    "        Params\n",
    "        ======\n",
    "        - state: the current state of the environment\n",
    "        \n",
    "        Returns\n",
    "        =======\n",
    "        - action: an integer, compatible with the task's action space\n",
    "        \"\"\"\n",
    "        if state in self.policy and random.uniform(0,1)> self.epsilon:\n",
    "            return self.policy[state]\n",
    "        else:\n",
    "            return env.action_space.sample()\n",
    "\n",
    "    def step(self, state, action, reward, next_state, next_action,done,i):\n",
    "        \"\"\" Update the agent's knowledge, using the most recently sampled tuple.\n",
    "        Params\n",
    "        ======\n",
    "        - state: the previous state of the environment\n",
    "        - action: the agent's previous choice of action\n",
    "        - reward: last reward received\n",
    "        - next_state: the current state of the environment\n",
    "        - done: whether the episode is complete (True or False)\n",
    "        \n",
    "        \"\"\"\n",
    "        if done:\n",
    "            self.Q[state,action]=self.Q[state][action]+self.alpha*(reward+(self.gamma*0)-self.Q[state][action])\n",
    "        else:\n",
    "            if i < 10000:\n",
    "                self.Q[state,action]=self.sarsa(state, action, reward, next_state, next_action)\n",
    "            else:\n",
    "                self.Q[state,action]=self.sarsamax(state, action, reward, next_state, next_action)                \n",
    "#             if self.algo==0:\n",
    "#                 self.Q[state,action]=self.sarsa(state, action, reward, next_state, next_action)\n",
    "#             elif self.algo==1:\n",
    "#                 self.Q[state,action]=self.sarsamax(state, action, reward, next_state, next_action)\n",
    "#             else:\n",
    "#                 self.Q[state,action]=self.esarsa(state, action, reward, next_state, next_action)\n",
    "\n",
    "    def update_policy(self,state):\n",
    "        max_acts=[]\n",
    "        max_val=max(self.Q[state])\n",
    "        for act in range(len(self.Q[state])):\n",
    "            if self.Q[state][act]==max_val:\n",
    "                max_acts.append(act)\n",
    "        if len(max_acts)>1:\n",
    "            action=random.choice(max_acts) # need to ensure that either of the actions having equal value are considered\n",
    "        else:\n",
    "            action=max_acts[0]\n",
    "        self.policy[state]=action\n",
    "        \n",
    "    def sarsa(self,state, action, reward, next_state, next_action):\n",
    "        self.Q[state][action]=self.Q[state][action]+self.alpha*(reward + self.gamma*self.Q[next_state][next_action]-self.Q[state][action])\n",
    "    \n",
    "    def sarsamax(self,state, action, reward, next_state, next_action):\n",
    "        self.Q[state][action]=self.Q[state][action]+self.alpha*(reward+self.gamma*max(self.Q[next_state])-self.Q[state][action])\n",
    "        \n",
    "    def esarsa(self,state, action, reward, next_state, next_action):\n",
    "        max_val=max(self.Q[next_state]*(1-self.epsilon+self.epsilon/len(self.Q[state])))\n",
    "        e_val=np.sum(self.Q[next_state]*(self.epsilon/len(self.Q[state])))+max_val\n",
    "        self.Q[state][action]=self.Q[state][action]+self.alpha*(reward+(self.gamma*e_val)-self.Q[state][action])\n",
    "    \n",
    "    def get_algo(self,state, action, reward, next_state, next_action):\n",
    "        return algo(self,state, action, reward, next_state, next_action)\n",
    "                     "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "# %load SK_monitor.py\n",
    "from collections import deque\n",
    "import sys\n",
    "import math\n",
    "import numpy as np\n",
    "\n",
    "def interact(env, agent, num_episodes=20000, window=100,epsilon_opt=1,epsilon_min=0.01,path=\" \"):\n",
    "    \"\"\" Monitor agent's performance.\n",
    "    \n",
    "    Params\n",
    "    ======\n",
    "    - env: instance of OpenAI Gym's Taxi-v1 environment\n",
    "    - agent: instance of class Agent (see Agent.py for details)\n",
    "    - num_episodes: number of episodes of agent-environment interaction\n",
    "    - window: number of episodes to consider when calculating average rewards\n",
    "\n",
    "    Returns\n",
    "    =======\n",
    "    - avg_rewards: deque containing average rewards\n",
    "    - best_avg_reward: largest value in the avg_rewards deque\n",
    "    \"\"\"\n",
    "    # initialize average rewards\n",
    "#     env = gym.wrappers.Monitor(env, \"./vid\"+path, force=True)\n",
    "#     vid = video_recorder.VideoRecorder(env,path=\"./recording\"+path+\"vid.mp4\")\n",
    "    avg_rewards = deque(maxlen=num_episodes)\n",
    "    # initialize best average reward\n",
    "    best_avg_reward = -math.inf\n",
    "    # initialize monitor for most recent rewards\n",
    "    samp_rewards = deque(maxlen=window)\n",
    "    J_step=[]\n",
    "    # for each episode\n",
    "    policies=[]\n",
    "    Full_state=[]\n",
    "    epsilon_all=[]\n",
    "    catch_ep=[]\n",
    "    total_rewards=[]\n",
    "    for i_episode in range(1, num_episodes+1):\n",
    "        catch= defaultdict(lambda: np.zeros(env.action_space.n))\n",
    "        states=[]\n",
    "        j=0\n",
    "        epsilon_ep=[]\n",
    "        # begin the episode\n",
    "        state = env.reset()\n",
    "        # initialize the sampled reward\n",
    "        samp_reward = 0\n",
    "        agent.epsilon=0.9 if i_episode%500==0 or i_episode<80000 else agent.epsilon #Sawtooth waveform for epsilon\n",
    "        if i_episode%500==0 and i_episode>66000:\n",
    "            agent.gamma=0.9  \n",
    "        elif i_episode>85000:\n",
    "            agent.gamma=0.6\n",
    "        else:\n",
    "            agent.gamma=agent.gamma\n",
    "        if i_episode>85000:\n",
    "            agent.alpha=0.1\n",
    "        while True:\n",
    "            # agent selects an action\n",
    "            action = agent.select_action(state,env)\n",
    "            # agent performs the selected action\n",
    "            next_state, reward, done, _ = env.step(action)\n",
    "            states.append((state,action,reward,next_state))\n",
    "            # agent performs internal updates based on sampled experience\n",
    "            next_action=agent.select_action(next_state,env)\n",
    "            agent.step(state, action, reward, next_state, next_action, done, i_episode)\n",
    "            # update the sampled reward\n",
    "            samp_reward += reward\n",
    "            # update the state (s <- s') to next time step\n",
    "            state = next_state\n",
    "            catch[state][action]+=1\n",
    "            if j>120:\n",
    "                agent.update_policy(state)\n",
    "            j+=1\n",
    "            if done:\n",
    "                # save final sampled reward\n",
    "                samp_rewards.append(samp_reward)\n",
    "                epsilon_ep.append(agent.epsilon)\n",
    "                if epsilon_opt==0:                         \n",
    "                    agent.epsilon=max(agent.epsilon*(-.05-np.exp(-.07)),epsilon_min)\n",
    "                else:\n",
    "                    agent.epsilon=max(np.exp(-(i_episode+0.0001)),epsilon_min)\n",
    "                J_step.append(j)\n",
    "                Full_state.append(states)\n",
    "                policies.append(agent.policy)\n",
    "                agent.update_policy(state)\n",
    "                catch_ep.append(catch)\n",
    "                total_rewards.append(samp_reward)\n",
    "                break\n",
    "        if (i_episode >= 100):\n",
    "            # get average reward from last 100 episodes\n",
    "            avg_reward = np.mean(samp_rewards)\n",
    "            # append to deque\n",
    "            avg_rewards.append(avg_reward)\n",
    "            # update best average reward\n",
    "            if avg_reward > best_avg_reward:\n",
    "                best_avg_reward = avg_reward\n",
    "        # monitor progress\n",
    "        print(\"\\rEpisode {}/{} || Best average reward {}\".format(i_episode,num_episodes, best_avg_reward), end=\"\")\n",
    "        sys.stdout.flush()\n",
    "        # check if task is solved (according to OpenAI Gym)\n",
    "        if best_avg_reward >= 9.7:\n",
    "            print('\\nEnvironment solved in {} episodes.'.format(i_episode), end=\"\")\n",
    "            break\n",
    "        if i_episode == num_episodes: print('\\n')\n",
    "    return avg_rewards, best_avg_reward , J_step, Full_state, policies,catch_ep,total_rewards"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # from SK_agent import Agent\n",
    "# # from monitor import interact\n",
    "import gym\n",
    "import numpy as np\n",
    "from gym import wrappers\n",
    "\n",
    "env = gym.make('Taxi-v3')\n",
    "# env = gym.wrappers.Monitor(env, \"./vid\", force=True)\n",
    "# agent = Agent(epsilon=1,gamma=0.9)\n",
    "# avg_rewards, best_avg_reward = interact(env, agent)\n",
    "# print(best_avg_reward)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Iteration 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "For alpha=0.01, gamma=0.4, algo=sarsamax and epsilon option=i_episode%500)+0.03\n",
      "Episode 100000/100000 || Best average reward 7.9136\n",
      "\n",
      "For alpha=0.01, gamma=0.2, algo=sarsamax and epsilon option=i_episode%500)+0.03\n",
      "Episode 100000/100000 || Best average reward 7.3618\n",
      "\n",
      "For alpha=0.005, gamma=0.4, algo=sarsamax and epsilon option=i_episode%500)+0.03\n",
      "Episode 100000/100000 || Best average reward 6.5123\n",
      "\n",
      "For alpha=0.005, gamma=0.2, algo=sarsamax and epsilon option=i_episode%500)+0.03\n",
      "Episode 100000/100000 || Best average reward 6.5524\n",
      "\n",
      "{'alpha': 0.01, 'gamma': 0.4, 'algo': 1, 'e_func': 1, 'average_reward': 7.91}\n",
      "[(0.01, 0.4, 1, 1, 7.91), (0.01, 0.2, 1, 1, 7.36), (0.005, 0.4, 1, 1, 6.51), (0.005, 0.2, 1, 1, 6.55)]\n"
     ]
    }
   ],
   "source": [
    "alpha_g=[0.01,0.005]#,  0.4,0.5,0.6,0.8,0.1]\n",
    "gamma_g=[0.4, 0.2]#,0.9,0.5 ,0.6 , 0.8]\n",
    "epsilon_opt_g=[1]#,0]\n",
    "epsilon_dict_g={0:'e*0.98',1:'i_episode%500)+0.03'}\n",
    "algo_g=[1]\n",
    "dict_alg_g={0:'sarsa',1:'sarsamax',2:'esarsa'}\n",
    "dict_eps_g={0:'e*0.999',1:'1/(i+0.005)'}\n",
    "avg_reward_g=-100\n",
    "best_dict_g={'alpha':0, 'gamma':0, 'algo':'','e_func':'','average_reward':0}\n",
    "good_rewards_g=[]\n",
    "all_states_g=[]\n",
    "all_J_g=[]#all timesteps per hyperparameter\n",
    "allpols=[]\n",
    "avg_reward=-math.inf\n",
    "all_avg_rewards=[]#See the trend of average rewards for each set of hyperparameters\n",
    "all_catch=[]#Catch is to identify how many times is each state action pair visited\n",
    "all_rewards=[]#This is to check what are the rewards per episode vs the states. Need to see why latter episodes are not generating any greater rewards\n",
    "all_Q=[]\n",
    "for a in alpha_g: # Perform a hyper parameter search to find initial hyper parameter conditions\n",
    "    for g in gamma_g:\n",
    "        for ep in epsilon_opt_g:\n",
    "            for al in algo_g:\n",
    "                    path='/'+str(a)+\" \"+str(g)+\" \"+str(al)+\" \"+str(ep)\n",
    "                    agent = Agent(algo=al,alpha=a,gamma=g)\n",
    "                    print(f'For alpha={a}, gamma={g}, algo={dict_alg_g[al]} and epsilon option={epsilon_dict_g[ep]}')\n",
    "                    avg_rewards, best_avg_reward, episode_step, episode_states, policies,count,total_rewards = interact(env, agent,num_episodes=100000, epsilon_opt=ep,path=path)\n",
    "                    all_states_g.append(episode_states)\n",
    "                    all_avg_rewards.append(avg_rewards)\n",
    "                    all_J_g.append(episode_step)\n",
    "                    allpols.append(policies)\n",
    "                    all_catch.append(count)\n",
    "                    all_Q.append(agent.Q)\n",
    "                    if best_avg_reward>-50:\n",
    "                        good_rewards_g.append((a,g,al,ep,best_avg_reward))\n",
    "                    if best_avg_reward>avg_reward:\n",
    "                        avg_reward=best_avg_reward\n",
    "                        best_dict_g['alpha']=a\n",
    "                        best_dict_g['gamma']=g\n",
    "                        best_dict_g['algo']=al\n",
    "                        best_dict_g['e_func']=ep\n",
    "                        best_dict_g['average_reward']=best_avg_reward\n",
    "print(best_dict_g)\n",
    "print(good_rewards_g)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_avg_rewards1=[]\n",
    "for a in all_avg_rewards:\n",
    "    ar=list(a)\n",
    "    all_avg_rewards1.append(ar)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "import copy\n",
    "all_catch_list=copy.deepcopy(all_catch)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "#all_catch - one value per hyperparameter combo\n",
    "#all_catch[0] - all states for each episode\n",
    "#all_catch[0][0] - dictionary of state_action pair for each step\n",
    "\n",
    "for i in all_catch_list:\n",
    "    for j in i:\n",
    "        for k in j.keys():\n",
    "            j[k]=j[k].tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_Q_list=copy.deepcopy(all_Q)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_Q_list[1].keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_Q_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dict_save={'all_states_g':all_states_g,\n",
    "           'all_avg_rewards':all_avg_rewards1, \n",
    "           'all_J_g':all_J_g,\n",
    "           \"allpols\":allpols,\n",
    "           'all_catch':all_catch_list}\n",
    "           #'all_Q':all_Q_list}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "json = json.dumps({'all_states_g':all_states_g,\n",
    "           'all_avg_rewards':all_avg_rewards1, \n",
    "           'all_J_g':all_J_g,\n",
    "           \"allpols\":allpols,\n",
    "           'all_catch':all_catch_list})\n",
    "            #'all_Q':all_Q_list}})\n",
    "f = open(\"dict_save.json\",\"w\")\n",
    "f.write(json)\n",
    "f.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "fig = plt.figure(figsize=[20,10])\n",
    "axes=fig.add_axes([0,0,1,1])\n",
    "plt.plot(range(100000),all_J_g[0],color='black')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}