22 - Approximation Q Control - Cart Pole (1).ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pip install gym==0.21.0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pip install pyglet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import pandas as pd\n",
    "from sklearn.kernel_approximation import RBFSampler\n",
    "from sklearn.preprocessing import PolynomialFeatures\n",
    "import time\n",
    "import gym"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "gym.version.VERSION"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "env = gym.make(\"CartPole-v0\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "s=env.reset()\n",
    "s"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "a=env.action_space.sample() #random action\n",
    "a"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "s_next, r, done, info = env.step(a)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "env.reset()\n",
    "done=False\n",
    "while not done:\n",
    "    a=env.action_space.sample() #random action\n",
    "    s_next, r, done, info = env.step(a)\n",
    "    time.sleep(0.05)\n",
    "    env.render()\n",
    "env.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def merge_state_action(s, a):\n",
    "    return np.concatenate((s, [a]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "s=env.reset()\n",
    "a=env.action_space.sample()\n",
    "sa=merge_state_action(s, a)\n",
    "sa"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "poly_features = PolynomialFeatures(degree=2)\n",
    "poly_features.fit([sa])\n",
    "poly_features.transform([sa])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def gather_samples(n_episodes=10000):\n",
    "    samples = []\n",
    "    for i in range(n_episodes):\n",
    "        s = env.reset()\n",
    "        done=False\n",
    "        while not done:\n",
    "            a=env.action_space.sample() #random action\n",
    "            sa = merge_state_action(s, a)\n",
    "            samples.append(sa)\n",
    "            s, r, done, info = env.step(a)\n",
    "    return samples"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "samples=gather_samples(n_episodes=10000)\n",
    "samples"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.kernel_approximation import Nystroem\n",
    "nystrom_featurizer = Nystroem(gamma=1,random_state=1,n_components=100)\n",
    "nystrom_featurizer.fit(samples)\n",
    "nystrom_featurizer.components_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "rbf_feature = RBFSampler(n_components=100)\n",
    "rbf_feature.fit(samples)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def predictV(s,a,w,kernel):\n",
    "    sa = merge_state_action(s, a)\n",
    "    if kernel=='poly':\n",
    "        poly_features.fit([sa])\n",
    "        x=poly_features.transform([sa])[0]\n",
    "    elif kernel=='rbf':\n",
    "        x = rbf_feature.transform([sa])[0]\n",
    "    else:\n",
    "        x=nystrom_featurizer.transform([sa])[0]\n",
    "    return np.dot(x,w)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def gradientV(s,a,kernel):\n",
    "    sa = merge_state_action(s, a)\n",
    "    if kernel=='poly':\n",
    "        poly_features.fit([sa])\n",
    "        x=poly_features.transform([sa])[0]\n",
    "    elif kernel=='rbf':\n",
    "        x = rbf_feature.transform([sa])[0]\n",
    "    else:\n",
    "        x=nystrom_featurizer.transform([sa])[0]\n",
    "    return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "env.action_space.n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def predictV_all_actions(s,w,kernel):\n",
    "    values=[]\n",
    "    for a in range(env.action_space.n):\n",
    "        values.append(predictV(s,a,w,kernel))\n",
    "    return values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def epsilon_greedy(s, w, kernel, eps=0.1):\n",
    "    values=[]\n",
    "    p = np.random.random()\n",
    "    if p < (1 - eps):\n",
    "        values = predictV_all_actions(s,w,kernel)\n",
    "        return np.argmax(values)\n",
    "    else:\n",
    "        return env.action_space.sample()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "GAMMA=0.9\n",
    "ALPHA = 0.1\n",
    "#Initialize weights as zero\n",
    "#RBFSampler has 100 dimensions as default, hence we have 100 features and weights\n",
    "w = np.zeros(poly_features.n_output_features_) \n",
    "n_episodes = 1500\n",
    "for it in range(n_episodes):\n",
    "    # begin a new episode\n",
    "    s=env.reset()\n",
    "    done=False\n",
    "    while not done:\n",
    "        a = epsilon_greedy(s, w, kernel='poly',eps=0.1)\n",
    "        s2, r, done, info = env.step(a)\n",
    "        if done:\n",
    "            target = r\n",
    "        else:\n",
    "            values = predictV_all_actions(s2,w,kernel='poly')\n",
    "            target = r + GAMMA * np.max(values)               \n",
    "        # update the weights\n",
    "        g = gradientV(s,a,kernel='poly')\n",
    "        err=target - predictV(s,a,w,'poly')\n",
    "        w += ALPHA * err * g\n",
    "        # update state\n",
    "        s = s2      "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "w"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def watch_agent(w,kernel):\n",
    "    done = False\n",
    "    episode_reward = 0\n",
    "    s = env.reset()\n",
    "    while not done:\n",
    "        a = epsilon_greedy(s, w, kernel,eps=0)\n",
    "        s, r, done, info = env.step(a)\n",
    "        time.sleep(0.05)\n",
    "        env.render()\n",
    "        episode_reward += r\n",
    "    print(\"Episode reward:\", episode_reward)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "watch_agent(w,kernel='poly')\n",
    "env.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "GAMMA=0.9\n",
    "ALPHA = 0.1\n",
    "#Initialize weights as zero\n",
    "w = np.zeros(100) \n",
    "n_episodes = 500\n",
    "for it in range(n_episodes):\n",
    "    print(it)\n",
    "    # begin a new episode\n",
    "    s=env.reset()\n",
    "    done=False\n",
    "    while not done:\n",
    "        a = epsilon_greedy(s, w, kernel='nystrom',eps=0.1)\n",
    "        s2, r, done, info = env.step(a)\n",
    "        if done:\n",
    "            target = r\n",
    "        else:\n",
    "            values = predictV_all_actions(s2,w,kernel='nystrom')\n",
    "            target = r + GAMMA * np.max(values)               \n",
    "        # update the weights\n",
    "        g = gradientV(s,a,kernel='nystrom')\n",
    "        err=target - predictV(s,a,w,'nystrom')\n",
    "        w += ALPHA * err * g\n",
    "        #print(w[0])\n",
    "        # update state\n",
    "        s = s2  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "watch_agent(w,kernel='nystrom')\n",
    "env.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "GAMMA=0.9\n",
    "ALPHA = 0.1\n",
    "#Initialize weights as zero\n",
    "w = np.zeros(100) \n",
    "n_episodes = 500\n",
    "for it in range(n_episodes):\n",
    "    print(it)\n",
    "    # begin a new episode\n",
    "    s=env.reset()\n",
    "    done=False\n",
    "    while not done:\n",
    "        a = epsilon_greedy(s, w, kernel='rbf',eps=0.1)\n",
    "        s2, r, done, info = env.step(a)\n",
    "        if done:\n",
    "            target = r\n",
    "        else:\n",
    "            values = predictV_all_actions(s2,w,kernel='rbf')\n",
    "            target = r + GAMMA * np.max(values)               \n",
    "        # update the weights\n",
    "        g = gradientV(s,a,kernel='rbf')\n",
    "        err=target - predictV(s,a,w,'rbf')\n",
    "        w += ALPHA * err * g\n",
    "        # update state\n",
    "        s = s2  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "watch_agent(w,kernel='rbf')\n",
    "env.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}