test_dqn_implementation.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "And some CartPole DQN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/Users/theeimer/anaconda3/envs/mighty/lib/python3.10/site-packages/pygame/pkgdata.py:25: DeprecationWarning: pkg_resources is deprecated as an API. See https://setuptools.pypa.io/en/latest/pkg_resources.html\n",
      "  from pkg_resources import resource_stream, resource_exists\n",
      "/Users/theeimer/anaconda3/envs/mighty/lib/python3.10/site-packages/pkg_resources/__init__.py:2825: DeprecationWarning: Deprecated call to `pkg_resources.declare_namespace('sphinxcontrib')`.\n",
      "Implementing implicit namespace packages (as specified in PEP 420) is preferred to `pkg_resources.declare_namespace`. See https://setuptools.pypa.io/en/latest/references/keywords.html#keyword-namespace-packages\n",
      "  declare_namespace(pkg)\n",
      "/Users/theeimer/anaconda3/envs/mighty/lib/python3.10/site-packages/dacbench/envs/__init__.py:34: UserWarning: CMA-ES Benchmark not installed. If you want to use this benchmark, please follow the installation guide.\n",
      "  warnings.warn(  # noqa: B028\n",
      "/Users/theeimer/anaconda3/envs/mighty/lib/python3.10/site-packages/dacbench/benchmarks/__init__.py:26: UserWarning: CMA-ES Benchmark not installed. If you want to use this benchmark, please follow the installation guide.\n",
      "  warnings.warn(  # noqa: B028\n",
      "/Users/theeimer/anaconda3/envs/mighty/lib/python3.10/site-packages/carl/envs/__init__.py:36: UserWarning: Module py4j not found. If you want to use these environments, please follow the installation guide.\n",
      "  warnings.warn(\n",
      "/Users/theeimer/anaconda3/envs/mighty/lib/python3.10/site-packages/carl/envs/__init__.py:36: UserWarning: Module distance not found. If you want to use these environments, please follow the installation guide.\n",
      "  warnings.warn(\n",
      "[\u001b[36m2024-10-02 14:59:35,304\u001b[0m][\u001b[35mHYDRA\u001b[0m] Launching 5 jobs locally\u001b[0m\n",
      "[\u001b[36m2024-10-02 14:59:35,305\u001b[0m][\u001b[35mHYDRA\u001b[0m] \t#0 : experiment_name=cartpole_dqn algorithm=dqn environment=gymnasium/cartpole seed=0\u001b[0m\n",
      "\u001b[2;36m[10/02/24 14:59:35]\u001b[0m\u001b[2;36m \u001b[0m\u001b[34mINFO    \u001b[0m Output will be written to       \u001b]8;id=179097;file:///Users/theeimer/Documents/git/Mighty-DACS/mighty/mighty_runners/mighty_runner.py\u001b\\\u001b[2mmighty_runner.py\u001b[0m\u001b]8;;\u001b\\\u001b[2m:\u001b[0m\u001b]8;id=442161;file:///Users/theeimer/Documents/git/Mighty-DACS/mighty/mighty_runners/mighty_runner.py#33\u001b\\\u001b[2m33\u001b[0m\u001b]8;;\u001b\\\n",
      "\u001b[2;36m                    \u001b[0m         runs/cartpole_dqn_0             \u001b[2m                   \u001b[0m\n",
      "Initialized agent.\n",
      "\u001b[2;36m                   \u001b[0m\u001b[2;36m \u001b[0m\u001b[34mINFO    \u001b[0m ############################### \u001b]8;id=947079;file:///Users/theeimer/Documents/git/Mighty-DACS/mighty/mighty_runners/mighty_runner.py\u001b\\\u001b[2mmighty_runner.py\u001b[0m\u001b]8;;\u001b\\\u001b[2m:\u001b[0m\u001b]8;id=509370;file:///Users/theeimer/Documents/git/Mighty-DACS/mighty/mighty_runners/mighty_runner.py#76\u001b\\\u001b[2m76\u001b[0m\u001b]8;;\u001b\\\n",
      "\u001b[2;36m                    \u001b[0m         ############################### \u001b[2m                   \u001b[0m\n",
      "\u001b[2;36m                    \u001b[0m         ##################              \u001b[2m                   \u001b[0m\n",
      "\u001b[2;36m                   \u001b[0m\u001b[2;36m \u001b[0m\u001b[34mINFO    \u001b[0m Using agent type                \u001b]8;id=55912;file:///Users/theeimer/Documents/git/Mighty-DACS/mighty/mighty_runners/mighty_runner.py\u001b\\\u001b[2mmighty_runner.py\u001b[0m\u001b]8;;\u001b\\\u001b[2m:\u001b[0m\u001b]8;id=880722;file:///Users/theeimer/Documents/git/Mighty-DACS/mighty/mighty_runners/mighty_runner.py#77\u001b\\\u001b[2m77\u001b[0m\u001b]8;;\u001b\\\n",
      "\u001b[2;36m                    \u001b[0m         \u001b[32m\"\u001b[0m\u001b[32m<\u001b[0m\u001b[32mmighty.mighty_agents.dqn.Migh\u001b[0m \u001b[2m                   \u001b[0m\n",
      "\u001b[2;36m                    \u001b[0m         \u001b[32mtyDQNAgent\u001b[0m\u001b[32m object at \u001b[0m           \u001b[2m                   \u001b[0m\n",
      "\u001b[2;36m                    \u001b[0m         \u001b[32m0x339c65390\u001b[0m\u001b[32m>\u001b[0m\u001b[32m\"\u001b[0m to learn          \u001b[2m                   \u001b[0m\n",
      "\u001b[2;36m                   \u001b[0m\u001b[2;36m \u001b[0m\u001b[34mINFO    \u001b[0m ############################### \u001b]8;id=733453;file:///Users/theeimer/Documents/git/Mighty-DACS/mighty/mighty_runners/mighty_runner.py\u001b\\\u001b[2mmighty_runner.py\u001b[0m\u001b]8;;\u001b\\\u001b[2m:\u001b[0m\u001b]8;id=575680;file:///Users/theeimer/Documents/git/Mighty-DACS/mighty/mighty_runners/mighty_runner.py#78\u001b\\\u001b[2m78\u001b[0m\u001b]8;;\u001b\\\n",
      "\u001b[2;36m                    \u001b[0m         ############################### \u001b[2m                   \u001b[0m\n",
      "\u001b[2;36m                    \u001b[0m         ##################              \u001b[2m                   \u001b[0m\n",
      "\u001b[2KTrain Steps \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[35m  0%\u001b[0m Remaining: \u001b[36m-:--:--\u001b[0m Elapsed: \u001b[33m0:00:00\u001b[0m\n",
      "\u001b[?25hSteps: \u001b[1;36m5000\u001b[0m, Latest Episode Reward: \u001b[1;36m9.0\u001b[0m, Latest Step Reward: \u001b[1;36m2.25\u001b[0m\n",
      "Saved checkpoint at runs/cartpole_dqn_0/checkpoints/5000\n",
      "\n",
      "------------------------------------------------------------------------------\n",
      "Evaluation performance after \u001b[1;36m10000\u001b[0m steps:\n",
      "                \u001b[1;36m18.7\u001b[0m\n",
      "Evaluation performance per step after \u001b[1;36m10000\u001b[0m steps:\n",
      "                \u001b[1;36m1.0\u001b[0m\n",
      "------------------------------------------------------------------------------\n",
      "\n",
      "Steps: \u001b[1;36m10000\u001b[0m, Latest Episode Reward: \u001b[1;36m10.0\u001b[0m, Latest Step Reward: \u001b[1;36m2.5\u001b[0m\n",
      "Saved checkpoint at runs/cartpole_dqn_0/checkpoints/10000\n"
     ]
    }
   ],
   "source": [
    "!python mighty/run_mighty.py experiment_name=cartpole_dqn algorithm=dqn environment=gymnasium/cartpole seed=0,1,2,3,4 -m"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Then let's load the data:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "import pandas as pd\n",
    "from mighty.mighty_utils.logger import to_dataframe\n",
    "\n",
    "def load_results(experiment_name, seeds=[0,1,2,3,4]):\n",
    "    dfs = []\n",
    "    for s in seeds:\n",
    "        logs = []\n",
    "        with open(f\"runs/{experiment_name}_{s}/eval.jsonl\", \"r\") as f:\n",
    "            for line in f:\n",
    "                logs.append(json.loads(line))\n",
    "        df = to_dataframe(logs)\n",
    "        df[\"seed\"] = s\n",
    "        dfs.append(df)\n",
    "    return pd.concat(dfs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "ename": "FileNotFoundError",
     "evalue": "[Errno 2] No such file or directory: 'runs/cartpole_dqn_0/eval.jsonl'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mFileNotFoundError\u001b[0m                         Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[11], line 2\u001b[0m\n\u001b[1;32m      1\u001b[0m ppo_cartpole \u001b[38;5;241m=\u001b[39m load_results(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcartpole_ppo\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[0;32m----> 2\u001b[0m dqn_cartpole \u001b[38;5;241m=\u001b[39m \u001b[43mload_results\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mcartpole_dqn\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m)\u001b[49m\n",
      "Cell \u001b[0;32mIn[10], line 9\u001b[0m, in \u001b[0;36mload_results\u001b[0;34m(experiment_name, seeds)\u001b[0m\n\u001b[1;32m      7\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m s \u001b[38;5;129;01min\u001b[39;00m seeds:\n\u001b[1;32m      8\u001b[0m     logs \u001b[38;5;241m=\u001b[39m []\n\u001b[0;32m----> 9\u001b[0m     \u001b[38;5;28;01mwith\u001b[39;00m \u001b[38;5;28;43mopen\u001b[39;49m\u001b[43m(\u001b[49m\u001b[38;5;124;43mf\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mruns/\u001b[39;49m\u001b[38;5;132;43;01m{\u001b[39;49;00m\u001b[43mexperiment_name\u001b[49m\u001b[38;5;132;43;01m}\u001b[39;49;00m\u001b[38;5;124;43m_\u001b[39;49m\u001b[38;5;132;43;01m{\u001b[39;49;00m\u001b[43ms\u001b[49m\u001b[38;5;132;43;01m}\u001b[39;49;00m\u001b[38;5;124;43m/eval.jsonl\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mr\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m)\u001b[49m \u001b[38;5;28;01mas\u001b[39;00m f:\n\u001b[1;32m     10\u001b[0m         \u001b[38;5;28;01mfor\u001b[39;00m line \u001b[38;5;129;01min\u001b[39;00m f:\n\u001b[1;32m     11\u001b[0m             logs\u001b[38;5;241m.\u001b[39mappend(json\u001b[38;5;241m.\u001b[39mloads(line))\n",
      "File \u001b[0;32m~/anaconda3/envs/mighty/lib/python3.10/site-packages/IPython/core/interactiveshell.py:324\u001b[0m, in \u001b[0;36m_modified_open\u001b[0;34m(file, *args, **kwargs)\u001b[0m\n\u001b[1;32m    317\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m file \u001b[38;5;129;01min\u001b[39;00m {\u001b[38;5;241m0\u001b[39m, \u001b[38;5;241m1\u001b[39m, \u001b[38;5;241m2\u001b[39m}:\n\u001b[1;32m    318\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(\n\u001b[1;32m    319\u001b[0m         \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mIPython won\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mt let you open fd=\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mfile\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m by default \u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m    320\u001b[0m         \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mas it is likely to crash IPython. If you know what you are doing, \u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m    321\u001b[0m         \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124myou can use builtins\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m open.\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m    322\u001b[0m     )\n\u001b[0;32m--> 324\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mio_open\u001b[49m\u001b[43m(\u001b[49m\u001b[43mfile\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
      "\u001b[0;31mFileNotFoundError\u001b[0m: [Errno 2] No such file or directory: 'runs/cartpole_dqn_0/eval.jsonl'"
     ]
    }
   ],
   "source": [
    "dqn_cartpole = load_results(\"cartpole_dqn\")\n",
    "dqn_cartpole[\"algorithm\"] = \"baseline\"\n",
    "\n",
    "#dqn_cartpole_latest = load_results(\"cartpole_dqn_latest\")\n",
    "#dqn_cartpole_latest[\"algorithm\"] = \"update\"\n",
    "#dqn_cartpole = pd.concat([dqn_cartpole, dqn_cartpole_latest])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "And check how performance looks:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from mighty.mighty_utils.plotting import plot_performance_over_time\n",
    "\n",
    "# Make episode reward a scalar instead of list\n",
    "dqn_cartpole['episode_reward'] = [np.array(x).mean() for x in dqn_cartpole.episode_reward.values]\n",
    "# Just take eval points for plotting\n",
    "plot_data = dqn_cartpole[dqn_cartpole.episode_reward.isna() == False]\n",
    "\n",
    "plot_performance_over_time(plot_data, x=\"total_steps\", y=\"episode_reward\", hue=\"algorithm\", aggregation=\"mean\", logx=False, xlabel=\"Steps\", ylabel=\"Mean Reward\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Score history:\n",
    "\n",
    "DQN:"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "mighty",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}