Add notebooks for GECCO 2022 conference publication

environment-base.yml

@@ -16,6 +16,6 @@ dependencies:
   - ipympl
   - tqdm
   - pip:
-    - gym>=0.18
-  imports:
-    - requirements/base.
+      - gym>=0.18
+  - imports:
+      - requirements/base.

notebooks/publications/2022_preliminary_tests_of_acs2_with_experience_replay/.gitignore

@@ -0,0 +1,2 @@
+*.dill
+RESULTS/*

notebooks/publications/2022_preliminary_tests_of_acs2_with_experience_replay/MPX11_Run.ipynb

@@ -0,0 +1,259 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 31,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import os\n",
+    "import itertools\n",
+    "import numpy as np\n",
+    "from utils.run_utils import Runner\n",
+    "\n",
+    "import gym\n",
+    "import gym_multiplexer\n",
+    "from gym_multiplexer.utils import get_correct_answer\n",
+    "\n",
+    "from lcs import Perception\n",
+    "from lcs.agents.acs2 import ACS2, Configuration as CFG_ACS2\n",
+    "from lcs.agents.acs2er import ACS2ER, Configuration as CFG_ACS2ER\n",
+    "from lcs.metrics import population_metrics\n",
+    "\n",
+    "# Logger\n",
+    "import logging\n",
+    "logging.basicConfig(level=logging.INFO)\n"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# EXPERIMENT CONFIGURATION"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 32,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "BITS = 11 # 6 | 11 | 20 | 37\n",
+    "MPX = f'boolean-multiplexer-{BITS}bit-v0'\n",
+    "EXPLORE_TRIALS = 5000\n",
+    "EXPLOIT_TRIALS = 1000\n",
+    "METRICS_FREQUENCY = 1\n",
+    "KNOWLEDGE_STATE_SAMPLES = 1000 # applies only when 20 or 37 bits, otherwise all possible states verified\n",
+    "\n",
+    "# The size of ER replay memory buffer\n",
+    "ER_BUFFER_SIZE = 10000\n",
+    "# The minimum number of samples of ER replay memory buffer to start replying samples (warm-up phase)\n",
+    "ER_BUFFER_MIN_SAMPLES = 1000\n",
+    "# The number of samples to be replayed druing ER phase\n",
+    "ER_SAMPLES_NUMBER_LIST = [1,3,5,13]\n",
+    "\n",
+    "\n",
+    "\n",
+    "#######\n",
+    "\n",
+    "REPEAT_START = 1\n",
+    "REPEAT = 1\n",
+    "\n",
+    "EXPERIMENT_NAME = \"MPX11_EXP1\" # Please edit if running new experiment to do not override saved results."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "runner = Runner('MPX', EXPERIMENT_NAME, MPX)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## METRICS"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 33,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "class MpxObservationWrapper(gym.ObservationWrapper):\n",
+    "    def observation(self, observation):\n",
+    "        return [str(x) for x in observation]\n",
+    "        \n",
+    "knowledge_env = MpxObservationWrapper(gym.make(MPX))\n",
+    "\n",
+    "def get_transitions(states):\n",
+    "    transitions = list(map(lambda s: \n",
+    "            (Perception([str(float(x)) for x in s] + ['0.0']), \n",
+    "            get_correct_answer(list(s) + [0], knowledge_env.env.env.control_bits), \n",
+    "            Perception([str(float(x)) for x in s] + ['1.0'])), \n",
+    "        states))\n",
+    "\n",
+    "    return transitions\n",
+    "\n",
+    "def _mpx_knowledge(population, transitions, transitions_length) -> float:\n",
+    "    # Take into consideration only reliable classifiers\n",
+    "    reliable_classifiers = [c for c in population if c.is_reliable()]\n",
+    "\n",
+    "    if(len(reliable_classifiers) == 0):\n",
+    "        return 0\n",
+    "\n",
+    "    nr_correct = 0\n",
+    "\n",
+    "    for p0, correct_answer, p1 in transitions:\n",
+    "        if any([True for cl in reliable_classifiers if\n",
+    "                cl.predicts_successfully(\n",
+    "                    p0,\n",
+    "                    correct_answer,\n",
+    "                    p1)]):\n",
+    "\n",
+    "            nr_correct += 1\n",
+    "\n",
+    "    return nr_correct / transitions_length\n",
+    "\n",
+    "if BITS == 6 or BITS == 11: # Verify all \n",
+    "    def get_all_transitions():\n",
+    "        states = list(itertools.product([0, 1], repeat=BITS))\n",
+    "        return get_transitions(states)\n",
+    "\n",
+    "    TRANSITIONS = get_all_transitions()\n",
+    "    TRANSITIONS_LENGTH = len(TRANSITIONS)\n",
+    "    \n",
+    "    def mpx_knowledge(population) -> float:\n",
+    "        return _mpx_knowledge(population, TRANSITIONS, TRANSITIONS_LENGTH)\n",
+    "\n",
+    "elif BITS == 20 or BITS == 37: # Verify samples\n",
+    "    def get_sampled_transitions():\n",
+    "        states = np.random.randint(2, size=(KNOWLEDGE_STATE_SAMPLES, BITS))\n",
+    "        return get_transitions(states)\n",
+    "\n",
+    "    def mpx_knowledge(population) -> float:\n",
+    "        return _mpx_knowledge(population, get_sampled_transitions(), KNOWLEDGE_STATE_SAMPLES)\n",
+    "else:\n",
+    "    raise Exception(f'Unsupported BITS number: {BITS}')\n",
+    "    \n",
+    "\n",
+    "def mpx_metrics(agent, env):\n",
+    "    metrics = {\n",
+    "        \"knowledge\": mpx_knowledge(agent.population)\n",
+    "    }\n",
+    "    metrics.update(population_metrics(agent.population, env))\n",
+    "\n",
+    "    return metrics\n"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## EXPERIMENT"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 34,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def _run_experiment(agent, path):\n",
+    "    runner.run_experiment(agent, MpxObservationWrapper(gym.make(MPX)), EXPLORE_TRIALS, EXPLOIT_TRIALS, path)\n",
+    "    \n",
+    "def run_acs2_experiment():\n",
+    "    for i in range(REPEAT_START, REPEAT_START + REPEAT):\n",
+    "        # Create agent \n",
+    "        cfg = CFG_ACS2(\n",
+    "            classifier_length=knowledge_env.env.observation_space.n,\n",
+    "            number_of_possible_actions=2,\n",
+    "            do_ga=True,\n",
+    "            metrics_trial_frequency=METRICS_FREQUENCY,\n",
+    "            user_metrics_collector_fcn=mpx_metrics)\n",
+    "        agent = ACS2(cfg)\n",
+    "\n",
+    "        _run_experiment(agent, f'{i}')\n",
+    "\n",
+    "def _run_acs2er_experiment(er_samples_number: int):\n",
+    "    for i in range(REPEAT_START, REPEAT_START + REPEAT):\n",
+    "        # Create agent \n",
+    "        cfg = CFG_ACS2ER(    \n",
+    "            classifier_length=knowledge_env.env.observation_space.n,\n",
+    "            number_of_possible_actions=2,\n",
+    "            do_ga=True,\n",
+    "            metrics_trial_frequency=METRICS_FREQUENCY,\n",
+    "            er_buffer_size=ER_BUFFER_SIZE,\n",
+    "            er_min_samples=ER_BUFFER_MIN_SAMPLES,\n",
+    "            er_samples_number=er_samples_number,\n",
+    "            user_metrics_collector_fcn=mpx_metrics)\n",
+    "        agent = ACS2ER(cfg)\n",
+    "\n",
+    "        _run_experiment(agent, os.path.join(f'm_{er_samples_number}', f'{i}'))\n",
+    "\n",
+    "def run_acs2er_experiments():\n",
+    "    for er_samples_number in ER_SAMPLES_NUMBER_LIST:\n",
+    "        print(f\"START - ACS2ER - {er_samples_number}\")\n",
+    "        _run_acs2er_experiment(er_samples_number)\n",
+    "        print(f\"END - ACS2ER - {er_samples_number}\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### RUN ACS2 Experiments"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "run_acs2_experiment()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### RUN ACS2ER Experiments"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "run_acs2er_experiments()"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.7.12"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 4
+}