LE_notebook.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from MuJoCo_Gym.mujoco_rl import MuJoCoRL\n",
    "from MuJoCo_Gym.wrappers import GymnasiumWrapper, GymWrapper\n",
    "from gymnasium.wrappers.frame_stack import FrameStack\n",
    "from gymnasium.experimental.wrappers import NormalizeObservationV0\n",
    "from dynamics import *\n",
    "import argparse\n",
    "import os\n",
    "import random\n",
    "import time\n",
    "from distutils.util import strtobool\n",
    "import gym\n",
    "import numpy as np\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "from torch.distributions.normal import Normal\n",
    "from torch.utils.tensorboard import SummaryWriter\n",
    "\n",
    "from wrappers.record_episode_statistics import RecordEpisodeStatistics\n",
    "from wrappers.frame_stack import FrameStack\n",
    "from wrappers.normalizeObservation import NormalizeObservation\n",
    "from wrappers.normalizeRewards import NormalizeReward\n",
    "\n",
    "from progressbar import progressbar"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Dynamics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import mean_squared_error\n",
    "from autoencoder import Autoencoder\n",
    "import math\n",
    "import mujoco"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Vision"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Image:\n",
    "    def __init__(self, environment):\n",
    "        self.environment = environment\n",
    "        self.observation_space = {\"low\": [0 for _ in range(50)], \"high\": [1 for _ in range(50)]}\n",
    "        self.action_space = {\"low\": [], \"high\": []}\n",
    "        self.autoencoder = Autoencoder(latent_dim=50, input_shape=(64, 64, 3))\n",
    "        self.autoencoder.encoder.load_weights(\"models/encoder50.h5\")\n",
    "        self.index = 0\n",
    "\n",
    "    def dynamic(self, agent, actions):\n",
    "        self.index = self.index + 1\n",
    "        image = self.environment.get_camera_data(agent + \"_camera\")\n",
    "        image = cv2.resize(image, (64, 64))\n",
    "        result = self.autoencoder.encoder.predict(np.array([image]), verbose=0)[0]\n",
    "        # cv2.imwrite(f\"/Users/cowolff/Documents/GitHub/s.language_experiments/images/{agent}_{self.index}.png\", image)\n",
    "        return 0, result, False, {}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Communication"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Communication:\n",
    "    def __init__(self, environment):\n",
    "        self.environment = environment\n",
    "        self.observation_space = {\"low\": [0, 0, 0, 0], \"high\": [1, 1, 1, 1]}\n",
    "        self.action_space = {\"low\": [0, 0, 0, 0], \"high\": [1, 1, 1, 1]}\n",
    "\n",
    "    def dynamic(self, agent, actions):\n",
    "        if \"utterance\" not in self.environment.data_store[agent].keys():\n",
    "            self.environment.data_store[agent][\"utterance\"] = None\n",
    "        if agent == \"receiver\":\n",
    "            utterance = [0, 0, 0, 0]\n",
    "            if \"target_color\" in self.environment.data_store.keys():\n",
    "                utterance[np.argmax(self.environment.data_store[\"target_color\"])] = 1\n",
    "            observation = utterance\n",
    "        elif agent == \"sender\":\n",
    "            utterance = [0, 0, 0, 0]\n",
    "            utterance[np.argmax(actions)] = 1\n",
    "            self.environment.data_store[agent][\"utterance\"] = actions\n",
    "            self.environment.data_store[agent][\"utterance_max\"] = utterance\n",
    "            observation = [0, 0, 0, 0]\n",
    "        else:\n",
    "            print(\"Dafaq is going on here?\")\n",
    "        return 0, observation, False, {}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Reward Dynamic Function\n",
    "The reward function for the receiver gives a positive reward for the agent to move closer to the current target object.<br/>\n",
    "The sender in turn gets a positive reward if it chooses the correct word encoding for the color of the current target object."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Reward:\n",
    "    def __init__(self, environment):\n",
    "        self.environment = environment\n",
    "        self.observation_space = {\"low\": [], \"high\": []}\n",
    "        self.action_space = {\"low\": [], \"high\": []}\n",
    "        self.choices = [\"choice_1\", \"choice_2\"]\n",
    "\n",
    "    def dynamic(self, agent, actions):\n",
    "        if not \"target\" in self.environment.data_store.keys():\n",
    "            color = self.environment.get_data(\"reference_geom\")[\"color\"]\n",
    "            for choice in self.choices:\n",
    "                if (color == self.environment.get_data(choice + \"_geom\")[\"color\"]).all():\n",
    "                    self.environment.data_store[\"target\"] = choice\n",
    "                    self.environment.data_store[\"target_color\"] = self.environment.get_data(choice + \"_geom\")[\"color\"]\n",
    "                    self.environment.data_store[\"last_distance\"] = copy.deepcopy(self.environment.distance(\"receiver_geom\", choice + \"_geom\"))\n",
    "        if agent == \"receiver\":\n",
    "            target = self.environment.data_store[\"target\"]\n",
    "            new_distance = self.environment.distance(\"receiver_geom\", target + \"_geom\")\n",
    "            reward = (self.environment.data_store[\"last_distance\"] - new_distance) * 10\n",
    "            self.environment.data_store[\"last_distance\"] = copy.deepcopy(new_distance)\n",
    "        elif agent == \"sender\":\n",
    "            reference = [0, 0, 0, 0]\n",
    "            color = self.environment.data_store[\"target_color\"]\n",
    "            reference[np.argmax(color)] = 1\n",
    "            reward = 0\n",
    "            if \"utterance\" in self.environment.data_store[agent].keys():\n",
    "                reward = -1 * mean_squared_error(reference, self.environment.data_store[agent][\"utterance\"])\n",
    "        return reward, [], False, {}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Accuracy Measurement Dynamic"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Accuracy:\n",
    "    def __init__(self, environment):\n",
    "        self.environment = environment\n",
    "        self.observation_space = {\"low\": [], \"high\": []}\n",
    "        self.action_space = {\"low\": [], \"high\": []}\n",
    "        self.accuracies = []\n",
    "        self.variances = []\n",
    "        self.sendAccuracies = []\n",
    "        self.sendVariances = []\n",
    "        self.currentSend = []\n",
    "        self.report_accuracy = {\"sender\": 0, \"receiver\": 0}\n",
    "\n",
    "    def dynamic(self, agent, actions):\n",
    "        choices = [\"choice_1\", \"choice_2\"]\n",
    "        variance = {\"choice_1\":1, \"choice_2\":-1}\n",
    "        if \"target\" in self.environment.data_store.keys():\n",
    "            if \"sendVariances\" not in self.environment.data_store.keys():\n",
    "                self.environment.data_store[\"sendVariances\"] = True\n",
    "                self.currentSend = [0, 0, 0, 0]\n",
    "            target = self.environment.data_store[\"target\"]\n",
    "            # if any(self.environment.collision(ankle, target + \"_geom\") for ankle in [\"left_leg_geom_2\", \"left_ankle_geom_2\", \"right_leg_geom_2\", \"right_ankle_geom_2\", \"back_leg_geom_2\", \"third_ankle_geom_2\", \"rightback_leg_geom_2\", \"fourth_ankle_geom_2\"]):\n",
    "            if self.environment.collision(\"receiver_geom\", target + \"_geom\"):\n",
    "                self.accuracies.append(1)\n",
    "                self.variances.append(variance[target])\n",
    "\n",
    "                if len(self.variances) > 50:\n",
    "                    report_variance = 1 - abs(sum(self.variances[-50:]) / 50)\n",
    "                    self.report_accuracy[\"receiver\"] = sum(self.accuracies[-50:]) / 50\n",
    "            # elif any(self.environment.collision(ankle, [choice for choice in choices if choice != target][0] + \"_geom\") for ankle in [\"left_leg_geom_2\", \"left_ankle_geom_2\", \"right_leg_geom_2\", \"right_ankle_geom_2\", \"back_leg_geom_2\", \"third_ankle_geom_2\", \"rightback_leg_geom_2\", \"fourth_ankle_geom_2\"]):\n",
    "            elif self.environment.collision(\"receiver_geom\", [choice for choice in choices if choice != target][0] + \"_geom\"):\n",
    "                self.accuracies.append(0)\n",
    "                self.variances.append(variance[[choice for choice in choices if choice != target][0]])\n",
    "\n",
    "                if len(self.variances) > 50:\n",
    "                    report_variance = 1 - abs(sum(self.variances[-50:]) / 50)\n",
    "                    self.report_accuracy[0] = sum(self.accuracies[-50:]) / 50\n",
    "            if \"utterance_max\" in self.environment.data_store[agent].keys():\n",
    "                reference = [0, 0, 0, 0]\n",
    "                color = self.environment.data_store[\"target_color\"]\n",
    "                reference[np.argmax(color)] = 1\n",
    "                self.currentSend = np.add(self.currentSend, self.environment.data_store[agent][\"utterance_max\"])\n",
    "\n",
    "                if self.environment.data_store[agent][\"utterance_max\"]  == reference:\n",
    "                    self.sendAccuracies.append(1)\n",
    "                else:\n",
    "                    self.sendAccuracies.append(0)\n",
    "        return 0, [], False, {}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Reward Functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def target_reward(mujoco_gym, agent):\n",
    "    if agent == \"receiver\":\n",
    "        choices = [\"choice_1\", \"choice_2\"]\n",
    "        if not \"target\" in mujoco_gym.data_store.keys():\n",
    "                color = mujoco_gym.get_data(\"reference_geom\")[\"color\"]\n",
    "                for choice in choices:\n",
    "                    if (color == mujoco_gym.get_data(choice + \"_geom\")[\"color\"]).all():\n",
    "                        mujoco_gym.data_store[\"target\"] = choice\n",
    "                        mujoco_gym.data_store[\"target_color\"] = mujoco_gym.get_data(choice + \"_geom\")[\"color\"]\n",
    "        \n",
    "        target = mujoco_gym.data_store[\"target\"]\n",
    "        # for ankle in [\"left_leg_geom_2\", \"left_ankle_geom_2\", \"right_leg_geom_2\", \"right_ankle_geom_2\", \"back_leg_geom_2\", \"third_ankle_geom_2\", \"rightback_leg_geom_2\", \"fourth_ankle_geom_2\"]:\n",
    "        for ankle in [\"receiver_geom\"]:\n",
    "            if mujoco_gym.collision(ankle, target + \"_geom\"):\n",
    "                return 1\n",
    "            elif mujoco_gym.collision(ankle, [choice for choice in choices if choice != target][0] + \"_geom\"):\n",
    "                return -1\n",
    "    return 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def collision_reward(mujoco_gym, agent):\n",
    "    for border in [\"border1_geom\", \"border2_geom\", \"border3_geom\", \"border4_geom\", \"border5_geom\"]:\n",
    "        # for ankle in [\"left_leg_geom_2\", \"left_ankle_geom_2\", \"right_leg_geom_2\", \"right_ankle_geom_2\", \"back_leg_geom_2\", \"third_ankle_geom_2\", \"rightback_leg_geom_2\", \"fourth_ankle_geom_2\"]:\n",
    "        for ankle in [agent + \"_geom\"]:\n",
    "            if mujoco_gym.collision(border, ankle):\n",
    "                return -0.1\n",
    "    return 0"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Environment Setup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def make_env(config_dict):\n",
    "    def thunk():\n",
    "        window = 5\n",
    "        env = MuJoCoRL(config_dict=config_dict)\n",
    "        # env = GymWrapper(env, \"receiver\")\n",
    "        # env = FrameStack(env, 4)\n",
    "        env = NormalizeObservation(env)\n",
    "        env = NormalizeReward(env)\n",
    "        # env = RecordEpisodeStatistics(env)\n",
    "        return env\n",
    "\n",
    "    return thunk"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/cowolff/miniconda3/envs/Ray/lib/python3.9/site-packages/gymnasium/spaces/box.py:130: UserWarning: \u001b[33mWARN: Box bound precision lowered by casting to float32\u001b[0m\n",
      "  gym.logger.warn(f\"Box bound precision lowered by casting to {self.dtype}\")\n",
      "/Users/cowolff/miniconda3/envs/Ray/lib/python3.9/site-packages/keras/src/engine/training_v1.py:2359: UserWarning: `Model.state_updates` will be removed in a future version. This property should not be used in TensorFlow 2.0, as `updates` are applied automatically.\n",
      "  updates=self.state_updates,\n"
     ]
    }
   ],
   "source": [
    "xml_files = [\"levels/\" + file for file in os.listdir(\"levels/\")]\n",
    "agents = [\"sender\", \"receiver\"]\n",
    "\n",
    "config_dict = {\"xmlPath\":xml_files, \n",
    "                   \"agents\":agents, \n",
    "                   \"rewardFunctions\":[collision_reward, target_reward], \n",
    "                   \"doneFunctions\":[target_done, border_done], \n",
    "                   \"skipFrames\":5,\n",
    "                   \"environmentDynamics\":[Image, Reward, Communication, Accuracy],\n",
    "                   \"freeJoint\":True,\n",
    "                   \"renderMode\":False,\n",
    "                   \"maxSteps\":1024,\n",
    "                   \"agentCameras\":True}\n",
    "\n",
    "env = make_env(config_dict)()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# RL Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def layer_init(layer, std=np.sqrt(2), bias_const=0.0):\n",
    "    torch.nn.init.orthogonal_(layer.weight, std)\n",
    "    torch.nn.init.constant_(layer.bias, bias_const)\n",
    "    return layer\n",
    "\n",
    "class Agent(nn.Module):\n",
    "    def __init__(self, envs):\n",
    "        super(Agent, self).__init__()\n",
    "        self.critic = nn.Sequential(\n",
    "            nn.Flatten(),\n",
    "            layer_init(nn.Linear(np.array(envs.observation_space.shape).prod(), 128)),\n",
    "            nn.Tanh(),\n",
    "            layer_init(nn.Linear(128, 128)),\n",
    "            nn.Tanh(),\n",
    "            layer_init(nn.Linear(128, 1), std=1.0),\n",
    "        )\n",
    "        self.actor_mean = nn.Sequential(\n",
    "            nn.Flatten(),\n",
    "            layer_init(nn.Linear(np.array(envs.observation_space.shape).prod(), 128)),\n",
    "            nn.Tanh(),\n",
    "            layer_init(nn.Linear(128, 128)),\n",
    "            nn.Tanh(),\n",
    "            layer_init(nn.Linear(128, np.prod(envs.action_space.shape)), std=0.01),\n",
    "        )\n",
    "        self.actor_logstd = nn.Parameter(torch.zeros(1, np.prod(envs.action_space.shape)))\n",
    "\n",
    "    def get_value(self, x):\n",
    "        return self.critic(x)\n",
    "\n",
    "    def get_action_and_value(self, x, action=None):\n",
    "        action_mean = self.actor_mean(x)\n",
    "        action_logstd = self.actor_logstd.expand_as(action_mean)\n",
    "        action_std = torch.exp(action_logstd)\n",
    "        probs = Normal(action_mean, action_std)\n",
    "        if action is None:\n",
    "            action = probs.sample()\n",
    "        return action, probs.log_prob(action).sum(1), probs.entropy().sum(1), self.critic(x)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Buffer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Buffer():\n",
    "    def __init__(self, num_steps, envs, num_envs, device):\n",
    "        self.obs = torch.zeros((num_steps, num_envs) + envs.observation_space.shape).to(device)\n",
    "        self.actions = torch.zeros((num_steps, num_envs) + envs.action_space.shape).to(device)\n",
    "        self.logprobs = torch.zeros((num_steps, num_envs)).to(device)\n",
    "        self.rewards = torch.zeros((num_steps, num_envs)).to(device)\n",
    "        self.dones = torch.zeros((num_steps, num_envs)).to(device)\n",
    "        self.values = torch.zeros((num_steps, num_envs)).to(device)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Update Function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def update_agent(agent, buffer, optimizer, next_obs, next_done, env, batch_size, update_epochs, minibatch_size, clip_coef, vf_coef, ent_coef, max_grad_norm, target_kl, clip_vloss, norm_adv, gae_lambda, gae, gamma, device, num_steps):\n",
    "\n",
    "    with torch.no_grad():\n",
    "        next_value = agent.get_value(next_obs).reshape(1, -1)\n",
    "        if gae:\n",
    "            advantages = torch.zeros_like(buffer.rewards).to(device)\n",
    "            lastgaelam = 0\n",
    "            for t in reversed(range(num_steps)):\n",
    "                if t == num_steps - 1:\n",
    "                    nextnonterminal = 1.0 - next_done\n",
    "                    nextvalues = next_value\n",
    "                else:\n",
    "                    nextnonterminal = 1.0 - buffer.dones[t + 1]\n",
    "                    nextvalues = buffer.values[t + 1]\n",
    "                delta = buffer.rewards[t] + gamma * nextvalues * nextnonterminal - buffer.values[t]\n",
    "                advantages[t] = lastgaelam = delta + gamma * gae_lambda * nextnonterminal * lastgaelam\n",
    "            returns = advantages + buffer.values\n",
    "        else:\n",
    "            returns = torch.zeros_like(buffer.rewards).to(device)\n",
    "            for t in reversed(range(num_steps)):\n",
    "                if t == num_steps - 1:\n",
    "                    nextnonterminal = 1.0 - next_done\n",
    "                    next_return = next_value\n",
    "                else:\n",
    "                    nextnonterminal = 1.0 - buffer.dones[t + 1]\n",
    "                    next_return = returns[t + 1]\n",
    "                returns[t] = buffer.rewards[t] + gamma * nextnonterminal * next_return\n",
    "            advantages = returns - buffer.values\n",
    "\n",
    "    # flatten the batch\n",
    "    b_obs = buffer.obs.reshape((-1,) + env.observation_space.shape)\n",
    "    b_logprobs = buffer.logprobs.reshape(-1)\n",
    "    b_actions = buffer.actions.reshape((-1,) + env.action_space.shape)\n",
    "    b_advantages = advantages.reshape(-1)\n",
    "    b_returns = returns.reshape(-1)\n",
    "    b_values = buffer.values.reshape(-1)\n",
    "\n",
    "    # Optimizing the policy and value network\n",
    "    b_inds = np.arange(batch_size)\n",
    "    clipfracs = []\n",
    "    for epoch in range(update_epochs):\n",
    "        np.random.shuffle(b_inds)\n",
    "        for start in range(0, batch_size, minibatch_size):\n",
    "            end = start + minibatch_size\n",
    "            mb_inds = b_inds[start:end]\n",
    "\n",
    "            _, newlogprob, entropy, newvalue = agent.get_action_and_value(b_obs[mb_inds], b_actions[mb_inds])\n",
    "            logratio = newlogprob - b_logprobs[mb_inds]\n",
    "            ratio = logratio.exp()\n",
    "\n",
    "            with torch.no_grad():\n",
    "                # calculate approx_kl http://joschu.net/blog/kl-approx.html\n",
    "                old_approx_kl = (-logratio).mean()\n",
    "                approx_kl = ((ratio - 1) - logratio).mean()\n",
    "                clipfracs += [((ratio - 1.0).abs() > clip_coef).float().mean().item()]\n",
    "\n",
    "            mb_advantages = b_advantages[mb_inds]\n",
    "            if norm_adv:\n",
    "                mb_advantages = (mb_advantages - mb_advantages.mean()) / (mb_advantages.std() + 1e-8)\n",
    "\n",
    "            # Policy loss\n",
    "            pg_loss1 = -mb_advantages * ratio\n",
    "            pg_loss2 = -mb_advantages * torch.clamp(ratio, 1 - clip_coef, 1 + clip_coef)\n",
    "            pg_loss = torch.max(pg_loss1, pg_loss2).mean()\n",
    "\n",
    "            # Value loss\n",
    "            newvalue = newvalue.view(-1)\n",
    "            if clip_vloss:\n",
    "                v_loss_unclipped = (newvalue - b_returns[mb_inds]) ** 2\n",
    "                v_clipped = b_values[mb_inds] + torch.clamp(\n",
    "                    newvalue - b_values[mb_inds],\n",
    "                    -clip_coef,\n",
    "                    clip_coef,\n",
    "                )\n",
    "                v_loss_clipped = (v_clipped - b_returns[mb_inds]) ** 2\n",
    "                v_loss_max = torch.max(v_loss_unclipped, v_loss_clipped)\n",
    "                v_loss = 0.5 * v_loss_max.mean()\n",
    "            else:\n",
    "                v_loss = 0.5 * ((newvalue - b_returns[mb_inds]) ** 2).mean()\n",
    "\n",
    "            entropy_loss = entropy.mean()\n",
    "            loss = pg_loss - ent_coef * entropy_loss + v_loss * vf_coef\n",
    "\n",
    "            optimizer.zero_grad()\n",
    "            loss.backward()\n",
    "            nn.utils.clip_grad_norm_(agent.parameters(), max_grad_norm)\n",
    "            optimizer.step()\n",
    "\n",
    "        if target_kl is not None:\n",
    "            if approx_kl > target_kl:\n",
    "                break\n",
    "\n",
    "    y_pred, y_true = b_values.cpu().numpy(), b_returns.cpu().numpy()\n",
    "    var_y = np.var(y_true)\n",
    "    explained_var = np.nan if var_y == 0 else 1 - np.var(y_true - y_pred) / var_y"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Training"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Helper Function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "def initialize_agent(env, device, learning_rate):\n",
    "    agent = Agent(env).to(device)\n",
    "    optimizer = optim.Adam(agent.parameters(), lr=learning_rate, eps=1e-5)\n",
    "    return agent, optimizer\n",
    "\n",
    "def get_action_and_update_buffer(agent, obs, buffer, step):\n",
    "    with torch.no_grad():\n",
    "        action, logprob, _, value = agent.get_action_and_value(obs)\n",
    "        buffer.values[step] = value.flatten()\n",
    "    buffer.actions[step] = action\n",
    "    buffer.logprobs[step] = logprob\n",
    "    return action\n",
    "\n",
    "def reset_environment(env, device):\n",
    "    next_obs, infos = env.reset()\n",
    "    next_obs = {k: torch.Tensor(v).unsqueeze(0).to(device) for k, v in next_obs.items()}\n",
    "    return next_obs, infos"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Hyperparameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "exp_name = \"Sender box\"\n",
    "\n",
    "learning_rate = 1e-5\n",
    "seed = 1\n",
    "# total_timesteps = 20000000\n",
    "total_timesteps = 1000000\n",
    "torch_deterministic = True\n",
    "cuda = False\n",
    "mps = False\n",
    "track = False\n",
    "wandb_project_name = \"ppo-implementation-details\"\n",
    "wandb_entity = None\n",
    "capture_video = False\n",
    "\n",
    "# Algorithm-specific arguments\n",
    "num_envs = 1\n",
    "num_steps = 2048\n",
    "anneal_lr = True\n",
    "gae = True\n",
    "gamma = 0.99\n",
    "gae_lambda = 0.95\n",
    "num_minibatches = 128\n",
    "update_epochs = 10\n",
    "norm_adv = True\n",
    "clip_coef = 0.2\n",
    "clip_vloss = True\n",
    "ent_coef = 0.0\n",
    "vf_coef = 0.5\n",
    "max_grad_norm = 0.5\n",
    "target_kl = None\n",
    "store_freq = 20\n",
    "\n",
    "# Calculate derived variables\n",
    "batch_size = int(num_envs * num_steps)\n",
    "minibatch_size = int(batch_size // num_minibatches)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0% (0 of 488) |                        | Elapsed Time: 0:00:00 ETA:  --:--:--\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 69 Average Reward: -21.402849167364266\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0% (1 of 488) |                        | Elapsed Time: 0:00:29 ETA:   4:00:47\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 73 Average Reward: -10.698389484964654\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0% (2 of 488) |                        | Elapsed Time: 0:00:55 ETA:   3:29:46\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 75 Average Reward: -10.883122260329063\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0% (3 of 488) |                        | Elapsed Time: 0:01:20 ETA:   3:25:26\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 76 Average Reward: -10.502174614277099\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0% (4 of 488) |                        | Elapsed Time: 0:01:46 ETA:   3:26:05\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 77 Average Reward: -10.577646543409038\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  1% (5 of 488) |                        | Elapsed Time: 0:02:11 ETA:   3:24:25\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 78 Average Reward: -10.761168199055467\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  1% (6 of 488) |                        | Elapsed Time: 0:02:37 ETA:   3:25:01\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 78 Average Reward: -10.47959500819738\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  1% (7 of 488) |                        | Elapsed Time: 0:03:02 ETA:   3:21:42\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 78 Average Reward: -11.351008226177058\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  1% (8 of 488) |                        | Elapsed Time: 0:03:28 ETA:   3:23:29\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -10.914489566813891\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  1% (9 of 488) |                        | Elapsed Time: 0:03:53 ETA:   3:20:29\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -10.579219104138666\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  2% (10 of 488) |                       | Elapsed Time: 0:04:18 ETA:   3:22:49\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -10.334683100041154\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  2% (11 of 488) |                       | Elapsed Time: 0:04:44 ETA:   3:22:12\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -11.152880037409814\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  2% (12 of 488) |                       | Elapsed Time: 0:05:09 ETA:   3:22:03\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -10.142538219656108\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  2% (13 of 488) |                       | Elapsed Time: 0:05:34 ETA:   3:20:27\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -9.959650946824503\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  2% (14 of 488) |                       | Elapsed Time: 0:06:00 ETA:   3:20:32\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -10.474531780867462\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  3% (15 of 488) |                       | Elapsed Time: 0:06:25 ETA:   3:19:28\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -9.933603193617023\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  3% (16 of 488) |                       | Elapsed Time: 0:06:51 ETA:   3:22:37\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -10.890004312295018\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  3% (17 of 488) |                       | Elapsed Time: 0:07:16 ETA:   3:19:14\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -10.47089198918784\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  3% (18 of 488) |                       | Elapsed Time: 0:07:42 ETA:   3:20:54\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -10.350919172348934\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  3% (19 of 488) |                       | Elapsed Time: 0:08:07 ETA:   3:17:16\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -9.879150254610835\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  4% (20 of 488) |                       | Elapsed Time: 0:08:33 ETA:   3:19:23\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -10.1137585897501\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  4% (21 of 488) |                       | Elapsed Time: 0:08:58 ETA:   3:17:33\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -10.53877992519145\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  4% (22 of 488) |#                      | Elapsed Time: 0:09:24 ETA:   3:18:21\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -11.326275804453918\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  4% (23 of 488) |#                      | Elapsed Time: 0:09:49 ETA:   3:15:59\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -11.198265384936299\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  4% (24 of 488) |#                      | Elapsed Time: 0:10:14 ETA:   3:16:52\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -10.161693275175722\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  5% (25 of 488) |#                      | Elapsed Time: 0:10:40 ETA:   3:20:13\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -11.728046947841676\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  5% (26 of 488) |#                      | Elapsed Time: 0:11:06 ETA:   3:15:40\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -11.39858994140274\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  5% (27 of 488) |#                      | Elapsed Time: 0:11:31 ETA:   3:12:32\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 80 Average Reward: -9.495085222786724\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  5% (28 of 488) |#                      | Elapsed Time: 0:11:56 ETA:   3:14:41\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 80 Average Reward: -11.139228523350516\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  5% (29 of 488) |#                      | Elapsed Time: 0:12:22 ETA:   3:15:15\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 80 Average Reward: -21.32112573614406\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  6% (30 of 488) |#                      | Elapsed Time: 0:12:47 ETA:   3:10:22\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 80 Average Reward: -27.85507888098484\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  6% (31 of 488) |#                      | Elapsed Time: 0:13:12 ETA:   3:12:42\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 80 Average Reward: -25.41796093525972\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  6% (32 of 488) |#                      | Elapsed Time: 0:13:37 ETA:   3:13:19\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 80 Average Reward: -23.197978561162813\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  6% (33 of 488) |#                      | Elapsed Time: 0:14:03 ETA:   3:12:10\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 80 Average Reward: -24.457453312100526\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  6% (34 of 488) |#                      | Elapsed Time: 0:14:28 ETA:   3:10:58\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 80 Average Reward: -24.160699151072258\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  7% (35 of 488) |#                      | Elapsed Time: 0:14:53 ETA:   3:12:54\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 80 Average Reward: -22.314837756779063\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  7% (36 of 488) |#                      | Elapsed Time: 0:15:19 ETA:   3:11:48\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 80 Average Reward: -23.630039977836518\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  7% (37 of 488) |#                      | Elapsed Time: 0:15:44 ETA:   3:11:25\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 80 Average Reward: -25.046174535451186\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  7% (38 of 488) |#                      | Elapsed Time: 0:16:10 ETA:   3:10:00\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 80 Average Reward: -22.072635204650467\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  7% (39 of 488) |#                      | Elapsed Time: 0:16:35 ETA:   3:11:19\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 80 Average Reward: -18.92851208891713\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  8% (40 of 488) |#                      | Elapsed Time: 0:17:03 ETA:   3:29:11\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -18.0437976079799\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  8% (41 of 488) |#                      | Elapsed Time: 0:17:30 ETA:   3:19:39\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -19.734297514233816\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  8% (42 of 488) |#                      | Elapsed Time: 0:17:57 ETA:   3:19:09\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -21.297956424784214\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  8% (43 of 488) |##                     | Elapsed Time: 0:18:23 ETA:   3:15:03\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -18.45910426547281\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  9% (44 of 488) |##                     | Elapsed Time: 0:18:49 ETA:   3:14:44\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -16.92371939366536\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  9% (45 of 488) |##                     | Elapsed Time: 0:19:16 ETA:   3:12:06\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -10.957537925434353\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  9% (46 of 488) |##                     | Elapsed Time: 0:19:42 ETA:   3:13:42\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -16.25550307257537\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  9% (47 of 488) |##                     | Elapsed Time: 0:20:08 ETA:   3:11:45\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -22.68007882932438\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  9% (48 of 488) |##                     | Elapsed Time: 0:20:34 ETA:   3:12:15\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -21.045077920331988\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 10% (49 of 488) |##                     | Elapsed Time: 0:21:00 ETA:   3:11:52\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -20.8593066927096\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 10% (50 of 488) |##                     | Elapsed Time: 0:21:27 ETA:   3:11:40\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -20.47932796215015\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 10% (51 of 488) |##                     | Elapsed Time: 0:21:53 ETA:   3:10:14\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -17.315842884378135\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 10% (52 of 488) |##                     | Elapsed Time: 0:22:19 ETA:   3:10:00\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -17.12718015352596\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 10% (53 of 488) |##                     | Elapsed Time: 0:22:45 ETA:   3:10:03\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -12.218091111651647\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 11% (54 of 488) |##                     | Elapsed Time: 0:23:11 ETA:   3:09:01\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -16.5737643137921\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 11% (55 of 488) |##                     | Elapsed Time: 0:23:38 ETA:   3:09:48\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -24.111954442889573\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 11% (56 of 488) |##                     | Elapsed Time: 0:24:04 ETA:   3:07:18\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -11.879696973053262\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 11% (57 of 488) |##                     | Elapsed Time: 0:24:30 ETA:   3:08:37\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -20.21109648246531\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 11% (58 of 488) |##                     | Elapsed Time: 0:24:56 ETA:   3:07:26\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -12.172337983757663\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 12% (59 of 488) |##                     | Elapsed Time: 0:25:22 ETA:   3:06:21\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -10.747591781552387\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 12% (60 of 488) |##                     | Elapsed Time: 0:25:48 ETA:   3:07:41\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -15.84645544570436\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 12% (61 of 488) |##                     | Elapsed Time: 0:26:15 ETA:   3:06:31\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -17.924605326708875\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 12% (62 of 488) |##                     | Elapsed Time: 0:26:41 ETA:   3:05:17\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -11.906230762121481\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 12% (63 of 488) |##                     | Elapsed Time: 0:27:07 ETA:   3:05:42\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -7.914220241634613\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 13% (64 of 488) |###                    | Elapsed Time: 0:27:33 ETA:   3:05:21\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -8.298650122851008\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 13% (65 of 488) |###                    | Elapsed Time: 0:27:59 ETA:   3:06:19\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -14.445058988761382\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 13% (66 of 488) |###                    | Elapsed Time: 0:28:25 ETA:   3:01:58\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -17.54290429595998\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 13% (67 of 488) |###                    | Elapsed Time: 0:28:52 ETA:   3:05:09\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -13.917034770893993\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 13% (68 of 488) |###                    | Elapsed Time: 0:29:18 ETA:   3:03:22\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -13.038045256449742\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 14% (69 of 488) |###                    | Elapsed Time: 0:29:44 ETA:   3:03:22\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -12.96237703072425\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 14% (70 of 488) |###                    | Elapsed Time: 0:30:10 ETA:   3:02:25\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -9.496543893538096\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 14% (71 of 488) |###                    | Elapsed Time: 0:30:37 ETA:   3:01:48\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -10.563489235111517\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 14% (72 of 488) |###                    | Elapsed Time: 0:31:03 ETA:   3:01:44\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -16.130911664065685\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 14% (73 of 488) |###                    | Elapsed Time: 0:31:29 ETA:   2:59:47\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -16.131473951257025\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 15% (74 of 488) |###                    | Elapsed Time: 0:31:55 ETA:   3:01:06\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -17.76894034446306\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 15% (75 of 488) |###                    | Elapsed Time: 0:32:21 ETA:   3:00:43\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -20.25255545610023\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 15% (76 of 488) |###                    | Elapsed Time: 0:32:47 ETA:   2:59:26\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -13.552829547877256\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 15% (77 of 488) |###                    | Elapsed Time: 0:33:13 ETA:   2:58:01\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -16.3398345639483\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 15% (78 of 488) |###                    | Elapsed Time: 0:33:40 ETA:   2:58:58\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -12.415059650214575\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 16% (79 of 488) |###                    | Elapsed Time: 0:34:06 ETA:   3:00:02\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -12.276517194972278\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 16% (80 of 488) |###                    | Elapsed Time: 0:34:32 ETA:   2:58:17\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -15.06842951983488\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 16% (81 of 488) |###                    | Elapsed Time: 0:34:59 ETA:   2:58:26\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -18.942076592720976\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 16% (82 of 488) |###                    | Elapsed Time: 0:35:25 ETA:   2:58:44\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 78 Average Reward: -19.200578370196606\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 17% (83 of 488) |###                    | Elapsed Time: 0:35:51 ETA:   2:57:41\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 78 Average Reward: -17.77053768985551\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 17% (84 of 488) |###                    | Elapsed Time: 0:36:17 ETA:   2:56:13\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 78 Average Reward: -14.120032235994035\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 17% (85 of 488) |####                   | Elapsed Time: 0:36:43 ETA:   2:54:15\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 78 Average Reward: -10.593796082269614\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 17% (86 of 488) |####                   | Elapsed Time: 0:37:09 ETA:   2:53:20\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 78 Average Reward: -13.519048129731205\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 17% (87 of 488) |####                   | Elapsed Time: 0:37:36 ETA:   2:56:06\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 78 Average Reward: -19.965427781778068\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 18% (88 of 488) |####                   | Elapsed Time: 0:38:02 ETA:   2:54:25\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 78 Average Reward: -12.250514698868898\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 18% (89 of 488) |####                   | Elapsed Time: 0:38:28 ETA:   2:54:02\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 78 Average Reward: -18.18392260685477\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 18% (90 of 488) |####                   | Elapsed Time: 0:38:54 ETA:   2:51:51\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -13.838666242395187\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 18% (91 of 488) |####                   | Elapsed Time: 0:39:18 ETA:   2:41:35\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -19.009275659762007\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 18% (92 of 488) |####                   | Elapsed Time: 0:39:43 ETA:   2:41:13\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -19.400083536001684\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 19% (93 of 488) |####                   | Elapsed Time: 0:40:07 ETA:   2:39:48\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -13.338895219498653\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 19% (94 of 488) |####                   | Elapsed Time: 0:40:31 ETA:   2:40:09\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -12.707403576226703\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 19% (95 of 488) |####                   | Elapsed Time: 0:40:56 ETA:   2:39:24\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -11.723402400673304\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 19% (96 of 488) |####                   | Elapsed Time: 0:41:20 ETA:   2:40:14\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -18.47949827980288\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 19% (97 of 488) |####                   | Elapsed Time: 0:41:45 ETA:   2:40:09\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -16.050229291256105\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 20% (98 of 488) |####                   | Elapsed Time: 0:42:09 ETA:   2:38:14\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -14.149422170837061\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 20% (99 of 488) |####                   | Elapsed Time: 0:42:33 ETA:   2:37:40\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -17.345473011001175\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 20% (100 of 488) |####                  | Elapsed Time: 0:42:58 ETA:   2:37:40\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -16.111156708621436\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 20% (101 of 488) |####                  | Elapsed Time: 0:43:22 ETA:   2:37:16\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -14.300665541014206\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 20% (102 of 488) |####                  | Elapsed Time: 0:43:47 ETA:   2:36:37\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -10.544998387838255\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 21% (103 of 488) |####                  | Elapsed Time: 0:44:11 ETA:   2:35:53\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -12.077160065976056\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 21% (104 of 488) |####                  | Elapsed Time: 0:44:35 ETA:   2:37:25\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -11.831289197374522\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 21% (105 of 488) |####                  | Elapsed Time: 0:45:00 ETA:   2:35:49\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -16.48306529073408\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 21% (106 of 488) |####                  | Elapsed Time: 0:45:24 ETA:   2:36:14\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -13.648004843308968\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 21% (107 of 488) |####                  | Elapsed Time: 0:45:49 ETA:   2:35:06\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -15.25362536176685\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 22% (108 of 488) |####                  | Elapsed Time: 0:46:13 ETA:   2:34:08\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -12.486678189079132\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 22% (109 of 488) |####                  | Elapsed Time: 0:46:38 ETA:   2:33:53\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -10.915727306083264\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 22% (110 of 488) |####                  | Elapsed Time: 0:47:02 ETA:   2:32:40\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -13.809790217490114\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 22% (111 of 488) |#####                 | Elapsed Time: 0:47:26 ETA:   2:33:07\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -10.913440002943432\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 22% (112 of 488) |#####                 | Elapsed Time: 0:47:51 ETA:   2:34:10\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -14.41434308817573\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 23% (113 of 488) |#####                 | Elapsed Time: 0:48:15 ETA:   2:32:20\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -13.1855469885212\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 23% (114 of 488) |#####                 | Elapsed Time: 0:48:40 ETA:   2:32:51\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -12.251851363879533\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 23% (115 of 488) |#####                 | Elapsed Time: 0:49:04 ETA:   2:31:28\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 80 Average Reward: -13.166505239690132\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 23% (116 of 488) |#####                 | Elapsed Time: 0:49:28 ETA:   2:30:54\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 80 Average Reward: -11.462833226278315\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 23% (117 of 488) |#####                 | Elapsed Time: 0:49:53 ETA:   2:31:22\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 80 Average Reward: -14.229632621381851\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 24% (118 of 488) |#####                 | Elapsed Time: 0:50:17 ETA:   2:30:24\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 80 Average Reward: -11.788167679611949\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 24% (119 of 488) |#####                 | Elapsed Time: 0:50:42 ETA:   2:29:39\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 80 Average Reward: -14.270398063958277\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 24% (120 of 488) |#####                 | Elapsed Time: 0:51:06 ETA:   2:29:50\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 80 Average Reward: -10.519570593894699\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 24% (121 of 488) |#####                 | Elapsed Time: 0:51:31 ETA:   2:30:10\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 80 Average Reward: -11.821842866975354\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 25% (122 of 488) |#####                 | Elapsed Time: 0:51:55 ETA:   2:28:45\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 80 Average Reward: -8.088668302021746\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 25% (123 of 488) |#####                 | Elapsed Time: 0:52:19 ETA:   2:28:26\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 80 Average Reward: -7.574297121725365\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 25% (124 of 488) |#####                 | Elapsed Time: 0:52:44 ETA:   2:28:36\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 80 Average Reward: -6.931462875472998\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 25% (125 of 488) |#####                 | Elapsed Time: 0:53:08 ETA:   2:28:23\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 80 Average Reward: -11.481559409516066\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 25% (126 of 488) |#####                 | Elapsed Time: 0:53:33 ETA:   2:28:09\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 80 Average Reward: -14.766126994054233\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 26% (127 of 488) |#####                 | Elapsed Time: 0:53:57 ETA:   2:27:48\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 80 Average Reward: -11.838753629840852\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 26% (128 of 488) |#####                 | Elapsed Time: 0:54:22 ETA:   2:26:30\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 80 Average Reward: -13.320312780499567\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 26% (129 of 488) |#####                 | Elapsed Time: 0:54:46 ETA:   2:27:13\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 80 Average Reward: -13.931359595403\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 26% (130 of 488) |#####                 | Elapsed Time: 0:55:11 ETA:   2:26:12\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 80 Average Reward: -10.843215289826981\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 26% (131 of 488) |#####                 | Elapsed Time: 0:55:35 ETA:   2:25:15\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 80 Average Reward: -13.060363045647362\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 27% (132 of 488) |#####                 | Elapsed Time: 0:56:00 ETA:   2:25:20\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 80 Average Reward: -11.545055176758504\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 27% (133 of 488) |#####                 | Elapsed Time: 0:56:24 ETA:   2:25:17\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 80 Average Reward: -11.679049286565094\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 27% (134 of 488) |######                | Elapsed Time: 0:56:49 ETA:   2:25:20\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 80 Average Reward: -13.560130396312482\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 27% (135 of 488) |######                | Elapsed Time: 0:57:14 ETA:   2:24:17\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 80 Average Reward: -13.553482336107809\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 27% (136 of 488) |######                | Elapsed Time: 0:57:38 ETA:   2:23:26\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 80 Average Reward: -11.971224320644101\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 28% (137 of 488) |######                | Elapsed Time: 0:58:03 ETA:   2:23:36\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 80 Average Reward: -10.730175822323865\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 28% (138 of 488) |######                | Elapsed Time: 0:58:27 ETA:   2:22:10\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 80 Average Reward: -11.711911181882646\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 28% (139 of 488) |######                | Elapsed Time: 0:58:51 ETA:   2:22:26\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 80 Average Reward: -11.286394310113616\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 28% (140 of 488) |######                | Elapsed Time: 0:59:16 ETA:   2:21:47\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 80 Average Reward: -10.182255766470215\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 28% (141 of 488) |######                | Elapsed Time: 0:59:40 ETA:   2:21:22\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 80 Average Reward: -14.021840017882742\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 29% (142 of 488) |######                | Elapsed Time: 1:00:05 ETA:   2:21:35\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 80 Average Reward: -11.668626382989386\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 29% (143 of 488) |######                | Elapsed Time: 1:00:29 ETA:   2:20:16\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 80 Average Reward: -11.140149264527825\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 29% (144 of 488) |######                | Elapsed Time: 1:00:54 ETA:   2:20:10\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 80 Average Reward: -11.764561337235122\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 29% (145 of 488) |######                | Elapsed Time: 1:01:18 ETA:   2:19:52\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 80 Average Reward: -12.040425098719354\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 29% (146 of 488) |######                | Elapsed Time: 1:01:43 ETA:   2:19:23\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 80 Average Reward: -10.414680679498142\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 30% (147 of 488) |######                | Elapsed Time: 1:02:07 ETA:   2:19:30\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 80 Average Reward: -12.82476609629429\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 30% (148 of 488) |######                | Elapsed Time: 1:02:32 ETA:   2:19:23\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 80 Average Reward: -10.53343596761457\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 30% (149 of 488) |######                | Elapsed Time: 1:02:56 ETA:   2:18:06\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 80 Average Reward: -11.001751404799766\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 30% (150 of 488) |######                | Elapsed Time: 1:03:21 ETA:   2:17:18\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 80 Average Reward: -14.143316823170153\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 30% (151 of 488) |######                | Elapsed Time: 1:03:45 ETA:   2:17:52\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 80 Average Reward: -11.14090830412924\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 31% (152 of 488) |######                | Elapsed Time: 1:04:10 ETA:   2:16:46\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 80 Average Reward: -12.044460811592899\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 31% (153 of 488) |######                | Elapsed Time: 1:04:34 ETA:   2:16:37\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 80 Average Reward: -11.23955074993042\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 31% (154 of 488) |######                | Elapsed Time: 1:04:59 ETA:   2:16:14\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 80 Average Reward: -11.742867331843183\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 31% (155 of 488) |######                | Elapsed Time: 1:05:23 ETA:   2:15:56\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 80 Average Reward: -12.149728723970519\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 31% (156 of 488) |#######               | Elapsed Time: 1:05:48 ETA:   2:15:24\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 80 Average Reward: -12.17748737696777\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 32% (157 of 488) |#######               | Elapsed Time: 1:06:12 ETA:   2:14:55\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 80 Average Reward: -11.291318442716294\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 32% (158 of 488) |#######               | Elapsed Time: 1:06:36 ETA:   2:13:14\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 80 Average Reward: -11.681043114860088\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 32% (159 of 488) |#######               | Elapsed Time: 1:07:00 ETA:   2:10:59\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 81 Average Reward: -12.485161885551582\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 32% (160 of 488) |#######               | Elapsed Time: 1:07:24 ETA:   2:12:17\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 81 Average Reward: -10.778550754121124\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 32% (161 of 488) |#######               | Elapsed Time: 1:07:49 ETA:   2:12:41\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 81 Average Reward: -13.28788652306643\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 33% (162 of 488) |#######               | Elapsed Time: 1:08:13 ETA:   2:12:01\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 81 Average Reward: -12.203469722499248\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 33% (163 of 488) |#######               | Elapsed Time: 1:08:37 ETA:   2:11:48\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 81 Average Reward: -10.32501861370644\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 33% (164 of 488) |#######               | Elapsed Time: 1:09:02 ETA:   2:10:51\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 81 Average Reward: -10.298205118725498\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 33% (165 of 488) |#######               | Elapsed Time: 1:09:26 ETA:   2:11:23\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 81 Average Reward: -10.63106882996433\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 34% (166 of 488) |#######               | Elapsed Time: 1:09:50 ETA:   2:09:05\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 81 Average Reward: -10.277707695162135\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 34% (167 of 488) |#######               | Elapsed Time: 1:10:14 ETA:   2:08:15\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 81 Average Reward: -10.071223619125762\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 34% (168 of 488) |#######               | Elapsed Time: 1:10:38 ETA:   2:09:48\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 81 Average Reward: -8.736326118936997\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 34% (169 of 488) |#######               | Elapsed Time: 1:11:03 ETA:   2:08:51\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 81 Average Reward: -12.614301409638545\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 34% (170 of 488) |#######               | Elapsed Time: 1:11:27 ETA:   2:08:58\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 81 Average Reward: -11.874014135200976\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 35% (171 of 488) |#######               | Elapsed Time: 1:11:51 ETA:   2:08:32\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 81 Average Reward: -9.62166523773738\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 35% (172 of 488) |#######               | Elapsed Time: 1:12:15 ETA:   2:07:27\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 81 Average Reward: -13.07476538761986\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 35% (173 of 488) |#######               | Elapsed Time: 1:12:40 ETA:   2:07:24\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 81 Average Reward: -11.290757053053184\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 35% (174 of 488) |#######               | Elapsed Time: 1:13:04 ETA:   2:06:43\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 81 Average Reward: -9.96610599204149\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 35% (175 of 488) |#######               | Elapsed Time: 1:13:28 ETA:   2:05:45\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 81 Average Reward: -10.8701041481098\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 36% (176 of 488) |#######               | Elapsed Time: 1:13:52 ETA:   2:06:07\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 81 Average Reward: -10.609369094425432\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 36% (177 of 488) |#######               | Elapsed Time: 1:14:16 ETA:   2:05:46\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 81 Average Reward: -9.419802313822132\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 36% (178 of 488) |########              | Elapsed Time: 1:14:41 ETA:   2:04:24\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 81 Average Reward: -9.973305791482678\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 36% (179 of 488) |########              | Elapsed Time: 1:15:05 ETA:   2:04:31\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 81 Average Reward: -10.72075348556624\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 36% (180 of 488) |########              | Elapsed Time: 1:15:29 ETA:   2:04:01\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 81 Average Reward: -11.650185564401951\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 37% (181 of 488) |########              | Elapsed Time: 1:15:53 ETA:   2:03:34\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 81 Average Reward: -11.867389796101115\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 37% (182 of 488) |########              | Elapsed Time: 1:16:17 ETA:   2:03:28\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 81 Average Reward: -13.972413224699803\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 37% (183 of 488) |########              | Elapsed Time: 1:16:42 ETA:   2:03:09\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 81 Average Reward: -10.890803913443461\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 37% (184 of 488) |########              | Elapsed Time: 1:17:06 ETA:   2:02:38\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 81 Average Reward: -11.585788561503911\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 37% (185 of 488) |########              | Elapsed Time: 1:17:30 ETA:   2:01:42\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 81 Average Reward: -9.762618971411282\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 38% (186 of 488) |########              | Elapsed Time: 1:17:54 ETA:   2:01:15\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 81 Average Reward: -9.963144634197702\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 38% (187 of 488) |########              | Elapsed Time: 1:18:18 ETA:   2:01:30\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 81 Average Reward: -9.288329112183678\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 38% (188 of 488) |########              | Elapsed Time: 1:18:42 ETA:   2:00:54\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 81 Average Reward: -9.098629075404183\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 38% (189 of 488) |########              | Elapsed Time: 1:19:06 ETA:   2:00:17\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 81 Average Reward: -10.056864248684782\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 38% (190 of 488) |########              | Elapsed Time: 1:19:30 ETA:   1:58:41\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 81 Average Reward: -9.562667321192562\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 39% (191 of 488) |########              | Elapsed Time: 1:19:54 ETA:   1:57:44\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 81 Average Reward: -9.751950338817517\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 39% (192 of 488) |########              | Elapsed Time: 1:20:18 ETA:   1:57:53\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 81 Average Reward: -10.323539422087423\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 39% (193 of 488) |########              | Elapsed Time: 1:20:42 ETA:   1:58:46\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 81 Average Reward: -9.738266084558777\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 39% (194 of 488) |########              | Elapsed Time: 1:21:06 ETA:   1:58:24\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 81 Average Reward: -9.752195450503027\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 39% (195 of 488) |########              | Elapsed Time: 1:21:31 ETA:   1:59:04\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 81 Average Reward: -13.775038632857731\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 40% (196 of 488) |########              | Elapsed Time: 1:21:55 ETA:   1:57:15\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 81 Average Reward: -10.534996015364985\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 40% (197 of 488) |########              | Elapsed Time: 1:22:19 ETA:   1:58:20\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 81 Average Reward: -6.033510279846951\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 40% (198 of 488) |########              | Elapsed Time: 1:22:46 ETA:   2:11:34\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 81 Average Reward: -8.489567907658122\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 40% (199 of 488) |########              | Elapsed Time: 1:23:12 ETA:   2:03:09\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 81 Average Reward: -9.012704255747913\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 40% (200 of 488) |#########             | Elapsed Time: 1:23:37 ETA:   2:00:13\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 81 Average Reward: -11.453843216794494\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 41% (201 of 488) |#########             | Elapsed Time: 1:24:02 ETA:   2:00:10\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 81 Average Reward: -9.419499265554467\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 41% (202 of 488) |#########             | Elapsed Time: 1:24:27 ETA:   1:59:33\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 81 Average Reward: -10.03494378520773\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 41% (203 of 488) |#########             | Elapsed Time: 1:24:52 ETA:   1:58:46\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 81 Average Reward: -9.057791612767003\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 41% (204 of 488) |#########             | Elapsed Time: 1:25:17 ETA:   1:59:08\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 81 Average Reward: -8.911093314768436\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 42% (205 of 488) |#########             | Elapsed Time: 1:25:42 ETA:   1:57:58\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 81 Average Reward: -11.503381203067905\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 42% (206 of 488) |#########             | Elapsed Time: 1:26:08 ETA:   1:57:49\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 81 Average Reward: -10.166716648102922\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 42% (207 of 488) |#########             | Elapsed Time: 1:26:33 ETA:   1:57:50\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 81 Average Reward: -10.47208602639438\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 42% (208 of 488) |#########             | Elapsed Time: 1:26:58 ETA:   1:57:10\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 81 Average Reward: -8.94687427010946\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 42% (209 of 488) |#########             | Elapsed Time: 1:27:23 ETA:   1:56:37\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 81 Average Reward: -8.134755207520241\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 43% (210 of 488) |#########             | Elapsed Time: 1:27:48 ETA:   1:55:35\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 81 Average Reward: -8.653692382343694\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 43% (211 of 488) |#########             | Elapsed Time: 1:28:13 ETA:   1:55:14\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 81 Average Reward: -9.125320196282985\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 43% (212 of 488) |#########             | Elapsed Time: 1:28:38 ETA:   1:55:23\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 81 Average Reward: -12.300560876842317\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 43% (213 of 488) |#########             | Elapsed Time: 1:29:03 ETA:   1:55:35\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 81 Average Reward: -8.035500310801293\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 43% (214 of 488) |#########             | Elapsed Time: 1:29:28 ETA:   1:53:47\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 81 Average Reward: -11.638617203012847\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 44% (215 of 488) |#########             | Elapsed Time: 1:29:53 ETA:   1:54:07\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 81 Average Reward: -9.790545315858031\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 44% (216 of 488) |#########             | Elapsed Time: 1:30:18 ETA:   1:53:42\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 81 Average Reward: -9.833016736159315\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 44% (217 of 488) |#########             | Elapsed Time: 1:30:43 ETA:   1:52:48\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 81 Average Reward: -9.989488186756262\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 44% (218 of 488) |#########             | Elapsed Time: 1:31:08 ETA:   1:52:48\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 81 Average Reward: -9.032112364542554\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 44% (219 of 488) |#########             | Elapsed Time: 1:31:33 ETA:   1:52:32\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 81 Average Reward: -9.702846264305307\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 45% (220 of 488) |#########             | Elapsed Time: 1:31:59 ETA:   1:52:28\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 81 Average Reward: -9.134241021590432\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 45% (221 of 488) |#########             | Elapsed Time: 1:32:24 ETA:   1:51:43\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 81 Average Reward: -10.013735327389853\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 45% (222 of 488) |##########            | Elapsed Time: 1:32:49 ETA:   1:50:52\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 81 Average Reward: -9.6247566289983\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 45% (223 of 488) |##########            | Elapsed Time: 1:33:14 ETA:   1:50:12\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 81 Average Reward: -8.452665869473027\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 45% (224 of 488) |##########            | Elapsed Time: 1:33:39 ETA:   1:50:26\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 81 Average Reward: -10.619268702969244\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 46% (225 of 488) |##########            | Elapsed Time: 1:34:04 ETA:   1:49:09\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 81 Average Reward: -11.449799845141474\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 46% (226 of 488) |##########            | Elapsed Time: 1:34:29 ETA:   1:49:12\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 81 Average Reward: -10.662326953464225\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 46% (227 of 488) |##########            | Elapsed Time: 1:34:54 ETA:   1:48:51\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 81 Average Reward: -7.241396975808244\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 46% (228 of 488) |##########            | Elapsed Time: 1:35:19 ETA:   1:47:50\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 81 Average Reward: -8.296005568653303\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 46% (229 of 488) |##########            | Elapsed Time: 1:35:43 ETA:   1:47:24\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 81 Average Reward: -9.22419839555304\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 47% (230 of 488) |##########            | Elapsed Time: 1:36:08 ETA:   1:45:59\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 81 Average Reward: -6.501785072780444\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 47% (231 of 488) |##########            | Elapsed Time: 1:36:33 ETA:   1:47:04\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 81 Average Reward: -10.939766079704864\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 47% (232 of 488) |##########            | Elapsed Time: 1:36:58 ETA:   1:45:51\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 81 Average Reward: -9.402205727914373\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 47% (233 of 488) |##########            | Elapsed Time: 1:37:23 ETA:   1:46:44\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 81 Average Reward: -8.309690938721175\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 47% (234 of 488) |##########            | Elapsed Time: 1:37:48 ETA:   1:45:54\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 81 Average Reward: -8.533090141870098\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 48% (235 of 488) |##########            | Elapsed Time: 1:38:13 ETA:   1:45:17\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 81 Average Reward: -6.835967261673679\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 48% (236 of 488) |##########            | Elapsed Time: 1:38:38 ETA:   1:44:26\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 81 Average Reward: -8.671394868924416\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 48% (237 of 488) |##########            | Elapsed Time: 1:39:03 ETA:   1:43:54\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 81 Average Reward: -8.52365426998128\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 48% (238 of 488) |##########            | Elapsed Time: 1:39:28 ETA:   1:44:14\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 81 Average Reward: -8.048263991376064\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 48% (239 of 488) |##########            | Elapsed Time: 1:39:53 ETA:   1:44:25\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 81 Average Reward: -9.270123274340882\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 49% (240 of 488) |##########            | Elapsed Time: 1:40:18 ETA:   1:43:20\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 81 Average Reward: -8.713325722109081\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 49% (241 of 488) |##########            | Elapsed Time: 1:40:43 ETA:   1:42:19\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 81 Average Reward: -8.179518549448355\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 49% (242 of 488) |##########            | Elapsed Time: 1:41:08 ETA:   1:42:37\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 81 Average Reward: -12.022622758213636\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 49% (243 of 488) |##########            | Elapsed Time: 1:41:33 ETA:   1:42:19\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 81 Average Reward: -8.546761801750689\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 50% (244 of 488) |###########           | Elapsed Time: 1:41:58 ETA:   1:41:42\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 81 Average Reward: -8.36661620057175\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 50% (245 of 488) |###########           | Elapsed Time: 1:42:23 ETA:   1:40:42\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 81 Average Reward: -7.82020362271282\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 50% (246 of 488) |###########           | Elapsed Time: 1:42:48 ETA:   1:41:07\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 81 Average Reward: -8.68031422870453\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 50% (247 of 488) |###########           | Elapsed Time: 1:43:13 ETA:   1:39:57\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 81 Average Reward: -8.637096091045072\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 50% (248 of 488) |###########           | Elapsed Time: 1:43:37 ETA:   1:38:02\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 81 Average Reward: -7.990747574071107\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 51% (249 of 488) |###########           | Elapsed Time: 1:44:02 ETA:   1:37:30\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 81 Average Reward: -8.259178479073888\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 51% (250 of 488) |###########           | Elapsed Time: 1:44:26 ETA:   1:37:38\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 81 Average Reward: -8.057382262064891\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 51% (251 of 488) |###########           | Elapsed Time: 1:44:51 ETA:   1:37:02\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 81 Average Reward: -7.382086180443579\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 51% (252 of 488) |###########           | Elapsed Time: 1:45:15 ETA:   1:36:22\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 81 Average Reward: -8.556954202394612\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 51% (253 of 488) |###########           | Elapsed Time: 1:45:40 ETA:   1:35:33\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 81 Average Reward: -8.61150660782471\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 52% (254 of 488) |###########           | Elapsed Time: 1:46:04 ETA:   1:35:18\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 81 Average Reward: -7.507759319521351\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 52% (255 of 488) |###########           | Elapsed Time: 1:46:29 ETA:   1:35:09\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 81 Average Reward: -8.011048569126194\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 52% (256 of 488) |###########           | Elapsed Time: 1:46:53 ETA:   1:34:37\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 81 Average Reward: -8.021385392930354\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 52% (257 of 488) |###########           | Elapsed Time: 1:47:18 ETA:   1:35:03\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 81 Average Reward: -9.272707036529496\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 52% (258 of 488) |###########           | Elapsed Time: 1:47:42 ETA:   1:33:44\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 81 Average Reward: -8.653583178089209\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 53% (259 of 488) |###########           | Elapsed Time: 1:48:07 ETA:   1:33:56\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 81 Average Reward: -9.653566796842277\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 53% (260 of 488) |###########           | Elapsed Time: 1:48:33 ETA:   1:40:12\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 81 Average Reward: -8.430332025715567\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 53% (261 of 488) |###########           | Elapsed Time: 1:49:00 ETA:   1:39:43\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 81 Average Reward: -8.78349057098966\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 53% (262 of 488) |###########           | Elapsed Time: 1:49:25 ETA:   1:35:40\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 81 Average Reward: -9.01493050549171\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 53% (263 of 488) |###########           | Elapsed Time: 1:49:50 ETA:   1:33:47\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 81 Average Reward: -7.613648852654522\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 54% (264 of 488) |###########           | Elapsed Time: 1:50:15 ETA:   1:33:21\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 81 Average Reward: -7.8162926686791785\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 54% (265 of 488) |###########           | Elapsed Time: 1:50:40 ETA:   1:32:48\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 81 Average Reward: -9.288035937303729\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 54% (266 of 488) |###########           | Elapsed Time: 1:51:05 ETA:   1:32:44\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 81 Average Reward: -6.7572939053007515\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 54% (267 of 488) |############          | Elapsed Time: 1:51:30 ETA:   1:32:17\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 81 Average Reward: -9.097997087768174\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 54% (268 of 488) |############          | Elapsed Time: 1:51:55 ETA:   1:32:08\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 81 Average Reward: -10.676870022648979\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 55% (269 of 488) |############          | Elapsed Time: 1:52:20 ETA:   1:30:54\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 81 Average Reward: -7.863493057221771\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 55% (270 of 488) |############          | Elapsed Time: 1:52:45 ETA:   1:30:42\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 81 Average Reward: -7.392702222662644\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 55% (271 of 488) |############          | Elapsed Time: 1:53:11 ETA:   1:32:22\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 81 Average Reward: -8.20863367251983\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 55% (272 of 488) |############          | Elapsed Time: 1:53:36 ETA:   1:31:23\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 81 Average Reward: -8.016690541791062\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 55% (273 of 488) |############          | Elapsed Time: 1:54:03 ETA:   1:37:39\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 81 Average Reward: -7.895475221123061\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 56% (274 of 488) |############          | Elapsed Time: 1:54:30 ETA:   1:35:33\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 81 Average Reward: -5.28727328687448\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 56% (275 of 488) |############          | Elapsed Time: 1:54:57 ETA:   1:35:19\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 81 Average Reward: -7.775653366469252\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 56% (276 of 488) |############          | Elapsed Time: 1:55:27 ETA:   1:47:21\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 81 Average Reward: -7.198665846527751\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 56% (277 of 488) |############          | Elapsed Time: 1:55:58 ETA:   1:49:15\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 81 Average Reward: -8.757828609937485\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 56% (278 of 488) |############          | Elapsed Time: 1:56:30 ETA:   1:50:10\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 81 Average Reward: -8.60339383898605\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 57% (279 of 488) |############          | Elapsed Time: 1:57:00 ETA:   1:45:40\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 81 Average Reward: -7.063925429929915\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 57% (280 of 488) |############          | Elapsed Time: 1:57:32 ETA:   1:48:34\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 81 Average Reward: -7.306997580565658\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 57% (281 of 488) |############          | Elapsed Time: 1:58:03 ETA:   1:49:08\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 81 Average Reward: -6.675431207067841\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 57% (282 of 488) |############          | Elapsed Time: 1:58:34 ETA:   1:47:22\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 81 Average Reward: -7.708521313646746\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 57% (283 of 488) |############          | Elapsed Time: 1:59:04 ETA:   1:41:10\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 81 Average Reward: -8.380777145392786\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 58% (284 of 488) |############          | Elapsed Time: 1:59:36 ETA:   1:50:05\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 80 Average Reward: -7.7142018921933015\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 58% (285 of 488) |############          | Elapsed Time: 2:00:08 ETA:   1:46:26\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 80 Average Reward: -7.751280900322351\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 58% (286 of 488) |############          | Elapsed Time: 2:00:39 ETA:   1:44:58\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 80 Average Reward: -6.7741915141415205\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 58% (287 of 488) |############          | Elapsed Time: 2:01:05 ETA:   1:28:29\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 80 Average Reward: -7.256862548614247\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 59% (288 of 488) |############          | Elapsed Time: 2:01:36 ETA:   1:41:37\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 80 Average Reward: -7.576839787450205\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 59% (289 of 488) |#############         | Elapsed Time: 2:02:06 ETA:   1:39:15\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 80 Average Reward: -7.25570108690498\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 59% (290 of 488) |#############         | Elapsed Time: 2:02:36 ETA:   1:40:15\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 80 Average Reward: -7.553187829870736\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 59% (291 of 488) |#############         | Elapsed Time: 2:03:07 ETA:   1:41:57\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 80 Average Reward: -7.239657309105354\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 59% (292 of 488) |#############         | Elapsed Time: 2:03:35 ETA:   1:29:36\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 80 Average Reward: -7.368170568555964\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 60% (293 of 488) |#############         | Elapsed Time: 2:04:02 ETA:   1:28:19\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 80 Average Reward: -6.784728551223794\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 60% (294 of 488) |#############         | Elapsed Time: 2:04:29 ETA:   1:28:21\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 80 Average Reward: -7.3273815062748975\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 60% (295 of 488) |#############         | Elapsed Time: 2:04:56 ETA:   1:25:54\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 80 Average Reward: -6.971732332070348\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 60% (296 of 488) |#############         | Elapsed Time: 2:05:22 ETA:   1:22:20\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 80 Average Reward: -7.604883751079195\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 60% (297 of 488) |#############         | Elapsed Time: 2:05:47 ETA:   1:20:38\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 80 Average Reward: -6.663658422864284\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 61% (298 of 488) |#############         | Elapsed Time: 2:06:13 ETA:   1:23:13\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 80 Average Reward: -7.012658515189138\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 61% (299 of 488) |#############         | Elapsed Time: 2:06:40 ETA:   1:25:04\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 80 Average Reward: -7.1847979425742325\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 61% (300 of 488) |#############         | Elapsed Time: 2:07:12 ETA:   1:37:56\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 80 Average Reward: -7.052911074811015\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 61% (301 of 488) |#############         | Elapsed Time: 2:07:39 ETA:   1:26:57\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 80 Average Reward: -7.376019006318532\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 61% (302 of 488) |#############         | Elapsed Time: 2:08:09 ETA:   1:31:45\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 80 Average Reward: -9.861319607200054\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 62% (303 of 488) |#############         | Elapsed Time: 2:08:37 ETA:   1:27:08\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 80 Average Reward: -7.584579122191912\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 62% (304 of 488) |#############         | Elapsed Time: 2:09:05 ETA:   1:24:28\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 80 Average Reward: -7.492098796092352\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 62% (305 of 488) |#############         | Elapsed Time: 2:09:31 ETA:   1:18:28\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 80 Average Reward: -7.509821275881747\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 62% (306 of 488) |#############         | Elapsed Time: 2:09:57 ETA:   1:20:45\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 80 Average Reward: -6.677667996021956\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 62% (307 of 488) |#############         | Elapsed Time: 2:10:24 ETA:   1:19:28\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 80 Average Reward: -7.611712288113586\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 63% (308 of 488) |#############         | Elapsed Time: 2:10:53 ETA:   1:27:06\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 80 Average Reward: -6.925904210553053\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 63% (309 of 488) |#############         | Elapsed Time: 2:11:22 ETA:   1:28:03\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 80 Average Reward: -7.064628300423746\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 63% (310 of 488) |#############         | Elapsed Time: 2:11:51 ETA:   1:24:44\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 80 Average Reward: -7.168985575866686\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 63% (311 of 488) |##############        | Elapsed Time: 2:12:20 ETA:   1:25:45\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 80 Average Reward: -6.536346101855568\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 63% (312 of 488) |##############        | Elapsed Time: 2:12:48 ETA:   1:22:17\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 80 Average Reward: -6.931799010673007\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 64% (313 of 488) |##############        | Elapsed Time: 2:13:18 ETA:   1:26:55\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 80 Average Reward: -8.02822015546001\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 64% (314 of 488) |##############        | Elapsed Time: 2:13:47 ETA:   1:25:48\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 80 Average Reward: -7.31806070634171\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 64% (315 of 488) |##############        | Elapsed Time: 2:14:15 ETA:   1:20:48\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 80 Average Reward: -6.673606839964101\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 64% (316 of 488) |##############        | Elapsed Time: 2:14:41 ETA:   1:13:11\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 80 Average Reward: -7.8906630676282905\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 64% (317 of 488) |##############        | Elapsed Time: 2:15:06 ETA:   1:12:55\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 80 Average Reward: -7.221977540407271\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 65% (318 of 488) |##############        | Elapsed Time: 2:15:32 ETA:   1:12:43\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 80 Average Reward: -7.194369028602907\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 65% (319 of 488) |##############        | Elapsed Time: 2:15:59 ETA:   1:16:43\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 80 Average Reward: -7.031292247308819\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 65% (320 of 488) |##############        | Elapsed Time: 2:16:30 ETA:   1:26:26\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 80 Average Reward: -6.686451821283068\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 65% (321 of 488) |##############        | Elapsed Time: 2:16:56 ETA:   1:12:12\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 80 Average Reward: -7.887788731676035\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 65% (322 of 488) |##############        | Elapsed Time: 2:17:21 ETA:   1:10:12\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 80 Average Reward: -6.22304534865312\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 66% (323 of 488) |##############        | Elapsed Time: 2:17:47 ETA:   1:10:22\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 80 Average Reward: -8.20251885437458\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 66% (324 of 488) |##############        | Elapsed Time: 2:18:12 ETA:   1:09:19\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 80 Average Reward: -6.63034517227835\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 66% (325 of 488) |##############        | Elapsed Time: 2:18:38 ETA:   1:09:19\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -6.541760206881805\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 66% (326 of 488) |##############        | Elapsed Time: 2:19:08 ETA:   1:22:23\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -6.616845038432025\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 67% (327 of 488) |##############        | Elapsed Time: 2:19:36 ETA:   1:13:38\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -7.259087125222428\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 67% (328 of 488) |##############        | Elapsed Time: 2:20:05 ETA:   1:16:55\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -7.032676732400766\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 67% (329 of 488) |##############        | Elapsed Time: 2:20:33 ETA:   1:13:37\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -6.7648893096041585\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 67% (330 of 488) |##############        | Elapsed Time: 2:21:00 ETA:   1:13:18\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -7.023548649401114\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 67% (331 of 488) |##############        | Elapsed Time: 2:21:28 ETA:   1:13:13\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -6.781181128223895\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 68% (332 of 488) |##############        | Elapsed Time: 2:21:56 ETA:   1:10:58\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -6.691319662846832\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 68% (333 of 488) |###############       | Elapsed Time: 2:22:22 ETA:   1:08:18\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -6.205960871882736\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 68% (334 of 488) |###############       | Elapsed Time: 2:22:48 ETA:   1:07:20\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -6.346989449203773\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 68% (335 of 488) |###############       | Elapsed Time: 2:23:15 ETA:   1:08:28\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -7.063210828649829\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 68% (336 of 488) |###############       | Elapsed Time: 2:23:42 ETA:   1:08:46\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -7.058478827060715\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 69% (337 of 488) |###############       | Elapsed Time: 2:24:09 ETA:   1:07:57\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -7.206381261531837\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 69% (338 of 488) |###############       | Elapsed Time: 2:24:37 ETA:   1:08:42\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -7.037468522877365\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 69% (339 of 488) |###############       | Elapsed Time: 2:25:06 ETA:   1:12:04\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -6.251368452696069\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 69% (340 of 488) |###############       | Elapsed Time: 2:25:34 ETA:   1:10:17\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -7.975925942158981\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 69% (341 of 488) |###############       | Elapsed Time: 2:26:05 ETA:   1:14:08\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -5.986974943013772\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 70% (342 of 488) |###############       | Elapsed Time: 2:26:35 ETA:   1:13:03\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -7.699363130480667\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 70% (343 of 488) |###############       | Elapsed Time: 2:27:04 ETA:   1:11:31\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -6.595009702055849\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 70% (344 of 488) |###############       | Elapsed Time: 2:27:31 ETA:   1:04:02\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -7.01385323792928\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 70% (345 of 488) |###############       | Elapsed Time: 2:28:00 ETA:   1:08:55\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -6.452022870670923\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 70% (346 of 488) |###############       | Elapsed Time: 2:28:32 ETA:   1:15:05\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -6.576697920260868\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 71% (347 of 488) |###############       | Elapsed Time: 2:28:59 ETA:   1:04:42\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -6.674441491948219\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 71% (348 of 488) |###############       | Elapsed Time: 2:29:27 ETA:   1:04:09\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -6.325074948955675\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 71% (349 of 488) |###############       | Elapsed Time: 2:29:54 ETA:   1:02:44\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -6.553872362537692\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 71% (350 of 488) |###############       | Elapsed Time: 2:30:21 ETA:   1:03:30\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -7.108850398612037\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 71% (351 of 488) |###############       | Elapsed Time: 2:30:47 ETA:   0:58:32\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -6.764710785078666\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 72% (352 of 488) |###############       | Elapsed Time: 2:31:13 ETA:   0:58:15\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -6.5965542081404775\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 72% (353 of 488) |###############       | Elapsed Time: 2:31:39 ETA:   0:59:24\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -6.688167956883168\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 72% (354 of 488) |###############       | Elapsed Time: 2:32:06 ETA:   0:59:31\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -6.246359417198467\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 72% (355 of 488) |################      | Elapsed Time: 2:32:32 ETA:   0:59:23\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -4.607700427983458\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 72% (356 of 488) |################      | Elapsed Time: 2:32:59 ETA:   0:57:55\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -7.308346035139783\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 73% (357 of 488) |################      | Elapsed Time: 2:33:25 ETA:   0:57:31\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -5.808391263577196\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 73% (358 of 488) |################      | Elapsed Time: 2:33:51 ETA:   0:56:42\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -8.793226213297807\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 73% (359 of 488) |################      | Elapsed Time: 2:34:18 ETA:   0:57:08\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -6.009487211125516\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 73% (360 of 488) |################      | Elapsed Time: 2:34:44 ETA:   0:56:02\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -6.223435264829043\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 73% (361 of 488) |################      | Elapsed Time: 2:35:10 ETA:   0:55:27\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -6.302289385798267\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 74% (362 of 488) |################      | Elapsed Time: 2:35:36 ETA:   0:54:52\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -7.96943542950773\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 74% (363 of 488) |################      | Elapsed Time: 2:36:03 ETA:   0:54:30\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -6.897313836655182\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 74% (364 of 488) |################      | Elapsed Time: 2:36:29 ETA:   0:53:51\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -6.349000220098301\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 74% (365 of 488) |################      | Elapsed Time: 2:36:55 ETA:   0:53:23\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -6.498005459374307\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 75% (366 of 488) |################      | Elapsed Time: 2:37:21 ETA:   0:52:51\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -6.358798482945225\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 75% (367 of 488) |################      | Elapsed Time: 2:37:48 ETA:   0:54:50\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -6.373837557651872\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 75% (368 of 488) |################      | Elapsed Time: 2:38:14 ETA:   0:53:04\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -4.483241134267354\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 75% (369 of 488) |################      | Elapsed Time: 2:38:41 ETA:   0:52:21\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -5.670510356401244\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 75% (370 of 488) |################      | Elapsed Time: 2:39:07 ETA:   0:51:22\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -6.125032675417029\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 76% (371 of 488) |################      | Elapsed Time: 2:39:33 ETA:   0:51:33\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -6.289266383929137\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 76% (372 of 488) |################      | Elapsed Time: 2:40:00 ETA:   0:50:38\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -6.401228821124611\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 76% (373 of 488) |################      | Elapsed Time: 2:40:26 ETA:   0:50:31\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -6.23158883283551\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 76% (374 of 488) |################      | Elapsed Time: 2:40:52 ETA:   0:49:29\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -6.939438779427646\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 76% (375 of 488) |################      | Elapsed Time: 2:41:18 ETA:   0:48:30\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -6.601326318558466\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 77% (376 of 488) |################      | Elapsed Time: 2:41:44 ETA:   0:48:55\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -6.162510920844313\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 77% (377 of 488) |################      | Elapsed Time: 2:42:12 ETA:   0:51:06\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -6.976172859789063\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 77% (378 of 488) |#################     | Elapsed Time: 2:42:38 ETA:   0:47:51\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -5.563055385709624\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 77% (379 of 488) |#################     | Elapsed Time: 2:43:04 ETA:   0:48:06\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -6.579078444179672\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 77% (380 of 488) |#################     | Elapsed Time: 2:43:30 ETA:   0:47:13\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -6.064053210008717\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 78% (381 of 488) |#################     | Elapsed Time: 2:43:57 ETA:   0:46:36\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -6.914175643719061\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 78% (382 of 488) |#################     | Elapsed Time: 2:44:23 ETA:   0:45:50\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -6.489702279075612\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 78% (383 of 488) |#################     | Elapsed Time: 2:44:49 ETA:   0:46:26\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -5.744281880557263\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 78% (384 of 488) |#################     | Elapsed Time: 2:45:16 ETA:   0:46:00\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -5.816433412668751\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 78% (385 of 488) |#################     | Elapsed Time: 2:45:41 ETA:   0:43:54\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -6.148166612682571\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 79% (386 of 488) |#################     | Elapsed Time: 2:46:07 ETA:   0:43:46\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -8.070940305543077\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 79% (387 of 488) |#################     | Elapsed Time: 2:46:34 ETA:   0:46:01\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -6.41336985852309\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 79% (388 of 488) |#################     | Elapsed Time: 2:47:02 ETA:   0:46:05\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -6.125020191431578\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 79% (389 of 488) |#################     | Elapsed Time: 2:47:28 ETA:   0:43:23\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -6.347321245698887\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 79% (390 of 488) |#################     | Elapsed Time: 2:47:55 ETA:   0:43:29\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -5.683878218589449\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 80% (391 of 488) |#################     | Elapsed Time: 2:48:22 ETA:   0:43:23\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -6.235150775411751\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 80% (392 of 488) |#################     | Elapsed Time: 2:48:48 ETA:   0:42:47\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -5.917022583933819\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 80% (393 of 488) |#################     | Elapsed Time: 2:49:15 ETA:   0:41:26\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -5.9428762493458365\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 80% (394 of 488) |#################     | Elapsed Time: 2:49:40 ETA:   0:40:11\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -6.221027760283997\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 80% (395 of 488) |#################     | Elapsed Time: 2:50:07 ETA:   0:40:54\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -4.984749748908119\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 81% (396 of 488) |#################     | Elapsed Time: 2:50:33 ETA:   0:39:49\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -6.110382357432633\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 81% (397 of 488) |#################     | Elapsed Time: 2:50:58 ETA:   0:39:12\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -5.3864587680560625\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 81% (398 of 488) |#################     | Elapsed Time: 2:51:25 ETA:   0:39:02\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -6.3008103943054845\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 81% (399 of 488) |#################     | Elapsed Time: 2:51:50 ETA:   0:37:20\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -7.39350773018565\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 81% (400 of 488) |##################    | Elapsed Time: 2:52:14 ETA:   0:36:00\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -6.773593231005891\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 82% (401 of 488) |##################    | Elapsed Time: 2:52:39 ETA:   0:35:38\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -5.855597285348773\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 82% (402 of 488) |##################    | Elapsed Time: 2:53:03 ETA:   0:35:04\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -6.260192740137036\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 82% (403 of 488) |##################    | Elapsed Time: 2:53:28 ETA:   0:34:54\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -5.561554018171522\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 82% (404 of 488) |##################    | Elapsed Time: 2:53:53 ETA:   0:34:34\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -8.43961131068419\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 82% (405 of 488) |##################    | Elapsed Time: 2:54:17 ETA:   0:33:53\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -5.424821663986793\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 83% (406 of 488) |##################    | Elapsed Time: 2:54:42 ETA:   0:33:48\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -6.834771119260453\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 83% (407 of 488) |##################    | Elapsed Time: 2:55:07 ETA:   0:33:18\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -5.9618172558492475\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 83% (408 of 488) |##################    | Elapsed Time: 2:55:31 ETA:   0:32:30\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -6.592575237555419\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 83% (409 of 488) |##################    | Elapsed Time: 2:55:55 ETA:   0:32:05\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -5.570590460417624\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 84% (410 of 488) |##################    | Elapsed Time: 2:56:20 ETA:   0:31:38\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -8.0104352845995\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 84% (411 of 488) |##################    | Elapsed Time: 2:56:44 ETA:   0:31:50\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -6.649744005530981\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 84% (412 of 488) |##################    | Elapsed Time: 2:57:09 ETA:   0:31:24\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -5.810235887354148\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 84% (413 of 488) |##################    | Elapsed Time: 2:57:34 ETA:   0:30:30\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -6.73520141388478\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 84% (414 of 488) |##################    | Elapsed Time: 2:57:58 ETA:   0:30:20\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -5.693495263579117\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 85% (415 of 488) |##################    | Elapsed Time: 2:58:23 ETA:   0:30:01\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -6.634524068481318\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 85% (416 of 488) |##################    | Elapsed Time: 2:58:48 ETA:   0:29:34\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -5.191748239632661\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 85% (417 of 488) |##################    | Elapsed Time: 2:59:12 ETA:   0:28:53\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -5.794008078738425\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 85% (418 of 488) |##################    | Elapsed Time: 2:59:37 ETA:   0:28:42\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -5.9282886343418175\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 85% (419 of 488) |##################    | Elapsed Time: 3:00:01 ETA:   0:28:11\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -6.494327391033606\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 86% (420 of 488) |##################    | Elapsed Time: 3:00:26 ETA:   0:27:53\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -6.281006777948236\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 86% (421 of 488) |##################    | Elapsed Time: 3:00:50 ETA:   0:27:29\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -5.855831881383112\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 86% (422 of 488) |###################   | Elapsed Time: 3:01:15 ETA:   0:27:15\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -6.17496478116321\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 86% (423 of 488) |###################   | Elapsed Time: 3:01:40 ETA:   0:26:43\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -6.391978141912364\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 86% (424 of 488) |###################   | Elapsed Time: 3:02:04 ETA:   0:26:11\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -4.871630822027277\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 87% (425 of 488) |###################   | Elapsed Time: 3:02:29 ETA:   0:25:38\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -6.7676106599829335\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 87% (426 of 488) |###################   | Elapsed Time: 3:02:53 ETA:   0:25:22\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -6.404536869343452\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 87% (427 of 488) |###################   | Elapsed Time: 3:03:18 ETA:   0:25:02\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -6.27774819238607\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 87% (428 of 488) |###################   | Elapsed Time: 3:03:43 ETA:   0:24:34\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -5.828977234910374\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 87% (429 of 488) |###################   | Elapsed Time: 3:04:07 ETA:   0:24:06\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -6.102573014244138\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 88% (430 of 488) |###################   | Elapsed Time: 3:04:32 ETA:   0:23:42\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -5.985301156623192\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 88% (431 of 488) |###################   | Elapsed Time: 3:04:56 ETA:   0:23:10\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -6.337937620711821\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 88% (432 of 488) |###################   | Elapsed Time: 3:05:20 ETA:   0:22:49\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -5.835121645663985\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 88% (433 of 488) |###################   | Elapsed Time: 3:05:45 ETA:   0:22:27\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -6.2454352060149105\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 88% (434 of 488) |###################   | Elapsed Time: 3:06:09 ETA:   0:21:56\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -6.8709799106923155\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 89% (435 of 488) |###################   | Elapsed Time: 3:06:34 ETA:   0:21:37\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -5.985820385459563\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 89% (436 of 488) |###################   | Elapsed Time: 3:06:58 ETA:   0:21:10\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -6.170702127173934\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 89% (437 of 488) |###################   | Elapsed Time: 3:07:23 ETA:   0:20:45\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -6.707021851123377\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 89% (438 of 488) |###################   | Elapsed Time: 3:07:50 ETA:   0:22:41\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -5.868767528383828\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 89% (439 of 488) |###################   | Elapsed Time: 3:08:19 ETA:   0:23:23\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -6.381443660423972\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 90% (440 of 488) |###################   | Elapsed Time: 3:08:49 ETA:   0:24:35\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -6.353237156341375\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 90% (441 of 488) |###################   | Elapsed Time: 3:09:16 ETA:   0:21:07\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -5.284013594397724\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 90% (442 of 488) |###################   | Elapsed Time: 3:09:42 ETA:   0:19:41\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -5.499277969175691\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 90% (443 of 488) |###################   | Elapsed Time: 3:10:07 ETA:   0:18:57\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -6.019618042693104\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 90% (444 of 488) |####################  | Elapsed Time: 3:10:33 ETA:   0:18:45\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -6.465685860637925\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 91% (445 of 488) |####################  | Elapsed Time: 3:10:58 ETA:   0:18:15\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -6.384383447893982\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 91% (446 of 488) |####################  | Elapsed Time: 3:11:24 ETA:   0:17:42\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -6.403292332661736\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 91% (447 of 488) |####################  | Elapsed Time: 3:11:49 ETA:   0:17:13\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -6.323647104011281\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 91% (448 of 488) |####################  | Elapsed Time: 3:12:14 ETA:   0:16:39\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -5.518634614828147\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 92% (449 of 488) |####################  | Elapsed Time: 3:12:39 ETA:   0:16:23\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -5.642365001496192\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 92% (450 of 488) |####################  | Elapsed Time: 3:13:05 ETA:   0:16:26\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -5.746148305362948\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 92% (451 of 488) |####################  | Elapsed Time: 3:13:31 ETA:   0:15:50\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -6.25738166970035\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 92% (452 of 488) |####################  | Elapsed Time: 3:13:57 ETA:   0:15:53\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -6.863586100003262\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 92% (453 of 488) |####################  | Elapsed Time: 3:14:24 ETA:   0:15:27\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -6.239107687436265\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 93% (454 of 488) |####################  | Elapsed Time: 3:14:49 ETA:   0:14:29\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -6.182221548588061\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 93% (455 of 488) |####################  | Elapsed Time: 3:15:15 ETA:   0:14:09\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -5.920366056791171\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 93% (456 of 488) |####################  | Elapsed Time: 3:15:41 ETA:   0:13:50\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -6.192457650913481\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 93% (457 of 488) |####################  | Elapsed Time: 3:16:06 ETA:   0:13:10\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -6.094104770534354\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 93% (458 of 488) |####################  | Elapsed Time: 3:16:34 ETA:   0:13:47\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -5.1652291173412666\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 94% (459 of 488) |####################  | Elapsed Time: 3:17:01 ETA:   0:12:51\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -6.508958517007764\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 94% (460 of 488) |####################  | Elapsed Time: 3:17:27 ETA:   0:12:21\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -5.719719095329275\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 94% (461 of 488) |####################  | Elapsed Time: 3:17:54 ETA:   0:12:12\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -5.852381720056047\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 94% (462 of 488) |####################  | Elapsed Time: 3:18:20 ETA:   0:11:21\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -5.5942839416871175\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 94% (463 of 488) |####################  | Elapsed Time: 3:18:47 ETA:   0:11:11\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -6.184588975418336\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 95% (464 of 488) |####################  | Elapsed Time: 3:19:13 ETA:   0:10:20\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -5.832540948036906\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 95% (465 of 488) |####################  | Elapsed Time: 3:19:41 ETA:   0:10:46\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -6.0233163035812805\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 95% (466 of 488) |##################### | Elapsed Time: 3:20:08 ETA:   0:09:58\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -6.335728804768392\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 95% (467 of 488) |##################### | Elapsed Time: 3:20:35 ETA:   0:09:19\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -6.0495708226779215\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 95% (468 of 488) |##################### | Elapsed Time: 3:21:03 ETA:   0:09:24\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -6.100532816215209\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 96% (469 of 488) |##################### | Elapsed Time: 3:21:29 ETA:   0:08:11\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -6.27826165120222\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 96% (470 of 488) |##################### | Elapsed Time: 3:21:55 ETA:   0:07:37\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -5.881495148318199\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 96% (471 of 488) |##################### | Elapsed Time: 3:22:20 ETA:   0:07:05\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -5.76903123865247\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 96% (472 of 488) |##################### | Elapsed Time: 3:22:44 ETA:   0:06:35\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -5.90190198787751\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 96% (473 of 488) |##################### | Elapsed Time: 3:23:14 ETA:   0:07:31\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -5.904979956914093\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 97% (474 of 488) |##################### | Elapsed Time: 3:23:45 ETA:   0:07:06\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -5.633161835444236\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 97% (475 of 488) |##################### | Elapsed Time: 3:24:16 ETA:   0:06:47\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -6.013968407198345\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 97% (476 of 488) |##################### | Elapsed Time: 3:24:47 ETA:   0:06:14\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -6.104715834015341\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 97% (477 of 488) |##################### | Elapsed Time: 3:25:18 ETA:   0:05:34\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -5.895402888652134\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 97% (478 of 488) |##################### | Elapsed Time: 3:25:47 ETA:   0:04:55\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -6.850759630429255\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 98% (479 of 488) |##################### | Elapsed Time: 3:26:17 ETA:   0:04:23\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -6.451214222751775\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 98% (480 of 488) |##################### | Elapsed Time: 3:26:46 ETA:   0:03:56\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -6.02283574092978\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 98% (481 of 488) |##################### | Elapsed Time: 3:27:16 ETA:   0:03:27\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -6.132948894808731\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 98% (482 of 488) |##################### | Elapsed Time: 3:27:46 ETA:   0:03:02\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -6.345383583578386\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 98% (483 of 488) |##################### | Elapsed Time: 3:28:17 ETA:   0:02:31\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -5.794877282078398\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 99% (484 of 488) |##################### | Elapsed Time: 3:28:47 ETA:   0:02:01\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -6.272782411815232\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 99% (485 of 488) |##################### | Elapsed Time: 3:29:18 ETA:   0:01:31\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -5.684101138313562\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 99% (486 of 488) |##################### | Elapsed Time: 3:29:48 ETA:   0:01:01\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -5.872491653256822\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 99% (487 of 488) |##################### | Elapsed Time: 3:30:20 ETA:   0:00:31\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SPS: 79 Average Reward: -5.96250430684163\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100% (488 of 488) |######################| Elapsed Time: 3:30:50 Time:  3:30:50\n"
     ]
    }
   ],
   "source": [
    "torch.set_default_dtype(torch.float32)\n",
    "\n",
    "run_name = f\"{exp_name}__{seed}__{int(time.time())}\"\n",
    "\n",
    "writer = SummaryWriter(f\"runs/{run_name}\")\n",
    "\n",
    "writer.add_text(\"environment/level_number\", str(len(xml_files)), 0)\n",
    "writer.add_text(\"environment/agents\", ', '.join(agents), 0)\n",
    "writer.add_text(\"hyperparameters/learning_rate\", str(learning_rate), 0)\n",
    "writer.add_text(\"hyperparameters/network_size\", ', '.join(str(e) for e in [512, 256]), 0)\n",
    "writer.add_text(\"hyperparameters/batch\", str(minibatch_size), 0)\n",
    "\n",
    "# TRY NOT TO MODIFY: seeding\n",
    "random.seed(seed)\n",
    "np.random.seed(seed)\n",
    "torch.manual_seed(seed)\n",
    "torch.backends.cudnn.deterministic = torch_deterministic\n",
    "\n",
    "device = torch.device(\"cuda\" if torch.cuda.is_available() and cuda else \"cpu\")\n",
    "\n",
    "obs, infos = env.reset()\n",
    "\n",
    "sender, sender_optimizer = initialize_agent(env, device, learning_rate)\n",
    "receiver, receiver_optimizer = initialize_agent(env, device, learning_rate)\n",
    "\n",
    "buffer_sender = Buffer(num_steps, env, num_envs, device)\n",
    "buffer_receiver = Buffer(num_steps, env, num_envs, device)\n",
    "\n",
    "global_step = 0\n",
    "start_time = time.time()\n",
    "next_obs, infos = reset_environment(env, device)\n",
    "\n",
    "next_done = {\"sender\": torch.zeros(num_envs).to(device), \"receiver\": torch.zeros(num_envs).to(device)}\n",
    "\n",
    "num_updates = total_timesteps // batch_size\n",
    "train_start = time.time()\n",
    "\n",
    "epoch_lengths = []\n",
    "current_length = 0\n",
    "\n",
    "for update in progressbar(range(1, num_updates + 1), redirect_stdout=True):\n",
    "    # Annealing the rate if instructed to do so.\n",
    "    if anneal_lr:\n",
    "        frac = 1.0 - (update - 1.0) / num_updates\n",
    "        lrnow = frac * learning_rate\n",
    "        sender_optimizer.param_groups[0][\"lr\"] = lrnow\n",
    "        receiver_optimizer.param_groups[0][\"lr\"] = lrnow\n",
    "    \n",
    "    epoch_rewards = {\"sender\":0, \"receiver\":0}\n",
    "    current_rewards = {\"sender\":[], \"receiver\":[]}\n",
    "    variances = {\"sender\":[], \"receiver\":[]}\n",
    "    epoch_runs = 0\n",
    "    episode_accuracies = 0\n",
    "    episode_sendAccuracies = 0\n",
    "    for step in range(0, num_steps):\n",
    "        global_step += 1 * num_envs\n",
    "        current_length += 1\n",
    "        buffer_sender.obs[step] = next_obs[\"sender\"]\n",
    "        buffer_receiver.obs[step] = next_obs[\"receiver\"]\n",
    "\n",
    "\n",
    "        sender_action = get_action_and_update_buffer(sender, next_obs[\"sender\"], buffer_sender, step)\n",
    "        receiver_action = get_action_and_update_buffer(receiver, next_obs[\"receiver\"], buffer_receiver, step)\n",
    "\n",
    "        next_obs, reward, terminations, truncations, info = env.step({\"sender\": sender_action.cpu().numpy()[0], \"receiver\": receiver_action.cpu().numpy()[0]})\n",
    "        current_rewards[\"sender\"].append(reward[\"sender\"])\n",
    "        current_rewards[\"receiver\"].append(reward[\"receiver\"])\n",
    "        next_obs = {\"sender\": torch.Tensor(next_obs[\"sender\"]).unsqueeze(0).to(device), \"receiver\": torch.Tensor(next_obs[\"receiver\"]).unsqueeze(0).to(device)}\n",
    "\n",
    "        if terminations[\"sender\"] or terminations[\"receiver\"] or truncations[\"sender\"] or truncations[\"receiver\"]:\n",
    "            next_obs, infos = reset_environment(env, device)\n",
    "            epoch_rewards[\"sender\"] += sum(current_rewards[\"sender\"])\n",
    "            epoch_rewards[\"receiver\"] += sum(current_rewards[\"receiver\"])\n",
    "\n",
    "            epoch_lengths.append(current_length)\n",
    "            current_length = 0\n",
    "\n",
    "            dynamic = env.env.env.environment_dynamics[3]\n",
    "\n",
    "            if len(dynamic.sendAccuracies) > 512:\n",
    "                episode_sendAccuracies = sum(dynamic.sendAccuracies[-512:]) / 512\n",
    "                del dynamic.sendAccuracies[:-513]\n",
    "                writer.add_scalar(\"charts/sender/accuracies\", episode_sendAccuracies, global_step)\n",
    "\n",
    "            if len(dynamic.accuracies) > 4:\n",
    "                window = min(15, len(dynamic.accuracies))\n",
    "                episode_accuracies = sum(dynamic.accuracies[-1 * window:]) / window\n",
    "                writer.add_scalar(\"charts/receiver/accuracies\", episode_accuracies, global_step)\n",
    "                if window == 15:\n",
    "                    del dynamic.accuracies[:-16]\n",
    "\n",
    "            if len(dynamic.variances) > 4:\n",
    "                window = min(15, len(dynamic.variances))\n",
    "                current_variance = sum(dynamic.variances[-1 * window:]) / window\n",
    "                writer.add_scalar(\"charts/receiver_variance\", current_variance, global_step)\n",
    "                if window == 15:\n",
    "                    del dynamic.variances[:-16]\n",
    "\n",
    "            if len(epoch_lengths) > 3:\n",
    "                window = min(10, len(epoch_lengths))\n",
    "                epoch_length = sum(epoch_lengths[-1 * window:]) / window\n",
    "                writer.add_scalar(\"charts/episodic_length\", epoch_length, global_step)\n",
    "                if window == 10:\n",
    "                    del epoch_lengths[:-11]\n",
    "            epoch_runs += 1\n",
    "        \n",
    "        buffer_sender.rewards[step] = torch.tensor(reward[\"sender\"]).to(device).view(-1)\n",
    "        buffer_receiver.rewards[step] = torch.tensor(reward[\"receiver\"]).to(device).view(-1)\n",
    "        next_done = {\"sender\": torch.Tensor([terminations[\"sender\"]]).to(device), \"receiver\": torch.Tensor([terminations[\"receiver\"]]).to(device)}\n",
    "    if update % store_freq == 0:\n",
    "        torch.save(sender, \"models/model\" + str(start_time) + \".pth\")\n",
    "        torch.save(receiver, \"models/model\" + str(start_time) + \".pth\")\n",
    "\n",
    "    update_agent(sender, buffer_sender, sender_optimizer, next_obs[\"sender\"], next_done[\"sender\"], env, batch_size, update_epochs, minibatch_size, clip_coef, vf_coef, ent_coef, max_grad_norm, target_kl, clip_vloss, norm_adv, gae_lambda, gae, gamma, device, step)\n",
    "    update_agent(receiver, buffer_receiver, receiver_optimizer, next_obs[\"receiver\"], next_done[\"receiver\"], env, batch_size, update_epochs, minibatch_size, clip_coef, vf_coef, ent_coef, max_grad_norm, target_kl, clip_vloss, norm_adv, gae_lambda, gae, gamma, device, step)\n",
    "\n",
    "    writer.add_scalar(\"charts/learning_rate\", sender_optimizer.param_groups[0][\"lr\"], global_step)\n",
    "    writer.add_scalar(\"charts/sender/episodic_return\", epoch_rewards[\"sender\"] / epoch_runs, global_step)\n",
    "    writer.add_scalar(\"charts/receiver/episodic_return\", epoch_rewards[\"receiver\"] / epoch_runs, global_step)\n",
    "    print(\"SPS:\", int(global_step / (time.time() - start_time)), \"Average Reward:\", epoch_rewards[\"sender\"] / epoch_runs)\n",
    "    writer.add_scalar(\"charts/SPS\", int(global_step / (time.time() - start_time)), global_step)\n",
    "\n",
    "torch.save(sender, \"models/model\" + str(start_time) + \".pth\")\n",
    "torch.save(receiver, \"models/model\" + str(start_time) + \".pth\")\n",
    "writer.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "torch.save(sender, \"models/sender\" + str(start_time) + \".pth\")\n",
    "torch.save(receiver, \"models/receiver\" + str(start_time) + \".pth\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Direct communication test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Test_Communication:\n",
    "    def __init__(self, environment):\n",
    "        self.environment = environment\n",
    "        self.observation_space = {\"low\": [0, 0, 0, 0], \"high\": [1, 1, 1, 1]}\n",
    "        self.action_space = {\"low\": [0, 0, 0, 0], \"high\": [1, 1, 1, 1]}\n",
    "\n",
    "    def dynamic(self, agent, actions):\n",
    "        if \"utterance\" not in self.environment.data_store[agent].keys():\n",
    "            self.environment.data_store[agent][\"utterance\"] = None\n",
    "        if agent == \"receiver\":\n",
    "            utterance = [0, 0, 0, 0]\n",
    "            if \"utterance_max\" in self.environment.data_store[\"sender\"].keys():\n",
    "                observation = self.environment.data_store[\"sender\"][\"utterance_max\"]\n",
    "            else:\n",
    "                observation = utterance\n",
    "        elif agent == \"sender\":\n",
    "            utterance = [0, 0, 0, 0]\n",
    "            utterance[np.argmax(actions)] = 1\n",
    "            self.environment.data_store[agent][\"utterance\"] = actions\n",
    "            self.environment.data_store[agent][\"utterance_max\"] = utterance\n",
    "            observation = [0, 0, 0, 0]\n",
    "        else:\n",
    "            print(\"Dafaq is going on here?\")\n",
    "        return 0, observation, False, {}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/cowolff/miniconda3/envs/Ray/lib/python3.9/site-packages/gymnasium/spaces/box.py:130: UserWarning: \u001b[33mWARN: Box bound precision lowered by casting to float32\u001b[0m\n",
      "  gym.logger.warn(f\"Box bound precision lowered by casting to {self.dtype}\")\n",
      "/Users/cowolff/miniconda3/envs/Ray/lib/python3.9/site-packages/keras/src/engine/training_v1.py:2359: UserWarning: `Model.state_updates` will be removed in a future version. This property should not be used in TensorFlow 2.0, as `updates` are applied automatically.\n",
      "  updates=self.state_updates,\n"
     ]
    }
   ],
   "source": [
    "xml_files = [\"levels/\" + file for file in os.listdir(\"levels/\")]\n",
    "agents = [\"sender\", \"receiver\"]\n",
    "\n",
    "config_dict = {\"xmlPath\":xml_files, \n",
    "                   \"agents\":agents, \n",
    "                   \"rewardFunctions\":[collision_reward, target_reward], \n",
    "                   \"doneFunctions\":[target_done, border_done], \n",
    "                   \"skipFrames\":5,\n",
    "                   \"environmentDynamics\":[Image, Reward, Test_Communication, Accuracy],\n",
    "                   \"freeJoint\":True,\n",
    "                   \"renderMode\":False,\n",
    "                   \"maxSteps\":1024,\n",
    "                   \"agentCameras\":True}\n",
    "\n",
    "env = make_env(config_dict)()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "epochs = 30\n",
    "num_steps = 1024\n",
    "lengths = []\n",
    "\n",
    "for epoch in range(epochs):\n",
    "    next_obs, infos = reset_environment(env, device)\n",
    "\n",
    "    next_obs = {k: torch.Tensor(v).unsqueeze(0).to(device) for k, v in next_obs.items()}\n",
    "    next_done = {\"sender\": torch.zeros(num_envs).to(device), \"receiver\": torch.zeros(num_envs).to(device)}\n",
    "\n",
    "    for step in range(0, num_steps):\n",
    "        sender_action = sender.get_action_and_value(next_obs[\"sender\"])[0]\n",
    "        receiver_action = receiver.get_action_and_value(next_obs[\"receiver\"])[0]\n",
    "\n",
    "        next_obs, reward, terminations, truncations, info = env.step({\"sender\": sender_action.cpu().numpy()[0], \"receiver\": receiver_action.cpu().numpy()[0]})\n",
    "        next_obs = {\"sender\": torch.Tensor(next_obs[\"sender\"]).unsqueeze(0).to(device), \"receiver\": torch.Tensor(next_obs[\"receiver\"]).unsqueeze(0).to(device)}\n",
    "\n",
    "        if terminations[\"sender\"] or terminations[\"receiver\"] or truncations[\"sender\"] or truncations[\"receiver\"]:\n",
    "            next_obs, infos = reset_environment(env, device)\n",
    "            lengths.append(step)\n",
    "            break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "dynamic = env.env.env.environment_dynamics[3]\n",
    "print(\"Accuracy:\", sum(dynamic.accuracies) / len(dynamic.accuracies))\n",
    "print(\"Variance:\", sum(dynamic.variances) / len(dynamic.variances))\n",
    "print(\"Send Accuracy:\", sum(dynamic.sendAccuracies) / len(dynamic.sendAccuracies))\n",
    "print(\"Length:\", sum(lengths) / len(lengths))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Ray",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.16"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}